diff --git "a/checkpoint-25000/trainer_state.json" "b/checkpoint-25000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-25000/trainer_state.json" @@ -0,0 +1,205056 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9929697740000795, + "global_step": 25000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-06, + "loss": 4.3952, + "theoretical_loss": 17.566201104328645, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 4.3745, + "theoretical_loss": 14.89231675598857, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809525e-06, + "loss": 4.1183, + "theoretical_loss": 13.552561472550224, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 3.9369, + "theoretical_loss": 12.690129625483323, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-06, + "loss": 4.0965, + "theoretical_loss": 12.067412607035077, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 3.5987, + "theoretical_loss": 11.586719208706729, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 1.3888888888888888e-05, + "loss": 3.6811, + "theoretical_loss": 11.199011702111871, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 3.7779, + "theoretical_loss": 10.87642808645695, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 1.7857142857142855e-05, + "loss": 3.8919, + "theoretical_loss": 10.60172987623028, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 4.0009, + "theoretical_loss": 10.36356394376333, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 2.1825396825396824e-05, + "loss": 3.75, + "theoretical_loss": 10.15408655327002, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 3.8353, + "theoretical_loss": 9.967669178840278, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.36052054166793823, + "objective/train/avg_token_score": 0.01564830169081688, + "objective/train/docs_used": 13112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.4145798683166504, + "objective/train/original_loss": 9.366209983825684, + "objective/train/theoretical_loss": 9.881763126393109, + "objective/train/tokens_used": 1638400, + "objective/train/unlikelihood_loss": 0.004730077460408211, + "theoretical_loss": 9.881763126393109, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.5793650793650793e-05, + "loss": 3.9383, + "theoretical_loss": 9.80014659154056, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 2.7777777777777776e-05, + "loss": 3.6506, + "theoretical_loss": 9.648356759081546, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904762e-05, + "loss": 3.5887, + "theoretical_loss": 9.509847046764852, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 3.7839, + "theoretical_loss": 9.382679790910457, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 3.373015873015873e-05, + "loss": 3.3506, + "theoretical_loss": 9.265299666660276, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428571e-05, + "loss": 3.6369, + "theoretical_loss": 9.156440812508292, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 3.76984126984127e-05, + "loss": 3.1405, + "theoretical_loss": 9.055060296533734, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 3.5961, + "theoretical_loss": 8.960289489909357, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 4.1666666666666665e-05, + "loss": 3.8762, + "theoretical_loss": 8.871397892478225, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 4.365079365079365e-05, + "loss": 3.1495, + "theoretical_loss": 8.787765792778412, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 4.563492063492063e-05, + "loss": 3.2901, + "theoretical_loss": 8.708863309520833, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 3.7481, + "theoretical_loss": 8.634234117735474, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.3303396999835968, + "objective/train/avg_token_score": 0.04211259260773659, + "objective/train/docs_used": 13755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.995987892150879, + "objective/train/original_loss": 9.143326759338379, + "objective/train/theoretical_loss": 8.563482664611069, + "objective/train/tokens_used": 3276800, + "objective/train/unlikelihood_loss": 0.01486299093812704, + "theoretical_loss": 8.563482664611069, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-05, + "loss": 3.5744, + "theoretical_loss": 8.563482664611069, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 5.1587301587301586e-05, + "loss": 3.5767, + "theoretical_loss": 8.496264019646002, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 5.357142857142857e-05, + "loss": 3.0622, + "theoretical_loss": 8.432275737672779, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555555e-05, + "loss": 3.7336, + "theoretical_loss": 8.371251277120209, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 5.753968253968254e-05, + "loss": 3.411, + "theoretical_loss": 8.31295463228533, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809524e-05, + "loss": 3.38, + "theoretical_loss": 8.257175922251864, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 6.15079365079365e-05, + "loss": 3.1689, + "theoretical_loss": 8.20372774027797, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 3.3772, + "theoretical_loss": 8.152442112639616, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 6.547619047619048e-05, + "loss": 2.919, + "theoretical_loss": 8.10316794961571, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 6.746031746031745e-05, + "loss": 3.1994, + "theoretical_loss": 8.055768896701416, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-05, + "loss": 3.6138, + "theoretical_loss": 8.010121513461836, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857142e-05, + "loss": 3.0735, + "theoretical_loss": 7.966113722271801, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 7.341269841269842e-05, + "loss": 3.063, + "theoretical_loss": 7.9236434806675184, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.3146994113922119, + "objective/train/avg_token_score": 0.016941465437412262, + "objective/train/docs_used": 15099, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.699511766433716, + "objective/train/original_loss": 8.369515419006348, + "objective/train/theoretical_loss": 7.902955512484067, + "objective/train/tokens_used": 4915200, + "objective/train/unlikelihood_loss": 0.032859548926353455, + "theoretical_loss": 7.902955512484067, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.53968253968254e-05, + "loss": 3.196, + "theoretical_loss": 7.882617639989203, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 7.738095238095239e-05, + "loss": 2.9935, + "theoretical_loss": 7.842950960027937, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 3.4802, + "theoretical_loss": 7.804565254954165, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 8.134920634920635e-05, + "loss": 3.2085, + "theoretical_loss": 7.767388650235364, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 3.3755, + "theoretical_loss": 7.731354933799318, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 8.531746031746032e-05, + "loss": 3.6461, + "theoretical_loss": 7.696402987558934, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-05, + "loss": 3.0517, + "theoretical_loss": 7.662476287731328, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571429e-05, + "loss": 3.0863, + "theoretical_loss": 7.629522464270861, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 9.126984126984126e-05, + "loss": 2.7058, + "theoretical_loss": 7.59749291128028, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 9.325396825396825e-05, + "loss": 3.3358, + "theoretical_loss": 7.5663424415343705, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 3.0732, + "theoretical_loss": 7.536028979299919, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 9.722222222222223e-05, + "loss": 3.0706, + "theoretical_loss": 7.506513286506497, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.41287878155708313, + "objective/train/avg_token_score": 0.03952885419130325, + "objective/train/docs_used": 15777, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.0034613609313965, + "objective/train/original_loss": 7.385845184326172, + "objective/train/theoretical_loss": 7.4777587180480305, + "objective/train/tokens_used": 6553600, + "objective/train/unlikelihood_loss": 0.09759358316659927, + "theoretical_loss": 7.4777587180480305, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-05, + "loss": 3.0284, + "theoretical_loss": 7.4777587180480305, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001011904761904762, + "loss": 2.8636, + "theoretical_loss": 7.449731002601916, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460317, + "loss": 2.724, + "theoretical_loss": 7.422398045861905, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010515873015873016, + "loss": 2.9955, + "theoretical_loss": 7.395729753510345, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010714285714285714, + "loss": 2.967, + "theoretical_loss": 7.369697871618373, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010912698412698413, + "loss": 3.2671, + "theoretical_loss": 7.3442758424706875, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001111111111111111, + "loss": 3.0036, + "theoretical_loss": 7.319438674073677, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011309523809523809, + "loss": 3.0592, + "theoretical_loss": 7.295162821829564, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011507936507936508, + "loss": 3.2368, + "theoretical_loss": 7.271426081050832, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011706349206349207, + "loss": 3.1647, + "theoretical_loss": 7.24820748915387, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 2.9356, + "theoretical_loss": 7.225487236512497, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012103174603174604, + "loss": 2.8582, + "theoretical_loss": 7.2032465850744005, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.000123015873015873, + "loss": 2.7811, + "theoretical_loss": 7.1814677939495155, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4550953209400177, + "objective/train/avg_token_score": 0.02127336896955967, + "objective/train/docs_used": 17166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.3246047496795654, + "objective/train/original_loss": 7.3253278732299805, + "objective/train/theoretical_loss": 7.1707463077509646, + "objective/train/tokens_used": 8192000, + "objective/train/unlikelihood_loss": 0.044195178896188736, + "theoretical_loss": 7.1707463077509646, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000125, + "loss": 3.3126, + "theoretical_loss": 7.160134051271272, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 3.0628, + "theoretical_loss": 7.139229411711638, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012896825396825398, + "loss": 2.7911, + "theoretical_loss": 7.118738739100616, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013095238095238096, + "loss": 2.8125, + "theoretical_loss": 7.09864765366177, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013293650793650793, + "loss": 2.6533, + "theoretical_loss": 7.078942483428749, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001349206349206349, + "loss": 2.9263, + "theoretical_loss": 7.059610219454568, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001369047619047619, + "loss": 2.8425, + "theoretical_loss": 7.040638474466625, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001388888888888889, + "loss": 3.1806, + "theoretical_loss": 7.022015444656678, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014087301587301586, + "loss": 3.0648, + "theoretical_loss": 7.003729874327071, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714284, + "loss": 3.0663, + "theoretical_loss": 6.98577102314278, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014484126984126984, + "loss": 2.7552, + "theoretical_loss": 6.968128635764015, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014682539682539685, + "loss": 3.0583, + "theoretical_loss": 6.950792913656309, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.36840176582336426, + "objective/train/avg_token_score": 0.020729999989271164, + "objective/train/docs_used": 17861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.664196014404297, + "objective/train/original_loss": 7.552342891693115, + "objective/train/theoretical_loss": 6.9337544888949, + "objective/train/tokens_used": 9830400, + "objective/train/unlikelihood_loss": 0.06143795698881149, + "theoretical_loss": 6.9337544888949, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014880952380952382, + "loss": 2.7889, + "theoretical_loss": 6.9337544888949, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 2.387, + "theoretical_loss": 6.917004399797798, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001527777777777778, + "loss": 2.846, + "theoretical_loss": 6.900534068237688, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015476190476190478, + "loss": 2.8487, + "theoretical_loss": 6.884335278496871, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015674603174603175, + "loss": 2.8823, + "theoretical_loss": 6.868400157541997, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 2.799, + "theoretical_loss": 6.85272115660663, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016071428571428573, + "loss": 2.9633, + "theoretical_loss": 6.8372910339797945, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001626984126984127, + "loss": 2.5692, + "theoretical_loss": 6.8221028389077185, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016468253968253969, + "loss": 2.8166, + "theoretical_loss": 6.807149896524181, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 3.0421, + "theoretical_loss": 6.7924257937322245, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016865079365079366, + "loss": 2.6027, + "theoretical_loss": 6.777924365966638, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017063492063492064, + "loss": 2.9229, + "theoretical_loss": 6.763639684772625, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017261904761904762, + "loss": 2.8946, + "theoretical_loss": 6.749566046141486, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4332233667373657, + "objective/train/avg_token_score": 0.019017228856682777, + "objective/train/docs_used": 18493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.7806520462036133, + "objective/train/original_loss": 7.158781051635742, + "objective/train/theoretical_loss": 6.7426066439450905, + "objective/train/tokens_used": 11468800, + "objective/train/unlikelihood_loss": 0.05772477760910988, + "theoretical_loss": 6.7426066439450905, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001746031746031746, + "loss": 2.8532, + "theoretical_loss": 6.735697959549075, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001765873015873016, + "loss": 2.5473, + "theoretical_loss": 6.722030137647226, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017857142857142857, + "loss": 2.4778, + "theoretical_loss": 6.7085574865624125, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018055555555555555, + "loss": 2.9024, + "theoretical_loss": 6.695275096759559, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968252, + "loss": 2.7962, + "theoretical_loss": 6.682178234432274, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018452380952380953, + "loss": 2.6879, + "theoretical_loss": 6.669262333383815, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001865079365079365, + "loss": 2.9128, + "theoretical_loss": 6.656522987365879, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001884920634920635, + "loss": 2.4778, + "theoretical_loss": 6.643955942844831, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 2.5956, + "theoretical_loss": 6.631557092167304, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019246031746031748, + "loss": 2.8293, + "theoretical_loss": 6.619322467099223, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019444444444444446, + "loss": 3.043, + "theoretical_loss": 6.607248232714213, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019642857142857144, + "loss": 2.7412, + "theoretical_loss": 6.59533068160918, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4409824013710022, + "objective/train/avg_token_score": 0.007921183481812477, + "objective/train/docs_used": 19731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.8607468605041504, + "objective/train/original_loss": 5.793218612670898, + "objective/train/theoretical_loss": 6.583566228426414, + "objective/train/tokens_used": 13107200, + "objective/train/unlikelihood_loss": 0.14077132940292358, + "theoretical_loss": 6.583566228426414, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001984126984126984, + "loss": 2.5021, + "theoretical_loss": 6.583566228426414, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020039682539682542, + "loss": 2.5258, + "theoretical_loss": 6.571951404663098, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002023809523809524, + "loss": 2.8582, + "theoretical_loss": 6.560482853750463, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020436507936507937, + "loss": 2.828, + "theoretical_loss": 6.549157326386091, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920634, + "loss": 2.8331, + "theoretical_loss": 6.537971676104026, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020833333333333335, + "loss": 2.0467, + "theoretical_loss": 6.5269228550684195, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021031746031746032, + "loss": 2.6811, + "theoretical_loss": 6.516007910077416, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002123015873015873, + "loss": 2.7855, + "theoretical_loss": 6.50522397876491, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021428571428571427, + "loss": 2.7877, + "theoretical_loss": 6.494568285988618, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021626984126984128, + "loss": 2.624, + "theoretical_loss": 6.484038140393699, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021825396825396825, + "loss": 2.7998, + "theoretical_loss": 6.473630931141869, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022023809523809523, + "loss": 2.71, + "theoretical_loss": 6.463344124796616, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002222222222222222, + "loss": 2.5073, + "theoretical_loss": 6.45317526235573, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.49395161867141724, + "objective/train/avg_token_score": 0.028753770515322685, + "objective/train/docs_used": 20428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.2851853370666504, + "objective/train/original_loss": 6.822603225708008, + "objective/train/theoretical_loss": 6.448134311315593, + "objective/train/tokens_used": 14745600, + "objective/train/unlikelihood_loss": 0.016451282426714897, + "theoretical_loss": 6.448134311315593, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002242063492063492, + "loss": 2.5962, + "theoretical_loss": 6.443121956422939, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022619047619047618, + "loss": 2.6017, + "theoretical_loss": 6.433181888510964, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022817460317460316, + "loss": 2.7709, + "theoretical_loss": 6.4233528064687855, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 2.5309, + "theoretical_loss": 6.413632522026391, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023214285714285717, + "loss": 2.5149, + "theoretical_loss": 6.404018908450656, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023412698412698414, + "loss": 2.5856, + "theoretical_loss": 6.394509898306452, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023611111111111112, + "loss": 2.5106, + "theoretical_loss": 6.385103481317387, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 2.559, + "theoretical_loss": 6.375797702320966, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002400793650793651, + "loss": 2.2329, + "theoretical_loss": 6.366590659313248, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024206349206349207, + "loss": 2.6909, + "theoretical_loss": 6.357480501578371, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024404761904761905, + "loss": 2.6294, + "theoretical_loss": 6.348465427898629, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246031746031746, + "loss": 2.464, + "theoretical_loss": 6.33954368484097, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.39424487948417664, + "objective/train/avg_token_score": 0.013139091432094574, + "objective/train/docs_used": 21641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.531733751296997, + "objective/train/original_loss": 6.893299102783203, + "objective/train/theoretical_loss": 6.330713565116083, + "objective/train/tokens_used": 16384000, + "objective/train/unlikelihood_loss": 0.03236621245741844, + "theoretical_loss": 6.330713565116083, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248015873015873, + "loss": 2.6212, + "theoretical_loss": 6.330713565116083, + "tokens_seen": 16384000 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 2.7361, + "theoretical_loss": 6.32197340600647, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000251984126984127, + "loss": 2.4975, + "theoretical_loss": 6.313321587860021, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025396825396825396, + "loss": 2.75, + "theoretical_loss": 6.304756532645939, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025595238095238096, + "loss": 2.38, + "theoretical_loss": 6.296276702569918, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025793650793650796, + "loss": 2.4812, + "theoretical_loss": 6.28788059874573, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025992063492063497, + "loss": 2.5431, + "theoretical_loss": 6.279566759920507, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002619047619047619, + "loss": 2.9019, + "theoretical_loss": 6.271333761251142, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002638888888888889, + "loss": 2.454, + "theoretical_loss": 6.2631802131294085, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026587301587301587, + "loss": 2.6175, + "theoretical_loss": 6.255104760053497, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026785714285714287, + "loss": 2.6316, + "theoretical_loss": 6.247106079543801, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002698412698412698, + "loss": 2.4119, + "theoretical_loss": 6.239182881100916, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002718253968253968, + "loss": 2.7574, + "theoretical_loss": 6.231333905203899, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3819648027420044, + "objective/train/avg_token_score": 0.008278077468276024, + "objective/train/docs_used": 22253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.420483112335205, + "objective/train/original_loss": 6.370981693267822, + "objective/train/theoretical_loss": 6.227436865342643, + "objective/train/tokens_used": 18022400, + "objective/train/unlikelihood_loss": 0.04339008778333664, + "theoretical_loss": 6.227436865342643, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002738095238095238, + "loss": 2.6268, + "theoretical_loss": 6.223557922346955, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027579365079365083, + "loss": 2.5336, + "theoretical_loss": 6.215853732112821, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002777777777777778, + "loss": 2.3994, + "theoretical_loss": 6.208220162281178, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002797619047619048, + "loss": 2.3323, + "theoretical_loss": 6.20065606797053, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028174603174603173, + "loss": 2.5059, + "theoretical_loss": 6.1931603308120975, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028373015873015873, + "loss": 2.28, + "theoretical_loss": 6.185731858154261, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002857142857142857, + "loss": 2.4878, + "theoretical_loss": 6.17836958229627, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002876984126984127, + "loss": 2.4956, + "theoretical_loss": 6.171072459749913, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002896825396825397, + "loss": 2.2678, + "theoretical_loss": 6.163839470527964, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002916666666666667, + "loss": 2.1494, + "theoretical_loss": 6.156669617458243, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002936507936507937, + "loss": 2.3674, + "theoretical_loss": 6.149561925522211, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029563492063492064, + "loss": 2.3958, + "theoretical_loss": 6.142515441217064, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4558895528316498, + "objective/train/avg_token_score": 0.004920742008835077, + "objective/train/docs_used": 23612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.9035089015960693, + "objective/train/original_loss": 6.259584426879883, + "objective/train/theoretical_loss": 6.135529231940326, + "objective/train/tokens_used": 19660800, + "objective/train/unlikelihood_loss": 0.0633547455072403, + "theoretical_loss": 6.135529231940326, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029761904761904765, + "loss": 2.5385, + "theoretical_loss": 6.135529231940326, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002996031746031746, + "loss": 2.3807, + "theoretical_loss": 6.128602385396022, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003015873015873016, + "loss": 2.0421, + "theoretical_loss": 6.121734009021521, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030357142857142855, + "loss": 2.5316, + "theoretical_loss": 6.114923229434213, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003055555555555556, + "loss": 2.4607, + "theoretical_loss": 6.108169191897195, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030753968253968255, + "loss": 2.5377, + "theoretical_loss": 6.101471059803204, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030952380952380956, + "loss": 2.7593, + "theoretical_loss": 6.094828014176053, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003115079365079365, + "loss": 2.6748, + "theoretical_loss": 6.088239253188885, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003134920634920635, + "loss": 2.387, + "theoretical_loss": 6.0817039916985465, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031547619047619046, + "loss": 2.3745, + "theoretical_loss": 6.075221460795472, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031746031746031746, + "loss": 2.3548, + "theoretical_loss": 6.068790907368448, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003194444444444444, + "loss": 2.5693, + "theoretical_loss": 6.062411593683687, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032142857142857147, + "loss": 2.754, + "theoretical_loss": 6.056082796977648, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.5116080045700073, + "objective/train/avg_token_score": 0.0047623817808926105, + "objective/train/docs_used": 24330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 3.2419803142547607, + "objective/train/original_loss": 5.9219136238098145, + "objective/train/theoretical_loss": 6.052937120624258, + "objective/train/tokens_used": 21299200, + "objective/train/unlikelihood_loss": 0.10302968323230743, + "theoretical_loss": 6.052937120624258, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003234126984126984, + "loss": 2.6761, + "theoretical_loss": 6.049803809063083, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003253968253968254, + "loss": 2.3248, + "theoretical_loss": 6.04357393594778, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032738095238095237, + "loss": 2.5682, + "theoretical_loss": 6.037392497465552, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032936507936507937, + "loss": 2.2946, + "theoretical_loss": 6.031258826918979, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003313492063492063, + "loss": 2.6749, + "theoretical_loss": 6.025172270733464, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003333333333333333, + "loss": 2.6852, + "theoretical_loss": 6.0191321881221995, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003353174603174603, + "loss": 2.5813, + "theoretical_loss": 6.01313795076161, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033730158730158733, + "loss": 2.6132, + "theoretical_loss": 6.007188942476907, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033928571428571433, + "loss": 2.1199, + "theoretical_loss": 6.001284558937368, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003412698412698413, + "loss": 2.5638, + "theoretical_loss": 5.995424207360987, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003432539682539683, + "loss": 2.4583, + "theoretical_loss": 5.989607306228168, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034523809523809523, + "loss": 2.6098, + "theoretical_loss": 5.983833285004112, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.374450147151947, + "objective/train/avg_token_score": 0.010239812545478344, + "objective/train/docs_used": 24983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.322624921798706, + "objective/train/original_loss": 6.397256374359131, + "objective/train/theoretical_loss": 5.978101583869607, + "objective/train/tokens_used": 22937600, + "objective/train/unlikelihood_loss": 0.04547956585884094, + "theoretical_loss": 5.978101583869607, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034722222222222224, + "loss": 2.6236, + "theoretical_loss": 5.978101583869607, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003492063492063492, + "loss": 2.2842, + "theoretical_loss": 5.972411653459913, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003511904761904762, + "loss": 2.4053, + "theoretical_loss": 5.966762954611432, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003531746031746032, + "loss": 2.4468, + "theoretical_loss": 5.961154958115937, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003551587301587302, + "loss": 2.7427, + "theoretical_loss": 5.955587144482044, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035714285714285714, + "loss": 2.6201, + "theoretical_loss": 5.950059003703704, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035912698412698415, + "loss": 2.6046, + "theoretical_loss": 5.944570035035458, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003611111111111111, + "loss": 2.4813, + "theoretical_loss": 5.939119746774228, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003630952380952381, + "loss": 2.464, + "theoretical_loss": 5.933707656047414, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036507936507936505, + "loss": 2.5273, + "theoretical_loss": 5.928333288607086, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036706349206349205, + "loss": 2.6318, + "theoretical_loss": 5.92299617863006, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036904761904761905, + "loss": 2.5566, + "theoretical_loss": 5.9176958685236585, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037103174603174606, + "loss": 2.3378, + "theoretical_loss": 5.912431908736972, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3531891405582428, + "objective/train/avg_token_score": 0.034930985420942307, + "objective/train/docs_used": 26439, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0727763175964355, + "objective/train/original_loss": 6.117815971374512, + "objective/train/theoretical_loss": 5.909813421882534, + "objective/train/tokens_used": 24576000, + "objective/train/unlikelihood_loss": 0.0909704715013504, + "theoretical_loss": 5.909813421882534, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000373015873015873, + "loss": 2.2126, + "theoretical_loss": 5.907203857577422, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000375, + "loss": 2.7691, + "theoretical_loss": 5.902011281032472, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.000376984126984127, + "loss": 2.3763, + "theoretical_loss": 5.896853752596286, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037896825396825396, + "loss": 2.1913, + "theoretical_loss": 5.891730853101199, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038095238095238096, + "loss": 2.6541, + "theoretical_loss": 5.8866421705538325, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038293650793650797, + "loss": 2.3709, + "theoretical_loss": 5.881587299975694, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038492063492063497, + "loss": 2.0019, + "theoretical_loss": 5.876565843248124, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003869047619047619, + "loss": 2.3081, + "theoretical_loss": 5.871577408961457, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003888888888888889, + "loss": 2.1361, + "theoretical_loss": 5.866621612268246, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039087301587301587, + "loss": 2.4905, + "theoretical_loss": 5.8616980747404295, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003928571428571429, + "loss": 2.2697, + "theoretical_loss": 5.856806424230314, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003948412698412698, + "loss": 2.3797, + "theoretical_loss": 5.851946294735258, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.48717010021209717, + "objective/train/avg_token_score": 0.020001767203211784, + "objective/train/docs_used": 27060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.8721370697021484, + "objective/train/original_loss": 5.758991718292236, + "objective/train/theoretical_loss": 5.8471173262659235, + "objective/train/tokens_used": 26214400, + "objective/train/unlikelihood_loss": 0.08089856803417206, + "theoretical_loss": 5.8471173262659235, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003968253968253968, + "loss": 2.4683, + "theoretical_loss": 5.8471173262659235, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039880952380952383, + "loss": 2.3909, + "theoretical_loss": 5.842319164718004, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040079365079365083, + "loss": 2.3413, + "theoretical_loss": 5.83755146174731, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004027777777777778, + "loss": 2.6428, + "theoretical_loss": 5.832813874648102, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 2.5088, + "theoretical_loss": 5.828106066234588, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040674603174603173, + "loss": 2.5131, + "theoretical_loss": 5.823427704725473, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040873015873015874, + "loss": 2.8614, + "theoretical_loss": 5.818778463631473, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004107142857142857, + "loss": 2.3157, + "theoretical_loss": 5.8141580216457065, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004126984126984127, + "loss": 2.4281, + "theoretical_loss": 5.809566062536868, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004146825396825397, + "loss": 2.6006, + "theoretical_loss": 5.805002275045111, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004166666666666667, + "loss": 2.2208, + "theoretical_loss": 5.800466352780546, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004186507936507937, + "loss": 2.4363, + "theoretical_loss": 5.795957994124291, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 2.5718, + "theoretical_loss": 5.791476902131985, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.39797165989875793, + "objective/train/avg_token_score": 0.010648269206285477, + "objective/train/docs_used": 28453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.4116973876953125, + "objective/train/original_loss": 6.296491622924805, + "objective/train/theoretical_loss": 5.789246489594688, + "objective/train/tokens_used": 27852800, + "objective/train/unlikelihood_loss": 0.0855846107006073, + "theoretical_loss": 5.789246489594688, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042261904761904765, + "loss": 2.3402, + "theoretical_loss": 5.787022784439701, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004246031746031746, + "loss": 2.5929, + "theoretical_loss": 5.782595353172176, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004265873015873016, + "loss": 2.671, + "theoretical_loss": 5.778194324853311, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042857142857142855, + "loss": 2.5018, + "theoretical_loss": 5.773819420318858, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004305555555555556, + "loss": 2.2753, + "theoretical_loss": 5.769470364631225, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043253968253968256, + "loss": 2.0779, + "theoretical_loss": 5.765146886996363, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043452380952380956, + "loss": 2.1053, + "theoretical_loss": 5.760848720682651, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 2.3278, + "theoretical_loss": 5.756575602941732, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004384920634920635, + "loss": 2.4976, + "theoretical_loss": 5.752327274931249, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044047619047619046, + "loss": 2.2979, + "theoretical_loss": 5.7481034816394105, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044246031746031746, + "loss": 2.4162, + "theoretical_loss": 5.74390397181136, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004444444444444444, + "loss": 2.4311, + "theoretical_loss": 5.739728497877267, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.28812316060066223, + "objective/train/avg_token_score": 0.02659354917705059, + "objective/train/docs_used": 29115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.703152060508728, + "objective/train/original_loss": 6.603550434112549, + "objective/train/theoretical_loss": 5.7355768158821245, + "objective/train/tokens_used": 29491200, + "objective/train/unlikelihood_loss": 0.04069405049085617, + "theoretical_loss": 5.7355768158821245, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044642857142857147, + "loss": 2.4414, + "theoretical_loss": 5.7355768158821245, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004484126984126984, + "loss": 2.4985, + "theoretical_loss": 5.731448685417178, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004503968253968254, + "loss": 2.2532, + "theoretical_loss": 5.7273438695529535, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 2.4096, + "theoretical_loss": 5.7232621347738455, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045436507936507937, + "loss": 2.1835, + "theoretical_loss": 5.719203250914208, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004563492063492063, + "loss": 2.4457, + "theoretical_loss": 5.715166991095922, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004583333333333333, + "loss": 2.598, + "theoretical_loss": 5.71115313166738, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 2.5442, + "theoretical_loss": 5.707161452143879, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046230158730158733, + "loss": 2.2213, + "theoretical_loss": 5.7031917351493515, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046428571428571433, + "loss": 2.178, + "theoretical_loss": 5.699243766359421, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004662698412698413, + "loss": 2.4453, + "theoretical_loss": 5.695317334445736, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 2.4695, + "theoretical_loss": 5.691412231021549, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047023809523809523, + "loss": 2.4932, + "theoretical_loss": 5.687528250588518, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.21346530318260193, + "objective/train/avg_token_score": 0.01769358292222023, + "objective/train/docs_used": 30152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2651164531707764, + "objective/train/original_loss": 6.490805149078369, + "objective/train/theoretical_loss": 5.6855941180524265, + "objective/train/tokens_used": 31129600, + "objective/train/unlikelihood_loss": 0.0511309988796711, + "theoretical_loss": 5.6855941180524265, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047222222222222224, + "loss": 2.1794, + "theoretical_loss": 5.683665190484683, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004742063492063492, + "loss": 2.251, + "theoretical_loss": 5.679822850833591, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 2.3855, + "theoretical_loss": 5.676001034494554, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004781746031746032, + "loss": 2.352, + "theoretical_loss": 5.672199547013983, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004801587301587302, + "loss": 2.5004, + "theoretical_loss": 5.6684181965778, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048214285714285715, + "loss": 2.3015, + "theoretical_loss": 5.6646567939648715, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048412698412698415, + "loss": 2.358, + "theoretical_loss": 5.660915152501465, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004861111111111111, + "loss": 2.496, + "theoretical_loss": 5.657193088016677, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004880952380952381, + "loss": 2.3284, + "theoretical_loss": 5.653490418798825, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004900793650793651, + "loss": 2.2022, + "theoretical_loss": 5.649806965552774, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492063492063492, + "loss": 2.1381, + "theoretical_loss": 5.6461425513581665, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004940476190476191, + "loss": 2.6351, + "theoretical_loss": 5.6424970016285485, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4376832842826843, + "objective/train/avg_token_score": 0.009709020145237446, + "objective/train/docs_used": 30758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.5283288955688477, + "objective/train/original_loss": 5.68566370010376, + "objective/train/theoretical_loss": 5.638870144071353, + "objective/train/tokens_used": 32768000, + "objective/train/unlikelihood_loss": 0.07930290699005127, + "theoretical_loss": 5.638870144071353, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496031746031746, + "loss": 2.3134, + "theoretical_loss": 5.638870144071353, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498015873015873, + "loss": 2.3275, + "theoretical_loss": 5.635261808648728, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 2.2282, + "theoretical_loss": 5.631671827539186, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999799398194584, + "loss": 2.256, + "theoretical_loss": 5.628100035100061, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999598796389167, + "loss": 2.2104, + "theoretical_loss": 5.6245462678307385, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999398194583751, + "loss": 2.2591, + "theoretical_loss": 5.621010364336651, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999197592778335, + "loss": 2.4118, + "theoretical_loss": 5.61749216529402, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998996990972919, + "loss": 2.2046, + "theoretical_loss": 5.61399151341532, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998796389167503, + "loss": 2.3628, + "theoretical_loss": 5.610508253415453, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998595787362087, + "loss": 2.3158, + "theoretical_loss": 5.6070422319786095, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499839518555667, + "loss": 2.2646, + "theoretical_loss": 5.603593297725807, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998194583751254, + "loss": 2.4531, + "theoretical_loss": 5.600161301183084, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997993981945837, + "loss": 2.3033, + "theoretical_loss": 5.596746094750342, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.31781524419784546, + "objective/train/avg_token_score": 0.06681742519140244, + "objective/train/docs_used": 31355, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7797188758850098, + "objective/train/original_loss": 5.599770545959473, + "objective/train/theoretical_loss": 5.595044742205037, + "objective/train/tokens_used": 34406400, + "objective/train/unlikelihood_loss": 0.1539888083934784, + "theoretical_loss": 5.595044742205037, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997793380140421, + "loss": 2.0253, + "theoretical_loss": 5.5933475326707995, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997592778335005, + "loss": 2.3513, + "theoretical_loss": 5.589965471001077, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997392176529588, + "loss": 2.385, + "theoretical_loss": 5.586599767581859, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997191574724173, + "loss": 2.4408, + "theoretical_loss": 5.583250282009159, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996990972918757, + "loss": 2.2335, + "theoretical_loss": 5.579916875606134, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499679037111334, + "loss": 2.2096, + "theoretical_loss": 5.576599411395472, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996589769307924, + "loss": 2.5149, + "theoretical_loss": 5.5732977540723105, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996389167502507, + "loss": 2.1646, + "theoretical_loss": 5.570011769977693, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996188565697091, + "loss": 2.2061, + "theoretical_loss": 5.566741327072535, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995987963891675, + "loss": 2.462, + "theoretical_loss": 5.563486294912105, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995787362086258, + "loss": 2.1712, + "theoretical_loss": 5.560246544620993, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995586760280842, + "loss": 2.4396, + "theoretical_loss": 5.557021948868571, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.37939882278442383, + "objective/train/avg_token_score": 0.02247542515397072, + "objective/train/docs_used": 32385, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2949864864349365, + "objective/train/original_loss": 5.736522674560547, + "objective/train/theoretical_loss": 5.553812381844907, + "objective/train/tokens_used": 36044800, + "objective/train/unlikelihood_loss": 0.08487144112586975, + "theoretical_loss": 5.553812381844907, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995386158475427, + "loss": 2.2289, + "theoretical_loss": 5.553812381844907, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499518555667001, + "loss": 2.3559, + "theoretical_loss": 5.550617719237167, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994984954864594, + "loss": 2.1262, + "theoretical_loss": 5.547437838206435, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994784353059178, + "loss": 2.2879, + "theoretical_loss": 5.544272617365014, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994583751253761, + "loss": 2.2899, + "theoretical_loss": 5.541121936754111, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994383149448345, + "loss": 2.1249, + "theoretical_loss": 5.537985677821986, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994182547642928, + "loss": 2.5925, + "theoretical_loss": 5.5348637234024824, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993981945837512, + "loss": 2.3988, + "theoretical_loss": 5.5317559576939725, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993781344032096, + "loss": 2.1127, + "theoretical_loss": 5.5286622662386975, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499358074222668, + "loss": 2.2598, + "theoretical_loss": 5.525582535902489, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993380140421264, + "loss": 2.2746, + "theoretical_loss": 5.52251665485486, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993179538615848, + "loss": 2.0758, + "theoretical_loss": 5.519464512549478, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992978936810431, + "loss": 2.0689, + "theoretical_loss": 5.516425999704987, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4496578574180603, + "objective/train/avg_token_score": 0.006494433153420687, + "objective/train/docs_used": 32883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.570333480834961, + "objective/train/original_loss": 5.788158416748047, + "objective/train/theoretical_loss": 5.514911820521407, + "objective/train/tokens_used": 37683200, + "objective/train/unlikelihood_loss": 0.06332194805145264, + "theoretical_loss": 5.514911820521407, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992778335005015, + "loss": 2.2661, + "theoretical_loss": 5.5134010082861895, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992577733199598, + "loss": 2.4036, + "theoretical_loss": 5.51038943148556, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992377131394183, + "loss": 2.6818, + "theoretical_loss": 5.50739116370511, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992176529588767, + "loss": 2.1309, + "theoretical_loss": 5.5044061005385725, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499197592778335, + "loss": 2.3905, + "theoretical_loss": 5.501434138753918, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991775325977934, + "loss": 2.5159, + "theoretical_loss": 5.498475176276176, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991574724172518, + "loss": 2.6962, + "theoretical_loss": 5.495529112170568, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991374122367101, + "loss": 2.1093, + "theoretical_loss": 5.492595846625951, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991173520561685, + "loss": 2.4076, + "theoretical_loss": 5.489675280938547, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499097291875627, + "loss": 2.3561, + "theoretical_loss": 5.486767317495966, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990772316950853, + "loss": 2.4408, + "theoretical_loss": 5.483871859761511, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990571715145437, + "loss": 2.1575, + "theoretical_loss": 5.480988812258763, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.41135141253471375, + "objective/train/avg_token_score": 0.026532353833317757, + "objective/train/docs_used": 34166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.235038995742798, + "objective/train/original_loss": 5.851961612701416, + "objective/train/theoretical_loss": 5.478118080556438, + "objective/train/tokens_used": 39321600, + "objective/train/unlikelihood_loss": 0.04781878739595413, + "theoretical_loss": 5.478118080556438, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499037111334002, + "loss": 2.1814, + "theoretical_loss": 5.478118080556438, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990170511534604, + "loss": 2.2678, + "theoretical_loss": 5.475259571253502, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989969909729188, + "loss": 2.4555, + "theoretical_loss": 5.4724131919645576, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989769307923771, + "loss": 2.2421, + "theoretical_loss": 5.4695788513054815, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989568706118355, + "loss": 2.2966, + "theoretical_loss": 5.466756458879306, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989368104312939, + "loss": 2.578, + "theoretical_loss": 5.463945925262355, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989167502507523, + "loss": 1.9973, + "theoretical_loss": 5.461147161990611, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988966900702107, + "loss": 2.4388, + "theoretical_loss": 5.458360081546321, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988766298896691, + "loss": 2.4184, + "theoretical_loss": 5.455584597344835, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988565697091274, + "loss": 2.1967, + "theoretical_loss": 5.452820623721662, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988365095285858, + "loss": 2.3451, + "theoretical_loss": 5.450068075919752, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988164493480441, + "loss": 2.0896, + "theoretical_loss": 5.447326870076996, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987963891675025, + "loss": 2.1861, + "theoretical_loss": 5.444596923213931, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.33492180705070496, + "objective/train/avg_token_score": 0.03136708214879036, + "objective/train/docs_used": 34866, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9246761798858643, + "objective/train/original_loss": 5.993686199188232, + "objective/train/theoretical_loss": 5.443236146207074, + "objective/train/tokens_used": 40960000, + "objective/train/unlikelihood_loss": 0.09010782837867737, + "theoretical_loss": 5.443236146207074, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987763289869609, + "loss": 2.2846, + "theoretical_loss": 5.441878153221662, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987562688064192, + "loss": 2.4934, + "theoretical_loss": 5.439170478849976, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987362086258777, + "loss": 1.8671, + "theoretical_loss": 5.4364738196956655, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987161484453361, + "loss": 2.2663, + "theoretical_loss": 5.433788096191039, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986960882647944, + "loss": 2.3139, + "theoretical_loss": 5.4311132295926345, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986760280842528, + "loss": 2.5888, + "theoretical_loss": 5.428449141970107, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986559679037111, + "loss": 2.1469, + "theoretical_loss": 5.42579575619531, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986359077231695, + "loss": 2.3618, + "theoretical_loss": 5.423152995931552, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986158475426279, + "loss": 2.3092, + "theoretical_loss": 5.420520785623031, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985957873620862, + "loss": 2.3179, + "theoretical_loss": 5.417899050484451, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985757271815446, + "loss": 2.4495, + "theoretical_loss": 5.415287716490787, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985556670010031, + "loss": 2.2482, + "theoretical_loss": 5.412686710367245, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3299120366573334, + "objective/train/avg_token_score": 0.04832848906517029, + "objective/train/docs_used": 35868, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8417558670043945, + "objective/train/original_loss": 5.99447774887085, + "objective/train/theoretical_loss": 5.410095959579362, + "objective/train/tokens_used": 42598400, + "objective/train/unlikelihood_loss": 0.052462801337242126, + "theoretical_loss": 5.410095959579362, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985356068204614, + "loss": 1.9885, + "theoretical_loss": 5.410095959579362, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985155466399198, + "loss": 2.4452, + "theoretical_loss": 5.407515392323276, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984954864593782, + "loss": 2.3031, + "theoretical_loss": 5.404944937516161, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984754262788365, + "loss": 2.2327, + "theoretical_loss": 5.402384524786797, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984553660982949, + "loss": 1.8473, + "theoretical_loss": 5.399834084466306, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984353059177532, + "loss": 2.2829, + "theoretical_loss": 5.397293547579041, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984152457372116, + "loss": 2.2311, + "theoretical_loss": 5.394762845833601, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049839518555667, + "loss": 2.3256, + "theoretical_loss": 5.392241911614005, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983751253761284, + "loss": 2.2566, + "theoretical_loss": 5.389730677971002, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983550651955868, + "loss": 2.5088, + "theoretical_loss": 5.387229078613521, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983350050150452, + "loss": 2.1468, + "theoretical_loss": 5.384737047900243, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983149448345035, + "loss": 2.3067, + "theoretical_loss": 5.382254520831328, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982948846539619, + "loss": 1.9734, + "theoretical_loss": 5.379781433040252, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.345246821641922, + "objective/train/avg_token_score": 0.005966670345515013, + "objective/train/docs_used": 36544, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9924170970916748, + "objective/train/original_loss": 5.951955795288086, + "objective/train/theoretical_loss": 5.378548408930558, + "objective/train/tokens_used": 44236800, + "objective/train/unlikelihood_loss": 0.06286505609750748, + "theoretical_loss": 5.378548408930558, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982748244734202, + "loss": 2.169, + "theoretical_loss": 5.377317720785777, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982547642928786, + "loss": 2.5078, + "theoretical_loss": 5.374863320944057, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498234704112337, + "loss": 2.2855, + "theoretical_loss": 5.372418171000847, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982146439317953, + "loss": 2.2421, + "theoretical_loss": 5.369982209043851, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981945837512538, + "loss": 2.4581, + "theoretical_loss": 5.367555373755179, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981745235707122, + "loss": 2.3194, + "theoretical_loss": 5.365137604403923, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981544633901705, + "loss": 2.1544, + "theoretical_loss": 5.362728840838843, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981344032096289, + "loss": 2.0627, + "theoretical_loss": 5.360329023481169, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981143430290873, + "loss": 2.2282, + "theoretical_loss": 5.357938093317518, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980942828485456, + "loss": 2.1852, + "theoretical_loss": 5.355555991892905, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498074222668004, + "loss": 2.147, + "theoretical_loss": 5.353182661303873, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980541624874623, + "loss": 2.3338, + "theoretical_loss": 5.350818044191721, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4638318717479706, + "objective/train/avg_token_score": 0.017905525863170624, + "objective/train/docs_used": 37915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.485879421234131, + "objective/train/original_loss": 5.818847179412842, + "objective/train/theoretical_loss": 5.348462083735834, + "objective/train/tokens_used": 45875200, + "objective/train/unlikelihood_loss": 0.05798850208520889, + "theoretical_loss": 5.348462083735834, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980341023069207, + "loss": 2.2988, + "theoretical_loss": 5.348462083735834, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980140421263792, + "loss": 2.3518, + "theoretical_loss": 5.346114723647119, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979939819458375, + "loss": 2.4436, + "theoretical_loss": 5.343775908161532, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979739217652959, + "loss": 2.5148, + "theoretical_loss": 5.341445582033705, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979538615847543, + "loss": 2.2889, + "theoretical_loss": 5.339123690530673, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979338014042126, + "loss": 2.4053, + "theoretical_loss": 5.336810179425685, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497913741223671, + "loss": 2.5862, + "theoretical_loss": 5.334504994992115, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978936810431293, + "loss": 2.2837, + "theoretical_loss": 5.332208083997459, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978736208625877, + "loss": 2.2526, + "theoretical_loss": 5.329919393697422, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978535606820461, + "loss": 2.2839, + "theoretical_loss": 5.327638871830089, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978335005015045, + "loss": 2.1244, + "theoretical_loss": 5.32536646661019, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978134403209629, + "loss": 2.3265, + "theoretical_loss": 5.323102126723439, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977933801404213, + "loss": 2.1693, + "theoretical_loss": 5.320845801320959, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.5400782227516174, + "objective/train/avg_token_score": 0.00798075646162033, + "objective/train/docs_used": 38651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.944997549057007, + "objective/train/original_loss": 5.1642985343933105, + "objective/train/theoretical_loss": 5.319720628286955, + "objective/train/tokens_used": 47513600, + "objective/train/unlikelihood_loss": 0.09301882237195969, + "theoretical_loss": 5.319720628286955, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977733199598796, + "loss": 2.4382, + "theoretical_loss": 5.318597440013795, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497753259779338, + "loss": 2.4145, + "theoretical_loss": 5.316356992867491, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977331995987965, + "loss": 2.3468, + "theoretical_loss": 5.314124410396767, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977131394182548, + "loss": 1.8894, + "theoretical_loss": 5.311899643560251, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976930792377132, + "loss": 2.3309, + "theoretical_loss": 5.3096826437553, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976730190571715, + "loss": 2.1528, + "theoretical_loss": 5.3074733628129005, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976529588766299, + "loss": 2.1967, + "theoretical_loss": 5.305271752992619, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976328986960883, + "loss": 2.3363, + "theoretical_loss": 5.303077766977653, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976128385155466, + "loss": 2.1614, + "theoretical_loss": 5.300891357869929, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497592778335005, + "loss": 2.4155, + "theoretical_loss": 5.298712479185288, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975727181544635, + "loss": 2.1652, + "theoretical_loss": 5.296541084848727, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975526579739218, + "loss": 2.2516, + "theoretical_loss": 5.294377129189715, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4233260154724121, + "objective/train/avg_token_score": 0.01209855917841196, + "objective/train/docs_used": 39958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.3168957233428955, + "objective/train/original_loss": 5.3540496826171875, + "objective/train/theoretical_loss": 5.292220566937567, + "objective/train/tokens_used": 49152000, + "objective/train/unlikelihood_loss": 0.07595264166593552, + "theoretical_loss": 5.292220566937567, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975325977933802, + "loss": 2.3305, + "theoretical_loss": 5.292220566937567, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975125376128386, + "loss": 2.3771, + "theoretical_loss": 5.290071353216895, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974924774322969, + "loss": 2.1448, + "theoretical_loss": 5.28792944354311, + "tokens_seen": 49414144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974724172517553, + "loss": 2.3501, + "theoretical_loss": 5.285794793817999, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974523570712136, + "loss": 2.0919, + "theoretical_loss": 5.283667360325351, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497432296890672, + "loss": 2.1672, + "theoretical_loss": 5.281547099726654, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974122367101304, + "loss": 2.2725, + "theoretical_loss": 5.279433969056848, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973921765295888, + "loss": 2.3125, + "theoretical_loss": 5.277327925720137, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973721163490472, + "loss": 2.1681, + "theoretical_loss": 5.275228927485855, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973520561685056, + "loss": 2.1653, + "theoretical_loss": 5.273136932484399, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973319959879639, + "loss": 2.4031, + "theoretical_loss": 5.271051899203207, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973119358074223, + "loss": 2.0327, + "theoretical_loss": 5.268973786482794, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972918756268806, + "loss": 2.1619, + "theoretical_loss": 5.266902553512847, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.25061094760894775, + "objective/train/avg_token_score": 0.02978723682463169, + "objective/train/docs_used": 40584, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3529692888259888, + "objective/train/original_loss": 4.955077648162842, + "objective/train/theoretical_loss": 5.265869504275602, + "objective/train/tokens_used": 50790400, + "objective/train/unlikelihood_loss": 0.16319940984249115, + "theoretical_loss": 5.265869504275602, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497271815446339, + "loss": 2.1136, + "theoretical_loss": 5.264838159828369, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972517552657974, + "loss": 2.3096, + "theoretical_loss": 5.262780565305875, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972316950852557, + "loss": 2.3798, + "theoretical_loss": 5.260729730159641, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972116349047142, + "loss": 2.3046, + "theoretical_loss": 5.2586856149380035, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971915747241726, + "loss": 2.3706, + "theoretical_loss": 5.256648180519708, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971715145436309, + "loss": 2.0973, + "theoretical_loss": 5.25461738811031, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971514543630893, + "loss": 2.6331, + "theoretical_loss": 5.252593199238619, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313941825477, + "loss": 2.2023, + "theoretical_loss": 5.2505755757531904, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497111334002006, + "loss": 2.1155, + "theoretical_loss": 5.248564479818876, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970912738214644, + "loss": 2.3576, + "theoretical_loss": 5.246559873913396, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970712136409227, + "loss": 2.0642, + "theoretical_loss": 5.24456172082399, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970511534603811, + "loss": 2.393, + "theoretical_loss": 5.242569983644074, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.42399805784225464, + "objective/train/avg_token_score": 0.009449027478694916, + "objective/train/docs_used": 41941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1671478748321533, + "objective/train/original_loss": 5.030263423919678, + "objective/train/theoretical_loss": 5.240584625769978, + "objective/train/tokens_used": 52428800, + "objective/train/unlikelihood_loss": 0.07760672271251678, + "theoretical_loss": 5.240584625769978, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970310932798396, + "loss": 2.2038, + "theoretical_loss": 5.240584625769978, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970110330992979, + "loss": 2.1783, + "theoretical_loss": 5.238605610897698, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969909729187563, + "loss": 2.1103, + "theoretical_loss": 5.2366329030197125, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969709127382147, + "loss": 2.1368, + "theoretical_loss": 5.2346664664218245, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496950852557673, + "loss": 2.2045, + "theoretical_loss": 5.232706265680049, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969307923771314, + "loss": 1.911, + "theoretical_loss": 5.230752265657554, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969107321965897, + "loss": 2.103, + "theoretical_loss": 5.228804431501619, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968906720160481, + "loss": 2.4849, + "theoretical_loss": 5.226862728640651, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968706118355065, + "loss": 2.005, + "theoretical_loss": 5.2249271227812315, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968505516549649, + "loss": 2.2629, + "theoretical_loss": 5.222997579905204, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968304914744233, + "loss": 2.2283, + "theoretical_loss": 5.2210740662667945, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968104312938817, + "loss": 2.399, + "theoretical_loss": 5.219156548389775, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049679037111334, + "loss": 2.3043, + "theoretical_loss": 5.217244993064664, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.42821359634399414, + "objective/train/avg_token_score": 0.012333965860307217, + "objective/train/docs_used": 42708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.223114490509033, + "objective/train/original_loss": 5.237239837646484, + "objective/train/theoretical_loss": 5.216291441053366, + "objective/train/tokens_used": 54067200, + "objective/train/unlikelihood_loss": 0.06622930616140366, + "theoretical_loss": 5.216291441053366, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967703109327984, + "loss": 2.5708, + "theoretical_loss": 5.215339367345955, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967502507522568, + "loss": 2.3715, + "theoretical_loss": 5.2134396385493815, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967301905717151, + "loss": 2.1358, + "theoretical_loss": 5.211545774249233, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967101303911735, + "loss": 2.4727, + "theoretical_loss": 5.209657742275683, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496690070210632, + "loss": 2.102, + "theoretical_loss": 5.207775510712159, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966700100300903, + "loss": 2.2568, + "theoretical_loss": 5.205899047892753, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966499498495487, + "loss": 1.8779, + "theoretical_loss": 5.204028322399658, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496629889669007, + "loss": 2.4808, + "theoretical_loss": 5.202163303060633, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966098294884654, + "loss": 2.1792, + "theoretical_loss": 5.20030395894651, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965897693079238, + "loss": 2.3935, + "theoretical_loss": 5.198450259368721, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965697091273821, + "loss": 2.2388, + "theoretical_loss": 5.196602173876867, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965496489468405, + "loss": 2.209, + "theoretical_loss": 5.194759672256309, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.34781280159950256, + "objective/train/avg_token_score": 0.039099015295505524, + "objective/train/docs_used": 44116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9161739349365234, + "objective/train/original_loss": 5.530422687530518, + "objective/train/theoretical_loss": 5.192922724525789, + "objective/train/tokens_used": 55705600, + "objective/train/unlikelihood_loss": 0.06317880749702454, + "theoretical_loss": 5.192922724525789, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965295887662988, + "loss": 2.2517, + "theoretical_loss": 5.192922724525789, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965095285857573, + "loss": 2.2552, + "theoretical_loss": 5.19109130093509, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964894684052157, + "loss": 2.5353, + "theoretical_loss": 5.189265371962712, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496469408224674, + "loss": 2.2548, + "theoretical_loss": 5.187444908313586, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964493480441324, + "loss": 2.3618, + "theoretical_loss": 5.185629880916814, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964292878635908, + "loss": 1.9889, + "theoretical_loss": 5.18382026092343, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964092276830491, + "loss": 2.6223, + "theoretical_loss": 5.182016019704204, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963891675025075, + "loss": 2.2145, + "theoretical_loss": 5.180217128847451, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963691073219659, + "loss": 2.0782, + "theoretical_loss": 5.178423560156894, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963490471414242, + "loss": 2.1989, + "theoretical_loss": 5.176635285649521, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963289869608827, + "loss": 2.174, + "theoretical_loss": 5.174852277553498, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496308926780341, + "loss": 2.2559, + "theoretical_loss": 5.17307450830609, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962888665997994, + "loss": 2.221, + "theoretical_loss": 5.1713019505516105, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.39821603894233704, + "objective/train/avg_token_score": 0.009783880785107613, + "objective/train/docs_used": 44698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.974114179611206, + "objective/train/original_loss": 5.170925617218018, + "objective/train/theoretical_loss": 5.170417617491682, + "objective/train/tokens_used": 57344000, + "objective/train/unlikelihood_loss": 0.07565154135227203, + "theoretical_loss": 5.170417617491682, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962688064192578, + "loss": 2.1205, + "theoretical_loss": 5.169534577139395, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962487462387161, + "loss": 1.8749, + "theoretical_loss": 5.167772361121805, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962286860581746, + "loss": 2.1519, + "theoretical_loss": 5.1660152757522475, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496208625877633, + "loss": 2.2882, + "theoretical_loss": 5.164263294483226, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961885656970913, + "loss": 2.175, + "theoretical_loss": 5.162516390964408, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961685055165497, + "loss": 1.834, + "theoretical_loss": 5.160774539040716, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961484453360081, + "loss": 2.0703, + "theoretical_loss": 5.159037712750455, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961283851554664, + "loss": 2.0747, + "theoretical_loss": 5.157305886323435, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961083249749248, + "loss": 2.2992, + "theoretical_loss": 5.155579034179144, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960882647943831, + "loss": 2.192, + "theoretical_loss": 5.153857130924929, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960682046138415, + "loss": 2.4796, + "theoretical_loss": 5.152140151354191, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960481444333, + "loss": 2.053, + "theoretical_loss": 5.150428070444621, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.42919111251831055, + "objective/train/avg_token_score": 0.023179521784186363, + "objective/train/docs_used": 46033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9980182647705078, + "objective/train/original_loss": 5.299570560455322, + "objective/train/theoretical_loss": 5.1487208633564405, + "objective/train/tokens_used": 58982400, + "objective/train/unlikelihood_loss": 0.05876965448260307, + "theoretical_loss": 5.1487208633564405, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960280842527583, + "loss": 2.0719, + "theoretical_loss": 5.1487208633564405, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960080240722167, + "loss": 2.1807, + "theoretical_loss": 5.147018505430666, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959879638916751, + "loss": 1.8661, + "theoretical_loss": 5.145320972187402, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959679037111334, + "loss": 2.313, + "theoretical_loss": 5.143628239324139, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959478435305918, + "loss": 2.3328, + "theoretical_loss": 5.1419402827140885, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959277833500501, + "loss": 2.192, + "theoretical_loss": 5.140257078404524, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959077231695085, + "loss": 2.0898, + "theoretical_loss": 5.138578602615146, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958876629889669, + "loss": 2.1709, + "theoretical_loss": 5.1369048317364685, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958676028084253, + "loss": 2.0217, + "theoretical_loss": 5.135235742328217, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958475426278837, + "loss": 2.1712, + "theoretical_loss": 5.133571311117755, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958274824473421, + "loss": 2.1534, + "theoretical_loss": 5.131911514998518, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958074222668004, + "loss": 2.0424, + "theoretical_loss": 5.130256331028474, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957873620862588, + "loss": 2.0909, + "theoretical_loss": 5.128605736428597, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.31408846378326416, + "objective/train/avg_token_score": 0.03549068421125412, + "objective/train/docs_used": 46630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6117610931396484, + "objective/train/original_loss": 5.108785152435303, + "objective/train/theoretical_loss": 5.127782153068225, + "objective/train/tokens_used": 60620800, + "objective/train/unlikelihood_loss": 0.07724561542272568, + "theoretical_loss": 5.127782153068225, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957673019057172, + "loss": 2.0965, + "theoretical_loss": 5.126959708581356, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957472417251755, + "loss": 2.104, + "theoretical_loss": 5.125318225029231, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957271815446339, + "loss": 2.107, + "theoretical_loss": 5.12368126347323, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957071213640923, + "loss": 2.6384, + "theoretical_loss": 5.122048801771443, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956870611835507, + "loss": 2.4012, + "theoretical_loss": 5.120420817937591, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956670010030091, + "loss": 2.2013, + "theoretical_loss": 5.118797290139605, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956469408224674, + "loss": 2.0302, + "theoretical_loss": 5.1171781966982195, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956268806419258, + "loss": 2.0847, + "theoretical_loss": 5.11556351608558, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956068204613842, + "loss": 2.2934, + "theoretical_loss": 5.113953226923864, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955867602808425, + "loss": 2.5557, + "theoretical_loss": 5.112347307983919, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955667001003009, + "loss": 2.1684, + "theoretical_loss": 5.110745738183919, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955466399197592, + "loss": 2.2634, + "theoretical_loss": 5.109148496588032, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.44507575035095215, + "objective/train/avg_token_score": 0.011363647878170013, + "objective/train/docs_used": 47896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.073281764984131, + "objective/train/original_loss": 4.909066200256348, + "objective/train/theoretical_loss": 5.107555562405102, + "objective/train/tokens_used": 62259200, + "objective/train/unlikelihood_loss": 0.08025174587965012, + "theoretical_loss": 5.107555562405102, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955265797392177, + "loss": 2.0933, + "theoretical_loss": 5.107555562405102, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955065195586761, + "loss": 2.0567, + "theoretical_loss": 5.105966914987349, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954864593781344, + "loss": 2.1778, + "theoretical_loss": 5.10438253382908, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954663991975928, + "loss": 2.406, + "theoretical_loss": 5.102802398565418, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954463390170512, + "loss": 2.2442, + "theoretical_loss": 5.101226488971042, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954262788365095, + "loss": 1.9896, + "theoretical_loss": 5.09965478495894, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954062186559679, + "loss": 1.906, + "theoretical_loss": 5.09808726657918, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953861584754263, + "loss": 2.3592, + "theoretical_loss": 5.096523914017688, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953660982948846, + "loss": 2.2084, + "theoretical_loss": 5.094964707595047, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953460381143431, + "loss": 2.1077, + "theoretical_loss": 5.093409627765306, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953259779338014, + "loss": 2.1148, + "theoretical_loss": 5.091858655114796, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953059177532598, + "loss": 2.2873, + "theoretical_loss": 5.090311770360971, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952858575727182, + "loss": 2.1432, + "theoretical_loss": 5.088768954351249, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.5087976455688477, + "objective/train/avg_token_score": 0.013270500116050243, + "objective/train/docs_used": 48551, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.6218936443328857, + "objective/train/original_loss": 5.463879108428955, + "objective/train/theoretical_loss": 5.087999066175502, + "objective/train/tokens_used": 63897600, + "objective/train/unlikelihood_loss": 0.0694500133395195, + "theoretical_loss": 5.087999066175502, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952657973921765, + "loss": 2.1419, + "theoretical_loss": 5.0872301880618735, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952457372116349, + "loss": 1.9254, + "theoretical_loss": 5.085695452596788, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952256770310933, + "loss": 1.9951, + "theoretical_loss": 5.0841647291865115, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952056168505516, + "loss": 2.2865, + "theoretical_loss": 5.082637999187046, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049518555667001, + "loss": 2.0893, + "theoretical_loss": 5.0811152440787755, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951654964894685, + "loss": 2.2674, + "theoretical_loss": 5.079596445465386, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951454363089268, + "loss": 2.0007, + "theoretical_loss": 5.078081585072802, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951253761283852, + "loss": 2.0058, + "theoretical_loss": 5.076570644748123, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951053159478435, + "loss": 1.895, + "theoretical_loss": 5.075063606458576, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950852557673019, + "loss": 2.2423, + "theoretical_loss": 5.07356045229049, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950651955867603, + "loss": 2.0065, + "theoretical_loss": 5.072061164448261, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950451354062186, + "loss": 2.1188, + "theoretical_loss": 5.070565725253344, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.34683528542518616, + "objective/train/avg_token_score": 0.024793358519673347, + "objective/train/docs_used": 49128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.808795690536499, + "objective/train/original_loss": 5.487620830535889, + "objective/train/theoretical_loss": 5.069074117143246, + "objective/train/tokens_used": 65536000, + "objective/train/unlikelihood_loss": 0.0768253281712532, + "theoretical_loss": 5.069074117143246, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495025075225677, + "loss": 1.8184, + "theoretical_loss": 5.069074117143246, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950050150451354, + "loss": 1.9084, + "theoretical_loss": 5.067586322670541, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949849548645938, + "loss": 2.3579, + "theoretical_loss": 5.066102324501883, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949648946840522, + "loss": 2.1562, + "theoretical_loss": 5.064622105417033, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949448345035105, + "loss": 2.0672, + "theoretical_loss": 5.063145648307904, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949247743229689, + "loss": 1.9244, + "theoretical_loss": 5.061672936177604, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949047141424273, + "loss": 1.8588, + "theoretical_loss": 5.060203952139497, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948846539618856, + "loss": 1.9714, + "theoretical_loss": 5.0587386794162725, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494864593781344, + "loss": 2.1933, + "theoretical_loss": 5.057277101339023, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948445336008024, + "loss": 1.9907, + "theoretical_loss": 5.055819201346331, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948244734202607, + "loss": 2.2394, + "theoretical_loss": 5.054364962983367, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948044132397192, + "loss": 2.1289, + "theoretical_loss": 5.052914369900997, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947843530591776, + "loss": 2.362, + "theoretical_loss": 5.051467405854897, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3599095940589905, + "objective/train/avg_token_score": 0.01841607131063938, + "objective/train/docs_used": 50658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9209699630737305, + "objective/train/original_loss": 5.435420036315918, + "objective/train/theoretical_loss": 5.05074527967254, + "objective/train/tokens_used": 67174400, + "objective/train/unlikelihood_loss": 0.08842965960502625, + "theoretical_loss": 5.05074527967254, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947642928786359, + "loss": 2.086, + "theoretical_loss": 5.050024054704677, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947442326980943, + "loss": 1.9711, + "theoretical_loss": 5.048584300413019, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947241725175527, + "loss": 2.2735, + "theoretical_loss": 5.0471481270448155, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947041123370111, + "loss": 2.0652, + "theoretical_loss": 5.045715518766322, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946840521564695, + "loss": 1.9284, + "theoretical_loss": 5.044286459844319, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946639919759278, + "loss": 2.1492, + "theoretical_loss": 5.0428609346452795, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946439317953862, + "loss": 2.3224, + "theoretical_loss": 5.041438927634549, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946238716148446, + "loss": 2.1179, + "theoretical_loss": 5.040020423375525, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946038114343029, + "loss": 2.1086, + "theoretical_loss": 5.038605406528857, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945837512537613, + "loss": 2.042, + "theoretical_loss": 5.037193861851646, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945636910732196, + "loss": 2.3074, + "theoretical_loss": 5.035785774196654, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945436308926781, + "loss": 2.2202, + "theoretical_loss": 5.034381128511525, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3374877870082855, + "objective/train/avg_token_score": 0.023196162655949593, + "objective/train/docs_used": 51428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4241688251495361, + "objective/train/original_loss": 4.484879493713379, + "objective/train/theoretical_loss": 5.032979909838007, + "objective/train/tokens_used": 68812800, + "objective/train/unlikelihood_loss": 0.11043024063110352, + "theoretical_loss": 5.032979909838007, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945235707121365, + "loss": 1.958, + "theoretical_loss": 5.032979909838007, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945035105315948, + "loss": 2.023, + "theoretical_loss": 5.031582103311187, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944834503510532, + "loss": 1.984, + "theoretical_loss": 5.030187694158739, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944633901705116, + "loss": 2.2235, + "theoretical_loss": 5.028796667700159, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944433299899699, + "loss": 2.0468, + "theoretical_loss": 5.0274090093460355, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944232698094283, + "loss": 2.2206, + "theoretical_loss": 5.0260247045973045, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944032096288867, + "loss": 2.1891, + "theoretical_loss": 5.024643739044526, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494383149448345, + "loss": 2.2133, + "theoretical_loss": 5.023266098367161, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943630892678035, + "loss": 2.1522, + "theoretical_loss": 5.0218917683328534, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943430290872618, + "loss": 1.9963, + "theoretical_loss": 5.020520734796728, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943229689067202, + "loss": 2.0446, + "theoretical_loss": 5.019152983700687, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943029087261786, + "loss": 2.1678, + "theoretical_loss": 5.01778850107272, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828485456369, + "loss": 2.3371, + "theoretical_loss": 5.016427273026212, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4471529722213745, + "objective/train/avg_token_score": 0.013587483204901218, + "objective/train/docs_used": 52769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.070784568786621, + "objective/train/original_loss": 5.132371425628662, + "objective/train/theoretical_loss": 5.015747875154622, + "objective/train/tokens_used": 70451200, + "objective/train/unlikelihood_loss": 0.07507549971342087, + "theoretical_loss": 5.015747875154622, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942627883650953, + "loss": 1.8887, + "theoretical_loss": 5.015069285759269, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942427281845537, + "loss": 2.0888, + "theoretical_loss": 5.0137145255540405, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494222668004012, + "loss": 2.0747, + "theoretical_loss": 5.012362978776057, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942026078234704, + "loss": 1.994, + "theoretical_loss": 5.011014631873566, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941825476429289, + "loss": 2.2445, + "theoretical_loss": 5.009669471376882, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941624874623872, + "loss": 2.2045, + "theoretical_loss": 5.008327483897736, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941424272818456, + "loss": 2.0795, + "theoretical_loss": 5.006988656128635, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941223671013039, + "loss": 2.2727, + "theoretical_loss": 5.00565297484223, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941023069207623, + "loss": 2.0863, + "theoretical_loss": 5.004320426890686, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940822467402207, + "loss": 2.2777, + "theoretical_loss": 5.002990999205057, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494062186559679, + "loss": 2.0031, + "theoretical_loss": 5.001664678794671, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940421263791374, + "loss": 2.0639, + "theoretical_loss": 5.0003414527465235, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4258919954299927, + "objective/train/avg_token_score": 0.020259426906704903, + "objective/train/docs_used": 53229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.168090343475342, + "objective/train/original_loss": 5.140798091888428, + "objective/train/theoretical_loss": 4.999021308224664, + "objective/train/tokens_used": 72089600, + "objective/train/unlikelihood_loss": 0.11368393152952194, + "theoretical_loss": 4.999021308224664, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940220661985958, + "loss": 2.0618, + "theoretical_loss": 4.999021308224664, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940020060180542, + "loss": 1.9749, + "theoretical_loss": 4.997704232469606, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939819458375126, + "loss": 2.1991, + "theoretical_loss": 4.996390212797728, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493961885656971, + "loss": 2.288, + "theoretical_loss": 4.995079236600686, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939418254764293, + "loss": 1.9311, + "theoretical_loss": 4.993771291344839, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939217652958877, + "loss": 1.9251, + "theoretical_loss": 4.992466364570659, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493901705115346, + "loss": 1.8568, + "theoretical_loss": 4.991164443892175, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938816449348044, + "loss": 2.2996, + "theoretical_loss": 4.989865516996396, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938615847542628, + "loss": 2.1793, + "theoretical_loss": 4.988569571642756, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938415245737211, + "loss": 2.0888, + "theoretical_loss": 4.9872765956625615, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938214643931796, + "loss": 2.0231, + "theoretical_loss": 4.9859865769584335, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493801404212638, + "loss": 2.1265, + "theoretical_loss": 4.984699503503771, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937813440320963, + "loss": 2.1259, + "theoretical_loss": 4.9834153633422105, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.43444526195526123, + "objective/train/avg_token_score": 0.0062065767124295235, + "objective/train/docs_used": 53865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2150537967681885, + "objective/train/original_loss": 5.1299591064453125, + "objective/train/theoretical_loss": 4.982774389529053, + "objective/train/tokens_used": 73728000, + "objective/train/unlikelihood_loss": 0.08083882927894592, + "theoretical_loss": 4.982774389529053, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937612838515547, + "loss": 1.9495, + "theoretical_loss": 4.9821341445870875, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493741223671013, + "loss": 1.9215, + "theoretical_loss": 4.98085583542091, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937211634904714, + "loss": 2.021, + "theoretical_loss": 4.979580424094836, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937011033099298, + "loss": 1.8973, + "theoretical_loss": 4.978307898928149, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936810431293881, + "loss": 2.1481, + "theoretical_loss": 4.97703824830775, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936609829488465, + "loss": 1.8232, + "theoretical_loss": 4.975771460687641, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493640922768305, + "loss": 1.8557, + "theoretical_loss": 4.974507524588424, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936208625877633, + "loss": 1.8265, + "theoretical_loss": 4.973246428596802, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936008024072217, + "loss": 2.192, + "theoretical_loss": 4.971988161365077, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049358074222668, + "loss": 1.9407, + "theoretical_loss": 4.970732711610667, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935606820461384, + "loss": 2.2415, + "theoretical_loss": 4.96948006811561, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935406218655968, + "loss": 2.0764, + "theoretical_loss": 4.968230219726093, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.41452834010124207, + "objective/train/avg_token_score": 0.014995137229561806, + "objective/train/docs_used": 54881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9440230131149292, + "objective/train/original_loss": 4.934903144836426, + "objective/train/theoretical_loss": 4.966983155351962, + "objective/train/tokens_used": 75366400, + "objective/train/unlikelihood_loss": 0.07033995538949966, + "theoretical_loss": 4.966983155351962, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935205616850551, + "loss": 2.1972, + "theoretical_loss": 4.966983155351962, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935005015045135, + "loss": 2.145, + "theoretical_loss": 4.96573886396626, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934804413239719, + "loss": 1.9567, + "theoretical_loss": 4.964497334604748, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934603811434303, + "loss": 1.9129, + "theoretical_loss": 4.963258556365449, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934403209628887, + "loss": 2.0695, + "theoretical_loss": 4.962022518408183, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493420260782347, + "loss": 2.2744, + "theoretical_loss": 4.9607892099541075, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934002006018054, + "loss": 2.2838, + "theoretical_loss": 4.959558620285274, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933801404212638, + "loss": 1.9697, + "theoretical_loss": 4.958330738744172, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933600802407221, + "loss": 2.2717, + "theoretical_loss": 4.957105554733289, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933400200601805, + "loss": 2.0457, + "theoretical_loss": 4.955883057714669, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933199598796389, + "loss": 2.0651, + "theoretical_loss": 4.954663237209477, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932998996990972, + "loss": 2.0885, + "theoretical_loss": 4.9534460827975675, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932798395185557, + "loss": 2.2084, + "theoretical_loss": 4.952231584117056, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3369990289211273, + "objective/train/avg_token_score": 0.013007127679884434, + "objective/train/docs_used": 55424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6116856336593628, + "objective/train/original_loss": 4.758896350860596, + "objective/train/theoretical_loss": 4.951625327453812, + "objective/train/tokens_used": 77004800, + "objective/train/unlikelihood_loss": 0.08679111301898956, + "theoretical_loss": 4.951625327453812, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932597793380141, + "loss": 1.8865, + "theoretical_loss": 4.951019730863894, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932397191574724, + "loss": 2.1724, + "theoretical_loss": 4.94981051279145, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932196589769308, + "loss": 2.1451, + "theoretical_loss": 4.948603919710088, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931995987963893, + "loss": 1.9967, + "theoretical_loss": 4.947399941486762, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931795386158476, + "loss": 2.005, + "theoretical_loss": 4.946198568044602, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493159478435306, + "loss": 2.2193, + "theoretical_loss": 4.944999789362508, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931394182547643, + "loss": 1.9879, + "theoretical_loss": 4.9438035954747495, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931193580742227, + "loss": 2.0917, + "theoretical_loss": 4.942609976470566, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930992978936811, + "loss": 2.0912, + "theoretical_loss": 4.941418922493774, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930792377131394, + "loss": 2.1054, + "theoretical_loss": 4.940230423742372, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930591775325978, + "loss": 2.302, + "theoretical_loss": 4.939044470468156, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930391173520562, + "loss": 2.1726, + "theoretical_loss": 4.937861052976332, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4615102708339691, + "objective/train/avg_token_score": 0.007493212353438139, + "objective/train/docs_used": 56825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.250938892364502, + "objective/train/original_loss": 4.587989330291748, + "objective/train/theoretical_loss": 4.9366801616251355, + "objective/train/tokens_used": 78643200, + "objective/train/unlikelihood_loss": 0.1308807134628296, + "theoretical_loss": 4.9366801616251355, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930190571715146, + "loss": 2.0669, + "theoretical_loss": 4.9366801616251355, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492998996990973, + "loss": 1.8013, + "theoretical_loss": 4.935501786825457, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929789368104313, + "loss": 1.9622, + "theoretical_loss": 4.934325919040461, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929588766298897, + "loss": 1.9327, + "theoretical_loss": 4.933152548785222, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929388164493481, + "loss": 2.1009, + "theoretical_loss": 4.931981666626351, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929187562688064, + "loss": 2.2199, + "theoretical_loss": 4.930813263181631, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928986960882648, + "loss": 1.9262, + "theoretical_loss": 4.929647329119659, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928786359077232, + "loss": 2.1295, + "theoretical_loss": 4.928483855159485, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928585757271815, + "loss": 2.0885, + "theoretical_loss": 4.9273228320702565, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049283851554664, + "loss": 2.2262, + "theoretical_loss": 4.926164250670868, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928184553660984, + "loss": 2.1799, + "theoretical_loss": 4.925008101829608, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927983951855567, + "loss": 2.0411, + "theoretical_loss": 4.923854376463816, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927783350050151, + "loss": 2.0662, + "theoretical_loss": 4.92270306553954, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.46413734555244446, + "objective/train/avg_token_score": 0.03281039744615555, + "objective/train/docs_used": 57296, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.3605399131774902, + "objective/train/original_loss": 5.119383811950684, + "objective/train/theoretical_loss": 4.922128312683096, + "objective/train/tokens_used": 80281600, + "objective/train/unlikelihood_loss": 0.08986959606409073, + "theoretical_loss": 4.922128312683096, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927582748244734, + "loss": 1.9299, + "theoretical_loss": 4.921554160071194, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927382146439318, + "loss": 2.0743, + "theoretical_loss": 4.920407651121222, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927181544633902, + "loss": 2.0166, + "theoretical_loss": 4.919263529799759, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926980942828485, + "loss": 1.9821, + "theoretical_loss": 4.91812178726431, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926780341023069, + "loss": 1.962, + "theoretical_loss": 4.916982414719408, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926579739217654, + "loss": 1.8903, + "theoretical_loss": 4.915845403416299, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926379137412237, + "loss": 2.0539, + "theoretical_loss": 4.914710744652614, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926178535606821, + "loss": 2.0817, + "theoretical_loss": 4.913578429772047, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925977933801404, + "loss": 2.0058, + "theoretical_loss": 4.912448450164041, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925777331995988, + "loss": 1.741, + "theoretical_loss": 4.911320797263471, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925576730190572, + "loss": 2.1737, + "theoretical_loss": 4.910195462550334, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925376128385155, + "loss": 2.246, + "theoretical_loss": 4.909072437549434, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.46144917607307434, + "objective/train/avg_token_score": 0.00516983587294817, + "objective/train/docs_used": 58605, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2914438247680664, + "objective/train/original_loss": 4.962143421173096, + "objective/train/theoretical_loss": 4.907951713830082, + "objective/train/tokens_used": 81920000, + "objective/train/unlikelihood_loss": 0.07921816408634186, + "theoretical_loss": 4.907951713830082, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925175526579739, + "loss": 2.07, + "theoretical_loss": 4.907951713830082, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924974924774323, + "loss": 2.0655, + "theoretical_loss": 4.906833283005785, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924774322968907, + "loss": 2.093, + "theoretical_loss": 4.90571713673395, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924573721163491, + "loss": 2.3312, + "theoretical_loss": 4.904603266715578, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924373119358075, + "loss": 2.0983, + "theoretical_loss": 4.903491664694977, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924172517552658, + "loss": 1.9503, + "theoretical_loss": 4.902382322459456, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923971915747242, + "loss": 2.1464, + "theoretical_loss": 4.90127523183904, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923771313941825, + "loss": 2.1856, + "theoretical_loss": 4.900170384706181, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923570712136409, + "loss": 1.9381, + "theoretical_loss": 4.899067772975469, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923370110330993, + "loss": 2.3138, + "theoretical_loss": 4.897967388603346, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923169508525576, + "loss": 2.0845, + "theoretical_loss": 4.896869223587828, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922968906720161, + "loss": 2.1112, + "theoretical_loss": 4.895773269968219, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922768304914745, + "loss": 1.9807, + "theoretical_loss": 4.894679519824841, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4049975574016571, + "objective/train/avg_token_score": 0.012781642377376556, + "objective/train/docs_used": 59310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9555177688598633, + "objective/train/original_loss": 5.0520477294921875, + "objective/train/theoretical_loss": 4.894133468592984, + "objective/train/tokens_used": 83558400, + "objective/train/unlikelihood_loss": 0.09169325232505798, + "theoretical_loss": 4.894133468592984, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922567703109328, + "loss": 1.7533, + "theoretical_loss": 4.8935879652787495, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922367101303912, + "loss": 1.9519, + "theoretical_loss": 4.892498598491473, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922166499498495, + "loss": 2.1345, + "theoretical_loss": 4.891411411664727, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921965897693079, + "loss": 2.1, + "theoretical_loss": 4.890326397040158, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921765295887663, + "loss": 2.0142, + "theoretical_loss": 4.8892435468990705, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921564694082246, + "loss": 2.0119, + "theoretical_loss": 4.888162853562166, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492136409227683, + "loss": 2.2767, + "theoretical_loss": 4.88708430938928, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921163490471415, + "loss": 2.1413, + "theoretical_loss": 4.8860079067791204, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920962888665998, + "loss": 2.0281, + "theoretical_loss": 4.884933638169014, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920762286860582, + "loss": 1.802, + "theoretical_loss": 4.883861496034644, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920561685055166, + "loss": 1.8336, + "theoretical_loss": 4.8827914728898065, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920361083249749, + "loss": 2.0471, + "theoretical_loss": 4.881723561286149, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.35856547951698303, + "objective/train/avg_token_score": 0.008686823770403862, + "objective/train/docs_used": 60492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7955634593963623, + "objective/train/original_loss": 4.565763473510742, + "objective/train/theoretical_loss": 4.880657753812926, + "objective/train/tokens_used": 85196800, + "objective/train/unlikelihood_loss": 0.19698937237262726, + "theoretical_loss": 4.880657753812926, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920160481444333, + "loss": 1.9216, + "theoretical_loss": 4.880657753812926, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919959879638916, + "loss": 2.1886, + "theoretical_loss": 4.879594043096755, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049197592778335, + "loss": 2.0444, + "theoretical_loss": 4.87853242180136, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919558676028084, + "loss": 1.9928, + "theoretical_loss": 4.877472882627343, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919358074222668, + "loss": 1.9912, + "theoretical_loss": 4.876415418311928, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919157472417252, + "loss": 1.9497, + "theoretical_loss": 4.875360021628733, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918956870611836, + "loss": 2.1111, + "theoretical_loss": 4.874306685387525, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918756268806419, + "loss": 2.2713, + "theoretical_loss": 4.87325540243399, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918555667001003, + "loss": 2.0874, + "theoretical_loss": 4.872206165649493, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918355065195586, + "loss": 1.9654, + "theoretical_loss": 4.871158967950852, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491815446339017, + "loss": 2.0332, + "theoretical_loss": 4.8701138022901045, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917953861584754, + "loss": 2.2051, + "theoretical_loss": 4.8690706616542805, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917753259779337, + "loss": 2.0213, + "theoretical_loss": 4.868029539065176, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.45521751046180725, + "objective/train/avg_token_score": 0.008304348215460777, + "objective/train/docs_used": 61005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1370649337768555, + "objective/train/original_loss": 4.633736610412598, + "objective/train/theoretical_loss": 4.867509732366907, + "objective/train/tokens_used": 86835200, + "objective/train/unlikelihood_loss": 0.09587273001670837, + "theoretical_loss": 4.867509732366907, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917552657973922, + "loss": 1.7873, + "theoretical_loss": 4.866990427579129, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917352056168506, + "loss": 1.9276, + "theoretical_loss": 4.8659533202867955, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917151454363089, + "loss": 1.7595, + "theoretical_loss": 4.864918210312927, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916950852557673, + "loss": 2.029, + "theoretical_loss": 4.863885090816158, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916750250752258, + "loss": 1.8672, + "theoretical_loss": 4.862853954988781, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916549648946841, + "loss": 1.9145, + "theoretical_loss": 4.861824796056533, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916349047141425, + "loss": 1.8864, + "theoretical_loss": 4.860797607278385, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916148445336008, + "loss": 2.0223, + "theoretical_loss": 4.859772381946323, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915947843530592, + "loss": 1.8535, + "theoretical_loss": 4.858749113385144, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915747241725176, + "loss": 1.8087, + "theoretical_loss": 4.8577277949522415, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915546639919759, + "loss": 1.8308, + "theoretical_loss": 4.856708420037402, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915346038114343, + "loss": 1.8766, + "theoretical_loss": 4.8556909820625975, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.44208210706710815, + "objective/train/avg_token_score": 0.02312779426574707, + "objective/train/docs_used": 61597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.139789581298828, + "objective/train/original_loss": 4.875903129577637, + "objective/train/theoretical_loss": 4.854675474481779, + "objective/train/tokens_used": 88473600, + "objective/train/unlikelihood_loss": 0.07984444499015808, + "theoretical_loss": 4.854675474481779, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915145436308927, + "loss": 2.214, + "theoretical_loss": 4.854675474481779, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914944834503511, + "loss": 1.9395, + "theoretical_loss": 4.85366189078068, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914744232698095, + "loss": 1.9757, + "theoretical_loss": 4.852650224476609, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914543630892679, + "loss": 2.0743, + "theoretical_loss": 4.851640469118255, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343029087262, + "loss": 2.041, + "theoretical_loss": 4.850632618285486, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914142427281846, + "loss": 2.0281, + "theoretical_loss": 4.849626665589156, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913941825476429, + "loss": 1.8351, + "theoretical_loss": 4.8486226046709024, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913741223671013, + "loss": 2.1069, + "theoretical_loss": 4.847620429202967, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913540621865597, + "loss": 1.7571, + "theoretical_loss": 4.846620132887992, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491334002006018, + "loss": 2.0421, + "theoretical_loss": 4.845621709458831, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913139418254765, + "loss": 1.859, + "theoretical_loss": 4.844625152678364, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912938816449349, + "loss": 2.0505, + "theoretical_loss": 4.84363045633931, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912738214643932, + "loss": 1.7393, + "theoretical_loss": 4.84263761426404, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.34017595648765564, + "objective/train/avg_token_score": 0.00968579389154911, + "objective/train/docs_used": 63098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.632453441619873, + "objective/train/original_loss": 4.724969387054443, + "objective/train/theoretical_loss": 4.842141886652876, + "objective/train/tokens_used": 90112000, + "objective/train/unlikelihood_loss": 0.09992973506450653, + "theoretical_loss": 4.842141886652876, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912537612838516, + "loss": 2.0554, + "theoretical_loss": 4.841646620304388, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049123370110331, + "loss": 2.1008, + "theoretical_loss": 4.840657468341476, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912136409227683, + "loss": 1.9921, + "theoretical_loss": 4.839670152285526, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911935807422267, + "loss": 2.072, + "theoretical_loss": 4.838684666075682, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491173520561685, + "loss": 1.9095, + "theoretical_loss": 4.837701003679829, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911534603811434, + "loss": 1.877, + "theoretical_loss": 4.836719159094422, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911334002006019, + "loss": 1.9558, + "theoretical_loss": 4.835739126344298, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911133400200602, + "loss": 1.9039, + "theoretical_loss": 4.834760899482514, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910932798395186, + "loss": 2.1059, + "theoretical_loss": 4.833784472590165, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491073219658977, + "loss": 1.9337, + "theoretical_loss": 4.832809839776213, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910531594784353, + "loss": 2.2944, + "theoretical_loss": 4.831836995177319, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910330992978937, + "loss": 1.9808, + "theoretical_loss": 4.8308659329576695, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.19037145376205444, + "objective/train/avg_token_score": 0.031226102262735367, + "objective/train/docs_used": 63840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9143338203430176, + "objective/train/original_loss": 4.837706089019775, + "objective/train/theoretical_loss": 4.8298966473088125, + "objective/train/tokens_used": 91750400, + "objective/train/unlikelihood_loss": 0.17378956079483032, + "theoretical_loss": 4.8298966473088125, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491013039117352, + "loss": 1.7083, + "theoretical_loss": 4.8298966473088125, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909929789368104, + "loss": 1.9804, + "theoretical_loss": 4.8289291324494865, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909729187562688, + "loss": 1.9059, + "theoretical_loss": 4.827963382625454, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909528585757272, + "loss": 1.9218, + "theoretical_loss": 4.826999392109344, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909327983951856, + "loss": 1.8156, + "theoretical_loss": 4.826037155200478, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490912738214644, + "loss": 1.9604, + "theoretical_loss": 4.825076666224717, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908926780341023, + "loss": 1.8492, + "theoretical_loss": 4.824117919534297, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908726178535607, + "loss": 2.0128, + "theoretical_loss": 4.823160909507665, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490852557673019, + "loss": 1.7759, + "theoretical_loss": 4.822205630549329, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908324974924774, + "loss": 1.9094, + "theoretical_loss": 4.821252077089696, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908124373119358, + "loss": 1.9877, + "theoretical_loss": 4.820300243584913, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907923771313941, + "loss": 1.8244, + "theoretical_loss": 4.819350124516717, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907723169508526, + "loss": 1.9063, + "theoretical_loss": 4.818401714392279, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.49071359634399414, + "objective/train/avg_token_score": 0.014383784495294094, + "objective/train/docs_used": 65224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1994926929473877, + "objective/train/original_loss": 4.691728115081787, + "objective/train/theoretical_loss": 4.817928148474378, + "objective/train/tokens_used": 93388800, + "objective/train/unlikelihood_loss": 0.06607113778591156, + "theoretical_loss": 4.817928148474378, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490752256770311, + "loss": 1.9917, + "theoretical_loss": 4.817455007744052, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907321965897693, + "loss": 2.0181, + "theoretical_loss": 4.816509999129618, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907121364092277, + "loss": 1.9882, + "theoretical_loss": 4.815566683131536, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490692076228686, + "loss": 1.9407, + "theoretical_loss": 4.814625054357199, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906720160481444, + "loss": 1.7596, + "theoretical_loss": 4.813685107438679, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906519558676028, + "loss": 1.8142, + "theoretical_loss": 4.812746837032582, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906318956870611, + "loss": 1.6129, + "theoretical_loss": 4.811810237819904, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906118355065195, + "loss": 1.9627, + "theoretical_loss": 4.810875304505881, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490591775325978, + "loss": 1.9738, + "theoretical_loss": 4.809942031819853, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905717151454363, + "loss": 1.7409, + "theoretical_loss": 4.809010414515113, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905516549648947, + "loss": 2.0122, + "theoretical_loss": 4.808080447368766, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905315947843531, + "loss": 1.8492, + "theoretical_loss": 4.807152125181597, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.5782013535499573, + "objective/train/avg_token_score": 0.0024078439455479383, + "objective/train/docs_used": 65494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.5631401538848877, + "objective/train/original_loss": 4.700851917266846, + "objective/train/theoretical_loss": 4.8062254427779205, + "objective/train/tokens_used": 95027200, + "objective/train/unlikelihood_loss": 0.061158839613199234, + "theoretical_loss": 4.8062254427779205, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905115346038114, + "loss": 1.9437, + "theoretical_loss": 4.8062254427779205, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904914744232698, + "loss": 1.9594, + "theoretical_loss": 4.805300395005444, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904714142427281, + "loss": 1.9803, + "theoretical_loss": 4.8043769767351385, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904513540621865, + "loss": 1.9562, + "theoretical_loss": 4.803455182861087, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904312938816449, + "loss": 2.0377, + "theoretical_loss": 4.802535008300364, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904112337011033, + "loss": 2.0507, + "theoretical_loss": 4.801616447992888, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903911735205617, + "loss": 2.073, + "theoretical_loss": 4.8006994969013, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903711133400201, + "loss": 2.0056, + "theoretical_loss": 4.799784150010819, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903510531594784, + "loss": 1.9311, + "theoretical_loss": 4.798870402329115, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903309929789368, + "loss": 1.7706, + "theoretical_loss": 4.797958248886179, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903109327983952, + "loss": 1.8998, + "theoretical_loss": 4.797047684734192, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902908726178535, + "loss": 1.9212, + "theoretical_loss": 4.796138704947397, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902708124373119, + "loss": 1.6952, + "theoretical_loss": 4.795231304621968, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.32086998224258423, + "objective/train/avg_token_score": 0.05917775630950928, + "objective/train/docs_used": 66055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3923687934875488, + "objective/train/original_loss": 5.291764259338379, + "objective/train/theoretical_loss": 4.794778195230787, + "objective/train/tokens_used": 96665600, + "objective/train/unlikelihood_loss": 0.06859153509140015, + "theoretical_loss": 4.794778195230787, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902507522567703, + "loss": 1.8366, + "theoretical_loss": 4.794325478875885, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902306920762287, + "loss": 2.0257, + "theoretical_loss": 4.793421222848808, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902106318956871, + "loss": 1.9104, + "theoretical_loss": 4.792518531701948, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901905717151454, + "loss": 1.9913, + "theoretical_loss": 4.791617400617948, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901705115346038, + "loss": 1.8546, + "theoretical_loss": 4.790717824800755, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901504513540623, + "loss": 1.8143, + "theoretical_loss": 4.789819799475499, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901303911735206, + "loss": 1.8901, + "theoretical_loss": 4.788923319888369, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490110330992979, + "loss": 1.8487, + "theoretical_loss": 4.788028381306497, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900902708124374, + "loss": 1.868, + "theoretical_loss": 4.787134979017832, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900702106318957, + "loss": 1.8523, + "theoretical_loss": 4.786243108331024, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900501504513541, + "loss": 1.7257, + "theoretical_loss": 4.785352764575304, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900300902708124, + "loss": 1.9306, + "theoretical_loss": 4.784463943100367, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4745234549045563, + "objective/train/avg_token_score": 0.020105870440602303, + "objective/train/docs_used": 66835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.3115997314453125, + "objective/train/original_loss": 4.947286128997803, + "objective/train/theoretical_loss": 4.783576639276257, + "objective/train/tokens_used": 98304000, + "objective/train/unlikelihood_loss": 0.07566744834184647, + "theoretical_loss": 4.783576639276257, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900100300902708, + "loss": 1.7746, + "theoretical_loss": 4.783576639276257, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899899699097292, + "loss": 1.8378, + "theoretical_loss": 4.782690848493245, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899699097291876, + "loss": 2.0137, + "theoretical_loss": 4.781806566161723, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489949849548646, + "loss": 2.0675, + "theoretical_loss": 4.78092378771208, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899297893681044, + "loss": 2.0216, + "theoretical_loss": 4.780042508594596, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899097291875627, + "loss": 1.7411, + "theoretical_loss": 4.779162724279324, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898896690070211, + "loss": 1.7954, + "theoretical_loss": 4.778284430255981, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898696088264794, + "loss": 1.9314, + "theoretical_loss": 4.777407622033838, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898495486459378, + "loss": 1.8941, + "theoretical_loss": 4.776532295141601, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898294884653962, + "loss": 2.0467, + "theoretical_loss": 4.775658445127318, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898094282848545, + "loss": 1.875, + "theoretical_loss": 4.7747860675582485, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489789368104313, + "loss": 1.8207, + "theoretical_loss": 4.773915158020776, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897693079237714, + "loss": 1.8993, + "theoretical_loss": 4.773045712120284, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3229472041130066, + "objective/train/avg_token_score": 0.02979409135878086, + "objective/train/docs_used": 67525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.47669517993927, + "objective/train/original_loss": 4.8227763175964355, + "objective/train/theoretical_loss": 4.77261153666576, + "objective/train/tokens_used": 99942400, + "objective/train/unlikelihood_loss": 0.1576315313577652, + "theoretical_loss": 4.77261153666576, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897492477432297, + "loss": 1.9601, + "theoretical_loss": 4.772177725481062, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897291875626881, + "loss": 1.7757, + "theoretical_loss": 4.771311193746191, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897091273821465, + "loss": 1.8233, + "theoretical_loss": 4.770446112577445, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896890672016048, + "loss": 1.9513, + "theoretical_loss": 4.769582477655177, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896690070210632, + "loss": 1.8272, + "theoretical_loss": 4.768720284678228, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896489468405215, + "loss": 1.764, + "theoretical_loss": 4.767859529363809, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896288866599799, + "loss": 1.956, + "theoretical_loss": 4.767000207447417, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896088264794384, + "loss": 1.9563, + "theoretical_loss": 4.766142314682716, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895887662988967, + "loss": 1.8045, + "theoretical_loss": 4.765285846841444, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895687061183551, + "loss": 2.0327, + "theoretical_loss": 4.764430799713314, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895486459378135, + "loss": 1.6355, + "theoretical_loss": 4.763577169105912, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895285857572718, + "loss": 1.9126, + "theoretical_loss": 4.762724950844598, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.361131489276886, + "objective/train/avg_token_score": 0.010854127816855907, + "objective/train/docs_used": 68617, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5994964838027954, + "objective/train/original_loss": 4.556946754455566, + "objective/train/theoretical_loss": 4.761874140772408, + "objective/train/tokens_used": 101580800, + "objective/train/unlikelihood_loss": 0.12181789427995682, + "theoretical_loss": 4.761874140772408, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895085255767302, + "loss": 2.0572, + "theoretical_loss": 4.761874140772408, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894884653961885, + "loss": 1.6931, + "theoretical_loss": 4.761024734749958, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894684052156469, + "loss": 1.842, + "theoretical_loss": 4.760176728655345, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894483450351053, + "loss": 1.7938, + "theoretical_loss": 4.759330118384053, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894282848545637, + "loss": 1.8328, + "theoretical_loss": 4.758484899848854, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894082246740221, + "loss": 1.8125, + "theoretical_loss": 4.7576410689797175, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893881644934805, + "loss": 1.6074, + "theoretical_loss": 4.756798621723712, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893681043129388, + "loss": 1.7143, + "theoretical_loss": 4.755957554044917, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893480441323972, + "loss": 2.0152, + "theoretical_loss": 4.755117861924321, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893279839518556, + "loss": 1.8091, + "theoretical_loss": 4.754279541359738, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893079237713139, + "loss": 1.8848, + "theoretical_loss": 4.75344258836571, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892878635907723, + "loss": 1.9612, + "theoretical_loss": 4.752606998973421, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892678034102307, + "loss": 1.7873, + "theoretical_loss": 4.7517727692306035, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3702346086502075, + "objective/train/avg_token_score": 0.0168976578861475, + "objective/train/docs_used": 69265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7375133037567139, + "objective/train/original_loss": 4.848740100860596, + "objective/train/theoretical_loss": 4.751356162997164, + "objective/train/tokens_used": 103219200, + "objective/train/unlikelihood_loss": 0.1075160950422287, + "theoretical_loss": 4.751356162997164, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892477432296891, + "loss": 2.047, + "theoretical_loss": 4.750939895201443, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892276830491475, + "loss": 1.7528, + "theoretical_loss": 4.750108372966501, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892076228686058, + "loss": 1.9577, + "theoretical_loss": 4.749278198622617, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891875626880642, + "loss": 1.7456, + "theoretical_loss": 4.748449368282822, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891675025075226, + "loss": 1.6757, + "theoretical_loss": 4.747621878076252, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891474423269809, + "loss": 1.6139, + "theoretical_loss": 4.746795724148061, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891273821464393, + "loss": 2.0421, + "theoretical_loss": 4.745970902659338, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891073219658976, + "loss": 1.6816, + "theoretical_loss": 4.7451474097870125, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890872617853561, + "loss": 1.763, + "theoretical_loss": 4.744325241723777, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890672016048145, + "loss": 1.7075, + "theoretical_loss": 4.743504394678, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890471414242728, + "loss": 1.8281, + "theoretical_loss": 4.742684864873641, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890270812437312, + "loss": 1.7182, + "theoretical_loss": 4.741866648550168, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.35599949955940247, + "objective/train/avg_token_score": 0.05716551095247269, + "objective/train/docs_used": 70556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.608228087425232, + "objective/train/original_loss": 4.970082759857178, + "objective/train/theoretical_loss": 4.741049741962473, + "objective/train/tokens_used": 104857600, + "objective/train/unlikelihood_loss": 0.09267803281545639, + "theoretical_loss": 4.741049741962473, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890070210631896, + "loss": 1.6529, + "theoretical_loss": 4.741049741962473, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889869608826479, + "loss": 1.8281, + "theoretical_loss": 4.740234141380794, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889669007021063, + "loss": 1.7417, + "theoretical_loss": 4.739419843090626, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889468405215647, + "loss": 1.7041, + "theoretical_loss": 4.738606843392644, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488926780341023, + "loss": 1.5751, + "theoretical_loss": 4.737795138602624, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889067201604815, + "loss": 1.8483, + "theoretical_loss": 4.736984725051357, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888866599799398, + "loss": 1.7904, + "theoretical_loss": 4.736175599084576, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888665997993982, + "loss": 1.9943, + "theoretical_loss": 4.735367757062869, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888465396188566, + "loss": 1.7043, + "theoretical_loss": 4.734561195361609, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888264794383149, + "loss": 1.9254, + "theoretical_loss": 4.733755910370867, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888064192577733, + "loss": 1.7947, + "theoretical_loss": 4.732951898495341, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887863590772317, + "loss": 1.7915, + "theoretical_loss": 4.732149156154276, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048876629889669, + "loss": 1.7682, + "theoretical_loss": 4.731347679781386, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.429068922996521, + "objective/train/avg_token_score": 0.009868655353784561, + "objective/train/docs_used": 70950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9191758632659912, + "objective/train/original_loss": 4.865683078765869, + "objective/train/theoretical_loss": 4.730947415222481, + "objective/train/tokens_used": 106496000, + "objective/train/unlikelihood_loss": 0.10545031726360321, + "theoretical_loss": 4.730947415222481, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887462387161484, + "loss": 1.8063, + "theoretical_loss": 4.730547465824781, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887261785356069, + "loss": 2.0496, + "theoretical_loss": 4.72974851074689, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887061183550652, + "loss": 1.869, + "theoretical_loss": 4.728950811024383, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886860581745236, + "loss": 1.82, + "theoretical_loss": 4.728154363148102, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886659979939819, + "loss": 1.8344, + "theoretical_loss": 4.72735916362298, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886459378134403, + "loss": 1.6656, + "theoretical_loss": 4.726565208967973, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886258776328988, + "loss": 1.9221, + "theoretical_loss": 4.725772495715983, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886058174523571, + "loss": 1.8658, + "theoretical_loss": 4.724981020413787, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885857572718155, + "loss": 1.8208, + "theoretical_loss": 4.7241907796219635, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656970912739, + "loss": 1.748, + "theoretical_loss": 4.723401769914824, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885456369107322, + "loss": 1.9757, + "theoretical_loss": 4.722613987880335, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885255767301906, + "loss": 1.7293, + "theoretical_loss": 4.721827430120053, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4198435842990875, + "objective/train/avg_token_score": 0.03244104981422424, + "objective/train/docs_used": 72030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9712913036346436, + "objective/train/original_loss": 5.18431282043457, + "objective/train/theoretical_loss": 4.721042093249051, + "objective/train/tokens_used": 108134400, + "objective/train/unlikelihood_loss": 0.07222175598144531, + "theoretical_loss": 4.721042093249051, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488505516549649, + "loss": 2.0079, + "theoretical_loss": 4.721042093249051, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884854563691073, + "loss": 1.8136, + "theoretical_loss": 4.7202579738958494, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884653961885657, + "loss": 1.6593, + "theoretical_loss": 4.719475068702346, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884453360080241, + "loss": 1.8464, + "theoretical_loss": 4.718693374323747, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884252758274825, + "loss": 1.552, + "theoretical_loss": 4.717912887428501, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884052156469409, + "loss": 1.7877, + "theoretical_loss": 4.717133604698222, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883851554663992, + "loss": 1.6993, + "theoretical_loss": 4.716355522827633, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883650952858576, + "loss": 1.87, + "theoretical_loss": 4.715578638524491, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488345035105316, + "loss": 1.7024, + "theoretical_loss": 4.714802948509522, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883249749247743, + "loss": 1.7157, + "theoretical_loss": 4.714028449516356, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883049147442327, + "loss": 1.8335, + "theoretical_loss": 4.713255138291454, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882848545636911, + "loss": 1.7999, + "theoretical_loss": 4.712483011594056, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048826479438314946, + "loss": 1.6147, + "theoretical_loss": 4.7117120661961005, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3784213066101074, + "objective/train/avg_token_score": 0.07327920943498611, + "objective/train/docs_used": 72698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.72416353225708, + "objective/train/original_loss": 4.901858806610107, + "objective/train/theoretical_loss": 4.711327035479103, + "objective/train/tokens_used": 109772800, + "objective/train/unlikelihood_loss": 0.09020828455686569, + "theoretical_loss": 4.711327035479103, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882447342026078, + "loss": 1.836, + "theoretical_loss": 4.710942298882169, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822467402206624, + "loss": 1.8356, + "theoretical_loss": 4.710173706449419, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882046138415246, + "loss": 1.6668, + "theoretical_loss": 4.7094062857075185, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818455366098296, + "loss": 1.843, + "theoretical_loss": 4.708640033478584, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881644934804413, + "loss": 1.6348, + "theoretical_loss": 4.7078749465971175, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048814443329989974, + "loss": 1.885, + "theoretical_loss": 4.707111021909941, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881243731193581, + "loss": 1.4851, + "theoretical_loss": 4.706348256276138, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048810431293881646, + "loss": 1.9024, + "theoretical_loss": 4.705586646566987, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880842527582748, + "loss": 1.8789, + "theoretical_loss": 4.704826189665905, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880641925777332, + "loss": 1.9667, + "theoretical_loss": 4.7040668824683785, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880441323971916, + "loss": 1.7731, + "theoretical_loss": 4.7033087218819105, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802407221664997, + "loss": 1.782, + "theoretical_loss": 4.702551704825957, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.38880741596221924, + "objective/train/avg_token_score": 0.002843292895704508, + "objective/train/docs_used": 74080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.625144124031067, + "objective/train/original_loss": 4.229738235473633, + "objective/train/theoretical_loss": 4.701795828231866, + "objective/train/tokens_used": 111411200, + "objective/train/unlikelihood_loss": 0.14396952092647552, + "theoretical_loss": 4.701795828231866, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048800401203610833, + "loss": 1.9266, + "theoretical_loss": 4.701795828231866, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879839518555667, + "loss": 1.5747, + "theoretical_loss": 4.701041089042813, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879638916750251, + "loss": 1.7214, + "theoretical_loss": 4.700287484213753, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048794383149448347, + "loss": 1.9717, + "theoretical_loss": 4.6995350107113545, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048792377131394183, + "loss": 1.7515, + "theoretical_loss": 4.698783665513934, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879037111334002, + "loss": 1.8924, + "theoretical_loss": 4.698033445611415, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048788365095285856, + "loss": 1.9578, + "theoretical_loss": 4.697284348005253, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048786359077231697, + "loss": 1.5521, + "theoretical_loss": 4.696536369708386, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048784353059177533, + "loss": 1.7932, + "theoretical_loss": 4.695789507745176, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878234704112337, + "loss": 1.8788, + "theoretical_loss": 4.695043759151353, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048780341023069206, + "loss": 1.851, + "theoretical_loss": 4.694299120973957, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877833500501505, + "loss": 1.775, + "theoretical_loss": 4.693555590271282, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048776328986960884, + "loss": 1.7527, + "theoretical_loss": 4.692813164112819, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.32972875237464905, + "objective/train/avg_token_score": 0.019512729719281197, + "objective/train/docs_used": 74817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5578219890594482, + "objective/train/original_loss": 4.993590354919434, + "objective/train/theoretical_loss": 4.692442364324931, + "objective/train/tokens_used": 113049600, + "objective/train/unlikelihood_loss": 0.07117574661970139, + "theoretical_loss": 4.692442364324931, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877432296890672, + "loss": 1.6282, + "theoretical_loss": 4.692071839579201, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048772316950852556, + "loss": 1.7212, + "theoretical_loss": 4.691331613762153, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877031093279839, + "loss": 1.9666, + "theoretical_loss": 4.690592483764427, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048768304914744234, + "loss": 1.8759, + "theoretical_loss": 4.689854446699757, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876629889669007, + "loss": 1.9312, + "theoretical_loss": 4.689117499692798, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048764292878635907, + "loss": 1.9425, + "theoretical_loss": 4.688381639879076, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048762286860581743, + "loss": 1.6478, + "theoretical_loss": 4.687646864404934, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048760280842527584, + "loss": 1.9203, + "theoretical_loss": 4.686913170427477, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875827482447342, + "loss": 1.5842, + "theoretical_loss": 4.68618055511452, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048756268806419257, + "loss": 1.7842, + "theoretical_loss": 4.685449015644537, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048754262788365093, + "loss": 1.6122, + "theoretical_loss": 4.684718549206607, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048752256770310935, + "loss": 1.6103, + "theoretical_loss": 4.6839891530003595, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.42418134212493896, + "objective/train/avg_token_score": 0.013869252055883408, + "objective/train/docs_used": 76169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.927834153175354, + "objective/train/original_loss": 4.830661296844482, + "objective/train/theoretical_loss": 4.68326082423593, + "objective/train/tokens_used": 114688000, + "objective/train/unlikelihood_loss": 0.13057447969913483, + "theoretical_loss": 4.68326082423593, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875025075225677, + "loss": 1.8594, + "theoretical_loss": 4.68326082423593, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048748244734202607, + "loss": 1.8205, + "theoretical_loss": 4.682533560133901, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048746238716148443, + "loss": 1.8782, + "theoretical_loss": 4.681807357925257, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874423269809428, + "loss": 1.8902, + "theoretical_loss": 4.68108221485133, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874222668004012, + "loss": 1.7744, + "theoretical_loss": 4.680358128163747, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874022066198596, + "loss": 1.7876, + "theoretical_loss": 4.6796350951243895, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048738214643931794, + "loss": 1.6889, + "theoretical_loss": 4.678913113005333, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873620862587763, + "loss": 2.0774, + "theoretical_loss": 4.678192179088802, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873420260782347, + "loss": 1.8764, + "theoretical_loss": 4.677472290667122, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873219658976931, + "loss": 1.8666, + "theoretical_loss": 4.676753445042669, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048730190571715144, + "loss": 1.7627, + "theoretical_loss": 4.6760356395278215, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872818455366098, + "loss": 1.9188, + "theoretical_loss": 4.675318871444908, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048726178535606816, + "loss": 1.9219, + "theoretical_loss": 4.674603138126168, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.44238758087158203, + "objective/train/avg_token_score": 0.016831381246447563, + "objective/train/docs_used": 76931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.068297863006592, + "objective/train/original_loss": 4.9380340576171875, + "objective/train/theoretical_loss": 4.674245658672382, + "objective/train/tokens_used": 116326400, + "objective/train/unlikelihood_loss": 0.09188218414783478, + "theoretical_loss": 4.674245658672382, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872417251755266, + "loss": 1.8908, + "theoretical_loss": 4.673888436913694, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487221664994985, + "loss": 1.5917, + "theoretical_loss": 4.673174765159393, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048720160481444336, + "loss": 1.7106, + "theoretical_loss": 4.6724621202249335, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871815446339017, + "loss": 1.7435, + "theoretical_loss": 4.6717504994817, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048716148445336014, + "loss": 2.0931, + "theoretical_loss": 4.671039900310747, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871414242728185, + "loss": 1.889, + "theoretical_loss": 4.670330320102753, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048712136409227686, + "loss": 1.8596, + "theoretical_loss": 4.669621756257971, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871013039117352, + "loss": 1.8924, + "theoretical_loss": 4.668914206186189, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870812437311936, + "loss": 1.848, + "theoretical_loss": 4.66820766730668, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487061183550652, + "loss": 1.6911, + "theoretical_loss": 4.667502137048155, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048704112337011037, + "loss": 1.7743, + "theoretical_loss": 4.666797612848723, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048702106318956873, + "loss": 1.6988, + "theoretical_loss": 4.666094092155843, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5015273690223694, + "objective/train/avg_token_score": 0.006345341447740793, + "objective/train/docs_used": 78354, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.065946578979492, + "objective/train/original_loss": 3.913723945617676, + "objective/train/theoretical_loss": 4.665391572426282, + "objective/train/tokens_used": 117964800, + "objective/train/unlikelihood_loss": 0.22304238379001617, + "theoretical_loss": 4.665391572426282, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870010030090271, + "loss": 1.9779, + "theoretical_loss": 4.665391572426282, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869809428284855, + "loss": 1.8665, + "theoretical_loss": 4.664690051126065, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048696088264794387, + "loss": 1.5626, + "theoretical_loss": 4.66398952573044, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048694082246740223, + "loss": 1.8862, + "theoretical_loss": 4.663289993723826, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869207622868606, + "loss": 1.88, + "theoretical_loss": 4.662591452599774, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048690070210631896, + "loss": 1.8471, + "theoretical_loss": 4.661893899860923, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688064192577737, + "loss": 1.5922, + "theoretical_loss": 4.661197333018957, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686058174523573, + "loss": 1.8316, + "theoretical_loss": 4.6605017495945615, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868405215646941, + "loss": 1.7042, + "theoretical_loss": 4.659807147117382, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682046138415246, + "loss": 1.6569, + "theoretical_loss": 4.659113523125981, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868004012036109, + "loss": 1.6006, + "theoretical_loss": 4.658420875167799, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678034102306924, + "loss": 1.6682, + "theoretical_loss": 4.657729200799105, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867602808425276, + "loss": 1.7433, + "theoretical_loss": 4.657038497584967, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.359115332365036, + "objective/train/avg_token_score": 0.027856703847646713, + "objective/train/docs_used": 78905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.605509638786316, + "objective/train/original_loss": 4.950808048248291, + "objective/train/theoretical_loss": 4.656693509402331, + "objective/train/tokens_used": 119603200, + "objective/train/unlikelihood_loss": 0.09441498667001724, + "theoretical_loss": 4.656693509402331, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048674022066198596, + "loss": 1.599, + "theoretical_loss": 4.6563487630991975, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867201604814443, + "loss": 1.8791, + "theoretical_loss": 4.655659994924323, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048670010030090274, + "loss": 1.6709, + "theoretical_loss": 4.65497219065154, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866800401203611, + "loss": 1.7202, + "theoretical_loss": 4.654285347880672, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048665997993981947, + "loss": 1.8856, + "theoretical_loss": 4.653599464220129, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048663991975927783, + "loss": 1.8096, + "theoretical_loss": 4.6529145372868745, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048661985957873624, + "loss": 1.6635, + "theoretical_loss": 4.652230564706377, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865997993981946, + "loss": 1.6887, + "theoretical_loss": 4.651547544112575, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048657973921765297, + "loss": 1.6575, + "theoretical_loss": 4.650865473147837, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048655967903711133, + "loss": 1.7481, + "theoretical_loss": 4.650184349462922, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048653961885656975, + "loss": 1.7805, + "theoretical_loss": 4.6495041707169396, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865195586760281, + "loss": 1.5922, + "theoretical_loss": 4.648824934577313, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.28109726309776306, + "objective/train/avg_token_score": 0.07095440477132797, + "objective/train/docs_used": 79692, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.170123815536499, + "objective/train/original_loss": 4.673448085784912, + "objective/train/theoretical_loss": 4.648146638719739, + "objective/train/tokens_used": 121241600, + "objective/train/unlikelihood_loss": 0.12229027599096298, + "theoretical_loss": 4.648146638719739, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048649949849548647, + "loss": 1.7261, + "theoretical_loss": 4.648146638719739, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048647943831494483, + "loss": 1.8013, + "theoretical_loss": 4.647469280828153, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864593781344032, + "loss": 1.9773, + "theoretical_loss": 4.646792858594686, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864393179538616, + "loss": 1.6408, + "theoretical_loss": 4.646117369719629, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048641925777332, + "loss": 1.7331, + "theoretical_loss": 4.6454428119113995, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048639919759277834, + "loss": 1.6656, + "theoretical_loss": 4.644769182886495, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863791374122367, + "loss": 1.8518, + "theoretical_loss": 4.644096480369466, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863590772316951, + "loss": 1.8672, + "theoretical_loss": 4.6434247020928705, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863390170511535, + "loss": 1.8686, + "theoretical_loss": 4.642753845797243, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048631895687061184, + "loss": 1.9569, + "theoretical_loss": 4.642083909231053, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862988966900702, + "loss": 1.659, + "theoretical_loss": 4.641414890150675, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048627883650952857, + "loss": 1.8805, + "theoretical_loss": 4.6407467863203475, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486258776328987, + "loss": 1.8608, + "theoretical_loss": 4.6400795955121374, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.46236559748649597, + "objective/train/avg_token_score": 0.02207406610250473, + "objective/train/docs_used": 81051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.061617851257324, + "objective/train/original_loss": 4.578189849853516, + "objective/train/theoretical_loss": 4.639746341797229, + "objective/train/tokens_used": 122880000, + "objective/train/unlikelihood_loss": 0.10073511302471161, + "theoretical_loss": 4.639746341797229, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048623871614844534, + "loss": 1.9091, + "theoretical_loss": 4.639413315505905, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862186559679037, + "loss": 1.7783, + "theoretical_loss": 4.638747944089273, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048619859578736207, + "loss": 1.9328, + "theoretical_loss": 4.638083479057579, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861785356068205, + "loss": 1.8101, + "theoretical_loss": 4.6374199182138565, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048615847542627885, + "loss": 1.7789, + "theoretical_loss": 4.636757259368787, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861384152457372, + "loss": 1.9448, + "theoretical_loss": 4.636095500340669, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048611835506519557, + "loss": 1.7024, + "theoretical_loss": 4.635434638955388, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048609829488465393, + "loss": 1.8975, + "theoretical_loss": 4.634774673046376, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048607823470411235, + "loss": 1.6453, + "theoretical_loss": 4.634115600454582, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860581745235707, + "loss": 1.8373, + "theoretical_loss": 4.6334574190284314, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860381143430291, + "loss": 1.7563, + "theoretical_loss": 4.632800126623803, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048601805416248744, + "loss": 1.832, + "theoretical_loss": 4.632143721103983, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3342497646808624, + "objective/train/avg_token_score": 0.010767764411866665, + "objective/train/docs_used": 81846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.500130295753479, + "objective/train/original_loss": 4.459724426269531, + "objective/train/theoretical_loss": 4.631488200339643, + "objective/train/tokens_used": 124518400, + "objective/train/unlikelihood_loss": 0.17901530861854553, + "theoretical_loss": 4.631488200339643, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048599799398194585, + "loss": 1.8062, + "theoretical_loss": 4.631488200339643, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859779338014042, + "loss": 1.8382, + "theoretical_loss": 4.630833562208797, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859578736208626, + "loss": 1.7831, + "theoretical_loss": 4.630179804596775, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048593781344032094, + "loss": 1.7324, + "theoretical_loss": 4.629526925396189, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859177532597793, + "loss": 1.7767, + "theoretical_loss": 4.628874922506897, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858976930792377, + "loss": 1.801, + "theoretical_loss": 4.628223793835975, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858776328986961, + "loss": 1.8394, + "theoretical_loss": 4.627573537297678, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048585757271815444, + "loss": 1.8139, + "theoretical_loss": 4.6269241508134185, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858375125376128, + "loss": 1.8361, + "theoretical_loss": 4.6262756323117245, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858174523570712, + "loss": 2.1512, + "theoretical_loss": 4.625627979728212, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857973921765296, + "loss": 1.7211, + "theoretical_loss": 4.624981191005554, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577733199598795, + "loss": 1.7506, + "theoretical_loss": 4.624335264093447, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857572718154463, + "loss": 1.719, + "theoretical_loss": 4.623690196948582, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.2474951148033142, + "objective/train/avg_token_score": 0.018130095675587654, + "objective/train/docs_used": 83055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1577845811843872, + "objective/train/original_loss": 4.456354141235352, + "objective/train/theoretical_loss": 4.623367985152234, + "objective/train/tokens_used": 126156800, + "objective/train/unlikelihood_loss": 0.11982196569442749, + "theoretical_loss": 4.623367985152234, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573721163490467, + "loss": 1.7005, + "theoretical_loss": 4.623045987534609, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048571715145436314, + "loss": 2.0355, + "theoretical_loss": 4.622402633822114, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856970912738215, + "loss": 1.7999, + "theoretical_loss": 4.62176013378858, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048567703109327987, + "loss": 1.6439, + "theoretical_loss": 4.621118485418362, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048565697091273823, + "loss": 2.0235, + "theoretical_loss": 4.620477686702651, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563691073219664, + "loss": 1.805, + "theoretical_loss": 4.619837735639452, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485616850551655, + "loss": 1.8306, + "theoretical_loss": 4.619198630233547, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559679037111337, + "loss": 1.7948, + "theoretical_loss": 4.618560368496466, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048557673019057173, + "loss": 1.7666, + "theoretical_loss": 4.617922948446459, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048555667001003015, + "loss": 1.8654, + "theoretical_loss": 4.617286368108466, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855366098294885, + "loss": 1.7467, + "theoretical_loss": 4.616650625514091, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048551654964894687, + "loss": 1.7312, + "theoretical_loss": 4.616015718701563, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.39668866991996765, + "objective/train/avg_token_score": 0.008423499763011932, + "objective/train/docs_used": 83592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7686882019042969, + "objective/train/original_loss": 4.729675769805908, + "objective/train/theoretical_loss": 4.615381645715717, + "objective/train/tokens_used": 127795200, + "objective/train/unlikelihood_loss": 0.08303800970315933, + "theoretical_loss": 4.615381645715717, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048549648946840523, + "loss": 1.8034, + "theoretical_loss": 4.615381645715717, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854764292878636, + "loss": 1.7946, + "theoretical_loss": 4.61474840460796, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485456369107322, + "loss": 1.6519, + "theoretical_loss": 4.614115993436242, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854363089267804, + "loss": 1.9487, + "theoretical_loss": 4.613484410265032, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048541624874623874, + "loss": 1.6912, + "theoretical_loss": 4.612853653165283, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853961885656971, + "loss": 1.7975, + "theoretical_loss": 4.612223720214407, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853761283851555, + "loss": 1.6292, + "theoretical_loss": 4.61159460949625, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853560682046139, + "loss": 1.5301, + "theoretical_loss": 4.610966319101056, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533600802407224, + "loss": 1.8054, + "theoretical_loss": 4.610338847125445, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853159478435306, + "loss": 1.6353, + "theoretical_loss": 4.6097121916723856, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048529588766298897, + "loss": 1.8334, + "theoretical_loss": 4.609086350851165, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852758274824474, + "loss": 1.7218, + "theoretical_loss": 4.6084613227773605, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525576730190574, + "loss": 1.8052, + "theoretical_loss": 4.607837105572816, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5973240733146667, + "objective/train/avg_token_score": 0.010894504375755787, + "objective/train/docs_used": 84814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.5378358364105225, + "objective/train/original_loss": 4.310730934143066, + "objective/train/theoretical_loss": 4.607525300461299, + "objective/train/tokens_used": 129433600, + "objective/train/unlikelihood_loss": 0.1317676305770874, + "theoretical_loss": 4.607525300461299, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852357071213641, + "loss": 1.9714, + "theoretical_loss": 4.607213697365613, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521564694082247, + "loss": 1.7143, + "theoretical_loss": 4.6065910962900425, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851955867602809, + "loss": 2.0327, + "theoretical_loss": 4.60596930048658, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048517552657973925, + "loss": 1.8787, + "theoretical_loss": 4.6053483081018545, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851554663991976, + "loss": 1.9663, + "theoretical_loss": 4.604728117288631, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513540621865597, + "loss": 1.7322, + "theoretical_loss": 4.604108726205774, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048511534603811433, + "loss": 1.5917, + "theoretical_loss": 4.6034901330182265, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048509528585757275, + "loss": 1.7381, + "theoretical_loss": 4.6028723358969845, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850752256770311, + "loss": 1.8999, + "theoretical_loss": 4.602255333019068, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850551654964895, + "loss": 1.7749, + "theoretical_loss": 4.601639122567497, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503510531594784, + "loss": 2.0268, + "theoretical_loss": 4.601023702731264, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048501504513540625, + "loss": 1.8285, + "theoretical_loss": 4.600409071705312, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4414711594581604, + "objective/train/avg_token_score": 0.013736131601035595, + "objective/train/docs_used": 85560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.943097710609436, + "objective/train/original_loss": 4.5531487464904785, + "objective/train/theoretical_loss": 4.599795227690505, + "objective/train/tokens_used": 131072000, + "objective/train/unlikelihood_loss": 0.18566669523715973, + "theoretical_loss": 4.599795227690505, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849949849548646, + "loss": 1.7317, + "theoretical_loss": 4.599795227690505, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484974924774323, + "loss": 2.0678, + "theoretical_loss": 4.599182168893604, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048495486459378134, + "loss": 2.0053, + "theoretical_loss": 4.5985698935272445, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849348044132397, + "loss": 1.7232, + "theoretical_loss": 4.597958399809908, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849147442326981, + "loss": 1.7742, + "theoretical_loss": 4.597347685965897, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848946840521565, + "loss": 1.7775, + "theoretical_loss": 4.596737750225311, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048487462387161484, + "loss": 1.6808, + "theoretical_loss": 4.596128590824026, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848545636910732, + "loss": 1.6192, + "theoretical_loss": 4.595520206003663, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848345035105316, + "loss": 1.7225, + "theoretical_loss": 4.594912594011566, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048481444332999, + "loss": 1.9255, + "theoretical_loss": 4.594305753100782, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048479438314944835, + "loss": 1.7942, + "theoretical_loss": 4.59369968153003, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847743229689067, + "loss": 1.7619, + "theoretical_loss": 4.593094377563681, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048475426278836507, + "loss": 1.7052, + "theoretical_loss": 4.592489839471735, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4741568863391876, + "objective/train/avg_token_score": 0.008817518129944801, + "objective/train/docs_used": 86181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0522453784942627, + "objective/train/original_loss": 4.405289649963379, + "objective/train/theoretical_loss": 4.592187857089571, + "objective/train/tokens_used": 132710400, + "objective/train/unlikelihood_loss": 0.15506720542907715, + "theoretical_loss": 4.592187857089571, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847342026078235, + "loss": 1.6979, + "theoretical_loss": 4.591886065529795, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048471414242728185, + "loss": 1.6689, + "theoretical_loss": 4.591283054019041, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846940822467402, + "loss": 1.4962, + "theoretical_loss": 4.590680803226213, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846740220661986, + "loss": 1.6976, + "theoretical_loss": 4.590079311443583, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484653961885657, + "loss": 1.7468, + "theoretical_loss": 4.589478576968932, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048463390170511535, + "loss": 1.7804, + "theoretical_loss": 4.588878598105527, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846138415245737, + "loss": 1.6541, + "theoretical_loss": 4.588279373162101, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845937813440321, + "loss": 1.8706, + "theoretical_loss": 4.587680900452824, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048457372116349044, + "loss": 1.8239, + "theoretical_loss": 4.587083178297288, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048455366098294886, + "loss": 1.8302, + "theoretical_loss": 4.586486205020474, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845336008024072, + "loss": 1.772, + "theoretical_loss": 4.585889978952741, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845135406218656, + "loss": 1.6578, + "theoretical_loss": 4.585294498429796, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.49590665102005005, + "objective/train/avg_token_score": 0.008106932044029236, + "objective/train/docs_used": 87535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1396756172180176, + "objective/train/original_loss": 4.32695198059082, + "objective/train/theoretical_loss": 4.584699761792674, + "objective/train/tokens_used": 134348800, + "objective/train/unlikelihood_loss": 0.16700398921966553, + "theoretical_loss": 4.584699761792674, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048449348044132394, + "loss": 1.8098, + "theoretical_loss": 4.584699761792674, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048447342026078236, + "loss": 1.9279, + "theoretical_loss": 4.5841057673877135, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844533600802407, + "loss": 1.8587, + "theoretical_loss": 4.5835125135665375, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844332998996991, + "loss": 1.6187, + "theoretical_loss": 4.58291999868603, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048441323971915745, + "loss": 1.58, + "theoretical_loss": 4.582328221108318, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048439317953861586, + "loss": 1.9187, + "theoretical_loss": 4.581737179200739, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843731193580742, + "loss": 1.8012, + "theoretical_loss": 4.581146871335832, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843530591775326, + "loss": 1.745, + "theoretical_loss": 4.5805572958913086, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048433299899699095, + "loss": 1.8915, + "theoretical_loss": 4.579968451250032, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843129388164493, + "loss": 1.745, + "theoretical_loss": 4.579380335800001, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048429287863590773, + "loss": 1.6448, + "theoretical_loss": 4.5787929479343195, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842728184553661, + "loss": 1.9537, + "theoretical_loss": 4.578206286051184, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048425275827482445, + "loss": 1.8237, + "theoretical_loss": 4.577620348553859, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3742668628692627, + "objective/train/avg_token_score": 0.01183388289064169, + "objective/train/docs_used": 88189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5616800785064697, + "objective/train/original_loss": 4.133111476898193, + "objective/train/theoretical_loss": 4.577327650952276, + "objective/train/tokens_used": 135987200, + "objective/train/unlikelihood_loss": 0.14664104580879211, + "theoretical_loss": 4.577327650952276, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048423269809428287, + "loss": 1.6142, + "theoretical_loss": 4.5770351338506545, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842126379137413, + "loss": 1.8337, + "theoretical_loss": 4.57645064035491, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048419257773319965, + "loss": 1.8521, + "theoretical_loss": 4.575866866484967, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484172517552658, + "loss": 1.6806, + "theoretical_loss": 4.575283810664155, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048415245737211637, + "loss": 1.6517, + "theoretical_loss": 4.574701471320768, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048413239719157473, + "loss": 1.5807, + "theoretical_loss": 4.574119846888045, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048411233701103315, + "loss": 1.6995, + "theoretical_loss": 4.573538935804146, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840922768304915, + "loss": 1.7618, + "theoretical_loss": 4.5729587365121365, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840722166499499, + "loss": 1.7269, + "theoretical_loss": 4.572379247459969, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048405215646940824, + "loss": 1.629, + "theoretical_loss": 4.571800467100456, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048403209628886665, + "loss": 1.7945, + "theoretical_loss": 4.571222393891253, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484012036108325, + "loss": 1.6366, + "theoretical_loss": 4.570645026294844, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4571114480495453, + "objective/train/avg_token_score": 0.013456963934004307, + "objective/train/docs_used": 88907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0327019691467285, + "objective/train/original_loss": 4.131555557250977, + "objective/train/theoretical_loss": 4.570068362778516, + "objective/train/tokens_used": 137625600, + "objective/train/unlikelihood_loss": 0.16605950891971588, + "theoretical_loss": 4.570068362778516, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839919759277834, + "loss": 1.9336, + "theoretical_loss": 4.570068362778516, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397191574724174, + "loss": 1.7372, + "theoretical_loss": 4.569492401814339, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839518555667001, + "loss": 1.7945, + "theoretical_loss": 4.568917141879149, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839317953861585, + "loss": 1.8875, + "theoretical_loss": 4.568342581454532, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839117352056169, + "loss": 1.5855, + "theoretical_loss": 4.567768719026797, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048389167502507524, + "loss": 1.796, + "theoretical_loss": 4.567195553086961, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838716148445336, + "loss": 1.8751, + "theoretical_loss": 4.566623082130729, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483851554663992, + "loss": 1.7671, + "theoretical_loss": 4.56605130465848, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838314944834504, + "loss": 1.7724, + "theoretical_loss": 4.565480219175237, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381143430290875, + "loss": 1.8898, + "theoretical_loss": 4.56490982419066, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837913741223671, + "loss": 1.766, + "theoretical_loss": 4.564340118219022, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377131394182547, + "loss": 1.6873, + "theoretical_loss": 4.563771099779187, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837512537612839, + "loss": 1.7903, + "theoretical_loss": 4.563202767394597, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4491080045700073, + "objective/train/avg_token_score": 0.0068944301456213, + "objective/train/docs_used": 90119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.943935751914978, + "objective/train/original_loss": 4.3753509521484375, + "objective/train/theoretical_loss": 4.562918858012843, + "objective/train/tokens_used": 139264000, + "objective/train/unlikelihood_loss": 0.1535024493932724, + "theoretical_loss": 4.562918858012843, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373119358074225, + "loss": 1.6177, + "theoretical_loss": 4.562635119593255, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837111334002006, + "loss": 1.5881, + "theoretical_loss": 4.5620681549076965, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483691073219659, + "loss": 1.8352, + "theoretical_loss": 4.561501871874984, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836710130391174, + "loss": 1.7988, + "theoretical_loss": 4.560936269036679, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048365095285857575, + "loss": 1.694, + "theoretical_loss": 4.560371344938831, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836308926780341, + "loss": 1.5882, + "theoretical_loss": 4.559807098131953, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836108324974925, + "loss": 1.6214, + "theoretical_loss": 4.559243527171011, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359077231695084, + "loss": 1.8377, + "theoretical_loss": 4.558680630615397, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357071213640926, + "loss": 1.8401, + "theoretical_loss": 4.558118407028921, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835506519558676, + "loss": 1.6105, + "theoretical_loss": 4.557556854979786, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483530591775326, + "loss": 1.7269, + "theoretical_loss": 4.556995973040574, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048351053159478434, + "loss": 1.6745, + "theoretical_loss": 4.556435759788229, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3440249264240265, + "objective/train/avg_token_score": 0.03104650229215622, + "objective/train/docs_used": 90792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5081690549850464, + "objective/train/original_loss": 4.372777462005615, + "objective/train/theoretical_loss": 4.555876213804037, + "objective/train/tokens_used": 140902400, + "objective/train/unlikelihood_loss": 0.17843031883239746, + "theoretical_loss": 4.555876213804037, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349047141424276, + "loss": 1.682, + "theoretical_loss": 4.555876213804037, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834704112337011, + "loss": 1.8494, + "theoretical_loss": 4.555317333673611, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834503510531595, + "loss": 1.6859, + "theoretical_loss": 4.5547591179868725, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343029087261785, + "loss": 1.7201, + "theoretical_loss": 4.554201565338033, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048341023069207626, + "loss": 1.6618, + "theoretical_loss": 4.553644674325584, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833901705115346, + "loss": 1.6429, + "theoretical_loss": 4.553088443552269, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483370110330993, + "loss": 1.6037, + "theoretical_loss": 4.552532871625077, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048335005015045135, + "loss": 1.564, + "theoretical_loss": 4.551977957155217, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833299899699097, + "loss": 1.7074, + "theoretical_loss": 4.551423698758111, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048330992978936813, + "loss": 1.6408, + "theoretical_loss": 4.550870095053366, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832898696088265, + "loss": 1.6711, + "theoretical_loss": 4.550317144664766, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048326980942828485, + "loss": 1.649, + "theoretical_loss": 4.5497648462202545, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832497492477432, + "loss": 1.714, + "theoretical_loss": 4.549213198351914, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.31085044145584106, + "objective/train/avg_token_score": 0.008633834309875965, + "objective/train/docs_used": 92175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2824527025222778, + "objective/train/original_loss": 4.095202922821045, + "objective/train/theoretical_loss": 4.548937617957463, + "objective/train/tokens_used": 142540800, + "objective/train/unlikelihood_loss": 0.16536656022071838, + "theoretical_loss": 4.548937617957463, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048322968906720163, + "loss": 1.6286, + "theoretical_loss": 4.548662199695954, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048320962888666, + "loss": 1.7102, + "theoretical_loss": 4.548111848892693, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048318956870611836, + "loss": 1.6976, + "theoretical_loss": 4.547562144586539, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831695085255767, + "loss": 1.8329, + "theoretical_loss": 4.5470130854259825, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831494483450351, + "loss": 1.8789, + "theoretical_loss": 4.546464670063569, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831293881644935, + "loss": 1.9697, + "theoretical_loss": 4.545916897155894, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048310932798395186, + "loss": 1.7363, + "theoretical_loss": 4.545369765363578, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830892678034102, + "loss": 1.889, + "theoretical_loss": 4.544823273351257, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830692076228686, + "loss": 1.6188, + "theoretical_loss": 4.544277419787566, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483049147442327, + "loss": 1.8209, + "theoretical_loss": 4.543732203345119, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048302908726178536, + "loss": 1.6722, + "theoretical_loss": 4.543187622700497, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830090270812437, + "loss": 1.6843, + "theoretical_loss": 4.542643676534234, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.35123410820961, + "objective/train/avg_token_score": 0.024587556719779968, + "objective/train/docs_used": 92798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.466112732887268, + "objective/train/original_loss": 4.362775802612305, + "objective/train/theoretical_loss": 4.542100363530799, + "objective/train/tokens_used": 144179200, + "objective/train/unlikelihood_loss": 0.18418121337890625, + "theoretical_loss": 4.542100363530799, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829889669007021, + "loss": 1.8888, + "theoretical_loss": 4.542100363530799, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048296890672016045, + "loss": 1.8105, + "theoretical_loss": 4.5415576823785795, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294884653961887, + "loss": 1.7692, + "theoretical_loss": 4.541015631769872, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292878635907723, + "loss": 1.7058, + "theoretical_loss": 4.540474210400859, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829087261785356, + "loss": 1.8536, + "theoretical_loss": 4.5399334169716, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048288866599799395, + "loss": 1.7873, + "theoretical_loss": 4.539393250186015, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048286860581745237, + "loss": 1.7762, + "theoretical_loss": 4.538853708751866, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048284854563691073, + "loss": 1.7058, + "theoretical_loss": 4.538314791380748, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828284854563691, + "loss": 1.9382, + "theoretical_loss": 4.537776496788071, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048280842527582746, + "loss": 1.8217, + "theoretical_loss": 4.537238823693045, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827883650952858, + "loss": 1.7635, + "theoretical_loss": 4.536701770818665, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048276830491474423, + "loss": 1.7094, + "theoretical_loss": 4.536165336891699, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827482447342026, + "loss": 1.8869, + "theoretical_loss": 4.53562952064267, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4389662742614746, + "objective/train/avg_token_score": 0.00564260920509696, + "objective/train/docs_used": 94036, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9034613370895386, + "objective/train/original_loss": 4.32117223739624, + "objective/train/theoretical_loss": 4.535361843751696, + "objective/train/tokens_used": 145817600, + "objective/train/unlikelihood_loss": 0.2049456685781479, + "theoretical_loss": 4.535361843751696, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482728184553661, + "loss": 1.8672, + "theoretical_loss": 4.535094320805847, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827081243731194, + "loss": 1.8281, + "theoretical_loss": 4.5345597361192205, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826880641925778, + "loss": 1.8617, + "theoretical_loss": 4.5340257653244995, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266800401203615, + "loss": 1.8899, + "theoretical_loss": 4.533492407167093, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826479438314945, + "loss": 1.6389, + "theoretical_loss": 4.5329596603960916, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826278836509529, + "loss": 1.831, + "theoretical_loss": 4.532427523764261, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048260782347041124, + "loss": 1.67, + "theoretical_loss": 4.5318959960280205, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258776328986966, + "loss": 1.4795, + "theoretical_loss": 4.531365075947434, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482567703109328, + "loss": 1.9673, + "theoretical_loss": 4.5308347622861955, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825476429287864, + "loss": 1.6775, + "theoretical_loss": 4.530305053811611, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252758274824474, + "loss": 1.8903, + "theoretical_loss": 4.529775949294593, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048250752256770316, + "loss": 1.5669, + "theoretical_loss": 4.529247447509637, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5044599175453186, + "objective/train/avg_token_score": 0.009645281359553337, + "objective/train/docs_used": 94726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1166269779205322, + "objective/train/original_loss": 4.383207321166992, + "objective/train/theoretical_loss": 4.528719547234816, + "objective/train/tokens_used": 147456000, + "objective/train/unlikelihood_loss": 0.12163429707288742, + "theoretical_loss": 4.528719547234816, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824874623871615, + "loss": 1.874, + "theoretical_loss": 4.528719547234816, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824674022066199, + "loss": 1.5731, + "theoretical_loss": 4.528192247251763, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244734202607825, + "loss": 1.7182, + "theoretical_loss": 4.527665546345656, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048242728184553666, + "loss": 1.6724, + "theoretical_loss": 4.527139443305209, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482407221664995, + "loss": 1.8344, + "theoretical_loss": 4.526613936922654, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823871614844534, + "loss": 1.729, + "theoretical_loss": 4.526089025993732, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048236710130391175, + "loss": 1.914, + "theoretical_loss": 4.525564709317678, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823470411233701, + "loss": 1.8067, + "theoretical_loss": 4.525040985697203, + "tokens_seen": 148373504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048232698094282853, + "loss": 1.7183, + "theoretical_loss": 4.524517853938489, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823069207622869, + "loss": 1.6597, + "theoretical_loss": 4.523995312851174, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048228686058174525, + "loss": 1.8709, + "theoretical_loss": 4.52347336124833, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822668004012036, + "loss": 1.8413, + "theoretical_loss": 4.522951997946466, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224674022066203, + "loss": 1.938, + "theoretical_loss": 4.522431221765498, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.5625610947608948, + "objective/train/avg_token_score": 0.0176337081938982, + "objective/train/docs_used": 95869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.122063159942627, + "objective/train/original_loss": 4.070887088775635, + "objective/train/theoretical_loss": 4.522171053477507, + "objective/train/tokens_used": 149094400, + "objective/train/unlikelihood_loss": 0.15213339030742645, + "theoretical_loss": 4.522171053477507, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822266800401204, + "loss": 1.9062, + "theoretical_loss": 4.5219110315287505, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048220661985957876, + "loss": 1.7105, + "theoretical_loss": 4.521391426062934, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821865596790371, + "loss": 1.8112, + "theoretical_loss": 4.520872404198139, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821664994984955, + "loss": 1.6997, + "theoretical_loss": 4.520353964767814, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821464393179539, + "loss": 1.6329, + "theoretical_loss": 4.519836106608768, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048212637913741226, + "loss": 1.7948, + "theoretical_loss": 4.519318828561142, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821063189568706, + "loss": 1.9672, + "theoretical_loss": 4.518802129468405, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482086258776329, + "loss": 1.7178, + "theoretical_loss": 4.518286008177341, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820661985957874, + "loss": 1.8528, + "theoretical_loss": 4.517770463538038, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204613841524576, + "loss": 1.7364, + "theoretical_loss": 4.517255494403868, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820260782347041, + "loss": 1.7236, + "theoretical_loss": 4.516741099631485, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820060180541625, + "loss": 1.6208, + "theoretical_loss": 4.5162272780808035, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4750122129917145, + "objective/train/avg_token_score": 0.008325014263391495, + "objective/train/docs_used": 96442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0012364387512207, + "objective/train/original_loss": 4.350079536437988, + "objective/train/theoretical_loss": 4.515714028614996, + "objective/train/tokens_used": 150732800, + "objective/train/unlikelihood_loss": 0.13368818163871765, + "theoretical_loss": 4.515714028614996, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048198595787362085, + "loss": 1.7642, + "theoretical_loss": 4.515714028614996, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048196589769307927, + "loss": 1.986, + "theoretical_loss": 4.515201350100471, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048194583751253763, + "loss": 1.6876, + "theoretical_loss": 4.5146892414068684, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481925777331996, + "loss": 1.3922, + "theoretical_loss": 4.514177701407042, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048190571715145435, + "loss": 1.7374, + "theoretical_loss": 4.513666728977054, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048188565697091277, + "loss": 1.7462, + "theoretical_loss": 4.513156322996155, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048186559679037113, + "loss": 1.5728, + "theoretical_loss": 4.5126464823467805, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818455366098295, + "loss": 1.7909, + "theoretical_loss": 4.5121372059145335, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048182547642928786, + "loss": 1.5978, + "theoretical_loss": 4.511628492588174, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818054162487462, + "loss": 1.6053, + "theoretical_loss": 4.511120341259608, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048178535606820463, + "loss": 1.8598, + "theoretical_loss": 4.510612750823878, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481765295887663, + "loss": 1.864, + "theoretical_loss": 4.510105720179144, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048174523570712136, + "loss": 1.7646, + "theoretical_loss": 4.509599248226683, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.5002443790435791, + "objective/train/avg_token_score": 0.033866036683321, + "objective/train/docs_used": 97778, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1443092823028564, + "objective/train/original_loss": 4.667900562286377, + "objective/train/theoretical_loss": 4.509346221417552, + "objective/train/tokens_used": 152371200, + "objective/train/unlikelihood_loss": 0.1134328842163086, + "theoretical_loss": 4.509346221417552, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817251755265797, + "loss": 1.7244, + "theoretical_loss": 4.509093333870869, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048170511534603814, + "loss": 1.5927, + "theoretical_loss": 4.508587976019164, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816850551654965, + "loss": 1.6639, + "theoretical_loss": 4.508083173582105, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048166499498495486, + "loss": 1.8064, + "theoretical_loss": 4.5075789254733, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816449348044132, + "loss": 1.6705, + "theoretical_loss": 4.507075230609407, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816248746238716, + "loss": 1.6887, + "theoretical_loss": 4.506572087910127, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048160481444333, + "loss": 1.7163, + "theoretical_loss": 4.506069496298198, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048158475426278837, + "loss": 1.5463, + "theoretical_loss": 4.505567454699373, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048156469408224673, + "loss": 1.7644, + "theoretical_loss": 4.505065962042418, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815446339017051, + "loss": 1.5152, + "theoretical_loss": 4.504565017259097, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815245737211635, + "loss": 1.6447, + "theoretical_loss": 4.504064619284163, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048150451354062187, + "loss": 1.4444, + "theoretical_loss": 4.5035647670553445, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.584799587726593, + "objective/train/avg_token_score": 0.008578718639910221, + "objective/train/docs_used": 98099, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.4139657020568848, + "objective/train/original_loss": 4.249912261962891, + "objective/train/theoretical_loss": 4.503065459513339, + "objective/train/tokens_used": 154009600, + "objective/train/unlikelihood_loss": 0.09866306930780411, + "theoretical_loss": 4.503065459513339, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048148445336008023, + "loss": 1.8128, + "theoretical_loss": 4.503065459513339, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814643931795386, + "loss": 1.8889, + "theoretical_loss": 4.502566695601795, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048144433299899696, + "loss": 1.8156, + "theoretical_loss": 4.502068474267309, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048142427281845537, + "loss": 1.8757, + "theoretical_loss": 4.501570794459411, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048140421263791373, + "loss": 1.8008, + "theoretical_loss": 4.501073655130554, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813841524573721, + "loss": 1.781, + "theoretical_loss": 4.500577055236104, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048136409227683046, + "loss": 1.7119, + "theoretical_loss": 4.500080993734329, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813440320962889, + "loss": 1.8122, + "theoretical_loss": 4.499585469586387, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048132397191574724, + "loss": 1.9573, + "theoretical_loss": 4.499090481756321, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813039117352056, + "loss": 1.6949, + "theoretical_loss": 4.498596029211041, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048128385155466396, + "loss": 1.8511, + "theoretical_loss": 4.4981021109203185, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812637913741223, + "loss": 1.716, + "theoretical_loss": 4.497608725856776, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048124373119358074, + "loss": 1.7856, + "theoretical_loss": 4.497115872995876, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4549120366573334, + "objective/train/avg_token_score": 0.0024577039293944836, + "objective/train/docs_used": 99496, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9255015850067139, + "objective/train/original_loss": 4.110994338989258, + "objective/train/theoretical_loss": 4.496869645822022, + "objective/train/tokens_used": 155648000, + "objective/train/unlikelihood_loss": 0.1798098087310791, + "theoretical_loss": 4.496869645822022, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048122367101303916, + "loss": 1.6885, + "theoretical_loss": 4.496623551315908, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812036108324975, + "loss": 1.7518, + "theoretical_loss": 4.496131759797984, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811835506519559, + "loss": 1.696, + "theoretical_loss": 4.495640497426023, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811634904714143, + "loss": 1.7442, + "theoretical_loss": 4.4951497631867445, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048114343029087266, + "loss": 1.6777, + "theoretical_loss": 4.4946595560696565, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481123370110331, + "loss": 1.6765, + "theoretical_loss": 4.494169875067046, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811033099297894, + "loss": 1.6892, + "theoretical_loss": 4.493680719173968, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810832497492478, + "loss": 1.5711, + "theoretical_loss": 4.493192087388239, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048106318956870616, + "loss": 1.6435, + "theoretical_loss": 4.4927039787104235, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810431293881645, + "loss": 1.4946, + "theoretical_loss": 4.492216392143826, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810230692076229, + "loss": 1.9506, + "theoretical_loss": 4.49172932669448, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048100300902708125, + "loss": 1.5955, + "theoretical_loss": 4.491242781371138, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3397482931613922, + "objective/train/avg_token_score": 0.02855628915131092, + "objective/train/docs_used": 100000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4549492597579956, + "objective/train/original_loss": 4.49015998840332, + "objective/train/theoretical_loss": 4.4907567551852665, + "objective/train/tokens_used": 157286400, + "objective/train/unlikelihood_loss": 0.11378736048936844, + "theoretical_loss": 4.4907567551852665, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048098294884653967, + "loss": 1.5109, + "theoretical_loss": 4.4907567551852665, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048096288866599803, + "loss": 1.7852, + "theoretical_loss": 4.490271247151027, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809428284854564, + "loss": 1.6, + "theoretical_loss": 4.489786256285276, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048092276830491475, + "loss": 1.8271, + "theoretical_loss": 4.489301781607551, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048090270812437317, + "loss": 1.8531, + "theoretical_loss": 4.48881782214006, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048088264794383153, + "loss": 1.7978, + "theoretical_loss": 4.488334376907673, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808625877632899, + "loss": 1.5617, + "theoretical_loss": 4.487851444937916, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084252758274826, + "loss": 1.7742, + "theoretical_loss": 4.487369025260954, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808224674022066, + "loss": 1.7228, + "theoretical_loss": 4.48688711690959, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048080240722166503, + "loss": 1.819, + "theoretical_loss": 4.48640571891925, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807823470411234, + "loss": 1.7198, + "theoretical_loss": 4.485924830327974, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048076228686058176, + "loss": 1.7289, + "theoretical_loss": 4.485444450176413, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807422266800401, + "loss": 1.694, + "theoretical_loss": 4.484964577507808, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.47953322529792786, + "objective/train/avg_token_score": 0.00853870902210474, + "objective/train/docs_used": 101476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9475176334381104, + "objective/train/original_loss": 4.236670970916748, + "objective/train/theoretical_loss": 4.484724831181337, + "objective/train/tokens_used": 158924800, + "objective/train/unlikelihood_loss": 0.1266087293624878, + "theoretical_loss": 4.484724831181337, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072216649949854, + "loss": 1.8989, + "theoretical_loss": 4.4844852113679945, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807021063189569, + "loss": 1.5987, + "theoretical_loss": 4.484006350805385, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068204613841526, + "loss": 1.7375, + "theoretical_loss": 4.483527994870958, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806619859578736, + "loss": 1.5585, + "theoretical_loss": 4.483050142618255, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480641925777332, + "loss": 1.7186, + "theoretical_loss": 4.482572793103373, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806218655967904, + "loss": 1.6185, + "theoretical_loss": 4.482095945384946, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048060180541624877, + "loss": 1.6661, + "theoretical_loss": 4.4816195985241425, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058174523570713, + "loss": 1.7461, + "theoretical_loss": 4.481143751584659, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805616850551655, + "loss": 1.8502, + "theoretical_loss": 4.480668403632706, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805416248746239, + "loss": 1.7936, + "theoretical_loss": 4.480193553736999, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052156469408227, + "loss": 1.9024, + "theoretical_loss": 4.479719200968757, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048050150451354063, + "loss": 1.7003, + "theoretical_loss": 4.479245344401685, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.53665691614151, + "objective/train/avg_token_score": 0.011694345623254776, + "objective/train/docs_used": 102054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.992788553237915, + "objective/train/original_loss": 3.9252665042877197, + "objective/train/theoretical_loss": 4.478771983111967, + "objective/train/tokens_used": 160563200, + "objective/train/unlikelihood_loss": 0.1462445855140686, + "theoretical_loss": 4.478771983111967, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480481444332999, + "loss": 1.6986, + "theoretical_loss": 4.478771983111967, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048046138415245736, + "loss": 1.4835, + "theoretical_loss": 4.478299116178265, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048044132397191577, + "loss": 1.5806, + "theoretical_loss": 4.4778267426817, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042126379137413, + "loss": 1.7454, + "theoretical_loss": 4.47735486170585, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804012036108325, + "loss": 1.6559, + "theoretical_loss": 4.4768834723367394, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038114343029086, + "loss": 1.7753, + "theoretical_loss": 4.476412573662829, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803610832497493, + "loss": 1.6249, + "theoretical_loss": 4.475942164775013, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034102306920764, + "loss": 1.6971, + "theoretical_loss": 4.475472244766601, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480320962888666, + "loss": 1.6015, + "theoretical_loss": 4.47500281273332, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048030090270812436, + "loss": 1.8606, + "theoretical_loss": 4.474533867773299, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802808425275827, + "loss": 1.615, + "theoretical_loss": 4.474065408987063, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048026078234704114, + "loss": 1.5313, + "theoretical_loss": 4.473597435477526, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802407221664995, + "loss": 1.8292, + "theoretical_loss": 4.473129946349982, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.436950147151947, + "objective/train/avg_token_score": 0.005409137345850468, + "objective/train/docs_used": 102718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9111195802688599, + "objective/train/original_loss": 4.506403923034668, + "objective/train/theoretical_loss": 4.472896383150508, + "objective/train/tokens_used": 162201600, + "objective/train/unlikelihood_loss": 0.16208896040916443, + "theoretical_loss": 4.472896383150508, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048022066198595786, + "loss": 1.6402, + "theoretical_loss": 4.472662940712091, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020060180541623, + "loss": 1.5722, + "theoretical_loss": 4.472196417673883, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048018054162487464, + "loss": 1.6823, + "theoretical_loss": 4.471730376347738, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480160481444333, + "loss": 1.6869, + "theoretical_loss": 4.471264815848384, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014042126379137, + "loss": 1.794, + "theoretical_loss": 4.470799735292889, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048012036108324973, + "loss": 1.7949, + "theoretical_loss": 4.470335133800649, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048010030090270815, + "loss": 1.6686, + "theoretical_loss": 4.469871010493383, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800802407221665, + "loss": 1.7866, + "theoretical_loss": 4.4694073644951295, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048006018054162487, + "loss": 1.6188, + "theoretical_loss": 4.468944194932225, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004012036108323, + "loss": 1.7558, + "theoretical_loss": 4.46848150093331, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800200601805416, + "loss": 1.9446, + "theoretical_loss": 4.468019281629316, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 1.9535, + "theoretical_loss": 4.467557536153457, + "tokens_seen": 163708928 + }, + { + "debugging/length": 491.6, + "debugging/num_segments": 5, + "epoch": 0.05, + "objective/train/aligned_token_count": 0.416177898645401, + "objective/train/avg_token_score": 0.005864487960934639, + "objective/train/docs_used": 104069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5953806638717651, + "objective/train/original_loss": 3.946671962738037, + "objective/train/theoretical_loss": 4.467096263641219, + "objective/train/tokens_used": 163840000, + "objective/train/unlikelihood_loss": 0.20240885019302368, + "theoretical_loss": 4.467096263641219, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799799398194584, + "loss": 1.8006, + "theoretical_loss": 4.467096263641219, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047995987963891674, + "loss": 1.7427, + "theoretical_loss": 4.466635463230359, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799398194583751, + "loss": 1.7973, + "theoretical_loss": 4.466175134060894, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799197592778335, + "loss": 1.7691, + "theoretical_loss": 4.4657152752750875, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798996990972919, + "loss": 1.8554, + "theoretical_loss": 4.465255886017452, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047987963891675024, + "loss": 1.9336, + "theoretical_loss": 4.464796965434738, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798595787362086, + "loss": 1.5818, + "theoretical_loss": 4.464338512675919, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983951855566696, + "loss": 1.8526, + "theoretical_loss": 4.4638805268921935, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798194583751254, + "loss": 1.8442, + "theoretical_loss": 4.463423007236974, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047979939819458374, + "loss": 1.418, + "theoretical_loss": 4.462965952865879, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797793380140421, + "loss": 1.7218, + "theoretical_loss": 4.462509362936723, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975927783350047, + "loss": 1.7421, + "theoretical_loss": 4.462053236609516, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797392176529589, + "loss": 1.7061, + "theoretical_loss": 4.461597573046449, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4883309006690979, + "objective/train/avg_token_score": 0.004766791593283415, + "objective/train/docs_used": 105300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9741791486740112, + "objective/train/original_loss": 3.7647905349731445, + "objective/train/theoretical_loss": 4.461369914540247, + "objective/train/tokens_used": 165478400, + "objective/train/unlikelihood_loss": 0.15376895666122437, + "theoretical_loss": 4.461369914540247, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797191574724173, + "loss": 1.7765, + "theoretical_loss": 4.4611423714118885, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047969909729187566, + "loss": 2.1214, + "theoretical_loss": 4.460687630872371, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479679037111334, + "loss": 1.8148, + "theoretical_loss": 4.460233350596599, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796589769307924, + "loss": 1.8356, + "theoretical_loss": 4.459779529755423, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796389167502508, + "loss": 1.847, + "theoretical_loss": 4.459326167521844, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047961885656970917, + "loss": 1.5851, + "theoretical_loss": 4.458873263071002, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047959879638916753, + "loss": 1.87, + "theoretical_loss": 4.458420815580169, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795787362086259, + "loss": 1.7718, + "theoretical_loss": 4.457968824228743, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795586760280843, + "loss": 1.9016, + "theoretical_loss": 4.4575172881982414, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953861584754267, + "loss": 1.5086, + "theoretical_loss": 4.457066206672291, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047951855566700103, + "loss": 1.8205, + "theoretical_loss": 4.456615578836625, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794984954864594, + "loss": 1.617, + "theoretical_loss": 4.45616540387907, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.5096529722213745, + "objective/train/avg_token_score": 0.009488563984632492, + "objective/train/docs_used": 106148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0118892192840576, + "objective/train/original_loss": 4.2763590812683105, + "objective/train/theoretical_loss": 4.455715680989545, + "objective/train/tokens_used": 167116800, + "objective/train/unlikelihood_loss": 0.12322863936424255, + "theoretical_loss": 4.455715680989545, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047947843530591776, + "loss": 1.6269, + "theoretical_loss": 4.455715680989545, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047945837512537617, + "loss": 1.809, + "theoretical_loss": 4.4552664093600525, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943831494483453, + "loss": 1.7982, + "theoretical_loss": 4.454817588184669, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794182547642929, + "loss": 1.6824, + "theoretical_loss": 4.454369216659542, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939819458375126, + "loss": 1.82, + "theoretical_loss": 4.453921293982877, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793781344032097, + "loss": 1.4183, + "theoretical_loss": 4.453473819354942, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047935807422266804, + "loss": 1.6876, + "theoretical_loss": 4.453026791978045, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793380140421264, + "loss": 1.4265, + "theoretical_loss": 4.452580211056542, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047931795386158476, + "loss": 1.7324, + "theoretical_loss": 4.45213407579682, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792978936810431, + "loss": 1.7566, + "theoretical_loss": 4.451688385407296, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047927783350050154, + "loss": 1.6371, + "theoretical_loss": 4.4512431390984055, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792577733199599, + "loss": 1.6714, + "theoretical_loss": 4.450798336082601, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923771313941827, + "loss": 1.6945, + "theoretical_loss": 4.450353975574341, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3742057681083679, + "objective/train/avg_token_score": 0.02583669312298298, + "objective/train/docs_used": 106755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.461486577987671, + "objective/train/original_loss": 4.418996334075928, + "objective/train/theoretical_loss": 4.450131961015606, + "objective/train/tokens_used": 168755200, + "objective/train/unlikelihood_loss": 0.12685106694698334, + "theoretical_loss": 4.450131961015606, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047921765295887663, + "loss": 1.7889, + "theoretical_loss": 4.449910056790086, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919759277833504, + "loss": 1.6048, + "theoretical_loss": 4.4494665789482895, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791775325977934, + "loss": 1.5793, + "theoretical_loss": 4.449023541269395, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047915747241725177, + "loss": 1.6751, + "theoretical_loss": 4.448580942975825, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913741223671013, + "loss": 1.7455, + "theoretical_loss": 4.448138783291979, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047911735205616855, + "loss": 1.7699, + "theoretical_loss": 4.44769706144422, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790972918756269, + "loss": 1.6299, + "theoretical_loss": 4.447255776660878, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047907723169508527, + "loss": 1.5542, + "theoretical_loss": 4.446814928172234, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905717151454363, + "loss": 1.7944, + "theoretical_loss": 4.446374515210521, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479037111334002, + "loss": 1.6775, + "theoretical_loss": 4.445934537009911, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790170511534604, + "loss": 1.8059, + "theoretical_loss": 4.445494992806513, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789969909729188, + "loss": 1.8668, + "theoretical_loss": 4.445055881838365, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.41752198338508606, + "objective/train/avg_token_score": 0.02483731135725975, + "objective/train/docs_used": 107830, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6519962549209595, + "objective/train/original_loss": 4.723080635070801, + "objective/train/theoretical_loss": 4.44461720334543, + "objective/train/tokens_used": 170393600, + "objective/train/unlikelihood_loss": 0.11609357595443726, + "theoretical_loss": 4.44461720334543, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047897693079237714, + "loss": 1.7401, + "theoretical_loss": 4.44461720334543, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789568706118355, + "loss": 1.6675, + "theoretical_loss": 4.444178956569585, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789368104312939, + "loss": 1.7224, + "theoretical_loss": 4.44374114075462, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789167502507523, + "loss": 1.6234, + "theoretical_loss": 4.443303755146225, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047889669007021064, + "loss": 1.6098, + "theoretical_loss": 4.442866798991993, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478876629889669, + "loss": 1.749, + "theoretical_loss": 4.442430271541404, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047885656970912736, + "loss": 1.6635, + "theoretical_loss": 4.4419941720458285, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788365095285858, + "loss": 1.8943, + "theoretical_loss": 4.441558499758511, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047881644934804414, + "loss": 1.6301, + "theoretical_loss": 4.441123253934572, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787963891675025, + "loss": 1.5459, + "theoretical_loss": 4.440688433830999, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047877632898696087, + "loss": 1.7699, + "theoretical_loss": 4.440254038706639, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787562688064193, + "loss": 1.6063, + "theoretical_loss": 4.439820067822195, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047873620862587765, + "loss": 1.7779, + "theoretical_loss": 4.439386520440218, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3144550323486328, + "objective/train/avg_token_score": 0.00989236868917942, + "objective/train/docs_used": 108471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1930038928985596, + "objective/train/original_loss": 4.118610382080078, + "objective/train/theoretical_loss": 4.439169905332706, + "objective/train/tokens_used": 172032000, + "objective/train/unlikelihood_loss": 0.1678958386182785, + "theoretical_loss": 4.439169905332706, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478716148445336, + "loss": 1.4924, + "theoretical_loss": 4.438953395825102, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047869608826479437, + "loss": 1.6309, + "theoretical_loss": 4.438520693243079, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047867602808425273, + "loss": 1.5903, + "theoretical_loss": 4.438088411962211, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047865596790371115, + "loss": 1.5514, + "theoretical_loss": 4.437656551252381, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786359077231695, + "loss": 1.8921, + "theoretical_loss": 4.437225110385297, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786158475426279, + "loss": 1.6624, + "theoretical_loss": 4.436794088634477, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047859578736208624, + "loss": 1.653, + "theoretical_loss": 4.436363485275246, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047857572718154465, + "loss": 1.6515, + "theoretical_loss": 4.435933299584729, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478555667001003, + "loss": 1.6443, + "theoretical_loss": 4.435503530841849, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785356068204614, + "loss": 1.6424, + "theoretical_loss": 4.4350741783273175, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047851554663991974, + "loss": 1.7407, + "theoretical_loss": 4.434645241323629, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784954864593781, + "loss": 1.6988, + "theoretical_loss": 4.434216719115057, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.38972383737564087, + "objective/train/avg_token_score": 0.009662693366408348, + "objective/train/docs_used": 109645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5914331674575806, + "objective/train/original_loss": 4.367075443267822, + "objective/train/theoretical_loss": 4.433788610987646, + "objective/train/tokens_used": 173670400, + "objective/train/unlikelihood_loss": 0.1352538764476776, + "theoretical_loss": 4.433788610987646, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784754262788365, + "loss": 1.7224, + "theoretical_loss": 4.433788610987646, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784553660982949, + "loss": 1.7154, + "theoretical_loss": 4.433360916229209, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047843530591775324, + "loss": 1.6758, + "theoretical_loss": 4.432933634129318, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784152457372116, + "loss": 1.6071, + "theoretical_loss": 4.432506763979299, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047839518555667, + "loss": 1.6384, + "theoretical_loss": 4.432080305072233, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783751253761284, + "loss": 1.5316, + "theoretical_loss": 4.431654256702938, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047835506519558675, + "loss": 1.614, + "theoretical_loss": 4.4312286181679745, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783350050150451, + "loss": 1.9548, + "theoretical_loss": 4.430803388765636, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047831494483450347, + "loss": 1.5284, + "theoretical_loss": 4.430378567795938, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782948846539619, + "loss": 1.6997, + "theoretical_loss": 4.429954154560624, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047827482447342025, + "loss": 1.6483, + "theoretical_loss": 4.429530148363151, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782547642928786, + "loss": 1.5161, + "theoretical_loss": 4.429106548508685, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478234704112337, + "loss": 1.7931, + "theoretical_loss": 4.428683354304098, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3965053856372833, + "objective/train/avg_token_score": 0.01913469098508358, + "objective/train/docs_used": 110166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7019623517990112, + "objective/train/original_loss": 4.4854278564453125, + "objective/train/theoretical_loss": 4.428471909104372, + "objective/train/tokens_used": 175308800, + "objective/train/unlikelihood_loss": 0.11818500608205795, + "theoretical_loss": 4.428471909104372, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047821464393179544, + "loss": 1.5788, + "theoretical_loss": 4.428260565057964, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781945837512538, + "loss": 1.786, + "theoretical_loss": 4.427838180080547, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047817452357071217, + "loss": 1.7903, + "theoretical_loss": 4.427416198683803, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047815446339017053, + "loss": 1.7291, + "theoretical_loss": 4.42699462018137, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047813440320962895, + "loss": 1.8156, + "theoretical_loss": 4.426573443888563, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781143430290873, + "loss": 1.6172, + "theoretical_loss": 4.426152669122374, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047809428284854567, + "loss": 1.5866, + "theoretical_loss": 4.425732295201455, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047807422266800403, + "loss": 1.6032, + "theoretical_loss": 4.425312321446127, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780541624874624, + "loss": 1.7259, + "theoretical_loss": 4.424892747178365, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780341023069208, + "loss": 1.5495, + "theoretical_loss": 4.424473571721794, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780140421263792, + "loss": 1.5823, + "theoretical_loss": 4.424054794401689, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799398194583754, + "loss": 1.5334, + "theoretical_loss": 4.42363641454496, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.27394917607307434, + "objective/train/avg_token_score": 0.024683641269803047, + "objective/train/docs_used": 110915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1538923978805542, + "objective/train/original_loss": 4.269205570220947, + "objective/train/theoretical_loss": 4.42321843148016, + "objective/train/tokens_used": 176947200, + "objective/train/unlikelihood_loss": 0.17650169134140015, + "theoretical_loss": 4.42321843148016, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779739217652959, + "loss": 1.5801, + "theoretical_loss": 4.42321843148016, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779538615847543, + "loss": 1.743, + "theoretical_loss": 4.422800844537466, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779338014042127, + "loss": 1.7594, + "theoretical_loss": 4.422383653048685, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047791374122367104, + "loss": 1.7924, + "theoretical_loss": 4.421966856347243, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778936810431294, + "loss": 1.9118, + "theoretical_loss": 4.421550453768181, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047787362086258776, + "loss": 1.6296, + "theoretical_loss": 4.42113444464815, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778535606820462, + "loss": 1.512, + "theoretical_loss": 4.420718828325403, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783350050150454, + "loss": 1.7201, + "theoretical_loss": 4.4203036041398, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778134403209629, + "loss": 1.6327, + "theoretical_loss": 4.419888771432789, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047779338014042127, + "loss": 1.5254, + "theoretical_loss": 4.419474329547413, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777733199598797, + "loss": 1.8315, + "theoretical_loss": 4.419060277828295, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775325977933805, + "loss": 1.6243, + "theoretical_loss": 4.4186466156216415, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777331995987964, + "loss": 1.6067, + "theoretical_loss": 4.418233342275233, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4303519129753113, + "objective/train/avg_token_score": 0.005776857491582632, + "objective/train/docs_used": 112413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6707626581192017, + "objective/train/original_loss": 3.843381643295288, + "objective/train/theoretical_loss": 4.418026851221231, + "objective/train/tokens_used": 178585600, + "objective/train/unlikelihood_loss": 0.19406434893608093, + "theoretical_loss": 4.418026851221231, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771313941825477, + "loss": 1.5482, + "theoretical_loss": 4.4178204571384185, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047769307923771313, + "loss": 1.5164, + "theoretical_loss": 4.417407959562116, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047767301905717155, + "loss": 1.6927, + "theoretical_loss": 4.416995848898797, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776529588766299, + "loss": 1.496, + "theoretical_loss": 4.416584124502495, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776328986960883, + "loss": 1.5685, + "theoretical_loss": 4.41617278572879, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761283851554664, + "loss": 1.4586, + "theoretical_loss": 4.415761831934808, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047759277833500505, + "loss": 1.4191, + "theoretical_loss": 4.415351262479216, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775727181544634, + "loss": 1.5665, + "theoretical_loss": 4.414941076722219, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775526579739218, + "loss": 1.545, + "theoretical_loss": 4.414531274025548, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753259779338014, + "loss": 1.5296, + "theoretical_loss": 4.414121853752466, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775125376128385, + "loss": 1.5963, + "theoretical_loss": 4.413712815267752, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774924774322969, + "loss": 1.8008, + "theoretical_loss": 4.4133041579377075, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.39986559748649597, + "objective/train/avg_token_score": 0.06428433954715729, + "objective/train/docs_used": 112982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4951305389404297, + "objective/train/original_loss": 4.130880355834961, + "objective/train/theoretical_loss": 4.412895881130142, + "objective/train/tokens_used": 180224000, + "objective/train/unlikelihood_loss": 0.13160917162895203, + "theoretical_loss": 4.412895881130142, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774724172517553, + "loss": 1.6337, + "theoretical_loss": 4.412895881130142, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745235707121364, + "loss": 1.658, + "theoretical_loss": 4.412487984214373, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477432296890672, + "loss": 1.5547, + "theoretical_loss": 4.412080466561221, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774122367101304, + "loss": 1.7956, + "theoretical_loss": 4.411673327543005, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773921765295888, + "loss": 1.7189, + "theoretical_loss": 4.411266566533539, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047737211634904715, + "loss": 1.6131, + "theoretical_loss": 4.41086018290812, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773520561685055, + "loss": 1.7809, + "theoretical_loss": 4.410454176043537, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733199598796387, + "loss": 1.5577, + "theoretical_loss": 4.410048545318052, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773119358074223, + "loss": 1.4906, + "theoretical_loss": 4.409643290111404, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729187562688065, + "loss": 1.5827, + "theoretical_loss": 4.409238409804804, + "tokens_seen": 181403648 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477271815446339, + "loss": 1.6267, + "theoretical_loss": 4.408833903780926, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772517552657974, + "loss": 1.6512, + "theoretical_loss": 4.408429771423909, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772316950852558, + "loss": 1.7169, + "theoretical_loss": 4.408026012119344, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3759775161743164, + "objective/train/avg_token_score": 0.011839372105896473, + "objective/train/docs_used": 114216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4656167030334473, + "objective/train/original_loss": 4.162503719329834, + "objective/train/theoretical_loss": 4.407824272170128, + "objective/train/tokens_used": 181862400, + "objective/train/unlikelihood_loss": 0.16261805593967438, + "theoretical_loss": 4.407824272170128, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047721163490471415, + "loss": 1.7268, + "theoretical_loss": 4.407622625254279, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771915747241725, + "loss": 1.6943, + "theoretical_loss": 4.407219610217206, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771715145436309, + "loss": 1.7274, + "theoretical_loss": 4.406816966398064, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047715145436308924, + "loss": 1.539, + "theoretical_loss": 4.4064146931882275, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713139418254766, + "loss": 1.6115, + "theoretical_loss": 4.406012789980506, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477111334002006, + "loss": 1.6946, + "theoretical_loss": 4.405611256169143, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770912738214644, + "loss": 1.69, + "theoretical_loss": 4.405210091149802, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047707121364092274, + "loss": 1.7164, + "theoretical_loss": 4.404809294319572, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047705115346038116, + "loss": 1.7673, + "theoretical_loss": 4.404408865076955, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770310932798395, + "loss": 1.7797, + "theoretical_loss": 4.404008802821871, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770110330992979, + "loss": 1.7046, + "theoretical_loss": 4.403609106955645, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047699097291875625, + "loss": 1.5983, + "theoretical_loss": 4.403209776881004, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4590664803981781, + "objective/train/avg_token_score": 0.02654801867902279, + "objective/train/docs_used": 114775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.934823751449585, + "objective/train/original_loss": 4.4886064529418945, + "objective/train/theoretical_loss": 4.4028108120020795, + "objective/train/tokens_used": 183500800, + "objective/train/unlikelihood_loss": 0.1138959601521492, + "theoretical_loss": 4.4028108120020795, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047697091273821466, + "loss": 1.7135, + "theoretical_loss": 4.4028108120020795, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476950852557673, + "loss": 1.6734, + "theoretical_loss": 4.4024122117243945, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769307923771314, + "loss": 1.8335, + "theoretical_loss": 4.4020139754548655, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047691073219658975, + "loss": 1.6858, + "theoretical_loss": 4.4016161026017935, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768906720160481, + "loss": 1.7323, + "theoretical_loss": 4.401218592574865, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047687061183550653, + "loss": 1.8249, + "theoretical_loss": 4.400821444785143, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768505516549649, + "loss": 1.588, + "theoretical_loss": 4.400424658645065, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047683049147442325, + "loss": 1.6622, + "theoretical_loss": 4.40002823356844, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768104312938816, + "loss": 1.6254, + "theoretical_loss": 4.39963216897044, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047679037111334003, + "loss": 1.7437, + "theoretical_loss": 4.399236464267602, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767703109327984, + "loss": 1.6554, + "theoretical_loss": 4.398841118877819, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047675025075225676, + "loss": 1.6995, + "theoretical_loss": 4.398446132220338, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767301905717151, + "loss": 1.9342, + "theoretical_loss": 4.398051503715753, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.5482649207115173, + "objective/train/avg_token_score": 0.00839152093976736, + "objective/train/docs_used": 115939, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.310957908630371, + "objective/train/original_loss": 4.36484956741333, + "objective/train/theoretical_loss": 4.397854323590102, + "objective/train/tokens_used": 185139200, + "objective/train/unlikelihood_loss": 0.10018301010131836, + "theoretical_loss": 4.397854323590102, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047671013039117353, + "loss": 1.7113, + "theoretical_loss": 4.397657232786008, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047669007021063195, + "loss": 1.6987, + "theoretical_loss": 4.397263318854384, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766700100300903, + "loss": 1.5346, + "theoretical_loss": 4.396869761345503, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766499498495487, + "loss": 1.7575, + "theoretical_loss": 4.396476559685315, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662988966900704, + "loss": 1.6408, + "theoretical_loss": 4.3960837133011035, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047660982948846545, + "loss": 1.8307, + "theoretical_loss": 4.395691221621476, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765897693079238, + "loss": 1.8426, + "theoretical_loss": 4.39529908407636, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765697091273822, + "loss": 1.6831, + "theoretical_loss": 4.394907300097002, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654964894684054, + "loss": 1.6989, + "theoretical_loss": 4.39451586911596, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765295887662989, + "loss": 1.7103, + "theoretical_loss": 4.394124790567101, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765095285857573, + "loss": 1.7273, + "theoretical_loss": 4.393734063885599, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764894684052157, + "loss": 1.6129, + "theoretical_loss": 4.39334368850793, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.47330155968666077, + "objective/train/avg_token_score": 0.007206100504845381, + "objective/train/docs_used": 116562, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.913322925567627, + "objective/train/original_loss": 4.169583797454834, + "objective/train/theoretical_loss": 4.392953663871862, + "objective/train/tokens_used": 186777600, + "objective/train/unlikelihood_loss": 0.11674590408802032, + "theoretical_loss": 4.392953663871862, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047646940822467404, + "loss": 1.7131, + "theoretical_loss": 4.392953663871862, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764493480441324, + "loss": 1.7657, + "theoretical_loss": 4.392563989416462, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764292878635908, + "loss": 1.7479, + "theoretical_loss": 4.392174664582085, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764092276830492, + "loss": 1.7493, + "theoretical_loss": 4.391785688810373, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047638916750250755, + "loss": 1.7387, + "theoretical_loss": 4.391397061544247, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763691073219659, + "loss": 1.6152, + "theoretical_loss": 4.3910087822279085, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047634904714142427, + "loss": 1.5353, + "theoretical_loss": 4.390620850306832, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763289869608827, + "loss": 1.5725, + "theoretical_loss": 4.390233265227764, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047630892678034105, + "loss": 1.631, + "theoretical_loss": 4.389846026438715, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762888665997994, + "loss": 1.7898, + "theoretical_loss": 4.389459133388962, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762688064192578, + "loss": 1.5287, + "theoretical_loss": 4.389072585529037, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762487462387162, + "loss": 1.4302, + "theoretical_loss": 4.388686382310732, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047622868605817455, + "loss": 1.8187, + "theoretical_loss": 4.388300523187087, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.37365591526031494, + "objective/train/avg_token_score": 0.0153102558106184, + "objective/train/docs_used": 117858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5269169807434082, + "objective/train/original_loss": 4.470926284790039, + "objective/train/theoretical_loss": 4.3881077224901786, + "objective/train/tokens_used": 188416000, + "objective/train/unlikelihood_loss": 0.11220584064722061, + "theoretical_loss": 4.3881077224901786, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762086258776329, + "loss": 1.5687, + "theoretical_loss": 4.38791500761239, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761885656970913, + "loss": 1.5935, + "theoretical_loss": 4.3875298350421765, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047616850551654964, + "loss": 1.7741, + "theoretical_loss": 4.387145004933218, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614844533600806, + "loss": 1.4192, + "theoretical_loss": 4.386760516743526, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761283851554664, + "loss": 1.8067, + "theoretical_loss": 4.386376369932344, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761083249749248, + "loss": 1.6904, + "theoretical_loss": 4.385992563960145, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047608826479438314, + "loss": 1.7294, + "theoretical_loss": 4.385609098288628, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047606820461384156, + "loss": 1.5139, + "theoretical_loss": 4.385225972380715, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760481444332999, + "loss": 1.5383, + "theoretical_loss": 4.384843185700544, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760280842527583, + "loss": 1.745, + "theoretical_loss": 4.384460737713471, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047600802407221665, + "loss": 1.6476, + "theoretical_loss": 4.384078627886062, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047598796389167506, + "loss": 1.7629, + "theoretical_loss": 4.38369685568609, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3760386109352112, + "objective/train/avg_token_score": 0.010193110443651676, + "objective/train/docs_used": 118609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4706860780715942, + "objective/train/original_loss": 3.930041790008545, + "objective/train/theoretical_loss": 4.383315420582533, + "objective/train/tokens_used": 190054400, + "objective/train/unlikelihood_loss": 0.24754564464092255, + "theoretical_loss": 4.383315420582533, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759679037111334, + "loss": 1.862, + "theoretical_loss": 4.383315420582533, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759478435305918, + "loss": 1.5947, + "theoretical_loss": 4.38293432204557, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047592778335005015, + "loss": 1.7051, + "theoretical_loss": 4.3825535595465785, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759077231695085, + "loss": 1.6088, + "theoretical_loss": 4.382173132558126, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047588766298896693, + "loss": 1.6397, + "theoretical_loss": 4.381793040553973, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758676028084253, + "loss": 1.662, + "theoretical_loss": 4.381413283009065, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047584754262788365, + "loss": 1.5988, + "theoretical_loss": 4.381033859399532, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475827482447342, + "loss": 1.7682, + "theoretical_loss": 4.380654769202683, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047580742226680043, + "loss": 1.6793, + "theoretical_loss": 4.380276011897003, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757873620862588, + "loss": 1.5176, + "theoretical_loss": 4.379897586962148, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047576730190571716, + "loss": 1.5289, + "theoretical_loss": 4.379519493878948, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757472417251755, + "loss": 1.646, + "theoretical_loss": 4.379141732129394, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757271815446339, + "loss": 1.6823, + "theoretical_loss": 4.378764301196642, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.46615347266197205, + "objective/train/avg_token_score": 0.02856089174747467, + "objective/train/docs_used": 119339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6883468627929688, + "objective/train/original_loss": 4.204484462738037, + "objective/train/theoretical_loss": 4.37857570962537, + "objective/train/tokens_used": 191692800, + "objective/train/unlikelihood_loss": 0.11059554666280746, + "theoretical_loss": 4.37857570962537, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757071213640923, + "loss": 1.699, + "theoretical_loss": 4.378387200565006, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047568706118355066, + "loss": 1.8261, + "theoretical_loss": 4.378010429719957, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475667001003009, + "loss": 1.7145, + "theoretical_loss": 4.377633988148117, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756469408224674, + "loss": 1.5378, + "theoretical_loss": 4.377257875337257, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756268806419258, + "loss": 1.8218, + "theoretical_loss": 4.376882090776293, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047560682046138416, + "loss": 1.7853, + "theoretical_loss": 4.376506633955286, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755867602808425, + "loss": 1.4124, + "theoretical_loss": 4.37613150436543, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755667001003009, + "loss": 1.6803, + "theoretical_loss": 4.375756701499063, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047554663991975925, + "loss": 1.5873, + "theoretical_loss": 4.375382224849648, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047552657973921766, + "loss": 1.5583, + "theoretical_loss": 4.375008073911781, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047550651955867603, + "loss": 1.6886, + "theoretical_loss": 4.374634248181182, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754864593781344, + "loss": 1.6532, + "theoretical_loss": 4.374260747154692, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4188660681247711, + "objective/train/avg_token_score": 0.00520719401538372, + "objective/train/docs_used": 120055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6985862255096436, + "objective/train/original_loss": 4.125784873962402, + "objective/train/theoretical_loss": 4.373887570330275, + "objective/train/tokens_used": 193331200, + "objective/train/unlikelihood_loss": 0.16551171243190765, + "theoretical_loss": 4.373887570330275, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047546639919759275, + "loss": 1.6246, + "theoretical_loss": 4.373887570330275, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047544633901705117, + "loss": 1.4885, + "theoretical_loss": 4.373514717207009, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047542627883650953, + "loss": 1.4034, + "theoretical_loss": 4.373142187285083, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754062186559679, + "loss": 1.4975, + "theoretical_loss": 4.372769980065797, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047538615847542625, + "loss": 1.6481, + "theoretical_loss": 4.372398095051559, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753660982948846, + "loss": 1.4715, + "theoretical_loss": 4.372026531745877, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047534603811434303, + "loss": 1.4546, + "theoretical_loss": 4.37165528965336, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753259779338014, + "loss": 1.4513, + "theoretical_loss": 4.371284368279714, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047530591775325976, + "loss": 1.7305, + "theoretical_loss": 4.3709137671317375, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752858575727181, + "loss": 1.4355, + "theoretical_loss": 4.370543485717322, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047526579739217654, + "loss": 1.5039, + "theoretical_loss": 4.370173523545443, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752457372116349, + "loss": 1.6506, + "theoretical_loss": 4.369803880126162, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047522567703109326, + "loss": 1.4202, + "theoretical_loss": 4.369434554970621, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.42467010021209717, + "objective/train/avg_token_score": 0.023077575489878654, + "objective/train/docs_used": 121265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6124404668807983, + "objective/train/original_loss": 3.863985300064087, + "objective/train/theoretical_loss": 4.369250011589279, + "objective/train/tokens_used": 194969600, + "objective/train/unlikelihood_loss": 0.3769527077674866, + "theoretical_loss": 4.369250011589279, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752056168505517, + "loss": 1.6853, + "theoretical_loss": 4.369065547591038, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047518555667001004, + "loss": 1.7059, + "theoretical_loss": 4.36869685750071, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047516549648946846, + "loss": 1.6439, + "theoretical_loss": 4.368328484214002, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751454363089268, + "loss": 1.5728, + "theoretical_loss": 4.3679604272463495, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751253761283852, + "loss": 1.6146, + "theoretical_loss": 4.367592686114252, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047510531594784354, + "loss": 1.563, + "theoretical_loss": 4.367225260335272, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047508525576730196, + "loss": 1.6781, + "theoretical_loss": 4.366858149428032, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750651955867603, + "loss": 1.5683, + "theoretical_loss": 4.366491352912211, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750451354062187, + "loss": 1.7887, + "theoretical_loss": 4.366124870308541, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047502507522567705, + "loss": 1.5592, + "theoretical_loss": 4.3657587011388035, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047500501504513546, + "loss": 1.7372, + "theoretical_loss": 4.365392844925829, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749849548645938, + "loss": 1.6439, + "theoretical_loss": 4.365027301193491, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4978005886077881, + "objective/train/avg_token_score": 0.0068794176913797855, + "objective/train/docs_used": 121711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8076094388961792, + "objective/train/original_loss": 4.219160556793213, + "objective/train/theoretical_loss": 4.364662069466704, + "objective/train/tokens_used": 196608000, + "objective/train/unlikelihood_loss": 0.12803588807582855, + "theoretical_loss": 4.364662069466704, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749648946840522, + "loss": 1.6509, + "theoretical_loss": 4.364662069466704, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494483450351055, + "loss": 1.4878, + "theoretical_loss": 4.364297149271423, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749247743229689, + "loss": 1.5957, + "theoretical_loss": 4.363932540134638, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047490471414242733, + "loss": 1.705, + "theoretical_loss": 4.363568241584368, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748846539618857, + "loss": 1.5305, + "theoretical_loss": 4.363204253149667, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047486459378134405, + "loss": 1.7872, + "theoretical_loss": 4.362840574360612, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748445336008024, + "loss": 1.7201, + "theoretical_loss": 4.362477204748305, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047482447342026083, + "loss": 1.7059, + "theoretical_loss": 4.362114143844867, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748044132397192, + "loss": 1.557, + "theoretical_loss": 4.3617513911834385, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478435305917756, + "loss": 1.6276, + "theoretical_loss": 4.3613889462981765, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747642928786359, + "loss": 1.5583, + "theoretical_loss": 4.361026808724247, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747442326980943, + "loss": 1.7629, + "theoretical_loss": 4.360664977997828, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747241725175527, + "loss": 1.4869, + "theoretical_loss": 4.360303453656103, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3821480870246887, + "objective/train/avg_token_score": 0.010640091262757778, + "objective/train/docs_used": 122248, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6288254261016846, + "objective/train/original_loss": 4.3907952308654785, + "objective/train/theoretical_loss": 4.360122806235145, + "objective/train/tokens_used": 198246400, + "objective/train/unlikelihood_loss": 0.1243598461151123, + "theoretical_loss": 4.360122806235145, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470411233701106, + "loss": 1.6806, + "theoretical_loss": 4.359942235237257, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746840521564694, + "loss": 1.6821, + "theoretical_loss": 4.359581322280479, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746639919759278, + "loss": 1.7165, + "theoretical_loss": 4.359220714325954, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746439317953862, + "loss": 1.6884, + "theoretical_loss": 4.358860410914861, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047462387161484456, + "loss": 1.6764, + "theoretical_loss": 4.358500411589375, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746038114343029, + "loss": 1.874, + "theoretical_loss": 4.358140715892658, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745837512537613, + "loss": 1.5337, + "theoretical_loss": 4.357781323368857, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047456369107321965, + "loss": 1.916, + "theoretical_loss": 4.357422233563106, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454363089267807, + "loss": 1.5149, + "theoretical_loss": 4.35706344602152, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047452357071213643, + "loss": 1.6638, + "theoretical_loss": 4.356704960291191, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745035105315948, + "loss": 1.7224, + "theoretical_loss": 4.356346775920185, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047448345035105315, + "loss": 1.6415, + "theoretical_loss": 4.355988892457546, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.5615224838256836, + "objective/train/avg_token_score": 0.008392857387661934, + "objective/train/docs_used": 123553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.331106185913086, + "objective/train/original_loss": 4.441834449768066, + "objective/train/theoretical_loss": 4.355631309453283, + "objective/train/tokens_used": 199884800, + "objective/train/unlikelihood_loss": 0.08705087006092072, + "theoretical_loss": 4.355631309453283, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047446339017051157, + "loss": 1.9122, + "theoretical_loss": 4.355631309453283, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444332998996993, + "loss": 1.8121, + "theoretical_loss": 4.355274026458375, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744232698094283, + "loss": 1.7729, + "theoretical_loss": 4.354917043024765, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047440320962888666, + "loss": 1.6087, + "theoretical_loss": 4.354560358705358, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474383149448345, + "loss": 1.4376, + "theoretical_loss": 4.35420397305402, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436308926780343, + "loss": 1.8095, + "theoretical_loss": 4.353847885625571, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743430290872618, + "loss": 1.672, + "theoretical_loss": 4.353492095975787, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432296890672016, + "loss": 1.7417, + "theoretical_loss": 4.353136603661392, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743029087261785, + "loss": 1.6302, + "theoretical_loss": 4.352781408240065, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047428284854563694, + "loss": 1.7316, + "theoretical_loss": 4.352426509270425, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742627883650953, + "loss": 1.8957, + "theoretical_loss": 4.352071906312037, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047424272818455366, + "loss": 1.7722, + "theoretical_loss": 4.351717598925406, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474222668004012, + "loss": 1.6547, + "theoretical_loss": 4.351363586671976, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4540567100048065, + "objective/train/avg_token_score": 0.013070252723991871, + "objective/train/docs_used": 124253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9555007219314575, + "objective/train/original_loss": 4.570303916931152, + "objective/train/theoretical_loss": 4.351186691083417, + "objective/train/tokens_used": 201523200, + "objective/train/unlikelihood_loss": 0.11967580020427704, + "theoretical_loss": 4.351186691083417, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742026078234704, + "loss": 1.6592, + "theoretical_loss": 4.351009869114124, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741825476429288, + "loss": 1.652, + "theoretical_loss": 4.350656445815164, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416248746238716, + "loss": 1.6998, + "theoretical_loss": 4.350303316339337, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741424272818455, + "loss": 1.637, + "theoretical_loss": 4.349950480251813, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741223671013039, + "loss": 1.443, + "theoretical_loss": 4.349597937118687, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741023069207623, + "loss": 1.719, + "theoretical_loss": 4.349245686506976, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047408224674022067, + "loss": 1.6887, + "theoretical_loss": 4.348893727984619, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047406218655967903, + "loss": 1.7137, + "theoretical_loss": 4.348542061120469, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740421263791374, + "loss": 1.5756, + "theoretical_loss": 4.3481906854842975, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047402206619859575, + "loss": 1.8621, + "theoretical_loss": 4.347839600646786, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047400200601805417, + "loss": 1.6352, + "theoretical_loss": 4.347488806179528, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047398194583751253, + "loss": 1.7664, + "theoretical_loss": 4.347138301655021, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4085410535335541, + "objective/train/avg_token_score": 0.01795566827058792, + "objective/train/docs_used": 125838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5744194984436035, + "objective/train/original_loss": 3.834841251373291, + "objective/train/theoretical_loss": 4.346788086646671, + "objective/train/tokens_used": 203161600, + "objective/train/unlikelihood_loss": 0.23488780856132507, + "theoretical_loss": 4.346788086646671, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739618856569709, + "loss": 1.4374, + "theoretical_loss": 4.346788086646671, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047394182547642926, + "loss": 1.6623, + "theoretical_loss": 4.346438160728785, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739217652958877, + "loss": 1.8131, + "theoretical_loss": 4.346088523476569, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047390170511534604, + "loss": 1.6644, + "theoretical_loss": 4.345739174466127, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738816449348044, + "loss": 1.6942, + "theoretical_loss": 4.3453901132744575, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047386158475426276, + "loss": 1.6128, + "theoretical_loss": 4.345041339479453, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738415245737211, + "loss": 1.8607, + "theoretical_loss": 4.344692852659895, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047382146439317954, + "loss": 1.6397, + "theoretical_loss": 4.344344652395451, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738014042126379, + "loss": 1.5671, + "theoretical_loss": 4.343996738266677, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047378134403209626, + "loss": 1.6248, + "theoretical_loss": 4.343649109855009, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737612838515546, + "loss": 1.6748, + "theoretical_loss": 4.343301766742763, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047374122367101304, + "loss": 1.6671, + "theoretical_loss": 4.342954708513136, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047372116349047146, + "loss": 1.5574, + "theoretical_loss": 4.3426079347501965, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3620479106903076, + "objective/train/avg_token_score": 0.014170818962156773, + "objective/train/docs_used": 126453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4098846912384033, + "objective/train/original_loss": 3.9272265434265137, + "objective/train/theoretical_loss": 4.342434654413995, + "objective/train/tokens_used": 204800000, + "objective/train/unlikelihood_loss": 0.20976464450359344, + "theoretical_loss": 4.342434654413995, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737011033099298, + "loss": 1.5437, + "theoretical_loss": 4.342261445038888, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736810431293882, + "loss": 1.5255, + "theoretical_loss": 4.341915238965026, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736609829488466, + "loss": 1.7084, + "theoretical_loss": 4.34156931611529, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047364092276830496, + "loss": 1.6739, + "theoretical_loss": 4.341223676077232, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736208625877633, + "loss": 1.6936, + "theoretical_loss": 4.340878318439261, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736008024072217, + "loss": 1.7022, + "theoretical_loss": 4.3405332427906504, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047358074222668005, + "loss": 1.3979, + "theoretical_loss": 4.340188448721532, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047356068204613847, + "loss": 1.7449, + "theoretical_loss": 4.339843935822895, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047354062186559683, + "loss": 1.5221, + "theoretical_loss": 4.339499703686579, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735205616850552, + "loss": 1.6496, + "theoretical_loss": 4.339155751905282, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047350050150451355, + "loss": 1.6696, + "theoretical_loss": 4.338812080072545, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047348044132397197, + "loss": 1.7524, + "theoretical_loss": 4.3384686877827585, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.37756597995758057, + "objective/train/avg_token_score": 0.015428772196173668, + "objective/train/docs_used": 127848, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4948502779006958, + "objective/train/original_loss": 4.056982040405273, + "objective/train/theoretical_loss": 4.33812557463116, + "objective/train/tokens_used": 206438400, + "objective/train/unlikelihood_loss": 0.15866130590438843, + "theoretical_loss": 4.33812557463116, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047346038114343033, + "loss": 1.5575, + "theoretical_loss": 4.33812557463116, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734403209628887, + "loss": 1.4098, + "theoretical_loss": 4.337782740213827, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047342026078234706, + "loss": 1.6327, + "theoretical_loss": 4.337440184127679, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734002006018054, + "loss": 1.3983, + "theoretical_loss": 4.337097905970471, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047338014042126383, + "loss": 1.7989, + "theoretical_loss": 4.336755905340797, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733600802407222, + "loss": 1.6815, + "theoretical_loss": 4.336414181838082, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047334002006018056, + "loss": 1.6673, + "theoretical_loss": 4.336072735062583, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733199598796389, + "loss": 1.7324, + "theoretical_loss": 4.335731564615387, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047329989969909734, + "loss": 1.5223, + "theoretical_loss": 4.335390670098407, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732798395185557, + "loss": 1.7556, + "theoretical_loss": 4.335050051114379, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325977933801406, + "loss": 1.6275, + "theoretical_loss": 4.334709707266865, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732397191574724, + "loss": 1.7843, + "theoretical_loss": 4.334369638160242, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732196589769308, + "loss": 1.6139, + "theoretical_loss": 4.334029843399709, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4079912006855011, + "objective/train/avg_token_score": 0.016537239775061607, + "objective/train/docs_used": 128338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6732548475265503, + "objective/train/original_loss": 4.391384124755859, + "objective/train/theoretical_loss": 4.333860048776074, + "objective/train/tokens_used": 208076800, + "objective/train/unlikelihood_loss": 0.11190986633300781, + "theoretical_loss": 4.333860048776074, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731995987963892, + "loss": 1.8005, + "theoretical_loss": 4.33369032259128, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047317953861584757, + "loss": 1.4683, + "theoretical_loss": 4.33335107534178, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047315947843530593, + "loss": 1.5492, + "theoretical_loss": 4.3330121012588485, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731394182547643, + "loss": 1.6122, + "theoretical_loss": 4.332673399950932, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731193580742227, + "loss": 1.6955, + "theoretical_loss": 4.332334971027284, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047309929789368107, + "loss": 1.4314, + "theoretical_loss": 4.331996814097963, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047307923771313943, + "loss": 1.599, + "theoretical_loss": 4.331658928773831, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730591775325978, + "loss": 1.7127, + "theoretical_loss": 4.33132131466655, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303911735205615, + "loss": 1.5623, + "theoretical_loss": 4.330983971388578, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047301905717151457, + "loss": 1.4685, + "theoretical_loss": 4.330646898553173, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047299899699097293, + "loss": 1.5697, + "theoretical_loss": 4.330310095774383, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729789368104313, + "loss": 1.652, + "theoretical_loss": 4.329973562667053, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.45869991183280945, + "objective/train/avg_token_score": 0.025695886462926865, + "objective/train/docs_used": 129483, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.688183307647705, + "objective/train/original_loss": 4.105484485626221, + "objective/train/theoretical_loss": 4.329637298846812, + "objective/train/tokens_used": 209715200, + "objective/train/unlikelihood_loss": 0.14652951061725616, + "theoretical_loss": 4.329637298846812, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047295887662988966, + "loss": 1.675, + "theoretical_loss": 4.329637298846812, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729388164493481, + "loss": 1.5683, + "theoretical_loss": 4.32930130393008, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047291875626880644, + "loss": 1.6511, + "theoretical_loss": 4.3289655775340625, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728986960882648, + "loss": 1.5679, + "theoretical_loss": 4.328630119276747, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047287863590772316, + "loss": 1.6666, + "theoretical_loss": 4.328294928776903, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728585757271815, + "loss": 1.6358, + "theoretical_loss": 4.327960005654081, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047283851554663994, + "loss": 1.5076, + "theoretical_loss": 4.327625349528605, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728184553660983, + "loss": 1.5038, + "theoretical_loss": 4.3272909600215765, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047279839518555666, + "loss": 1.7096, + "theoretical_loss": 4.326956836754871, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472778335005015, + "loss": 1.4226, + "theoretical_loss": 4.326622979351132, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047275827482447344, + "loss": 1.5037, + "theoretical_loss": 4.326289387433776, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727382146439318, + "loss": 1.7344, + "theoretical_loss": 4.325956060626982, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047271815446339017, + "loss": 1.7053, + "theoretical_loss": 4.325622998555697, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.433956503868103, + "objective/train/avg_token_score": 0.010776055976748466, + "objective/train/docs_used": 130263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.608245849609375, + "objective/train/original_loss": 4.053385257720947, + "objective/train/theoretical_loss": 4.3254565666788745, + "objective/train/tokens_used": 211353600, + "objective/train/unlikelihood_loss": 0.1744779348373413, + "theoretical_loss": 4.3254565666788745, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047269809428284853, + "loss": 1.6731, + "theoretical_loss": 4.325290200845629, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047267803410230695, + "loss": 1.7337, + "theoretical_loss": 4.324957667123249, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726579739217653, + "loss": 1.7319, + "theoretical_loss": 4.324625397015783, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047263791374122367, + "loss": 1.6882, + "theoretical_loss": 4.324293390151218, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047261785356068203, + "loss": 1.5198, + "theoretical_loss": 4.323961646158294, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725977933801404, + "loss": 1.5977, + "theoretical_loss": 4.323630164666502, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725777331995988, + "loss": 1.6756, + "theoretical_loss": 4.323298945306089, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725576730190572, + "loss": 1.6358, + "theoretical_loss": 4.322967987708043, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047253761283851554, + "loss": 1.6253, + "theoretical_loss": 4.322637291504106, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725175526579739, + "loss": 1.5415, + "theoretical_loss": 4.322306856326761, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724974924774323, + "loss": 1.7128, + "theoretical_loss": 4.321976681809236, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724774322968907, + "loss": 1.658, + "theoretical_loss": 4.321646767585497, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.32704055309295654, + "objective/train/avg_token_score": 0.014311397448182106, + "objective/train/docs_used": 131630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.350407600402832, + "objective/train/original_loss": 4.261425971984863, + "objective/train/theoretical_loss": 4.321317113290252, + "objective/train/tokens_used": 212992000, + "objective/train/unlikelihood_loss": 0.19203782081604004, + "theoretical_loss": 4.321317113290252, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047245737211634904, + "loss": 1.4489, + "theoretical_loss": 4.321317113290252, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724373119358074, + "loss": 1.6253, + "theoretical_loss": 4.320987718558945, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047241725175526576, + "loss": 1.6789, + "theoretical_loss": 4.320658583027755, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723971915747242, + "loss": 1.4694, + "theoretical_loss": 4.3203297063335935, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047237713139418254, + "loss": 1.515, + "theoretical_loss": 4.320001088114105, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723570712136409, + "loss": 1.7089, + "theoretical_loss": 4.31967272800766, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047233701103309927, + "loss": 1.527, + "theoretical_loss": 4.319344625653361, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723169508525577, + "loss": 1.7138, + "theoretical_loss": 4.319016780691033, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047229689067201605, + "loss": 1.7267, + "theoretical_loss": 4.318689192761225, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722768304914744, + "loss": 1.6687, + "theoretical_loss": 4.318361861505207, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047225677031093277, + "loss": 1.6014, + "theoretical_loss": 4.318034786564971, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047223671013039113, + "loss": 1.5851, + "theoretical_loss": 4.3177079675832255, + "tokens_seen": 214433792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722166499498496, + "loss": 1.3161, + "theoretical_loss": 4.3173814042033944, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4389662742614746, + "objective/train/avg_token_score": 0.023373806849122047, + "objective/train/docs_used": 132228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6921892166137695, + "objective/train/original_loss": 4.019752502441406, + "objective/train/theoretical_loss": 4.317218218252963, + "objective/train/tokens_used": 214630400, + "objective/train/unlikelihood_loss": 0.20437592267990112, + "theoretical_loss": 4.317218218252963, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047219658976930797, + "loss": 1.6002, + "theoretical_loss": 4.317055096069618, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047217652958876633, + "loss": 1.6065, + "theoretical_loss": 4.316729042826745, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721564694082247, + "loss": 1.7483, + "theoretical_loss": 4.316403244120339, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721364092276831, + "loss": 1.631, + "theoretical_loss": 4.316077699596671, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047211634904714147, + "loss": 1.8237, + "theoretical_loss": 4.315752408902716, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047209628886659983, + "loss": 1.6567, + "theoretical_loss": 4.315427371686157, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720762286860582, + "loss": 1.6742, + "theoretical_loss": 4.315102587595379, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047205616850551656, + "loss": 1.7232, + "theoretical_loss": 4.314778056279468, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047203610832497497, + "loss": 1.5351, + "theoretical_loss": 4.314453777388209, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047201604814443333, + "loss": 1.5639, + "theoretical_loss": 4.314129750572087, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719959879638917, + "loss": 1.4961, + "theoretical_loss": 4.313805975482278, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047197592778335006, + "loss": 1.7063, + "theoretical_loss": 4.313482451770659, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.47482892870903015, + "objective/train/avg_token_score": 0.027175776660442352, + "objective/train/docs_used": 133529, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9989675283432007, + "objective/train/original_loss": 4.329239845275879, + "objective/train/theoretical_loss": 4.3131591790897925, + "objective/train/tokens_used": 216268800, + "objective/train/unlikelihood_loss": 0.155557319521904, + "theoretical_loss": 4.3131591790897925, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719558676028085, + "loss": 1.4553, + "theoretical_loss": 4.3131591790897925, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193580742226684, + "loss": 1.6868, + "theoretical_loss": 4.312836157092934, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719157472417252, + "loss": 1.4115, + "theoretical_loss": 4.31251338543403, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047189568706118356, + "loss": 1.6705, + "theoretical_loss": 4.312190863767708, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718756268806419, + "loss": 1.6034, + "theoretical_loss": 4.311868591749287, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047185556670010034, + "loss": 1.7697, + "theoretical_loss": 4.311546569034767, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718355065195587, + "loss": 1.6095, + "theoretical_loss": 4.311224795280825, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047181544633901706, + "loss": 1.7137, + "theoretical_loss": 4.310903270144825, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717953861584754, + "loss": 1.5225, + "theoretical_loss": 4.310581993284805, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177532597793384, + "loss": 1.8289, + "theoretical_loss": 4.310260964359479, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717552657973922, + "loss": 1.6959, + "theoretical_loss": 4.309940183028236, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173520561685057, + "loss": 1.604, + "theoretical_loss": 4.309619648951139, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047171514543630893, + "loss": 1.6158, + "theoretical_loss": 4.30929936178892, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.5201612710952759, + "objective/train/avg_token_score": 0.025473540648818016, + "objective/train/docs_used": 134228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.066497325897217, + "objective/train/original_loss": 3.987381935119629, + "objective/train/theoretical_loss": 4.309139310695053, + "objective/train/tokens_used": 217907200, + "objective/train/unlikelihood_loss": 0.17281749844551086, + "theoretical_loss": 4.309139310695053, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169508525576735, + "loss": 1.7519, + "theoretical_loss": 4.308979321202983, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716750250752257, + "loss": 1.7713, + "theoretical_loss": 4.308659526855396, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047165496489468407, + "loss": 1.6655, + "theoretical_loss": 4.308339978408897, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163490471414243, + "loss": 1.7458, + "theoretical_loss": 4.308020675526883, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716148445336008, + "loss": 1.6181, + "theoretical_loss": 4.30770161787342, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715947843530592, + "loss": 1.5035, + "theoretical_loss": 4.307382805113228, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715747241725176, + "loss": 1.616, + "theoretical_loss": 4.307064236911692, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047155466399197594, + "loss": 1.806, + "theoretical_loss": 4.306745912934849, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715346038114343, + "loss": 1.6041, + "theoretical_loss": 4.306427832849394, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715145436308927, + "loss": 1.8418, + "theoretical_loss": 4.306109996322679, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714944834503511, + "loss": 1.5547, + "theoretical_loss": 4.305792403022703, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047147442326980944, + "loss": 1.2979, + "theoretical_loss": 4.305475052618119, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4022482931613922, + "objective/train/avg_token_score": 0.006948302499949932, + "objective/train/docs_used": 135676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5557243824005127, + "objective/train/original_loss": 3.7284510135650635, + "objective/train/theoretical_loss": 4.305157944778228, + "objective/train/tokens_used": 219545600, + "objective/train/unlikelihood_loss": 0.15928247570991516, + "theoretical_loss": 4.305157944778228, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714543630892678, + "loss": 1.86, + "theoretical_loss": 4.305157944778228, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047143430290872616, + "loss": 1.7673, + "theoretical_loss": 4.304841079172979, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714142427281846, + "loss": 1.6245, + "theoretical_loss": 4.304524455472965, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139418254764294, + "loss": 1.8468, + "theoretical_loss": 4.304208073349426, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713741223671013, + "loss": 1.5617, + "theoretical_loss": 4.30389193247424, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135406218655967, + "loss": 1.7598, + "theoretical_loss": 4.303576032519931, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713340020060181, + "loss": 1.7091, + "theoretical_loss": 4.303260373159659, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047131394182547645, + "loss": 1.7758, + "theoretical_loss": 4.30294495406722, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712938816449348, + "loss": 1.6308, + "theoretical_loss": 4.302629774917049, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047127382146439317, + "loss": 1.6361, + "theoretical_loss": 4.302314835384214, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125376128385153, + "loss": 1.6959, + "theoretical_loss": 4.302000135144416, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047123370110330995, + "loss": 1.7791, + "theoretical_loss": 4.301685673873987, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712136409227683, + "loss": 1.6592, + "theoretical_loss": 4.301371451249888, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4833821952342987, + "objective/train/avg_token_score": 0.009588075801730156, + "objective/train/docs_used": 136371, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.763645887374878, + "objective/train/original_loss": 3.7894089221954346, + "objective/train/theoretical_loss": 4.301214429329433, + "objective/train/tokens_used": 221184000, + "objective/train/unlikelihood_loss": 0.19660082459449768, + "theoretical_loss": 4.301214429329433, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711935807422267, + "loss": 1.6616, + "theoretical_loss": 4.301057466949707, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047117352056168504, + "loss": 1.7754, + "theoretical_loss": 4.30074372065166, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115346038114345, + "loss": 1.5822, + "theoretical_loss": 4.300430212034587, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711334002006018, + "loss": 1.7638, + "theoretical_loss": 4.300116940777951, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711133400200602, + "loss": 1.8272, + "theoretical_loss": 4.299803906561835, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047109327983951854, + "loss": 1.6185, + "theoretical_loss": 4.299491109066947, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710732196589769, + "loss": 1.6955, + "theoretical_loss": 4.2991785479746065, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710531594784353, + "loss": 1.7621, + "theoretical_loss": 4.298866222966755, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710330992978937, + "loss": 1.5053, + "theoretical_loss": 4.298554133725946, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047101303911735204, + "loss": 1.7638, + "theoretical_loss": 4.298242279935349, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709929789368104, + "loss": 1.7638, + "theoretical_loss": 4.297930661278745, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709729187562688, + "loss": 1.6024, + "theoretical_loss": 4.297619277440523, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4119012653827667, + "objective/train/avg_token_score": 0.012975508347153664, + "objective/train/docs_used": 137650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5277689695358276, + "objective/train/original_loss": 4.178741931915283, + "objective/train/theoretical_loss": 4.297308128105687, + "objective/train/tokens_used": 222822400, + "objective/train/unlikelihood_loss": 0.15160509943962097, + "theoretical_loss": 4.297308128105687, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709528585757272, + "loss": 1.4614, + "theoretical_loss": 4.297308128105687, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047093279839518555, + "loss": 1.6429, + "theoretical_loss": 4.296997212959842, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709127382146439, + "loss": 1.6713, + "theoretical_loss": 4.296686531689204, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047089267803410227, + "loss": 1.7281, + "theoretical_loss": 4.296376083980589, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708726178535607, + "loss": 1.7329, + "theoretical_loss": 4.296065869521421, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047085255767301905, + "loss": 1.6212, + "theoretical_loss": 4.2957558879997215, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708324974924774, + "loss": 1.7222, + "theoretical_loss": 4.295446139104112, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708124373119358, + "loss": 1.5936, + "theoretical_loss": 4.295136622523817, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707923771313942, + "loss": 1.5582, + "theoretical_loss": 4.294827337948651, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047077231695085255, + "loss": 1.621, + "theoretical_loss": 4.29451828506903, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707522567703109, + "loss": 1.7204, + "theoretical_loss": 4.29420946357596, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707321965897693, + "loss": 1.6165, + "theoretical_loss": 4.293900873161043, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047071213640922775, + "loss": 1.6774, + "theoretical_loss": 4.293592513516469, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.3668743968009949, + "objective/train/avg_token_score": 0.00708651402965188, + "objective/train/docs_used": 138458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4711482524871826, + "objective/train/original_loss": 4.133864879608154, + "objective/train/theoretical_loss": 4.293438420137031, + "objective/train/tokens_used": 224460800, + "objective/train/unlikelihood_loss": 0.21583233773708344, + "theoretical_loss": 4.293438420137031, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706920762286861, + "loss": 1.4837, + "theoretical_loss": 4.293284384335017, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047067201604814447, + "loss": 1.646, + "theoretical_loss": 4.292976485310057, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047065195586760283, + "loss": 1.4897, + "theoretical_loss": 4.292668816135545, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706318956870612, + "loss": 1.6201, + "theoretical_loss": 4.29236137650602, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706118355065196, + "loss": 1.7653, + "theoretical_loss": 4.292054166116605, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470591775325978, + "loss": 1.5181, + "theoretical_loss": 4.2917471846630075, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047057171514543634, + "loss": 1.5244, + "theoretical_loss": 4.291440431841513, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705516549648947, + "loss": 1.3208, + "theoretical_loss": 4.291133907348989, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705315947843531, + "loss": 1.6231, + "theoretical_loss": 4.29082761088288, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705115346038115, + "loss": 1.4922, + "theoretical_loss": 4.290521542141203, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047049147442326984, + "loss": 1.63, + "theoretical_loss": 4.290215700822556, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704714142427282, + "loss": 1.6671, + "theoretical_loss": 4.289910086626108, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.519305944442749, + "objective/train/avg_token_score": 0.011661575175821781, + "objective/train/docs_used": 139641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.168332815170288, + "objective/train/original_loss": 4.076672077178955, + "objective/train/theoretical_loss": 4.2896046992515995, + "objective/train/tokens_used": 226099200, + "objective/train/unlikelihood_loss": 0.17042392492294312, + "theoretical_loss": 4.2896046992515995, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047045135406218656, + "loss": 1.5969, + "theoretical_loss": 4.2896046992515995, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470431293881645, + "loss": 1.5568, + "theoretical_loss": 4.289299538399341, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047041123370110334, + "loss": 1.654, + "theoretical_loss": 4.288994603770215, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703911735205617, + "loss": 1.651, + "theoretical_loss": 4.28868989506567, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047037111334002007, + "loss": 1.8949, + "theoretical_loss": 4.288385411987722, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703510531594785, + "loss": 1.5494, + "theoretical_loss": 4.2880811542389505, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047033099297893685, + "loss": 1.4705, + "theoretical_loss": 4.287777121522501, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703109327983952, + "loss": 1.6097, + "theoretical_loss": 4.287473313542077, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047029087261785357, + "loss": 1.7498, + "theoretical_loss": 4.287169730001949, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047027081243731193, + "loss": 1.7013, + "theoretical_loss": 4.286866370606943, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047025075225677035, + "loss": 1.8046, + "theoretical_loss": 4.286563235062444, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702306920762287, + "loss": 1.6791, + "theoretical_loss": 4.286260323074394, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702106318956871, + "loss": 1.578, + "theoretical_loss": 4.285957634349289, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.46957477927207947, + "objective/train/avg_token_score": 0.018808742985129356, + "objective/train/docs_used": 140376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8826521635055542, + "objective/train/original_loss": 4.291516304016113, + "objective/train/theoretical_loss": 4.285806373618774, + "objective/train/tokens_used": 227737600, + "objective/train/unlikelihood_loss": 0.15742744505405426, + "theoretical_loss": 4.285806373618774, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047019057171514544, + "loss": 1.8432, + "theoretical_loss": 4.285655168594182, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047017051153460385, + "loss": 1.539, + "theoretical_loss": 4.285352925516676, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701504513540622, + "loss": 1.6382, + "theoretical_loss": 4.285050904824925, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701303911735206, + "loss": 1.5511, + "theoretical_loss": 4.284749106227636, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047011033099297894, + "loss": 1.6545, + "theoretical_loss": 4.284447529434061, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700902708124373, + "loss": 1.6648, + "theoretical_loss": 4.284146174154003, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700702106318957, + "loss": 1.5866, + "theoretical_loss": 4.283845040097807, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700501504513541, + "loss": 1.6361, + "theoretical_loss": 4.2835441269763646, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047003009027081244, + "loss": 1.6915, + "theoretical_loss": 4.283243434501112, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700100300902708, + "loss": 1.6013, + "theoretical_loss": 4.282942962384023, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699899699097292, + "loss": 1.6088, + "theoretical_loss": 4.282642710337618, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699699097291876, + "loss": 1.7422, + "theoretical_loss": 4.282342678074951, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.34762951731681824, + "objective/train/avg_token_score": 0.020571375265717506, + "objective/train/docs_used": 141693, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.228954553604126, + "objective/train/original_loss": 4.2740325927734375, + "objective/train/theoretical_loss": 4.282042865309616, + "objective/train/tokens_used": 229376000, + "objective/train/unlikelihood_loss": 0.13970565795898438, + "theoretical_loss": 4.282042865309616, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046994984954864595, + "loss": 1.5481, + "theoretical_loss": 4.282042865309616, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699297893681043, + "loss": 1.7614, + "theoretical_loss": 4.2817432717557455, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046990972918756267, + "loss": 1.4883, + "theoretical_loss": 4.281443897128004, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698896690070211, + "loss": 1.5377, + "theoretical_loss": 4.281144741141593, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046986960882647945, + "loss": 1.6032, + "theoretical_loss": 4.2808458035122445, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698495486459378, + "loss": 1.6431, + "theoretical_loss": 4.280547083956224, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698294884653962, + "loss": 1.7657, + "theoretical_loss": 4.280248582190324, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698094282848546, + "loss": 1.6122, + "theoretical_loss": 4.279950297931869, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046978936810431295, + "loss": 1.5755, + "theoretical_loss": 4.279652230898709, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697693079237713, + "loss": 1.557, + "theoretical_loss": 4.27935438080922, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697492477432297, + "loss": 1.722, + "theoretical_loss": 4.279056747382306, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046972918756268804, + "loss": 1.5124, + "theoretical_loss": 4.27875933033739, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046970912738214646, + "loss": 1.7483, + "theoretical_loss": 4.278462129394423, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.44086021184921265, + "objective/train/avg_token_score": 0.02034401148557663, + "objective/train/docs_used": 142407, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9175533056259155, + "objective/train/original_loss": 4.461835861206055, + "objective/train/theoretical_loss": 4.278313609873795, + "objective/train/tokens_used": 231014400, + "objective/train/unlikelihood_loss": 0.13161636888980865, + "theoretical_loss": 4.278313609873795, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696890672016048, + "loss": 1.6861, + "theoretical_loss": 4.278165144273871, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696690070210632, + "loss": 1.6634, + "theoretical_loss": 4.277868374696725, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046964894684052154, + "loss": 1.7192, + "theoretical_loss": 4.277571820384491, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046962888665997996, + "loss": 1.5473, + "theoretical_loss": 4.277275481059195, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696088264794383, + "loss": 1.6103, + "theoretical_loss": 4.276979356443377, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695887662988967, + "loss": 1.6004, + "theoretical_loss": 4.276683446260093, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046956870611835505, + "loss": 1.5718, + "theoretical_loss": 4.276387750232913, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046954864593781346, + "loss": 1.4526, + "theoretical_loss": 4.276092268085918, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695285857572718, + "loss": 1.7252, + "theoretical_loss": 4.275796999543703, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695085255767302, + "loss": 1.6623, + "theoretical_loss": 4.275501944331367, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046948846539618855, + "loss": 1.6335, + "theoretical_loss": 4.275207102174525, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694684052156469, + "loss": 1.9984, + "theoretical_loss": 4.2749124727992935, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.44855815172195435, + "objective/train/avg_token_score": 0.007418564986437559, + "objective/train/docs_used": 143093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7657521963119507, + "objective/train/original_loss": 3.696604013442993, + "objective/train/theoretical_loss": 4.274618055932298, + "objective/train/tokens_used": 232652800, + "objective/train/unlikelihood_loss": 0.1868204027414322, + "theoretical_loss": 4.274618055932298, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694483450351053, + "loss": 1.7135, + "theoretical_loss": 4.274618055932298, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694282848545637, + "loss": 1.6273, + "theoretical_loss": 4.27432385130067, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046940822467402205, + "loss": 1.6464, + "theoretical_loss": 4.2740298586320415, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693881644934804, + "loss": 1.6506, + "theoretical_loss": 4.27373607765455, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046936810431293883, + "loss": 1.5552, + "theoretical_loss": 4.273442508096833, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693480441323972, + "loss": 1.6676, + "theoretical_loss": 4.273149149688028, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046932798395185555, + "loss": 1.9249, + "theoretical_loss": 4.272856002157772, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693079237713139, + "loss": 1.7331, + "theoretical_loss": 4.2725630652362, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692878635907723, + "loss": 1.6774, + "theoretical_loss": 4.272270338653942, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692678034102307, + "loss": 1.5886, + "theoretical_loss": 4.271977822142125, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046924774322968906, + "loss": 1.6229, + "theoretical_loss": 4.27168551543237, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692276830491474, + "loss": 1.6078, + "theoretical_loss": 4.27139341825679, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046920762286860584, + "loss": 1.672, + "theoretical_loss": 4.27110153034799, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4761730134487152, + "objective/train/avg_token_score": 0.01180332899093628, + "objective/train/docs_used": 144323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8356056213378906, + "objective/train/original_loss": 4.213988780975342, + "objective/train/theoretical_loss": 4.270955664785207, + "objective/train/tokens_used": 234291200, + "objective/train/unlikelihood_loss": 0.13796842098236084, + "theoretical_loss": 4.270955664785207, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046918756268806425, + "loss": 1.6133, + "theoretical_loss": 4.2708098514390676, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691675025075226, + "loss": 1.7033, + "theoretical_loss": 4.2705183812636065, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469147442326981, + "loss": 1.7059, + "theoretical_loss": 4.270227119555681, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046912738214643934, + "loss": 1.7052, + "theoretical_loss": 4.269936066049852, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691073219658977, + "loss": 1.4875, + "theoretical_loss": 4.269645220481166, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690872617853561, + "loss": 1.7967, + "theoretical_loss": 4.269354582585156, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690672016048145, + "loss": 1.5462, + "theoretical_loss": 4.269064152097835, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046904714142427284, + "loss": 1.642, + "theoretical_loss": 4.268773928755701, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690270812437312, + "loss": 1.7563, + "theoretical_loss": 4.268483912295735, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690070210631896, + "loss": 1.7788, + "theoretical_loss": 4.268194102455395, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468986960882648, + "loss": 1.758, + "theoretical_loss": 4.267904498972618, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046896690070210635, + "loss": 1.5842, + "theoretical_loss": 4.267615101585821, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.48765885829925537, + "objective/train/avg_token_score": 0.008590947836637497, + "objective/train/docs_used": 145084, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9980686902999878, + "objective/train/original_loss": 4.121566295623779, + "objective/train/theoretical_loss": 4.267325910033897, + "objective/train/tokens_used": 235929600, + "objective/train/unlikelihood_loss": 0.1698838621377945, + "theoretical_loss": 4.267325910033897, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689468405215647, + "loss": 1.6593, + "theoretical_loss": 4.267325910033897, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892678034102307, + "loss": 1.6164, + "theoretical_loss": 4.267036924056215, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689067201604815, + "loss": 1.5894, + "theoretical_loss": 4.266748143392617, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046888665997993985, + "loss": 1.6229, + "theoretical_loss": 4.26645956778342, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688665997993982, + "loss": 1.805, + "theoretical_loss": 4.266171196969412, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688465396188566, + "loss": 1.7046, + "theoretical_loss": 4.265883030691853, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468826479438315, + "loss": 1.6652, + "theoretical_loss": 4.265595068692473, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046880641925777335, + "loss": 1.7922, + "theoretical_loss": 4.265307310713471, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687863590772317, + "loss": 1.5168, + "theoretical_loss": 4.265019756497512, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687662988966901, + "loss": 1.7535, + "theoretical_loss": 4.264732405787731, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046874623871614844, + "loss": 1.6919, + "theoretical_loss": 4.264445258327724, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872617853560686, + "loss": 1.7658, + "theoretical_loss": 4.264158313861557, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687061183550652, + "loss": 1.6452, + "theoretical_loss": 4.2638715721337554, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4448924660682678, + "objective/train/avg_token_score": 0.019487597048282623, + "objective/train/docs_used": 146518, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6499921083450317, + "objective/train/original_loss": 4.144484519958496, + "objective/train/theoretical_loss": 4.263728277217032, + "objective/train/tokens_used": 237568000, + "objective/train/unlikelihood_loss": 0.14790794253349304, + "theoretical_loss": 4.263728277217032, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686860581745236, + "loss": 1.6665, + "theoretical_loss": 4.263585032889306, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046866599799398194, + "loss": 1.5501, + "theoretical_loss": 4.2632986958736625, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864593781344036, + "loss": 1.788, + "theoretical_loss": 4.263012560832733, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686258776328987, + "loss": 1.2767, + "theoretical_loss": 4.262726627512886, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686058174523571, + "loss": 1.7164, + "theoretical_loss": 4.26244089566095, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046858575727181545, + "loss": 1.6782, + "theoretical_loss": 4.262155365024207, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656970912738, + "loss": 1.6008, + "theoretical_loss": 4.261870035350399, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685456369107322, + "loss": 1.5916, + "theoretical_loss": 4.26158490638772, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685255767301906, + "loss": 1.6809, + "theoretical_loss": 4.261299977884816, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046850551654964895, + "loss": 1.5857, + "theoretical_loss": 4.261015249590789, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684854563691073, + "loss": 1.5282, + "theoretical_loss": 4.260730721255191, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046846539618856573, + "loss": 1.7735, + "theoretical_loss": 4.260446392628026, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.43187928199768066, + "objective/train/avg_token_score": 0.025535602122545242, + "objective/train/docs_used": 147039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7066162824630737, + "objective/train/original_loss": 4.2467427253723145, + "objective/train/theoretical_loss": 4.260162263459744, + "objective/train/tokens_used": 239206400, + "objective/train/unlikelihood_loss": 0.11330122500658035, + "theoretical_loss": 4.260162263459744, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684453360080241, + "loss": 1.475, + "theoretical_loss": 4.260162263459744, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046842527582748245, + "loss": 1.5486, + "theoretical_loss": 4.259878333501247, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684052156469408, + "loss": 1.4455, + "theoretical_loss": 4.259594602503881, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838515546639923, + "loss": 1.6137, + "theoretical_loss": 4.259311070219441, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683650952858576, + "loss": 1.4483, + "theoretical_loss": 4.259027736400165, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834503510531595, + "loss": 1.618, + "theoretical_loss": 4.258744600798739, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683249749247743, + "loss": 1.6814, + "theoretical_loss": 4.258461663168285, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683049147442327, + "loss": 1.5598, + "theoretical_loss": 4.258178923262376, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682848545636911, + "loss": 1.6794, + "theoretical_loss": 4.257896380835018, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046826479438314946, + "loss": 1.5066, + "theoretical_loss": 4.257614035640662, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682447342026078, + "loss": 1.7484, + "theoretical_loss": 4.257331887434198, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682246740220662, + "loss": 1.6482, + "theoretical_loss": 4.25704993597095, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682046138415246, + "loss": 1.5145, + "theoretical_loss": 4.256768181006683, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.35654935240745544, + "objective/train/avg_token_score": 0.024610688909888268, + "objective/train/docs_used": 148118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2514936923980713, + "objective/train/original_loss": 3.9919378757476807, + "objective/train/theoretical_loss": 4.2566273771354615, + "objective/train/tokens_used": 240844800, + "objective/train/unlikelihood_loss": 0.20598264038562775, + "theoretical_loss": 4.2566273771354615, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046818455366098296, + "loss": 1.5226, + "theoretical_loss": 4.256486622297595, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681644934804413, + "loss": 1.7672, + "theoretical_loss": 4.256205259600321, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681444332998997, + "loss": 1.6025, + "theoretical_loss": 4.255924092671931, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046812437311935805, + "loss": 1.5735, + "theoretical_loss": 4.255643121269924, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810431293881646, + "loss": 1.8846, + "theoretical_loss": 4.255362345152234, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680842527582748, + "loss": 1.5332, + "theoretical_loss": 4.255081764077224, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680641925777332, + "loss": 1.6185, + "theoretical_loss": 4.254801377803689, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046804413239719155, + "loss": 1.5774, + "theoretical_loss": 4.254521186090852, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802407221664997, + "loss": 1.728, + "theoretical_loss": 4.254241188698361, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800401203610833, + "loss": 1.5526, + "theoretical_loss": 4.2539613853862965, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679839518555667, + "loss": 1.6072, + "theoretical_loss": 4.253681775915161, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046796389167502505, + "loss": 1.7814, + "theoretical_loss": 4.253402360045882, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.2602028250694275, + "objective/train/avg_token_score": 0.019304223358631134, + "objective/train/docs_used": 148826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9984758496284485, + "objective/train/original_loss": 4.527791500091553, + "objective/train/theoretical_loss": 4.253123137539814, + "objective/train/tokens_used": 242483200, + "objective/train/unlikelihood_loss": 0.15593910217285156, + "theoretical_loss": 4.253123137539814, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679438314944834, + "loss": 1.4314, + "theoretical_loss": 4.253123137539814, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046792377131394183, + "loss": 1.5239, + "theoretical_loss": 4.25284410815873, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679037111334002, + "loss": 1.5362, + "theoretical_loss": 4.252565271664828, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046788365095285856, + "loss": 1.663, + "theoretical_loss": 4.252286627820727, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678635907723169, + "loss": 1.4544, + "theoretical_loss": 4.252008176389465, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046784353059177534, + "loss": 1.5947, + "theoretical_loss": 4.251729917134498, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678234704112337, + "loss": 1.4033, + "theoretical_loss": 4.251451849819704, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046780341023069206, + "loss": 1.618, + "theoretical_loss": 4.251173974209375, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677833500501504, + "loss": 1.5327, + "theoretical_loss": 4.250896290068218, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677632898696088, + "loss": 1.7856, + "theoretical_loss": 4.25061879716136, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677432296890672, + "loss": 1.9125, + "theoretical_loss": 4.250341495254337, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046772316950852556, + "loss": 1.6721, + "theoretical_loss": 4.250064384113102, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467703109327984, + "loss": 1.4838, + "theoretical_loss": 4.249787463504019, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.20821113884449005, + "objective/train/avg_token_score": 0.05237054452300072, + "objective/train/docs_used": 150000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8499055504798889, + "objective/train/original_loss": 5.0128703117370605, + "objective/train/theoretical_loss": 4.249649074576134, + "objective/train/tokens_used": 244121600, + "objective/train/unlikelihood_loss": 0.0956830307841301, + "theoretical_loss": 4.249649074576134, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046768304914744234, + "loss": 1.5865, + "theoretical_loss": 4.249510733193862, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046766298896690076, + "loss": 1.6801, + "theoretical_loss": 4.249234192949818, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676429287863591, + "loss": 1.5263, + "theoretical_loss": 4.248957842539484, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676228686058175, + "loss": 1.5385, + "theoretical_loss": 4.2486816817308615, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046760280842527585, + "loss": 1.5574, + "theoretical_loss": 4.248405710292364, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675827482447342, + "loss": 1.5474, + "theoretical_loss": 4.248129927992808, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675626880641926, + "loss": 1.64, + "theoretical_loss": 4.247854334601421, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467542627883651, + "loss": 1.6484, + "theoretical_loss": 4.247578929887829, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046752256770310935, + "loss": 1.5901, + "theoretical_loss": 4.247303713622067, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675025075225677, + "loss": 1.697, + "theoretical_loss": 4.247028685574569, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046748244734202613, + "loss": 1.8329, + "theoretical_loss": 4.246753845516174, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674623871614845, + "loss": 1.582, + "theoretical_loss": 4.246479193218123, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.35355570912361145, + "objective/train/avg_token_score": 0.023441534489393234, + "objective/train/docs_used": 150657, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.178994059562683, + "objective/train/original_loss": 3.2855653762817383, + "objective/train/theoretical_loss": 4.246204728452055, + "objective/train/tokens_used": 245760000, + "objective/train/unlikelihood_loss": 0.2683996260166168, + "theoretical_loss": 4.246204728452055, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046744232698094285, + "loss": 1.4603, + "theoretical_loss": 4.246204728452055, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674222668004012, + "loss": 1.5375, + "theoretical_loss": 4.245930450990007, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046740220661985963, + "loss": 1.4696, + "theoretical_loss": 4.245656360604417, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467382146439318, + "loss": 1.641, + "theoretical_loss": 4.2453824570681205, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046736208625877636, + "loss": 1.5838, + "theoretical_loss": 4.2451087401543495, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673420260782347, + "loss": 1.736, + "theoretical_loss": 4.24483520963673, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673219658976931, + "loss": 1.4829, + "theoretical_loss": 4.244561865289285, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673019057171515, + "loss": 1.6817, + "theoretical_loss": 4.2442887068864295, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046728184553660986, + "loss": 1.5743, + "theoretical_loss": 4.244015734202973, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672617853560682, + "loss": 1.6287, + "theoretical_loss": 4.243742947014117, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672417251755266, + "loss": 1.5413, + "theoretical_loss": 4.243470345095453, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467221664994985, + "loss": 1.5116, + "theoretical_loss": 4.243197928222964, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046720160481444336, + "loss": 1.6491, + "theoretical_loss": 4.242925696173021, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.2963709533214569, + "objective/train/avg_token_score": 0.031853631138801575, + "objective/train/docs_used": 151862, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1483128070831299, + "objective/train/original_loss": 4.083296775817871, + "objective/train/theoretical_loss": 4.2427896493867285, + "objective/train/tokens_used": 247398400, + "objective/train/unlikelihood_loss": 0.13969635963439941, + "theoretical_loss": 4.2427896493867285, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671815446339017, + "loss": 1.597, + "theoretical_loss": 4.242653648722387, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671614844533601, + "loss": 1.5181, + "theoretical_loss": 4.24238178564821, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046714142427281845, + "loss": 1.7096, + "theoretical_loss": 4.242110106728022, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046712136409227686, + "loss": 1.7045, + "theoretical_loss": 4.241838611739748, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671013039117352, + "loss": 1.4473, + "theoretical_loss": 4.241567300461693, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670812437311936, + "loss": 1.4892, + "theoretical_loss": 4.241296172672547, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046706118355065195, + "loss": 1.4327, + "theoretical_loss": 4.241025228151383, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046704112337011037, + "loss": 1.4395, + "theoretical_loss": 4.240754466677659, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046702106318956873, + "loss": 1.3767, + "theoretical_loss": 4.240483888031212, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670010030090271, + "loss": 1.6805, + "theoretical_loss": 4.240213491992261, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046698094282848545, + "loss": 1.7161, + "theoretical_loss": 4.239943278341404, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669608826479438, + "loss": 1.6768, + "theoretical_loss": 4.239673246859619, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.40518084168434143, + "objective/train/avg_token_score": 0.00906757079064846, + "objective/train/docs_used": 152583, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6137572526931763, + "objective/train/original_loss": 3.96043062210083, + "objective/train/theoretical_loss": 4.239403397328261, + "objective/train/tokens_used": 249036800, + "objective/train/unlikelihood_loss": 0.250114768743515, + "theoretical_loss": 4.239403397328261, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694082246740223, + "loss": 1.6961, + "theoretical_loss": 4.239403397328261, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669207622868606, + "loss": 1.5261, + "theoretical_loss": 4.239133729529064, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046690070210631896, + "loss": 1.7208, + "theoretical_loss": 4.2388642432441355, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668806419257773, + "loss": 1.4695, + "theoretical_loss": 4.238594938255963, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046686058174523574, + "loss": 1.5757, + "theoretical_loss": 4.238325814347404, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668405215646941, + "loss": 1.5495, + "theoretical_loss": 4.238056871301695, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682046138415246, + "loss": 1.4589, + "theoretical_loss": 4.237788108902441, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668004012036108, + "loss": 1.5079, + "theoretical_loss": 4.237519526933622, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667803410230692, + "loss": 1.6051, + "theoretical_loss": 4.237251125179588, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667602808425276, + "loss": 1.6881, + "theoretical_loss": 4.23698290342506, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046674022066198596, + "loss": 1.4733, + "theoretical_loss": 4.23671486145513, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667201604814443, + "loss": 1.4943, + "theoretical_loss": 4.236446999055257, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667001003009027, + "loss": 1.5532, + "theoretical_loss": 4.2361793160112695, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.38782989978790283, + "objective/train/avg_token_score": 0.009073655121028423, + "objective/train/docs_used": 153242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.569201111793518, + "objective/train/original_loss": 4.112845420837402, + "objective/train/theoretical_loss": 4.236045541680905, + "objective/train/tokens_used": 250675200, + "objective/train/unlikelihood_loss": 0.14907237887382507, + "theoretical_loss": 4.236045541680905, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666800401203611, + "loss": 1.5928, + "theoretical_loss": 4.235911812109363, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046665997993981947, + "loss": 1.5379, + "theoretical_loss": 4.235644487136098, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046663991975927783, + "loss": 1.5322, + "theoretical_loss": 4.235377340878404, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666198595787362, + "loss": 1.4665, + "theoretical_loss": 4.235110373123572, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046659979939819455, + "loss": 1.4523, + "theoretical_loss": 4.2348435836592575, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046657973921765297, + "loss": 1.6582, + "theoretical_loss": 4.234576972273481, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046655967903711133, + "loss": 1.662, + "theoretical_loss": 4.234310538754624, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665396188565697, + "loss": 1.4291, + "theoretical_loss": 4.234044282891429, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046651955867602806, + "loss": 1.4925, + "theoretical_loss": 4.233778204473002, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664994984954865, + "loss": 1.6234, + "theoretical_loss": 4.233512303288807, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046647943831494484, + "loss": 1.5104, + "theoretical_loss": 4.233246579128666, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664593781344032, + "loss": 1.5029, + "theoretical_loss": 4.232981031782761, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.3658357858657837, + "objective/train/avg_token_score": 0.005103480536490679, + "objective/train/docs_used": 154565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2179532051086426, + "objective/train/original_loss": 3.7866077423095703, + "objective/train/theoretical_loss": 4.232715661041632, + "objective/train/tokens_used": 252313600, + "objective/train/unlikelihood_loss": 0.20572546124458313, + "theoretical_loss": 4.232715661041632, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046643931795386156, + "loss": 1.6638, + "theoretical_loss": 4.232715661041632, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664192577733199, + "loss": 1.4777, + "theoretical_loss": 4.232450466696174, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046639919759277834, + "loss": 1.4621, + "theoretical_loss": 4.23218544853764, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663791374122367, + "loss": 1.815, + "theoretical_loss": 4.231920606357638, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046635907723169506, + "loss": 1.5611, + "theoretical_loss": 4.231655939948127, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663390170511534, + "loss": 1.6306, + "theoretical_loss": 4.231391449101425, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046631895687061184, + "loss": 1.5308, + "theoretical_loss": 4.231127133610198, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662988966900702, + "loss": 1.4716, + "theoretical_loss": 4.230862993267468, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046627883650952857, + "loss": 1.5696, + "theoretical_loss": 4.230599027866606, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046625877632898693, + "loss": 1.6201, + "theoretical_loss": 4.2303352372013325, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046623871614844535, + "loss": 1.5034, + "theoretical_loss": 4.230071621065721, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662186559679037, + "loss": 1.422, + "theoretical_loss": 4.229808179254192, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661985957873621, + "loss": 1.49, + "theoretical_loss": 4.229544911561513, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5327468514442444, + "objective/train/avg_token_score": 0.0074698301032185555, + "objective/train/docs_used": 156106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.954049825668335, + "objective/train/original_loss": 3.7289023399353027, + "objective/train/theoretical_loss": 4.229413342945703, + "objective/train/tokens_used": 253952000, + "objective/train/unlikelihood_loss": 0.149134561419487, + "theoretical_loss": 4.229413342945703, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661785356068205, + "loss": 1.7898, + "theoretical_loss": 4.229281817782801, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046615847542627885, + "loss": 1.6864, + "theoretical_loss": 4.229018897713519, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046613841524573727, + "loss": 1.3615, + "theoretical_loss": 4.228756151149475, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046611835506519563, + "loss": 1.615, + "theoretical_loss": 4.228493577886824, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466098294884654, + "loss": 1.5096, + "theoretical_loss": 4.228231177722063, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046607823470411235, + "loss": 1.4544, + "theoretical_loss": 4.227968950452035, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046605817452357077, + "loss": 1.661, + "theoretical_loss": 4.227706895873924, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046603811434302913, + "loss": 1.4282, + "theoretical_loss": 4.227445013785257, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660180541624875, + "loss": 1.6551, + "theoretical_loss": 4.227183303983901, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046599799398194586, + "loss": 1.4951, + "theoretical_loss": 4.226921766268067, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659779338014042, + "loss": 1.449, + "theoretical_loss": 4.226660400436302, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595787362086263, + "loss": 1.5032, + "theoretical_loss": 4.226399206287493, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4268695116043091, + "objective/train/avg_token_score": 0.012001312337815762, + "objective/train/docs_used": 156630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8062608242034912, + "objective/train/original_loss": 4.282796382904053, + "objective/train/theoretical_loss": 4.226138183620867, + "objective/train/tokens_used": 255590400, + "objective/train/unlikelihood_loss": 0.11538192629814148, + "theoretical_loss": 4.226138183620867, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465937813440321, + "loss": 1.5151, + "theoretical_loss": 4.226138183620867, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591775325977936, + "loss": 1.6603, + "theoretical_loss": 4.225877332235987, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658976930792377, + "loss": 1.4763, + "theoretical_loss": 4.225616651932753, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046587763289869614, + "loss": 1.6467, + "theoretical_loss": 4.225356142511402, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658575727181545, + "loss": 1.5337, + "theoretical_loss": 4.225095803772507, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046583751253761286, + "loss": 1.5934, + "theoretical_loss": 4.224835635516973, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658174523570712, + "loss": 1.4188, + "theoretical_loss": 4.224575637546041, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657973921765296, + "loss": 1.3615, + "theoretical_loss": 4.2243158096612845, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465777331995988, + "loss": 1.7877, + "theoretical_loss": 4.22405615166461, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575727181544636, + "loss": 1.652, + "theoretical_loss": 4.223796663358255, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657372116349047, + "loss": 1.5669, + "theoretical_loss": 4.223537344544788, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171514543631, + "loss": 1.5028, + "theoretical_loss": 4.22327819502711, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656970912738215, + "loss": 1.5612, + "theoretical_loss": 4.223019214608446, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.35031768679618835, + "objective/train/avg_token_score": 0.06552369147539139, + "objective/train/docs_used": 157258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4199466705322266, + "objective/train/original_loss": 4.115721225738525, + "objective/train/theoretical_loss": 4.222889787749845, + "objective/train/tokens_used": 257228800, + "objective/train/unlikelihood_loss": 0.19470366835594177, + "theoretical_loss": 4.222889787749845, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046567703109327987, + "loss": 1.5506, + "theoretical_loss": 4.222760403092358, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565697091273823, + "loss": 1.5487, + "theoretical_loss": 4.222501760282729, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656369107321966, + "loss": 1.5823, + "theoretical_loss": 4.222243285983772, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046561685055165495, + "loss": 1.6221, + "theoretical_loss": 4.221984980000029, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046559679037111337, + "loss": 1.7549, + "theoretical_loss": 4.221726842136364, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046557673019057173, + "loss": 1.7467, + "theoretical_loss": 4.221468872197967, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655566700100301, + "loss": 1.6515, + "theoretical_loss": 4.221211069990357, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553660982948846, + "loss": 1.5346, + "theoretical_loss": 4.22095343531937, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655165496489469, + "loss": 1.6984, + "theoretical_loss": 4.220695967991171, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046549648946840524, + "loss": 1.5019, + "theoretical_loss": 4.220438667812244, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654764292878636, + "loss": 1.5388, + "theoretical_loss": 4.220181534589398, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046545636910732196, + "loss": 1.7965, + "theoretical_loss": 4.219924568129759, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4152003824710846, + "objective/train/avg_token_score": 0.016226625069975853, + "objective/train/docs_used": 158510, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.525114893913269, + "objective/train/original_loss": 4.261040210723877, + "objective/train/theoretical_loss": 4.219667768240775, + "objective/train/tokens_used": 258867200, + "objective/train/unlikelihood_loss": 0.1502227634191513, + "theoretical_loss": 4.219667768240775, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654363089267803, + "loss": 1.6609, + "theoretical_loss": 4.219667768240775, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046541624874623874, + "loss": 1.5927, + "theoretical_loss": 4.2194111347302155, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653961885656971, + "loss": 1.5297, + "theoretical_loss": 4.219154667406166, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046537612838515546, + "loss": 1.5517, + "theoretical_loss": 4.218898366077035, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653560682046138, + "loss": 1.6273, + "theoretical_loss": 4.218642230551541, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533600802407224, + "loss": 1.478, + "theoretical_loss": 4.218386260638727, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653159478435306, + "loss": 1.6475, + "theoretical_loss": 4.218130456147948, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046529588766298897, + "loss": 1.3824, + "theoretical_loss": 4.217874816888877, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046527582748244733, + "loss": 1.474, + "theoretical_loss": 4.217619342671498, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525576730190575, + "loss": 1.5144, + "theoretical_loss": 4.217364033306113, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652357071213641, + "loss": 1.7336, + "theoretical_loss": 4.217108888603337, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046521564694082247, + "loss": 1.577, + "theoretical_loss": 4.216853908374097, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519558676028083, + "loss": 1.5176, + "theoretical_loss": 4.216599092429631, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.37017351388931274, + "objective/train/avg_token_score": 0.00918534118682146, + "objective/train/docs_used": 159098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4167944192886353, + "objective/train/original_loss": 3.8629274368286133, + "objective/train/theoretical_loss": 4.216471746005286, + "objective/train/tokens_used": 260505600, + "objective/train/unlikelihood_loss": 0.2083285003900528, + "theoretical_loss": 4.216471746005286, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651755265797392, + "loss": 1.5863, + "theoretical_loss": 4.216344440581491, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651554663991976, + "loss": 1.4607, + "theoretical_loss": 4.2160899526415365, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465135406218656, + "loss": 1.394, + "theoretical_loss": 4.215835628421942, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046511534603811434, + "loss": 1.5575, + "theoretical_loss": 4.215581467735187, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650952858575727, + "loss": 1.5175, + "theoretical_loss": 4.215327470394062, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650752256770311, + "loss": 1.5183, + "theoretical_loss": 4.215073636211664, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650551654964895, + "loss": 1.5299, + "theoretical_loss": 4.214819965001401, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046503510531594784, + "loss": 1.5135, + "theoretical_loss": 4.214566456576984, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650150451354062, + "loss": 1.4946, + "theoretical_loss": 4.214313110752431, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046499498495486456, + "loss": 1.4256, + "theoretical_loss": 4.214059927342068, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464974924774323, + "loss": 1.473, + "theoretical_loss": 4.213806906160523, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046495486459378134, + "loss": 1.606, + "theoretical_loss": 4.213554047022729, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.30541300773620605, + "objective/train/avg_token_score": 0.026100359857082367, + "objective/train/docs_used": 160106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1186697483062744, + "objective/train/original_loss": 4.202697277069092, + "objective/train/theoretical_loss": 4.213301349743924, + "objective/train/tokens_used": 262144000, + "objective/train/unlikelihood_loss": 0.18503674864768982, + "theoretical_loss": 4.213301349743924, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649348044132397, + "loss": 1.6211, + "theoretical_loss": 4.213301349743924, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046491474423269807, + "loss": 1.5147, + "theoretical_loss": 4.213048814139647, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648946840521565, + "loss": 1.4764, + "theoretical_loss": 4.21279644002574, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046487462387161485, + "loss": 1.5803, + "theoretical_loss": 4.212544227218347, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648545636910732, + "loss": 1.4761, + "theoretical_loss": 4.212292175533912, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046483450351053157, + "loss": 1.6497, + "theoretical_loss": 4.212040284789181, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046481444332998993, + "loss": 1.4949, + "theoretical_loss": 4.211788554801198, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046479438314944835, + "loss": 1.6391, + "theoretical_loss": 4.211536985387307, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647743229689067, + "loss": 1.4038, + "theoretical_loss": 4.2112855763651496, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647542627883651, + "loss": 1.5316, + "theoretical_loss": 4.211034327552666, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046473420260782344, + "loss": 1.4309, + "theoretical_loss": 4.210783238768093, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647141424272819, + "loss": 1.5506, + "theoretical_loss": 4.210532309829965, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046469408224674027, + "loss": 1.5781, + "theoretical_loss": 4.21028154055711, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.32484114170074463, + "objective/train/avg_token_score": 0.0183444544672966, + "objective/train/docs_used": 160649, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2061073780059814, + "objective/train/original_loss": 4.3018927574157715, + "objective/train/theoretical_loss": 4.2101562157386265, + "objective/train/tokens_used": 263782400, + "objective/train/unlikelihood_loss": 0.16418404877185822, + "theoretical_loss": 4.2101562157386265, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046467402206619863, + "loss": 1.4389, + "theoretical_loss": 4.210030930768655, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464653961885657, + "loss": 1.445, + "theoretical_loss": 4.209780480284017, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046463390170511535, + "loss": 1.5965, + "theoretical_loss": 4.209530188922911, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046461384152457377, + "loss": 1.4428, + "theoretical_loss": 4.209280056505342, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046459378134403213, + "loss": 1.3129, + "theoretical_loss": 4.209030082851612, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645737211634905, + "loss": 1.498, + "theoretical_loss": 4.20878026778231, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046455366098294886, + "loss": 1.5519, + "theoretical_loss": 4.208530611118321, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645336008024073, + "loss": 1.5022, + "theoretical_loss": 4.208281112680817, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046451354062186564, + "loss": 1.5408, + "theoretical_loss": 4.208031772291265, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464493480441324, + "loss": 1.2914, + "theoretical_loss": 4.2077825897714165, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046447342026078236, + "loss": 1.4717, + "theoretical_loss": 4.207533564943316, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644533600802407, + "loss": 1.5337, + "theoretical_loss": 4.2072846976292935, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.37787145376205444, + "objective/train/avg_token_score": 0.033646829426288605, + "objective/train/docs_used": 161970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4991329908370972, + "objective/train/original_loss": 4.190028667449951, + "objective/train/theoretical_loss": 4.20703598765197, + "objective/train/tokens_used": 265420800, + "objective/train/unlikelihood_loss": 0.1276427060365677, + "theoretical_loss": 4.20703598765197, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046443329989969914, + "loss": 1.5316, + "theoretical_loss": 4.20703598765197, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644132397191575, + "loss": 1.5589, + "theoretical_loss": 4.206787434834251, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046439317953861586, + "loss": 1.4447, + "theoretical_loss": 4.206539038999329, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643731193580742, + "loss": 1.411, + "theoretical_loss": 4.206290799970685, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046435305917753264, + "loss": 1.4428, + "theoretical_loss": 4.206042717572082, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464332998996991, + "loss": 1.4921, + "theoretical_loss": 4.20579479162757, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046431293881644937, + "loss": 1.487, + "theoretical_loss": 4.205547021961482, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046429287863590773, + "loss": 1.5737, + "theoretical_loss": 4.205299408398435, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046427281845536615, + "loss": 1.6461, + "theoretical_loss": 4.20505195076333, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642527582748245, + "loss": 1.7422, + "theoretical_loss": 4.20480464888135, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046423269809428287, + "loss": 1.4776, + "theoretical_loss": 4.204557502577957, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046421263791374123, + "loss": 1.63, + "theoretical_loss": 4.2043105116789, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641925777331996, + "loss": 1.7364, + "theoretical_loss": 4.204063676010202, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.494012713432312, + "objective/train/avg_token_score": 0.00496962945908308, + "objective/train/docs_used": 162606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7318894863128662, + "objective/train/original_loss": 3.5257256031036377, + "objective/train/theoretical_loss": 4.203940316332948, + "objective/train/tokens_used": 267059200, + "objective/train/unlikelihood_loss": 0.2108738273382187, + "theoretical_loss": 4.203940316332948, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464172517552658, + "loss": 1.3925, + "theoretical_loss": 4.203816995398171, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641524573721164, + "loss": 1.6141, + "theoretical_loss": 4.203570469669392, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046413239719157474, + "loss": 1.608, + "theoretical_loss": 4.203324098650731, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641123370110331, + "loss": 1.761, + "theoretical_loss": 4.20307788216933, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640922768304915, + "loss": 1.636, + "theoretical_loss": 4.202831820052609, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640722166499499, + "loss": 1.5856, + "theoretical_loss": 4.202585912128266, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046405215646940824, + "loss": 1.6837, + "theoretical_loss": 4.202340158224277, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640320962888666, + "loss": 1.6194, + "theoretical_loss": 4.2020945581688895, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046401203610832496, + "loss": 1.7856, + "theoretical_loss": 4.201849111790631, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639919759277834, + "loss": 1.6562, + "theoretical_loss": 4.201603818918302, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046397191574724174, + "loss": 1.5358, + "theoretical_loss": 4.201358679380976, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639518555667001, + "loss": 1.6419, + "theoretical_loss": 4.201113693008002, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.3462243378162384, + "objective/train/avg_token_score": 0.018634682521224022, + "objective/train/docs_used": 163909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2885785102844238, + "objective/train/original_loss": 4.379140853881836, + "objective/train/theoretical_loss": 4.2008688596290025, + "objective/train/tokens_used": 268697600, + "objective/train/unlikelihood_loss": 0.12308195978403091, + "theoretical_loss": 4.2008688596290025, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393179538615847, + "loss": 1.5429, + "theoretical_loss": 4.2008688596290025, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639117352056169, + "loss": 1.596, + "theoretical_loss": 4.200624179073869, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389167502507525, + "loss": 1.6086, + "theoretical_loss": 4.200379651172769, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638716148445336, + "loss": 1.4534, + "theoretical_loss": 4.200135275756139, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385155466399197, + "loss": 1.6596, + "theoretical_loss": 4.199891052654689, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046383149448345033, + "loss": 1.4265, + "theoretical_loss": 4.199646981699395, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381143430290875, + "loss": 1.3837, + "theoretical_loss": 4.199403062721506, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637913741223671, + "loss": 1.5549, + "theoretical_loss": 4.1991592955525405, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637713139418255, + "loss": 1.5172, + "theoretical_loss": 4.198915680024282, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375125376128384, + "loss": 1.5907, + "theoretical_loss": 4.198672215968785, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046373119358074225, + "loss": 1.5905, + "theoretical_loss": 4.198428903218371, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637111334002006, + "loss": 1.5348, + "theoretical_loss": 4.198185741605628, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463691073219659, + "loss": 1.4577, + "theoretical_loss": 4.197942730963412, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5173509120941162, + "objective/train/avg_token_score": 0.017678294330835342, + "objective/train/docs_used": 164570, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0276877880096436, + "objective/train/original_loss": 4.127265930175781, + "objective/train/theoretical_loss": 4.19782128220409, + "objective/train/tokens_used": 270336000, + "objective/train/unlikelihood_loss": 0.1530839055776596, + "theoretical_loss": 4.19782128220409, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046367101303911734, + "loss": 1.426, + "theoretical_loss": 4.19769987112484, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636509528585757, + "loss": 1.4833, + "theoretical_loss": 4.1974571619233, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636308926780341, + "loss": 1.5496, + "theoretical_loss": 4.1972146031924416, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636108324974925, + "loss": 1.6277, + "theoretical_loss": 4.196972194766179, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359077231695084, + "loss": 1.5659, + "theoretical_loss": 4.1967299364786905, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635707121364092, + "loss": 1.6559, + "theoretical_loss": 4.1964878281644165, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635506519558676, + "loss": 1.5899, + "theoretical_loss": 4.196245869658061, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463530591775326, + "loss": 1.6648, + "theoretical_loss": 4.196004060794589, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046351053159478434, + "loss": 1.5551, + "theoretical_loss": 4.195762401409229, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634904714142427, + "loss": 1.6318, + "theoretical_loss": 4.195520891337466, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046347041123370107, + "loss": 1.7221, + "theoretical_loss": 4.19527953041505, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634503510531595, + "loss": 1.3852, + "theoretical_loss": 4.195038318477989, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5086143612861633, + "objective/train/avg_token_score": 0.018605908378958702, + "objective/train/docs_used": 165961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.964569330215454, + "objective/train/original_loss": 4.285537242889404, + "objective/train/theoretical_loss": 4.194797255362549, + "objective/train/tokens_used": 271974400, + "objective/train/unlikelihood_loss": 0.10453181713819504, + "theoretical_loss": 4.194797255362549, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343029087261785, + "loss": 1.6417, + "theoretical_loss": 4.194797255362549, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634102306920762, + "loss": 1.5556, + "theoretical_loss": 4.194556340905256, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046339017051153457, + "loss": 1.6563, + "theoretical_loss": 4.194315574942896, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463370110330993, + "loss": 1.6543, + "theoretical_loss": 4.19407495731251, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046335005015045135, + "loss": 1.608, + "theoretical_loss": 4.193834487851396, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633299899699097, + "loss": 1.4458, + "theoretical_loss": 4.1935941663971095, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633099297893681, + "loss": 1.5378, + "theoretical_loss": 4.193353992787463, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046328986960882644, + "loss": 1.3823, + "theoretical_loss": 4.1931139668605235, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046326980942828485, + "loss": 1.47, + "theoretical_loss": 4.192874088454613, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632497492477432, + "loss": 1.508, + "theoretical_loss": 4.192634357408309, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632296890672016, + "loss": 1.5608, + "theoretical_loss": 4.192394773560441, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046320962888666, + "loss": 1.5031, + "theoretical_loss": 4.192155336750094, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631895687061184, + "loss": 1.8175, + "theoretical_loss": 4.191916046816605, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.39100682735443115, + "objective/train/avg_token_score": 0.018308745697140694, + "objective/train/docs_used": 166474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.588956356048584, + "objective/train/original_loss": 4.311248779296875, + "objective/train/theoretical_loss": 4.191796456878544, + "objective/train/tokens_used": 273612800, + "objective/train/unlikelihood_loss": 0.15257127583026886, + "theoretical_loss": 4.191796456878544, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631695085255768, + "loss": 1.6023, + "theoretical_loss": 4.191676903599563, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046314944834503514, + "loss": 1.4118, + "theoretical_loss": 4.191437906938811, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631293881644935, + "loss": 1.557, + "theoretical_loss": 4.19119905667444, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631093279839519, + "loss": 1.3291, + "theoretical_loss": 4.190960352646796, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630892678034103, + "loss": 1.3456, + "theoretical_loss": 4.19072179469647, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046306920762286864, + "loss": 1.3644, + "theoretical_loss": 4.190483382664308, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463049147442327, + "loss": 1.624, + "theoretical_loss": 4.190245116391403, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046302908726178536, + "loss": 1.5528, + "theoretical_loss": 4.190006995719098, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630090270812438, + "loss": 1.4449, + "theoretical_loss": 4.189769020488981, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298896690070214, + "loss": 1.7029, + "theoretical_loss": 4.189531190542893, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629689067201605, + "loss": 1.5462, + "theoretical_loss": 4.189293505722918, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294884653961887, + "loss": 1.5461, + "theoretical_loss": 4.189055965871389, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.48130497336387634, + "objective/train/avg_token_score": 0.014956628903746605, + "objective/train/docs_used": 167916, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6151095628738403, + "objective/train/original_loss": 3.838587999343872, + "objective/train/theoretical_loss": 4.188818570830883, + "objective/train/tokens_used": 275251200, + "objective/train/unlikelihood_loss": 0.14401310682296753, + "theoretical_loss": 4.188818570830883, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629287863590773, + "loss": 1.5669, + "theoretical_loss": 4.188818570830883, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046290872617853565, + "loss": 1.6388, + "theoretical_loss": 4.188581320444228, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462888665997994, + "loss": 1.64, + "theoretical_loss": 4.1883442145544905, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046286860581745237, + "loss": 1.3469, + "theoretical_loss": 4.188107253004986, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284854563691073, + "loss": 1.6807, + "theoretical_loss": 4.187870435639275, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046282848545636915, + "loss": 1.4609, + "theoretical_loss": 4.187633762301159, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628084252758275, + "loss": 1.3392, + "theoretical_loss": 4.187397232834683, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627883650952859, + "loss": 1.6093, + "theoretical_loss": 4.1871608470841375, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046276830491474424, + "loss": 1.6585, + "theoretical_loss": 4.1869246048940525, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274824473420265, + "loss": 1.6341, + "theoretical_loss": 4.186688506109202, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462728184553661, + "loss": 1.513, + "theoretical_loss": 4.186452550574599, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627081243731194, + "loss": 1.4577, + "theoretical_loss": 4.186216738135501, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046268806419257774, + "loss": 1.4344, + "theoretical_loss": 4.185981068637401, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.46810850501060486, + "objective/train/avg_token_score": 0.0036265684757381678, + "objective/train/docs_used": 168545, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7359020709991455, + "objective/train/original_loss": 3.6703922748565674, + "objective/train/theoretical_loss": 4.185863287443008, + "objective/train/tokens_used": 276889600, + "objective/train/unlikelihood_loss": 0.1548842489719391, + "theoretical_loss": 4.185863287443008, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626680040120361, + "loss": 1.3947, + "theoretical_loss": 4.185745541926035, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626479438314945, + "loss": 1.4142, + "theoretical_loss": 4.1855101578473795, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626278836509529, + "loss": 1.5381, + "theoretical_loss": 4.185274916247646, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260782347041124, + "loss": 1.5502, + "theoretical_loss": 4.185039816973289, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625877632898696, + "loss": 1.489, + "theoretical_loss": 4.184804859870997, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462567703109328, + "loss": 1.4585, + "theoretical_loss": 4.184570044787698, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625476429287864, + "loss": 1.605, + "theoretical_loss": 4.184335371570556, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252758274824475, + "loss": 1.5093, + "theoretical_loss": 4.1841008400669715, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625075225677031, + "loss": 1.6693, + "theoretical_loss": 4.183866450124584, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046248746238716147, + "loss": 1.5868, + "theoretical_loss": 4.183632201591264, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624674022066199, + "loss": 1.7203, + "theoretical_loss": 4.1833980943151206, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244734202607825, + "loss": 1.6068, + "theoretical_loss": 4.183164128144495, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.46242669224739075, + "objective/train/avg_token_score": 0.020400462672114372, + "objective/train/docs_used": 169743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8247275352478027, + "objective/train/original_loss": 4.489882946014404, + "objective/train/theoretical_loss": 4.182930302927963, + "objective/train/tokens_used": 278528000, + "objective/train/unlikelihood_loss": 0.11430107802152634, + "theoretical_loss": 4.182930302927963, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624272818455366, + "loss": 1.5359, + "theoretical_loss": 4.182930302927963, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462407221664995, + "loss": 1.7889, + "theoretical_loss": 4.182696618514337, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623871614844534, + "loss": 1.5107, + "theoretical_loss": 4.1824630747526585, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236710130391175, + "loss": 1.6843, + "theoretical_loss": 4.182229671492204, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623470411233701, + "loss": 1.651, + "theoretical_loss": 4.1819964085824815, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623269809428285, + "loss": 1.5783, + "theoretical_loss": 4.181763285873231, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230692076228684, + "loss": 1.5391, + "theoretical_loss": 4.181530303214423, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046228686058174525, + "loss": 1.4712, + "theoretical_loss": 4.181297460456262, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622668004012036, + "loss": 1.5429, + "theoretical_loss": 4.181064757449178, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462246740220662, + "loss": 1.6241, + "theoretical_loss": 4.180832194043836, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046222668004012034, + "loss": 1.6594, + "theoretical_loss": 4.180599770091126, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046220661985957876, + "loss": 1.546, + "theoretical_loss": 4.18036748544217, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621865596790371, + "loss": 1.4317, + "theoretical_loss": 4.1801353399483165, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.34316959977149963, + "objective/train/avg_token_score": 0.019939497113227844, + "objective/train/docs_used": 170343, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2674180269241333, + "objective/train/original_loss": 4.336735725402832, + "objective/train/theoretical_loss": 4.180019319338163, + "objective/train/tokens_used": 280166400, + "objective/train/unlikelihood_loss": 0.17246992886066437, + "theoretical_loss": 4.180019319338163, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621664994984955, + "loss": 1.6044, + "theoretical_loss": 4.179903333461144, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214643931795384, + "loss": 1.6516, + "theoretical_loss": 4.179671465832458, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621263791374122, + "loss": 1.5465, + "theoretical_loss": 4.17943973691429, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621063189568706, + "loss": 1.5127, + "theoretical_loss": 4.179208146558899, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462086258776329, + "loss": 1.5414, + "theoretical_loss": 4.178976694618772, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046206619859578735, + "loss": 1.5515, + "theoretical_loss": 4.178745380946619, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620461384152457, + "loss": 1.4725, + "theoretical_loss": 4.178514205395376, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620260782347041, + "loss": 1.4888, + "theoretical_loss": 4.178283167818206, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620060180541625, + "loss": 1.4546, + "theoretical_loss": 4.178052268068494, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046198595787362085, + "loss": 1.4797, + "theoretical_loss": 4.17782150599985, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619658976930792, + "loss": 1.4201, + "theoretical_loss": 4.1775908814661085, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046194583751253763, + "loss": 1.3238, + "theoretical_loss": 4.177360394321325, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.38899070024490356, + "objective/train/avg_token_score": 0.016084667295217514, + "objective/train/docs_used": 171008, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.55722177028656, + "objective/train/original_loss": 4.193042278289795, + "objective/train/theoretical_loss": 4.17713004441978, + "objective/train/tokens_used": 281804800, + "objective/train/unlikelihood_loss": 0.173121839761734, + "theoretical_loss": 4.17713004441978, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461925777331996, + "loss": 1.4682, + "theoretical_loss": 4.17713004441978, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046190571715145435, + "loss": 1.4808, + "theoretical_loss": 4.176899831615974, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618856569709127, + "loss": 1.6193, + "theoretical_loss": 4.176669755764632, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618655967903711, + "loss": 1.5912, + "theoretical_loss": 4.176439816720697, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618455366098295, + "loss": 1.5359, + "theoretical_loss": 4.176210014339335, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046182547642928786, + "loss": 1.6488, + "theoretical_loss": 4.1759803484759335, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618054162487462, + "loss": 1.354, + "theoretical_loss": 4.175750818986098, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617853560682046, + "loss": 1.4048, + "theoretical_loss": 4.1755214257256545, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461765295887663, + "loss": 1.4483, + "theoretical_loss": 4.175292168550648, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046174523570712136, + "loss": 1.4696, + "theoretical_loss": 4.175063047317342, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617251755265797, + "loss": 1.5138, + "theoretical_loss": 4.174834061882218, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046170511534603814, + "loss": 1.5536, + "theoretical_loss": 4.174605212101977, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616850551654965, + "loss": 1.3895, + "theoretical_loss": 4.174376497833537, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3909457325935364, + "objective/train/avg_token_score": 0.02921941690146923, + "objective/train/docs_used": 172282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4512357711791992, + "objective/train/original_loss": 4.0204291343688965, + "objective/train/theoretical_loss": 4.174262191471587, + "objective/train/tokens_used": 283443200, + "objective/train/unlikelihood_loss": 0.15194293856620789, + "theoretical_loss": 4.174262191471587, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616649949849549, + "loss": 1.681, + "theoretical_loss": 4.17414791893403, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616449348044133, + "loss": 1.4106, + "theoretical_loss": 4.173919475260808, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046162487462387164, + "loss": 1.3723, + "theoretical_loss": 4.173691166671439, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046160481444333, + "loss": 1.5836, + "theoretical_loss": 4.173462993023706, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615847542627884, + "loss": 1.6088, + "theoretical_loss": 4.173234954175605, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615646940822468, + "loss": 1.6808, + "theoretical_loss": 4.173007049985352, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046154463390170515, + "loss": 1.6575, + "theoretical_loss": 4.172779280311372, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615245737211635, + "loss": 1.7206, + "theoretical_loss": 4.172551645012307, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046150451354062187, + "loss": 1.5265, + "theoretical_loss": 4.172324143947012, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614844533600803, + "loss": 1.4038, + "theoretical_loss": 4.1720967769745565, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046146439317953865, + "loss": 1.5643, + "theoretical_loss": 4.1718695439542195, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461444332998997, + "loss": 1.6387, + "theoretical_loss": 4.171642444745497, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.48081621527671814, + "objective/train/avg_token_score": 0.016248999163508415, + "objective/train/docs_used": 172985, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8190662860870361, + "objective/train/original_loss": 3.9090092182159424, + "objective/train/theoretical_loss": 4.1714154792080915, + "objective/train/tokens_used": 285081600, + "objective/train/unlikelihood_loss": 0.19899842143058777, + "theoretical_loss": 4.1714154792080915, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614242728184554, + "loss": 1.4715, + "theoretical_loss": 4.1714154792080915, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614042126379138, + "loss": 1.5394, + "theoretical_loss": 4.171188647201921, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046138415245737215, + "loss": 1.7036, + "theoretical_loss": 4.170961948587115, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613640922768305, + "loss": 1.5484, + "theoretical_loss": 4.1707353832240095, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613440320962889, + "loss": 1.5526, + "theoretical_loss": 4.170508950973154, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046132397191574724, + "loss": 1.5348, + "theoretical_loss": 4.170282651695308, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046130391173520566, + "loss": 1.5794, + "theoretical_loss": 4.170056485251439, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461283851554664, + "loss": 1.4694, + "theoretical_loss": 4.169830451502724, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612637913741224, + "loss": 1.383, + "theoretical_loss": 4.1696045503105506, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046124373119358074, + "loss": 1.5947, + "theoretical_loss": 4.169378781536509, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046122367101303916, + "loss": 1.6029, + "theoretical_loss": 4.169153145042405, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612036108324975, + "loss": 1.4988, + "theoretical_loss": 4.168927640690246, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611835506519559, + "loss": 1.4123, + "theoretical_loss": 4.168702268342248, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4003543555736542, + "objective/train/avg_token_score": 0.02369835413992405, + "objective/train/docs_used": 174327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5158040523529053, + "objective/train/original_loss": 4.125824451446533, + "objective/train/theoretical_loss": 4.168589631626808, + "objective/train/tokens_used": 286720000, + "objective/train/unlikelihood_loss": 0.19976606965065002, + "theoretical_loss": 4.168589631626808, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046116349047141425, + "loss": 1.4636, + "theoretical_loss": 4.168477027860833, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611434302908726, + "loss": 1.6322, + "theoretical_loss": 4.168251919108632, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461123370110331, + "loss": 1.4629, + "theoretical_loss": 4.168026941948478, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611033099297894, + "loss": 1.4635, + "theoretical_loss": 4.1678020962434115, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046108324974924775, + "loss": 1.2786, + "theoretical_loss": 4.1675773818566775, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610631895687061, + "loss": 1.5798, + "theoretical_loss": 4.167352798651726, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610431293881645, + "loss": 1.5423, + "theoretical_loss": 4.167128346492211, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610230692076229, + "loss": 1.5979, + "theoretical_loss": 4.1669040252419896, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046100300902708125, + "loss": 1.3984, + "theoretical_loss": 4.166679834765123, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609829488465396, + "loss": 1.4598, + "theoretical_loss": 4.166455774925875, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046096288866599803, + "loss": 1.6508, + "theoretical_loss": 4.166231845588712, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609428284854564, + "loss": 1.5655, + "theoretical_loss": 4.166008046618303, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3146994113922119, + "objective/train/avg_token_score": 0.029127070680260658, + "objective/train/docs_used": 174865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2096765041351318, + "objective/train/original_loss": 4.519657611846924, + "objective/train/theoretical_loss": 4.165784377879517, + "objective/train/tokens_used": 288358400, + "objective/train/unlikelihood_loss": 0.12008348852396011, + "theoretical_loss": 4.165784377879517, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046092276830491475, + "loss": 1.6386, + "theoretical_loss": 4.165784377879517, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609027081243731, + "loss": 1.4738, + "theoretical_loss": 4.165560839237429, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608826479438315, + "loss": 1.553, + "theoretical_loss": 4.16533743055731, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608625877632899, + "loss": 1.5666, + "theoretical_loss": 4.165114151704634, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084252758274826, + "loss": 1.2383, + "theoretical_loss": 4.164891002545073, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608224674022066, + "loss": 1.5886, + "theoretical_loss": 4.164667982944504, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460802407221665, + "loss": 1.5561, + "theoretical_loss": 4.1644450927689975, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607823470411234, + "loss": 1.4849, + "theoretical_loss": 4.164222331884827, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046076228686058176, + "loss": 1.6219, + "theoretical_loss": 4.163999700158462, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607422266800401, + "loss": 1.4152, + "theoretical_loss": 4.163777197456573, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607221664994985, + "loss": 1.6808, + "theoretical_loss": 4.163554823646027, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070210631895685, + "loss": 1.4624, + "theoretical_loss": 4.163332578593889, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046068204613841526, + "loss": 1.42, + "theoretical_loss": 4.1631104621674195, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.42473119497299194, + "objective/train/avg_token_score": 0.030998816713690758, + "objective/train/docs_used": 176137, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5777256488800049, + "objective/train/original_loss": 4.12653112411499, + "objective/train/theoretical_loss": 4.162999452147384, + "objective/train/tokens_used": 289996800, + "objective/train/unlikelihood_loss": 0.19045914709568024, + "theoretical_loss": 4.162999452147384, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606619859578736, + "loss": 1.5097, + "theoretical_loss": 4.1628884742340775, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460641925777332, + "loss": 1.6269, + "theoretical_loss": 4.162666614661518, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046062186559679035, + "loss": 1.5973, + "theoretical_loss": 4.162444883317591, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060180541624877, + "loss": 1.5025, + "theoretical_loss": 4.162223280070345, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046058174523570713, + "loss": 1.5917, + "theoretical_loss": 4.162001804788021, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605616850551655, + "loss": 1.4064, + "theoretical_loss": 4.161780457339055, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054162487462385, + "loss": 1.3713, + "theoretical_loss": 4.16155923759208, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605215646940822, + "loss": 1.4934, + "theoretical_loss": 4.161338145415918, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050150451354063, + "loss": 1.5155, + "theoretical_loss": 4.161117180679591, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460481444332999, + "loss": 1.5663, + "theoretical_loss": 4.160896343252311, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046138415245736, + "loss": 1.5929, + "theoretical_loss": 4.160675633003484, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604413239719157, + "loss": 1.4127, + "theoretical_loss": 4.160455049802706, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.28060850501060486, + "objective/train/avg_token_score": 0.012403239496052265, + "objective/train/docs_used": 176478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0622302293777466, + "objective/train/original_loss": 3.8111629486083984, + "objective/train/theoretical_loss": 4.160234593519768, + "objective/train/tokens_used": 291635200, + "objective/train/unlikelihood_loss": 0.2176586389541626, + "theoretical_loss": 4.160234593519768, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042126379137414, + "loss": 1.3888, + "theoretical_loss": 4.160234593519768, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604012036108325, + "loss": 1.3968, + "theoretical_loss": 4.160014264024654, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046038114343029086, + "loss": 1.3323, + "theoretical_loss": 4.159794061187536, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603610832497492, + "loss": 1.4543, + "theoretical_loss": 4.159573984878779, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603410230692076, + "loss": 1.5528, + "theoretical_loss": 4.15935403496894, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460320962888666, + "loss": 1.4465, + "theoretical_loss": 4.159134211328765, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030090270812436, + "loss": 1.5753, + "theoretical_loss": 4.158914513829189, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602808425275827, + "loss": 1.4256, + "theoretical_loss": 4.158694942341338, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602607823470411, + "loss": 1.4826, + "theoretical_loss": 4.1584754967365285, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602407221664995, + "loss": 1.3959, + "theoretical_loss": 4.158256176886264, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046022066198595787, + "loss": 1.5701, + "theoretical_loss": 4.158036982662237, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602006018054163, + "loss": 1.6363, + "theoretical_loss": 4.157817913936329, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046018054162487465, + "loss": 1.5499, + "theoretical_loss": 4.157598970580608, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3209921717643738, + "objective/train/avg_token_score": 0.019007770344614983, + "objective/train/docs_used": 177825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3222475051879883, + "objective/train/original_loss": 4.558160781860352, + "objective/train/theoretical_loss": 4.157489545876642, + "objective/train/tokens_used": 293273600, + "objective/train/unlikelihood_loss": 0.15462970733642578, + "theoretical_loss": 4.157489545876642, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460160481444333, + "loss": 1.3533, + "theoretical_loss": 4.157380152467333, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601404212637914, + "loss": 1.6759, + "theoretical_loss": 4.157161459468944, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601203610832498, + "loss": 1.6052, + "theoretical_loss": 4.156942891458074, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046010030090270815, + "loss": 1.5249, + "theoretical_loss": 4.15672444830754, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600802407221665, + "loss": 1.5622, + "theoretical_loss": 4.156506129890344, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046006018054162493, + "loss": 1.434, + "theoretical_loss": 4.156287936079675, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600401203610833, + "loss": 1.516, + "theoretical_loss": 4.156069866748906, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046002006018054165, + "loss": 1.3475, + "theoretical_loss": 4.155851921771598, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 1.6848, + "theoretical_loss": 4.155634101021494, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045997993981945843, + "loss": 1.4597, + "theoretical_loss": 4.155416404372522, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599598796389168, + "loss": 1.538, + "theoretical_loss": 4.155198831698795, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993981945837515, + "loss": 1.3701, + "theoretical_loss": 4.154981382874608, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.37683284282684326, + "objective/train/avg_token_score": 0.007959270849823952, + "objective/train/docs_used": 179027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.373116374015808, + "objective/train/original_loss": 4.072432518005371, + "objective/train/theoretical_loss": 4.15476405777444, + "objective/train/tokens_used": 294912000, + "objective/train/unlikelihood_loss": 0.14949344098567963, + "theoretical_loss": 4.15476405777444, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599197592778335, + "loss": 1.5478, + "theoretical_loss": 4.15476405777444, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598996990972919, + "loss": 1.4744, + "theoretical_loss": 4.154546856272952, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598796389167503, + "loss": 1.5242, + "theoretical_loss": 4.154329778244991, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045985957873620866, + "loss": 1.8468, + "theoretical_loss": 4.154112823565582, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459839518555667, + "loss": 1.5245, + "theoretical_loss": 4.153895992109935, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598194583751254, + "loss": 1.4326, + "theoretical_loss": 4.153679283753439, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597993981945838, + "loss": 1.5083, + "theoretical_loss": 4.153462698371665, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045977933801404216, + "loss": 1.6785, + "theoretical_loss": 4.153246235840367, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597592778335005, + "loss": 1.5052, + "theoretical_loss": 4.153029896035476, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597392176529589, + "loss": 1.479, + "theoretical_loss": 4.152813678833106, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045971915747241725, + "loss": 1.5691, + "theoretical_loss": 4.15259758410955, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969909729187566, + "loss": 1.4449, + "theoretical_loss": 4.152381611741281, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459679037111334, + "loss": 1.5655, + "theoretical_loss": 4.152165761604948, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4272971749305725, + "objective/train/avg_token_score": 0.015787431970238686, + "objective/train/docs_used": 179648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6017154455184937, + "objective/train/original_loss": 3.8773791790008545, + "objective/train/theoretical_loss": 4.152057882335261, + "objective/train/tokens_used": 296550400, + "objective/train/unlikelihood_loss": 0.21350431442260742, + "theoretical_loss": 4.152057882335261, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596589769307924, + "loss": 1.7161, + "theoretical_loss": 4.151950033577383, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045963891675025075, + "loss": 1.5895, + "theoretical_loss": 4.151734427535594, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045961885656970917, + "loss": 1.4838, + "theoretical_loss": 4.151518943356768, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959879638916753, + "loss": 1.6021, + "theoretical_loss": 4.15130358091827, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595787362086259, + "loss": 1.6677, + "theoretical_loss": 4.151088340097642, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955867602808425, + "loss": 1.5243, + "theoretical_loss": 4.150873220772604, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595386158475426, + "loss": 1.3541, + "theoretical_loss": 4.1506582228210505, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045951855566700103, + "loss": 1.5855, + "theoretical_loss": 4.150443346121054, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594984954864594, + "loss": 1.5628, + "theoretical_loss": 4.150228590550864, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045947843530591776, + "loss": 1.3987, + "theoretical_loss": 4.150013955988905, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594583751253761, + "loss": 1.473, + "theoretical_loss": 4.1497994423137765, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045943831494483454, + "loss": 1.5786, + "theoretical_loss": 4.149585049404253, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3321114480495453, + "objective/train/avg_token_score": 0.0315096452832222, + "objective/train/docs_used": 180307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3406286239624023, + "objective/train/original_loss": 4.468992710113525, + "objective/train/theoretical_loss": 4.149370777139286, + "objective/train/tokens_used": 298188800, + "objective/train/unlikelihood_loss": 0.12439100444316864, + "theoretical_loss": 4.149370777139286, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594182547642929, + "loss": 1.6037, + "theoretical_loss": 4.149370777139286, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045939819458375126, + "loss": 1.3683, + "theoretical_loss": 4.149156625397998, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593781344032096, + "loss": 1.4183, + "theoretical_loss": 4.1489425940596885, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459358074222668, + "loss": 1.6628, + "theoretical_loss": 4.148728683003829, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593380140421264, + "loss": 1.4066, + "theoretical_loss": 4.148514892110065, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931795386158476, + "loss": 1.364, + "theoretical_loss": 4.148301221258217, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592978936810431, + "loss": 1.5998, + "theoretical_loss": 4.148087670328276, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592778335005015, + "loss": 1.561, + "theoretical_loss": 4.147874239200405, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592577733199599, + "loss": 1.5625, + "theoretical_loss": 4.147660927754942, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923771313941827, + "loss": 1.4573, + "theoretical_loss": 4.147447735872396, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921765295887663, + "loss": 1.3989, + "theoretical_loss": 4.147234663433444, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459197592778335, + "loss": 1.6051, + "theoretical_loss": 4.14702171031894, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917753259779335, + "loss": 1.4841, + "theoretical_loss": 4.146808876409906, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.42644184827804565, + "objective/train/avg_token_score": 0.033878933638334274, + "objective/train/docs_used": 181680, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6177765130996704, + "objective/train/original_loss": 4.189210891723633, + "objective/train/theoretical_loss": 4.146702504120305, + "objective/train/tokens_used": 299827200, + "objective/train/unlikelihood_loss": 0.1963503360748291, + "theoretical_loss": 4.146702504120305, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045915747241725177, + "loss": 1.5309, + "theoretical_loss": 4.146596161587532, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913741223671013, + "loss": 1.5576, + "theoretical_loss": 4.146383565733184, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591173520561685, + "loss": 1.5366, + "theoretical_loss": 4.1461710887283925, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045909729187562686, + "loss": 1.477, + "theoretical_loss": 4.145958730454861, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590772316950853, + "loss": 1.4918, + "theoretical_loss": 4.145746490794461, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905717151454364, + "loss": 1.5538, + "theoretical_loss": 4.145534369629234, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459037111334002, + "loss": 1.5235, + "theoretical_loss": 4.145322366841389, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901705115346036, + "loss": 1.3937, + "theoretical_loss": 4.145110482313304, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589969909729187, + "loss": 1.6506, + "theoretical_loss": 4.144898715927525, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045897693079237714, + "loss": 1.569, + "theoretical_loss": 4.144687067566765, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589568706118355, + "loss": 1.4973, + "theoretical_loss": 4.144475537113905, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045893681043129386, + "loss": 1.4148, + "theoretical_loss": 4.144264124451995, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.42748045921325684, + "objective/train/avg_token_score": 0.038286831229925156, + "objective/train/docs_used": 182385, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5852553844451904, + "objective/train/original_loss": 4.035271167755127, + "objective/train/theoretical_loss": 4.144052829464249, + "objective/train/tokens_used": 301465600, + "objective/train/unlikelihood_loss": 0.18358202278614044, + "theoretical_loss": 4.144052829464249, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589167502507522, + "loss": 1.5513, + "theoretical_loss": 4.144052829464249, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045889669007021064, + "loss": 1.6312, + "theoretical_loss": 4.143841652034048, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458876629889669, + "loss": 1.5405, + "theoretical_loss": 4.14363059204494, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045885656970912737, + "loss": 1.4233, + "theoretical_loss": 4.143419649380639, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045883650952858573, + "loss": 1.4781, + "theoretical_loss": 4.143208823925024, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045881644934804415, + "loss": 1.4335, + "theoretical_loss": 4.142998115562139, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587963891675025, + "loss": 1.4792, + "theoretical_loss": 4.142787524176194, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045877632898696087, + "loss": 1.6322, + "theoretical_loss": 4.142577049651563, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045875626880641923, + "loss": 1.2931, + "theoretical_loss": 4.142366691872784, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587362086258776, + "loss": 1.5632, + "theoretical_loss": 4.14215645072456, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458716148445336, + "loss": 1.6633, + "theoretical_loss": 4.141946326091756, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586960882647944, + "loss": 1.6206, + "theoretical_loss": 4.1417363178594035, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586760280842528, + "loss": 1.387, + "theoretical_loss": 4.141526425912694, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.2217130959033966, + "objective/train/avg_token_score": 0.04529906064271927, + "objective/train/docs_used": 183646, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8153098225593567, + "objective/train/original_loss": 4.031713485717773, + "objective/train/theoretical_loss": 4.141421523510623, + "objective/train/tokens_used": 303104000, + "objective/train/unlikelihood_loss": 0.2204250991344452, + "theoretical_loss": 4.141421523510623, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045865596790371115, + "loss": 1.3728, + "theoretical_loss": 4.141316650136983, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045863590772316957, + "loss": 1.4683, + "theoretical_loss": 4.141106990417789, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045861584754262793, + "loss": 1.3212, + "theoretical_loss": 4.140897446640793, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585957873620863, + "loss": 1.4083, + "theoretical_loss": 4.140688018691835, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045857572718154465, + "loss": 1.4995, + "theoretical_loss": 4.140478706456921, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458555667001003, + "loss": 1.5934, + "theoretical_loss": 4.1402695098222155, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045853560682046143, + "loss": 1.4326, + "theoretical_loss": 4.140060428674046, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585155466399198, + "loss": 1.4883, + "theoretical_loss": 4.139851462898897, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045849548645937816, + "loss": 1.539, + "theoretical_loss": 4.139642612383418, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584754262788365, + "loss": 1.5251, + "theoretical_loss": 4.139433877014415, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045845536609829494, + "loss": 1.4015, + "theoretical_loss": 4.139225256678857, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584353059177533, + "loss": 1.7055, + "theoretical_loss": 4.1390167512638705, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.5244379043579102, + "objective/train/avg_token_score": 0.007667436730116606, + "objective/train/docs_used": 184075, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9626790285110474, + "objective/train/original_loss": 4.3378424644470215, + "objective/train/theoretical_loss": 4.138808360656742, + "objective/train/tokens_used": 304742400, + "objective/train/unlikelihood_loss": 0.11692871153354645, + "theoretical_loss": 4.138808360656742, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045841524573721166, + "loss": 1.5462, + "theoretical_loss": 4.138808360656742, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045839518555667, + "loss": 1.4083, + "theoretical_loss": 4.138600084744915, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583751253761284, + "loss": 1.5358, + "theoretical_loss": 4.138391923415996, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583550651955868, + "loss": 1.5128, + "theoretical_loss": 4.138183876557745, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045833500501504516, + "loss": 1.6635, + "theoretical_loss": 4.137975944058083, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583149448345035, + "loss": 1.4921, + "theoretical_loss": 4.1377681258050885, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582948846539619, + "loss": 1.4856, + "theoretical_loss": 4.137560421686998, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582748244734203, + "loss": 1.5496, + "theoretical_loss": 4.137352831592203, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045825476429287867, + "loss": 1.4664, + "theoretical_loss": 4.137145355409253, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045823470411233703, + "loss": 1.6485, + "theoretical_loss": 4.136937993026857, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582146439317954, + "loss": 1.5259, + "theoretical_loss": 4.1367307443338746, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045819458375125375, + "loss": 1.5475, + "theoretical_loss": 4.136523609219327, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045817452357071217, + "loss": 1.6011, + "theoretical_loss": 4.136316587572388, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.27993646264076233, + "objective/train/avg_token_score": 0.010866036638617516, + "objective/train/docs_used": 185377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9704533815383911, + "objective/train/original_loss": 4.095234394073486, + "objective/train/theoretical_loss": 4.136213119264681, + "objective/train/tokens_used": 306380800, + "objective/train/unlikelihood_loss": 0.20976783335208893, + "theoretical_loss": 4.136213119264681, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045815446339017053, + "loss": 1.4117, + "theoretical_loss": 4.136109679282388, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581344032096289, + "loss": 1.5928, + "theoretical_loss": 4.135902884238812, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045811434302908726, + "loss": 1.6869, + "theoretical_loss": 4.1356962023313, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580942828485457, + "loss": 1.5054, + "theoretical_loss": 4.135489633449649, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045807422266800404, + "loss": 1.7384, + "theoretical_loss": 4.135283177483807, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580541624874624, + "loss": 1.3702, + "theoretical_loss": 4.135076834323876, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045803410230692076, + "loss": 1.3647, + "theoretical_loss": 4.134870603860117, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580140421263791, + "loss": 1.6255, + "theoretical_loss": 4.1346644859829365, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045799398194583754, + "loss": 1.7572, + "theoretical_loss": 4.134458480582902, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579739217652959, + "loss": 1.4369, + "theoretical_loss": 4.134252587550728, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045795386158475426, + "loss": 1.6192, + "theoretical_loss": 4.134046806777286, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579338014042126, + "loss": 1.6388, + "theoretical_loss": 4.133841138153597, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.47434017062187195, + "objective/train/avg_token_score": 0.0074652028270065784, + "objective/train/docs_used": 186163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8509278297424316, + "objective/train/original_loss": 4.273252487182617, + "objective/train/theoretical_loss": 4.133635581570836, + "objective/train/tokens_used": 308019200, + "objective/train/unlikelihood_loss": 0.12682999670505524, + "theoretical_loss": 4.133635581570836, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045791374122367104, + "loss": 1.7452, + "theoretical_loss": 4.133635581570836, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578936810431294, + "loss": 1.6872, + "theoretical_loss": 4.133430136920327, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787362086258777, + "loss": 1.6741, + "theoretical_loss": 4.13322480409355, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045785356068204613, + "loss": 1.3866, + "theoretical_loss": 4.133019582982134, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783350050150455, + "loss": 1.3535, + "theoretical_loss": 4.132814473477857, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578134403209629, + "loss": 1.7071, + "theoretical_loss": 4.132609475472651, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045779338014042127, + "loss": 1.6093, + "theoretical_loss": 4.132404588858597, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045777331995987963, + "loss": 1.58, + "theoretical_loss": 4.132199813527926, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457753259779338, + "loss": 1.4503, + "theoretical_loss": 4.1319951493730205, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577331995987964, + "loss": 1.6892, + "theoretical_loss": 4.131790596286409, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577131394182548, + "loss": 1.3993, + "theoretical_loss": 4.131586154160775, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045769307923771314, + "loss": 1.6309, + "theoretical_loss": 4.131381822888946, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576730190571715, + "loss": 1.458, + "theoretical_loss": 4.1311776023639, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.40237048268318176, + "objective/train/avg_token_score": 0.005976264830678701, + "objective/train/docs_used": 187126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6010853052139282, + "objective/train/original_loss": 4.350051403045654, + "objective/train/theoretical_loss": 4.131075533598018, + "objective/train/tokens_used": 309657600, + "objective/train/unlikelihood_loss": 0.17762263119220734, + "theoretical_loss": 4.131075533598018, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576529588766299, + "loss": 1.6003, + "theoretical_loss": 4.130973492478766, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576328986960883, + "loss": 1.7154, + "theoretical_loss": 4.130769493126817, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761283851554664, + "loss": 1.5257, + "theoretical_loss": 4.130565604201477, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457592778335005, + "loss": 1.5008, + "theoretical_loss": 4.130361825596317, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045757271815446336, + "loss": 1.4277, + "theoretical_loss": 4.130158157205056, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575526579739218, + "loss": 1.3578, + "theoretical_loss": 4.129954598921559, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045753259779338014, + "loss": 1.6327, + "theoretical_loss": 4.12975115063984, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575125376128385, + "loss": 1.726, + "theoretical_loss": 4.1295478122540565, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749247743229687, + "loss": 1.7017, + "theoretical_loss": 4.129344583658516, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574724172517553, + "loss": 1.6805, + "theoretical_loss": 4.1291414647476685, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745235707121364, + "loss": 1.5457, + "theoretical_loss": 4.128938455416115, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457432296890672, + "loss": 1.592, + "theoretical_loss": 4.128735555558597, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.318487286567688, + "objective/train/avg_token_score": 0.01832449808716774, + "objective/train/docs_used": 187875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2443170547485352, + "objective/train/original_loss": 4.4912567138671875, + "objective/train/theoretical_loss": 4.128532765070004, + "objective/train/tokens_used": 311296000, + "objective/train/unlikelihood_loss": 0.08960556238889694, + "theoretical_loss": 4.128532765070004, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045741223671013037, + "loss": 1.4941, + "theoretical_loss": 4.128532765070004, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739217652958873, + "loss": 1.582, + "theoretical_loss": 4.12833008384537, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045737211634904715, + "loss": 1.7372, + "theoretical_loss": 4.128127511779873, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573520561685055, + "loss": 1.6319, + "theoretical_loss": 4.127925048768839, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045733199598796387, + "loss": 1.5441, + "theoretical_loss": 4.1277226947077335, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731193580742223, + "loss": 1.6482, + "theoretical_loss": 4.12752044949217, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045729187562688065, + "loss": 1.7088, + "theoretical_loss": 4.127318313017904, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457271815446339, + "loss": 1.7473, + "theoretical_loss": 4.1271162851808345, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572517552657974, + "loss": 1.6738, + "theoretical_loss": 4.126914365877004, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723169508525574, + "loss": 1.6325, + "theoretical_loss": 4.1267125550026, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572116349047141, + "loss": 1.5452, + "theoretical_loss": 4.126510852453949, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045719157472417257, + "loss": 1.634, + "theoretical_loss": 4.126309258127524, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717151454363093, + "loss": 1.3362, + "theoretical_loss": 4.126107771919935, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.38135385513305664, + "objective/train/avg_token_score": 0.02425443008542061, + "objective/train/docs_used": 188927, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5612069368362427, + "objective/train/original_loss": 4.165873050689697, + "objective/train/theoretical_loss": 4.126007069328436, + "objective/train/tokens_used": 312934400, + "objective/train/unlikelihood_loss": 0.16727052628993988, + "theoretical_loss": 4.126007069328436, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571514543630893, + "loss": 1.4993, + "theoretical_loss": 4.125906393727941, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713139418254766, + "loss": 1.4884, + "theoretical_loss": 4.125705123448437, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571113340020061, + "loss": 1.6256, + "theoretical_loss": 4.125503960978464, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045709127382146444, + "loss": 1.6268, + "theoretical_loss": 4.125302906215199, + "tokens_seen": 313393152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570712136409228, + "loss": 1.6244, + "theoretical_loss": 4.125101959055965, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045705115346038116, + "loss": 1.5808, + "theoretical_loss": 4.124901119398222, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570310932798395, + "loss": 1.6653, + "theoretical_loss": 4.124700387139574, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701103309929794, + "loss": 1.4647, + "theoretical_loss": 4.124499762177764, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569909729187563, + "loss": 1.5096, + "theoretical_loss": 4.124299244410672, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045697091273821466, + "loss": 1.5286, + "theoretical_loss": 4.124098833736321, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456950852557673, + "loss": 1.6552, + "theoretical_loss": 4.123898530052874, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045693079237713144, + "loss": 1.5479, + "theoretical_loss": 4.123698333258631, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3544110357761383, + "objective/train/avg_token_score": 0.01743168756365776, + "objective/train/docs_used": 189547, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3702281713485718, + "objective/train/original_loss": 4.191202640533447, + "objective/train/theoretical_loss": 4.123498243252032, + "objective/train/tokens_used": 314572800, + "objective/train/unlikelihood_loss": 0.13637718558311462, + "theoretical_loss": 4.123498243252032, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569107321965898, + "loss": 1.4601, + "theoretical_loss": 4.123498243252032, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045689067201604817, + "loss": 1.5157, + "theoretical_loss": 4.123298259931657, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045687061183550653, + "loss": 1.5577, + "theoretical_loss": 4.123098383196222, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045685055165496495, + "loss": 1.7436, + "theoretical_loss": 4.122898612944582, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568304914744233, + "loss": 1.3226, + "theoretical_loss": 4.122698949075732, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045681043129388167, + "loss": 1.5545, + "theoretical_loss": 4.1224993914888035, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045679037111334003, + "loss": 1.5553, + "theoretical_loss": 4.122299940083065, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567703109327984, + "loss": 1.4723, + "theoretical_loss": 4.122100594757921, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567502507522568, + "loss": 1.632, + "theoretical_loss": 4.121901355412917, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567301905717152, + "loss": 1.7226, + "theoretical_loss": 4.121702221947732, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045671013039117354, + "loss": 1.4859, + "theoretical_loss": 4.121503194262183, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566900702106319, + "loss": 1.4012, + "theoretical_loss": 4.121304272256222, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566700100300903, + "loss": 1.474, + "theoretical_loss": 4.121105455829939, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.42247068881988525, + "objective/train/avg_token_score": 0.02578432485461235, + "objective/train/docs_used": 190733, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6070023775100708, + "objective/train/original_loss": 4.354072570800781, + "objective/train/theoretical_loss": 4.121006087177992, + "objective/train/tokens_used": 316211200, + "objective/train/unlikelihood_loss": 0.11510311812162399, + "theoretical_loss": 4.121006087177992, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566499498495487, + "loss": 1.6307, + "theoretical_loss": 4.120906744883559, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045662988966900704, + "loss": 1.5513, + "theoretical_loss": 4.120708139317441, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566098294884654, + "loss": 1.6868, + "theoretical_loss": 4.120509639032081, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045658976930792376, + "loss": 1.5259, + "theoretical_loss": 4.120311243928111, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565697091273822, + "loss": 1.4202, + "theoretical_loss": 4.120112953906296, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654964894684054, + "loss": 1.6459, + "theoretical_loss": 4.119914768867536, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565295887662989, + "loss": 1.4654, + "theoretical_loss": 4.119716688712866, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045650952858575727, + "loss": 1.5123, + "theoretical_loss": 4.119518713343455, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564894684052157, + "loss": 1.5729, + "theoretical_loss": 4.119320842660606, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045646940822467405, + "loss": 1.5038, + "theoretical_loss": 4.119123076565755, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564493480441324, + "loss": 1.6548, + "theoretical_loss": 4.118925414960472, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045642928786359077, + "loss": 1.4868, + "theoretical_loss": 4.11872785774646, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5064760446548462, + "objective/train/avg_token_score": 0.0031210684683173895, + "objective/train/docs_used": 191330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9102715253829956, + "objective/train/original_loss": 4.087852954864502, + "objective/train/theoretical_loss": 4.118530404825556, + "objective/train/tokens_used": 317849600, + "objective/train/unlikelihood_loss": 0.16944539546966553, + "theoretical_loss": 4.118530404825556, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640922768304913, + "loss": 1.4363, + "theoretical_loss": 4.118530404825556, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045638916750250755, + "loss": 1.6041, + "theoretical_loss": 4.118333056099728, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563691073219659, + "loss": 1.609, + "theoretical_loss": 4.1181358114710775, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045634904714142427, + "loss": 1.7263, + "theoretical_loss": 4.117938670841838, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045632898696088263, + "loss": 1.5765, + "theoretical_loss": 4.117741634114376, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630892678034105, + "loss": 1.4432, + "theoretical_loss": 4.117544701191187, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562888665997994, + "loss": 1.7524, + "theoretical_loss": 4.117347871974903, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562688064192578, + "loss": 1.7409, + "theoretical_loss": 4.117151146368282, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045624874623871614, + "loss": 1.3192, + "theoretical_loss": 4.116954524274216, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562286860581745, + "loss": 1.4386, + "theoretical_loss": 4.116758005595727, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562086258776329, + "loss": 1.462, + "theoretical_loss": 4.116561590235969, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561885656970913, + "loss": 1.699, + "theoretical_loss": 4.116365278098225, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616850551654964, + "loss": 1.4585, + "theoretical_loss": 4.11616906908591, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.43585044145584106, + "objective/train/avg_token_score": 0.006533558014780283, + "objective/train/docs_used": 192764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4908214807510376, + "objective/train/original_loss": 3.6467506885528564, + "objective/train/theoretical_loss": 4.11607100322164, + "objective/train/tokens_used": 319488000, + "objective/train/unlikelihood_loss": 0.22543899714946747, + "theoretical_loss": 4.11607100322164, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456148445336008, + "loss": 1.6432, + "theoretical_loss": 4.115972963102565, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561283851554664, + "loss": 1.3954, + "theoretical_loss": 4.115776960051864, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561083249749248, + "loss": 1.4829, + "theoretical_loss": 4.115581059837612, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045608826479438314, + "loss": 1.6191, + "theoretical_loss": 4.115385262363739, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560682046138415, + "loss": 1.6366, + "theoretical_loss": 4.115189567534307, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604814443329987, + "loss": 1.6404, + "theoretical_loss": 4.114993975253505, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560280842527583, + "loss": 1.6653, + "theoretical_loss": 4.114798485425652, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045600802407221665, + "loss": 1.5634, + "theoretical_loss": 4.114603097955197, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455987963891675, + "loss": 1.5859, + "theoretical_loss": 4.1144078127467125, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045596790371113337, + "loss": 1.6533, + "theoretical_loss": 4.114212629704902, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559478435305918, + "loss": 1.6941, + "theoretical_loss": 4.114017548734598, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045592778335005015, + "loss": 1.5966, + "theoretical_loss": 4.113822569740757, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3504398763179779, + "objective/train/avg_token_score": 0.021333487704396248, + "objective/train/docs_used": 193449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3244068622589111, + "objective/train/original_loss": 4.0752177238464355, + "objective/train/theoretical_loss": 4.113627692628464, + "objective/train/tokens_used": 321126400, + "objective/train/unlikelihood_loss": 0.22843573987483978, + "theoretical_loss": 4.113627692628464, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559077231695085, + "loss": 1.321, + "theoretical_loss": 4.113627692628464, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558876629889669, + "loss": 1.564, + "theoretical_loss": 4.113432917302934, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045586760280842524, + "loss": 1.662, + "theoretical_loss": 4.113238243669504, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045584754262788365, + "loss": 1.393, + "theoretical_loss": 4.113043671633641, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455827482447342, + "loss": 1.4109, + "theoretical_loss": 4.112849201100938, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558074222668004, + "loss": 1.4717, + "theoretical_loss": 4.112654831977112, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045578736208625874, + "loss": 1.5382, + "theoretical_loss": 4.112460564168009, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045576730190571716, + "loss": 1.5993, + "theoretical_loss": 4.112266397579598, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557472417251755, + "loss": 1.5618, + "theoretical_loss": 4.1120723321179735, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557271815446339, + "loss": 1.3682, + "theoretical_loss": 4.111878367689359, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045570712136409224, + "loss": 1.6175, + "theoretical_loss": 4.111684504200099, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556870611835507, + "loss": 1.605, + "theoretical_loss": 4.111490741556663, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556670010030091, + "loss": 1.5167, + "theoretical_loss": 4.11129707966565, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5134408473968506, + "objective/train/avg_token_score": 0.037201035767793655, + "objective/train/docs_used": 194727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7789528369903564, + "objective/train/original_loss": 3.8756916522979736, + "objective/train/theoretical_loss": 4.111200286473145, + "objective/train/tokens_used": 322764800, + "objective/train/unlikelihood_loss": 0.13461580872535706, + "theoretical_loss": 4.111200286473145, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045564694082246744, + "loss": 1.5212, + "theoretical_loss": 4.111103518433776, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556268806419258, + "loss": 1.5896, + "theoretical_loss": 4.110910057767887, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045560682046138416, + "loss": 1.526, + "theoretical_loss": 4.110716697574951, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555867602808426, + "loss": 1.458, + "theoretical_loss": 4.110523437762059, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045556670010030094, + "loss": 1.4805, + "theoretical_loss": 4.110330278236427, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555466399197593, + "loss": 1.3518, + "theoretical_loss": 4.110137218905393, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045552657973921767, + "loss": 1.5839, + "theoretical_loss": 4.109944259676419, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555065195586761, + "loss": 1.4871, + "theoretical_loss": 4.109751400457089, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045548645937813445, + "loss": 1.487, + "theoretical_loss": 4.109558641155112, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554663991975928, + "loss": 1.5072, + "theoretical_loss": 4.109365981678316, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045544633901705117, + "loss": 1.4531, + "theoretical_loss": 4.109173421934654, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045542627883650953, + "loss": 1.5629, + "theoretical_loss": 4.1089809618321995, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5034213066101074, + "objective/train/avg_token_score": 0.00968634057790041, + "objective/train/docs_used": 195463, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7959951162338257, + "objective/train/original_loss": 3.7963671684265137, + "objective/train/theoretical_loss": 4.108788601279149, + "objective/train/tokens_used": 324403200, + "objective/train/unlikelihood_loss": 0.1474306285381317, + "theoretical_loss": 4.108788601279149, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045540621865596795, + "loss": 1.5169, + "theoretical_loss": 4.108788601279149, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553861584754263, + "loss": 1.7446, + "theoretical_loss": 4.108596340183819, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553660982948847, + "loss": 1.6527, + "theoretical_loss": 4.108404178454651, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045534603811434304, + "loss": 1.5472, + "theoretical_loss": 4.108212116000203, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045532597793380145, + "loss": 1.6305, + "theoretical_loss": 4.108020152729157, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553059177532598, + "loss": 1.6545, + "theoretical_loss": 4.107828288550314, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552858575727182, + "loss": 1.4823, + "theoretical_loss": 4.107636523372598, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045526579739217654, + "loss": 1.4818, + "theoretical_loss": 4.107444857105052, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552457372116349, + "loss": 1.6236, + "theoretical_loss": 4.107253289656838, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552256770310933, + "loss": 1.4742, + "theoretical_loss": 4.10706182093724, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552056168505517, + "loss": 1.5021, + "theoretical_loss": 4.106870450855661, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045518555667001004, + "loss": 1.5357, + "theoretical_loss": 4.106679179321622, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551654964894684, + "loss": 1.7192, + "theoretical_loss": 4.106488006244767, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.49883919954299927, + "objective/train/avg_token_score": 0.02902386523783207, + "objective/train/docs_used": 196731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7118357419967651, + "objective/train/original_loss": 3.6418302059173584, + "objective/train/theoretical_loss": 4.106392456599577, + "objective/train/tokens_used": 326041600, + "objective/train/unlikelihood_loss": 0.22379085421562195, + "theoretical_loss": 4.106392456599577, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551454363089268, + "loss": 1.5264, + "theoretical_loss": 4.106296931534854, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551253761283852, + "loss": 1.5085, + "theoretical_loss": 4.106105955101766, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045510531594784354, + "loss": 1.375, + "theoretical_loss": 4.105915076855499, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550852557673019, + "loss": 1.5322, + "theoretical_loss": 4.105724296706172, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045506519558676027, + "loss": 1.6069, + "theoretical_loss": 4.1055336145640196, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550451354062187, + "loss": 1.6159, + "theoretical_loss": 4.105343030339395, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045502507522567705, + "loss": 1.5891, + "theoretical_loss": 4.10515254394277, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550050150451354, + "loss": 1.5234, + "theoretical_loss": 4.104962155284734, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045498495486459377, + "loss": 1.5651, + "theoretical_loss": 4.104771864275993, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549648946840522, + "loss": 1.407, + "theoretical_loss": 4.104581670827372, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045494483450351055, + "loss": 1.6212, + "theoretical_loss": 4.104391574849812, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549247743229689, + "loss": 1.5187, + "theoretical_loss": 4.104201576254369, + "tokens_seen": 327548928 + }, + { + "debugging/length": 505.3636363636364, + "debugging/num_segments": 11, + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5214442610740662, + "objective/train/avg_token_score": 0.01770769990980625, + "objective/train/docs_used": 197513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0139143466949463, + "objective/train/original_loss": 4.308063983917236, + "objective/train/theoretical_loss": 4.10401167495222, + "objective/train/tokens_used": 327680000, + "objective/train/unlikelihood_loss": 0.12178285419940948, + "theoretical_loss": 4.10401167495222, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549047141424273, + "loss": 1.6542, + "theoretical_loss": 4.10401167495222, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045488465396188564, + "loss": 1.6045, + "theoretical_loss": 4.103821870854656, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486459378134405, + "loss": 1.5895, + "theoretical_loss": 4.103632163873083, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548445336008024, + "loss": 1.6359, + "theoretical_loss": 4.103442553919026, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548244734202608, + "loss": 1.6449, + "theoretical_loss": 4.103253040904124, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480441323971914, + "loss": 1.5452, + "theoretical_loss": 4.103063624740133, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045478435305917756, + "loss": 1.4296, + "theoretical_loss": 4.102874305338923, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547642928786359, + "loss": 1.532, + "theoretical_loss": 4.10268508261248, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547442326980943, + "loss": 1.6441, + "theoretical_loss": 4.1024959564729055, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045472417251755264, + "loss": 1.5003, + "theoretical_loss": 4.102306926832417, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454704112337011, + "loss": 1.5285, + "theoretical_loss": 4.1021179936033425, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546840521564694, + "loss": 1.5756, + "theoretical_loss": 4.10192915669813, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546639919759278, + "loss": 1.5778, + "theoretical_loss": 4.101740416029338, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.48374876379966736, + "objective/train/avg_token_score": 0.012546883895993233, + "objective/train/docs_used": 198138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9386160373687744, + "objective/train/original_loss": 4.203801155090332, + "objective/train/theoretical_loss": 4.101646081756305, + "objective/train/tokens_used": 329318400, + "objective/train/unlikelihood_loss": 0.1514439433813095, + "theoretical_loss": 4.101646081756305, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464393179538615, + "loss": 1.4654, + "theoretical_loss": 4.101551771509641, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546238716148445, + "loss": 1.6519, + "theoretical_loss": 4.101363223051826, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546038114343029, + "loss": 1.4715, + "theoretical_loss": 4.101174770568795, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545837512537613, + "loss": 1.4744, + "theoretical_loss": 4.100986413973564, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045456369107321965, + "loss": 1.7026, + "theoretical_loss": 4.1007981531792606, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454543630892678, + "loss": 1.4468, + "theoretical_loss": 4.1006099880991265, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045452357071213643, + "loss": 1.5817, + "theoretical_loss": 4.100421918646517, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545035105315948, + "loss": 1.5614, + "theoretical_loss": 4.100233944734899, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045448345035105315, + "loss": 1.6208, + "theoretical_loss": 4.100046066277853, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544633901705115, + "loss": 1.5886, + "theoretical_loss": 4.0998582831890715, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544433299899699, + "loss": 1.5603, + "theoretical_loss": 4.09967059538236, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544232698094283, + "loss": 1.5664, + "theoretical_loss": 4.099483002771633, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.34042033553123474, + "objective/train/avg_token_score": 0.018601443618535995, + "objective/train/docs_used": 199416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2680652141571045, + "objective/train/original_loss": 3.715749740600586, + "objective/train/theoretical_loss": 4.099295505270921, + "objective/train/tokens_used": 330956800, + "objective/train/unlikelihood_loss": 0.23622927069664001, + "theoretical_loss": 4.099295505270921, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045440320962888666, + "loss": 1.3597, + "theoretical_loss": 4.099295505270921, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454383149448345, + "loss": 1.4881, + "theoretical_loss": 4.099108102794363, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543630892678034, + "loss": 1.5775, + "theoretical_loss": 4.098920795256213, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543430290872618, + "loss": 1.6801, + "theoretical_loss": 4.098733582570831, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045432296890672016, + "loss": 1.6044, + "theoretical_loss": 4.098546464652693, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543029087261785, + "loss": 1.5239, + "theoretical_loss": 4.098359441416383, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828485456369, + "loss": 1.7179, + "theoretical_loss": 4.098172512776597, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426278836509525, + "loss": 1.5754, + "theoretical_loss": 4.097985678648142, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045424272818455366, + "loss": 1.5853, + "theoretical_loss": 4.097798938945933, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454222668004012, + "loss": 1.4991, + "theoretical_loss": 4.097612293584998, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420260782347044, + "loss": 1.389, + "theoretical_loss": 4.097425742480472, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541825476429288, + "loss": 1.7117, + "theoretical_loss": 4.0972392855476025, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541624874623872, + "loss": 1.5459, + "theoretical_loss": 4.097052922701746, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3013196587562561, + "objective/train/avg_token_score": 0.018458493053913116, + "objective/train/docs_used": 200524, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0405497550964355, + "objective/train/original_loss": 3.868088960647583, + "objective/train/theoretical_loss": 4.0969597765350265, + "objective/train/tokens_used": 332595200, + "objective/train/unlikelihood_loss": 0.16207247972488403, + "theoretical_loss": 4.0969597765350265, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541424272818456, + "loss": 1.367, + "theoretical_loss": 4.096866653858368, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045412236710130395, + "loss": 1.617, + "theoretical_loss": 4.096680478933042, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541023069207623, + "loss": 1.5846, + "theoretical_loss": 4.09649439784145, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045408224674022067, + "loss": 1.6318, + "theoretical_loss": 4.09630841049939, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540621865596791, + "loss": 1.655, + "theoretical_loss": 4.096122516822757, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045404212637913745, + "loss": 1.4591, + "theoretical_loss": 4.095936716727564, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540220661985958, + "loss": 1.4523, + "theoretical_loss": 4.095751010129929, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400200601805417, + "loss": 1.6392, + "theoretical_loss": 4.0955653969460775, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539819458375126, + "loss": 1.5289, + "theoretical_loss": 4.095379877092343, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045396188565697095, + "loss": 1.7503, + "theoretical_loss": 4.0951944504851685, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539418254764293, + "loss": 1.4979, + "theoretical_loss": 4.095009117041102, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539217652958877, + "loss": 1.7049, + "theoretical_loss": 4.094823876676802, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5921309590339661, + "objective/train/avg_token_score": 0.006544371135532856, + "objective/train/docs_used": 201196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2652223110198975, + "objective/train/original_loss": 4.068779468536377, + "objective/train/theoretical_loss": 4.094638729309031, + "objective/train/tokens_used": 334233600, + "objective/train/unlikelihood_loss": 0.11625197529792786, + "theoretical_loss": 4.094638729309031, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045390170511534604, + "loss": 1.4098, + "theoretical_loss": 4.094638729309031, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045388164493480445, + "loss": 1.6587, + "theoretical_loss": 4.09445367485466, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538615847542628, + "loss": 1.5036, + "theoretical_loss": 4.094268713230667, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538415245737212, + "loss": 1.4698, + "theoretical_loss": 4.094083844354137, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045382146439317954, + "loss": 1.4929, + "theoretical_loss": 4.093899068142262, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045380140421263796, + "loss": 1.6098, + "theoretical_loss": 4.093714384512337, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537813440320963, + "loss": 1.6043, + "theoretical_loss": 4.093529793381768, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537612838515547, + "loss": 1.6377, + "theoretical_loss": 4.093345294668063, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045374122367101304, + "loss": 1.7649, + "theoretical_loss": 4.093160888288837, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537211634904714, + "loss": 1.5487, + "theoretical_loss": 4.0929765741618125, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537011033099298, + "loss": 1.519, + "theoretical_loss": 4.092792352204814, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536810431293882, + "loss": 1.5532, + "theoretical_loss": 4.092608222335774, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045366098294884655, + "loss": 1.5037, + "theoretical_loss": 4.09242418447273, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4757453501224518, + "objective/train/avg_token_score": 0.009879210032522678, + "objective/train/docs_used": 202460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8038827180862427, + "objective/train/original_loss": 4.1414971351623535, + "objective/train/theoretical_loss": 4.092332200017871, + "objective/train/tokens_used": 335872000, + "objective/train/unlikelihood_loss": 0.12569139897823334, + "theoretical_loss": 4.092332200017871, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536409227683049, + "loss": 1.7663, + "theoretical_loss": 4.092240238533822, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536208625877633, + "loss": 1.5856, + "theoretical_loss": 4.092056384437297, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536008024072217, + "loss": 1.63, + "theoretical_loss": 4.091872622101506, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045358074222668005, + "loss": 1.6248, + "theoretical_loss": 4.091688951444904, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535606820461384, + "loss": 1.5825, + "theoretical_loss": 4.091505372386051, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045354062186559683, + "loss": 1.6858, + "theoretical_loss": 4.091321884843609, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535205616850552, + "loss": 1.5953, + "theoretical_loss": 4.0911384887363464, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045350050150451355, + "loss": 1.6498, + "theoretical_loss": 4.0909551839831355, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534804413239719, + "loss": 1.5834, + "theoretical_loss": 4.090771970502948, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534603811434303, + "loss": 1.5166, + "theoretical_loss": 4.090588848214865, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534403209628887, + "loss": 1.5447, + "theoretical_loss": 4.090405817038065, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045342026078234706, + "loss": 1.4795, + "theoretical_loss": 4.090222876891834, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4949902296066284, + "objective/train/avg_token_score": 0.009650841355323792, + "objective/train/docs_used": 203033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7750602960586548, + "objective/train/original_loss": 3.9335618019104004, + "objective/train/theoretical_loss": 4.090040027695556, + "objective/train/tokens_used": 337510400, + "objective/train/unlikelihood_loss": 0.13872487843036652, + "theoretical_loss": 4.090040027695556, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534002006018054, + "loss": 1.712, + "theoretical_loss": 4.090040027695556, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533801404212638, + "loss": 1.874, + "theoretical_loss": 4.089857269368725, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533600802407222, + "loss": 1.5283, + "theoretical_loss": 4.089674601830929, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045334002006018056, + "loss": 1.6553, + "theoretical_loss": 4.089492025001864, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533199598796389, + "loss": 1.5234, + "theoretical_loss": 4.089309538801327, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532998996990973, + "loss": 1.5957, + "theoretical_loss": 4.0891271431492155, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045327983951855565, + "loss": 1.7097, + "theoretical_loss": 4.08894483796553, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325977933801406, + "loss": 1.6363, + "theoretical_loss": 4.088762623170373, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532397191574724, + "loss": 1.6686, + "theoretical_loss": 4.088580498683948, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532196589769308, + "loss": 1.6399, + "theoretical_loss": 4.088398464426559, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045319959879638915, + "loss": 1.6365, + "theoretical_loss": 4.088216520318612, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045317953861584757, + "loss": 1.5666, + "theoretical_loss": 4.088034666280614, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315947843530593, + "loss": 1.6203, + "theoretical_loss": 4.087852902233173, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4335288405418396, + "objective/train/avg_token_score": 0.013492316007614136, + "objective/train/docs_used": 204232, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5629843473434448, + "objective/train/original_loss": 3.536388874053955, + "objective/train/theoretical_loss": 4.087762053931129, + "objective/train/tokens_used": 339148800, + "objective/train/unlikelihood_loss": 0.25973203778266907, + "theoretical_loss": 4.087762053931129, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531394182547643, + "loss": 1.6724, + "theoretical_loss": 4.087671228096997, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311935807422265, + "loss": 1.5495, + "theoretical_loss": 4.087489643792894, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453099297893681, + "loss": 1.6182, + "theoretical_loss": 4.087308149241774, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045307923771313943, + "loss": 1.6192, + "theoretical_loss": 4.087126744364646, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530591775325978, + "loss": 1.5072, + "theoretical_loss": 4.086945429082618, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045303911735205616, + "loss": 1.6224, + "theoretical_loss": 4.086764203316902, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530190571715145, + "loss": 1.5255, + "theoretical_loss": 4.086583066988802, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045299899699097294, + "loss": 1.5888, + "theoretical_loss": 4.0864020200197295, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529789368104313, + "loss": 1.6588, + "theoretical_loss": 4.086221062331192, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045295887662988966, + "loss": 1.6366, + "theoretical_loss": 4.086040193844794, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452938816449348, + "loss": 1.4848, + "theoretical_loss": 4.0858594144822415, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529187562688064, + "loss": 1.5903, + "theoretical_loss": 4.085678724165341, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.32972875237464905, + "objective/train/avg_token_score": 0.018639761954545975, + "objective/train/docs_used": 205032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0958448648452759, + "objective/train/original_loss": 3.859755754470825, + "objective/train/theoretical_loss": 4.085498122815992, + "objective/train/tokens_used": 340787200, + "objective/train/unlikelihood_loss": 0.2070046067237854, + "theoretical_loss": 4.085498122815992, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528986960882648, + "loss": 1.5739, + "theoretical_loss": 4.085498122815992, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045287863590772316, + "loss": 1.6326, + "theoretical_loss": 4.085317610356199, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528585757271815, + "loss": 1.4208, + "theoretical_loss": 4.08513718670806, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528385155466399, + "loss": 1.5432, + "theoretical_loss": 4.084956851793773, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528184553660983, + "loss": 1.5042, + "theoretical_loss": 4.0847766055356365, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045279839518555667, + "loss": 1.6324, + "theoretical_loss": 4.0845964478560415, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045277833500501503, + "loss": 1.4951, + "theoretical_loss": 4.0844163786774805, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527582748244734, + "loss": 1.6173, + "theoretical_loss": 4.084236397922544, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045273821464393175, + "loss": 1.5795, + "theoretical_loss": 4.0840565055139155, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045271815446339017, + "loss": 1.5769, + "theoretical_loss": 4.083876701374381, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526980942828486, + "loss": 1.505, + "theoretical_loss": 4.08369698542682, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045267803410230695, + "loss": 1.5581, + "theoretical_loss": 4.083517357594212, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526579739217653, + "loss": 1.3931, + "theoretical_loss": 4.083337817799629, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.339259535074234, + "objective/train/avg_token_score": 0.027388932183384895, + "objective/train/docs_used": 206109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2726820707321167, + "objective/train/original_loss": 4.560004234313965, + "objective/train/theoretical_loss": 4.083248080892584, + "objective/train/tokens_used": 342425600, + "objective/train/unlikelihood_loss": 0.11201095581054688, + "theoretical_loss": 4.083248080892584, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526379137412237, + "loss": 1.5391, + "theoretical_loss": 4.083158365966243, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526178535606821, + "loss": 1.6968, + "theoretical_loss": 4.082979002017321, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045259779338014045, + "loss": 1.6573, + "theoretical_loss": 4.082799725876228, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525777331995988, + "loss": 1.6508, + "theoretical_loss": 4.082620537466422, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045255767301905723, + "loss": 1.6432, + "theoretical_loss": 4.082441436711459, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525376128385156, + "loss": 1.5502, + "theoretical_loss": 4.082262423534992, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045251755265797395, + "loss": 1.4503, + "theoretical_loss": 4.082083497860766, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524974924774323, + "loss": 1.5342, + "theoretical_loss": 4.081904659612624, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524774322968907, + "loss": 1.3966, + "theoretical_loss": 4.081725908714505, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524573721163491, + "loss": 1.4216, + "theoretical_loss": 4.08154724509044, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045243731193580746, + "loss": 1.3198, + "theoretical_loss": 4.08136866866456, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524172517552658, + "loss": 1.5836, + "theoretical_loss": 4.0811901793610845, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.34237536787986755, + "objective/train/avg_token_score": 0.0033272351138293743, + "objective/train/docs_used": 206760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2112702131271362, + "objective/train/original_loss": 3.6624834537506104, + "objective/train/theoretical_loss": 4.081011777104333, + "objective/train/tokens_used": 344064000, + "objective/train/unlikelihood_loss": 0.18153946101665497, + "theoretical_loss": 4.081011777104333, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523971915747242, + "loss": 1.4097, + "theoretical_loss": 4.081011777104333, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523771313941826, + "loss": 1.7856, + "theoretical_loss": 4.080833461818718, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045235707121364096, + "loss": 1.5064, + "theoretical_loss": 4.080655233428745, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523370110330993, + "loss": 1.5193, + "theoretical_loss": 4.080477091859015, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523169508525577, + "loss": 1.5518, + "theoretical_loss": 4.080299037034224, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045229689067201605, + "loss": 1.6474, + "theoretical_loss": 4.0801210688791585, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045227683049147446, + "loss": 1.5025, + "theoretical_loss": 4.079943187318703, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522567703109328, + "loss": 1.6411, + "theoretical_loss": 4.0797653922778325, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522367101303912, + "loss": 1.6179, + "theoretical_loss": 4.079587683681617, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045221664994984955, + "loss": 1.4868, + "theoretical_loss": 4.07941006145522, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045219658976930797, + "loss": 1.357, + "theoretical_loss": 4.0792325255238975, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045217652958876633, + "loss": 1.4853, + "theoretical_loss": 4.079055075812997, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521564694082247, + "loss": 1.6133, + "theoretical_loss": 4.078877712247963, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.1969086080789566, + "objective/train/avg_token_score": 0.01607353612780571, + "objective/train/docs_used": 207336, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7453041076660156, + "objective/train/original_loss": 4.304450988769531, + "objective/train/theoretical_loss": 4.078789062746872, + "objective/train/tokens_used": 345702400, + "objective/train/unlikelihood_loss": 0.1982918530702591, + "theoretical_loss": 4.078789062746872, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045213640922768305, + "loss": 1.3566, + "theoretical_loss": 4.07870043475433, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521163490471414, + "loss": 1.4831, + "theoretical_loss": 4.078523243257723, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045209628886659983, + "loss": 1.6821, + "theoretical_loss": 4.078346137683862, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520762286860582, + "loss": 1.5421, + "theoretical_loss": 4.078169117958562, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045205616850551656, + "loss": 1.4795, + "theoretical_loss": 4.077992184007725, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520361083249749, + "loss": 1.6154, + "theoretical_loss": 4.077815335757347, + "tokens_seen": 346423296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045201604814443334, + "loss": 1.5112, + "theoretical_loss": 4.077638573133515, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519959879638917, + "loss": 1.5462, + "theoretical_loss": 4.0774618960624105, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045197592778335006, + "loss": 1.6378, + "theoretical_loss": 4.077285304470304, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519558676028084, + "loss": 1.5589, + "theoretical_loss": 4.077108798283557, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519358074222668, + "loss": 1.3784, + "theoretical_loss": 4.076932377428625, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519157472417252, + "loss": 1.4946, + "theoretical_loss": 4.076756041832049, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3781769275665283, + "objective/train/avg_token_score": 0.007616470567882061, + "objective/train/docs_used": 208796, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4440606832504272, + "objective/train/original_loss": 3.906726360321045, + "objective/train/theoretical_loss": 4.076579791420469, + "objective/train/tokens_used": 347340800, + "objective/train/unlikelihood_loss": 0.1508537232875824, + "theoretical_loss": 4.076579791420469, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189568706118356, + "loss": 1.6182, + "theoretical_loss": 4.076579791420469, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518756268806419, + "loss": 1.5178, + "theoretical_loss": 4.07640362612061, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518555667001003, + "loss": 1.529, + "theoretical_loss": 4.076227545859288, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518355065195587, + "loss": 1.6502, + "theoretical_loss": 4.076051550563411, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181544633901707, + "loss": 1.3819, + "theoretical_loss": 4.075875640159978, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045179538615847543, + "loss": 1.5274, + "theoretical_loss": 4.075699814576076, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517753259779338, + "loss": 1.5282, + "theoretical_loss": 4.075524073738883, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175526579739215, + "loss": 1.5711, + "theoretical_loss": 4.075348417575669, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173520561685057, + "loss": 1.6626, + "theoretical_loss": 4.075172846013789, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171514543630893, + "loss": 1.4903, + "theoretical_loss": 4.074997358980694, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516950852557673, + "loss": 1.6022, + "theoretical_loss": 4.074821956403917, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167502507522566, + "loss": 1.5102, + "theoretical_loss": 4.074646638211087, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165496489468407, + "loss": 1.6114, + "theoretical_loss": 4.074471404329918, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5323802828788757, + "objective/train/avg_token_score": 0.004500185139477253, + "objective/train/docs_used": 209597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1040139198303223, + "objective/train/original_loss": 4.173861980438232, + "objective/train/theoretical_loss": 4.074383818983643, + "objective/train/tokens_used": 348979200, + "objective/train/unlikelihood_loss": 0.11532780528068542, + "theoretical_loss": 4.074383818983643, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163490471414244, + "loss": 1.7397, + "theoretical_loss": 4.074296254688216, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516148445336008, + "loss": 1.5713, + "theoretical_loss": 4.074121189213873, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045159478435305916, + "loss": 1.4462, + "theoretical_loss": 4.073946207834872, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515747241725175, + "loss": 1.5493, + "theoretical_loss": 4.073771310479284, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045155466399197594, + "loss": 1.4127, + "theoretical_loss": 4.073596497075267, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515346038114343, + "loss": 1.3747, + "theoretical_loss": 4.07342176755107, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151454363089266, + "loss": 1.4241, + "theoretical_loss": 4.073247121835028, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451494483450351, + "loss": 1.5795, + "theoretical_loss": 4.073072559855565, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147442326980944, + "loss": 1.5063, + "theoretical_loss": 4.072898081541193, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514543630892678, + "loss": 1.4867, + "theoretical_loss": 4.0727236868205114, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045143430290872617, + "loss": 1.4653, + "theoretical_loss": 4.072549375622207, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141424272818453, + "loss": 1.5842, + "theoretical_loss": 4.072375147875054, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3748167157173157, + "objective/train/avg_token_score": 0.026631662622094154, + "objective/train/docs_used": 210243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2623140811920166, + "objective/train/original_loss": 4.159633636474609, + "objective/train/theoretical_loss": 4.0722010035079155, + "objective/train/tokens_used": 350617600, + "objective/train/unlikelihood_loss": 0.13227179646492004, + "theoretical_loss": 4.0722010035079155, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045139418254764294, + "loss": 1.4965, + "theoretical_loss": 4.0722010035079155, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513741223671013, + "loss": 1.4648, + "theoretical_loss": 4.07202694244974, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135406218655967, + "loss": 1.2912, + "theoretical_loss": 4.0718529646295645, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045133400200601803, + "loss": 1.3967, + "theoretical_loss": 4.071679069976511, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513139418254764, + "loss": 1.4605, + "theoretical_loss": 4.07150525841979, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512938816449348, + "loss": 1.4834, + "theoretical_loss": 4.071331529888698, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045127382146439317, + "loss": 1.4192, + "theoretical_loss": 4.071157884312619, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125376128385153, + "loss": 1.5059, + "theoretical_loss": 4.070984321621021, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512337011033099, + "loss": 1.5377, + "theoretical_loss": 4.0708108417434605, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512136409227683, + "loss": 1.4409, + "theoretical_loss": 4.07063744460958, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119358074222673, + "loss": 1.4988, + "theoretical_loss": 4.070464130149106, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511735205616851, + "loss": 1.5584, + "theoretical_loss": 4.070290898291852, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115346038114345, + "loss": 1.5342, + "theoretical_loss": 4.07011774896772, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5329301357269287, + "objective/train/avg_token_score": 0.007351990323513746, + "objective/train/docs_used": 211560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9991000890731812, + "objective/train/original_loss": 4.122071743011475, + "objective/train/theoretical_loss": 4.07003120523369, + "objective/train/tokens_used": 352256000, + "objective/train/unlikelihood_loss": 0.16980284452438354, + "theoretical_loss": 4.07003120523369, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511334002006018, + "loss": 1.599, + "theoretical_loss": 4.069944682106691, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045111334002006023, + "loss": 1.4121, + "theoretical_loss": 4.069771697638838, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510932798395186, + "loss": 1.5508, + "theoretical_loss": 4.069598795494315, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045107321965897696, + "loss": 1.3157, + "theoretical_loss": 4.069425975603364, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510531594784353, + "loss": 1.4493, + "theoretical_loss": 4.0692532378963095, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045103309929789374, + "loss": 1.4906, + "theoretical_loss": 4.069080582303563, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510130391173521, + "loss": 1.5558, + "theoretical_loss": 4.068908008755619, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045099297893681046, + "loss": 1.4319, + "theoretical_loss": 4.0687355171830575, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509729187562688, + "loss": 1.6799, + "theoretical_loss": 4.068563107516544, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509528585757272, + "loss": 1.5795, + "theoretical_loss": 4.068390779686826, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509327983951856, + "loss": 1.2278, + "theoretical_loss": 4.068218533624737, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045091273821464396, + "loss": 1.5487, + "theoretical_loss": 4.068046369261194, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3893572688102722, + "objective/train/avg_token_score": 0.027582136914134026, + "objective/train/docs_used": 212189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.257407784461975, + "objective/train/original_loss": 3.744044065475464, + "objective/train/theoretical_loss": 4.067874286527197, + "objective/train/tokens_used": 353894400, + "objective/train/unlikelihood_loss": 0.16484889388084412, + "theoretical_loss": 4.067874286527197, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508926780341023, + "loss": 1.5564, + "theoretical_loss": 4.067874286527197, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508726178535607, + "loss": 1.4902, + "theoretical_loss": 4.067702285353833, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508525576730191, + "loss": 1.4888, + "theoretical_loss": 4.067530365672267, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045083249749247747, + "loss": 1.4045, + "theoretical_loss": 4.067358527413753, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045081243731193583, + "loss": 1.5815, + "theoretical_loss": 4.067186770509627, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507923771313942, + "loss": 1.4217, + "theoretical_loss": 4.067015094891306, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045077231695085255, + "loss": 1.5362, + "theoretical_loss": 4.066843500490292, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045075225677031097, + "loss": 1.4491, + "theoretical_loss": 4.066671987238171, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045073219658976933, + "loss": 1.3298, + "theoretical_loss": 4.066500555066609, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507121364092277, + "loss": 1.4452, + "theoretical_loss": 4.066329203907358, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045069207622868606, + "loss": 1.6151, + "theoretical_loss": 4.066157933692249, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506720160481445, + "loss": 1.4978, + "theoretical_loss": 4.0659867443532, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045065195586760284, + "loss": 1.6458, + "theoretical_loss": 4.065815635822206, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5819892287254333, + "objective/train/avg_token_score": 0.017979789525270462, + "objective/train/docs_used": 213521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.2102198600769043, + "objective/train/original_loss": 4.2459540367126465, + "objective/train/theoretical_loss": 4.065730111838502, + "objective/train/tokens_used": 355532800, + "objective/train/unlikelihood_loss": 0.09294069558382034, + "theoretical_loss": 4.065730111838502, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506318956870612, + "loss": 1.6467, + "theoretical_loss": 4.0656446080313495, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045061183550651956, + "loss": 1.509, + "theoretical_loss": 4.065473660912791, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505917753259779, + "loss": 1.3671, + "theoretical_loss": 4.065302794398775, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045057171514543634, + "loss": 1.3185, + "theoretical_loss": 4.065132008421628, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505516549648947, + "loss": 1.4506, + "theoretical_loss": 4.0649613029137575, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045053159478435306, + "loss": 1.31, + "theoretical_loss": 4.064790677807652, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505115346038114, + "loss": 1.3887, + "theoretical_loss": 4.064620133035883, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045049147442326984, + "loss": 1.4987, + "theoretical_loss": 4.064449668531103, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504714142427282, + "loss": 1.6307, + "theoretical_loss": 4.064279284226044, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045045135406218657, + "loss": 1.4173, + "theoretical_loss": 4.06410898005352, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045043129388164493, + "loss": 1.7445, + "theoretical_loss": 4.0639387559464275, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045041123370110334, + "loss": 1.7331, + "theoretical_loss": 4.063768611837742, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.2592253088951111, + "objective/train/avg_token_score": 0.012151106260716915, + "objective/train/docs_used": 214188, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0094984769821167, + "objective/train/original_loss": 4.309907913208008, + "objective/train/theoretical_loss": 4.063598547660519, + "objective/train/tokens_used": 357171200, + "objective/train/unlikelihood_loss": 0.2334572970867157, + "theoretical_loss": 4.063598547660519, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503911735205617, + "loss": 1.4034, + "theoretical_loss": 4.063598547660519, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045037111334002007, + "loss": 1.5863, + "theoretical_loss": 4.063428563347898, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045035105315947843, + "loss": 1.4359, + "theoretical_loss": 4.063258658833096, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503309929789368, + "loss": 1.468, + "theoretical_loss": 4.0630888340494105, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503109327983952, + "loss": 1.4797, + "theoretical_loss": 4.06291908893022, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045029087261785357, + "loss": 1.5528, + "theoretical_loss": 4.062749423408981, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045027081243731193, + "loss": 1.4224, + "theoretical_loss": 4.062579837419235, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502507522567703, + "loss": 1.5149, + "theoretical_loss": 4.062410330894597, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502306920762287, + "loss": 1.5902, + "theoretical_loss": 4.062240903768767, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502106318956871, + "loss": 1.5077, + "theoretical_loss": 4.062071555975519, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045019057171514544, + "loss": 1.3566, + "theoretical_loss": 4.061902287448714, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501705115346038, + "loss": 1.5275, + "theoretical_loss": 4.061733098122284, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045015045135406216, + "loss": 1.525, + "theoretical_loss": 4.0615639879302465, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5088587403297424, + "objective/train/avg_token_score": 0.01710294932126999, + "objective/train/docs_used": 215284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8704804182052612, + "objective/train/original_loss": 3.966881513595581, + "objective/train/theoretical_loss": 4.061479462489026, + "objective/train/tokens_used": 358809600, + "objective/train/unlikelihood_loss": 0.14359432458877563, + "theoretical_loss": 4.061479462489026, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501303911735206, + "loss": 1.5061, + "theoretical_loss": 4.061394956806694, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045011033099297894, + "loss": 1.3324, + "theoretical_loss": 4.0612260046858015, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500902708124373, + "loss": 1.3371, + "theoretical_loss": 4.06105713150182, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045007021063189567, + "loss": 1.4478, + "theoretical_loss": 4.0608883371890805, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500501504513541, + "loss": 1.4508, + "theoretical_loss": 4.06071962168199, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045003009027081244, + "loss": 1.5374, + "theoretical_loss": 4.060550984915039, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500100300902708, + "loss": 1.3958, + "theoretical_loss": 4.060382426822792, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044998996990972917, + "loss": 1.4925, + "theoretical_loss": 4.060213947339894, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044996990972918753, + "loss": 1.4814, + "theoretical_loss": 4.060045546401065, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044994984954864595, + "loss": 1.5105, + "theoretical_loss": 4.0598772239411085, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499297893681043, + "loss": 1.3412, + "theoretical_loss": 4.0597089798949, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044990972918756267, + "loss": 1.443, + "theoretical_loss": 4.059540814197396, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3905791640281677, + "objective/train/avg_token_score": 0.048172734677791595, + "objective/train/docs_used": 215704, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4849482774734497, + "objective/train/original_loss": 4.436104774475098, + "objective/train/theoretical_loss": 4.05937272678363, + "objective/train/tokens_used": 360448000, + "objective/train/unlikelihood_loss": 0.11548055708408356, + "theoretical_loss": 4.05937272678363, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044988966900702103, + "loss": 1.4311, + "theoretical_loss": 4.05937272678363, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044986960882647945, + "loss": 1.552, + "theoretical_loss": 4.059204717588712, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498495486459378, + "loss": 1.4372, + "theoretical_loss": 4.059036786547831, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498294884653962, + "loss": 1.481, + "theoretical_loss": 4.058868933596252, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044980942828485454, + "loss": 1.598, + "theoretical_loss": 4.058701158669318, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497893681043129, + "loss": 1.4096, + "theoretical_loss": 4.0585334617024476, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497693079237713, + "loss": 1.5581, + "theoretical_loss": 4.058365842631137, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497492477432297, + "loss": 1.4585, + "theoretical_loss": 4.058198301390959, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044972918756268804, + "loss": 1.4782, + "theoretical_loss": 4.058030837917565, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497091273821464, + "loss": 1.2856, + "theoretical_loss": 4.057863452146679, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496890672016049, + "loss": 1.3921, + "theoretical_loss": 4.057696144014106, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044966900702106324, + "loss": 1.6007, + "theoretical_loss": 4.057528913455723, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496489468405216, + "loss": 1.6407, + "theoretical_loss": 4.0573617604074865, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.47562316060066223, + "objective/train/avg_token_score": 0.012415323406457901, + "objective/train/docs_used": 216313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8667166233062744, + "objective/train/original_loss": 4.416706085205078, + "objective/train/theoretical_loss": 4.0572782129296785, + "objective/train/tokens_used": 362086400, + "objective/train/unlikelihood_loss": 0.12603291869163513, + "theoretical_loss": 4.0572782129296785, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044962888665997996, + "loss": 1.5666, + "theoretical_loss": 4.057194684805426, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496088264794383, + "loss": 1.4589, + "theoretical_loss": 4.0570276865856485, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044958876629889674, + "loss": 1.6308, + "theoretical_loss": 4.056860765684338, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495687061183551, + "loss": 1.5062, + "theoretical_loss": 4.056693922037753, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044954864593781346, + "loss": 1.3438, + "theoretical_loss": 4.056527155582226, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495285857572718, + "loss": 1.3656, + "theoretical_loss": 4.056360466254168, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044950852557673024, + "loss": 1.5156, + "theoretical_loss": 4.056193853990063, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494884653961886, + "loss": 1.5408, + "theoretical_loss": 4.056027318726472, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044946840521564697, + "loss": 1.6677, + "theoretical_loss": 4.0558608604000295, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044944834503510533, + "loss": 1.4249, + "theoretical_loss": 4.055694478947445, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044942828485456375, + "loss": 1.4203, + "theoretical_loss": 4.055528174305505, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494082246740221, + "loss": 1.5913, + "theoretical_loss": 4.055361946411068, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.37341153621673584, + "objective/train/avg_token_score": 0.040493469685316086, + "objective/train/docs_used": 217479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2378941774368286, + "objective/train/original_loss": 3.728134870529175, + "objective/train/theoretical_loss": 4.055195795201069, + "objective/train/tokens_used": 363724800, + "objective/train/unlikelihood_loss": 0.2129121869802475, + "theoretical_loss": 4.055195795201069, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044938816449348047, + "loss": 1.6193, + "theoretical_loss": 4.055195795201069, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044936810431293883, + "loss": 1.465, + "theoretical_loss": 4.0550297206125165, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493480441323972, + "loss": 1.3898, + "theoretical_loss": 4.054863722582494, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493279839518556, + "loss": 1.3853, + "theoretical_loss": 4.05469780104816, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044930792377131397, + "loss": 1.5886, + "theoretical_loss": 4.054531955946745, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044928786359077234, + "loss": 1.6761, + "theoretical_loss": 4.054366187215556, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492678034102307, + "loss": 1.5347, + "theoretical_loss": 4.054200494791972, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492477432296891, + "loss": 1.3878, + "theoretical_loss": 4.054034878613447, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492276830491475, + "loss": 1.4606, + "theoretical_loss": 4.053869338617508, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044920762286860584, + "loss": 1.5716, + "theoretical_loss": 4.053703874741757, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491875626880642, + "loss": 1.5397, + "theoretical_loss": 4.053538486923868, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044916750250752256, + "loss": 1.4733, + "theoretical_loss": 4.053373175101589, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449147442326981, + "loss": 1.5254, + "theoretical_loss": 4.05320793921274, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4299853444099426, + "objective/train/avg_token_score": 0.02031981386244297, + "objective/train/docs_used": 218231, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5544748306274414, + "objective/train/original_loss": 4.155126571655273, + "objective/train/theoretical_loss": 4.053125349723942, + "objective/train/tokens_used": 365363200, + "objective/train/unlikelihood_loss": 0.17956046760082245, + "theoretical_loss": 4.053125349723942, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044912738214643934, + "loss": 1.4319, + "theoretical_loss": 4.053042779195217, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491073219658977, + "loss": 1.4881, + "theoretical_loss": 4.052877694986987, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044908726178535607, + "loss": 1.4094, + "theoretical_loss": 4.05271268652609, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490672016048145, + "loss": 1.671, + "theoretical_loss": 4.052547753750639, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044904714142427284, + "loss": 1.4686, + "theoretical_loss": 4.05238289659882, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490270812437312, + "loss": 1.3926, + "theoretical_loss": 4.052218115008892, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044900702106318957, + "loss": 1.5302, + "theoretical_loss": 4.0520534089191855, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044898696088264793, + "loss": 1.6902, + "theoretical_loss": 4.051888778268103, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044896690070210635, + "loss": 1.5773, + "theoretical_loss": 4.051724222994122, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489468405215647, + "loss": 1.4207, + "theoretical_loss": 4.051559743035789, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892678034102307, + "loss": 1.513, + "theoretical_loss": 4.051395338331723, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890672016048143, + "loss": 1.4494, + "theoretical_loss": 4.051231008820618, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.34115347266197205, + "objective/train/avg_token_score": 0.10597312450408936, + "objective/train/docs_used": 219467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.169438123703003, + "objective/train/original_loss": 5.253897190093994, + "objective/train/theoretical_loss": 4.051066754441235, + "objective/train/tokens_used": 367001600, + "objective/train/unlikelihood_loss": 0.0806141048669815, + "theoretical_loss": 4.051066754441235, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044888665997993985, + "loss": 1.3879, + "theoretical_loss": 4.051066754441235, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488665997993982, + "loss": 1.6292, + "theoretical_loss": 4.050902575132412, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488465396188566, + "loss": 1.5422, + "theoretical_loss": 4.050738470833054, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882647943831494, + "loss": 1.5571, + "theoretical_loss": 4.05057444148214, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488064192577733, + "loss": 1.5531, + "theoretical_loss": 4.05041048701872, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487863590772317, + "loss": 1.4186, + "theoretical_loss": 4.050246607381913, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487662988966901, + "loss": 1.6269, + "theoretical_loss": 4.050082802510915, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044874623871614844, + "loss": 1.5642, + "theoretical_loss": 4.049919072344987, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487261785356068, + "loss": 1.5353, + "theoretical_loss": 4.049755416823463, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487061183550652, + "loss": 1.4776, + "theoretical_loss": 4.0495918358857494, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486860581745236, + "loss": 1.465, + "theoretical_loss": 4.04942832947132, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866599799398194, + "loss": 1.5645, + "theoretical_loss": 4.049264897519723, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486459378134403, + "loss": 1.425, + "theoretical_loss": 4.049101539970576, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.47611191868782043, + "objective/train/avg_token_score": 0.012634911574423313, + "objective/train/docs_used": 220098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.645254135131836, + "objective/train/original_loss": 4.079015731811523, + "objective/train/theoretical_loss": 4.04901988907807, + "objective/train/tokens_used": 368640000, + "objective/train/unlikelihood_loss": 0.1734001785516739, + "theoretical_loss": 4.04901988907807, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044862587763289867, + "loss": 1.7137, + "theoretical_loss": 4.048938256763565, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486058174523571, + "loss": 1.6932, + "theoretical_loss": 4.048775047838447, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044858575727181545, + "loss": 1.705, + "theoretical_loss": 4.048611913135052, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485656970912738, + "loss": 1.7297, + "theoretical_loss": 4.048448852593277, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044854563691073217, + "loss": 1.7156, + "theoretical_loss": 4.0482858661530905, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485255767301906, + "loss": 1.4436, + "theoretical_loss": 4.04812295375453, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850551654964895, + "loss": 1.5983, + "theoretical_loss": 4.047960115337702, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484854563691073, + "loss": 1.5025, + "theoretical_loss": 4.047797350842786, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484653961885657, + "loss": 1.5012, + "theoretical_loss": 4.0476346602100275, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844533600802404, + "loss": 1.5321, + "theoretical_loss": 4.047472043379743, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842527582748245, + "loss": 1.3779, + "theoretical_loss": 4.04730950029232, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484052156469408, + "loss": 1.7209, + "theoretical_loss": 4.04714703088821, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4204545319080353, + "objective/train/avg_token_score": 0.013336265459656715, + "objective/train/docs_used": 221122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5654873847961426, + "objective/train/original_loss": 4.028966903686523, + "objective/train/theoretical_loss": 4.04698463510794, + "objective/train/tokens_used": 370278400, + "objective/train/unlikelihood_loss": 0.19616243243217468, + "theoretical_loss": 4.04698463510794, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483851554663992, + "loss": 1.3742, + "theoretical_loss": 4.04698463510794, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836509528585754, + "loss": 1.3853, + "theoretical_loss": 4.046822312892102, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834503510531596, + "loss": 1.5133, + "theoretical_loss": 4.046660064181359, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483249749247743, + "loss": 1.4851, + "theoretical_loss": 4.0464978889164405, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483049147442327, + "loss": 1.5003, + "theoretical_loss": 4.046335787038148, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828485456369104, + "loss": 1.6905, + "theoretical_loss": 4.046173758487347, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482647943831494, + "loss": 1.4056, + "theoretical_loss": 4.046011803204976, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482447342026078, + "loss": 1.4829, + "theoretical_loss": 4.045849921132042, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482246740220662, + "loss": 1.6773, + "theoretical_loss": 4.045688112209615, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044820461384152455, + "loss": 1.4661, + "theoretical_loss": 4.04552637637884, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818455366098296, + "loss": 1.6595, + "theoretical_loss": 4.045364713580923, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481644934804414, + "loss": 1.6206, + "theoretical_loss": 4.045203123757146, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814443329989974, + "loss": 1.6652, + "theoretical_loss": 4.0450416068488515, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4794110357761383, + "objective/train/avg_token_score": 0.004854011349380016, + "objective/train/docs_used": 221664, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7908248901367188, + "objective/train/original_loss": 3.7173187732696533, + "objective/train/theoretical_loss": 4.044960875719701, + "objective/train/tokens_used": 371916800, + "objective/train/unlikelihood_loss": 0.17050793766975403, + "theoretical_loss": 4.044960875719701, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481243731193581, + "loss": 1.493, + "theoretical_loss": 4.0448801627974555, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044810431293881647, + "loss": 1.5649, + "theoretical_loss": 4.044718791544438, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480842527582749, + "loss": 1.7985, + "theoretical_loss": 4.044557493031347, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806419257773324, + "loss": 1.5757, + "theoretical_loss": 4.0443962671998, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480441323971916, + "loss": 1.5035, + "theoretical_loss": 4.04423511399148, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802407221664997, + "loss": 1.5995, + "theoretical_loss": 4.044074033348139, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044800401203610833, + "loss": 1.7926, + "theoretical_loss": 4.0439130252115945, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044798395185556675, + "loss": 1.7443, + "theoretical_loss": 4.043752089523732, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479638916750251, + "loss": 1.6664, + "theoretical_loss": 4.043591226226503, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044794383149448347, + "loss": 1.55, + "theoretical_loss": 4.043430435261927, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044792377131394183, + "loss": 1.5638, + "theoretical_loss": 4.043269716572091, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044790371113340025, + "loss": 1.5155, + "theoretical_loss": 4.043109070099146, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5913978219032288, + "objective/train/avg_token_score": 0.00555779505521059, + "objective/train/docs_used": 222906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1588377952575684, + "objective/train/original_loss": 3.770258665084839, + "objective/train/theoretical_loss": 4.042948495785312, + "objective/train/tokens_used": 373555200, + "objective/train/unlikelihood_loss": 0.16115084290504456, + "theoretical_loss": 4.042948495785312, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478836509528586, + "loss": 1.5525, + "theoretical_loss": 4.042948495785312, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447863590772317, + "loss": 1.5479, + "theoretical_loss": 4.042787993572875, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044784353059177534, + "loss": 1.4116, + "theoretical_loss": 4.042627563404187, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478234704112337, + "loss": 1.5096, + "theoretical_loss": 4.042467205221665, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478034102306921, + "loss": 1.5331, + "theoretical_loss": 4.042306918967795, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477833500501505, + "loss": 1.6504, + "theoretical_loss": 4.042146704585127, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044776328986960884, + "loss": 1.6323, + "theoretical_loss": 4.041986562016278, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477432296890672, + "loss": 1.4255, + "theoretical_loss": 4.0418264912039295, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477231695085256, + "loss": 1.7105, + "theoretical_loss": 4.041666492090831, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447703109327984, + "loss": 1.4864, + "theoretical_loss": 4.041506564619796, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044768304914744234, + "loss": 1.5879, + "theoretical_loss": 4.041346708733704, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476629889669007, + "loss": 1.6726, + "theoretical_loss": 4.041186924375499, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044764292878635907, + "loss": 1.7443, + "theoretical_loss": 4.041027211488194, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.2611192464828491, + "objective/train/avg_token_score": 0.02251238189637661, + "objective/train/docs_used": 223164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0515928268432617, + "objective/train/original_loss": 4.479992389678955, + "objective/train/theoretical_loss": 4.040947381828337, + "objective/train/tokens_used": 375193600, + "objective/train/unlikelihood_loss": 0.1949537694454193, + "theoretical_loss": 4.040947381828337, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476228686058175, + "loss": 1.406, + "theoretical_loss": 4.040867570014862, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044760280842527585, + "loss": 1.4454, + "theoretical_loss": 4.040707999898647, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475827482447342, + "loss": 1.7935, + "theoretical_loss": 4.0405485010827515, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044756268806419257, + "loss": 1.5507, + "theoretical_loss": 4.040389073510449, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447542627883651, + "loss": 1.3859, + "theoretical_loss": 4.040229717125074, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044752256770310935, + "loss": 1.6392, + "theoretical_loss": 4.040070431870029, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475025075225677, + "loss": 1.5432, + "theoretical_loss": 4.039911217688777, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474824473420261, + "loss": 1.4319, + "theoretical_loss": 4.039752074524851, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044746238716148444, + "loss": 1.5912, + "theoretical_loss": 4.039593002321844, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044744232698094285, + "loss": 1.5599, + "theoretical_loss": 4.039434001023415, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474222668004012, + "loss": 1.3618, + "theoretical_loss": 4.039275070573288, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474022066198596, + "loss": 1.5952, + "theoretical_loss": 4.03911621091525, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4695136845111847, + "objective/train/avg_token_score": 0.009801458567380905, + "objective/train/docs_used": 224842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.732696771621704, + "objective/train/original_loss": 4.10999059677124, + "objective/train/theoretical_loss": 4.038957421993153, + "objective/train/tokens_used": 376832000, + "objective/train/unlikelihood_loss": 0.14230914413928986, + "theoretical_loss": 4.038957421993153, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044738214643931794, + "loss": 1.7543, + "theoretical_loss": 4.038957421993153, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044736208625877636, + "loss": 1.4096, + "theoretical_loss": 4.038798703750913, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473420260782347, + "loss": 1.6097, + "theoretical_loss": 4.038640056132509, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473219658976931, + "loss": 1.6665, + "theoretical_loss": 4.0384814790819865, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044730190571715144, + "loss": 1.5166, + "theoretical_loss": 4.038322972543451, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472818455366098, + "loss": 1.7482, + "theoretical_loss": 4.038164536461075, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472617853560682, + "loss": 1.5796, + "theoretical_loss": 4.038006170779091, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472417251755266, + "loss": 1.4874, + "theoretical_loss": 4.037847875441798, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044722166499498495, + "loss": 1.6282, + "theoretical_loss": 4.037689650393559, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472016048144433, + "loss": 1.6695, + "theoretical_loss": 4.037531495578797, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471815446339017, + "loss": 1.4349, + "theoretical_loss": 4.037373410942001, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471614844533601, + "loss": 1.7403, + "theoretical_loss": 4.037215396427721, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044714142427281845, + "loss": 1.5023, + "theoretical_loss": 4.037057451980571, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4757453501224518, + "objective/train/avg_token_score": 0.021771633997559547, + "objective/train/docs_used": 225570, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.804304599761963, + "objective/train/original_loss": 4.102111339569092, + "objective/train/theoretical_loss": 4.036978506014879, + "objective/train/tokens_used": 378470400, + "objective/train/unlikelihood_loss": 0.18979206681251526, + "theoretical_loss": 4.036978506014879, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471213640922768, + "loss": 1.7116, + "theoretical_loss": 4.036899577545229, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044710130391173523, + "loss": 1.6378, + "theoretical_loss": 4.036741773066434, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470812437311936, + "loss": 1.6524, + "theoretical_loss": 4.036584038488988, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044706118355065195, + "loss": 1.4253, + "theoretical_loss": 4.036426373757759, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470411233701103, + "loss": 1.6715, + "theoretical_loss": 4.0362687788176705, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470210631895687, + "loss": 1.8991, + "theoretical_loss": 4.036111253613715, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470010030090271, + "loss": 1.6512, + "theoretical_loss": 4.0359537980909455, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698094282848546, + "loss": 1.4178, + "theoretical_loss": 4.0357964121944745, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469608826479438, + "loss": 1.5667, + "theoretical_loss": 4.035639095869481, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469408224674022, + "loss": 1.5906, + "theoretical_loss": 4.0354818490612026, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469207622868606, + "loss": 1.5448, + "theoretical_loss": 4.035324671714942, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044690070210631896, + "loss": 1.6204, + "theoretical_loss": 4.03516756377606, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4055474102497101, + "objective/train/avg_token_score": 0.01529501099139452, + "objective/train/docs_used": 226798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4889705181121826, + "objective/train/original_loss": 3.754312515258789, + "objective/train/theoretical_loss": 4.035010525189982, + "objective/train/tokens_used": 380108800, + "objective/train/unlikelihood_loss": 0.2511163651943207, + "theoretical_loss": 4.035010525189982, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468806419257773, + "loss": 1.5714, + "theoretical_loss": 4.035010525189982, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468605817452357, + "loss": 1.3369, + "theoretical_loss": 4.0348535559021945, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684052156469405, + "loss": 1.5174, + "theoretical_loss": 4.034696655858246, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682046138415246, + "loss": 1.607, + "theoretical_loss": 4.034539825003746, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468004012036108, + "loss": 1.5884, + "theoretical_loss": 4.034383063284364, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467803410230692, + "loss": 1.5842, + "theoretical_loss": 4.0342263706458334, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676028084252755, + "loss": 1.6033, + "theoretical_loss": 4.034069747033948, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674022066198597, + "loss": 1.5395, + "theoretical_loss": 4.033913192394563, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672016048144433, + "loss": 1.7102, + "theoretical_loss": 4.033756706673593, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467001003009027, + "loss": 1.5634, + "theoretical_loss": 4.033600289817015, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466800401203611, + "loss": 1.4824, + "theoretical_loss": 4.033443941770867, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044665997993981947, + "loss": 1.6288, + "theoretical_loss": 4.033287662481247, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466399197592779, + "loss": 1.653, + "theoretical_loss": 4.033131451894317, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4856427311897278, + "objective/train/avg_token_score": 0.012365730479359627, + "objective/train/docs_used": 227456, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6744704246520996, + "objective/train/original_loss": 3.758188486099243, + "objective/train/theoretical_loss": 4.03305337234755, + "objective/train/tokens_used": 381747200, + "objective/train/unlikelihood_loss": 0.1711551696062088, + "theoretical_loss": 4.03305337234755, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044661985957873625, + "loss": 1.5343, + "theoretical_loss": 4.032975309956292, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465997993981946, + "loss": 1.5904, + "theoretical_loss": 4.032819236613457, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044657973921765297, + "loss": 1.4637, + "theoretical_loss": 4.03266323181215, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465596790371114, + "loss": 1.5051, + "theoretical_loss": 4.032507295498774, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044653961885656975, + "loss": 1.6082, + "theoretical_loss": 4.032351427619789, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465195586760281, + "loss": 1.5642, + "theoretical_loss": 4.032195628121718, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464994984954865, + "loss": 1.5348, + "theoretical_loss": 4.032039896951142, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044647943831494484, + "loss": 1.6252, + "theoretical_loss": 4.0318842340547025, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044645937813440325, + "loss": 1.4619, + "theoretical_loss": 4.031728639379103, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464393179538616, + "loss": 1.4193, + "theoretical_loss": 4.031573112871103, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044641925777332, + "loss": 1.7358, + "theoretical_loss": 4.031417654477526, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044639919759277834, + "loss": 1.6106, + "theoretical_loss": 4.03126226414525, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4290078282356262, + "objective/train/avg_token_score": 0.0078100161626935005, + "objective/train/docs_used": 228876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6035910844802856, + "objective/train/original_loss": 4.235310077667236, + "objective/train/theoretical_loss": 4.031106941821218, + "objective/train/tokens_used": 383385600, + "objective/train/unlikelihood_loss": 0.13287082314491272, + "theoretical_loss": 4.031106941821218, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044637913741223676, + "loss": 1.7171, + "theoretical_loss": 4.031106941821218, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463590772316951, + "loss": 1.6425, + "theoretical_loss": 4.030951687452429, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463390170511535, + "loss": 1.5361, + "theoretical_loss": 4.030796500985943, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044631895687061184, + "loss": 1.6445, + "theoretical_loss": 4.030641382368878, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462988966900702, + "loss": 1.5612, + "theoretical_loss": 4.030486331548413, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462788365095286, + "loss": 1.649, + "theoretical_loss": 4.030331348471784, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446258776328987, + "loss": 1.4784, + "theoretical_loss": 4.030176433086289, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044623871614844535, + "loss": 1.4671, + "theoretical_loss": 4.030021585339281, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462186559679037, + "loss": 1.7199, + "theoretical_loss": 4.029866805178175, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461985957873621, + "loss": 1.4296, + "theoretical_loss": 4.029712092550443, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461785356068205, + "loss": 1.4691, + "theoretical_loss": 4.029557447403618, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044615847542627885, + "loss": 1.4171, + "theoretical_loss": 4.029402869685288, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461384152457372, + "loss": 1.4194, + "theoretical_loss": 4.029248359343105, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.589442789554596, + "objective/train/avg_token_score": 0.008675182238221169, + "objective/train/docs_used": 229454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1350181102752686, + "objective/train/original_loss": 3.9059112071990967, + "objective/train/theoretical_loss": 4.029171129421724, + "objective/train/tokens_used": 385024000, + "objective/train/unlikelihood_loss": 0.1192452609539032, + "theoretical_loss": 4.029171129421724, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044611835506519563, + "loss": 1.8329, + "theoretical_loss": 4.029093916324774, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446098294884654, + "loss": 1.5253, + "theoretical_loss": 4.02893954057806, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044607823470411235, + "loss": 1.5488, + "theoretical_loss": 4.0287852320507875, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460581745235707, + "loss": 1.6576, + "theoretical_loss": 4.028630990690838, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460381143430291, + "loss": 1.5913, + "theoretical_loss": 4.028476816446153, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460180541624875, + "loss": 1.5977, + "theoretical_loss": 4.028322709264729, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044599799398194586, + "loss": 1.4783, + "theoretical_loss": 4.028168669094623, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459779338014042, + "loss": 1.3898, + "theoretical_loss": 4.0280146958839484, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459578736208626, + "loss": 1.3856, + "theoretical_loss": 4.027860789580878, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445937813440321, + "loss": 1.3978, + "theoretical_loss": 4.0277069501336396, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591775325977936, + "loss": 1.4399, + "theoretical_loss": 4.027553177490521, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458976930792377, + "loss": 1.5377, + "theoretical_loss": 4.027399471599867, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.48521506786346436, + "objective/train/avg_token_score": 0.010920585133135319, + "objective/train/docs_used": 230121, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8535081148147583, + "objective/train/original_loss": 4.084407329559326, + "objective/train/theoretical_loss": 4.027245832410079, + "objective/train/tokens_used": 386662400, + "objective/train/unlikelihood_loss": 0.12329123914241791, + "theoretical_loss": 4.027245832410079, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458776328986961, + "loss": 1.6056, + "theoretical_loss": 4.027245832410079, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585757271815445, + "loss": 1.4379, + "theoretical_loss": 4.027092259869616, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044583751253761286, + "loss": 1.6012, + "theoretical_loss": 4.026938753926997, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458174523570712, + "loss": 1.5904, + "theoretical_loss": 4.026785314530792, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457973921765296, + "loss": 1.5274, + "theoretical_loss": 4.026631941629635, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044577733199598795, + "loss": 1.6511, + "theoretical_loss": 4.026478635172212, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575727181544637, + "loss": 1.4566, + "theoretical_loss": 4.0263253951072695, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044573721163490473, + "loss": 1.3738, + "theoretical_loss": 4.026172221383607, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171514543631, + "loss": 1.3676, + "theoretical_loss": 4.026019113950086, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569709127382145, + "loss": 1.5872, + "theoretical_loss": 4.025866072755619, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456770310932798, + "loss": 1.509, + "theoretical_loss": 4.02571309774918, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565697091273823, + "loss": 1.4532, + "theoretical_loss": 4.025560188879796, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456369107321966, + "loss": 1.572, + "theoretical_loss": 4.025407346096551, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3402370512485504, + "objective/train/avg_token_score": 0.010277241468429565, + "objective/train/docs_used": 231531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1777230501174927, + "objective/train/original_loss": 4.140812397003174, + "objective/train/theoretical_loss": 4.025330949471336, + "objective/train/tokens_used": 388300800, + "objective/train/unlikelihood_loss": 0.12250532954931259, + "theoretical_loss": 4.025330949471336, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561685055165496, + "loss": 1.5369, + "theoretical_loss": 4.025254569348588, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455967903711133, + "loss": 1.5416, + "theoretical_loss": 4.025101858585104, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044557673019057173, + "loss": 1.5914, + "theoretical_loss": 4.0249492137553515, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455566700100301, + "loss": 1.3606, + "theoretical_loss": 4.0247966348086415, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553660982948846, + "loss": 1.5075, + "theoretical_loss": 4.02464412169434, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455165496489468, + "loss": 1.5384, + "theoretical_loss": 4.024491674361868, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454964894684052, + "loss": 1.4394, + "theoretical_loss": 4.024339292760703, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454764292878636, + "loss": 1.5151, + "theoretical_loss": 4.0241869768403795, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545636910732196, + "loss": 1.6464, + "theoretical_loss": 4.024034726550486, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454363089267803, + "loss": 1.5365, + "theoretical_loss": 4.023882541840667, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454162487462387, + "loss": 1.4798, + "theoretical_loss": 4.023730422660624, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453961885656971, + "loss": 1.4354, + "theoretical_loss": 4.023578368960113, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.34365835785865784, + "objective/train/avg_token_score": 0.01555408164858818, + "objective/train/docs_used": 232298, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2046136856079102, + "objective/train/original_loss": 3.8979644775390625, + "objective/train/theoretical_loss": 4.023426380688943, + "objective/train/tokens_used": 389939200, + "objective/train/unlikelihood_loss": 0.1804252713918686, + "theoretical_loss": 4.023426380688943, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537612838515547, + "loss": 1.3799, + "theoretical_loss": 4.023426380688943, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044535606820461383, + "loss": 1.6791, + "theoretical_loss": 4.023274457796983, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453360080240722, + "loss": 1.45, + "theoretical_loss": 4.023122600234155, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531594784353055, + "loss": 1.5732, + "theoretical_loss": 4.022970807950434, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044529588766298897, + "loss": 1.5985, + "theoretical_loss": 4.022819080895853, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527582748244733, + "loss": 1.5921, + "theoretical_loss": 4.022667419020499, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452557673019057, + "loss": 1.5946, + "theoretical_loss": 4.022515822274515, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044523570712136406, + "loss": 1.5798, + "theoretical_loss": 4.022364290608098, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521564694082247, + "loss": 1.4749, + "theoretical_loss": 4.022212823971498, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044519558676028083, + "loss": 1.7443, + "theoretical_loss": 4.022061422315023, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517552657973925, + "loss": 1.5852, + "theoretical_loss": 4.021910085589032, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451554663991976, + "loss": 1.6875, + "theoretical_loss": 4.021758813743942, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044513540621865603, + "loss": 1.4254, + "theoretical_loss": 4.021607606730223, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3057795763015747, + "objective/train/avg_token_score": 0.016242092475295067, + "objective/train/docs_used": 233436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1886520385742188, + "objective/train/original_loss": 4.423647880554199, + "objective/train/theoretical_loss": 4.0215320275196635, + "objective/train/tokens_used": 391577600, + "objective/train/unlikelihood_loss": 0.1317605823278427, + "theoretical_loss": 4.0215320275196635, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451153460381144, + "loss": 1.3848, + "theoretical_loss": 4.0214564644983986, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044509528585757275, + "loss": 1.7228, + "theoretical_loss": 4.021305386999048, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450752256770311, + "loss": 1.475, + "theoretical_loss": 4.021154374182803, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450551654964895, + "loss": 1.4708, + "theoretical_loss": 4.021003426000353, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450351053159479, + "loss": 1.7547, + "theoretical_loss": 4.0208525424024355, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044501504513540626, + "loss": 1.5899, + "theoretical_loss": 4.020701723339848, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449949849548646, + "loss": 1.4629, + "theoretical_loss": 4.020550968763439, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444974924774323, + "loss": 1.5781, + "theoretical_loss": 4.020400278624112, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449548645937814, + "loss": 1.3149, + "theoretical_loss": 4.020249652872822, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044493480441323976, + "loss": 1.7288, + "theoretical_loss": 4.02009909146058, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449147442326981, + "loss": 1.6467, + "theoretical_loss": 4.01994859433845, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448946840521565, + "loss": 1.7551, + "theoretical_loss": 4.019798161457548, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4335899353027344, + "objective/train/avg_token_score": 0.016622694209218025, + "objective/train/docs_used": 234056, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6204777956008911, + "objective/train/original_loss": 3.885645866394043, + "objective/train/theoretical_loss": 4.019647792769048, + "objective/train/tokens_used": 393216000, + "objective/train/unlikelihood_loss": 0.16480211913585663, + "theoretical_loss": 4.019647792769048, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044487462387161485, + "loss": 1.5896, + "theoretical_loss": 4.019647792769048, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044485456369107326, + "loss": 1.5453, + "theoretical_loss": 4.01949748822417, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448345035105316, + "loss": 1.4782, + "theoretical_loss": 4.019347247774195, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044481444332999, + "loss": 1.4937, + "theoretical_loss": 4.019197071370451, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044479438314944835, + "loss": 1.438, + "theoretical_loss": 4.019046958964324, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044477432296890677, + "loss": 1.3938, + "theoretical_loss": 4.01889691050725, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044475426278836513, + "loss": 1.7109, + "theoretical_loss": 4.018746925950719, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447342026078235, + "loss": 1.3604, + "theoretical_loss": 4.018597005246273, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044471414242728185, + "loss": 1.6068, + "theoretical_loss": 4.01844714834551, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446940822467402, + "loss": 1.5331, + "theoretical_loss": 4.018297355200076, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044467402206619863, + "loss": 1.465, + "theoretical_loss": 4.018147625761673, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444653961885657, + "loss": 1.5658, + "theoretical_loss": 4.017997959982055, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044463390170511536, + "loss": 1.7217, + "theoretical_loss": 4.017848357813028, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.45723363757133484, + "objective/train/avg_token_score": 0.017939608544111252, + "objective/train/docs_used": 235431, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7509286403656006, + "objective/train/original_loss": 4.263587474822998, + "objective/train/theoretical_loss": 4.01777358056744, + "objective/train/tokens_used": 394854400, + "objective/train/unlikelihood_loss": 0.13543236255645752, + "theoretical_loss": 4.01777358056744, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446138415245737, + "loss": 1.635, + "theoretical_loss": 4.017698819206451, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044459378134403214, + "loss": 1.5588, + "theoretical_loss": 4.017549344114236, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445737211634905, + "loss": 1.5377, + "theoretical_loss": 4.0173999324883445, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044455366098294886, + "loss": 1.5516, + "theoretical_loss": 4.017250584280794, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445336008024072, + "loss": 1.5806, + "theoretical_loss": 4.0171012994436515, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445135406218656, + "loss": 1.574, + "theoretical_loss": 4.016952077929037, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444493480441324, + "loss": 1.7532, + "theoretical_loss": 4.016802919689124, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044447342026078236, + "loss": 1.4473, + "theoretical_loss": 4.016653824676135, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444533600802407, + "loss": 1.5449, + "theoretical_loss": 4.016504792842346, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444332998996991, + "loss": 1.7389, + "theoretical_loss": 4.016355824140085, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444132397191575, + "loss": 1.6257, + "theoretical_loss": 4.016206918521731, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044439317953861587, + "loss": 1.5358, + "theoretical_loss": 4.016058075939716, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3450024425983429, + "objective/train/avg_token_score": 0.024041682481765747, + "objective/train/docs_used": 235840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.240441083908081, + "objective/train/original_loss": 3.7224650382995605, + "objective/train/theoretical_loss": 4.015909296346521, + "objective/train/tokens_used": 396492800, + "objective/train/unlikelihood_loss": 0.21158063411712646, + "theoretical_loss": 4.015909296346521, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044437311935807423, + "loss": 1.5572, + "theoretical_loss": 4.015909296346521, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443530591775326, + "loss": 1.5529, + "theoretical_loss": 4.015760579694683, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044433299899699095, + "loss": 1.5808, + "theoretical_loss": 4.015611925936785, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044431293881644937, + "loss": 1.449, + "theoretical_loss": 4.015463335025465, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044429287863590773, + "loss": 1.3055, + "theoretical_loss": 4.015314806913411, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442728184553661, + "loss": 1.3535, + "theoretical_loss": 4.015166341553362, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044425275827482446, + "loss": 1.536, + "theoretical_loss": 4.01501793889811, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044423269809428287, + "loss": 1.4453, + "theoretical_loss": 4.014869598900496, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044421263791374123, + "loss": 1.6041, + "theoretical_loss": 4.014721321513412, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441925777331996, + "loss": 1.6268, + "theoretical_loss": 4.014573106689803, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044417251755265796, + "loss": 1.5815, + "theoretical_loss": 4.014424954382663, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441524573721163, + "loss": 1.4915, + "theoretical_loss": 4.014276864545036, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044413239719157474, + "loss": 1.5733, + "theoretical_loss": 4.014128837130021, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.42839688062667847, + "objective/train/avg_token_score": 0.010799635201692581, + "objective/train/docs_used": 236468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.580099105834961, + "objective/train/original_loss": 3.822235107421875, + "objective/train/theoretical_loss": 4.0140548468163475, + "objective/train/tokens_used": 398131200, + "objective/train/unlikelihood_loss": 0.15448230504989624, + "theoretical_loss": 4.0140548468163475, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441123370110331, + "loss": 1.613, + "theoretical_loss": 4.0139808720907615, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044409227683049146, + "loss": 1.4741, + "theoretical_loss": 4.013832969380456, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440722166499498, + "loss": 1.4448, + "theoretical_loss": 4.013685128952353, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044405215646940824, + "loss": 1.3947, + "theoretical_loss": 4.01353735075975, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440320962888666, + "loss": 1.5525, + "theoretical_loss": 4.013389634755995, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044401203610832497, + "loss": 1.5298, + "theoretical_loss": 4.013241980894488, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044399197592778333, + "loss": 1.4135, + "theoretical_loss": 4.013094389128677, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439719157472417, + "loss": 1.5502, + "theoretical_loss": 4.012946859412062, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439518555667001, + "loss": 1.5869, + "theoretical_loss": 4.012799391698191, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044393179538615847, + "loss": 1.4827, + "theoretical_loss": 4.012651985940666, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391173520561683, + "loss": 1.6088, + "theoretical_loss": 4.012504642093133, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438916750250752, + "loss": 1.4438, + "theoretical_loss": 4.012357360109293, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4299853444099426, + "objective/train/avg_token_score": 0.011655104346573353, + "objective/train/docs_used": 237759, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6880359649658203, + "objective/train/original_loss": 3.903928279876709, + "objective/train/theoretical_loss": 4.012210139942894, + "objective/train/tokens_used": 399769600, + "objective/train/unlikelihood_loss": 0.17748136818408966, + "theoretical_loss": 4.012210139942894, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438716148445336, + "loss": 1.6091, + "theoretical_loss": 4.012210139942894, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385155466399197, + "loss": 1.6209, + "theoretical_loss": 4.012062981547736, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044383149448345033, + "loss": 1.6198, + "theoretical_loss": 4.0119158848776655, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438114343029087, + "loss": 1.3378, + "theoretical_loss": 4.011768849886582, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437913741223671, + "loss": 1.6247, + "theoretical_loss": 4.011621876528431, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437713139418255, + "loss": 1.7281, + "theoretical_loss": 4.011474964757211, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375125376128384, + "loss": 1.477, + "theoretical_loss": 4.011328114526968, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437311935807422, + "loss": 1.6307, + "theoretical_loss": 4.011181325791797, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044371113340020056, + "loss": 1.3736, + "theoretical_loss": 4.011034598505843, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369107321965903, + "loss": 1.4352, + "theoretical_loss": 4.010887932623299, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436710130391174, + "loss": 1.6627, + "theoretical_loss": 4.01074132809841, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365095285857576, + "loss": 1.6658, + "theoretical_loss": 4.010594784885466, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436308926780341, + "loss": 1.4219, + "theoretical_loss": 4.01044830293881, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.31763195991516113, + "objective/train/avg_token_score": 0.008867236785590649, + "objective/train/docs_used": 238664, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1373664140701294, + "objective/train/original_loss": 3.9792494773864746, + "objective/train/theoretical_loss": 4.0103750849260855, + "objective/train/tokens_used": 401408000, + "objective/train/unlikelihood_loss": 0.17361101508140564, + "theoretical_loss": 4.0103750849260855, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044361083249749254, + "loss": 1.4158, + "theoretical_loss": 4.010301882212831, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435907723169509, + "loss": 1.6034, + "theoretical_loss": 4.010155522661968, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044357071213640926, + "loss": 1.6155, + "theoretical_loss": 4.010009224240708, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435506519558676, + "loss": 1.7323, + "theoretical_loss": 4.009862986903588, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443530591775326, + "loss": 1.5855, + "theoretical_loss": 4.009716810605193, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435105315947844, + "loss": 1.481, + "theoretical_loss": 4.009570695300155, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044349047141424276, + "loss": 1.5404, + "theoretical_loss": 4.009424640943157, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434704112337011, + "loss": 1.5359, + "theoretical_loss": 4.00927864748893, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434503510531595, + "loss": 1.5011, + "theoretical_loss": 4.009132714892251, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434302908726179, + "loss": 1.6215, + "theoretical_loss": 4.008986843107949, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341023069207627, + "loss": 1.5534, + "theoretical_loss": 4.008841032090898, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044339017051153463, + "loss": 1.4099, + "theoretical_loss": 4.0086952817960215, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4825879633426666, + "objective/train/avg_token_score": 0.010442715138196945, + "objective/train/docs_used": 240054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6706886291503906, + "objective/train/original_loss": 3.7450501918792725, + "objective/train/theoretical_loss": 4.008549592178291, + "objective/train/tokens_used": 403046400, + "objective/train/unlikelihood_loss": 0.15135115385055542, + "theoretical_loss": 4.008549592178291, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443370110330993, + "loss": 1.4657, + "theoretical_loss": 4.008549592178291, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335005015045135, + "loss": 1.4444, + "theoretical_loss": 4.008403963192725, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044332998996990977, + "loss": 1.6658, + "theoretical_loss": 4.008258394794392, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044330992978936813, + "loss": 1.5749, + "theoretical_loss": 4.008112886938409, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432898696088265, + "loss": 1.4549, + "theoretical_loss": 4.007967439579936, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044326980942828486, + "loss": 1.3103, + "theoretical_loss": 4.007822052674185, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044324974924774327, + "loss": 1.6347, + "theoretical_loss": 4.007676726176415, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044322968906720163, + "loss": 1.3592, + "theoretical_loss": 4.007531460041933, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044320962888666, + "loss": 1.4303, + "theoretical_loss": 4.00738625422609, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044318956870611836, + "loss": 1.2863, + "theoretical_loss": 4.0072411086842905, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431695085255767, + "loss": 1.4716, + "theoretical_loss": 4.007096023371981, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044314944834503514, + "loss": 1.4291, + "theoretical_loss": 4.006950998244658, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431293881644935, + "loss": 1.5443, + "theoretical_loss": 4.006806033257866, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.4843597114086151, + "objective/train/avg_token_score": 0.03304527327418327, + "objective/train/docs_used": 240788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6086971759796143, + "objective/train/original_loss": 3.774994134902954, + "objective/train/theoretical_loss": 4.006733573303289, + "objective/train/tokens_used": 404684800, + "objective/train/unlikelihood_loss": 0.15561363101005554, + "theoretical_loss": 4.006733573303289, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044310932798395186, + "loss": 1.4442, + "theoretical_loss": 4.0066611283671945, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430892678034102, + "loss": 1.498, + "theoretical_loss": 4.00651628352828, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044306920762286864, + "loss": 1.5164, + "theoretical_loss": 4.00637149869681, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443049147442327, + "loss": 1.468, + "theoretical_loss": 4.006226773828514, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044302908726178537, + "loss": 1.517, + "theoretical_loss": 4.0060821088791725, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044300902708124373, + "loss": 1.6745, + "theoretical_loss": 4.00593750380461, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429889669007021, + "loss": 1.6433, + "theoretical_loss": 4.005792958560699, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429689067201605, + "loss": 1.5431, + "theoretical_loss": 4.00564847310336, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294884653961887, + "loss": 1.4441, + "theoretical_loss": 4.005504047388558, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292878635907723, + "loss": 1.4946, + "theoretical_loss": 4.005359681372306, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429087261785356, + "loss": 1.5999, + "theoretical_loss": 4.005215375010662, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442888665997994, + "loss": 1.5072, + "theoretical_loss": 4.005071128259734, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.388868510723114, + "objective/train/avg_token_score": 0.012570223771035671, + "objective/train/docs_used": 241796, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4023349285125732, + "objective/train/original_loss": 3.9673821926116943, + "objective/train/theoretical_loss": 4.004926941075674, + "objective/train/tokens_used": 406323200, + "objective/train/unlikelihood_loss": 0.16696441173553467, + "theoretical_loss": 4.004926941075674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044286860581745237, + "loss": 1.536, + "theoretical_loss": 4.004926941075674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284854563691073, + "loss": 1.482, + "theoretical_loss": 4.004782813414678, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428284854563691, + "loss": 1.5833, + "theoretical_loss": 4.0046387452329935, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428084252758275, + "loss": 1.5516, + "theoretical_loss": 4.00449473648691, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427883650952859, + "loss": 1.5163, + "theoretical_loss": 4.004350787132766, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044276830491474424, + "loss": 1.4299, + "theoretical_loss": 4.004206897126943, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427482447342026, + "loss": 1.3877, + "theoretical_loss": 4.004063066425874, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272818455366096, + "loss": 1.5866, + "theoretical_loss": 4.003919294986032, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427081243731194, + "loss": 1.3976, + "theoretical_loss": 4.003775582763939, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044268806419257774, + "loss": 1.4836, + "theoretical_loss": 4.003631929716162, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426680040120361, + "loss": 1.6081, + "theoretical_loss": 4.0034883357993145, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264794383149447, + "loss": 1.5082, + "theoretical_loss": 4.003344800970056, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426278836509529, + "loss": 1.6963, + "theoretical_loss": 4.00320132518509, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3863636255264282, + "objective/train/avg_token_score": 0.017943046987056732, + "objective/train/docs_used": 242638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4344366788864136, + "objective/train/original_loss": 4.310576915740967, + "objective/train/theoretical_loss": 4.0031296094207, + "objective/train/tokens_used": 407961600, + "objective/train/unlikelihood_loss": 0.1840338259935379, + "theoretical_loss": 4.0031296094207, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044260782347041124, + "loss": 1.5529, + "theoretical_loss": 4.003057908401168, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425877632898696, + "loss": 1.5715, + "theoretical_loss": 4.002914550575085, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256770310932797, + "loss": 1.5628, + "theoretical_loss": 4.002771251663681, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044254764292878633, + "loss": 1.4782, + "theoretical_loss": 4.0026280116238455, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252758274824475, + "loss": 1.6979, + "theoretical_loss": 4.0024848304125085, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425075225677031, + "loss": 1.4791, + "theoretical_loss": 4.002341707986648, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044248746238716147, + "loss": 1.6751, + "theoretical_loss": 4.002198644303288, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246740220661983, + "loss": 1.5135, + "theoretical_loss": 4.002055639319494, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044244734202607825, + "loss": 1.6368, + "theoretical_loss": 4.001912692992379, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424272818455366, + "loss": 1.5111, + "theoretical_loss": 4.001769805279103, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442407221664995, + "loss": 1.4949, + "theoretical_loss": 4.001626976136868, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044238716148445334, + "loss": 1.5723, + "theoretical_loss": 4.001484205522922, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.42625856399536133, + "objective/train/avg_token_score": 0.018668072298169136, + "objective/train/docs_used": 244086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4982153177261353, + "objective/train/original_loss": 3.7382147312164307, + "objective/train/theoretical_loss": 4.001341493394558, + "objective/train/tokens_used": 409600000, + "objective/train/unlikelihood_loss": 0.17688016593456268, + "theoretical_loss": 4.001341493394558, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423671013039117, + "loss": 1.6406, + "theoretical_loss": 4.001341493394558, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423470411233701, + "loss": 1.59, + "theoretical_loss": 4.001198839709113, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423269809428285, + "loss": 1.439, + "theoretical_loss": 4.001056244423971, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230692076228684, + "loss": 1.5345, + "theoretical_loss": 4.000913707496556, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422868605817452, + "loss": 1.5954, + "theoretical_loss": 4.000771228884344, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422668004012036, + "loss": 1.6115, + "theoretical_loss": 4.000628808544848, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442246740220662, + "loss": 1.4791, + "theoretical_loss": 4.000486446435629, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222668004012034, + "loss": 1.4222, + "theoretical_loss": 4.000344142514294, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422066198595787, + "loss": 1.3933, + "theoretical_loss": 4.000201896738492, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421865596790371, + "loss": 1.6559, + "theoretical_loss": 4.0000597090659165, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216649949849554, + "loss": 1.5433, + "theoretical_loss": 3.9999175794543063, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421464393179539, + "loss": 1.4679, + "theoretical_loss": 3.999775507861443, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212637913741226, + "loss": 1.3713, + "theoretical_loss": 3.9996334942451544, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.39326736330986023, + "objective/train/avg_token_score": 0.010529226623475552, + "objective/train/docs_used": 244630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.416571021080017, + "objective/train/original_loss": 3.8771302700042725, + "objective/train/theoretical_loss": 3.999562509165058, + "objective/train/tokens_used": 411238400, + "objective/train/unlikelihood_loss": 0.1982622891664505, + "theoretical_loss": 3.999562509165058, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421063189568706, + "loss": 1.6983, + "theoretical_loss": 3.9994915385633103, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044208625877632904, + "loss": 1.5469, + "theoretical_loss": 3.9993496407738256, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420661985957874, + "loss": 1.3864, + "theoretical_loss": 3.999207800834659, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044204613841524577, + "loss": 1.5093, + "theoretical_loss": 3.999066018703814, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202607823470413, + "loss": 1.617, + "theoretical_loss": 3.9989242943393357, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420060180541625, + "loss": 1.5732, + "theoretical_loss": 3.998782627699315, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419859578736209, + "loss": 1.657, + "theoretical_loss": 3.9986410187418855, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044196589769307927, + "loss": 1.5478, + "theoretical_loss": 3.9984994674252254, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044194583751253763, + "loss": 1.4761, + "theoretical_loss": 3.9983579737075554, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441925777331996, + "loss": 1.4249, + "theoretical_loss": 3.9982165375471403, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419057171514544, + "loss": 1.5888, + "theoretical_loss": 3.9980751589022887, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044188565697091277, + "loss": 1.4857, + "theoretical_loss": 3.9979338377313525, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4978005886077881, + "objective/train/avg_token_score": 0.016746150329709053, + "objective/train/docs_used": 245807, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8077481985092163, + "objective/train/original_loss": 4.005298614501953, + "objective/train/theoretical_loss": 3.997792573992726, + "objective/train/tokens_used": 412876800, + "objective/train/unlikelihood_loss": 0.1058061420917511, + "theoretical_loss": 3.997792573992726, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044186559679037113, + "loss": 1.5612, + "theoretical_loss": 3.997792573992726, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418455366098295, + "loss": 1.3567, + "theoretical_loss": 3.997651367644848, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418254764292879, + "loss": 1.674, + "theoretical_loss": 3.9975102186462004, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418054162487463, + "loss": 1.4358, + "theoretical_loss": 3.9973691269553076, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044178535606820464, + "loss": 1.4689, + "theoretical_loss": 3.9972280925307384, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441765295887663, + "loss": 1.6083, + "theoretical_loss": 3.9970871153311025, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044174523570712136, + "loss": 1.434, + "theoretical_loss": 3.996946195315055, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417251755265798, + "loss": 1.6717, + "theoretical_loss": 3.996805332441293, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044170511534603814, + "loss": 1.5077, + "theoretical_loss": 3.996664526668555, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416850551654965, + "loss": 1.5328, + "theoretical_loss": 3.996523777955626, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044166499498495487, + "loss": 1.4311, + "theoretical_loss": 3.9963830862613303, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416449348044133, + "loss": 1.5412, + "theoretical_loss": 3.9962424515445356, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044162487462387164, + "loss": 1.4443, + "theoretical_loss": 3.9961018737641547, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4602883756160736, + "objective/train/avg_token_score": 0.015043980441987514, + "objective/train/docs_used": 246540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.704790472984314, + "objective/train/original_loss": 4.011831283569336, + "objective/train/theoretical_loss": 3.99603160621229, + "objective/train/tokens_used": 414515200, + "objective/train/unlikelihood_loss": 0.13677598536014557, + "theoretical_loss": 3.99603160621229, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044160481444333, + "loss": 1.4581, + "theoretical_loss": 3.9959613528791396, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044158475426278837, + "loss": 1.4903, + "theoretical_loss": 3.9958208888484865, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044156469408224673, + "loss": 1.5552, + "theoretical_loss": 3.9956804816312355, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044154463390170515, + "loss": 1.5021, + "theoretical_loss": 3.9955401311864662, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415245737211635, + "loss": 1.5002, + "theoretical_loss": 3.995399837473302, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044150451354062187, + "loss": 1.5115, + "theoretical_loss": 3.99525960045091, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044148445336008023, + "loss": 1.5434, + "theoretical_loss": 3.9951194200784976, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044146439317953865, + "loss": 1.5491, + "theoretical_loss": 3.9949792963153143, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441444332998997, + "loss": 1.5161, + "theoretical_loss": 3.994839229120654, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414242728184554, + "loss": 1.5123, + "theoretical_loss": 3.99469921845385, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044140421263791374, + "loss": 1.4513, + "theoretical_loss": 3.9945592642742795, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413841524573721, + "loss": 1.5792, + "theoretical_loss": 3.9944193665413605, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.38825756311416626, + "objective/train/avg_token_score": 0.025925040245056152, + "objective/train/docs_used": 247824, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4125205278396606, + "objective/train/original_loss": 3.7366883754730225, + "objective/train/theoretical_loss": 3.994279525214554, + "objective/train/tokens_used": 416153600, + "objective/train/unlikelihood_loss": 0.2123020440340042, + "theoretical_loss": 3.994279525214554, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413640922768305, + "loss": 1.4642, + "theoretical_loss": 3.994279525214554, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413440320962889, + "loss": 1.6119, + "theoretical_loss": 3.994139740253363, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044132397191574724, + "loss": 1.4493, + "theoretical_loss": 3.99400001161733, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413039117352056, + "loss": 1.4857, + "theoretical_loss": 3.9938603392660426, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441283851554664, + "loss": 1.5087, + "theoretical_loss": 3.993720723159127, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412637913741224, + "loss": 1.5877, + "theoretical_loss": 3.9935811632562532, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044124373119358074, + "loss": 1.5386, + "theoretical_loss": 3.9934416595171323, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412236710130391, + "loss": 1.5819, + "theoretical_loss": 3.993302211901516, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044120361083249747, + "loss": 1.5589, + "theoretical_loss": 3.993162820369199, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411835506519559, + "loss": 1.5473, + "theoretical_loss": 3.993023484880016, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044116349047141425, + "loss": 1.5015, + "theoretical_loss": 3.9928842053938443, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411434302908726, + "loss": 1.3862, + "theoretical_loss": 3.9927449818706005, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044112337011033097, + "loss": 1.5006, + "theoretical_loss": 3.9926058142702456, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4471529722213745, + "objective/train/avg_token_score": 0.010523061268031597, + "objective/train/docs_used": 248559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6317007541656494, + "objective/train/original_loss": 3.7148232460021973, + "objective/train/theoretical_loss": 3.9925362514286498, + "objective/train/tokens_used": 417792000, + "objective/train/unlikelihood_loss": 0.19979330897331238, + "theoretical_loss": 3.9925362514286498, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411033099297894, + "loss": 1.4831, + "theoretical_loss": 3.992466702552779, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044108324974924775, + "loss": 1.5975, + "theoretical_loss": 3.9923276466782425, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410631895687061, + "loss": 1.6912, + "theoretical_loss": 3.992188646606719, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410431293881645, + "loss": 1.4185, + "theoretical_loss": 3.992049702298332, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044102306920762284, + "loss": 1.4789, + "theoretical_loss": 3.991910813713246, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044100300902708125, + "loss": 1.4325, + "theoretical_loss": 3.991771980811667, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409829488465396, + "loss": 1.6477, + "theoretical_loss": 3.9916332035538415, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440962888665998, + "loss": 1.6004, + "theoretical_loss": 3.991494481900056, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094282848545634, + "loss": 1.4793, + "theoretical_loss": 3.99135581581064, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092276830491476, + "loss": 1.5306, + "theoretical_loss": 3.9912172052459614, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409027081243731, + "loss": 1.6564, + "theoretical_loss": 3.991078650166429, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408826479438315, + "loss": 1.5377, + "theoretical_loss": 3.990940150532494, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3536168038845062, + "objective/train/avg_token_score": 0.01034875400364399, + "objective/train/docs_used": 249785, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2462135553359985, + "objective/train/original_loss": 4.00803804397583, + "objective/train/theoretical_loss": 3.990801706304647, + "objective/train/tokens_used": 419430400, + "objective/train/unlikelihood_loss": 0.17689494788646698, + "theoretical_loss": 3.990801706304647, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086258776328984, + "loss": 1.5763, + "theoretical_loss": 3.990801706304647, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408425275827482, + "loss": 1.4898, + "theoretical_loss": 3.990663317443418, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408224674022066, + "loss": 1.5875, + "theoretical_loss": 3.9905249839093786, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440802407221665, + "loss": 1.5456, + "theoretical_loss": 3.990386705663142, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044078234704112335, + "loss": 1.6324, + "theoretical_loss": 3.990248482665359, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407622868605817, + "loss": 1.6974, + "theoretical_loss": 3.9901103148767225, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407422266800401, + "loss": 1.5273, + "theoretical_loss": 3.989972202257966, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407221664994985, + "loss": 1.321, + "theoretical_loss": 3.989834144769861, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044070210631895685, + "loss": 1.5629, + "theoretical_loss": 3.989696142373221, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068204613841527, + "loss": 1.6409, + "theoretical_loss": 3.9895581950288994, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406619859578737, + "loss": 1.4717, + "theoretical_loss": 3.989420302697789, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064192577733204, + "loss": 1.5218, + "theoretical_loss": 3.989282465340823, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406218655967904, + "loss": 1.4617, + "theoretical_loss": 3.9891446829189734, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4630987346172333, + "objective/train/avg_token_score": 0.02803061157464981, + "objective/train/docs_used": 250392, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7076679468154907, + "objective/train/original_loss": 4.153679370880127, + "objective/train/theoretical_loss": 3.989075812296533, + "objective/train/tokens_used": 421068800, + "objective/train/unlikelihood_loss": 0.13347437977790833, + "theoretical_loss": 3.989075812296533, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044060180541624877, + "loss": 1.6244, + "theoretical_loss": 3.989006955393254, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044058174523570713, + "loss": 1.7096, + "theoretical_loss": 3.9888692827247167, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044056168505516555, + "loss": 1.4498, + "theoretical_loss": 3.988731664874454, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405416248746239, + "loss": 1.3687, + "theoretical_loss": 3.988594101803598, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052156469408227, + "loss": 1.687, + "theoretical_loss": 3.9884565934733196, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044050150451354063, + "loss": 1.444, + "theoretical_loss": 3.9883191398448306, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044048144433299905, + "loss": 1.3943, + "theoretical_loss": 3.988181740879382, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404613841524574, + "loss": 1.4322, + "theoretical_loss": 3.988044396538263, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404413239719158, + "loss": 1.5675, + "theoretical_loss": 3.9879071067828034, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042126379137414, + "loss": 1.5548, + "theoretical_loss": 3.987769871574373, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404012036108325, + "loss": 1.4586, + "theoretical_loss": 3.987632690874379, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403811434302909, + "loss": 1.6742, + "theoretical_loss": 3.98749556464427, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3960777223110199, + "objective/train/avg_token_score": 0.019322294741868973, + "objective/train/docs_used": 251199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4102435111999512, + "objective/train/original_loss": 4.012388706207275, + "objective/train/theoretical_loss": 3.987358492845532, + "objective/train/tokens_used": 422707200, + "objective/train/unlikelihood_loss": 0.17995640635490417, + "theoretical_loss": 3.987358492845532, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403610832497493, + "loss": 1.4228, + "theoretical_loss": 3.987358492845532, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034102306920764, + "loss": 1.4906, + "theoretical_loss": 3.987221475439691, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440320962888666, + "loss": 1.3949, + "theoretical_loss": 3.9870845123883116, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403009027081244, + "loss": 1.3426, + "theoretical_loss": 3.9869476036529994, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402808425275828, + "loss": 1.562, + "theoretical_loss": 3.986810749195396, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026078234704114, + "loss": 1.8139, + "theoretical_loss": 3.9866739489771845, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402407221664995, + "loss": 1.6235, + "theoretical_loss": 3.986537202960085, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044022066198595787, + "loss": 1.6967, + "theoretical_loss": 3.986400511105858, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402006018054163, + "loss": 1.3629, + "theoretical_loss": 3.986263873376301, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044018054162487465, + "loss": 1.6447, + "theoretical_loss": 3.9861272897332527, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440160481444333, + "loss": 1.5283, + "theoretical_loss": 3.9859907601385887, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014042126379137, + "loss": 1.4187, + "theoretical_loss": 3.985854284554223, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401203610832498, + "loss": 1.6492, + "theoretical_loss": 3.98571786294211, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4866202473640442, + "objective/train/avg_token_score": 0.008693613111972809, + "objective/train/docs_used": 252247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6004599332809448, + "objective/train/original_loss": 3.3173439502716064, + "objective/train/theoretical_loss": 3.9856496723637687, + "objective/train/tokens_used": 424345600, + "objective/train/unlikelihood_loss": 0.19723816215991974, + "theoretical_loss": 3.9856496723637687, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044010030090270815, + "loss": 1.6577, + "theoretical_loss": 3.985581495264241, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400802407221665, + "loss": 1.5225, + "theoretical_loss": 3.9854451814826457, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400601805416249, + "loss": 1.4646, + "theoretical_loss": 3.985308921559394, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004012036108324, + "loss": 1.4294, + "theoretical_loss": 3.9851727154565926, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002006018054165, + "loss": 1.6139, + "theoretical_loss": 3.9850365631363873, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 1.5823, + "theoretical_loss": 3.984900464560961, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399799398194584, + "loss": 1.6172, + "theoretical_loss": 3.984764419692537, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995987963891674, + "loss": 1.6824, + "theoretical_loss": 3.984628428493374, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043993981945837516, + "loss": 1.5701, + "theoretical_loss": 3.984492490925772, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399197592778335, + "loss": 1.5669, + "theoretical_loss": 3.9843566069520664, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398996990972919, + "loss": 1.3379, + "theoretical_loss": 3.984220776534632, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987963891675024, + "loss": 1.6932, + "theoretical_loss": 3.9840849996358814, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.35037878155708313, + "objective/train/avg_token_score": 0.010006745345890522, + "objective/train/docs_used": 253004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2638418674468994, + "objective/train/original_loss": 3.8855535984039307, + "objective/train/theoretical_loss": 3.9839492762182647, + "objective/train/tokens_used": 425984000, + "objective/train/unlikelihood_loss": 0.18474146723747253, + "theoretical_loss": 3.9839492762182647, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398595787362086, + "loss": 1.5895, + "theoretical_loss": 3.9839492762182647, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439839518555667, + "loss": 1.5817, + "theoretical_loss": 3.9838136062442704, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398194583751254, + "loss": 1.6915, + "theoretical_loss": 3.9836779896764254, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043979939819458375, + "loss": 1.7129, + "theoretical_loss": 3.983542426477292, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397793380140421, + "loss": 1.6312, + "theoretical_loss": 3.983406916609473, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397592778335005, + "loss": 1.5439, + "theoretical_loss": 3.983271460035608, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397392176529589, + "loss": 1.5743, + "theoretical_loss": 3.983136056718373, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971915747241725, + "loss": 1.5065, + "theoretical_loss": 3.9830007066204836, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396990972918756, + "loss": 1.5345, + "theoretical_loss": 3.982865409704692, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967903711133403, + "loss": 1.6371, + "theoretical_loss": 3.9827301659337864, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396589769307924, + "loss": 1.626, + "theoretical_loss": 3.982594975270595, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963891675025075, + "loss": 1.5697, + "theoretical_loss": 3.9824598376779825, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396188565697091, + "loss": 1.4803, + "theoretical_loss": 3.98232475311885, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.33889296650886536, + "objective/train/avg_token_score": 0.04264990985393524, + "objective/train/docs_used": 254260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.204171061515808, + "objective/train/original_loss": 4.482820987701416, + "objective/train/theoretical_loss": 3.9822572307152564, + "objective/train/tokens_used": 427622400, + "objective/train/unlikelihood_loss": 0.09470682591199875, + "theoretical_loss": 3.9822572307152564, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395987963891675, + "loss": 1.5879, + "theoretical_loss": 3.9821897215561375, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395787362086259, + "loss": 1.4706, + "theoretical_loss": 3.98205474295282, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955867602808426, + "loss": 1.558, + "theoretical_loss": 3.9819198172719124, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395386158475426, + "loss": 1.6488, + "theoretical_loss": 3.9817849444764644, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439518555667001, + "loss": 1.641, + "theoretical_loss": 3.9816501245295646, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394984954864594, + "loss": 1.6366, + "theoretical_loss": 3.981515357394338, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043947843530591776, + "loss": 1.3471, + "theoretical_loss": 3.981380643033945, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394583751253761, + "loss": 1.5653, + "theoretical_loss": 3.981245981411586, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394383149448345, + "loss": 1.573, + "theoretical_loss": 3.981111372490496, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043941825476429285, + "loss": 1.6642, + "theoretical_loss": 3.9809768162339476, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043939819458375126, + "loss": 1.5006, + "theoretical_loss": 3.9808423126052506, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393781344032096, + "loss": 1.4711, + "theoretical_loss": 3.9807078615677502, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4456256031990051, + "objective/train/avg_token_score": 0.0028587188571691513, + "objective/train/docs_used": 255066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7612452507019043, + "objective/train/original_loss": 4.201564311981201, + "objective/train/theoretical_loss": 3.9805734630848306, + "objective/train/tokens_used": 429260800, + "objective/train/unlikelihood_loss": 0.11549005657434464, + "theoretical_loss": 3.9805734630848306, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439358074222668, + "loss": 1.6208, + "theoretical_loss": 3.9805734630848306, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933801404212635, + "loss": 1.3378, + "theoretical_loss": 3.9804391171199103, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043931795386158477, + "loss": 1.4226, + "theoretical_loss": 3.9803048236364456, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043929789368104313, + "loss": 1.437, + "theoretical_loss": 3.98017058259793, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392778335005015, + "loss": 1.6993, + "theoretical_loss": 3.9800363939678918, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925777331995985, + "loss": 1.454, + "theoretical_loss": 3.979902257709897, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392377131394182, + "loss": 1.5987, + "theoretical_loss": 3.979768173787548, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921765295887663, + "loss": 1.7553, + "theoretical_loss": 3.9796341421644827, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439197592778335, + "loss": 1.5489, + "theoretical_loss": 3.979500162804377, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391775325977934, + "loss": 1.4274, + "theoretical_loss": 3.9793662356709407, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043915747241725177, + "loss": 1.6129, + "theoretical_loss": 3.9792323607279223, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391374122367102, + "loss": 1.4735, + "theoretical_loss": 3.9790985379391053, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911735205616855, + "loss": 1.4495, + "theoretical_loss": 3.978964767268309, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.35037878155708313, + "objective/train/avg_token_score": 0.03396159037947655, + "objective/train/docs_used": 256338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.339438557624817, + "objective/train/original_loss": 4.461732387542725, + "objective/train/theoretical_loss": 3.9788979014658716, + "objective/train/tokens_used": 430899200, + "objective/train/unlikelihood_loss": 0.13601770997047424, + "theoretical_loss": 3.9788979014658716, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390972918756269, + "loss": 1.6812, + "theoretical_loss": 3.978831048679389, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390772316950853, + "loss": 1.5037, + "theoretical_loss": 3.9786973821362377, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043905717151454364, + "loss": 1.3236, + "theoretical_loss": 3.978563767602782, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043903711133400205, + "loss": 1.7428, + "theoretical_loss": 3.9784302050429874, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390170511534604, + "loss": 1.2738, + "theoretical_loss": 3.9782966944208518, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389969909729188, + "loss": 1.6967, + "theoretical_loss": 3.9781632357004124, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043897693079237714, + "loss": 1.6041, + "theoretical_loss": 3.9780298288457394, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043895687061183556, + "loss": 1.4963, + "theoretical_loss": 3.977896473820941, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389368104312939, + "loss": 1.4479, + "theoretical_loss": 3.977763170590159, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389167502507523, + "loss": 1.5352, + "theoretical_loss": 3.9776299191175726, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043889669007021064, + "loss": 1.6516, + "theoretical_loss": 3.977496719367396, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438876629889669, + "loss": 1.6093, + "theoretical_loss": 3.9773635713038784, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.46212121844291687, + "objective/train/avg_token_score": 0.020001525059342384, + "objective/train/docs_used": 257001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5973520278930664, + "objective/train/original_loss": 3.78364634513855, + "objective/train/theoretical_loss": 3.9772304748913054, + "objective/train/tokens_used": 432537600, + "objective/train/unlikelihood_loss": 0.15700258314609528, + "theoretical_loss": 3.9772304748913054, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388565697091274, + "loss": 1.5176, + "theoretical_loss": 3.9772304748913054, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388365095285858, + "loss": 1.5094, + "theoretical_loss": 3.977097430093999, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043881644934804415, + "loss": 1.5875, + "theoretical_loss": 3.9769644368763135, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387963891675025, + "loss": 1.5692, + "theoretical_loss": 3.9768314952026422, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387763289869609, + "loss": 1.4838, + "theoretical_loss": 3.976698605037411, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387562688064193, + "loss": 1.4281, + "theoretical_loss": 3.976565766345083, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043873620862587765, + "loss": 1.4939, + "theoretical_loss": 3.976432979090155, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438716148445336, + "loss": 1.5756, + "theoretical_loss": 3.9763002432371604, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043869608826479443, + "loss": 1.6247, + "theoretical_loss": 3.9761675587506673, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386760280842528, + "loss": 1.4458, + "theoretical_loss": 3.976034925595278, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043865596790371115, + "loss": 1.7759, + "theoretical_loss": 3.9759023437356316, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386359077231695, + "loss": 1.5281, + "theoretical_loss": 3.9757698131364005, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386158475426279, + "loss": 1.6324, + "theoretical_loss": 3.9756373337622932, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4903470277786255, + "objective/train/avg_token_score": 0.012690459378063679, + "objective/train/docs_used": 257982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6558393239974976, + "objective/train/original_loss": 3.627277374267578, + "objective/train/theoretical_loss": 3.975571113273642, + "objective/train/tokens_used": 434176000, + "objective/train/unlikelihood_loss": 0.18798591196537018, + "theoretical_loss": 3.975571113273642, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385957873620863, + "loss": 1.5611, + "theoretical_loss": 3.975504905578053, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043857572718154466, + "loss": 1.6058, + "theoretical_loss": 3.9753725285484576, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438555667001003, + "loss": 1.6332, + "theoretical_loss": 3.9752402026383207, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385356068204614, + "loss": 1.6041, + "theoretical_loss": 3.9751079278124886, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385155466399198, + "loss": 1.4406, + "theoretical_loss": 3.9749757040358453, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043849548645937816, + "loss": 1.5341, + "theoretical_loss": 3.974843531273307, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384754262788365, + "loss": 1.5085, + "theoretical_loss": 3.974711409489826, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384553660982949, + "loss": 1.3682, + "theoretical_loss": 3.9745793386503885, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043843530591775325, + "loss": 1.5537, + "theoretical_loss": 3.9744473187200158, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043841524573721166, + "loss": 1.5423, + "theoretical_loss": 3.974315349663763, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043839518555667, + "loss": 1.5619, + "theoretical_loss": 3.974183431446721, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383751253761284, + "loss": 1.5113, + "theoretical_loss": 3.9740515640340135, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3175097703933716, + "objective/train/avg_token_score": 0.01103399321436882, + "objective/train/docs_used": 258654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1034122705459595, + "objective/train/original_loss": 3.728519916534424, + "objective/train/theoretical_loss": 3.973919747390801, + "objective/train/tokens_used": 435814400, + "objective/train/unlikelihood_loss": 0.20263297855854034, + "theoretical_loss": 3.973919747390801, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043835506519558675, + "loss": 1.4324, + "theoretical_loss": 3.973919747390801, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043833500501504517, + "loss": 1.7232, + "theoretical_loss": 3.9737879814822747, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043831494483450353, + "loss": 1.5674, + "theoretical_loss": 3.9736562662736636, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382948846539619, + "loss": 1.6733, + "theoretical_loss": 3.9735246017302295, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043827482447342025, + "loss": 1.6952, + "theoretical_loss": 3.9733929878172685, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382547642928786, + "loss": 1.6721, + "theoretical_loss": 3.9732614245001106, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043823470411233703, + "loss": 1.4846, + "theoretical_loss": 3.9731299117441203, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382146439317954, + "loss": 1.4607, + "theoretical_loss": 3.972998449514697, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043819458375125376, + "loss": 1.5541, + "theoretical_loss": 3.972867037777273, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381745235707121, + "loss": 1.5188, + "theoretical_loss": 3.9727356764973143, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043815446339017053, + "loss": 1.5887, + "theoretical_loss": 3.972604365640322, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381344032096289, + "loss": 1.5462, + "theoretical_loss": 3.97247310517183, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043811434302908726, + "loss": 1.6721, + "theoretical_loss": 3.972341895057408, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3990102708339691, + "objective/train/avg_token_score": 0.014954428188502789, + "objective/train/docs_used": 259916, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3668731451034546, + "objective/train/original_loss": 3.7425804138183594, + "objective/train/theoretical_loss": 3.972276308872222, + "objective/train/tokens_used": 437452800, + "objective/train/unlikelihood_loss": 0.18754182755947113, + "theoretical_loss": 3.972276308872222, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380942828485456, + "loss": 1.5532, + "theoretical_loss": 3.972210735262657, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438074222668004, + "loss": 1.7842, + "theoretical_loss": 3.9720796257532136, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380541624874624, + "loss": 1.5287, + "theoretical_loss": 3.9719485664947483, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043803410230692076, + "loss": 1.6341, + "theoretical_loss": 3.9718175574529635, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380140421263791, + "loss": 1.4245, + "theoretical_loss": 3.971686598593596, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379939819458375, + "loss": 1.5868, + "theoretical_loss": 3.9715556898824182, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379739217652959, + "loss": 1.6966, + "theoretical_loss": 3.9714248312852343, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795386158475427, + "loss": 1.494, + "theoretical_loss": 3.9712940227678803, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793380140421263, + "loss": 1.6852, + "theoretical_loss": 3.971163264296229, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437913741223671, + "loss": 1.4822, + "theoretical_loss": 3.9710325558361856, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043789368104312935, + "loss": 1.6556, + "theoretical_loss": 3.970901897353688, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787362086258777, + "loss": 1.7292, + "theoretical_loss": 3.9707712888147073, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3905791640281677, + "objective/train/avg_token_score": 0.019270719960331917, + "objective/train/docs_used": 260524, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4323198795318604, + "objective/train/original_loss": 4.0027384757995605, + "objective/train/theoretical_loss": 3.9706407301852487, + "objective/train/tokens_used": 439091200, + "objective/train/unlikelihood_loss": 0.153401181101799, + "theoretical_loss": 3.9706407301852487, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785356068204613, + "loss": 1.6313, + "theoretical_loss": 3.9706407301852487, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378335005015045, + "loss": 1.6079, + "theoretical_loss": 3.9705102214313506, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781344032096286, + "loss": 1.5786, + "theoretical_loss": 3.970379762519084, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043779338014042127, + "loss": 1.6811, + "theoretical_loss": 3.9702493534145544, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777331995987963, + "loss": 1.5142, + "theoretical_loss": 3.970118994083899, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437753259779338, + "loss": 1.5964, + "theoretical_loss": 3.969988684493289, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043773319959879636, + "loss": 1.4616, + "theoretical_loss": 3.969858424608928, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377131394182547, + "loss": 1.6263, + "theoretical_loss": 3.969728214397053, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043769307923771314, + "loss": 1.5717, + "theoretical_loss": 3.969598053823934, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767301905717155, + "loss": 1.5974, + "theoretical_loss": 3.969467942855875, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376529588766299, + "loss": 1.5539, + "theoretical_loss": 3.9693378814592104, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376328986960883, + "loss": 1.6919, + "theoretical_loss": 3.969207869600309, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376128385155467, + "loss": 1.5798, + "theoretical_loss": 3.9690779072455733, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4020650088787079, + "objective/train/avg_token_score": 0.015384318307042122, + "objective/train/docs_used": 261739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4516440629959106, + "objective/train/original_loss": 3.8263213634490967, + "objective/train/theoretical_loss": 3.969012944621776, + "objective/train/tokens_used": 440729600, + "objective/train/unlikelihood_loss": 0.15117530524730682, + "theoretical_loss": 3.969012944621776, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043759277833500506, + "loss": 1.5678, + "theoretical_loss": 3.968947994361436, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375727181544634, + "loss": 1.5547, + "theoretical_loss": 3.9688181309143657, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375526579739218, + "loss": 1.4414, + "theoretical_loss": 3.9686883168708613, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375325977933802, + "loss": 1.6541, + "theoretical_loss": 3.9685585521974556, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751253761283856, + "loss": 1.68, + "theoretical_loss": 3.968428836860712, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374924774322969, + "loss": 1.7028, + "theoretical_loss": 3.96829917082723, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374724172517553, + "loss": 1.6575, + "theoretical_loss": 3.968169554063639, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745235707121365, + "loss": 1.6168, + "theoretical_loss": 3.968039986536601, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743229689067206, + "loss": 1.7393, + "theoretical_loss": 3.9679104682128106, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374122367101304, + "loss": 1.7, + "theoretical_loss": 3.9677809990589967, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373921765295888, + "loss": 1.4573, + "theoretical_loss": 3.9676515790419176, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043737211634904715, + "loss": 1.5554, + "theoretical_loss": 3.967522208128366, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.43737781047821045, + "objective/train/avg_token_score": 0.016550948843359947, + "objective/train/docs_used": 262441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5659313201904297, + "objective/train/original_loss": 3.8397927284240723, + "objective/train/theoretical_loss": 3.9673928862851655, + "objective/train/tokens_used": 442368000, + "objective/train/unlikelihood_loss": 0.18158647418022156, + "theoretical_loss": 3.9673928862851655, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735205616850557, + "loss": 1.5389, + "theoretical_loss": 3.9673928862851655, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733199598796393, + "loss": 1.6709, + "theoretical_loss": 3.9672636134791737, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373119358074223, + "loss": 1.6792, + "theoretical_loss": 3.9671343896772786, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043729187562688065, + "loss": 1.5301, + "theoretical_loss": 3.967005214846401, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437271815446339, + "loss": 1.6247, + "theoretical_loss": 3.9668760889534944, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725175526579743, + "loss": 1.574, + "theoretical_loss": 3.966747011965544, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372316950852558, + "loss": 1.6382, + "theoretical_loss": 3.9666179838495657, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043721163490471416, + "loss": 1.5853, + "theoretical_loss": 3.96648900457261, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371915747241725, + "loss": 1.5827, + "theoretical_loss": 3.9663600741017575, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043717151454363093, + "loss": 1.5746, + "theoretical_loss": 3.9662311924041203, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371514543630893, + "loss": 1.4653, + "theoretical_loss": 3.966102359446845, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713139418254766, + "loss": 1.5448, + "theoretical_loss": 3.9659735751971064, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437111334002006, + "loss": 1.6434, + "theoretical_loss": 3.9658448396221138, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.5477761626243591, + "objective/train/avg_token_score": 0.017558608204126358, + "objective/train/docs_used": 263839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9970813989639282, + "objective/train/original_loss": 3.930330753326416, + "objective/train/theoretical_loss": 3.9657804900774085, + "objective/train/tokens_used": 444006400, + "objective/train/unlikelihood_loss": 0.15241247415542603, + "theoretical_loss": 3.9657804900774085, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370912738214644, + "loss": 1.8145, + "theoretical_loss": 3.9657161526891076, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370712136409228, + "loss": 1.6294, + "theoretical_loss": 3.9655875143653594, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043705115346038116, + "loss": 1.5046, + "theoretical_loss": 3.965458924618173, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370310932798395, + "loss": 1.842, + "theoretical_loss": 3.9653303834148828, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370110330992979, + "loss": 1.5623, + "theoretical_loss": 3.965201890722857, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369909729187563, + "loss": 1.5008, + "theoretical_loss": 3.965073446509493, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043697091273821467, + "loss": 1.5226, + "theoretical_loss": 3.9649450507422204, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043695085255767303, + "loss": 1.7281, + "theoretical_loss": 3.964816703388501, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369307923771314, + "loss": 1.5668, + "theoretical_loss": 3.9646884044158277, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043691073219658975, + "loss": 1.543, + "theoretical_loss": 3.9645601537917248, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043689067201604817, + "loss": 1.6472, + "theoretical_loss": 3.9644319514837467, + "tokens_seen": 445382656 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043687061183550653, + "loss": 1.6752, + "theoretical_loss": 3.9643037974594817, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3345552384853363, + "objective/train/avg_token_score": 0.010209061205387115, + "objective/train/docs_used": 264540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3382853269577026, + "objective/train/original_loss": 4.270390033721924, + "objective/train/theoretical_loss": 3.9641756916865463, + "objective/train/tokens_used": 445644800, + "objective/train/unlikelihood_loss": 0.15310260653495789, + "theoretical_loss": 3.9641756916865463, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368505516549649, + "loss": 1.6619, + "theoretical_loss": 3.9641756916865463, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043683049147442326, + "loss": 1.4821, + "theoretical_loss": 3.9640476341325916, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043681043129388167, + "loss": 1.6354, + "theoretical_loss": 3.9639196247652966, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043679037111334003, + "loss": 1.4849, + "theoretical_loss": 3.9637916635523736, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367703109327984, + "loss": 1.4699, + "theoretical_loss": 3.9636637504615653, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043675025075225676, + "loss": 1.4246, + "theoretical_loss": 3.9635358854606455, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367301905717151, + "loss": 1.6212, + "theoretical_loss": 3.963408068517419, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043671013039117354, + "loss": 1.7125, + "theoretical_loss": 3.963280299599722, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366900702106319, + "loss": 1.4525, + "theoretical_loss": 3.9631525786754214, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043667001003009026, + "loss": 1.5724, + "theoretical_loss": 3.9630249057124147, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366499498495486, + "loss": 1.3806, + "theoretical_loss": 3.9628972806786305, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662988966900704, + "loss": 1.7016, + "theoretical_loss": 3.9627697035420284, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366098294884654, + "loss": 1.6344, + "theoretical_loss": 3.9626421742705986, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.34390273690223694, + "objective/train/avg_token_score": 0.016636604443192482, + "objective/train/docs_used": 265849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2081854343414307, + "objective/train/original_loss": 4.167159557342529, + "objective/train/theoretical_loss": 3.9625784275743294, + "objective/train/tokens_used": 447283200, + "objective/train/unlikelihood_loss": 0.16087184846401215, + "theoretical_loss": 3.9625784275743294, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043658976930792376, + "loss": 1.4704, + "theoretical_loss": 3.9625146928323627, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656970912738213, + "loss": 1.3915, + "theoretical_loss": 3.9623872591953715, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365496489468405, + "loss": 1.6532, + "theoretical_loss": 3.9622598733277083, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365295887662989, + "loss": 1.5888, + "theoretical_loss": 3.962132535197486, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043650952858575727, + "loss": 1.3799, + "theoretical_loss": 3.9620052447728478, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648946840521563, + "loss": 1.5288, + "theoretical_loss": 3.9618780020219693, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436469408224674, + "loss": 1.5376, + "theoretical_loss": 3.961750806913054, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364493480441324, + "loss": 1.5963, + "theoretical_loss": 3.961623659414338, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642928786359077, + "loss": 1.5641, + "theoretical_loss": 3.961496559494087, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043640922768304913, + "loss": 1.3678, + "theoretical_loss": 3.9613695071205974, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363891675025075, + "loss": 1.4971, + "theoretical_loss": 3.9612425022621953, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363691073219659, + "loss": 1.5659, + "theoretical_loss": 3.961115544887238, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3955278694629669, + "objective/train/avg_token_score": 0.013740437105298042, + "objective/train/docs_used": 266141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3886208534240723, + "objective/train/original_loss": 3.7399308681488037, + "objective/train/theoretical_loss": 3.960988634964113, + "objective/train/tokens_used": 448921600, + "objective/train/unlikelihood_loss": 0.21553213894367218, + "theoretical_loss": 3.960988634964113, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363490471414243, + "loss": 1.6211, + "theoretical_loss": 3.960988634964113, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632898696088264, + "loss": 1.5591, + "theoretical_loss": 3.9608617724612367, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436308926780341, + "loss": 1.3739, + "theoretical_loss": 3.960734957347058, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043628886659979936, + "loss": 1.6942, + "theoretical_loss": 3.9606081895900545, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362688064192578, + "loss": 1.6639, + "theoretical_loss": 3.9604814691587347, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043624874623871614, + "loss": 1.4516, + "theoretical_loss": 3.960354796021636, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362286860581745, + "loss": 1.4179, + "theoretical_loss": 3.960228170147327, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043620862587763286, + "loss": 1.3733, + "theoretical_loss": 3.960101591504406, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361885656970913, + "loss": 1.5806, + "theoretical_loss": 3.9599750600615025, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361685055165497, + "loss": 1.3499, + "theoretical_loss": 3.959848575787273, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614844533600806, + "loss": 1.55, + "theoretical_loss": 3.9597221386504073, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361283851554664, + "loss": 1.5825, + "theoretical_loss": 3.9595957486196225, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361083249749248, + "loss": 1.4968, + "theoretical_loss": 3.9594694056636675, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3633919954299927, + "objective/train/avg_token_score": 0.05124437436461449, + "objective/train/docs_used": 266764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3248687982559204, + "objective/train/original_loss": 4.117973327636719, + "objective/train/theoretical_loss": 3.9594062518289923, + "objective/train/tokens_used": 450560000, + "objective/train/unlikelihood_loss": 0.1388237476348877, + "theoretical_loss": 3.9594062518289923, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360882647943832, + "loss": 1.3466, + "theoretical_loss": 3.9593431097513188, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606820461384156, + "loss": 1.5877, + "theoretical_loss": 3.9592168608513854, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360481444332999, + "loss": 1.4062, + "theoretical_loss": 3.9590906589327037, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360280842527583, + "loss": 1.3673, + "theoretical_loss": 3.958964503964141, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360080240722167, + "loss": 1.4466, + "theoretical_loss": 3.9588383959145945, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043598796389167507, + "loss": 1.4174, + "theoretical_loss": 3.95871233475299, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043596790371113343, + "loss": 1.5887, + "theoretical_loss": 3.9585863204482834, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359478435305918, + "loss": 1.5364, + "theoretical_loss": 3.9584603529694604, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043592778335005015, + "loss": 1.6187, + "theoretical_loss": 3.958334432285537, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043590772316950857, + "loss": 1.5349, + "theoretical_loss": 3.9582085583655564, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043588766298896693, + "loss": 1.3348, + "theoretical_loss": 3.9580827311785924, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358676028084253, + "loss": 1.4182, + "theoretical_loss": 3.9579569506937498, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.41373410820961, + "objective/train/avg_token_score": 0.03839399293065071, + "objective/train/docs_used": 268192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.432794213294983, + "objective/train/original_loss": 4.107253551483154, + "objective/train/theoretical_loss": 3.9578312168801597, + "objective/train/tokens_used": 452198400, + "objective/train/unlikelihood_loss": 0.13305017352104187, + "theoretical_loss": 3.9578312168801597, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043584754262788366, + "loss": 1.4714, + "theoretical_loss": 3.9578312168801597, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043582748244734207, + "loss": 1.683, + "theoretical_loss": 3.9577055297069856, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043580742226680043, + "loss": 1.4859, + "theoretical_loss": 3.9575798891434184, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357873620862588, + "loss": 1.4563, + "theoretical_loss": 3.957454295158679, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043576730190571716, + "loss": 1.5263, + "theoretical_loss": 3.9573287477220167, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357472417251755, + "loss": 1.6256, + "theoretical_loss": 3.957203246802711, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043572718154463394, + "loss": 1.6589, + "theoretical_loss": 3.9570777923700704, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357071213640923, + "loss": 1.3931, + "theoretical_loss": 3.956952384393432, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043568706118355066, + "loss": 1.5144, + "theoretical_loss": 3.9568270228421616, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435667001003009, + "loss": 1.4202, + "theoretical_loss": 3.9567017076856565, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043564694082246744, + "loss": 1.5056, + "theoretical_loss": 3.9565764388933395, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356268806419258, + "loss": 1.5587, + "theoretical_loss": 3.9564512164346652, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043560682046138417, + "loss": 1.6948, + "theoretical_loss": 3.956326040279116, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4516128897666931, + "objective/train/avg_token_score": 0.02117766998708248, + "objective/train/docs_used": 268717, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.552639126777649, + "objective/train/original_loss": 4.055312156677246, + "objective/train/theoretical_loss": 3.956263469555484, + "objective/train/tokens_used": 453836800, + "objective/train/unlikelihood_loss": 0.12762990593910217, + "theoretical_loss": 3.956263469555484, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043558676028084253, + "loss": 1.6062, + "theoretical_loss": 3.956200910396202, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355667001003009, + "loss": 1.3842, + "theoretical_loss": 3.956075826755465, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355466399197593, + "loss": 1.5806, + "theoretical_loss": 3.955950789326474, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043552657973921767, + "loss": 1.4914, + "theoretical_loss": 3.955825798078826, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043550651955867603, + "loss": 1.4215, + "theoretical_loss": 3.9557008529821474, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354864593781344, + "loss": 1.3912, + "theoretical_loss": 3.955575954006095, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354663991975928, + "loss": 1.5038, + "theoretical_loss": 3.955451101120352, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043544633901705117, + "loss": 1.6058, + "theoretical_loss": 3.9553262942946312, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043542627883650953, + "loss": 1.4004, + "theoretical_loss": 3.9552015334986743, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354062186559679, + "loss": 1.6557, + "theoretical_loss": 3.95507681870225, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353861584754263, + "loss": 1.3329, + "theoretical_loss": 3.954952149875159, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353660982948847, + "loss": 1.506, + "theoretical_loss": 3.954827526987226, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.35752686858177185, + "objective/train/avg_token_score": 0.016128363087773323, + "objective/train/docs_used": 269967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1892235279083252, + "objective/train/original_loss": 3.846442937850952, + "objective/train/theoretical_loss": 3.954702950008308, + "objective/train/tokens_used": 455475200, + "objective/train/unlikelihood_loss": 0.21875594556331635, + "theoretical_loss": 3.954702950008308, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043534603811434304, + "loss": 1.2484, + "theoretical_loss": 3.954702950008308, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353259779338014, + "loss": 1.3839, + "theoretical_loss": 3.9545784189082887, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043530591775325976, + "loss": 1.5678, + "theoretical_loss": 3.9544539336570796, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352858575727182, + "loss": 1.4671, + "theoretical_loss": 3.954329494224623, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043526579739217654, + "loss": 1.5832, + "theoretical_loss": 3.9542051005808867, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352457372116349, + "loss": 1.5173, + "theoretical_loss": 3.954080752695868, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043522567703109326, + "loss": 1.4201, + "theoretical_loss": 3.953956450539594, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352056168505517, + "loss": 1.634, + "theoretical_loss": 3.953832194082117, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043518555667001004, + "loss": 1.5989, + "theoretical_loss": 3.95370798329352, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351654964894684, + "loss": 1.5042, + "theoretical_loss": 3.9535838181439127, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043514543630892677, + "loss": 1.5895, + "theoretical_loss": 3.953459698603434, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043512537612838513, + "loss": 1.611, + "theoretical_loss": 3.95333562464225, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043510531594784355, + "loss": 1.5907, + "theoretical_loss": 3.953211596230556, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.5039711594581604, + "objective/train/avg_token_score": 0.018044468015432358, + "objective/train/docs_used": 270622, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7069590091705322, + "objective/train/original_loss": 3.6348330974578857, + "objective/train/theoretical_loss": 3.953149599096461, + "objective/train/tokens_used": 457113600, + "objective/train/unlikelihood_loss": 0.2166004627943039, + "theoretical_loss": 3.953149599096461, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350852557673019, + "loss": 1.5883, + "theoretical_loss": 3.9530876133385737, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043506519558676027, + "loss": 1.4272, + "theoretical_loss": 3.9529636759365543, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043504513540621863, + "loss": 1.448, + "theoretical_loss": 3.952839783994776, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043502507522567705, + "loss": 1.4977, + "theoretical_loss": 3.9527159374835463, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350050150451354, + "loss": 1.4147, + "theoretical_loss": 3.9525921363731977, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349849548645938, + "loss": 1.6583, + "theoretical_loss": 3.9524683806340937, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496489468405214, + "loss": 1.6028, + "theoretical_loss": 3.9523446702366245, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349448345035105, + "loss": 1.5376, + "theoretical_loss": 3.9522210051512077, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349247743229689, + "loss": 1.4196, + "theoretical_loss": 3.9520973853482886, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349047141424273, + "loss": 1.5257, + "theoretical_loss": 3.951973810798341, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043488465396188564, + "loss": 1.5334, + "theoretical_loss": 3.9518502814718652, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434864593781344, + "loss": 1.3228, + "theoretical_loss": 3.951726797339391, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.39320626854896545, + "objective/train/avg_token_score": 0.018131617456674576, + "objective/train/docs_used": 271952, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.354129433631897, + "objective/train/original_loss": 3.985093355178833, + "objective/train/theoretical_loss": 3.9516033583714734, + "objective/train/tokens_used": 458752000, + "objective/train/unlikelihood_loss": 0.19177991151809692, + "theoretical_loss": 3.9516033583714734, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348445336008024, + "loss": 1.514, + "theoretical_loss": 3.9516033583714734, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348244734202608, + "loss": 1.4862, + "theoretical_loss": 3.9514799645386973, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480441323971914, + "loss": 1.5574, + "theoretical_loss": 3.951356615811674, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347843530591775, + "loss": 1.3874, + "theoretical_loss": 3.951233312161042, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476429287863587, + "loss": 1.5508, + "theoretical_loss": 3.9511100535574686, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347442326980943, + "loss": 1.5419, + "theoretical_loss": 3.9509868399716472, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472417251755265, + "loss": 1.2042, + "theoretical_loss": 3.950863671374299, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434704112337011, + "loss": 1.5611, + "theoretical_loss": 3.9507405477361726, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346840521564694, + "loss": 1.4663, + "theoretical_loss": 3.9506174690280442, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466399197592784, + "loss": 1.5522, + "theoretical_loss": 3.9504944352207176, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346439317953862, + "loss": 1.5421, + "theoretical_loss": 3.9503714462850237, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462387161484457, + "loss": 1.557, + "theoretical_loss": 3.950248502191819, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043460381143430293, + "loss": 1.5097, + "theoretical_loss": 3.9501256029119896, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4094574749469757, + "objective/train/avg_token_score": 0.011406373232603073, + "objective/train/docs_used": 272724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6004735231399536, + "objective/train/original_loss": 4.182957649230957, + "objective/train/theoretical_loss": 3.950064170068, + "objective/train/tokens_used": 460390400, + "objective/train/unlikelihood_loss": 0.14253197610378265, + "theoretical_loss": 3.950064170068, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345837512537613, + "loss": 1.5761, + "theoretical_loss": 3.9500027484164484, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345636910732197, + "loss": 1.3552, + "theoretical_loss": 3.9498799386761334, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043454363089267807, + "loss": 1.4481, + "theoretical_loss": 3.9497571736620127, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452357071213643, + "loss": 1.4637, + "theoretical_loss": 3.9496344533450793, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345035105315948, + "loss": 1.6593, + "theoretical_loss": 3.9495117776963538, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344834503510532, + "loss": 1.495, + "theoretical_loss": 3.9493891466868845, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446339017051157, + "loss": 1.4484, + "theoretical_loss": 3.9492665602877457, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043444332998996993, + "loss": 1.4955, + "theoretical_loss": 3.9491440184700393, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344232698094283, + "loss": 1.4501, + "theoretical_loss": 3.9490215212048936, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344032096288867, + "loss": 1.4812, + "theoretical_loss": 3.948899068463465, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343831494483451, + "loss": 1.6095, + "theoretical_loss": 3.948776660216935, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043436308926780344, + "loss": 1.4606, + "theoretical_loss": 3.948654296436513, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4204545319080353, + "objective/train/avg_token_score": 0.009838702157139778, + "objective/train/docs_used": 273842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5137208700180054, + "objective/train/original_loss": 3.8193166255950928, + "objective/train/theoretical_loss": 3.9485319770934355, + "objective/train/tokens_used": 462028800, + "objective/train/unlikelihood_loss": 0.21145619451999664, + "theoretical_loss": 3.9485319770934355, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343430290872618, + "loss": 1.5253, + "theoretical_loss": 3.9485319770934355, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432296890672016, + "loss": 1.4442, + "theoretical_loss": 3.9484097021589646, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343029087261786, + "loss": 1.3611, + "theoretical_loss": 3.94828747160439, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428284854563694, + "loss": 1.5169, + "theoretical_loss": 3.9481652854010285, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342627883650953, + "loss": 1.602, + "theoretical_loss": 3.9480431435202226, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043424272818455367, + "loss": 1.4654, + "theoretical_loss": 3.9479210459333407, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342226680040121, + "loss": 1.4747, + "theoretical_loss": 3.94779899261178, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043420260782347044, + "loss": 1.6772, + "theoretical_loss": 3.9476769835269634, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341825476429288, + "loss": 1.6903, + "theoretical_loss": 3.94755501865034, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416248746238717, + "loss": 1.5698, + "theoretical_loss": 3.9474330979533843, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414242728184553, + "loss": 1.5586, + "theoretical_loss": 3.947311221407599, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412236710130395, + "loss": 1.5921, + "theoretical_loss": 3.9471893889845138, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341023069207623, + "loss": 1.3821, + "theoretical_loss": 3.947067600655682, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3299120366573334, + "objective/train/avg_token_score": 0.027434295043349266, + "objective/train/docs_used": 274430, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1526551246643066, + "objective/train/original_loss": 3.9500741958618164, + "objective/train/theoretical_loss": 3.94700672301773, + "objective/train/tokens_used": 463667200, + "objective/train/unlikelihood_loss": 0.1668168306350708, + "theoretical_loss": 3.94700672301773, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043408224674022067, + "loss": 1.4242, + "theoretical_loss": 3.946945856392687, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406218655967903, + "loss": 1.3769, + "theoretical_loss": 3.9468241561671342, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404212637913745, + "loss": 1.3464, + "theoretical_loss": 3.946702499950659, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340220661985958, + "loss": 1.5064, + "theoretical_loss": 3.946580887714921, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340020060180542, + "loss": 1.322, + "theoretical_loss": 3.946459319431608, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043398194583751254, + "loss": 1.5324, + "theoretical_loss": 3.9463377950724317, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339618856569709, + "loss": 1.3533, + "theoretical_loss": 3.9462163146091305, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339418254764293, + "loss": 1.5288, + "theoretical_loss": 3.9460948780134713, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339217652958877, + "loss": 1.5073, + "theoretical_loss": 3.9459734852572437, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043390170511534604, + "loss": 1.5032, + "theoretical_loss": 3.945852136312265, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338816449348044, + "loss": 1.6266, + "theoretical_loss": 3.94573083115038, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338615847542628, + "loss": 1.5327, + "theoretical_loss": 3.9456095697434566, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.32600194215774536, + "objective/train/avg_token_score": 0.02726374939084053, + "objective/train/docs_used": 275809, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0033814907073975, + "objective/train/original_loss": 3.863251209259033, + "objective/train/theoretical_loss": 3.945488352063391, + "objective/train/tokens_used": 465305600, + "objective/train/unlikelihood_loss": 0.22456102073192596, + "theoretical_loss": 3.945488352063391, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338415245737212, + "loss": 1.4332, + "theoretical_loss": 3.945488352063391, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043382146439317954, + "loss": 1.4683, + "theoretical_loss": 3.945367178082104, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338014042126379, + "loss": 1.4562, + "theoretical_loss": 3.945246047771543, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043378134403209627, + "loss": 1.6305, + "theoretical_loss": 3.945124961103682, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337612838515547, + "loss": 1.5113, + "theoretical_loss": 3.945003918050518, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043374122367101305, + "loss": 1.5304, + "theoretical_loss": 3.9448829185840784, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337211634904714, + "loss": 1.4951, + "theoretical_loss": 3.944761962676412, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043370110330992977, + "loss": 1.4054, + "theoretical_loss": 3.9446410502995963, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336810431293882, + "loss": 1.4125, + "theoretical_loss": 3.9445201814257334, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043366098294884655, + "loss": 1.4092, + "theoretical_loss": 3.9443993560269504, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336409227683049, + "loss": 1.3047, + "theoretical_loss": 3.944278574075402, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336208625877633, + "loss": 1.5973, + "theoretical_loss": 3.9441578355432663, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043360080240722164, + "loss": 1.6775, + "theoretical_loss": 3.9440371404027488, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4305962920188904, + "objective/train/avg_token_score": 0.008782853372395039, + "objective/train/docs_used": 276320, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.639618158340454, + "objective/train/original_loss": 4.319045543670654, + "objective/train/theoretical_loss": 3.943976809095668, + "objective/train/tokens_used": 466944000, + "objective/train/unlikelihood_loss": 0.18612205982208252, + "theoretical_loss": 3.943976809095668, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043358074222668005, + "loss": 1.5497, + "theoretical_loss": 3.94391648862608, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335606820461384, + "loss": 1.5624, + "theoretical_loss": 3.9437958801855157, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335406218655968, + "loss": 1.5378, + "theoretical_loss": 3.9436753150533375, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043352056168505514, + "loss": 1.7575, + "theoretical_loss": 3.9435547932018524, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043350050150451356, + "loss": 1.7, + "theoretical_loss": 3.943434314603394, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334804413239719, + "loss": 1.4659, + "theoretical_loss": 3.9433138792303177, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334603811434303, + "loss": 1.3896, + "theoretical_loss": 3.943193487055009, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043344032096288864, + "loss": 1.3853, + "theoretical_loss": 3.9430731380498765, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433420260782347, + "loss": 1.3526, + "theoretical_loss": 3.9429528321873537, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334002006018054, + "loss": 1.5852, + "theoretical_loss": 3.9428325694398993, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333801404212638, + "loss": 1.6151, + "theoretical_loss": 3.94271234978, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043336008024072215, + "loss": 1.5538, + "theoretical_loss": 3.942592173180163, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.5079423189163208, + "objective/train/avg_token_score": 0.011020613834261894, + "objective/train/docs_used": 277290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8621379137039185, + "objective/train/original_loss": 3.9069674015045166, + "objective/train/theoretical_loss": 3.942472039612926, + "objective/train/tokens_used": 468582400, + "objective/train/unlikelihood_loss": 0.13427872955799103, + "theoretical_loss": 3.942472039612926, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333400200601805, + "loss": 1.5221, + "theoretical_loss": 3.942472039612926, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333199598796389, + "loss": 1.543, + "theoretical_loss": 3.9423519490508476, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332998996990973, + "loss": 1.5703, + "theoretical_loss": 3.9422319014665144, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327983951855565, + "loss": 1.419, + "theoretical_loss": 3.942111896832537, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433259779338014, + "loss": 1.4121, + "theoretical_loss": 3.9419919351215498, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323971915747243, + "loss": 1.481, + "theoretical_loss": 3.941872016306215, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332196589769308, + "loss": 1.5854, + "theoretical_loss": 3.9417521403592186, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043319959879638915, + "loss": 1.7019, + "theoretical_loss": 3.94163230725327, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043317953861584757, + "loss": 1.5685, + "theoretical_loss": 3.941512516961107, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043315947843530593, + "loss": 1.4466, + "theoretical_loss": 3.9413927694554887, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313941825476435, + "loss": 1.5199, + "theoretical_loss": 3.9412730647092022, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331193580742227, + "loss": 1.4827, + "theoretical_loss": 3.9411534026950568, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309929789368107, + "loss": 1.6589, + "theoretical_loss": 3.9410337833858886, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3777492642402649, + "objective/train/avg_token_score": 0.027359778061509132, + "objective/train/docs_used": 277851, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.291692852973938, + "objective/train/original_loss": 4.366660118103027, + "objective/train/theoretical_loss": 3.9409739897371896, + "objective/train/tokens_used": 470220800, + "objective/train/unlikelihood_loss": 0.15265023708343506, + "theoretical_loss": 3.9409739897371896, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043307923771313943, + "loss": 1.3202, + "theoretical_loss": 3.9409142067545586, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043305917753259785, + "loss": 1.2957, + "theoretical_loss": 3.940794672773951, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330391173520562, + "loss": 1.5722, + "theoretical_loss": 3.9406751814169763, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330190571715146, + "loss": 1.4456, + "theoretical_loss": 3.940555732656569, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043299899699097294, + "loss": 1.4136, + "theoretical_loss": 3.9404363264656883, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329789368104313, + "loss": 1.3883, + "theoretical_loss": 3.9403169628173185, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329588766298897, + "loss": 1.5931, + "theoretical_loss": 3.9401976416844677, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329388164493481, + "loss": 1.5737, + "theoretical_loss": 3.94007836304017, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043291875626880644, + "loss": 1.6797, + "theoretical_loss": 3.9399591268574836, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328986960882648, + "loss": 1.5247, + "theoretical_loss": 3.93983993310949, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328786359077232, + "loss": 1.6422, + "theoretical_loss": 3.9397207817692967, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328585757271816, + "loss": 1.6177, + "theoretical_loss": 3.9396016728100363, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.33987048268318176, + "objective/train/avg_token_score": 0.018589701503515244, + "objective/train/docs_used": 279270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2118350267410278, + "objective/train/original_loss": 3.926861524581909, + "objective/train/theoretical_loss": 3.939482606204863, + "objective/train/tokens_used": 471859200, + "objective/train/unlikelihood_loss": 0.1467958688735962, + "theoretical_loss": 3.939482606204863, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043283851554663994, + "loss": 1.5283, + "theoretical_loss": 3.939482606204863, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328184553660983, + "loss": 1.4258, + "theoretical_loss": 3.939363581926959, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043279839518555667, + "loss": 1.4664, + "theoretical_loss": 3.939244599949528, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327783350050151, + "loss": 1.3412, + "theoretical_loss": 3.9391256602457996, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043275827482447345, + "loss": 1.4782, + "theoretical_loss": 3.9390067627890275, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327382146439318, + "loss": 1.4871, + "theoretical_loss": 3.9388879075524907, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043271815446339017, + "loss": 1.6205, + "theoretical_loss": 3.9387690945094906, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326980942828486, + "loss": 1.3513, + "theoretical_loss": 3.938650323633354, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043267803410230695, + "loss": 1.6774, + "theoretical_loss": 3.9385315948974315, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326579739217653, + "loss": 1.542, + "theoretical_loss": 3.938412908275099, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326379137412237, + "loss": 1.4591, + "theoretical_loss": 3.938294263739755, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043261785356068204, + "loss": 1.5059, + "theoretical_loss": 3.938175661264823, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043259779338014045, + "loss": 1.4335, + "theoretical_loss": 3.938057100823751, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4881476163864136, + "objective/train/avg_token_score": 0.017582368105649948, + "objective/train/docs_used": 279967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6766897439956665, + "objective/train/original_loss": 3.8044707775115967, + "objective/train/theoretical_loss": 3.9379978363576216, + "objective/train/tokens_used": 473497600, + "objective/train/unlikelihood_loss": 0.20410603284835815, + "theoretical_loss": 3.9379978363576216, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325777331995988, + "loss": 1.7232, + "theoretical_loss": 3.937938582390011, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325576730190572, + "loss": 1.5215, + "theoretical_loss": 3.9378201059370976, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043253761283851554, + "loss": 1.4315, + "theoretical_loss": 3.9377016714385316, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043251755265797396, + "loss": 1.5092, + "theoretical_loss": 3.937583278867857, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324974924774323, + "loss": 1.5942, + "theoretical_loss": 3.9374649281986405, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324774322968907, + "loss": 1.5666, + "theoretical_loss": 3.937346619404475, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043245737211634904, + "loss": 1.666, + "theoretical_loss": 3.937228352458976, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324373119358074, + "loss": 1.6075, + "theoretical_loss": 3.9371101273357825, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324172517552658, + "loss": 1.4956, + "theoretical_loss": 3.9369919440085592, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323971915747242, + "loss": 1.4977, + "theoretical_loss": 3.9368738024509926, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043237713139418255, + "loss": 1.6371, + "theoretical_loss": 3.936755702636794, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323570712136409, + "loss": 1.6294, + "theoretical_loss": 3.936637644539699, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.469696968793869, + "objective/train/avg_token_score": 0.015840815380215645, + "objective/train/docs_used": 280356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.593431830406189, + "objective/train/original_loss": 3.748237133026123, + "objective/train/theoretical_loss": 3.936519628133466, + "objective/train/tokens_used": 475136000, + "objective/train/unlikelihood_loss": 0.19854024052619934, + "theoretical_loss": 3.936519628133466, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323370110330993, + "loss": 1.6438, + "theoretical_loss": 3.936519628133466, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323169508525577, + "loss": 1.4211, + "theoretical_loss": 3.9364016533918766, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043229689067201605, + "loss": 1.4308, + "theoretical_loss": 3.9362837202887384, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322768304914744, + "loss": 1.5334, + "theoretical_loss": 3.9361658287978813, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043225677031093283, + "loss": 1.4563, + "theoretical_loss": 3.936047978893158, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322367101303912, + "loss": 1.3508, + "theoretical_loss": 3.9359301705484455, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043221664994984955, + "loss": 1.6533, + "theoretical_loss": 3.935812403737646, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321965897693079, + "loss": 1.7426, + "theoretical_loss": 3.9356946784346825, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321765295887663, + "loss": 1.5579, + "theoretical_loss": 3.9355769946135037, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321564694082247, + "loss": 1.5945, + "theoretical_loss": 3.9354593522480803, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043213640922768306, + "loss": 1.5677, + "theoretical_loss": 3.935341751312408, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321163490471414, + "loss": 1.4719, + "theoretical_loss": 3.935224191780505, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320962888665998, + "loss": 1.7156, + "theoretical_loss": 3.9351066736264126, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4970674514770508, + "objective/train/avg_token_score": 0.05302438512444496, + "objective/train/docs_used": 281398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6894476413726807, + "objective/train/original_loss": 4.0028076171875, + "objective/train/theoretical_loss": 3.93504793005794, + "objective/train/tokens_used": 476774400, + "objective/train/unlikelihood_loss": 0.18129520118236542, + "theoretical_loss": 3.93504793005794, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320762286860582, + "loss": 1.544, + "theoretical_loss": 3.9349891968241963, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043205616850551656, + "loss": 1.6091, + "theoretical_loss": 3.934871761347945, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320361083249749, + "loss": 1.3174, + "theoretical_loss": 3.93475436717177, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320160481444333, + "loss": 1.4658, + "theoretical_loss": 3.934637014269808, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043199598796389165, + "loss": 1.2387, + "theoretical_loss": 3.9345197026162158, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043197592778335006, + "loss": 1.3269, + "theoretical_loss": 3.934402432185176, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319558676028084, + "loss": 1.6676, + "theoretical_loss": 3.9342852029508935, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319358074222668, + "loss": 1.4119, + "theoretical_loss": 3.934168014887597, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043191574724172515, + "loss": 1.464, + "theoretical_loss": 3.934050867969537, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043189568706118357, + "loss": 1.5136, + "theoretical_loss": 3.93393376217099, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043187562688064193, + "loss": 1.5275, + "theoretical_loss": 3.933816697466252, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318555667001003, + "loss": 1.6569, + "theoretical_loss": 3.9336996738296444, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4156891405582428, + "objective/train/avg_token_score": 0.021102596074342728, + "objective/train/docs_used": 281924, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5788319110870361, + "objective/train/original_loss": 4.477541446685791, + "objective/train/theoretical_loss": 3.9335826912355114, + "objective/train/tokens_used": 478412800, + "objective/train/unlikelihood_loss": 0.12658530473709106, + "theoretical_loss": 3.9335826912355114, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183550651955865, + "loss": 1.3921, + "theoretical_loss": 3.9335826912355114, + "tokens_seen": 478412800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431815446339017, + "loss": 1.3283, + "theoretical_loss": 3.9334657496582195, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043179538615847543, + "loss": 1.3853, + "theoretical_loss": 3.9333488490721598, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317753259779338, + "loss": 1.4112, + "theoretical_loss": 3.933231989451744, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043175526579739215, + "loss": 1.4039, + "theoretical_loss": 3.933115170771409, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317352056168505, + "loss": 1.37, + "theoretical_loss": 3.932998393005613, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043171514543630893, + "loss": 1.3066, + "theoretical_loss": 3.9328816561288384, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316950852557673, + "loss": 1.4299, + "theoretical_loss": 3.93276496011559, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316750250752257, + "loss": 1.3555, + "theoretical_loss": 3.932648304940395, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316549648946841, + "loss": 1.2894, + "theoretical_loss": 3.9325316905778047, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163490471414244, + "loss": 1.3105, + "theoretical_loss": 3.932415117002391, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161484453360085, + "loss": 1.4595, + "theoretical_loss": 3.932298584188751, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315947843530592, + "loss": 1.4289, + "theoretical_loss": 3.9321820921115034, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.31689882278442383, + "objective/train/avg_token_score": 0.03435559570789337, + "objective/train/docs_used": 283309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2388397455215454, + "objective/train/original_loss": 4.061135768890381, + "objective/train/theoretical_loss": 3.932123861341101, + "objective/train/tokens_used": 480051200, + "objective/train/unlikelihood_loss": 0.1754794418811798, + "theoretical_loss": 3.932123861341101, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315747241725176, + "loss": 1.4026, + "theoretical_loss": 3.932065640745289, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043155466399197594, + "loss": 1.4035, + "theoretical_loss": 3.9319492300647725, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153460381143436, + "loss": 1.4297, + "theoretical_loss": 3.9318328600446413, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315145436308927, + "loss": 1.2719, + "theoretical_loss": 3.931716530659604, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314944834503511, + "loss": 1.4338, + "theoretical_loss": 3.9316002418843934, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147442326980944, + "loss": 1.4461, + "theoretical_loss": 3.931483993693764, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314543630892678, + "loss": 1.3509, + "theoretical_loss": 3.9313677860624936, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343029087262, + "loss": 1.3157, + "theoretical_loss": 3.9312516189653817, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314142427281846, + "loss": 1.5189, + "theoretical_loss": 3.9311354923772504, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043139418254764295, + "loss": 1.4483, + "theoretical_loss": 3.931019406272945, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313741223671013, + "loss": 1.3292, + "theoretical_loss": 3.9309033606273327, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313540621865597, + "loss": 1.3071, + "theoretical_loss": 3.930787355415304, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4172165095806122, + "objective/train/avg_token_score": 0.026619194075465202, + "objective/train/docs_used": 283520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5104230642318726, + "objective/train/original_loss": 3.943290948867798, + "objective/train/theoretical_loss": 3.93067139061177, + "objective/train/tokens_used": 481689600, + "objective/train/unlikelihood_loss": 0.14345265924930573, + "theoretical_loss": 3.93067139061177, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313340020060181, + "loss": 1.6205, + "theoretical_loss": 3.93067139061177, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131394182547645, + "loss": 1.3794, + "theoretical_loss": 3.9305554661916657, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312938816449348, + "loss": 1.563, + "theoretical_loss": 3.9304395821299485, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127382146439323, + "loss": 1.543, + "theoretical_loss": 3.930323738401597, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312537612838516, + "loss": 1.3568, + "theoretical_loss": 3.930207934981614, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123370110330995, + "loss": 1.5519, + "theoretical_loss": 3.930092171845022, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312136409227683, + "loss": 1.4007, + "theoretical_loss": 3.929976448966868, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311935807422267, + "loss": 1.3958, + "theoretical_loss": 3.92986076632222, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311735205616851, + "loss": 1.3824, + "theoretical_loss": 3.9297451238861685, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043115346038114346, + "loss": 1.4257, + "theoretical_loss": 3.929629521633827, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311334002006018, + "loss": 1.5184, + "theoretical_loss": 3.92951395954033, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311133400200602, + "loss": 1.402, + "theoretical_loss": 3.9293984375808337, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310932798395186, + "loss": 1.5764, + "theoretical_loss": 3.929282955730519, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4565615952014923, + "objective/train/avg_token_score": 0.03555590286850929, + "objective/train/docs_used": 284734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5730347633361816, + "objective/train/original_loss": 4.092532157897949, + "objective/train/theoretical_loss": 3.9292252298385533, + "objective/train/tokens_used": 483328000, + "objective/train/unlikelihood_loss": 0.2168586552143097, + "theoretical_loss": 3.9292252298385533, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043107321965897696, + "loss": 1.3438, + "theoretical_loss": 3.929167513964585, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310531594784353, + "loss": 1.3901, + "theoretical_loss": 3.9290521122582573, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310330992978937, + "loss": 1.3786, + "theoretical_loss": 3.928936750586779, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043101303911735205, + "loss": 1.4429, + "theoretical_loss": 3.9288214289254193, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043099297893681046, + "loss": 1.2574, + "theoretical_loss": 3.928706147249466, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309729187562688, + "loss": 1.2796, + "theoretical_loss": 3.9285909055342314, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309528585757272, + "loss": 1.4361, + "theoretical_loss": 3.928475703755048, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043093279839518555, + "loss": 1.5195, + "theoretical_loss": 3.9283605418872716, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043091273821464397, + "loss": 1.5687, + "theoretical_loss": 3.9282454199062777, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043089267803410233, + "loss": 1.5128, + "theoretical_loss": 3.928130337787467, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308726178535607, + "loss": 1.4754, + "theoretical_loss": 3.9280152955062584, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043085255767301905, + "loss": 1.3141, + "theoretical_loss": 3.9279002930380953, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4302908182144165, + "objective/train/avg_token_score": 0.014568953774869442, + "objective/train/docs_used": 285372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4252182245254517, + "objective/train/original_loss": 3.9690096378326416, + "objective/train/theoretical_loss": 3.927785330358441, + "objective/train/tokens_used": 484966400, + "objective/train/unlikelihood_loss": 0.10961988568305969, + "theoretical_loss": 3.927785330358441, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308324974924774, + "loss": 1.2884, + "theoretical_loss": 3.927785330358441, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043081243731193583, + "loss": 1.5775, + "theoretical_loss": 3.9276704074427826, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307923771313942, + "loss": 1.4638, + "theoretical_loss": 3.9275555242666265, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043077231695085256, + "loss": 1.4386, + "theoretical_loss": 3.927440680805503, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307522567703109, + "loss": 1.4193, + "theoretical_loss": 3.9273258770349626, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043073219658976933, + "loss": 1.6056, + "theoretical_loss": 3.9272111129305785, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307121364092277, + "loss": 1.4051, + "theoretical_loss": 3.9270963884679437, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043069207622868606, + "loss": 1.2638, + "theoretical_loss": 3.9269817036226757, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306720160481444, + "loss": 1.3818, + "theoretical_loss": 3.9268670583704113, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306519558676028, + "loss": 1.2313, + "theoretical_loss": 3.9267524526868085, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306318956870612, + "loss": 1.4043, + "theoretical_loss": 3.9266378865475495, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043061183550651956, + "loss": 1.3174, + "theoretical_loss": 3.926523359928335, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305917753259779, + "loss": 1.4299, + "theoretical_loss": 3.926408872804889, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4375610947608948, + "objective/train/avg_token_score": 0.00509691471233964, + "objective/train/docs_used": 285999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.344993233680725, + "objective/train/original_loss": 3.588568925857544, + "objective/train/theoretical_loss": 3.9263516440464987, + "objective/train/tokens_used": 486604800, + "objective/train/unlikelihood_loss": 0.1715584695339203, + "theoretical_loss": 3.9263516440464987, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305717151454363, + "loss": 1.468, + "theoretical_loss": 3.926294425152957, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305516549648947, + "loss": 1.5126, + "theoretical_loss": 3.9261800169483037, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043053159478435306, + "loss": 1.3765, + "theoretical_loss": 3.926065648166719, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305115346038114, + "loss": 1.6243, + "theoretical_loss": 3.9259513187840103, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304914744232698, + "loss": 1.4171, + "theoretical_loss": 3.9258370287760087, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043047141424272815, + "loss": 1.348, + "theoretical_loss": 3.9257227781185655, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043045135406218657, + "loss": 1.4053, + "theoretical_loss": 3.925608566787555, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043043129388164493, + "loss": 1.3852, + "theoretical_loss": 3.9254943947588705, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304112337011033, + "loss": 1.3728, + "theoretical_loss": 3.925380262008428, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043039117352056165, + "loss": 1.1108, + "theoretical_loss": 3.925266168512165, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043037111334002007, + "loss": 1.584, + "theoretical_loss": 3.9251521142460377, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043035105315947843, + "loss": 1.4063, + "theoretical_loss": 3.925038099186027, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.26423507928848267, + "objective/train/avg_token_score": 0.009158661589026451, + "objective/train/docs_used": 287428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9026793837547302, + "objective/train/original_loss": 3.2273669242858887, + "objective/train/theoretical_loss": 3.9249241233081333, + "objective/train/tokens_used": 488243200, + "objective/train/unlikelihood_loss": 0.31226837635040283, + "theoretical_loss": 3.9249241233081333, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303309929789368, + "loss": 1.4517, + "theoretical_loss": 3.9249241233081333, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043031093279839516, + "loss": 1.5534, + "theoretical_loss": 3.9248101865883767, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302908726178535, + "loss": 1.4797, + "theoretical_loss": 3.924696289002801, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043027081243731194, + "loss": 1.4941, + "theoretical_loss": 3.9245824305274697, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302507522567703, + "loss": 1.4144, + "theoretical_loss": 3.924468611138467, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043023069207622866, + "loss": 1.4611, + "theoretical_loss": 3.9243548308118994, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430210631895687, + "loss": 1.4122, + "theoretical_loss": 3.9242410895238935, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043019057171514544, + "loss": 1.3031, + "theoretical_loss": 3.9241273872505973, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043017051153460386, + "loss": 1.4607, + "theoretical_loss": 3.9240137239681787, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301504513540622, + "loss": 1.5392, + "theoretical_loss": 3.923900099652828, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301303911735206, + "loss": 1.5051, + "theoretical_loss": 3.9237865142807555, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430110330992979, + "loss": 1.6593, + "theoretical_loss": 3.9236729678281925, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043009027081243736, + "loss": 1.5331, + "theoretical_loss": 3.9235594602713926, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.5066593289375305, + "objective/train/avg_token_score": 0.013708590529859066, + "objective/train/docs_used": 288116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7811899185180664, + "objective/train/original_loss": 3.8481087684631348, + "objective/train/theoretical_loss": 3.923502721071488, + "objective/train/tokens_used": 489881600, + "objective/train/unlikelihood_loss": 0.17132464051246643, + "theoretical_loss": 3.923502721071488, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300702106318957, + "loss": 1.5335, + "theoretical_loss": 3.9234459915866275, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300501504513541, + "loss": 1.4411, + "theoretical_loss": 3.923332561750192, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043003009027081245, + "loss": 1.466, + "theoretical_loss": 3.9232191707384, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043001003009027086, + "loss": 1.3247, + "theoretical_loss": 3.9231058185275884, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299899699097292, + "loss": 1.5211, + "theoretical_loss": 3.922992505094112, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299699097291876, + "loss": 1.2627, + "theoretical_loss": 3.922879230414349, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042994984954864595, + "loss": 1.5509, + "theoretical_loss": 3.9227659944646964, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042992978936810437, + "loss": 1.504, + "theoretical_loss": 3.922652797221573, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042990972918756273, + "loss": 1.395, + "theoretical_loss": 3.922539638661418, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298896690070211, + "loss": 1.5169, + "theoretical_loss": 3.92242651876069, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042986960882647945, + "loss": 1.8001, + "theoretical_loss": 3.922313437495871, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298495486459378, + "loss": 1.3611, + "theoretical_loss": 3.9222003948434603, + "tokens_seen": 491388928 + }, + { + "debugging/length": 516.8571428571429, + "debugging/num_segments": 7, + "epoch": 0.15, + "objective/train/aligned_token_count": 0.45747801661491394, + "objective/train/avg_token_score": 0.004869875498116016, + "objective/train/docs_used": 289197, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5986454486846924, + "objective/train/original_loss": 4.020472526550293, + "objective/train/theoretical_loss": 3.92208739077998, + "objective/train/tokens_used": 491520000, + "objective/train/unlikelihood_loss": 0.21023108065128326, + "theoretical_loss": 3.92208739077998, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042982948846539623, + "loss": 1.4155, + "theoretical_loss": 3.92208739077998, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298094282848546, + "loss": 1.5174, + "theoretical_loss": 3.921974425281972, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042978936810431296, + "loss": 1.3887, + "theoretical_loss": 3.921861498325998, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297693079237713, + "loss": 1.4294, + "theoretical_loss": 3.9217486098886427, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042974924774322973, + "loss": 1.6334, + "theoretical_loss": 3.9216357599465073, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297291875626881, + "loss": 1.2926, + "theoretical_loss": 3.921522948476217, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042970912738214646, + "loss": 1.4125, + "theoretical_loss": 3.921410175454416, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296890672016048, + "loss": 1.4243, + "theoretical_loss": 3.9212974408577685, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296690070210632, + "loss": 1.3942, + "theoretical_loss": 3.92118474466296, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296489468405216, + "loss": 1.4767, + "theoretical_loss": 3.9210720868466953, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042962888665997996, + "loss": 1.5349, + "theoretical_loss": 3.9209594673857, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296088264794383, + "loss": 1.4584, + "theoretical_loss": 3.9208468862567205, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295887662988967, + "loss": 1.2407, + "theoretical_loss": 3.920734343436523, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.41605570912361145, + "objective/train/avg_token_score": 0.049754299223423004, + "objective/train/docs_used": 289918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4484484195709229, + "objective/train/original_loss": 4.115923881530762, + "objective/train/theoretical_loss": 3.920678086384963, + "objective/train/tokens_used": 493158400, + "objective/train/unlikelihood_loss": 0.13206537067890167, + "theoretical_loss": 3.920678086384963, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295687061183551, + "loss": 1.3909, + "theoretical_loss": 3.9206218389018943, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042954864593781347, + "loss": 1.6181, + "theoretical_loss": 3.9205093726296405, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042952858575727183, + "loss": 1.4534, + "theoretical_loss": 3.9203969445965887, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295085255767302, + "loss": 1.4879, + "theoretical_loss": 3.9202845547795864, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042948846539618855, + "loss": 1.5724, + "theoretical_loss": 3.9201722031555004, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042946840521564697, + "loss": 1.5877, + "theoretical_loss": 3.9200598897012187, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042944834503510533, + "loss": 1.6108, + "theoretical_loss": 3.9199476143936485, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294282848545637, + "loss": 1.4842, + "theoretical_loss": 3.9198353772097168, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042940822467402206, + "loss": 1.3961, + "theoretical_loss": 3.9197231781263726, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042938816449348047, + "loss": 1.4718, + "theoretical_loss": 3.919611017120583, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042936810431293883, + "loss": 1.172, + "theoretical_loss": 3.9194988941693354, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293480441323972, + "loss": 1.5461, + "theoretical_loss": 3.9193868092496382, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.38336998224258423, + "objective/train/avg_token_score": 0.007834085263311863, + "objective/train/docs_used": 291263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3380773067474365, + "objective/train/original_loss": 3.855975866317749, + "objective/train/theoretical_loss": 3.919274762338519, + "objective/train/tokens_used": 494796800, + "objective/train/unlikelihood_loss": 0.20109692215919495, + "theoretical_loss": 3.919274762338519, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042932798395185556, + "loss": 1.6039, + "theoretical_loss": 3.919274762338519, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293079237713139, + "loss": 1.2938, + "theoretical_loss": 3.9191627534130253, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042928786359077234, + "loss": 1.5174, + "theoretical_loss": 3.919050782450225, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292678034102307, + "loss": 1.3587, + "theoretical_loss": 3.9189388494272057, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042924774322968906, + "loss": 1.4829, + "theoretical_loss": 3.918826954321074, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292276830491474, + "loss": 1.5397, + "theoretical_loss": 3.9187150971089584, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042920762286860584, + "loss": 1.445, + "theoretical_loss": 3.9186032777680055, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291875626880642, + "loss": 1.6188, + "theoretical_loss": 3.918491496275383, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042916750250752256, + "loss": 1.4414, + "theoretical_loss": 3.918379752608276, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291474423269809, + "loss": 1.2727, + "theoretical_loss": 3.9182680467438926, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291273821464393, + "loss": 1.3956, + "theoretical_loss": 3.918156378659459, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291073219658977, + "loss": 1.4127, + "theoretical_loss": 3.9180447483322203, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042908726178535607, + "loss": 1.3812, + "theoretical_loss": 3.9179331557394432, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.36589688062667847, + "objective/train/avg_token_score": 0.019114283844828606, + "objective/train/docs_used": 291574, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3350896835327148, + "objective/train/original_loss": 4.14976167678833, + "objective/train/theoretical_loss": 3.9178773735863786, + "objective/train/tokens_used": 496435200, + "objective/train/unlikelihood_loss": 0.12984949350357056, + "theoretical_loss": 3.9178773735863786, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042906720160481443, + "loss": 1.3808, + "theoretical_loss": 3.917821600858413, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290471414242728, + "loss": 1.5891, + "theoretical_loss": 3.917710083666434, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290270812437312, + "loss": 1.279, + "theoretical_loss": 3.9175986041408315, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042900702106318957, + "loss": 1.3422, + "theoretical_loss": 3.9174871622589498, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898696088264793, + "loss": 1.4238, + "theoretical_loss": 3.9173757579981534, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289669007021063, + "loss": 1.4951, + "theoretical_loss": 3.9172643913358254, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289468405215647, + "loss": 1.4327, + "theoretical_loss": 3.917153062249368, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289267803410231, + "loss": 1.3294, + "theoretical_loss": 3.9170417707162057, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042890672016048144, + "loss": 1.5096, + "theoretical_loss": 3.9169305167137796, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288866599799398, + "loss": 1.587, + "theoretical_loss": 3.9168193002195513, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042886659979939816, + "loss": 1.3454, + "theoretical_loss": 3.916708121211002, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288465396188566, + "loss": 1.4182, + "theoretical_loss": 3.916596979665632, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.35257819294929504, + "objective/train/avg_token_score": 0.013382690958678722, + "objective/train/docs_used": 293024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3515236377716064, + "objective/train/original_loss": 3.946211576461792, + "objective/train/theoretical_loss": 3.9164858755609613, + "objective/train/tokens_used": 498073600, + "objective/train/unlikelihood_loss": 0.19796288013458252, + "theoretical_loss": 3.9164858755609613, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882647943831494, + "loss": 1.4908, + "theoretical_loss": 3.9164858755609613, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288064192577733, + "loss": 1.438, + "theoretical_loss": 3.9163748088745303, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878635907723166, + "loss": 1.3417, + "theoretical_loss": 3.916263779583896, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287662988966901, + "loss": 1.5937, + "theoretical_loss": 3.916152787666638, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042874623871614844, + "loss": 1.4663, + "theoretical_loss": 3.9160418331003526, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287261785356068, + "loss": 1.4601, + "theoretical_loss": 3.9159309158626576, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042870611835506517, + "loss": 1.2813, + "theoretical_loss": 3.915820035931188, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042868605817452353, + "loss": 1.2875, + "theoretical_loss": 3.9157091932836, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428665997993982, + "loss": 1.5178, + "theoretical_loss": 3.9155983878975675, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864593781344036, + "loss": 1.5188, + "theoretical_loss": 3.9154876197507846, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286258776328987, + "loss": 1.5098, + "theoretical_loss": 3.9153768888209646, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286058174523571, + "loss": 1.3121, + "theoretical_loss": 3.915266195085839, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285857572718155, + "loss": 1.5902, + "theoretical_loss": 3.91515553852316, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4228983521461487, + "objective/train/avg_token_score": 0.019886652007699013, + "objective/train/docs_used": 293514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.436700701713562, + "objective/train/original_loss": 3.5851337909698486, + "objective/train/theoretical_loss": 3.91510022417454, + "objective/train/tokens_used": 499712000, + "objective/train/unlikelihood_loss": 0.24415582418441772, + "theoretical_loss": 3.91510022417454, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856569709127387, + "loss": 1.4769, + "theoretical_loss": 3.915044919110697, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854563691073223, + "loss": 1.4117, + "theoretical_loss": 3.9149343368262404, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285255767301906, + "loss": 1.4151, + "theoretical_loss": 3.9148237916475983, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850551654964895, + "loss": 1.4644, + "theoretical_loss": 3.914713283552599, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042848545636910737, + "loss": 1.4707, + "theoretical_loss": 3.914602812519089, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846539618856573, + "loss": 1.3893, + "theoretical_loss": 3.914492378524934, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284453360080241, + "loss": 1.3221, + "theoretical_loss": 3.9143819815480194, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042842527582748246, + "loss": 1.4009, + "theoretical_loss": 3.9142716215662494, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042840521564694087, + "loss": 1.4476, + "theoretical_loss": 3.9141612985575454, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042838515546639923, + "loss": 1.3708, + "theoretical_loss": 3.91405101249985, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283650952858576, + "loss": 1.5202, + "theoretical_loss": 3.9139407633711243, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834503510531596, + "loss": 1.393, + "theoretical_loss": 3.9138305511493474, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.34488025307655334, + "objective/train/avg_token_score": 0.03448681905865669, + "objective/train/docs_used": 294722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1151436567306519, + "objective/train/original_loss": 3.941049337387085, + "objective/train/theoretical_loss": 3.9137203758125176, + "objective/train/tokens_used": 501350400, + "objective/train/unlikelihood_loss": 0.2037958949804306, + "theoretical_loss": 3.9137203758125176, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283249749247743, + "loss": 1.4004, + "theoretical_loss": 3.9137203758125176, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042830491474423274, + "loss": 1.2935, + "theoretical_loss": 3.913610237338653, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282848545636911, + "loss": 1.2546, + "theoretical_loss": 3.913500135705789, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042826479438314946, + "loss": 1.5618, + "theoretical_loss": 3.913390070891981, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282447342026078, + "loss": 1.3867, + "theoretical_loss": 3.9132800428753027, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822467402206624, + "loss": 1.2633, + "theoretical_loss": 3.9131700516338475, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282046138415246, + "loss": 1.4745, + "theoretical_loss": 3.913060097145725, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818455366098296, + "loss": 1.46, + "theoretical_loss": 3.9129501793890666, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281644934804413, + "loss": 1.2897, + "theoretical_loss": 3.9128402983420214, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281444332998997, + "loss": 1.2421, + "theoretical_loss": 3.912730453982756, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281243731193581, + "loss": 1.3616, + "theoretical_loss": 3.912620646289457, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042810431293881647, + "loss": 1.5323, + "theoretical_loss": 3.9125108752403293, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808425275827483, + "loss": 1.2956, + "theoretical_loss": 3.9124011408135964, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4918743968009949, + "objective/train/avg_token_score": 0.003718223888427019, + "objective/train/docs_used": 295112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6487371921539307, + "objective/train/original_loss": 4.021548748016357, + "objective/train/theoretical_loss": 3.9123462873268275, + "objective/train/tokens_used": 502988800, + "objective/train/unlikelihood_loss": 0.127858504652977, + "theoretical_loss": 3.9123462873268275, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280641925777332, + "loss": 1.3411, + "theoretical_loss": 3.9122914429874998, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280441323971916, + "loss": 1.5446, + "theoretical_loss": 3.9121817817403004, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802407221664997, + "loss": 1.2649, + "theoretical_loss": 3.912072157050278, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042800401203610833, + "loss": 1.2767, + "theoretical_loss": 3.91196256889573, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279839518555667, + "loss": 1.2984, + "theoretical_loss": 3.911853017254973, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279638916750251, + "loss": 1.3402, + "theoretical_loss": 3.911743502106341, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279438314944835, + "loss": 1.4761, + "theoretical_loss": 3.9116340234281886, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042792377131394184, + "loss": 1.4801, + "theoretical_loss": 3.9115245811988864, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279037111334002, + "loss": 1.4165, + "theoretical_loss": 3.911415175396825, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042788365095285856, + "loss": 1.4903, + "theoretical_loss": 3.9113058060004136, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427863590772317, + "loss": 1.3463, + "theoretical_loss": 3.911196472988079, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042784353059177534, + "loss": 1.4082, + "theoretical_loss": 3.911087176338266, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.2879398763179779, + "objective/train/avg_token_score": 0.030777588486671448, + "objective/train/docs_used": 296352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.946451723575592, + "objective/train/original_loss": 3.832697629928589, + "objective/train/theoretical_loss": 3.910977916029439, + "objective/train/tokens_used": 504627200, + "objective/train/unlikelihood_loss": 0.18606476485729218, + "theoretical_loss": 3.910977916029439, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278234704112337, + "loss": 1.31, + "theoretical_loss": 3.910977916029439, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042780341023069206, + "loss": 1.3279, + "theoretical_loss": 3.91086869204008, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277833500501505, + "loss": 1.4205, + "theoretical_loss": 3.9107595043486905, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042776328986960884, + "loss": 1.5258, + "theoretical_loss": 3.9106503529337875, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277432296890672, + "loss": 1.5069, + "theoretical_loss": 3.910541237773909, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042772316950852557, + "loss": 1.5058, + "theoretical_loss": 3.9104321588476108, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042770310932798393, + "loss": 1.4835, + "theoretical_loss": 3.9103231161334655, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042768304914744235, + "loss": 1.3955, + "theoretical_loss": 3.910214109610065, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276629889669007, + "loss": 1.4016, + "theoretical_loss": 3.9101051392560198, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042764292878635907, + "loss": 1.4402, + "theoretical_loss": 3.9099962050499584, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042762286860581743, + "loss": 1.4021, + "theoretical_loss": 3.9098873069705267, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042760280842527585, + "loss": 1.4597, + "theoretical_loss": 3.909778444996389, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275827482447342, + "loss": 1.469, + "theoretical_loss": 3.9096696191062277, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.37237292528152466, + "objective/train/avg_token_score": 0.01380022894591093, + "objective/train/docs_used": 297118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3744415044784546, + "objective/train/original_loss": 4.307300090789795, + "objective/train/theoretical_loss": 3.909615219685982, + "objective/train/tokens_used": 506265600, + "objective/train/unlikelihood_loss": 0.17518089711666107, + "theoretical_loss": 3.909615219685982, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275626880641926, + "loss": 1.4586, + "theoretical_loss": 3.9095608292787443, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042754262788365094, + "loss": 1.4717, + "theoretical_loss": 3.9094520754926574, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275225677031093, + "loss": 1.4785, + "theoretical_loss": 3.909343357726703, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275025075225677, + "loss": 1.4522, + "theoretical_loss": 3.9092346759596373, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274824473420261, + "loss": 1.4267, + "theoretical_loss": 3.9091260301702326, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042746238716148444, + "loss": 1.6183, + "theoretical_loss": 3.90901742033728, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274423269809428, + "loss": 1.4882, + "theoretical_loss": 3.9089088464395876, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274222668004012, + "loss": 1.3987, + "theoretical_loss": 3.9088003084559833, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274022066198596, + "loss": 1.6277, + "theoretical_loss": 3.9086918063653115, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042738214643931794, + "loss": 1.5008, + "theoretical_loss": 3.9085833401464347, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273620862587763, + "loss": 1.4856, + "theoretical_loss": 3.908474909778234, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042734202607823467, + "loss": 1.3557, + "theoretical_loss": 3.9083665152396074, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.35135629773139954, + "objective/train/avg_token_score": 0.02974272146821022, + "objective/train/docs_used": 298304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.351550579071045, + "objective/train/original_loss": 4.083728790283203, + "objective/train/theoretical_loss": 3.908258156509472, + "objective/train/tokens_used": 507904000, + "objective/train/unlikelihood_loss": 0.15935710072517395, + "theoretical_loss": 3.908258156509472, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273219658976931, + "loss": 1.406, + "theoretical_loss": 3.908258156509472, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042730190571715145, + "loss": 1.5871, + "theoretical_loss": 3.9081498335667613, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272818455366098, + "loss": 1.2993, + "theoretical_loss": 3.908041546390428, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042726178535606817, + "loss": 1.4989, + "theoretical_loss": 3.9079332949594408, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272417251755266, + "loss": 1.417, + "theoretical_loss": 3.907825079252789, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042722166499498495, + "loss": 1.4882, + "theoretical_loss": 3.907716899249477, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272016048144433, + "loss": 1.4883, + "theoretical_loss": 3.9076087549285274, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271815446339017, + "loss": 1.5095, + "theoretical_loss": 3.9075006462689825, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271614844533601, + "loss": 1.4736, + "theoretical_loss": 3.9073925732498997, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271414242728185, + "loss": 1.4869, + "theoretical_loss": 3.907284535850356, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042712136409227687, + "loss": 1.401, + "theoretical_loss": 3.9071765340494453, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042710130391173523, + "loss": 1.3473, + "theoretical_loss": 3.907068567826278, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270812437311936, + "loss": 1.3851, + "theoretical_loss": 3.906960637159985, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.29997557401657104, + "objective/train/avg_token_score": 0.05142614245414734, + "objective/train/docs_used": 298986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.076449990272522, + "objective/train/original_loss": 4.7175726890563965, + "objective/train/theoretical_loss": 3.9069066851541487, + "objective/train/tokens_used": 509542400, + "objective/train/unlikelihood_loss": 0.11390677094459534, + "theoretical_loss": 3.9069066851541487, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427061183550652, + "loss": 1.4249, + "theoretical_loss": 3.906852742029712, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042704112337011037, + "loss": 1.4852, + "theoretical_loss": 3.9067448824146243, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042702106318956873, + "loss": 1.411, + "theoretical_loss": 3.906637058293903, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270010030090271, + "loss": 1.31, + "theoretical_loss": 3.9065292696467475, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269809428284855, + "loss": 1.5244, + "theoretical_loss": 3.906421516452376, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269608826479439, + "loss": 1.5396, + "theoretical_loss": 3.9063137986900216, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042694082246740224, + "loss": 1.4904, + "theoretical_loss": 3.9062061163389377, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269207622868606, + "loss": 1.4771, + "theoretical_loss": 3.906098469378393, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042690070210631896, + "loss": 1.4655, + "theoretical_loss": 3.905990857787675, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268806419257774, + "loss": 1.5493, + "theoretical_loss": 3.905883281546088, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042686058174523574, + "loss": 1.604, + "theoretical_loss": 3.9057757406329534, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268405215646941, + "loss": 1.4769, + "theoretical_loss": 3.9056682350276106, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.5116080045700073, + "objective/train/avg_token_score": 0.008771687746047974, + "objective/train/docs_used": 299996, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8161802291870117, + "objective/train/original_loss": 3.781794548034668, + "objective/train/theoretical_loss": 3.905560764709417, + "objective/train/tokens_used": 511180800, + "objective/train/unlikelihood_loss": 0.15718472003936768, + "theoretical_loss": 3.905560764709417, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682046138415246, + "loss": 1.3467, + "theoretical_loss": 3.905560764709417, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268004012036109, + "loss": 1.2939, + "theoretical_loss": 3.905453329657745, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678034102306924, + "loss": 1.4485, + "theoretical_loss": 3.9053459298519875, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267602808425276, + "loss": 1.3541, + "theoretical_loss": 3.905238565271553, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042674022066198597, + "loss": 1.4509, + "theoretical_loss": 3.9051312358958663, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672016048144433, + "loss": 1.3144, + "theoretical_loss": 3.905023941704371, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042670010030090275, + "loss": 1.5859, + "theoretical_loss": 3.9049166826765283, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266800401203611, + "loss": 1.4643, + "theoretical_loss": 3.9048094587918154, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042665997993981947, + "loss": 1.62, + "theoretical_loss": 3.904702270029727, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042663991975927783, + "loss": 1.474, + "theoretical_loss": 3.9045951163697756, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042661985957873625, + "loss": 1.4682, + "theoretical_loss": 3.904487997791491, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265997993981946, + "loss": 1.313, + "theoretical_loss": 3.904380914274419, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426579739217653, + "loss": 1.5021, + "theoretical_loss": 3.9042738657981237, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.28787878155708313, + "objective/train/avg_token_score": 0.02153957076370716, + "objective/train/docs_used": 300643, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0317378044128418, + "objective/train/original_loss": 4.142201900482178, + "objective/train/theoretical_loss": 3.9042203546938854, + "objective/train/tokens_used": 512819200, + "objective/train/unlikelihood_loss": 0.20485782623291016, + "theoretical_loss": 3.9042203546938854, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042655967903711134, + "loss": 1.438, + "theoretical_loss": 3.9041668523421853, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265396188565697, + "loss": 1.5775, + "theoretical_loss": 3.9040598738862027, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265195586760281, + "loss": 1.4891, + "theoretical_loss": 3.90395293040979, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264994984954865, + "loss": 1.6842, + "theoretical_loss": 3.90384602189258, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042647943831494484, + "loss": 1.5024, + "theoretical_loss": 3.9037391483142208, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264593781344032, + "loss": 1.6385, + "theoretical_loss": 3.9036323096543795, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264393179538616, + "loss": 1.5277, + "theoretical_loss": 3.9035255058927394, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042641925777332, + "loss": 1.5275, + "theoretical_loss": 3.903418737009, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042639919759277834, + "loss": 1.3673, + "theoretical_loss": 3.903312002982879, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263791374122367, + "loss": 1.2568, + "theoretical_loss": 3.9032053037941106, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042635907723169507, + "loss": 1.3819, + "theoretical_loss": 3.903098639422445, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263390170511535, + "loss": 1.5151, + "theoretical_loss": 3.902992009847652, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.45747801661491394, + "objective/train/avg_token_score": 0.014842421747744083, + "objective/train/docs_used": 301818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6070836782455444, + "objective/train/original_loss": 3.8919403553009033, + "objective/train/theoretical_loss": 3.9028854150495143, + "objective/train/tokens_used": 514457600, + "objective/train/unlikelihood_loss": 0.16238826513290405, + "theoretical_loss": 3.9028854150495143, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042631895687061185, + "loss": 1.4055, + "theoretical_loss": 3.9028854150495143, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262988966900702, + "loss": 1.4375, + "theoretical_loss": 3.9027788550078357, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042627883650952857, + "loss": 1.461, + "theoretical_loss": 3.9026723297024337, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426258776328987, + "loss": 1.4347, + "theoretical_loss": 3.9025658391131444, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042623871614844535, + "loss": 1.5544, + "theoretical_loss": 3.9024593832198198, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262186559679037, + "loss": 1.4037, + "theoretical_loss": 3.9023529620023294, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261985957873621, + "loss": 1.4876, + "theoretical_loss": 3.902246575440559, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042617853560682044, + "loss": 1.3702, + "theoretical_loss": 3.9021402235144116, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042615847542627885, + "loss": 1.3778, + "theoretical_loss": 3.9020339062038065, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261384152457372, + "loss": 1.4036, + "theoretical_loss": 3.90192762348868, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261183550651956, + "loss": 1.4649, + "theoretical_loss": 3.901821375348985, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042609829488465394, + "loss": 1.5229, + "theoretical_loss": 3.9017151617646917, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042607823470411236, + "loss": 1.5503, + "theoretical_loss": 3.9016089827157865, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.44012707471847534, + "objective/train/avg_token_score": 0.02129729837179184, + "objective/train/docs_used": 302480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6454428434371948, + "objective/train/original_loss": 4.097105026245117, + "objective/train/theoretical_loss": 3.9015559061358545, + "objective/train/tokens_used": 516096000, + "objective/train/unlikelihood_loss": 0.17164163291454315, + "theoretical_loss": 3.9015559061358545, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260581745235707, + "loss": 1.5246, + "theoretical_loss": 3.901502838182272, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260381143430291, + "loss": 1.397, + "theoretical_loss": 3.9013967281441677, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042601805416248744, + "loss": 1.4353, + "theoretical_loss": 3.901290652581511, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259979939819458, + "loss": 1.5998, + "theoretical_loss": 3.9011846114743536, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259779338014042, + "loss": 1.5021, + "theoretical_loss": 3.9010786048027666, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259578736208626, + "loss": 1.5956, + "theoretical_loss": 3.900972632546835, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042593781344032095, + "loss": 1.5683, + "theoretical_loss": 3.9008666946866613, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259177532597793, + "loss": 1.486, + "theoretical_loss": 3.900760791202366, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258976930792377, + "loss": 1.378, + "theoretical_loss": 3.9006549220740836, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258776328986961, + "loss": 1.4012, + "theoretical_loss": 3.9005490872819673, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042585757271815445, + "loss": 1.3931, + "theoretical_loss": 3.9004432868061856, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258375125376128, + "loss": 1.5117, + "theoretical_loss": 3.9003375206269233, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3442082107067108, + "objective/train/avg_token_score": 0.01602189801633358, + "objective/train/docs_used": 303675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2318471670150757, + "objective/train/original_loss": 4.042038440704346, + "objective/train/theoretical_loss": 3.9002317887243834, + "objective/train/tokens_used": 517734400, + "objective/train/unlikelihood_loss": 0.17470526695251465, + "theoretical_loss": 3.9002317887243834, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258174523570712, + "loss": 1.4767, + "theoretical_loss": 3.9002317887243834, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257973921765296, + "loss": 1.3376, + "theoretical_loss": 3.900126091078783, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577733199598795, + "loss": 1.3587, + "theoretical_loss": 3.9000204276703565, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257572718154463, + "loss": 1.3704, + "theoretical_loss": 3.8999147984793554, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257372116349047, + "loss": 1.47, + "theoretical_loss": 3.8998092034860474, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171514543631, + "loss": 1.3936, + "theoretical_loss": 3.8997036426707155, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042569709127382145, + "loss": 1.4903, + "theoretical_loss": 3.89959811601366, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256770310932798, + "loss": 1.4924, + "theoretical_loss": 3.899492623495198, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042565697091273823, + "loss": 1.5132, + "theoretical_loss": 3.8993871650956606, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563691073219665, + "loss": 1.4211, + "theoretical_loss": 3.8992817407953986, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425616850551655, + "loss": 1.6041, + "theoretical_loss": 3.8991763505747765, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255967903711134, + "loss": 1.3711, + "theoretical_loss": 3.899070994414176, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042557673019057174, + "loss": 1.5216, + "theoretical_loss": 3.8989656722939943, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.5273093581199646, + "objective/train/avg_token_score": 0.004553263075649738, + "objective/train/docs_used": 304298, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.785210132598877, + "objective/train/original_loss": 3.7178406715393066, + "objective/train/theoretical_loss": 3.8989130239929395, + "objective/train/tokens_used": 519372800, + "objective/train/unlikelihood_loss": 0.14067475497722626, + "theoretical_loss": 3.8989130239929395, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255566700100301, + "loss": 1.581, + "theoretical_loss": 3.8988603841946463, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255366098294885, + "loss": 1.3521, + "theoretical_loss": 3.898755130096562, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255165496489469, + "loss": 1.5186, + "theoretical_loss": 3.8986499099801875, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549648946840524, + "loss": 1.5265, + "theoretical_loss": 3.8985447238259856, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254764292878636, + "loss": 1.5886, + "theoretical_loss": 3.8984395716144347, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425456369107322, + "loss": 1.2698, + "theoretical_loss": 3.8983344533260307, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254363089267804, + "loss": 1.4448, + "theoretical_loss": 3.8982293689412835, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042541624874623874, + "loss": 1.5163, + "theoretical_loss": 3.898124318440721, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253961885656971, + "loss": 1.527, + "theoretical_loss": 3.8980193018048857, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537612838515547, + "loss": 1.6545, + "theoretical_loss": 3.8979143190143377, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253560682046139, + "loss": 1.4184, + "theoretical_loss": 3.8978093700496514, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533600802407225, + "loss": 1.4471, + "theoretical_loss": 3.897704454891419, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.42644184827804565, + "objective/train/avg_token_score": 0.012661627493798733, + "objective/train/docs_used": 305033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3695693016052246, + "objective/train/original_loss": 3.7167294025421143, + "objective/train/theoretical_loss": 3.897599573520247, + "objective/train/tokens_used": 521011200, + "objective/train/unlikelihood_loss": 0.21720042824745178, + "theoretical_loss": 3.897599573520247, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253159478435306, + "loss": 1.4253, + "theoretical_loss": 3.897599573520247, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042529588766298897, + "loss": 1.4934, + "theoretical_loss": 3.8974947259167596, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252758274824474, + "loss": 1.4735, + "theoretical_loss": 3.8973899120615956, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525576730190575, + "loss": 1.4753, + "theoretical_loss": 3.897285131935411, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252357071213641, + "loss": 1.6052, + "theoretical_loss": 3.897180385518876, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252156469408225, + "loss": 1.4427, + "theoretical_loss": 3.8970756727926785, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042519558676028084, + "loss": 1.5753, + "theoretical_loss": 3.896970993737522, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517552657973925, + "loss": 1.5369, + "theoretical_loss": 3.8968663483341253, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251554663991976, + "loss": 1.3264, + "theoretical_loss": 3.8967617365632217, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425135406218656, + "loss": 1.3828, + "theoretical_loss": 3.8966571584055645, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511534603811434, + "loss": 1.5941, + "theoretical_loss": 3.8965526138419184, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042509528585757276, + "loss": 1.284, + "theoretical_loss": 3.896448102853067, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250752256770311, + "loss": 1.4964, + "theoretical_loss": 3.8963436254198074, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4902859330177307, + "objective/train/avg_token_score": 0.007528013549745083, + "objective/train/docs_used": 306243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7427791357040405, + "objective/train/original_loss": 3.7014918327331543, + "objective/train/theoretical_loss": 3.8962913992805284, + "objective/train/tokens_used": 522649600, + "objective/train/unlikelihood_loss": 0.16139429807662964, + "theoretical_loss": 3.8962913992805284, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250551654964895, + "loss": 1.5339, + "theoretical_loss": 3.896239181522954, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503510531594784, + "loss": 1.5216, + "theoretical_loss": 3.8961347711433376, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250150451354062, + "loss": 1.4623, + "theoretical_loss": 3.8960303942618024, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249949849548646, + "loss": 1.346, + "theoretical_loss": 3.8959260508592104, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424974924774323, + "loss": 1.6433, + "theoretical_loss": 3.895821740916439, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042495486459378135, + "loss": 1.3234, + "theoretical_loss": 3.89571746441438, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249348044132397, + "loss": 1.546, + "theoretical_loss": 3.8956132213339423, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249147442326981, + "loss": 1.4065, + "theoretical_loss": 3.8955090116560505, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248946840521565, + "loss": 1.494, + "theoretical_loss": 3.8954048353616435, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042487462387161485, + "loss": 1.6275, + "theoretical_loss": 3.895300692431677, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248545636910732, + "loss": 1.3638, + "theoretical_loss": 3.895196582847123, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248345035105316, + "loss": 1.4729, + "theoretical_loss": 3.8950925065889663, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4181329309940338, + "objective/train/avg_token_score": 0.01068844273686409, + "objective/train/docs_used": 306972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3113154172897339, + "objective/train/original_loss": 3.7316195964813232, + "objective/train/theoretical_loss": 3.8949884636382106, + "objective/train/tokens_used": 524288000, + "objective/train/unlikelihood_loss": 0.1870473474264145, + "theoretical_loss": 3.8949884636382106, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042481444332999, + "loss": 1.5167, + "theoretical_loss": 3.8949884636382106, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042479438314944835, + "loss": 1.4619, + "theoretical_loss": 3.894884453975873, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247743229689067, + "loss": 1.4227, + "theoretical_loss": 3.894780477582988, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247542627883651, + "loss": 1.5495, + "theoretical_loss": 3.8946765344406034, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247342026078235, + "loss": 1.362, + "theoretical_loss": 3.8945726245297836, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042471414242728186, + "loss": 1.4983, + "theoretical_loss": 3.894468747831609, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246940822467402, + "loss": 1.7403, + "theoretical_loss": 3.8943649043271744, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246740220661986, + "loss": 1.5129, + "theoretical_loss": 3.894261093997592, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424653961885657, + "loss": 1.535, + "theoretical_loss": 3.8941573168239874, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042463390170511536, + "loss": 1.4884, + "theoretical_loss": 3.894053572787502, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246138415245737, + "loss": 1.6132, + "theoretical_loss": 3.8939498618692934, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245937813440321, + "loss": 1.5335, + "theoretical_loss": 3.8938461840505347, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042457372116349044, + "loss": 1.4542, + "theoretical_loss": 3.893742539312414, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.43389540910720825, + "objective/train/avg_token_score": 0.013585958629846573, + "objective/train/docs_used": 308263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.377792477607727, + "objective/train/original_loss": 3.8568532466888428, + "objective/train/theoretical_loss": 3.8936907293427176, + "objective/train/tokens_used": 525926400, + "objective/train/unlikelihood_loss": 0.1510537713766098, + "theoretical_loss": 3.8936907293427176, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042455366098294886, + "loss": 1.3444, + "theoretical_loss": 3.893638927636134, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245336008024072, + "loss": 1.5525, + "theoretical_loss": 3.8935353490029136, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245135406218656, + "loss": 1.395, + "theoretical_loss": 3.8934318033939874, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042449348044132395, + "loss": 1.4722, + "theoretical_loss": 3.8933282907906044, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042447342026078236, + "loss": 1.4891, + "theoretical_loss": 3.89322481117403, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244533600802407, + "loss": 1.3796, + "theoretical_loss": 3.893121364525544, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244332998996991, + "loss": 1.4325, + "theoretical_loss": 3.893017950826441, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042441323971915745, + "loss": 1.519, + "theoretical_loss": 3.892914570058033, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243931795386158, + "loss": 1.5274, + "theoretical_loss": 3.8928112222016455, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042437311935807423, + "loss": 1.4975, + "theoretical_loss": 3.8927079072386186, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243530591775326, + "loss": 1.3574, + "theoretical_loss": 3.8926046251503092, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042433299899699095, + "loss": 1.4539, + "theoretical_loss": 3.892501375918089, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3826979398727417, + "objective/train/avg_token_score": 0.035878412425518036, + "objective/train/docs_used": 309026, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3176822662353516, + "objective/train/original_loss": 4.551613807678223, + "objective/train/theoretical_loss": 3.892398159523345, + "objective/train/tokens_used": 527564800, + "objective/train/unlikelihood_loss": 0.1040167510509491, + "theoretical_loss": 3.892398159523345, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243129388164493, + "loss": 1.4981, + "theoretical_loss": 3.892398159523345, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042429287863590773, + "loss": 1.5112, + "theoretical_loss": 3.892294975947477, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242728184553661, + "loss": 1.4593, + "theoretical_loss": 3.8921918251719054, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042425275827482446, + "loss": 1.6896, + "theoretical_loss": 3.8920887071780594, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242326980942828, + "loss": 1.4538, + "theoretical_loss": 3.891985621947388, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242126379137412, + "loss": 1.5069, + "theoretical_loss": 3.891882569461353, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241925777331996, + "loss": 1.6161, + "theoretical_loss": 3.8917795497014316, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424172517552658, + "loss": 1.315, + "theoretical_loss": 3.891676562649116, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241524573721164, + "loss": 1.4037, + "theoretical_loss": 3.891573608285915, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042413239719157474, + "loss": 1.5572, + "theoretical_loss": 3.89147068659335, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042411233701103316, + "loss": 1.5609, + "theoretical_loss": 3.891367797552959, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240922768304915, + "loss": 1.5477, + "theoretical_loss": 3.891264941146295, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240722166499499, + "loss": 1.4536, + "theoretical_loss": 3.8911621173549253, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.5902981162071228, + "objective/train/avg_token_score": 0.02185690961778164, + "objective/train/docs_used": 310459, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0812337398529053, + "objective/train/original_loss": 3.739985227584839, + "objective/train/theoretical_loss": 3.89111071768422, + "objective/train/tokens_used": 529203200, + "objective/train/unlikelihood_loss": 0.1313895732164383, + "theoretical_loss": 3.89111071768422, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042405215646940824, + "loss": 1.5481, + "theoretical_loss": 3.8910593261604323, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240320962888666, + "loss": 1.3968, + "theoretical_loss": 3.8909565675444138, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424012036108325, + "loss": 1.4023, + "theoretical_loss": 3.8908538414884823, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239919759277834, + "loss": 1.3563, + "theoretical_loss": 3.890751147974265, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397191574724175, + "loss": 1.6032, + "theoretical_loss": 3.890648486983405, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239518555667001, + "loss": 1.5371, + "theoretical_loss": 3.8905458584975583, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239317953861585, + "loss": 1.5931, + "theoretical_loss": 3.8904432624983976, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239117352056169, + "loss": 1.5401, + "theoretical_loss": 3.89034069896761, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042389167502507525, + "loss": 1.4977, + "theoretical_loss": 3.8902381678868974, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238716148445336, + "loss": 1.4138, + "theoretical_loss": 3.890135669237976, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423851554663992, + "loss": 1.4784, + "theoretical_loss": 3.8900332030025773, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238314944834504, + "loss": 1.3966, + "theoretical_loss": 3.889930769162448, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.38929617404937744, + "objective/train/avg_token_score": 0.033991601318120956, + "objective/train/docs_used": 311126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3811792135238647, + "objective/train/original_loss": 4.070007801055908, + "objective/train/theoretical_loss": 3.889828367699349, + "objective/train/tokens_used": 530841600, + "objective/train/unlikelihood_loss": 0.13424085080623627, + "theoretical_loss": 3.889828367699349, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381143430290875, + "loss": 1.4535, + "theoretical_loss": 3.889828367699349, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237913741223671, + "loss": 1.4853, + "theoretical_loss": 3.8897259985950554, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237713139418255, + "loss": 1.5492, + "theoretical_loss": 3.889623661831359, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237512537612839, + "loss": 1.432, + "theoretical_loss": 3.8895213573900635, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373119358074226, + "loss": 1.404, + "theoretical_loss": 3.8894190852529906, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237111334002006, + "loss": 1.3642, + "theoretical_loss": 3.889316845401974, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423691073219659, + "loss": 1.4066, + "theoretical_loss": 3.8892146378188643, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236710130391174, + "loss": 1.576, + "theoretical_loss": 3.8891124624855236, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042365095285857576, + "loss": 1.4427, + "theoretical_loss": 3.889010319383832, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236308926780341, + "loss": 1.3313, + "theoretical_loss": 3.888908208495683, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236108324974925, + "loss": 1.5414, + "theoretical_loss": 3.888806129802984, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042359077231695085, + "loss": 1.5081, + "theoretical_loss": 3.8887040832876587, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357071213640926, + "loss": 1.5332, + "theoretical_loss": 3.888602068931643, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3950391113758087, + "objective/train/avg_token_score": 0.011321122758090496, + "objective/train/docs_used": 312276, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4738465547561646, + "objective/train/original_loss": 3.9406888484954834, + "objective/train/theoretical_loss": 3.888551073807736, + "objective/train/tokens_used": 532480000, + "objective/train/unlikelihood_loss": 0.1509210616350174, + "theoretical_loss": 3.888551073807736, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235506519558676, + "loss": 1.4874, + "theoretical_loss": 3.8885000867168893, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423530591775326, + "loss": 1.4809, + "theoretical_loss": 3.888398136625364, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042351053159478435, + "loss": 1.4155, + "theoretical_loss": 3.8882962186390486, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349047141424276, + "loss": 1.4746, + "theoretical_loss": 3.8881943327399378, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042347041123370113, + "loss": 1.5315, + "theoretical_loss": 3.8880924789100417, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234503510531595, + "loss": 1.4988, + "theoretical_loss": 3.8879906571313847, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343029087261785, + "loss": 1.4565, + "theoretical_loss": 3.887888867386007, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234102306920762, + "loss": 1.4381, + "theoretical_loss": 3.88778710965596, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339017051153463, + "loss": 1.4021, + "theoretical_loss": 3.887685383923314, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423370110330993, + "loss": 1.5644, + "theoretical_loss": 3.887583690170149, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042335005015045135, + "loss": 1.5509, + "theoretical_loss": 3.887482028378564, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233299899699097, + "loss": 1.4937, + "theoretical_loss": 3.8873803985306683, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.49890029430389404, + "objective/train/avg_token_score": 0.006122749298810959, + "objective/train/docs_used": 312792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6340548992156982, + "objective/train/original_loss": 3.637239456176758, + "objective/train/theoretical_loss": 3.8872788006085894, + "objective/train/tokens_used": 534118400, + "objective/train/unlikelihood_loss": 0.22649259865283966, + "theoretical_loss": 3.8872788006085894, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042330992978936813, + "loss": 1.5323, + "theoretical_loss": 3.8872788006085894, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232898696088265, + "loss": 1.3563, + "theoretical_loss": 3.887177234594466, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042326980942828486, + "loss": 1.2005, + "theoretical_loss": 3.8870757004704526, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232497492477432, + "loss": 1.5123, + "theoretical_loss": 3.8869741982187183, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232296890672016, + "loss": 1.5522, + "theoretical_loss": 3.886872727821446, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042320962888666, + "loss": 1.5628, + "theoretical_loss": 3.886771289260834, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042318956870611836, + "loss": 1.6368, + "theoretical_loss": 3.886669882519093, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231695085255767, + "loss": 1.6142, + "theoretical_loss": 3.886568507578448, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231494483450351, + "loss": 1.4894, + "theoretical_loss": 3.886467164421142, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231293881644935, + "loss": 1.5009, + "theoretical_loss": 3.886365853029427, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042310932798395186, + "loss": 1.3925, + "theoretical_loss": 3.886264573385574, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230892678034102, + "loss": 1.4485, + "theoretical_loss": 3.886163325471864, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230692076228686, + "loss": 1.4131, + "theoretical_loss": 3.8860621092705956, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4310850501060486, + "objective/train/avg_token_score": 0.006272444501519203, + "objective/train/docs_used": 314102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.511539101600647, + "objective/train/original_loss": 3.6116085052490234, + "objective/train/theoretical_loss": 3.8860115130565984, + "objective/train/tokens_used": 535756800, + "objective/train/unlikelihood_loss": 0.3091060221195221, + "theoretical_loss": 3.8860115130565984, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042304914744232695, + "loss": 1.4495, + "theoretical_loss": 3.8859609247640794, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042302908726178537, + "loss": 1.4349, + "theoretical_loss": 3.885859771934642, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042300902708124373, + "loss": 1.1708, + "theoretical_loss": 3.8857586507646227, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229889669007021, + "loss": 1.4465, + "theoretical_loss": 3.8856575612363753, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042296890672016045, + "loss": 1.5385, + "theoretical_loss": 3.885556503332268, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294884653961887, + "loss": 1.5361, + "theoretical_loss": 3.885455477034683, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292878635907723, + "loss": 1.4312, + "theoretical_loss": 3.8853544823260178, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229087261785356, + "loss": 1.4874, + "theoretical_loss": 3.885253519188682, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042288866599799396, + "loss": 1.431, + "theoretical_loss": 3.8851525876050994, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686058174523, + "loss": 1.3144, + "theoretical_loss": 3.8850516875577092, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284854563691074, + "loss": 1.4203, + "theoretical_loss": 3.884950819028965, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228284854563691, + "loss": 1.5352, + "theoretical_loss": 3.8848499820013327, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.45100194215774536, + "objective/train/avg_token_score": 0.006222711876034737, + "objective/train/docs_used": 314596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7215205430984497, + "objective/train/original_loss": 3.9992692470550537, + "objective/train/theoretical_loss": 3.8847491764572926, + "objective/train/tokens_used": 537395200, + "objective/train/unlikelihood_loss": 0.1294986605644226, + "theoretical_loss": 3.8847491764572926, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042280842527582746, + "loss": 1.4693, + "theoretical_loss": 3.8847491764572926, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227883650952858, + "loss": 1.3508, + "theoretical_loss": 3.884648402379341, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042276830491474424, + "loss": 1.4451, + "theoretical_loss": 3.8845476597499857, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227482447342026, + "loss": 1.41, + "theoretical_loss": 3.8844469485517488, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272818455366096, + "loss": 1.4082, + "theoretical_loss": 3.8843462687671684, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227081243731193, + "loss": 1.3873, + "theoretical_loss": 3.884245620378794, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226880641925777, + "loss": 1.5178, + "theoretical_loss": 3.8841450033691913, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042266800401203616, + "loss": 1.4785, + "theoretical_loss": 3.884044417720938, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226479438314945, + "loss": 1.527, + "theoretical_loss": 3.8839438634166275, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226278836509529, + "loss": 1.4577, + "theoretical_loss": 3.883843340438865, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042260782347041125, + "loss": 1.3839, + "theoretical_loss": 3.8837428487702716, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258776328986966, + "loss": 1.5392, + "theoretical_loss": 3.883642388393481, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422567703109328, + "loss": 1.5504, + "theoretical_loss": 3.8835419592911413, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.32624632120132446, + "objective/train/avg_token_score": 0.007150240708142519, + "objective/train/docs_used": 315427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1273283958435059, + "objective/train/original_loss": 3.809842348098755, + "objective/train/theoretical_loss": 3.8834917564624716, + "objective/train/tokens_used": 539033600, + "objective/train/unlikelihood_loss": 0.1816972941160202, + "theoretical_loss": 3.8834917564624716, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225476429287864, + "loss": 1.3572, + "theoretical_loss": 3.883441561445914, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252758274824475, + "loss": 1.5028, + "theoretical_loss": 3.8833411948404755, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042250752256770317, + "loss": 1.3948, + "theoretical_loss": 3.8832408594575147, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248746238716153, + "loss": 1.2123, + "theoretical_loss": 3.883140555279735, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224674022066199, + "loss": 1.3314, + "theoretical_loss": 3.8830402822898535, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042244734202607825, + "loss": 1.4584, + "theoretical_loss": 3.8829400404706003, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224272818455366, + "loss": 1.4907, + "theoretical_loss": 3.8828398298047206, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042240722166499503, + "loss": 1.5874, + "theoretical_loss": 3.8827396502749734, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223871614844534, + "loss": 1.4599, + "theoretical_loss": 3.882639501864129, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042236710130391176, + "loss": 1.3443, + "theoretical_loss": 3.882539384554974, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223470411233701, + "loss": 1.4215, + "theoretical_loss": 3.8824392983303078, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042232698094282853, + "loss": 1.5224, + "theoretical_loss": 3.8823392431729435, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3801930546760559, + "objective/train/avg_token_score": 0.03133194521069527, + "objective/train/docs_used": 316530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4709320068359375, + "objective/train/original_loss": 4.609292030334473, + "objective/train/theoretical_loss": 3.882239219065708, + "objective/train/tokens_used": 540672000, + "objective/train/unlikelihood_loss": 0.10041511058807373, + "theoretical_loss": 3.882239219065708, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223069207622869, + "loss": 1.531, + "theoretical_loss": 3.882239219065708, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042228686058174526, + "loss": 1.3437, + "theoretical_loss": 3.882139225991441, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222668004012036, + "loss": 1.6213, + "theoretical_loss": 3.882039263932997, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422246740220662, + "loss": 1.3605, + "theoretical_loss": 3.8819393328732446, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222266800401204, + "loss": 1.459, + "theoretical_loss": 3.881839432795064, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220661985957876, + "loss": 1.525, + "theoretical_loss": 3.88173956368135, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221865596790371, + "loss": 1.541, + "theoretical_loss": 3.8816397255150115, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221664994984955, + "loss": 1.2984, + "theoretical_loss": 3.8815399182789703, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221464393179539, + "loss": 1.4195, + "theoretical_loss": 3.881440141956162, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042212637913741226, + "loss": 1.3955, + "theoretical_loss": 3.8813403965295357, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221063189568706, + "loss": 1.4605, + "theoretical_loss": 3.881240681982054, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422086258776329, + "loss": 1.4174, + "theoretical_loss": 3.881140998296694, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042206619859578735, + "loss": 1.4816, + "theoretical_loss": 3.881041345456444, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.44330400228500366, + "objective/train/avg_token_score": 0.020557841286063194, + "objective/train/docs_used": 317270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5593016147613525, + "objective/train/original_loss": 3.649369955062866, + "objective/train/theoretical_loss": 3.8809915305979237, + "objective/train/tokens_used": 542310400, + "objective/train/unlikelihood_loss": 0.21089822053909302, + "theoretical_loss": 3.8809915305979237, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204613841524577, + "loss": 1.4163, + "theoretical_loss": 3.880941723444308, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042202607823470413, + "loss": 1.5283, + "theoretical_loss": 3.8808421322433024, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220060180541625, + "loss": 1.469, + "theoretical_loss": 3.880742571836457, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042198595787362085, + "loss": 1.3293, + "theoretical_loss": 3.880643042206816, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042196589769307927, + "loss": 1.4443, + "theoretical_loss": 3.8805435433374353, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042194583751253763, + "loss": 1.4672, + "theoretical_loss": 3.8804440752113862, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421925777331996, + "loss": 1.4297, + "theoretical_loss": 3.880344637811752, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042190571715145436, + "loss": 1.4319, + "theoretical_loss": 3.8802452311216302, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218856569709127, + "loss": 1.5163, + "theoretical_loss": 3.8801458551241312, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042186559679037114, + "loss": 1.471, + "theoretical_loss": 3.880046509802378, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218455366098295, + "loss": 1.4938, + "theoretical_loss": 3.8799471951395095, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042182547642928786, + "loss": 1.5712, + "theoretical_loss": 3.8798479111186754, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3662634491920471, + "objective/train/avg_token_score": 0.021842803806066513, + "objective/train/docs_used": 318285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.198555588722229, + "objective/train/original_loss": 3.5977704524993896, + "objective/train/theoretical_loss": 3.879748657723039, + "objective/train/tokens_used": 543948800, + "objective/train/unlikelihood_loss": 0.24941086769104004, + "theoretical_loss": 3.879748657723039, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218054162487462, + "loss": 1.4546, + "theoretical_loss": 3.879748657723039, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042178535606820464, + "loss": 1.4712, + "theoretical_loss": 3.879649434935778, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421765295887663, + "loss": 1.6343, + "theoretical_loss": 3.879550242740083, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042174523570712136, + "loss": 1.5444, + "theoretical_loss": 3.879451081119158, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217251755265797, + "loss": 1.6409, + "theoretical_loss": 3.8793519500562192, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004217051153460381, + "loss": 1.521, + "theoretical_loss": 3.879252849534498, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216850551654965, + "loss": 1.4057, + "theoretical_loss": 3.8791537795372366, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042166499498495487, + "loss": 1.6129, + "theoretical_loss": 3.8790547400476925, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042164493480441323, + "loss": 1.5271, + "theoretical_loss": 3.878955731049136, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216248746238716, + "loss": 1.496, + "theoretical_loss": 3.8788567525248494, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042160481444333, + "loss": 1.3628, + "theoretical_loss": 3.8787578044581292, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042158475426278837, + "loss": 1.4522, + "theoretical_loss": 3.8786588868322847, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042156469408224673, + "loss": 1.3902, + "theoretical_loss": 3.8785599996306397, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4580278694629669, + "objective/train/avg_token_score": 0.007631353102624416, + "objective/train/docs_used": 318961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5138821601867676, + "objective/train/original_loss": 3.7409567832946777, + "objective/train/theoretical_loss": 3.8785105674336835, + "objective/train/tokens_used": 545587200, + "objective/train/unlikelihood_loss": 0.14674793183803558, + "theoretical_loss": 3.8785105674336835, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215446339017051, + "loss": 1.4147, + "theoretical_loss": 3.8784611428365285, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215245737211635, + "loss": 1.4939, + "theoretical_loss": 3.878362316433301, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215045135406219, + "loss": 1.4692, + "theoretical_loss": 3.8782635204043188, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042148445336008024, + "loss": 1.4711, + "theoretical_loss": 3.8781647547329565, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214643931795386, + "loss": 1.5982, + "theoretical_loss": 3.8780660194026035, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042144433299899696, + "loss": 1.5872, + "theoretical_loss": 3.8779673143966606, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214242728184554, + "loss": 1.6839, + "theoretical_loss": 3.8778686396985416, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042140421263791374, + "loss": 1.4686, + "theoretical_loss": 3.8777699952916738, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213841524573721, + "loss": 1.6047, + "theoretical_loss": 3.877671381159499, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042136409227683046, + "loss": 1.331, + "theoretical_loss": 3.877572797285469, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213440320962889, + "loss": 1.3915, + "theoretical_loss": 3.877474243653051, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042132397191574724, + "loss": 1.5564, + "theoretical_loss": 3.877375720245725, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.41086265444755554, + "objective/train/avg_token_score": 0.00896799098700285, + "objective/train/docs_used": 320353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4873528480529785, + "objective/train/original_loss": 3.9619879722595215, + "objective/train/theoretical_loss": 3.8772772270469824, + "objective/train/tokens_used": 547225600, + "objective/train/unlikelihood_loss": 0.1763414591550827, + "theoretical_loss": 3.8772772270469824, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213039117352056, + "loss": 1.5364, + "theoretical_loss": 3.8772772270469824, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042128385155466397, + "loss": 1.4582, + "theoretical_loss": 3.877178764040329, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042126379137412233, + "loss": 1.5835, + "theoretical_loss": 3.8770803312092834, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042124373119358075, + "loss": 1.3389, + "theoretical_loss": 3.8769819285373766, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212236710130391, + "loss": 1.518, + "theoretical_loss": 3.8768835560081527, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042120361083249747, + "loss": 1.417, + "theoretical_loss": 3.8767852136051686, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042118355065195583, + "loss": 1.46, + "theoretical_loss": 3.876686901311995, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211634904714143, + "loss": 1.6196, + "theoretical_loss": 3.8765886191122148, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042114343029087266, + "loss": 1.4089, + "theoretical_loss": 3.876490366989423, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042112337011033103, + "loss": 1.4457, + "theoretical_loss": 3.876392144927229, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211033099297894, + "loss": 1.4709, + "theoretical_loss": 3.8762939529092537, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042108324974924775, + "loss": 1.6226, + "theoretical_loss": 3.876195790919132, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042106318956870617, + "loss": 1.5583, + "theoretical_loss": 3.8760976589405107, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4123900234699249, + "objective/train/avg_token_score": 0.014366233721375465, + "objective/train/docs_used": 320989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.500076174736023, + "objective/train/original_loss": 3.8920931816101074, + "objective/train/theoretical_loss": 3.8760486042004056, + "objective/train/tokens_used": 548864000, + "objective/train/unlikelihood_loss": 0.15627138316631317, + "theoretical_loss": 3.8760486042004056, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042104312938816453, + "loss": 1.426, + "theoretical_loss": 3.8759995569570496, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210230692076229, + "loss": 1.4949, + "theoretical_loss": 3.875901484952422, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042100300902708125, + "loss": 1.5319, + "theoretical_loss": 3.8758034429103128, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042098294884653967, + "loss": 1.6009, + "theoretical_loss": 3.875705430814421, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042096288866599803, + "loss": 1.2854, + "theoretical_loss": 3.8756074486484566, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209428284854564, + "loss": 1.3986, + "theoretical_loss": 3.8755094963961447, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042092276830491476, + "loss": 1.4487, + "theoretical_loss": 3.8754115740412214, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209027081243731, + "loss": 1.4448, + "theoretical_loss": 3.8753136815674356, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042088264794383154, + "loss": 1.5002, + "theoretical_loss": 3.875215818958549, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208625877632899, + "loss": 1.6046, + "theoretical_loss": 3.8751179861983367, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084252758274826, + "loss": 1.3805, + "theoretical_loss": 3.8750201832705864, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208224674022066, + "loss": 1.6252, + "theoretical_loss": 3.8749224101590976, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.3899071216583252, + "objective/train/avg_token_score": 0.00873059593141079, + "objective/train/docs_used": 322374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3493757247924805, + "objective/train/original_loss": 3.7656335830688477, + "objective/train/theoretical_loss": 3.8748246668476827, + "objective/train/tokens_used": 550502400, + "objective/train/unlikelihood_loss": 0.2136422097682953, + "theoretical_loss": 3.8748246668476827, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042080240722166504, + "loss": 1.5383, + "theoretical_loss": 3.8748246668476827, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207823470411234, + "loss": 1.4345, + "theoretical_loss": 3.8747269533201676, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042076228686058176, + "loss": 1.4031, + "theoretical_loss": 3.8746292695603897, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207422266800401, + "loss": 1.6027, + "theoretical_loss": 3.8745316155521996, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207221664994985, + "loss": 1.6353, + "theoretical_loss": 3.874433991279461, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207021063189569, + "loss": 1.5923, + "theoretical_loss": 3.874336396726049, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068204613841527, + "loss": 1.5882, + "theoretical_loss": 3.874238831875852, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042066198595787363, + "loss": 1.5571, + "theoretical_loss": 3.8741412967127706, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420641925777332, + "loss": 1.3437, + "theoretical_loss": 3.8740437912207186, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206218655967904, + "loss": 1.463, + "theoretical_loss": 3.873946315383622, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042060180541624877, + "loss": 1.4278, + "theoretical_loss": 3.8738488691854194, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058174523570713, + "loss": 1.2151, + "theoretical_loss": 3.873751452610061, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205616850551655, + "loss": 1.4381, + "theoretical_loss": 3.873654065641511, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.48796433210372925, + "objective/train/avg_token_score": 0.016625795513391495, + "objective/train/docs_used": 322945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7378815412521362, + "objective/train/original_loss": 4.003293037414551, + "objective/train/theoretical_loss": 3.8736053832547808, + "objective/train/tokens_used": 552140800, + "objective/train/unlikelihood_loss": 0.14501631259918213, + "theoretical_loss": 3.8736053832547808, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205416248746239, + "loss": 1.4769, + "theoretical_loss": 3.873556708263745, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205215646940823, + "loss": 1.4547, + "theoretical_loss": 3.8734593804607513, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042050150451354064, + "loss": 1.5517, + "theoretical_loss": 3.8733620822165316, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420481444332999, + "loss": 1.4435, + "theoretical_loss": 3.8732648135150978, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046138415245736, + "loss": 1.565, + "theoretical_loss": 3.873167574340477, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204413239719158, + "loss": 1.3768, + "theoretical_loss": 3.873070364676707, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042126379137414, + "loss": 1.5248, + "theoretical_loss": 3.8729731845078383, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204012036108325, + "loss": 1.1907, + "theoretical_loss": 3.872876033817934, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038114343029086, + "loss": 1.432, + "theoretical_loss": 3.872778912591069, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203610832497493, + "loss": 1.4321, + "theoretical_loss": 3.8726818208113323, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034102306920764, + "loss": 1.3276, + "theoretical_loss": 3.872584758462823, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420320962888666, + "loss": 1.6415, + "theoretical_loss": 3.8724877255296533, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4739125072956085, + "objective/train/avg_token_score": 0.016837503761053085, + "objective/train/docs_used": 324208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4981387853622437, + "objective/train/original_loss": 3.6097116470336914, + "objective/train/theoretical_loss": 3.8723907219959486, + "objective/train/tokens_used": 553779200, + "objective/train/unlikelihood_loss": 0.127224862575531, + "theoretical_loss": 3.8723907219959486, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042030090270812437, + "loss": 1.446, + "theoretical_loss": 3.8723907219959486, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028084252758273, + "loss": 1.4663, + "theoretical_loss": 3.872293747845846, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042026078234704115, + "loss": 1.4597, + "theoretical_loss": 3.872196803063495, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202407221664995, + "loss": 1.5731, + "theoretical_loss": 3.8720998876330572, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042022066198595787, + "loss": 1.4404, + "theoretical_loss": 3.8720030015387064, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042020060180541623, + "loss": 1.4862, + "theoretical_loss": 3.8719061447646297, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042018054162487465, + "loss": 1.2866, + "theoretical_loss": 3.8718093172950248, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420160481444333, + "loss": 1.4176, + "theoretical_loss": 3.8717125191141024, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201404212637914, + "loss": 1.2709, + "theoretical_loss": 3.8716157502060864, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042012036108324974, + "loss": 1.4054, + "theoretical_loss": 3.871519010555211, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201003009027081, + "loss": 1.329, + "theoretical_loss": 3.871422300145725, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200802407221665, + "loss": 1.2811, + "theoretical_loss": 3.871325618961887, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200601805416249, + "loss": 1.4898, + "theoretical_loss": 3.871228966987969, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.40713587403297424, + "objective/train/avg_token_score": 0.009826954454183578, + "objective/train/docs_used": 324930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.531923532485962, + "objective/train/original_loss": 3.965177297592163, + "objective/train/theoretical_loss": 3.871180651949819, + "objective/train/tokens_used": 555417600, + "objective/train/unlikelihood_loss": 0.14890779554843903, + "theoretical_loss": 3.871180651949819, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004012036108324, + "loss": 1.4487, + "theoretical_loss": 3.871132344208256, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200200601805416, + "loss": 1.3508, + "theoretical_loss": 3.8710357506070427, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 1.5513, + "theoretical_loss": 3.870939186168638, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199799398194584, + "loss": 1.314, + "theoretical_loss": 3.8708426508773632, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041995987963891674, + "loss": 1.319, + "theoretical_loss": 3.8707461447175495, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199398194583751, + "loss": 1.4497, + "theoretical_loss": 3.870649667673543, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041991975927783347, + "loss": 1.3447, + "theoretical_loss": 3.8705532197296995, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198996990972919, + "loss": 1.375, + "theoretical_loss": 3.870456800870388, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987963891675025, + "loss": 1.4515, + "theoretical_loss": 3.8703604110799903, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198595787362086, + "loss": 1.3271, + "theoretical_loss": 3.8702640503428984, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983951855566697, + "loss": 1.3552, + "theoretical_loss": 3.8701677186435184, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198194583751254, + "loss": 1.4606, + "theoretical_loss": 3.8700714159662666, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.32086998224258423, + "objective/train/avg_token_score": 0.02666344866156578, + "objective/train/docs_used": 326252, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0482410192489624, + "objective/train/original_loss": 4.232210636138916, + "objective/train/theoretical_loss": 3.869975142295573, + "objective/train/tokens_used": 557056000, + "objective/train/unlikelihood_loss": 0.14249522984027863, + "theoretical_loss": 3.869975142295573, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041979939819458375, + "loss": 1.5389, + "theoretical_loss": 3.869975142295573, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197793380140421, + "loss": 1.3538, + "theoretical_loss": 3.8698788976158776, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975927783350047, + "loss": 1.315, + "theoretical_loss": 3.8697826819116345, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973921765295883, + "loss": 1.6038, + "theoretical_loss": 3.8696864951673087, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971915747241725, + "loss": 1.4843, + "theoretical_loss": 3.8695903373673772, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196990972918756, + "loss": 1.5741, + "theoretical_loss": 3.869494208496329, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419679037111334, + "loss": 1.5376, + "theoretical_loss": 3.8693981085386655, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196589769307924, + "loss": 1.4148, + "theoretical_loss": 3.8693020374788993, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196389167502508, + "loss": 1.5087, + "theoretical_loss": 3.8692059953015563, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961885656970917, + "loss": 1.2978, + "theoretical_loss": 3.8691099819911723, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041959879638916753, + "loss": 1.468, + "theoretical_loss": 3.8690139975322966, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195787362086259, + "loss": 1.275, + "theoretical_loss": 3.8689180419094895, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195586760280843, + "loss": 1.5295, + "theoretical_loss": 3.868822115107324, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.48692572116851807, + "objective/train/avg_token_score": 0.008406232111155987, + "objective/train/docs_used": 326886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.637534499168396, + "objective/train/original_loss": 3.788033962249756, + "objective/train/theoretical_loss": 3.8687741625091636, + "objective/train/tokens_used": 558694400, + "objective/train/unlikelihood_loss": 0.1396278291940689, + "theoretical_loss": 3.8687741625091636, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195386158475427, + "loss": 1.4419, + "theoretical_loss": 3.8687262171103836, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041951855566700104, + "loss": 1.6244, + "theoretical_loss": 3.868630347903266, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194984954864594, + "loss": 1.3994, + "theoretical_loss": 3.868534507470579, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041947843530591776, + "loss": 1.4045, + "theoretical_loss": 3.868438695796941, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194583751253762, + "loss": 1.3849, + "theoretical_loss": 3.8683429128669857, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943831494483454, + "loss": 1.3363, + "theoretical_loss": 3.868247158665356, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194182547642929, + "loss": 1.3883, + "theoretical_loss": 3.868151433176707, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041939819458375126, + "loss": 1.5001, + "theoretical_loss": 3.868055736385706, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193781344032097, + "loss": 1.4969, + "theoretical_loss": 3.867960068277033, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041935807422266804, + "loss": 1.3821, + "theoretical_loss": 3.867864428835377, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193380140421264, + "loss": 1.4965, + "theoretical_loss": 3.8677688180454415, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041931795386158477, + "loss": 1.5585, + "theoretical_loss": 3.8676732358919406, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.2959432899951935, + "objective/train/avg_token_score": 0.025653989985585213, + "objective/train/docs_used": 327930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9278034567832947, + "objective/train/original_loss": 3.611701011657715, + "objective/train/theoretical_loss": 3.8675776823595998, + "objective/train/tokens_used": 560332800, + "objective/train/unlikelihood_loss": 0.2902003228664398, + "theoretical_loss": 3.8675776823595998, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929789368104313, + "loss": 1.3127, + "theoretical_loss": 3.8675776823595998, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927783350050155, + "loss": 1.5624, + "theoretical_loss": 3.867482157433157, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192577733199599, + "loss": 1.3398, + "theoretical_loss": 3.8673866610973624, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923771313941827, + "loss": 1.4846, + "theoretical_loss": 3.867291193336976, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041921765295887663, + "loss": 1.5351, + "theoretical_loss": 3.867195754136771, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919759277833505, + "loss": 1.4848, + "theoretical_loss": 3.867100343481532, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191775325977934, + "loss": 1.3812, + "theoretical_loss": 3.867004961356054, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191574724172518, + "loss": 1.3775, + "theoretical_loss": 3.8669096077451464, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913741223671014, + "loss": 1.6188, + "theoretical_loss": 3.8668142826336274, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191173520561685, + "loss": 1.4986, + "theoretical_loss": 3.8667189860063282, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190972918756269, + "loss": 1.3638, + "theoretical_loss": 3.866623717848092, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190772316950853, + "loss": 1.4552, + "theoretical_loss": 3.8665284781437714, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041905717151454364, + "loss": 1.4597, + "theoretical_loss": 3.866433266878234, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.5047653913497925, + "objective/train/avg_token_score": 0.00482219597324729, + "objective/train/docs_used": 328746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.737789511680603, + "objective/train/original_loss": 3.9125659465789795, + "objective/train/theoretical_loss": 3.8663856719052827, + "objective/train/tokens_used": 561971200, + "objective/train/unlikelihood_loss": 0.14458067715168, + "theoretical_loss": 3.8663856719052827, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419037111334002, + "loss": 1.5193, + "theoretical_loss": 3.8663380840363564, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190170511534604, + "loss": 1.362, + "theoretical_loss": 3.866242929603027, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189969909729188, + "loss": 1.4048, + "theoretical_loss": 3.866147803563148, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041897693079237714, + "loss": 1.6211, + "theoretical_loss": 3.8660527059016294, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189568706118355, + "loss": 1.4695, + "theoretical_loss": 3.8659576366033956, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041893681043129387, + "loss": 1.4011, + "theoretical_loss": 3.8658625956533816, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189167502507523, + "loss": 1.3756, + "theoretical_loss": 3.8657675830365346, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041889669007021065, + "loss": 1.4237, + "theoretical_loss": 3.8656725987378118, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418876629889669, + "loss": 1.4832, + "theoretical_loss": 3.865577642742183, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041885656970912737, + "loss": 1.4726, + "theoretical_loss": 3.86548271503463, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188365095285858, + "loss": 1.3446, + "theoretical_loss": 3.865387815600144, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041881644934804415, + "loss": 1.5273, + "theoretical_loss": 3.86529294442373, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.6345918774604797, + "objective/train/avg_token_score": 0.00891077984124422, + "objective/train/docs_used": 329425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0925443172454834, + "objective/train/original_loss": 3.6826608180999756, + "objective/train/theoretical_loss": 3.8651981014904027, + "objective/train/tokens_used": 563609600, + "objective/train/unlikelihood_loss": 0.14516757428646088, + "theoretical_loss": 3.8651981014904027, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187963891675025, + "loss": 1.452, + "theoretical_loss": 3.8651981014904027, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187763289869609, + "loss": 1.2785, + "theoretical_loss": 3.8651032867851893, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041875626880641924, + "loss": 1.3895, + "theoretical_loss": 3.8650085002931283, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041873620862587765, + "loss": 1.4778, + "theoretical_loss": 3.8649137419992687, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418716148445336, + "loss": 1.4185, + "theoretical_loss": 3.8648190118886725, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186960882647944, + "loss": 1.4829, + "theoretical_loss": 3.8647243099464106, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041867602808425274, + "loss": 1.6106, + "theoretical_loss": 3.8646296361575687, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041865596790371115, + "loss": 1.3867, + "theoretical_loss": 3.8645349905072406, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186359077231695, + "loss": 1.4051, + "theoretical_loss": 3.864440372980533, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186158475426279, + "loss": 1.5175, + "theoretical_loss": 3.8643457835625634, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041859578736208624, + "loss": 1.4996, + "theoretical_loss": 3.864251222238462, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185757271815446, + "loss": 1.4721, + "theoretical_loss": 3.864156688993369, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418555667001003, + "loss": 1.4107, + "theoretical_loss": 3.864062183812436, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.37713831663131714, + "objective/train/avg_token_score": 0.01374147180467844, + "objective/train/docs_used": 329612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2935733795166016, + "objective/train/original_loss": 3.755645275115967, + "objective/train/theoretical_loss": 3.864014941741392, + "objective/train/tokens_used": 565248000, + "objective/train/unlikelihood_loss": 0.1588522344827652, + "theoretical_loss": 3.864014941741392, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185356068204614, + "loss": 1.5042, + "theoretical_loss": 3.8639677066808256, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041851554663991974, + "loss": 1.5188, + "theoretical_loss": 3.8638732575837125, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184954864593781, + "loss": 1.5116, + "theoretical_loss": 3.863778836506283, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184754262788365, + "loss": 1.5429, + "theoretical_loss": 3.8636844434337325, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184553660982949, + "loss": 1.6697, + "theoretical_loss": 3.863590078351271, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041843530591775325, + "loss": 1.4738, + "theoretical_loss": 3.8634957412441167, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184152457372116, + "loss": 1.5514, + "theoretical_loss": 3.8634014320975005, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041839518555666997, + "loss": 1.5185, + "theoretical_loss": 3.863307150896664, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183751253761284, + "loss": 1.4115, + "theoretical_loss": 3.8632128976268603, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041835506519558675, + "loss": 1.4309, + "theoretical_loss": 3.8631186722733535, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183350050150451, + "loss": 1.5064, + "theoretical_loss": 3.8630244748214198, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183149448345035, + "loss": 1.5129, + "theoretical_loss": 3.862930305256345, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4685361683368683, + "objective/train/avg_token_score": 0.020750796422362328, + "objective/train/docs_used": 331071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7844135761260986, + "objective/train/original_loss": 4.0298967361450195, + "objective/train/theoretical_loss": 3.8628361635634265, + "objective/train/tokens_used": 566886400, + "objective/train/unlikelihood_loss": 0.1526690572500229, + "theoretical_loss": 3.8628361635634265, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182948846539619, + "loss": 1.4882, + "theoretical_loss": 3.8628361635634265, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041827482447342025, + "loss": 1.5128, + "theoretical_loss": 3.8627420497279745, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182547642928786, + "loss": 1.6453, + "theoretical_loss": 3.8626479637353075, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418234704112337, + "loss": 1.5273, + "theoretical_loss": 3.8625539055707576, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182146439317954, + "loss": 1.5139, + "theoretical_loss": 3.862459875219667, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041819458375125376, + "loss": 1.4496, + "theoretical_loss": 3.8623658726673886, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181745235707121, + "loss": 1.5045, + "theoretical_loss": 3.862271897899287, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041815446339017054, + "loss": 1.5649, + "theoretical_loss": 3.8621779509007377, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181344032096289, + "loss": 1.4613, + "theoretical_loss": 3.862084031657128, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181143430290873, + "loss": 1.5159, + "theoretical_loss": 3.8619901401538543, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180942828485457, + "loss": 1.5407, + "theoretical_loss": 3.861896276376326, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041807422266800404, + "loss": 1.4418, + "theoretical_loss": 3.8618024403099636, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180541624874624, + "loss": 1.4974, + "theoretical_loss": 3.8617086319401968, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.3284457325935364, + "objective/train/avg_token_score": 0.011517138220369816, + "objective/train/docs_used": 331416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2020608186721802, + "objective/train/original_loss": 4.407777309417725, + "objective/train/theoretical_loss": 3.861661738136987, + "objective/train/tokens_used": 568524800, + "objective/train/unlikelihood_loss": 0.14563091099262238, + "theoretical_loss": 3.861661738136987, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180341023069208, + "loss": 1.498, + "theoretical_loss": 3.8616148512524675, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180140421263792, + "loss": 1.51, + "theoretical_loss": 3.8615210982322288, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799398194583754, + "loss": 1.59, + "theoretical_loss": 3.8614273728649446, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179739217652959, + "loss": 1.426, + "theoretical_loss": 3.8613336751360894, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041795386158475427, + "loss": 1.6531, + "theoretical_loss": 3.8612400050311493, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179338014042127, + "loss": 1.4923, + "theoretical_loss": 3.861146362535621, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041791374122367105, + "loss": 1.5301, + "theoretical_loss": 3.8610527476350116, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178936810431294, + "loss": 1.4552, + "theoretical_loss": 3.86095916031484, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041787362086258777, + "loss": 1.4689, + "theoretical_loss": 3.8608656005606354, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178535606820462, + "loss": 1.5132, + "theoretical_loss": 3.8607720683579396, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783350050150455, + "loss": 1.4345, + "theoretical_loss": 3.8606785636923027, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178134403209629, + "loss": 1.5175, + "theoretical_loss": 3.8605850865492872, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.43047410249710083, + "objective/train/avg_token_score": 0.02064175345003605, + "objective/train/docs_used": 332819, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4973698854446411, + "objective/train/original_loss": 3.5845766067504883, + "objective/train/theoretical_loss": 3.8604916369144666, + "objective/train/tokens_used": 570163200, + "objective/train/unlikelihood_loss": 0.2515372037887573, + "theoretical_loss": 3.8604916369144666, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177933801404213, + "loss": 1.4534, + "theoretical_loss": 3.8604916369144666, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041777331995987964, + "loss": 1.6288, + "theoretical_loss": 3.860398214773425, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041775325977933805, + "loss": 1.4872, + "theoretical_loss": 3.8603048201117565, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177331995987964, + "loss": 1.4164, + "theoretical_loss": 3.8602114529150673, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177131394182548, + "loss": 1.525, + "theoretical_loss": 3.8601181131689737, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041769307923771314, + "loss": 1.438, + "theoretical_loss": 3.8600248008591045, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041767301905717156, + "loss": 1.4738, + "theoretical_loss": 3.8599315159710965, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176529588766299, + "loss": 1.6646, + "theoretical_loss": 3.8598382584905995, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176328986960883, + "loss": 1.5638, + "theoretical_loss": 3.859745028403273, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761283851554664, + "loss": 1.6145, + "theoretical_loss": 3.859651825694787, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417592778335005, + "loss": 1.4472, + "theoretical_loss": 3.8595586503508246, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175727181544634, + "loss": 1.3499, + "theoretical_loss": 3.8594655023570765, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175526579739218, + "loss": 1.6236, + "theoretical_loss": 3.859372381699247, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.31506597995758057, + "objective/train/avg_token_score": 0.016959313303232193, + "objective/train/docs_used": 333556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9501151442527771, + "objective/train/original_loss": 3.510495901107788, + "objective/train/theoretical_loss": 3.859325831616836, + "objective/train/tokens_used": 571801600, + "objective/train/unlikelihood_loss": 0.19363252818584442, + "theoretical_loss": 3.859325831616836, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753259779338015, + "loss": 1.4159, + "theoretical_loss": 3.859279288363048, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175125376128385, + "loss": 1.6299, + "theoretical_loss": 3.8591862223342064, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174924774322969, + "loss": 1.5366, + "theoretical_loss": 3.859093183598455, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174724172517553, + "loss": 1.4375, + "theoretical_loss": 3.859000172141541, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745235707121365, + "loss": 1.5723, + "theoretical_loss": 3.858907187949221, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417432296890672, + "loss": 1.6735, + "theoretical_loss": 3.858814231007262, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041741223671013037, + "loss": 1.3602, + "theoretical_loss": 3.858721301301442, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173921765295888, + "loss": 1.4342, + "theoretical_loss": 3.8586283988175496, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041737211634904715, + "loss": 1.4224, + "theoretical_loss": 3.8585355235413843, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173520561685055, + "loss": 1.5072, + "theoretical_loss": 3.858442675458756, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173319959879639, + "loss": 1.5743, + "theoretical_loss": 3.858349854555486, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173119358074223, + "loss": 1.5621, + "theoretical_loss": 3.8582570608174045, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.29276636242866516, + "objective/train/avg_token_score": 0.03536292165517807, + "objective/train/docs_used": 334721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1436461210250854, + "objective/train/original_loss": 4.6639227867126465, + "objective/train/theoretical_loss": 3.858164294230354, + "objective/train/tokens_used": 573440000, + "objective/train/unlikelihood_loss": 0.10083737224340439, + "theoretical_loss": 3.858164294230354, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729187562688065, + "loss": 1.4795, + "theoretical_loss": 3.858164294230354, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417271815446339, + "loss": 1.5866, + "theoretical_loss": 3.8580715547801865, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172517552657974, + "loss": 1.678, + "theoretical_loss": 3.8579788424527655, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172316950852558, + "loss": 1.5994, + "theoretical_loss": 3.857886157233965, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041721163490471416, + "loss": 1.4772, + "theoretical_loss": 3.857793499109669, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171915747241725, + "loss": 1.5592, + "theoretical_loss": 3.8577008680657716, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171715145436309, + "loss": 1.5461, + "theoretical_loss": 3.8576082640881797, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041715145436308924, + "loss": 1.5339, + "theoretical_loss": 3.8575156871628082, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713139418254766, + "loss": 1.5063, + "theoretical_loss": 3.8574231372755836, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417111334002006, + "loss": 1.5664, + "theoretical_loss": 3.8573306144124437, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170912738214644, + "loss": 1.4587, + "theoretical_loss": 3.8572381185593354, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041707121364092275, + "loss": 1.5352, + "theoretical_loss": 3.8571456497022165, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041705115346038116, + "loss": 1.7719, + "theoretical_loss": 3.8570532078270565, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.410740464925766, + "objective/train/avg_token_score": 0.012464967556297779, + "objective/train/docs_used": 335233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4862878322601318, + "objective/train/original_loss": 3.868055820465088, + "objective/train/theoretical_loss": 3.857006997003329, + "objective/train/tokens_used": 575078400, + "objective/train/unlikelihood_loss": 0.2018526941537857, + "theoretical_loss": 3.857006997003329, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170310932798395, + "loss": 1.5556, + "theoretical_loss": 3.8569607929198337, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170110330992979, + "loss": 1.5718, + "theoretical_loss": 3.856868404966538, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041699097291875625, + "loss": 1.6003, + "theoretical_loss": 3.856776043953169, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169709127382146, + "loss": 1.6744, + "theoretical_loss": 3.856683709865737, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041695085255767303, + "loss": 1.4031, + "theoretical_loss": 3.856591402690264, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169307923771314, + "loss": 1.5336, + "theoretical_loss": 3.856499122412781, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041691073219658975, + "loss": 1.6005, + "theoretical_loss": 3.856406869019329, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168906720160481, + "loss": 1.2878, + "theoretical_loss": 3.856314642495961, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041687061183550653, + "loss": 1.4465, + "theoretical_loss": 3.8562224428287393, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168505516549649, + "loss": 1.3374, + "theoretical_loss": 3.856130270003736, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041683049147442326, + "loss": 1.6655, + "theoretical_loss": 3.8560381240070365, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168104312938816, + "loss": 1.6154, + "theoretical_loss": 3.855946004824733, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.3080400824546814, + "objective/train/avg_token_score": 0.017052171751856804, + "objective/train/docs_used": 335838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9714576005935669, + "objective/train/original_loss": 3.790057897567749, + "objective/train/theoretical_loss": 3.85585391244293, + "objective/train/tokens_used": 576716800, + "objective/train/unlikelihood_loss": 0.24715007841587067, + "theoretical_loss": 3.85585391244293, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041679037111334, + "loss": 1.4714, + "theoretical_loss": 3.85585391244293, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167703109327984, + "loss": 1.5795, + "theoretical_loss": 3.8557618468477415, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041675025075225676, + "loss": 1.5285, + "theoretical_loss": 3.8556698080252936, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167301905717151, + "loss": 1.4149, + "theoretical_loss": 3.8555777959617203, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167101303911735, + "loss": 1.3468, + "theoretical_loss": 3.8554858106431675, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166900702106319, + "loss": 1.5689, + "theoretical_loss": 3.8553938520557915, + "tokens_seen": 577372160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041667001003009026, + "loss": 1.5243, + "theoretical_loss": 3.8553019201857577, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166499498495487, + "loss": 1.4107, + "theoretical_loss": 3.8552100150192423, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041662988966900704, + "loss": 1.4474, + "theoretical_loss": 3.855118136542433, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166098294884654, + "loss": 1.4431, + "theoretical_loss": 3.855026284741526, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165897693079238, + "loss": 1.4762, + "theoretical_loss": 3.8549344596027293, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165697091273822, + "loss": 1.3957, + "theoretical_loss": 3.8548426611122597, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654964894684055, + "loss": 1.3942, + "theoretical_loss": 3.854750889256345, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.43218475580215454, + "objective/train/avg_token_score": 0.007149472367018461, + "objective/train/docs_used": 337192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.293778657913208, + "objective/train/original_loss": 4.05076265335083, + "objective/train/theoretical_loss": 3.854705013312045, + "objective/train/tokens_used": 578355200, + "objective/train/unlikelihood_loss": 0.13084176182746887, + "theoretical_loss": 3.854705013312045, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165295887662989, + "loss": 1.5097, + "theoretical_loss": 3.8546591440212237, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165095285857573, + "loss": 1.4131, + "theoretical_loss": 3.8545674253931432, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164894684052157, + "loss": 1.4509, + "theoretical_loss": 3.8544757333583632, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041646940822467405, + "loss": 1.3605, + "theoretical_loss": 3.854384067903151, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164493480441324, + "loss": 1.5583, + "theoretical_loss": 3.8542924290137863, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164292878635908, + "loss": 1.3599, + "theoretical_loss": 3.8542008166765576, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164092276830492, + "loss": 1.4714, + "theoretical_loss": 3.8541092308777647, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638916750250755, + "loss": 1.5355, + "theoretical_loss": 3.8540176716037164, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163691073219659, + "loss": 1.2708, + "theoretical_loss": 3.8539261388407327, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163490471414243, + "loss": 1.5444, + "theoretical_loss": 3.853834632575143, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163289869608827, + "loss": 1.4964, + "theoretical_loss": 3.853743152793287, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041630892678034105, + "loss": 1.4589, + "theoretical_loss": 3.8536516994815146, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.32331377267837524, + "objective/train/avg_token_score": 0.055630918592214584, + "objective/train/docs_used": 337878, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.113051414489746, + "objective/train/original_loss": 4.3019514083862305, + "objective/train/theoretical_loss": 3.8535602726261864, + "objective/train/tokens_used": 579993600, + "objective/train/unlikelihood_loss": 0.16012266278266907, + "theoretical_loss": 3.8535602726261864, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162888665997994, + "loss": 1.2567, + "theoretical_loss": 3.8535602726261864, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162688064192578, + "loss": 1.4235, + "theoretical_loss": 3.8534688722136723, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162487462387162, + "loss": 1.544, + "theoretical_loss": 3.8533774982303526, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622868605817456, + "loss": 1.4034, + "theoretical_loss": 3.8532861506626173, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162086258776329, + "loss": 1.5044, + "theoretical_loss": 3.8531948294968674, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161885656970913, + "loss": 1.5761, + "theoretical_loss": 3.853103534719513, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041616850551654964, + "loss": 1.4135, + "theoretical_loss": 3.8530122663169752, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614844533600806, + "loss": 1.41, + "theoretical_loss": 3.852921024275684, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161283851554664, + "loss": 1.3965, + "theoretical_loss": 3.8528298085820802, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161083249749248, + "loss": 1.4879, + "theoretical_loss": 3.852738619222615, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041608826479438315, + "loss": 1.4855, + "theoretical_loss": 3.8526474561837483, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041606820461384156, + "loss": 1.4659, + "theoretical_loss": 3.852556319451952, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160481444332999, + "loss": 1.5415, + "theoretical_loss": 3.852465209013705, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.415628045797348, + "objective/train/avg_token_score": 0.01404473278671503, + "objective/train/docs_used": 339259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.375450611114502, + "objective/train/original_loss": 3.8198952674865723, + "objective/train/theoretical_loss": 3.8524196636504415, + "objective/train/tokens_used": 581632000, + "objective/train/unlikelihood_loss": 0.21155038475990295, + "theoretical_loss": 3.8524196636504415, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160280842527583, + "loss": 1.3202, + "theoretical_loss": 3.8523741248555, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041600802407221665, + "loss": 1.3599, + "theoretical_loss": 3.852283066963836, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415987963891675, + "loss": 1.5542, + "theoretical_loss": 3.852192035325226, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041596790371113343, + "loss": 1.5125, + "theoretical_loss": 3.852101029926188, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159478435305918, + "loss": 1.4721, + "theoretical_loss": 3.852010050753254, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041592778335005015, + "loss": 1.3842, + "theoretical_loss": 3.8519190977929645, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159077231695085, + "loss": 1.4271, + "theoretical_loss": 3.85182817103187, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041588766298896693, + "loss": 1.385, + "theoretical_loss": 3.85173727045653, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158676028084253, + "loss": 1.3607, + "theoretical_loss": 3.851646396053516, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041584754262788366, + "loss": 1.6908, + "theoretical_loss": 3.851555547809408, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415827482447342, + "loss": 1.5575, + "theoretical_loss": 3.8514647257107955, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158074222668004, + "loss": 1.5368, + "theoretical_loss": 3.8513739297442786, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.37939882278442383, + "objective/train/avg_token_score": 0.012932621873915195, + "objective/train/docs_used": 340083, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3577672243118286, + "objective/train/original_loss": 3.419050931930542, + "objective/train/theoretical_loss": 3.851283159896468, + "objective/train/tokens_used": 583270400, + "objective/train/unlikelihood_loss": 0.294486939907074, + "theoretical_loss": 3.851283159896468, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157873620862588, + "loss": 1.4448, + "theoretical_loss": 3.851283159896468, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041576730190571716, + "loss": 1.4166, + "theoretical_loss": 3.8511924161539826, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157472417251755, + "loss": 1.4534, + "theoretical_loss": 3.8511016985034523, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157271815446339, + "loss": 1.5211, + "theoretical_loss": 3.8510110069315173, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157071213640923, + "loss": 1.2879, + "theoretical_loss": 3.8509203414248256, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041568706118355066, + "loss": 1.4893, + "theoretical_loss": 3.850829701970037, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415667001003009, + "loss": 1.2667, + "theoretical_loss": 3.85073908855382, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156469408224674, + "loss": 1.4526, + "theoretical_loss": 3.8506485011628544, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041562688064192575, + "loss": 1.4288, + "theoretical_loss": 3.850557939783828, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041560682046138417, + "loss": 1.6138, + "theoretical_loss": 3.8504674044034393, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041558676028084253, + "loss": 1.633, + "theoretical_loss": 3.850376895008396, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155667001003009, + "loss": 1.4531, + "theoretical_loss": 3.850286411585417, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041554663991975925, + "loss": 1.4509, + "theoretical_loss": 3.8501959541212294, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.31414955854415894, + "objective/train/avg_token_score": 0.012615097686648369, + "objective/train/docs_used": 341398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1601176261901855, + "objective/train/original_loss": 4.111655235290527, + "objective/train/theoretical_loss": 3.850150735119537, + "objective/train/tokens_used": 584908800, + "objective/train/unlikelihood_loss": 0.15587183833122253, + "theoretical_loss": 3.850150735119537, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041552657973921767, + "loss": 1.4771, + "theoretical_loss": 3.8501055226025707, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041550651955867603, + "loss": 1.454, + "theoretical_loss": 3.850015117016188, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154864593781344, + "loss": 1.5285, + "theoretical_loss": 3.8499247373488377, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041546639919759276, + "loss": 1.5182, + "theoretical_loss": 3.8498343835872872, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154463390170511, + "loss": 1.5975, + "theoretical_loss": 3.8497440557183134, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041542627883650954, + "loss": 1.6371, + "theoretical_loss": 3.849653753728701, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154062186559679, + "loss": 1.5461, + "theoretical_loss": 3.8495634776052463, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041538615847542626, + "loss": 1.2369, + "theoretical_loss": 3.849473227334755, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153660982948846, + "loss": 1.4595, + "theoretical_loss": 3.849383002904043, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041534603811434304, + "loss": 1.3872, + "theoretical_loss": 3.8492928042999335, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153259779338014, + "loss": 1.4741, + "theoretical_loss": 3.849202631509262, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041530591775325976, + "loss": 1.3709, + "theoretical_loss": 3.8491124845188716, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.35477760434150696, + "objective/train/avg_token_score": 0.018234847113490105, + "objective/train/docs_used": 342071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2699406147003174, + "objective/train/original_loss": 4.259076118469238, + "objective/train/theoretical_loss": 3.8490223633156173, + "objective/train/tokens_used": 586547200, + "objective/train/unlikelihood_loss": 0.1685103178024292, + "theoretical_loss": 3.8490223633156173, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152858575727181, + "loss": 1.4075, + "theoretical_loss": 3.8490223633156173, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152657973921765, + "loss": 1.3445, + "theoretical_loss": 3.8489322678863624, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152457372116349, + "loss": 1.4237, + "theoretical_loss": 3.848842198217979, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041522567703109327, + "loss": 1.4361, + "theoretical_loss": 3.84875215429735, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041520561685055163, + "loss": 1.3083, + "theoretical_loss": 3.8486621361113684, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041518555667001, + "loss": 1.3499, + "theoretical_loss": 3.848572143646936, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151654964894684, + "loss": 1.521, + "theoretical_loss": 3.8484821768909634, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151454363089268, + "loss": 1.5277, + "theoretical_loss": 3.8483922358303717, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151253761283852, + "loss": 1.4885, + "theoretical_loss": 3.8483023204520928, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041510531594784355, + "loss": 1.5283, + "theoretical_loss": 3.848212430743065, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041508525576730196, + "loss": 1.446, + "theoretical_loss": 3.848122566690239, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150651955867603, + "loss": 1.4859, + "theoretical_loss": 3.8480327282805744, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150451354062187, + "loss": 1.3935, + "theoretical_loss": 3.847942915501039, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4999389052391052, + "objective/train/avg_token_score": 0.008891026489436626, + "objective/train/docs_used": 342650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.745088815689087, + "objective/train/original_loss": 3.872380256652832, + "objective/train/theoretical_loss": 3.8478980187185003, + "objective/train/tokens_used": 588185600, + "objective/train/unlikelihood_loss": 0.1340382695198059, + "theoretical_loss": 3.8478980187185003, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041502507522567705, + "loss": 1.3831, + "theoretical_loss": 3.8478531283386124, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150050150451354, + "loss": 1.4038, + "theoretical_loss": 3.847763366780281, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498495486459383, + "loss": 1.5453, + "theoretical_loss": 3.8476736308130426, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149648946840522, + "loss": 1.4038, + "theoretical_loss": 3.8475839204239044, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041494483450351055, + "loss": 1.3637, + "theoretical_loss": 3.8474942355998825, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149247743229689, + "loss": 1.4316, + "theoretical_loss": 3.8474045763280023, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041490471414242733, + "loss": 1.6323, + "theoretical_loss": 3.8473149425953, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148846539618857, + "loss": 1.4029, + "theoretical_loss": 3.8472253343888196, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041486459378134406, + "loss": 1.4195, + "theoretical_loss": 3.8471357516956153, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148445336008024, + "loss": 1.5306, + "theoretical_loss": 3.8470461945027505, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148244734202608, + "loss": 1.4443, + "theoretical_loss": 3.8469566627972993, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148044132397192, + "loss": 1.4823, + "theoretical_loss": 3.8468671565663426, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.5726417303085327, + "objective/train/avg_token_score": 0.010895898565649986, + "objective/train/docs_used": 343935, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0607388019561768, + "objective/train/original_loss": 3.6878914833068848, + "objective/train/theoretical_loss": 3.846777675796974, + "objective/train/tokens_used": 589824000, + "objective/train/unlikelihood_loss": 0.14677737653255463, + "theoretical_loss": 3.846777675796974, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041478435305917756, + "loss": 1.3588, + "theoretical_loss": 3.846777675796974, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147642928786359, + "loss": 1.5026, + "theoretical_loss": 3.8466882204762936, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147442326980943, + "loss": 1.3238, + "theoretical_loss": 3.846598790591412, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147241725175527, + "loss": 1.5089, + "theoretical_loss": 3.8465093861294504, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041470411233701106, + "loss": 1.4212, + "theoretical_loss": 3.846420007077537, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146840521564694, + "loss": 1.4739, + "theoretical_loss": 3.846330653422812, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146639919759278, + "loss": 1.501, + "theoretical_loss": 3.8462413251524215, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464393179538615, + "loss": 1.5693, + "theoretical_loss": 3.8461520222535253, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041462387161484457, + "loss": 1.5297, + "theoretical_loss": 3.846062744713289, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041460381143430293, + "loss": 1.4618, + "theoretical_loss": 3.8459734925188895, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145837512537613, + "loss": 1.3261, + "theoretical_loss": 3.8458842656575123, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041456369107321965, + "loss": 1.5224, + "theoretical_loss": 3.8457950641163516, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454363089267807, + "loss": 1.3278, + "theoretical_loss": 3.845705887882612, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4558895528316498, + "objective/train/avg_token_score": 0.015658829361200333, + "objective/train/docs_used": 344744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6233446598052979, + "objective/train/original_loss": 3.805819511413574, + "objective/train/theoretical_loss": 3.845661309252029, + "objective/train/tokens_used": 591462400, + "objective/train/unlikelihood_loss": 0.17303699254989624, + "theoretical_loss": 3.845661309252029, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041452357071213643, + "loss": 1.5114, + "theoretical_loss": 3.8456167369435073, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145035105315948, + "loss": 1.6073, + "theoretical_loss": 3.84552761128626, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448345035105316, + "loss": 1.5835, + "theoretical_loss": 3.845438510898102, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144633901705115, + "loss": 1.2002, + "theoretical_loss": 3.8453494357662747, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444332998996994, + "loss": 1.4201, + "theoretical_loss": 3.8452603858780288, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144232698094283, + "loss": 1.5521, + "theoretical_loss": 3.8451713612206246, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041440320962888666, + "loss": 1.5349, + "theoretical_loss": 3.84508236178133, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414383149448345, + "loss": 1.5031, + "theoretical_loss": 3.844993387547425, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041436308926780344, + "loss": 1.5198, + "theoretical_loss": 3.8449044385061955, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143430290872618, + "loss": 1.3538, + "theoretical_loss": 3.8448155146449396, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432296890672016, + "loss": 1.5557, + "theoretical_loss": 3.8447266159509628, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143029087261785, + "loss": 1.4679, + "theoretical_loss": 3.84463774241158, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3541666567325592, + "objective/train/avg_token_score": 0.016156869009137154, + "objective/train/docs_used": 345976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.190863847732544, + "objective/train/original_loss": 3.7492716312408447, + "objective/train/theoretical_loss": 3.844548894014116, + "objective/train/tokens_used": 593100800, + "objective/train/unlikelihood_loss": 0.21690547466278076, + "theoretical_loss": 3.844548894014116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828485456369, + "loss": 1.5316, + "theoretical_loss": 3.844548894014116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142627883650953, + "loss": 1.5358, + "theoretical_loss": 3.844460070745905, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041424272818455367, + "loss": 1.4871, + "theoretical_loss": 3.8443712725942882, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422266800401203, + "loss": 1.5228, + "theoretical_loss": 3.844282499546619, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142026078234704, + "loss": 1.7323, + "theoretical_loss": 3.8441937515902573, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141825476429288, + "loss": 1.5594, + "theoretical_loss": 3.8441050287125744, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416248746238717, + "loss": 1.5317, + "theoretical_loss": 3.844016330900949, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041414242728184553, + "loss": 1.5874, + "theoretical_loss": 3.843927658142771, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141223671013039, + "loss": 1.3527, + "theoretical_loss": 3.843839010425436, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141023069207623, + "loss": 1.5098, + "theoretical_loss": 3.8437503877363524, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140822467402207, + "loss": 1.3849, + "theoretical_loss": 3.843661790062935, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406218655967904, + "loss": 1.3191, + "theoretical_loss": 3.84357321739261, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140421263791374, + "loss": 1.4866, + "theoretical_loss": 3.8434846697128098, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.38501954078674316, + "objective/train/avg_token_score": 0.03767390176653862, + "objective/train/docs_used": 346603, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2809624671936035, + "objective/train/original_loss": 3.974229335784912, + "objective/train/theoretical_loss": 3.843440405240433, + "objective/train/tokens_used": 594739200, + "objective/train/unlikelihood_loss": 0.15178601443767548, + "theoretical_loss": 3.843440405240433, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041402206619859576, + "loss": 1.5946, + "theoretical_loss": 3.843396147010979, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140020060180542, + "loss": 1.4606, + "theoretical_loss": 3.8433076492745695, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041398194583751254, + "loss": 1.6322, + "theoretical_loss": 3.8432191764910426, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139618856569709, + "loss": 1.5872, + "theoretical_loss": 3.8431307286478678, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041394182547642926, + "loss": 1.4489, + "theoretical_loss": 3.843042305732526, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139217652958877, + "loss": 1.3802, + "theoretical_loss": 3.8429539077325043, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041390170511534604, + "loss": 1.435, + "theoretical_loss": 3.842865534635301, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138816449348044, + "loss": 1.4674, + "theoretical_loss": 3.842777186428423, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041386158475426277, + "loss": 1.384, + "theoretical_loss": 3.8426888630993847, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041384152457372113, + "loss": 1.5661, + "theoretical_loss": 3.8426005646357106, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041382146439317954, + "loss": 1.3068, + "theoretical_loss": 3.842512291024935, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138014042126379, + "loss": 1.5246, + "theoretical_loss": 3.842424042254601, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.43139052391052246, + "objective/train/avg_token_score": 0.019480930641293526, + "objective/train/docs_used": 348035, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4563043117523193, + "objective/train/original_loss": 3.788301706314087, + "objective/train/theoretical_loss": 3.8423358183122582, + "objective/train/tokens_used": 596377600, + "objective/train/unlikelihood_loss": 0.16736803948879242, + "theoretical_loss": 3.8423358183122582, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041378134403209627, + "loss": 1.4645, + "theoretical_loss": 3.8423358183122582, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041376128385155463, + "loss": 1.3936, + "theoretical_loss": 3.8422476191854686, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041374122367101305, + "loss": 1.3855, + "theoretical_loss": 3.8421594448618013, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137211634904714, + "loss": 1.4532, + "theoretical_loss": 3.842071295328835, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041370110330992977, + "loss": 1.3864, + "theoretical_loss": 3.8419831705741556, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041368104312938813, + "loss": 1.4822, + "theoretical_loss": 3.841895070585361, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041366098294884655, + "loss": 1.6476, + "theoretical_loss": 3.8418069953500558, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041364092276830497, + "loss": 1.649, + "theoretical_loss": 3.841718944855854, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041362086258776333, + "loss": 1.372, + "theoretical_loss": 3.8416309190903783, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136008024072217, + "loss": 1.3341, + "theoretical_loss": 3.8415429180412617, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041358074222668005, + "loss": 1.3834, + "theoretical_loss": 3.8414549416961443, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041356068204613847, + "loss": 1.5413, + "theoretical_loss": 3.841366990042676, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041354062186559683, + "loss": 1.4008, + "theoretical_loss": 3.8412790630685154, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.28977271914482117, + "objective/train/avg_token_score": 0.01197260245680809, + "objective/train/docs_used": 348629, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9216458201408386, + "objective/train/original_loss": 3.9676473140716553, + "objective/train/theoretical_loss": 3.841235108832321, + "objective/train/tokens_used": 598016000, + "objective/train/unlikelihood_loss": 0.14898046851158142, + "theoretical_loss": 3.841235108832321, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135205616850552, + "loss": 1.4445, + "theoretical_loss": 3.84119116076133, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041350050150451356, + "loss": 1.3845, + "theoretical_loss": 3.8411032831087963, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134804413239719, + "loss": 1.4899, + "theoretical_loss": 3.841015430098599, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041346038114343034, + "loss": 1.4013, + "theoretical_loss": 3.8409276017184326, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134403209628887, + "loss": 1.4651, + "theoretical_loss": 3.840839797956, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041342026078234706, + "loss": 1.292, + "theoretical_loss": 3.8407520187990127, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134002006018054, + "loss": 1.5541, + "theoretical_loss": 3.8406642642351914, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041338014042126384, + "loss": 1.4107, + "theoretical_loss": 3.840576534252265, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133600802407222, + "loss": 1.5056, + "theoretical_loss": 3.840488828837973, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041334002006018056, + "loss": 1.5114, + "theoretical_loss": 3.840401147980061, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133199598796389, + "loss": 1.4337, + "theoretical_loss": 3.8403134916662847, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132998996990973, + "loss": 1.3905, + "theoretical_loss": 3.8402258598844097, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.36742424964904785, + "objective/train/avg_token_score": 0.029935631901025772, + "objective/train/docs_used": 349996, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2140491008758545, + "objective/train/original_loss": 4.204592704772949, + "objective/train/theoretical_loss": 3.840138252622208, + "objective/train/tokens_used": 599654400, + "objective/train/unlikelihood_loss": 0.1415853351354599, + "theoretical_loss": 3.840138252622208, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132798395185557, + "loss": 1.3492, + "theoretical_loss": 3.840138252622208, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325977933801407, + "loss": 1.5691, + "theoretical_loss": 3.8400506698674635, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323971915747243, + "loss": 1.4959, + "theoretical_loss": 3.839963111607965, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132196589769308, + "loss": 1.4411, + "theoretical_loss": 3.8398755778315135, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131995987963892, + "loss": 1.4184, + "theoretical_loss": 3.8397880685259165, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041317953861584757, + "loss": 1.4997, + "theoretical_loss": 3.839700583678991, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315947843530593, + "loss": 1.4387, + "theoretical_loss": 3.8396131232785637, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131394182547643, + "loss": 1.5275, + "theoretical_loss": 3.8395256873124683, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131193580742227, + "loss": 1.4305, + "theoretical_loss": 3.8394382757685483, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130992978936811, + "loss": 1.57, + "theoretical_loss": 3.8393508886346552, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041307923771313944, + "loss": 1.3022, + "theoretical_loss": 3.83926352589865, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130591775325978, + "loss": 1.4727, + "theoretical_loss": 3.839176187548402, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303911735205616, + "loss": 1.4899, + "theoretical_loss": 3.8390888735717885, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4315738081932068, + "objective/train/avg_token_score": 0.012260204181075096, + "objective/train/docs_used": 350686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4201757907867432, + "objective/train/original_loss": 3.8342316150665283, + "objective/train/theoretical_loss": 3.8390452257198096, + "objective/train/tokens_used": 601292800, + "objective/train/unlikelihood_loss": 0.11259658634662628, + "theoretical_loss": 3.8390452257198096, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130190571715146, + "loss": 1.375, + "theoretical_loss": 3.839001583956697, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041299899699097294, + "loss": 1.4189, + "theoretical_loss": 3.8389143186910224, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129789368104313, + "loss": 1.5252, + "theoretical_loss": 3.838827077762668, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041295887662988966, + "loss": 1.5391, + "theoretical_loss": 3.8387398611595476, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129388164493481, + "loss": 1.6135, + "theoretical_loss": 3.838652668869581, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041291875626880644, + "loss": 1.4085, + "theoretical_loss": 3.838565500880699, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128986960882648, + "loss": 1.4676, + "theoretical_loss": 3.83847835718084, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041287863590772317, + "loss": 1.3773, + "theoretical_loss": 3.8383912377579508, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041285857572718153, + "loss": 1.6076, + "theoretical_loss": 3.8383041425999864, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041283851554663995, + "loss": 1.3761, + "theoretical_loss": 3.8382170716949124, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128184553660983, + "loss": 1.3649, + "theoretical_loss": 3.8381300250307007, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041279839518555667, + "loss": 1.5752, + "theoretical_loss": 3.8380430025953327, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4177052676677704, + "objective/train/avg_token_score": 0.011318889446556568, + "objective/train/docs_used": 352047, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3438571691513062, + "objective/train/original_loss": 3.8496592044830322, + "objective/train/theoretical_loss": 3.837956004376799, + "objective/train/tokens_used": 602931200, + "objective/train/unlikelihood_loss": 0.19700142741203308, + "theoretical_loss": 3.837956004376799, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041277833500501503, + "loss": 1.6608, + "theoretical_loss": 3.837956004376799, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041275827482447345, + "loss": 1.441, + "theoretical_loss": 3.8378690303630973, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127382146439318, + "loss": 1.4288, + "theoretical_loss": 3.837782080542236, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041271815446339017, + "loss": 1.4916, + "theoretical_loss": 3.837695154902229, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041269809428284854, + "loss": 1.3033, + "theoretical_loss": 3.8376082534311022, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126780341023069, + "loss": 1.3031, + "theoretical_loss": 3.8375213761168867, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126579739217653, + "loss": 1.5044, + "theoretical_loss": 3.8374345229476248, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126379137412237, + "loss": 1.5035, + "theoretical_loss": 3.837347693911366, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041261785356068204, + "loss": 1.4344, + "theoretical_loss": 3.837260888996169, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125977933801404, + "loss": 1.5352, + "theoretical_loss": 3.8371741081900996, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125777331995988, + "loss": 1.5946, + "theoretical_loss": 3.837087351481234, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125576730190572, + "loss": 1.4719, + "theoretical_loss": 3.8370006188576555, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041253761283851554, + "loss": 1.2678, + "theoretical_loss": 3.836913910307456, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3923509418964386, + "objective/train/avg_token_score": 0.010112078860402107, + "objective/train/docs_used": 352465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4317771196365356, + "objective/train/original_loss": 4.348731994628906, + "objective/train/theoretical_loss": 3.836870565056154, + "objective/train/tokens_used": 604569600, + "objective/train/unlikelihood_loss": 0.11279493570327759, + "theoretical_loss": 3.836870565056154, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125175526579739, + "loss": 1.5122, + "theoretical_loss": 3.8368272258187366, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041249749247743227, + "loss": 1.2894, + "theoretical_loss": 3.836740565379606, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124774322968907, + "loss": 1.4996, + "theoretical_loss": 3.836653928978182, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041245737211634904, + "loss": 1.3454, + "theoretical_loss": 3.8365673166025913, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124373119358074, + "loss": 1.3027, + "theoretical_loss": 3.8364807282409674, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041241725175526577, + "loss": 1.5057, + "theoretical_loss": 3.8363941638814536, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123971915747242, + "loss": 1.3416, + "theoretical_loss": 3.8363076235122016, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041237713139418255, + "loss": 1.4237, + "theoretical_loss": 3.83622110712137, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123570712136409, + "loss": 1.3262, + "theoretical_loss": 3.8361346146971282, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041233701103309927, + "loss": 1.4146, + "theoretical_loss": 3.836048146227652, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041231695085255763, + "loss": 1.5451, + "theoretical_loss": 3.8359617017011267, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041229689067201605, + "loss": 1.3069, + "theoretical_loss": 3.8358752811057455, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.47061339020729065, + "objective/train/avg_token_score": 0.003165400819852948, + "objective/train/docs_used": 353634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.681175708770752, + "objective/train/original_loss": 3.5376367568969727, + "objective/train/theoretical_loss": 3.8357888844297094, + "objective/train/tokens_used": 606208000, + "objective/train/unlikelihood_loss": 0.1784207969903946, + "theoretical_loss": 3.8357888844297094, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122768304914744, + "loss": 1.3184, + "theoretical_loss": 3.8357888844297094, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122567703109328, + "loss": 1.5651, + "theoretical_loss": 3.8357025116612293, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041223671013039114, + "loss": 1.5169, + "theoretical_loss": 3.8356161627885235, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041221664994984955, + "loss": 1.2234, + "theoretical_loss": 3.835529837799818, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121965897693079, + "loss": 1.5053, + "theoretical_loss": 3.835443536683349, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121765295887663, + "loss": 1.4598, + "theoretical_loss": 3.8353572594273597, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121564694082247, + "loss": 1.5847, + "theoretical_loss": 3.835271006020101, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121364092276831, + "loss": 1.3336, + "theoretical_loss": 3.8351847764498332, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121163490471415, + "loss": 1.3501, + "theoretical_loss": 3.835098570704826, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041209628886659984, + "loss": 1.4099, + "theoretical_loss": 3.8350123887733543, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120762286860582, + "loss": 1.3004, + "theoretical_loss": 3.8349262306437044, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041205616850551656, + "loss": 1.464, + "theoretical_loss": 3.8348400963041684, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412036108324975, + "loss": 1.4531, + "theoretical_loss": 3.834753985743049, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.2646627426147461, + "objective/train/avg_token_score": 0.01583893597126007, + "objective/train/docs_used": 354263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9999536871910095, + "objective/train/original_loss": 4.469597816467285, + "objective/train/theoretical_loss": 3.834710939375742, + "objective/train/tokens_used": 607846400, + "objective/train/unlikelihood_loss": 0.14511200785636902, + "theoretical_loss": 3.834710939375742, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041201604814443334, + "loss": 1.4492, + "theoretical_loss": 3.8346678989486556, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119959879638917, + "loss": 1.3742, + "theoretical_loss": 3.8345818359093062, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041197592778335006, + "loss": 1.4287, + "theoretical_loss": 3.834495796613327, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119558676028085, + "loss": 1.4213, + "theoretical_loss": 3.834409781049053, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193580742226684, + "loss": 1.5101, + "theoretical_loss": 3.8343237892048263, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119157472417252, + "loss": 1.4667, + "theoretical_loss": 3.8342378210689994, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041189568706118357, + "loss": 1.4083, + "theoretical_loss": 3.83415187662993, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041187562688064193, + "loss": 1.463, + "theoretical_loss": 3.834065955875987, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041185556670010035, + "loss": 1.2794, + "theoretical_loss": 3.833980058795545, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118355065195587, + "loss": 1.5537, + "theoretical_loss": 3.8338941853769892, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041181544633901707, + "loss": 1.5483, + "theoretical_loss": 3.8338083356087105, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179538615847543, + "loss": 1.5858, + "theoretical_loss": 3.8337225094791103, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4015762507915497, + "objective/train/avg_token_score": 0.020319471135735512, + "objective/train/docs_used": 355721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4089401960372925, + "objective/train/original_loss": 4.095301628112793, + "objective/train/theoretical_loss": 3.8336367069765958, + "objective/train/tokens_used": 609484800, + "objective/train/unlikelihood_loss": 0.2028389275074005, + "theoretical_loss": 3.8336367069765958, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041177532597793385, + "loss": 1.273, + "theoretical_loss": 3.8336367069765958, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117552657973922, + "loss": 1.45, + "theoretical_loss": 3.8335509280895854, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117352056168506, + "loss": 1.4615, + "theoretical_loss": 3.833465172806503, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041171514543630894, + "loss": 1.4819, + "theoretical_loss": 3.833379441115781, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116950852557673, + "loss": 1.3545, + "theoretical_loss": 3.8332937330058616, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116750250752257, + "loss": 1.4763, + "theoretical_loss": 3.8332080484651945, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116549648946841, + "loss": 1.5326, + "theoretical_loss": 3.833122387482236, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163490471414244, + "loss": 1.3957, + "theoretical_loss": 3.833036750045452, + "tokens_seen": 610402304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116148445336008, + "loss": 1.4854, + "theoretical_loss": 3.8329511361433166, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115947843530592, + "loss": 1.4062, + "theoretical_loss": 3.832865545764311, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115747241725176, + "loss": 1.4865, + "theoretical_loss": 3.832779978896926, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041155466399197594, + "loss": 1.4416, + "theoretical_loss": 3.8326944355296595, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115346038114343, + "loss": 1.4191, + "theoretical_loss": 3.832608915651017, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4571114480495453, + "objective/train/avg_token_score": 0.008640832267701626, + "objective/train/docs_used": 356348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6731990575790405, + "objective/train/original_loss": 3.7301838397979736, + "objective/train/theoretical_loss": 3.83256616451634, + "objective/train/tokens_used": 611123200, + "objective/train/unlikelihood_loss": 0.17205381393432617, + "theoretical_loss": 3.83256616451634, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041151454363089267, + "loss": 1.4817, + "theoretical_loss": 3.8325234192495126, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114944834503511, + "loss": 1.4969, + "theoretical_loss": 3.8324379463136697, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041147442326980944, + "loss": 1.4105, + "theoretical_loss": 3.832352496832018, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114543630892678, + "loss": 1.5492, + "theoretical_loss": 3.8322670707930957, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143430290872617, + "loss": 1.5112, + "theoretical_loss": 3.83218166818545, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114142427281846, + "loss": 1.2652, + "theoretical_loss": 3.8320962889976347, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041139418254764295, + "loss": 1.58, + "theoretical_loss": 3.8320109332182124, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113741223671013, + "loss": 1.4196, + "theoretical_loss": 3.831925600835754, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135406218655967, + "loss": 1.4769, + "theoretical_loss": 3.8318402918388386, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041133400200601803, + "loss": 1.506, + "theoretical_loss": 3.831755006216052, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131394182547645, + "loss": 1.5132, + "theoretical_loss": 3.831669743955989, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112938816449348, + "loss": 1.3302, + "theoretical_loss": 3.8315845050472532, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4703079164028168, + "objective/train/avg_token_score": 0.01573745161294937, + "objective/train/docs_used": 356825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7282345294952393, + "objective/train/original_loss": 3.63925838470459, + "objective/train/theoretical_loss": 3.8314992894784536, + "objective/train/tokens_used": 612761600, + "objective/train/unlikelihood_loss": 0.17425817251205444, + "theoretical_loss": 3.8314992894784536, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112738214643932, + "loss": 1.4771, + "theoretical_loss": 3.8314992894784536, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125376128385154, + "loss": 1.4603, + "theoretical_loss": 3.8314140972382105, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041123370110330995, + "loss": 1.2944, + "theoretical_loss": 3.8313289283151497, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112136409227683, + "loss": 1.3469, + "theoretical_loss": 3.831243782697906, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111935807422267, + "loss": 1.2951, + "theoretical_loss": 3.831158660375122, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041117352056168504, + "loss": 1.3934, + "theoretical_loss": 3.8310735613354483, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111534603811434, + "loss": 1.2533, + "theoretical_loss": 3.830988485567543, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111334002006018, + "loss": 1.4875, + "theoretical_loss": 3.830903433060073, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111133400200602, + "loss": 1.4649, + "theoretical_loss": 3.830818403801713, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109327983951854, + "loss": 1.2906, + "theoretical_loss": 3.830733397781144, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110732196589769, + "loss": 1.2456, + "theoretical_loss": 3.830648414987057, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110531594784353, + "loss": 1.2733, + "theoretical_loss": 3.830563455408151, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110330992978937, + "loss": 1.3083, + "theoretical_loss": 3.8304785190331305, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.37115103006362915, + "objective/train/avg_token_score": 0.012984885834157467, + "objective/train/docs_used": 357491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3638612031936646, + "objective/train/original_loss": 3.861525535583496, + "objective/train/theoretical_loss": 3.830436059543551, + "objective/train/tokens_used": 614400000, + "objective/train/unlikelihood_loss": 0.16821658611297607, + "theoretical_loss": 3.830436059543551, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041101303911735205, + "loss": 1.7196, + "theoretical_loss": 3.830393605850711, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109929789368104, + "loss": 1.1773, + "theoretical_loss": 3.8303087158496134, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041097291875626877, + "loss": 1.173, + "theoretical_loss": 3.8302238490185676, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109528585757272, + "loss": 1.4127, + "theoretical_loss": 3.830139005346312, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041093279839518555, + "loss": 1.2904, + "theoretical_loss": 3.8300541848215905, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109127382146439, + "loss": 1.4453, + "theoretical_loss": 3.8299693874331573, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108926780341023, + "loss": 1.3533, + "theoretical_loss": 3.8298846131697744, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108726178535607, + "loss": 1.201, + "theoretical_loss": 3.8297998620202094, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041085255767301905, + "loss": 1.4987, + "theoretical_loss": 3.8297151339732407, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108324974924774, + "loss": 1.3022, + "theoretical_loss": 3.8296304290176524, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108124373119358, + "loss": 1.5003, + "theoretical_loss": 3.8295457471422365, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107923771313942, + "loss": 1.3209, + "theoretical_loss": 3.8294610883357945, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2765762507915497, + "objective/train/avg_token_score": 0.022694801911711693, + "objective/train/docs_used": 358741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9784660339355469, + "objective/train/original_loss": 3.9417734146118164, + "objective/train/theoretical_loss": 3.829376452587134, + "objective/train/tokens_used": 616038400, + "objective/train/unlikelihood_loss": 0.2532103657722473, + "theoretical_loss": 3.829376452587134, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041077231695085256, + "loss": 1.4609, + "theoretical_loss": 3.829376452587134, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107522567703109, + "loss": 1.4066, + "theoretical_loss": 3.8292918398850713, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107321965897693, + "loss": 1.3902, + "theoretical_loss": 3.82920725021843, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041071213640922764, + "loss": 1.2997, + "theoretical_loss": 3.829122683576042, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041069207622868606, + "loss": 1.4858, + "theoretical_loss": 3.8290381399467464, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106720160481444, + "loss": 1.3344, + "theoretical_loss": 3.8289536193193907, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041065195586760284, + "loss": 1.4025, + "theoretical_loss": 3.8288691216828297, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106318956870612, + "loss": 1.4296, + "theoretical_loss": 3.8287846470259264, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106118355065196, + "loss": 1.552, + "theoretical_loss": 3.828700195337551, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410591775325978, + "loss": 1.3554, + "theoretical_loss": 3.8286157666065828, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041057171514543634, + "loss": 1.3402, + "theoretical_loss": 3.8285313608219056, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105516549648947, + "loss": 1.3052, + "theoretical_loss": 3.8284469779724155, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041053159478435307, + "loss": 1.3052, + "theoretical_loss": 3.828362618047013, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.46590909361839294, + "objective/train/avg_token_score": 0.011999816633760929, + "objective/train/docs_used": 359404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5094083547592163, + "objective/train/original_loss": 3.980893135070801, + "objective/train/theoretical_loss": 3.828320446677378, + "objective/train/tokens_used": 617676800, + "objective/train/unlikelihood_loss": 0.12393984198570251, + "theoretical_loss": 3.828320446677378, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105115346038115, + "loss": 1.4529, + "theoretical_loss": 3.8282782810346063, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041049147442326985, + "loss": 1.3784, + "theoretical_loss": 3.8281939669241143, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104714142427282, + "loss": 1.3825, + "theoretical_loss": 3.82810967570446, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041045135406218657, + "loss": 1.3919, + "theoretical_loss": 3.828025407364577, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410431293881645, + "loss": 1.4041, + "theoretical_loss": 3.8279411618934045, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041041123370110335, + "loss": 1.4088, + "theoretical_loss": 3.82785693927989, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103911735205617, + "loss": 1.4916, + "theoretical_loss": 3.82777273951299, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041037111334002007, + "loss": 1.5682, + "theoretical_loss": 3.8276885625816663, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041035105315947844, + "loss": 1.538, + "theoretical_loss": 3.8276044084748904, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041033099297893685, + "loss": 1.3839, + "theoretical_loss": 3.827520277181641, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103109327983952, + "loss": 1.5539, + "theoretical_loss": 3.8274361686909035, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102908726178536, + "loss": 1.4889, + "theoretical_loss": 3.827352082991672, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.3732893466949463, + "objective/train/avg_token_score": 0.019104547798633575, + "objective/train/docs_used": 360794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2005456686019897, + "objective/train/original_loss": 3.98445725440979, + "objective/train/theoretical_loss": 3.827268020072948, + "objective/train/tokens_used": 619315200, + "objective/train/unlikelihood_loss": 0.14163470268249512, + "theoretical_loss": 3.827268020072948, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041027081243731194, + "loss": 1.3425, + "theoretical_loss": 3.827268020072948, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041025075225677035, + "loss": 1.299, + "theoretical_loss": 3.8271839799237397, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102306920762287, + "loss": 1.6187, + "theoretical_loss": 3.8270999625330644, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102106318956871, + "loss": 1.3738, + "theoretical_loss": 3.8270159678899462, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041019057171514544, + "loss": 1.3923, + "theoretical_loss": 3.8269319959834167, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101705115346038, + "loss": 1.4009, + "theoretical_loss": 3.8268480468025157, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101504513540622, + "loss": 1.5065, + "theoretical_loss": 3.82676412033629, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101303911735206, + "loss": 1.3505, + "theoretical_loss": 3.826680216573795, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041011033099297894, + "loss": 1.3851, + "theoretical_loss": 3.8265963355040915, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100902708124373, + "loss": 1.7003, + "theoretical_loss": 3.8265124771162506, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100702106318957, + "loss": 1.255, + "theoretical_loss": 3.8264286413993496, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100501504513541, + "loss": 1.5705, + "theoretical_loss": 3.8263448283424726, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041003009027081245, + "loss": 1.3712, + "theoretical_loss": 3.826261037934713, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.39937683939933777, + "objective/train/avg_token_score": 0.00654844893142581, + "objective/train/docs_used": 361341, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4888705015182495, + "objective/train/original_loss": 4.043774604797363, + "objective/train/theoretical_loss": 3.8262191512208448, + "objective/train/tokens_used": 620953600, + "objective/train/unlikelihood_loss": 0.12301285564899445, + "theoretical_loss": 3.8262191512208448, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100100300902708, + "loss": 1.3344, + "theoretical_loss": 3.8261772701651697, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040998996990972917, + "loss": 1.5005, + "theoretical_loss": 3.8260935250229515, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099699097291876, + "loss": 1.3902, + "theoretical_loss": 3.8260098024971727, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040994984954864595, + "loss": 1.4164, + "theoretical_loss": 3.8259261025769566, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099297893681043, + "loss": 1.4026, + "theoretical_loss": 3.825842425251434, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099097291875627, + "loss": 1.4462, + "theoretical_loss": 3.8257587705097404, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098896690070211, + "loss": 1.5376, + "theoretical_loss": 3.8256751383410235, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040986960882647945, + "loss": 1.3613, + "theoretical_loss": 3.825591528734434, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098495486459378, + "loss": 1.3915, + "theoretical_loss": 3.8255079416791338, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098294884653962, + "loss": 1.3391, + "theoretical_loss": 3.8254243771642895, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098094282848546, + "loss": 1.5791, + "theoretical_loss": 3.8253408351790767, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040978936810431296, + "loss": 1.4436, + "theoretical_loss": 3.825257315712678, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2656402587890625, + "objective/train/avg_token_score": 0.020830322057008743, + "objective/train/docs_used": 362042, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8504209518432617, + "objective/train/original_loss": 4.423110008239746, + "objective/train/theoretical_loss": 3.8251738187542843, + "objective/train/tokens_used": 622592000, + "objective/train/unlikelihood_loss": 0.12244540452957153, + "theoretical_loss": 3.8251738187542843, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097693079237713, + "loss": 1.261, + "theoretical_loss": 3.8251738187542843, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097492477432297, + "loss": 1.2915, + "theoretical_loss": 3.8250903442930917, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040972918756268804, + "loss": 1.4155, + "theoretical_loss": 3.8250068923183065, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040970912738214646, + "loss": 1.3704, + "theoretical_loss": 3.8249234628191404, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096890672016048, + "loss": 1.4364, + "theoretical_loss": 3.8248400557848146, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096690070210632, + "loss": 1.2654, + "theoretical_loss": 3.824756671204555, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040964894684052155, + "loss": 1.1298, + "theoretical_loss": 3.824673309067598, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040962888665997996, + "loss": 1.4316, + "theoretical_loss": 3.824589969363184, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096088264794383, + "loss": 1.4605, + "theoretical_loss": 3.824506652080564, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095887662988967, + "loss": 1.4785, + "theoretical_loss": 3.8244233572089943, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040956870611835505, + "loss": 1.4477, + "theoretical_loss": 3.8243400847377407, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095486459378134, + "loss": 1.2426, + "theoretical_loss": 3.8242568346560737, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040952858575727183, + "loss": 1.4633, + "theoretical_loss": 3.8241736069532726, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.38581377267837524, + "objective/train/avg_token_score": 0.025465743616223335, + "objective/train/docs_used": 363016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5282154083251953, + "objective/train/original_loss": 4.084991931915283, + "objective/train/theoretical_loss": 3.8241320014905993, + "objective/train/tokens_used": 624230400, + "objective/train/unlikelihood_loss": 0.14625848829746246, + "theoretical_loss": 3.8241320014905993, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095085255767302, + "loss": 1.3288, + "theoretical_loss": 3.824090401618625, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040948846539618855, + "loss": 1.6178, + "theoretical_loss": 3.8240072186414245, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094684052156469, + "loss": 1.4476, + "theoretical_loss": 3.8239240580109723, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040944834503510533, + "loss": 1.3995, + "theoretical_loss": 3.8238409197165772, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094282848545637, + "loss": 1.362, + "theoretical_loss": 3.8237578037475553, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040940822467402206, + "loss": 1.424, + "theoretical_loss": 3.823674710093231, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093881644934804, + "loss": 1.4847, + "theoretical_loss": 3.8235916387429336, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093681043129388, + "loss": 1.348, + "theoretical_loss": 3.8235085896860017, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093480441323972, + "loss": 1.322, + "theoretical_loss": 3.8234255629117815, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040932798395185556, + "loss": 1.3699, + "theoretical_loss": 3.8233425584096254, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093079237713139, + "loss": 1.385, + "theoretical_loss": 3.8232595761688932, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092878635907723, + "loss": 1.4204, + "theoretical_loss": 3.8231766161789524, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4359726309776306, + "objective/train/avg_token_score": 0.016611715778708458, + "objective/train/docs_used": 363672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.319413423538208, + "objective/train/original_loss": 3.72617506980896, + "objective/train/theoretical_loss": 3.8230936784291787, + "objective/train/tokens_used": 625868800, + "objective/train/unlikelihood_loss": 0.15357278287410736, + "theoretical_loss": 3.8230936784291787, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092678034102307, + "loss": 1.3302, + "theoretical_loss": 3.8230936784291787, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040924774322968906, + "loss": 1.4842, + "theoretical_loss": 3.823010762908953, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092276830491474, + "loss": 1.3199, + "theoretical_loss": 3.822927869607664, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092076228686058, + "loss": 1.2685, + "theoretical_loss": 3.822844998514711, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040918756268806415, + "loss": 1.5454, + "theoretical_loss": 3.8227621496194946, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040916750250752257, + "loss": 1.406, + "theoretical_loss": 3.822679322911429, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409147442326981, + "loss": 1.1559, + "theoretical_loss": 3.82259651837993, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040912738214643934, + "loss": 1.356, + "theoretical_loss": 3.8225137360144252, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091073219658977, + "loss": 1.4281, + "theoretical_loss": 3.8224309758043464, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090872617853561, + "loss": 1.2494, + "theoretical_loss": 3.822348237739134, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090672016048145, + "loss": 1.3822, + "theoretical_loss": 3.822265521808236, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040904714142427285, + "loss": 1.4783, + "theoretical_loss": 3.8221828280011065, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090270812437312, + "loss": 1.3068, + "theoretical_loss": 3.822100156307208, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.359665185213089, + "objective/train/avg_token_score": 0.019122768193483353, + "objective/train/docs_used": 364826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3461437225341797, + "objective/train/original_loss": 3.9998061656951904, + "objective/train/theoretical_loss": 3.8220588287494284, + "objective/train/tokens_used": 627507200, + "objective/train/unlikelihood_loss": 0.16339793801307678, + "theoretical_loss": 3.8220588287494284, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040900702106318957, + "loss": 1.3026, + "theoretical_loss": 3.8220175067160085, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408986960882648, + "loss": 1.4798, + "theoretical_loss": 3.821934879216985, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040896690070210635, + "loss": 1.3787, + "theoretical_loss": 3.821852273799621, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089468405215647, + "loss": 1.329, + "theoretical_loss": 3.821769690453407, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089267803410231, + "loss": 1.3329, + "theoretical_loss": 3.8216871291678416, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089067201604815, + "loss": 1.4236, + "theoretical_loss": 3.821604589932429, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040888665997993985, + "loss": 1.3524, + "theoretical_loss": 3.8215220727366823, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088665997993982, + "loss": 1.4028, + "theoretical_loss": 3.82143957757012, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088465396188566, + "loss": 1.4714, + "theoretical_loss": 3.8213571044222694, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408826479438315, + "loss": 1.3119, + "theoretical_loss": 3.8212746532826642, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040880641925777336, + "loss": 1.3945, + "theoretical_loss": 3.8211922241408454, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087863590772317, + "loss": 1.2536, + "theoretical_loss": 3.8211098169863607, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4605938494205475, + "objective/train/avg_token_score": 0.018184412270784378, + "objective/train/docs_used": 365375, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5830137729644775, + "objective/train/original_loss": 3.831573724746704, + "objective/train/theoretical_loss": 3.8210274318087656, + "objective/train/tokens_used": 629145600, + "objective/train/unlikelihood_loss": 0.12006247788667679, + "theoretical_loss": 3.8210274318087656, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087662988966901, + "loss": 1.4189, + "theoretical_loss": 3.8210274318087656, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040874623871614844, + "loss": 1.4326, + "theoretical_loss": 3.8209450685976227, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872617853560686, + "loss": 1.5258, + "theoretical_loss": 3.820862727342501, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087061183550652, + "loss": 1.2155, + "theoretical_loss": 3.8207804080329772, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086860581745236, + "loss": 1.3935, + "theoretical_loss": 3.8206981106586357, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866599799398195, + "loss": 1.4249, + "theoretical_loss": 3.8206158352090664, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864593781344036, + "loss": 1.4532, + "theoretical_loss": 3.8205335816738684, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086258776328987, + "loss": 1.4963, + "theoretical_loss": 3.820451350042646, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086058174523571, + "loss": 1.3688, + "theoretical_loss": 3.8203691403050106, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040858575727181545, + "loss": 1.2832, + "theoretical_loss": 3.8202869524505827, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656970912738, + "loss": 1.5181, + "theoretical_loss": 3.8202047864689885, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854563691073223, + "loss": 1.4243, + "theoretical_loss": 3.820122642349861, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085255767301906, + "loss": 1.4029, + "theoretical_loss": 3.8200405200828405, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2529325485229492, + "objective/train/avg_token_score": 0.023288333788514137, + "objective/train/docs_used": 366612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9088571071624756, + "objective/train/original_loss": 4.29040002822876, + "objective/train/theoretical_loss": 3.8199994671406348, + "objective/train/tokens_used": 630784000, + "objective/train/unlikelihood_loss": 0.16850118339061737, + "theoretical_loss": 3.8199994671406348, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040850551654964895, + "loss": 1.3071, + "theoretical_loss": 3.819958419657574, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084854563691073, + "loss": 1.4315, + "theoretical_loss": 3.819876341063718, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040846539618856573, + "loss": 1.2791, + "theoretical_loss": 3.8197942842909325, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084453360080241, + "loss": 1.3536, + "theoretical_loss": 3.8197122493288864, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040842527582748246, + "loss": 1.3218, + "theoretical_loss": 3.8196302361672556, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084052156469408, + "loss": 1.4322, + "theoretical_loss": 3.8195482447957225, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083851554663992, + "loss": 1.5428, + "theoretical_loss": 3.8194662752039776, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083650952858576, + "loss": 1.5233, + "theoretical_loss": 3.8193843273817167, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834503510531596, + "loss": 1.3592, + "theoretical_loss": 3.8193024013186445, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083249749247743, + "loss": 1.3766, + "theoretical_loss": 3.819220497004471, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083049147442327, + "loss": 1.4008, + "theoretical_loss": 3.8191386144289137, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082848545636911, + "loss": 1.4924, + "theoretical_loss": 3.819056753581699, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4759897291660309, + "objective/train/avg_token_score": 0.017946870997548103, + "objective/train/docs_used": 367323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5524520874023438, + "objective/train/original_loss": 3.585705041885376, + "objective/train/theoretical_loss": 3.818974914452557, + "objective/train/tokens_used": 632422400, + "objective/train/unlikelihood_loss": 0.13466675579547882, + "theoretical_loss": 3.818974914452557, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040826479438314946, + "loss": 1.3453, + "theoretical_loss": 3.818974914452557, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082447342026078, + "loss": 1.3149, + "theoretical_loss": 3.8188930970312267, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082246740220662, + "loss": 1.3998, + "theoretical_loss": 3.8188113013074547, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040820461384152455, + "loss": 1.2335, + "theoretical_loss": 3.8187295272709934, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040818455366098297, + "loss": 1.1875, + "theoretical_loss": 3.818647774911601, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040816449348044133, + "loss": 1.2951, + "theoretical_loss": 3.818566044219046, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081444332998997, + "loss": 1.4905, + "theoretical_loss": 3.8184843351831006, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040812437311935805, + "loss": 1.2673, + "theoretical_loss": 3.8184026477935458, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040810431293881647, + "loss": 1.338, + "theoretical_loss": 3.818320982040169, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040808425275827483, + "loss": 1.708, + "theoretical_loss": 3.8182393379127655, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080641925777332, + "loss": 1.5884, + "theoretical_loss": 3.8181577154011346, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804413239719156, + "loss": 1.4255, + "theoretical_loss": 3.818076114495086, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080240722166499, + "loss": 1.5113, + "theoretical_loss": 3.817994535184434, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.5092253088951111, + "objective/train/avg_token_score": 0.017480170354247093, + "objective/train/docs_used": 367883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8412432670593262, + "objective/train/original_loss": 3.7791481018066406, + "objective/train/theoretical_loss": 3.8179537536242005, + "objective/train/tokens_used": 634060800, + "objective/train/unlikelihood_loss": 0.15870891511440277, + "theoretical_loss": 3.8179537536242005, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040800401203610834, + "loss": 1.3694, + "theoretical_loss": 3.8179129774590006, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079839518555667, + "loss": 1.5419, + "theoretical_loss": 3.8178314413086154, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040796389167502506, + "loss": 1.4045, + "theoretical_loss": 3.817749926723114, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079438314944834, + "loss": 1.3526, + "theoretical_loss": 3.8176684336923383, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040792377131394184, + "loss": 1.4804, + "theoretical_loss": 3.8175869622061382, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079037111334002, + "loss": 1.5129, + "theoretical_loss": 3.817505512254371, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040788365095285856, + "loss": 1.4568, + "theoretical_loss": 3.817424083826899, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078635907723169, + "loss": 1.3891, + "theoretical_loss": 3.8173426769135927, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078435305917753, + "loss": 1.3367, + "theoretical_loss": 3.8172612915043294, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078234704112337, + "loss": 1.4437, + "theoretical_loss": 3.8171799275889926, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040780341023069207, + "loss": 1.3874, + "theoretical_loss": 3.8170985851574732, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040778335005015043, + "loss": 1.3605, + "theoretical_loss": 3.8170172641996682, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.39595553278923035, + "objective/train/avg_token_score": 0.020078817382454872, + "objective/train/docs_used": 369288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3277698755264282, + "objective/train/original_loss": 3.681797742843628, + "objective/train/theoretical_loss": 3.8169359647054835, + "objective/train/tokens_used": 635699200, + "objective/train/unlikelihood_loss": 0.16572554409503937, + "theoretical_loss": 3.8169359647054835, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077632898696088, + "loss": 1.3921, + "theoretical_loss": 3.8169359647054835, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077432296890672, + "loss": 1.2567, + "theoretical_loss": 3.8168546866648283, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040772316950852557, + "loss": 1.3112, + "theoretical_loss": 3.816773430067622, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040770310932798393, + "loss": 1.5763, + "theoretical_loss": 3.816692194903789, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076830491474423, + "loss": 1.3896, + "theoretical_loss": 3.816610981163261, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076629889669007, + "loss": 1.4929, + "theoretical_loss": 3.8165297888359766, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076429287863591, + "loss": 1.5615, + "theoretical_loss": 3.816448617911881, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076228686058175, + "loss": 1.5079, + "theoretical_loss": 3.816367468380926, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040760280842527585, + "loss": 1.2841, + "theoretical_loss": 3.8162863402330713, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075827482447342, + "loss": 1.4782, + "theoretical_loss": 3.816205233458281, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040756268806419263, + "loss": 1.1803, + "theoretical_loss": 3.816124148046529, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407542627883651, + "loss": 1.5352, + "theoretical_loss": 3.8160430839877937, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040752256770310935, + "loss": 1.3779, + "theoretical_loss": 3.815962041272061, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.5428274869918823, + "objective/train/avg_token_score": 0.006741791497915983, + "objective/train/docs_used": 369983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.891964077949524, + "objective/train/original_loss": 3.4886183738708496, + "objective/train/theoretical_loss": 3.815921527914693, + "objective/train/tokens_used": 637337600, + "objective/train/unlikelihood_loss": 0.1536000370979309, + "theoretical_loss": 3.815921527914693, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075025075225677, + "loss": 1.4306, + "theoretical_loss": 3.815881019889324, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040748244734202613, + "loss": 1.4211, + "theoretical_loss": 3.8158000198295814, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074623871614845, + "loss": 1.4194, + "theoretical_loss": 3.81571904108284, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040744232698094286, + "loss": 1.28, + "theoretical_loss": 3.8156380836391124, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074222668004012, + "loss": 1.4144, + "theoretical_loss": 3.815557147488418, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074022066198596, + "loss": 1.4748, + "theoretical_loss": 3.815476232620784, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407382146439318, + "loss": 1.4531, + "theoretical_loss": 3.8153953390262427, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040736208625877636, + "loss": 1.3289, + "theoretical_loss": 3.8153144666948338, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073420260782347, + "loss": 1.4632, + "theoretical_loss": 3.815233615616605, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073219658976931, + "loss": 1.5634, + "theoretical_loss": 3.8151527857816077, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073019057171515, + "loss": 1.483, + "theoretical_loss": 3.815071977179903, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040728184553660986, + "loss": 1.4585, + "theoretical_loss": 3.814991189801557, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2507942318916321, + "objective/train/avg_token_score": 0.031877949833869934, + "objective/train/docs_used": 370882, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.902974545955658, + "objective/train/original_loss": 4.440757751464844, + "objective/train/theoretical_loss": 3.8149104236366433, + "objective/train/tokens_used": 638976000, + "objective/train/unlikelihood_loss": 0.10956820845603943, + "theoretical_loss": 3.8149104236366433, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072617853560682, + "loss": 1.54, + "theoretical_loss": 3.8149104236366433, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072417251755266, + "loss": 1.4755, + "theoretical_loss": 3.814829678675242, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040722166499498495, + "loss": 1.2833, + "theoretical_loss": 3.814748954907439, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040720160481444337, + "loss": 1.3224, + "theoretical_loss": 3.8146682523233286, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040718154463390173, + "loss": 1.5405, + "theoretical_loss": 3.8145875709130093, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071614844533601, + "loss": 1.4171, + "theoretical_loss": 3.814506910666589, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040714142427281845, + "loss": 1.5556, + "theoretical_loss": 3.81442627157418, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040712136409227687, + "loss": 1.5124, + "theoretical_loss": 3.8143456536259035, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040710130391173523, + "loss": 1.4711, + "theoretical_loss": 3.8142650568118848, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070812437311936, + "loss": 1.4587, + "theoretical_loss": 3.814184481122257, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040706118355065196, + "loss": 1.4772, + "theoretical_loss": 3.814103926547161, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070411233701103, + "loss": 1.5276, + "theoretical_loss": 3.8140233930767424, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040702106318956874, + "loss": 1.4636, + "theoretical_loss": 3.8139428807011537, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2876954972743988, + "objective/train/avg_token_score": 0.005228789057582617, + "objective/train/docs_used": 371543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9728174805641174, + "objective/train/original_loss": 3.6427764892578125, + "objective/train/theoretical_loss": 3.8139026324208456, + "objective/train/tokens_used": 640614400, + "objective/train/unlikelihood_loss": 0.19972942769527435, + "theoretical_loss": 3.8139026324208456, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070010030090271, + "loss": 1.3805, + "theoretical_loss": 3.8138623894105557, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698094282848546, + "loss": 1.5623, + "theoretical_loss": 3.813781919195114, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069608826479438, + "loss": 1.5725, + "theoretical_loss": 3.813701470045001, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694082246740224, + "loss": 1.4707, + "theoretical_loss": 3.8136210419503977, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069207622868606, + "loss": 1.4157, + "theoretical_loss": 3.8135406349014884, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040690070210631896, + "loss": 1.4061, + "theoretical_loss": 3.8134602488884664, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068806419257773, + "loss": 1.3694, + "theoretical_loss": 3.813379883901531, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068605817452357, + "loss": 1.4924, + "theoretical_loss": 3.8132995399308873, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068405215646941, + "loss": 1.5049, + "theoretical_loss": 3.8132192169667483, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682046138415247, + "loss": 1.2999, + "theoretical_loss": 3.813138914999332, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040680040120361083, + "loss": 1.3115, + "theoretical_loss": 3.813058634018865, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067803410230692, + "loss": 1.3317, + "theoretical_loss": 3.812978374015578, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.43664467334747314, + "objective/train/avg_token_score": 0.01415205281227827, + "objective/train/docs_used": 373013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5176647901535034, + "objective/train/original_loss": 3.9611873626708984, + "objective/train/theoretical_loss": 3.8128981349797098, + "objective/train/tokens_used": 642252800, + "objective/train/unlikelihood_loss": 0.1576123982667923, + "theoretical_loss": 3.8128981349797098, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067602808425276, + "loss": 1.6215, + "theoretical_loss": 3.8128981349797098, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674022066198597, + "loss": 1.4195, + "theoretical_loss": 3.8128179169015057, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672016048144433, + "loss": 1.4247, + "theoretical_loss": 3.8127377197712176, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067001003009027, + "loss": 1.4622, + "theoretical_loss": 3.812657543579103, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066800401203611, + "loss": 1.3565, + "theoretical_loss": 3.812577388315426, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040665997993981947, + "loss": 1.5205, + "theoretical_loss": 3.812497253970459, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040663991975927783, + "loss": 1.5612, + "theoretical_loss": 3.812417140534478, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066198595787362, + "loss": 1.3668, + "theoretical_loss": 3.8123370479977687, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040659979939819456, + "loss": 1.4831, + "theoretical_loss": 3.8122569763506204, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406579739217653, + "loss": 1.5363, + "theoretical_loss": 3.812176925583331, + "tokens_seen": 643432448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040655967903711134, + "loss": 1.3226, + "theoretical_loss": 3.8120968956862034, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065396188565697, + "loss": 1.4465, + "theoretical_loss": 3.8120168866495483, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040651955867602806, + "loss": 1.5168, + "theoretical_loss": 3.8119368984636814, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.45222383737564087, + "objective/train/avg_token_score": 0.015421704389154911, + "objective/train/docs_used": 373533, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5024516582489014, + "objective/train/original_loss": 3.635314464569092, + "objective/train/theoretical_loss": 3.8118969121867696, + "objective/train/tokens_used": 643891200, + "objective/train/unlikelihood_loss": 0.1828690469264984, + "theoretical_loss": 3.8118969121867696, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064994984954865, + "loss": 1.2832, + "theoretical_loss": 3.8118569311189265, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040647943831494484, + "loss": 1.453, + "theoretical_loss": 3.811776984605612, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064593781344032, + "loss": 1.4285, + "theoretical_loss": 3.811697058914075, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040643931795386157, + "loss": 1.4884, + "theoretical_loss": 3.811617154034657, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040641925777331993, + "loss": 1.4966, + "theoretical_loss": 3.8115372699577073, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040639919759277834, + "loss": 1.4494, + "theoretical_loss": 3.8114574066735805, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063791374122367, + "loss": 1.4307, + "theoretical_loss": 3.8113775641726386, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040635907723169507, + "loss": 1.3986, + "theoretical_loss": 3.81129774244525, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040633901705115343, + "loss": 1.6454, + "theoretical_loss": 3.8112179414817877, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040631895687061185, + "loss": 1.6522, + "theoretical_loss": 3.8111381612726345, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062988966900702, + "loss": 1.6055, + "theoretical_loss": 3.811058401808177, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040627883650952857, + "loss": 1.479, + "theoretical_loss": 3.8109786630788083, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4299853444099426, + "objective/train/avg_token_score": 0.014909038320183754, + "objective/train/docs_used": 374553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5298352241516113, + "objective/train/original_loss": 4.025496006011963, + "objective/train/theoretical_loss": 3.8108989450749293, + "objective/train/tokens_used": 645529600, + "objective/train/unlikelihood_loss": 0.1589026004076004, + "theoretical_loss": 3.8108989450749293, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040625877632898693, + "loss": 1.5565, + "theoretical_loss": 3.8108989450749293, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062387161484453, + "loss": 1.3728, + "theoretical_loss": 3.810819247786946, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062186559679037, + "loss": 1.518, + "theoretical_loss": 3.8107395712052714, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061985957873621, + "loss": 1.3697, + "theoretical_loss": 3.8106599153203256, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040617853560682044, + "loss": 1.53, + "theoretical_loss": 3.810580280122533, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061584754262788, + "loss": 1.4548, + "theoretical_loss": 3.8105006656023264, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040613841524573727, + "loss": 1.4705, + "theoretical_loss": 3.8104210717501434, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040611835506519563, + "loss": 1.5978, + "theoretical_loss": 3.8103414985564292, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406098294884654, + "loss": 1.3603, + "theoretical_loss": 3.810261946011636, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040607823470411236, + "loss": 1.4798, + "theoretical_loss": 3.810182414106219, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060581745235707, + "loss": 1.4606, + "theoretical_loss": 3.8101029028306437, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040603811434302914, + "loss": 1.5778, + "theoretical_loss": 3.8100234121753798, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060180541624875, + "loss": 1.5656, + "theoretical_loss": 3.8099439421309036, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.45576736330986023, + "objective/train/avg_token_score": 0.028985600918531418, + "objective/train/docs_used": 375146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.656118392944336, + "objective/train/original_loss": 3.9800736904144287, + "objective/train/theoretical_loss": 3.8099042148347366, + "objective/train/tokens_used": 647168000, + "objective/train/unlikelihood_loss": 0.11932669579982758, + "theoretical_loss": 3.8099042148347366, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040599799398194586, + "loss": 1.4929, + "theoretical_loss": 3.809864492687698, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059779338014042, + "loss": 1.5617, + "theoretical_loss": 3.8097850638362516, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595787362086264, + "loss": 1.2915, + "theoretical_loss": 3.8097056555670608, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405937813440321, + "loss": 1.4292, + "theoretical_loss": 3.8096262678706267, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040591775325977936, + "loss": 1.5357, + "theoretical_loss": 3.809546900737457, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058976930792377, + "loss": 1.3051, + "theoretical_loss": 3.809467554158067, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058776328986961, + "loss": 1.4298, + "theoretical_loss": 3.8093882281229767, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058575727181545, + "loss": 1.3352, + "theoretical_loss": 3.809308922622713, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040583751253761287, + "loss": 1.512, + "theoretical_loss": 3.8092296376478094, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581745235707123, + "loss": 1.4593, + "theoretical_loss": 3.809150373188805, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057973921765296, + "loss": 1.3488, + "theoretical_loss": 3.8090711292362456, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405777331995988, + "loss": 1.4565, + "theoretical_loss": 3.808991905780683, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.455278605222702, + "objective/train/avg_token_score": 0.015043376013636589, + "objective/train/docs_used": 375860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5879684686660767, + "objective/train/original_loss": 3.877953290939331, + "objective/train/theoretical_loss": 3.8089127028126764, + "objective/train/tokens_used": 648806400, + "objective/train/unlikelihood_loss": 0.21014168858528137, + "theoretical_loss": 3.8089127028126764, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575727181544637, + "loss": 1.4726, + "theoretical_loss": 3.8089127028126764, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040573721163490473, + "loss": 1.4837, + "theoretical_loss": 3.8088335203227897, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171514543631, + "loss": 1.4837, + "theoretical_loss": 3.8087543583015933, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569709127382146, + "loss": 1.338, + "theoretical_loss": 3.808675216739665, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056770310932799, + "loss": 1.4928, + "theoretical_loss": 3.8085960956275873, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565697091273824, + "loss": 1.5526, + "theoretical_loss": 3.8085169949559505, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056369107321966, + "loss": 1.4267, + "theoretical_loss": 3.8084379147153493, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040561685055165496, + "loss": 1.3143, + "theoretical_loss": 3.8083588548963867, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055967903711134, + "loss": 1.4323, + "theoretical_loss": 3.808279815489671, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040557673019057174, + "loss": 1.2942, + "theoretical_loss": 3.808200796485815, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055566700100301, + "loss": 1.4689, + "theoretical_loss": 3.8081217978754407, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553660982948846, + "loss": 1.6256, + "theoretical_loss": 3.8080428196491747, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055165496489469, + "loss": 1.3445, + "theoretical_loss": 3.8079638617976492, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4335899353027344, + "objective/train/avg_token_score": 0.011522009037435055, + "objective/train/docs_used": 377374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4454277753829956, + "objective/train/original_loss": 3.9384045600891113, + "objective/train/theoretical_loss": 3.8079243905094886, + "objective/train/tokens_used": 650444800, + "objective/train/unlikelihood_loss": 0.13931621611118317, + "theoretical_loss": 3.8079243905094886, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040549648946840524, + "loss": 1.4248, + "theoretical_loss": 3.8078849243115043, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054764292878636, + "loss": 1.3241, + "theoretical_loss": 3.8078060071813846, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040545636910732197, + "loss": 1.6134, + "theoretical_loss": 3.807727110397942, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543630892678033, + "loss": 1.505, + "theoretical_loss": 3.807648233951835, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040541624874623874, + "loss": 1.5242, + "theoretical_loss": 3.807569377833726, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053961885656971, + "loss": 1.3798, + "theoretical_loss": 3.8074905420342864, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537612838515547, + "loss": 1.5358, + "theoretical_loss": 3.807411726544191, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040535606820461383, + "loss": 1.6171, + "theoretical_loss": 3.8073329313541233, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533600802407225, + "loss": 1.5088, + "theoretical_loss": 3.807254156454772, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053159478435306, + "loss": 1.504, + "theoretical_loss": 3.8071754018368305, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040529588766298897, + "loss": 1.4844, + "theoretical_loss": 3.8070966674910007, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040527582748244733, + "loss": 1.308, + "theoretical_loss": 3.807017953407989, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.30968964099884033, + "objective/train/avg_token_score": 0.01105769444257021, + "objective/train/docs_used": 377873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0822099447250366, + "objective/train/original_loss": 4.107913494110107, + "objective/train/theoretical_loss": 3.8069392595785083, + "objective/train/tokens_used": 652083200, + "objective/train/unlikelihood_loss": 0.13217970728874207, + "theoretical_loss": 3.8069392595785083, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052557673019057, + "loss": 1.5052, + "theoretical_loss": 3.8069392595785083, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052357071213641, + "loss": 1.4459, + "theoretical_loss": 3.8068605859932783, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052156469408225, + "loss": 1.5091, + "theoretical_loss": 3.806781932643024, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040519558676028084, + "loss": 1.3071, + "theoretical_loss": 3.8067032995184764, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051755265797392, + "loss": 1.4541, + "theoretical_loss": 3.806624686610374, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051554663991976, + "loss": 1.3498, + "theoretical_loss": 3.8065460939094597, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405135406218656, + "loss": 1.5227, + "theoretical_loss": 3.806467521406484, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040511534603811434, + "loss": 1.5705, + "theoretical_loss": 3.8063889690922013, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050952858575727, + "loss": 1.6055, + "theoretical_loss": 3.806310436957374, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040507522567703107, + "loss": 1.3488, + "theoretical_loss": 3.8062319249927707, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050551654964895, + "loss": 1.4862, + "theoretical_loss": 3.806153433189165, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040503510531594784, + "loss": 1.5408, + "theoretical_loss": 3.8060749615373375, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050150451354062, + "loss": 1.4192, + "theoretical_loss": 3.8059965100280735, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4617546498775482, + "objective/train/avg_token_score": 0.007299356162548065, + "objective/train/docs_used": 379015, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.598456621170044, + "objective/train/original_loss": 3.781627655029297, + "objective/train/theoretical_loss": 3.8059572918240248, + "objective/train/tokens_used": 653721600, + "objective/train/unlikelihood_loss": 0.2492816299200058, + "theoretical_loss": 3.8059572918240248, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040499498495486457, + "loss": 1.5622, + "theoretical_loss": 3.8059180786521654, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404974924774323, + "loss": 1.5489, + "theoretical_loss": 3.805839667400412, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040495486459378135, + "loss": 1.4142, + "theoretical_loss": 3.8057612762636173, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049348044132397, + "loss": 1.4089, + "theoretical_loss": 3.8056829052325916, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040491474423269807, + "loss": 1.4654, + "theoretical_loss": 3.8056045542981516, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040489468405215643, + "loss": 1.4081, + "theoretical_loss": 3.8055262234511193, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040487462387161485, + "loss": 1.4031, + "theoretical_loss": 3.805447912682324, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048545636910732, + "loss": 1.3817, + "theoretical_loss": 3.8053696219825994, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048345035105316, + "loss": 1.3633, + "theoretical_loss": 3.8052913513427864, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040481444332998994, + "loss": 1.4964, + "theoretical_loss": 3.805213100753731, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040479438314944835, + "loss": 1.3457, + "theoretical_loss": 3.8051348702062864, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047743229689067, + "loss": 1.3372, + "theoretical_loss": 3.805056659691311, + "tokens_seen": 655228928 + }, + { + "debugging/length": 515.6, + "debugging/num_segments": 5, + "epoch": 0.2, + "objective/train/aligned_token_count": 0.3613147735595703, + "objective/train/avg_token_score": 0.006067417562007904, + "objective/train/docs_used": 379813, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.339416742324829, + "objective/train/original_loss": 3.8491370677948, + "objective/train/theoretical_loss": 3.804978469199669, + "objective/train/tokens_used": 655360000, + "objective/train/unlikelihood_loss": 0.20670151710510254, + "theoretical_loss": 3.804978469199669, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047542627883651, + "loss": 1.4639, + "theoretical_loss": 3.804978469199669, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040473420260782344, + "loss": 1.3815, + "theoretical_loss": 3.8049002987222313, + "tokens_seen": 655491072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047141424272818, + "loss": 1.4452, + "theoretical_loss": 3.8048221482498743, + "tokens_seen": 655622144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046940822467402, + "loss": 1.4445, + "theoretical_loss": 3.8047440177734804, + "tokens_seen": 655753216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046740220661986, + "loss": 1.2184, + "theoretical_loss": 3.804665907283938, + "tokens_seen": 655884288 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404653961885657, + "loss": 1.2792, + "theoretical_loss": 3.804587816772142, + "tokens_seen": 656015360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040463390170511536, + "loss": 1.5185, + "theoretical_loss": 3.8045097462289923, + "tokens_seen": 656146432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046138415245738, + "loss": 1.4538, + "theoretical_loss": 3.8044316956453956, + "tokens_seen": 656277504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040459378134403214, + "loss": 1.4853, + "theoretical_loss": 3.804353665012264, + "tokens_seen": 656408576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045737211634905, + "loss": 1.4347, + "theoretical_loss": 3.8042756543205156, + "tokens_seen": 656539648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040455366098294886, + "loss": 1.5086, + "theoretical_loss": 3.8041976635610752, + "tokens_seen": 656670720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045336008024073, + "loss": 1.3484, + "theoretical_loss": 3.804119692724873, + "tokens_seen": 656801792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040451354062186564, + "loss": 1.4582, + "theoretical_loss": 3.8040417418028443, + "tokens_seen": 656932864 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.5032991170883179, + "objective/train/avg_token_score": 0.04026363417506218, + "objective/train/docs_used": 381022, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7246177196502686, + "objective/train/original_loss": 3.856398820877075, + "objective/train/theoretical_loss": 3.8040027738068143, + "objective/train/tokens_used": 656998400, + "objective/train/unlikelihood_loss": 0.13124938309192657, + "theoretical_loss": 3.8040027738068143, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404493480441324, + "loss": 1.4797, + "theoretical_loss": 3.8039638107859317, + "tokens_seen": 657063936 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040447342026078237, + "loss": 1.4157, + "theoretical_loss": 3.803885899665083, + "tokens_seen": 657195008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040445336008024073, + "loss": 1.4632, + "theoretical_loss": 3.803808008431252, + "tokens_seen": 657326080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040443329989969915, + "loss": 1.4103, + "theoretical_loss": 3.8037301370753993, + "tokens_seen": 657457152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044132397191575, + "loss": 1.3352, + "theoretical_loss": 3.8036522855884893, + "tokens_seen": 657588224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040439317953861587, + "loss": 1.4645, + "theoretical_loss": 3.803574453961495, + "tokens_seen": 657719296 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040437311935807423, + "loss": 1.4167, + "theoretical_loss": 3.803496642185393, + "tokens_seen": 657850368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040435305917753265, + "loss": 1.5585, + "theoretical_loss": 3.8034188502511665, + "tokens_seen": 657981440 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404332998996991, + "loss": 1.5436, + "theoretical_loss": 3.8033410781498054, + "tokens_seen": 658112512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040431293881644937, + "loss": 1.565, + "theoretical_loss": 3.8032633258723045, + "tokens_seen": 658243584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040429287863590773, + "loss": 1.6411, + "theoretical_loss": 3.8031855934096654, + "tokens_seen": 658374656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042728184553661, + "loss": 1.526, + "theoretical_loss": 3.8031078807528944, + "tokens_seen": 658505728 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.41080155968666077, + "objective/train/avg_token_score": 0.024585559964179993, + "objective/train/docs_used": 381585, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4388028383255005, + "objective/train/original_loss": 4.1186909675598145, + "objective/train/theoretical_loss": 3.803030187893005, + "objective/train/tokens_used": 658636800, + "objective/train/unlikelihood_loss": 0.14371395111083984, + "theoretical_loss": 3.803030187893005, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042527582748245, + "loss": 1.4232, + "theoretical_loss": 3.803030187893005, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042326980942829, + "loss": 1.5521, + "theoretical_loss": 3.8029525148210146, + "tokens_seen": 658767872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040421263791374124, + "loss": 1.4719, + "theoretical_loss": 3.8028748615279495, + "tokens_seen": 658898944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041925777331996, + "loss": 1.4519, + "theoretical_loss": 3.802797228004839, + "tokens_seen": 659030016 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404172517552658, + "loss": 1.2698, + "theoretical_loss": 3.8027196142427195, + "tokens_seen": 659161088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041524573721164, + "loss": 1.4676, + "theoretical_loss": 3.802642020232633, + "tokens_seen": 659292160 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040413239719157474, + "loss": 1.364, + "theoretical_loss": 3.8025644459656283, + "tokens_seen": 659423232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041123370110331, + "loss": 1.3945, + "theoretical_loss": 3.802486891432758, + "tokens_seen": 659554304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040409227683049147, + "loss": 1.3054, + "theoretical_loss": 3.802409356625082, + "tokens_seen": 659685376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040722166499499, + "loss": 1.4835, + "theoretical_loss": 3.802331841533666, + "tokens_seen": 659816448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040405215646940824, + "loss": 1.4453, + "theoretical_loss": 3.8022543461495806, + "tokens_seen": 659947520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040320962888666, + "loss": 1.498, + "theoretical_loss": 3.802176870463904, + "tokens_seen": 660078592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040401203610832497, + "loss": 1.4038, + "theoretical_loss": 3.8020994144677185, + "tokens_seen": 660209664 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.5050097703933716, + "objective/train/avg_token_score": 0.013619226403534412, + "objective/train/docs_used": 382875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6750119924545288, + "objective/train/original_loss": 3.8228859901428223, + "objective/train/theoretical_loss": 3.8020606938504, + "objective/train/tokens_used": 660275200, + "objective/train/unlikelihood_loss": 0.1518351137638092, + "theoretical_loss": 3.8020606938504, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039919759277834, + "loss": 1.3671, + "theoretical_loss": 3.8020219781521125, + "tokens_seen": 660340736 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040397191574724175, + "loss": 1.404, + "theoretical_loss": 3.801944561508181, + "tokens_seen": 660471808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039518555667001, + "loss": 1.3005, + "theoretical_loss": 3.8018671645270237, + "tokens_seen": 660602880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393179538615847, + "loss": 1.3154, + "theoretical_loss": 3.8017897871997466, + "tokens_seen": 660733952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040391173520561683, + "loss": 1.3829, + "theoretical_loss": 3.8017124295174622, + "tokens_seen": 660865024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389167502507525, + "loss": 1.4106, + "theoretical_loss": 3.8016350914712884, + "tokens_seen": 660996096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038716148445336, + "loss": 1.282, + "theoretical_loss": 3.8015577730523473, + "tokens_seen": 661127168 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403851554663992, + "loss": 1.3615, + "theoretical_loss": 3.801480474251769, + "tokens_seen": 661258240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040383149448345034, + "loss": 1.4237, + "theoretical_loss": 3.801403195060688, + "tokens_seen": 661389312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040381143430290875, + "loss": 1.4849, + "theoretical_loss": 3.8013259354702456, + "tokens_seen": 661520384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037913741223671, + "loss": 1.3307, + "theoretical_loss": 3.801248695471588, + "tokens_seen": 661651456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037713139418255, + "loss": 1.2963, + "theoretical_loss": 3.801171475055867, + "tokens_seen": 661782528 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4411656856536865, + "objective/train/avg_token_score": 0.005799155682325363, + "objective/train/docs_used": 383560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4003348350524902, + "objective/train/original_loss": 3.5925559997558594, + "objective/train/theoretical_loss": 3.8010942742142415, + "objective/train/tokens_used": 661913600, + "objective/train/unlikelihood_loss": 0.15024060010910034, + "theoretical_loss": 3.8010942742142415, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375125376128384, + "loss": 1.3566, + "theoretical_loss": 3.8010942742142415, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037311935807422, + "loss": 1.3069, + "theoretical_loss": 3.801017092937874, + "tokens_seen": 662044672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037111334002006, + "loss": 1.4094, + "theoretical_loss": 3.8009399312179353, + "tokens_seen": 662175744 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403691073219659, + "loss": 1.4527, + "theoretical_loss": 3.800862789045599, + "tokens_seen": 662306816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040367101303911734, + "loss": 1.5146, + "theoretical_loss": 3.8007856664120476, + "tokens_seen": 662437888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036509528585757, + "loss": 1.5428, + "theoretical_loss": 3.8007085633084663, + "tokens_seen": 662568960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036308926780341, + "loss": 1.4638, + "theoretical_loss": 3.8006314797260488, + "tokens_seen": 662700032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036108324974925, + "loss": 1.5226, + "theoretical_loss": 3.8005544156559923, + "tokens_seen": 662831104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359077231695085, + "loss": 1.4905, + "theoretical_loss": 3.8004773710895003, + "tokens_seen": 662962176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035707121364092, + "loss": 1.3048, + "theoretical_loss": 3.800400346017783, + "tokens_seen": 663093248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355065195586757, + "loss": 1.4244, + "theoretical_loss": 3.800323340432055, + "tokens_seen": 663224320 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403530591775326, + "loss": 1.4905, + "theoretical_loss": 3.800246354323538, + "tokens_seen": 663355392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040351053159478435, + "loss": 1.5756, + "theoretical_loss": 3.8001693876834572, + "tokens_seen": 663486464 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.44611436128616333, + "objective/train/avg_token_score": 0.022830946370959282, + "objective/train/docs_used": 384103, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4629120826721191, + "objective/train/original_loss": 3.9829959869384766, + "objective/train/theoretical_loss": 3.8001309116613413, + "objective/train/tokens_used": 663552000, + "objective/train/unlikelihood_loss": 0.18428486585617065, + "theoretical_loss": 3.8001309116613413, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034904714142427, + "loss": 1.5134, + "theoretical_loss": 3.8000924405030467, + "tokens_seen": 663617536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034704112337011, + "loss": 1.377, + "theoretical_loss": 3.8000155127735424, + "tokens_seen": 663748608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034503510531595, + "loss": 1.3325, + "theoretical_loss": 3.799938604486189, + "tokens_seen": 663879680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343029087261785, + "loss": 1.5118, + "theoretical_loss": 3.799861715632236, + "tokens_seen": 664010752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034102306920762, + "loss": 1.2592, + "theoretical_loss": 3.7997848462029378, + "tokens_seen": 664141824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033901705115346, + "loss": 1.1487, + "theoretical_loss": 3.799707996189555, + "tokens_seen": 664272896 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403370110330993, + "loss": 1.5225, + "theoretical_loss": 3.799631165583354, + "tokens_seen": 664403968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040335005015045136, + "loss": 1.4555, + "theoretical_loss": 3.7995543543756063, + "tokens_seen": 664535040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033299899699097, + "loss": 1.4798, + "theoretical_loss": 3.7994775625575903, + "tokens_seen": 664666112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033099297893681, + "loss": 1.4712, + "theoretical_loss": 3.799400790120588, + "tokens_seen": 664797184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040328986960882644, + "loss": 1.4632, + "theoretical_loss": 3.7993240370558894, + "tokens_seen": 664928256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040326980942828486, + "loss": 1.5863, + "theoretical_loss": 3.7992473033547887, + "tokens_seen": 665059328 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.5188171863555908, + "objective/train/avg_token_score": 0.013243277557194233, + "objective/train/docs_used": 385247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.709418535232544, + "objective/train/original_loss": 3.517648696899414, + "objective/train/theoretical_loss": 3.799170589008585, + "objective/train/tokens_used": 665190400, + "objective/train/unlikelihood_loss": 0.16377879679203033, + "theoretical_loss": 3.799170589008585, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032497492477432, + "loss": 1.4992, + "theoretical_loss": 3.799170589008585, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032296890672016, + "loss": 1.4591, + "theoretical_loss": 3.7990938940085845, + "tokens_seen": 665321472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040320962888665995, + "loss": 1.6708, + "theoretical_loss": 3.7990172183460995, + "tokens_seen": 665452544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040318956870611836, + "loss": 1.371, + "theoretical_loss": 3.7989405620124455, + "tokens_seen": 665583616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031695085255767, + "loss": 1.4103, + "theoretical_loss": 3.7988639249989458, + "tokens_seen": 665714688 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040314944834503514, + "loss": 1.4399, + "theoretical_loss": 3.7987873072969283, + "tokens_seen": 665845760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031293881644935, + "loss": 1.3848, + "theoretical_loss": 3.798710708897727, + "tokens_seen": 665976832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040310932798395187, + "loss": 1.4698, + "theoretical_loss": 3.798634129792681, + "tokens_seen": 666107904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030892678034103, + "loss": 1.4668, + "theoretical_loss": 3.798557569973135, + "tokens_seen": 666238976 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040306920762286864, + "loss": 1.3652, + "theoretical_loss": 3.79848102943044, + "tokens_seen": 666370048 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403049147442327, + "loss": 1.2849, + "theoretical_loss": 3.7984045081559517, + "tokens_seen": 666501120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040302908726178537, + "loss": 1.3247, + "theoretical_loss": 3.798328006141032, + "tokens_seen": 666632192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030090270812438, + "loss": 1.5208, + "theoretical_loss": 3.798251523377048, + "tokens_seen": 666763264 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.44214320182800293, + "objective/train/avg_token_score": 0.015530267730355263, + "objective/train/docs_used": 385855, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4893537759780884, + "objective/train/original_loss": 4.056307315826416, + "objective/train/theoretical_loss": 3.79821328921146, + "objective/train/tokens_used": 666828800, + "objective/train/unlikelihood_loss": 0.12036958336830139, + "theoretical_loss": 3.79821328921146, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298896690070215, + "loss": 1.4938, + "theoretical_loss": 3.798175059855372, + "tokens_seen": 666894336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029689067201605, + "loss": 1.4625, + "theoretical_loss": 3.7980986155673833, + "tokens_seen": 667025408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294884653961887, + "loss": 1.3747, + "theoretical_loss": 3.7980221905044655, + "tokens_seen": 667156480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040292878635907723, + "loss": 1.3319, + "theoretical_loss": 3.797945784658008, + "tokens_seen": 667287552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290872617853565, + "loss": 1.616, + "theoretical_loss": 3.7978693980194054, + "tokens_seen": 667418624 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402888665997994, + "loss": 1.3439, + "theoretical_loss": 3.7977930305800585, + "tokens_seen": 667549696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686058174524, + "loss": 1.4135, + "theoretical_loss": 3.7977166823313735, + "tokens_seen": 667680768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284854563691074, + "loss": 1.5208, + "theoretical_loss": 3.797640353264762, + "tokens_seen": 667811840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040282848545636915, + "loss": 1.3685, + "theoretical_loss": 3.7975640433716413, + "tokens_seen": 667942912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028084252758275, + "loss": 1.3658, + "theoretical_loss": 3.797487752643434, + "tokens_seen": 668073984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027883650952859, + "loss": 1.4006, + "theoretical_loss": 3.797411481071568, + "tokens_seen": 668205056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040276830491474424, + "loss": 1.2342, + "theoretical_loss": 3.7973352286474773, + "tokens_seen": 668336128 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4714687168598175, + "objective/train/avg_token_score": 0.017436988651752472, + "objective/train/docs_used": 387212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6166558265686035, + "objective/train/original_loss": 3.713994264602661, + "objective/train/theoretical_loss": 3.7972589953626006, + "objective/train/tokens_used": 668467200, + "objective/train/unlikelihood_loss": 0.19959448277950287, + "theoretical_loss": 3.7972589953626006, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027482447342026, + "loss": 1.5957, + "theoretical_loss": 3.7972589953626006, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402728184553661, + "loss": 1.1721, + "theoretical_loss": 3.7971827812083836, + "tokens_seen": 668598272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027081243731194, + "loss": 1.4366, + "theoretical_loss": 3.7971065861762754, + "tokens_seen": 668729344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040268806419257774, + "loss": 1.4566, + "theoretical_loss": 3.797030410257733, + "tokens_seen": 668860416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026680040120361, + "loss": 1.3538, + "theoretical_loss": 3.796954253444216, + "tokens_seen": 668991488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026479438314945, + "loss": 1.45, + "theoretical_loss": 3.7968781157271922, + "tokens_seen": 669122560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026278836509529, + "loss": 1.2487, + "theoretical_loss": 3.796801997098134, + "tokens_seen": 669253632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040260782347041125, + "loss": 1.4279, + "theoretical_loss": 3.7967258975485185, + "tokens_seen": 669384704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025877632898696, + "loss": 1.4366, + "theoretical_loss": 3.796649817069828, + "tokens_seen": 669515776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040256770310932797, + "loss": 1.5685, + "theoretical_loss": 3.796573755653553, + "tokens_seen": 669646848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025476429287864, + "loss": 1.3805, + "theoretical_loss": 3.7964977132911866, + "tokens_seen": 669777920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040252758274824475, + "loss": 1.4142, + "theoretical_loss": 3.7964216899742276, + "tokens_seen": 669908992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025075225677031, + "loss": 1.6182, + "theoretical_loss": 3.796345685694182, + "tokens_seen": 670040064 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.36100929975509644, + "objective/train/avg_token_score": 0.013192600570619106, + "objective/train/docs_used": 387914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2840262651443481, + "objective/train/original_loss": 3.754566192626953, + "objective/train/theoretical_loss": 3.796307690690348, + "objective/train/tokens_used": 670105600, + "objective/train/unlikelihood_loss": 0.21956370770931244, + "theoretical_loss": 3.796307690690348, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024874623871615, + "loss": 1.5326, + "theoretical_loss": 3.7962697004425596, + "tokens_seen": 670171136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024674022066199, + "loss": 1.4881, + "theoretical_loss": 3.7961937342108767, + "tokens_seen": 670302208 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040244734202607825, + "loss": 1.2563, + "theoretical_loss": 3.796117786990655, + "tokens_seen": 670433280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024272818455366, + "loss": 1.538, + "theoretical_loss": 3.79604185877342, + "tokens_seen": 670564352 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402407221664995, + "loss": 1.3223, + "theoretical_loss": 3.7959659495507045, + "tokens_seen": 670695424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023871614844534, + "loss": 1.2454, + "theoretical_loss": 3.795890059314047, + "tokens_seen": 670826496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040236710130391176, + "loss": 1.5497, + "theoretical_loss": 3.7958141880549885, + "tokens_seen": 670957568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023470411233701, + "loss": 1.5501, + "theoretical_loss": 3.79573833576508, + "tokens_seen": 671088640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023269809428285, + "loss": 1.4536, + "theoretical_loss": 3.7956625024358726, + "tokens_seen": 671219712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040230692076228684, + "loss": 1.3496, + "theoretical_loss": 3.795586688058928, + "tokens_seen": 671350784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040228686058174526, + "loss": 1.3002, + "theoretical_loss": 3.795510892625809, + "tokens_seen": 671481856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022668004012036, + "loss": 1.3597, + "theoretical_loss": 3.795435116128087, + "tokens_seen": 671612928 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.3643084168434143, + "objective/train/avg_token_score": 0.012728302739560604, + "objective/train/docs_used": 389368, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2042542695999146, + "objective/train/original_loss": 3.5068728923797607, + "objective/train/theoretical_loss": 3.795359358557337, + "objective/train/tokens_used": 671744000, + "objective/train/unlikelihood_loss": 0.2943571209907532, + "theoretical_loss": 3.795359358557337, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402246740220662, + "loss": 1.4169, + "theoretical_loss": 3.795359358557337, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040222668004012035, + "loss": 1.3901, + "theoretical_loss": 3.79528361990514, + "tokens_seen": 671875072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040220661985957876, + "loss": 1.5021, + "theoretical_loss": 3.7952079001630814, + "tokens_seen": 672006144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021865596790371, + "loss": 1.4185, + "theoretical_loss": 3.795132199322754, + "tokens_seen": 672137216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021664994984955, + "loss": 1.4172, + "theoretical_loss": 3.795056517375754, + "tokens_seen": 672268288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214643931795385, + "loss": 1.518, + "theoretical_loss": 3.7949808543136845, + "tokens_seen": 672399360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021263791374122, + "loss": 1.4102, + "theoretical_loss": 3.794905210128152, + "tokens_seen": 672530432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040210631895687063, + "loss": 1.4404, + "theoretical_loss": 3.7948295848107714, + "tokens_seen": 672661504 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402086258776329, + "loss": 1.3552, + "theoretical_loss": 3.7947539783531594, + "tokens_seen": 672792576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040206619859578735, + "loss": 1.4169, + "theoretical_loss": 3.794678390746941, + "tokens_seen": 672923648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020461384152457, + "loss": 1.1783, + "theoretical_loss": 3.7946028219837453, + "tokens_seen": 673054720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040202607823470413, + "loss": 1.336, + "theoretical_loss": 3.794527272055206, + "tokens_seen": 673185792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020060180541625, + "loss": 1.526, + "theoretical_loss": 3.7944517409529634, + "tokens_seen": 673316864 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.3549608886241913, + "objective/train/avg_token_score": 0.0124678248539567, + "objective/train/docs_used": 390091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1494449377059937, + "objective/train/original_loss": 3.64634370803833, + "objective/train/theoretical_loss": 3.7944139824590923, + "objective/train/tokens_used": 673382400, + "objective/train/unlikelihood_loss": 0.21259579062461853, + "theoretical_loss": 3.7944139824590923, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040198595787362086, + "loss": 1.3566, + "theoretical_loss": 3.794376228668663, + "tokens_seen": 673447936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019658976930792, + "loss": 1.413, + "theoretical_loss": 3.7943007351939553, + "tokens_seen": 673579008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019458375125376, + "loss": 1.5021, + "theoretical_loss": 3.794225260520496, + "tokens_seen": 673710080 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401925777331996, + "loss": 1.5602, + "theoretical_loss": 3.7941498046399458, + "tokens_seen": 673841152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040190571715145436, + "loss": 1.5949, + "theoretical_loss": 3.7940743675439714, + "tokens_seen": 673972224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018856569709127, + "loss": 1.384, + "theoretical_loss": 3.793998949224245, + "tokens_seen": 674103296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018655967903711, + "loss": 1.5939, + "theoretical_loss": 3.793923549672444, + "tokens_seen": 674234368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018455366098295, + "loss": 1.5378, + "theoretical_loss": 3.7938481688802503, + "tokens_seen": 674365440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040182547642928786, + "loss": 1.3974, + "theoretical_loss": 3.793772806839352, + "tokens_seen": 674496512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018054162487462, + "loss": 1.5573, + "theoretical_loss": 3.7936974635414416, + "tokens_seen": 674627584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017853560682046, + "loss": 1.383, + "theoretical_loss": 3.793622138978218, + "tokens_seen": 674758656 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040176529588766295, + "loss": 1.2955, + "theoretical_loss": 3.7935468331413844, + "tokens_seen": 674889728 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.31830400228500366, + "objective/train/avg_token_score": 0.01861387863755226, + "objective/train/docs_used": 391518, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1915409564971924, + "objective/train/original_loss": 4.150604724884033, + "objective/train/theoretical_loss": 3.79347154602265, + "objective/train/tokens_used": 675020800, + "objective/train/unlikelihood_loss": 0.1649107038974762, + "theoretical_loss": 3.79347154602265, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040174523570712137, + "loss": 1.4985, + "theoretical_loss": 3.79347154602265, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040172517552657973, + "loss": 1.4581, + "theoretical_loss": 3.7933962776137284, + "tokens_seen": 675151872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017051153460381, + "loss": 1.4327, + "theoretical_loss": 3.79332102790634, + "tokens_seen": 675282944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040168505516549645, + "loss": 1.4196, + "theoretical_loss": 3.793245796892209, + "tokens_seen": 675414016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040166499498495487, + "loss": 1.2892, + "theoretical_loss": 3.7931705845630654, + "tokens_seen": 675545088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016449348044133, + "loss": 1.4264, + "theoretical_loss": 3.7930953909106444, + "tokens_seen": 675676160 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040162487462387165, + "loss": 1.4377, + "theoretical_loss": 3.793020215926686, + "tokens_seen": 675807232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040160481444333, + "loss": 1.4299, + "theoretical_loss": 3.7929450596029373, + "tokens_seen": 675938304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040158475426278837, + "loss": 1.3796, + "theoretical_loss": 3.7928699219311484, + "tokens_seen": 676069376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015646940822468, + "loss": 1.3173, + "theoretical_loss": 3.7927948029030754, + "tokens_seen": 676200448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040154463390170515, + "loss": 1.4398, + "theoretical_loss": 3.792719702510481, + "tokens_seen": 676331520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015245737211635, + "loss": 1.4193, + "theoretical_loss": 3.79264462074513, + "tokens_seen": 676462592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015045135406219, + "loss": 1.1739, + "theoretical_loss": 3.7925695575987963, + "tokens_seen": 676593664 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.4125122129917145, + "objective/train/avg_token_score": 0.027648912742733955, + "objective/train/docs_used": 392138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0395374298095703, + "objective/train/original_loss": 3.6678740978240967, + "objective/train/theoretical_loss": 3.792532033005191, + "objective/train/tokens_used": 676659200, + "objective/train/unlikelihood_loss": 0.1877020001411438, + "theoretical_loss": 3.792532033005191, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014844533600803, + "loss": 1.4845, + "theoretical_loss": 3.7924945130632564, + "tokens_seen": 676724736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040146439317953865, + "loss": 1.3842, + "theoretical_loss": 3.792419487130292, + "tokens_seen": 676855808 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401444332998997, + "loss": 1.2873, + "theoretical_loss": 3.7923444797916916, + "tokens_seen": 676986880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014242728184554, + "loss": 1.3376, + "theoretical_loss": 3.7922694910392476, + "tokens_seen": 677117952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014042126379138, + "loss": 1.4501, + "theoretical_loss": 3.7921945208647583, + "tokens_seen": 677249024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040138415245737216, + "loss": 1.4055, + "theoretical_loss": 3.792119569260027, + "tokens_seen": 677380096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013640922768305, + "loss": 1.3605, + "theoretical_loss": 3.792044636216862, + "tokens_seen": 677511168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013440320962889, + "loss": 1.6221, + "theoretical_loss": 3.791969721727077, + "tokens_seen": 677642240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040132397191574724, + "loss": 1.475, + "theoretical_loss": 3.7918948257824914, + "tokens_seen": 677773312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040130391173520566, + "loss": 1.3805, + "theoretical_loss": 3.791819948374928, + "tokens_seen": 677904384 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401283851554664, + "loss": 1.32, + "theoretical_loss": 3.7917450894962177, + "tokens_seen": 678035456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012637913741224, + "loss": 1.2921, + "theoretical_loss": 3.7916702491381935, + "tokens_seen": 678166528 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.5511974692344666, + "objective/train/avg_token_score": 0.031187957152724266, + "objective/train/docs_used": 393619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8398100137710571, + "objective/train/original_loss": 3.7891223430633545, + "objective/train/theoretical_loss": 3.7915954272926955, + "objective/train/tokens_used": 678297600, + "objective/train/unlikelihood_loss": 0.11208107322454453, + "theoretical_loss": 3.7915954272926955, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040124373119358075, + "loss": 1.2979, + "theoretical_loss": 3.7915954272926955, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040122367101303916, + "loss": 1.4132, + "theoretical_loss": 3.791520623951569, + "tokens_seen": 678428672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012036108324975, + "loss": 1.4451, + "theoretical_loss": 3.7914458391066628, + "tokens_seen": 678559744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011835506519559, + "loss": 1.5308, + "theoretical_loss": 3.7913710727498335, + "tokens_seen": 678690816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040116349047141425, + "loss": 1.5374, + "theoretical_loss": 3.7912963248729397, + "tokens_seen": 678821888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011434302908726, + "loss": 1.4265, + "theoretical_loss": 3.7912215954678476, + "tokens_seen": 678952960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040112337011033103, + "loss": 1.4148, + "theoretical_loss": 3.791146884526428, + "tokens_seen": 679084032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011033099297894, + "loss": 1.4894, + "theoretical_loss": 3.7910721920405566, + "tokens_seen": 679215104 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040108324974924775, + "loss": 1.3246, + "theoretical_loss": 3.790997518002113, + "tokens_seen": 679346176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010631895687061, + "loss": 1.1998, + "theoretical_loss": 3.7909228624029856, + "tokens_seen": 679477248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040104312938816453, + "loss": 1.3263, + "theoretical_loss": 3.790848225235063, + "tokens_seen": 679608320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010230692076229, + "loss": 1.4284, + "theoretical_loss": 3.790773606490243, + "tokens_seen": 679739392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040100300902708126, + "loss": 1.4817, + "theoretical_loss": 3.790699006160427, + "tokens_seen": 679870464 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.4543621838092804, + "objective/train/avg_token_score": 0.015197647735476494, + "objective/train/docs_used": 393966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.633802890777588, + "objective/train/original_loss": 3.740381956100464, + "objective/train/theoretical_loss": 3.7906617128986153, + "objective/train/tokens_used": 679936000, + "objective/train/unlikelihood_loss": 0.21875910460948944, + "theoretical_loss": 3.7906617128986153, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009829488465396, + "loss": 1.3486, + "theoretical_loss": 3.79062442423752, + "tokens_seen": 680001536 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400962888665998, + "loss": 1.473, + "theoretical_loss": 3.790549860713435, + "tokens_seen": 680132608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009428284854564, + "loss": 1.4432, + "theoretical_loss": 3.7904753155800885, + "tokens_seen": 680263680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040092276830491476, + "loss": 1.3859, + "theoretical_loss": 3.7904007888294027, + "tokens_seen": 680394752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009027081243731, + "loss": 1.3828, + "theoretical_loss": 3.790326280453303, + "tokens_seen": 680525824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008826479438315, + "loss": 1.4742, + "theoretical_loss": 3.7902517904437225, + "tokens_seen": 680656896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008625877632899, + "loss": 1.3765, + "theoretical_loss": 3.790177318792599, + "tokens_seen": 680787968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040084252758274826, + "loss": 1.4612, + "theoretical_loss": 3.790102865491874, + "tokens_seen": 680919040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008224674022066, + "loss": 1.4717, + "theoretical_loss": 3.7900284305334937, + "tokens_seen": 681050112 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400802407221665, + "loss": 1.4703, + "theoretical_loss": 3.789954013909412, + "tokens_seen": 681181184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040078234704112335, + "loss": 1.5432, + "theoretical_loss": 3.7898796156115866, + "tokens_seen": 681312256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040076228686058177, + "loss": 1.418, + "theoretical_loss": 3.7898052356319787, + "tokens_seen": 681443328 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.49364614486694336, + "objective/train/avg_token_score": 0.006545716430991888, + "objective/train/docs_used": 395160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.661900281906128, + "objective/train/original_loss": 3.7815558910369873, + "objective/train/theoretical_loss": 3.789730873962557, + "objective/train/tokens_used": 681574400, + "objective/train/unlikelihood_loss": 0.14150911569595337, + "theoretical_loss": 3.789730873962557, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040074222668004013, + "loss": 1.468, + "theoretical_loss": 3.789730873962557, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007221664994985, + "loss": 1.4777, + "theoretical_loss": 3.7896565305952934, + "tokens_seen": 681705472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040070210631895685, + "loss": 1.4588, + "theoretical_loss": 3.7895822055221657, + "tokens_seen": 681836544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040068204613841527, + "loss": 1.2407, + "theoretical_loss": 3.789507898735158, + "tokens_seen": 681967616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040066198595787363, + "loss": 1.3401, + "theoretical_loss": 3.789433610226256, + "tokens_seen": 682098688 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400641925777332, + "loss": 1.5068, + "theoretical_loss": 3.7893593399874548, + "tokens_seen": 682229760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040062186559679036, + "loss": 1.3553, + "theoretical_loss": 3.7892850880107503, + "tokens_seen": 682360832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006018054162487, + "loss": 1.5858, + "theoretical_loss": 3.7892108542881475, + "tokens_seen": 682491904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040058174523570713, + "loss": 1.4714, + "theoretical_loss": 3.7891366388116525, + "tokens_seen": 682622976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005616850551655, + "loss": 1.5806, + "theoretical_loss": 3.7890624415732796, + "tokens_seen": 682754048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040054162487462386, + "loss": 1.2537, + "theoretical_loss": 3.7889882625650464, + "tokens_seen": 682885120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005215646940822, + "loss": 1.5051, + "theoretical_loss": 3.788914101778976, + "tokens_seen": 683016192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040050150451354064, + "loss": 1.2163, + "theoretical_loss": 3.788839959207097, + "tokens_seen": 683147264 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.3299120366573334, + "objective/train/avg_token_score": 0.011881964281201363, + "objective/train/docs_used": 395993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1687992811203003, + "objective/train/original_loss": 3.7518150806427, + "objective/train/theoretical_loss": 3.788802894748989, + "objective/train/tokens_used": 683212800, + "objective/train/unlikelihood_loss": 0.21577776968479156, + "theoretical_loss": 3.788802894748989, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400481444332999, + "loss": 1.3472, + "theoretical_loss": 3.788765834841442, + "tokens_seen": 683278336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040046138415245736, + "loss": 1.3949, + "theoretical_loss": 3.788691728674049, + "tokens_seen": 683409408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004413239719157, + "loss": 1.5839, + "theoretical_loss": 3.7886176406969625, + "tokens_seen": 683540480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004212637913741, + "loss": 1.4091, + "theoretical_loss": 3.7885435709022284, + "tokens_seen": 683671552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004012036108325, + "loss": 1.5031, + "theoretical_loss": 3.788469519281902, + "tokens_seen": 683802624 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040038114343029087, + "loss": 1.5656, + "theoretical_loss": 3.7883954858280404, + "tokens_seen": 683933696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040036108324974923, + "loss": 1.5519, + "theoretical_loss": 3.788321470532707, + "tokens_seen": 684064768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003410230692076, + "loss": 1.5119, + "theoretical_loss": 3.78824747338797, + "tokens_seen": 684195840 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400320962888666, + "loss": 1.4957, + "theoretical_loss": 3.7881734943859025, + "tokens_seen": 684326912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040030090270812437, + "loss": 1.4186, + "theoretical_loss": 3.788099533518582, + "tokens_seen": 684457984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040028084252758273, + "loss": 1.4072, + "theoretical_loss": 3.7880255907780924, + "tokens_seen": 684589056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002607823470411, + "loss": 1.4357, + "theoretical_loss": 3.7879516661565216, + "tokens_seen": 684720128 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.43261241912841797, + "objective/train/avg_token_score": 0.022025832906365395, + "objective/train/docs_used": 397274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5358208417892456, + "objective/train/original_loss": 4.505789756774902, + "objective/train/theoretical_loss": 3.787877759645963, + "objective/train/tokens_used": 684851200, + "objective/train/unlikelihood_loss": 0.07978002727031708, + "theoretical_loss": 3.787877759645963, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040024072216649946, + "loss": 1.4395, + "theoretical_loss": 3.787877759645963, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040022066198595787, + "loss": 1.5951, + "theoretical_loss": 3.787803871238513, + "tokens_seen": 684982272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040020060180541623, + "loss": 1.5471, + "theoretical_loss": 3.787730000926276, + "tokens_seen": 685113344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001805416248746, + "loss": 1.3414, + "theoretical_loss": 3.78765614870136, + "tokens_seen": 685244416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040016048144433296, + "loss": 1.4514, + "theoretical_loss": 3.787582314555877, + "tokens_seen": 685375488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040014042126379143, + "loss": 1.3401, + "theoretical_loss": 3.787508498481945, + "tokens_seen": 685506560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001203610832498, + "loss": 1.4518, + "theoretical_loss": 3.7874347004716866, + "tokens_seen": 685637632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040010030090270815, + "loss": 1.3673, + "theoretical_loss": 3.78736092051723, + "tokens_seen": 685768704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000802407221665, + "loss": 1.5298, + "theoretical_loss": 3.7872871586107077, + "tokens_seen": 685899776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040006018054162493, + "loss": 1.5167, + "theoretical_loss": 3.787213414744257, + "tokens_seen": 686030848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000401203610833, + "loss": 1.5146, + "theoretical_loss": 3.7871396889100204, + "tokens_seen": 686161920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040002006018054166, + "loss": 1.6207, + "theoretical_loss": 3.7870659811001457, + "tokens_seen": 686292992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004, + "loss": 1.4184, + "theoretical_loss": 3.7869922913067846, + "tokens_seen": 686424064 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.48973608016967773, + "objective/train/avg_token_score": 0.005983791314065456, + "objective/train/docs_used": 397954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6822431087493896, + "objective/train/original_loss": 3.8239264488220215, + "objective/train/theoretical_loss": 3.786955453163846, + "objective/train/tokens_used": 686489600, + "objective/train/unlikelihood_loss": 0.14866818487644196, + "theoretical_loss": 3.786955453163846, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999799398194584, + "loss": 1.3966, + "theoretical_loss": 3.7869186195220945, + "tokens_seen": 686555136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999598796389168, + "loss": 1.5624, + "theoretical_loss": 3.7868449657382377, + "tokens_seen": 686686208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039993981945837516, + "loss": 1.6192, + "theoretical_loss": 3.7867713299473813, + "tokens_seen": 686817280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999197592778335, + "loss": 1.5611, + "theoretical_loss": 3.786697712141697, + "tokens_seen": 686948352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998996990972919, + "loss": 1.511, + "theoretical_loss": 3.7866241123133615, + "tokens_seen": 687079424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998796389167503, + "loss": 1.7465, + "theoretical_loss": 3.786550530454557, + "tokens_seen": 687210496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039985957873620866, + "loss": 1.19, + "theoretical_loss": 3.78647696655747, + "tokens_seen": 687341568 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399839518555667, + "loss": 1.4229, + "theoretical_loss": 3.7864034206142914, + "tokens_seen": 687472640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998194583751254, + "loss": 1.4904, + "theoretical_loss": 3.7863298926172186, + "tokens_seen": 687603712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039979939819458375, + "loss": 1.3871, + "theoretical_loss": 3.786256382558452, + "tokens_seen": 687734784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039977933801404217, + "loss": 1.4157, + "theoretical_loss": 3.786182890430198, + "tokens_seen": 687865856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039975927783350053, + "loss": 1.4489, + "theoretical_loss": 3.786109416224668, + "tokens_seen": 687996928 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.2985703945159912, + "objective/train/avg_token_score": 0.009044165723025799, + "objective/train/docs_used": 398552, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0640056133270264, + "objective/train/original_loss": 4.266354084014893, + "objective/train/theoretical_loss": 3.7860359599340776, + "objective/train/tokens_used": 688128000, + "objective/train/unlikelihood_loss": 0.11285004019737244, + "theoretical_loss": 3.7860359599340776, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997392176529589, + "loss": 1.358, + "theoretical_loss": 3.7860359599340776, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039971915747241725, + "loss": 1.5435, + "theoretical_loss": 3.7859625215506476, + "tokens_seen": 688259072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969909729187567, + "loss": 1.4167, + "theoretical_loss": 3.785889101066603, + "tokens_seen": 688390144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039967903711133403, + "loss": 1.3661, + "theoretical_loss": 3.7858156984741758, + "tokens_seen": 688521216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996589769307924, + "loss": 1.5609, + "theoretical_loss": 3.7857423137655997, + "tokens_seen": 688652288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039963891675025076, + "loss": 1.5022, + "theoretical_loss": 3.7856689469331153, + "tokens_seen": 688783360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996188565697091, + "loss": 1.4285, + "theoretical_loss": 3.7855955979689684, + "tokens_seen": 688914432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039959879638916754, + "loss": 1.3415, + "theoretical_loss": 3.7855222668654074, + "tokens_seen": 689045504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995787362086259, + "loss": 1.388, + "theoretical_loss": 3.7854489536146883, + "tokens_seen": 689176576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039955867602808426, + "loss": 1.5714, + "theoretical_loss": 3.7853756582090705, + "tokens_seen": 689307648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995386158475426, + "loss": 1.2824, + "theoretical_loss": 3.7853023806408173, + "tokens_seen": 689438720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039951855566700104, + "loss": 1.4164, + "theoretical_loss": 3.7852291209021987, + "tokens_seen": 689569792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994984954864594, + "loss": 1.4732, + "theoretical_loss": 3.7851558789854884, + "tokens_seen": 689700864 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.3938172161579132, + "objective/train/avg_token_score": 0.022718848660588264, + "objective/train/docs_used": 399765, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.335110068321228, + "objective/train/original_loss": 3.6663994789123535, + "objective/train/theoretical_loss": 3.785119264707936, + "objective/train/tokens_used": 689766400, + "objective/train/unlikelihood_loss": 0.1674300730228424, + "theoretical_loss": 3.785119264707936, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039947843530591776, + "loss": 1.4126, + "theoretical_loss": 3.7850826548829652, + "tokens_seen": 689831936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994583751253761, + "loss": 1.3082, + "theoretical_loss": 3.785009448586913, + "tokens_seen": 689963008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994383149448345, + "loss": 1.4161, + "theoretical_loss": 3.78493626008962, + "tokens_seen": 690094080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994182547642929, + "loss": 1.4203, + "theoretical_loss": 3.7848630893833786, + "tokens_seen": 690225152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039939819458375127, + "loss": 1.2872, + "theoretical_loss": 3.7847899364604887, + "tokens_seen": 690356224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039937813440320963, + "loss": 1.4593, + "theoretical_loss": 3.7847168013132513, + "tokens_seen": 690487296 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399358074222668, + "loss": 1.5105, + "theoretical_loss": 3.7846436839339743, + "tokens_seen": 690618368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993380140421264, + "loss": 1.3915, + "theoretical_loss": 3.7845705843149715, + "tokens_seen": 690749440 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039931795386158477, + "loss": 1.395, + "theoretical_loss": 3.7844975024485583, + "tokens_seen": 690880512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039929789368104313, + "loss": 1.4911, + "theoretical_loss": 3.7844244383270578, + "tokens_seen": 691011584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992778335005015, + "loss": 1.4866, + "theoretical_loss": 3.7843513919427965, + "tokens_seen": 691142656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039925777331995986, + "loss": 1.3637, + "theoretical_loss": 3.784278363288105, + "tokens_seen": 691273728 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.4256476163864136, + "objective/train/avg_token_score": 0.01342714298516512, + "objective/train/docs_used": 400468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4376263618469238, + "objective/train/original_loss": 3.8832194805145264, + "objective/train/theoretical_loss": 3.784205352355321, + "objective/train/tokens_used": 691404800, + "objective/train/unlikelihood_loss": 0.142247274518013, + "theoretical_loss": 3.784205352355321, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039923771313941827, + "loss": 1.357, + "theoretical_loss": 3.784205352355321, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039921765295887663, + "loss": 1.5518, + "theoretical_loss": 3.7841323591367844, + "tokens_seen": 691535872 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399197592778335, + "loss": 1.2302, + "theoretical_loss": 3.7840593836248413, + "tokens_seen": 691666944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917753259779336, + "loss": 1.4572, + "theoretical_loss": 3.7839864258118427, + "tokens_seen": 691798016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991574724172518, + "loss": 1.4045, + "theoretical_loss": 3.7839134856901437, + "tokens_seen": 691929088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913741223671014, + "loss": 1.5864, + "theoretical_loss": 3.7838405632521037, + "tokens_seen": 692060160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991173520561685, + "loss": 1.5805, + "theoretical_loss": 3.7837676584900883, + "tokens_seen": 692191232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039909729187562686, + "loss": 1.4142, + "theoretical_loss": 3.7836947713964673, + "tokens_seen": 692322304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990772316950853, + "loss": 1.3941, + "theoretical_loss": 3.783621901963614, + "tokens_seen": 692453376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039905717151454364, + "loss": 1.2217, + "theoretical_loss": 3.7835490501839075, + "tokens_seen": 692584448 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399037111334002, + "loss": 1.3851, + "theoretical_loss": 3.7834762160497326, + "tokens_seen": 692715520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901705115346037, + "loss": 1.5086, + "theoretical_loss": 3.7834033995534764, + "tokens_seen": 692846592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039899699097291873, + "loss": 1.458, + "theoretical_loss": 3.7833306006875334, + "tokens_seen": 692977664 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.36968475580215454, + "objective/train/avg_token_score": 0.02488008886575699, + "objective/train/docs_used": 401917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3004066944122314, + "objective/train/original_loss": 3.9216160774230957, + "objective/train/theoretical_loss": 3.7832942078635527, + "objective/train/tokens_used": 693043200, + "objective/train/unlikelihood_loss": 0.17222432792186737, + "theoretical_loss": 3.7832942078635527, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039897693079237714, + "loss": 1.3995, + "theoretical_loss": 3.7832578194443007, + "tokens_seen": 693108736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989568706118355, + "loss": 1.4124, + "theoretical_loss": 3.783185055816181, + "tokens_seen": 693239808 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039893681043129387, + "loss": 1.3657, + "theoretical_loss": 3.783112309795582, + "tokens_seen": 693370880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039891675025075223, + "loss": 1.4883, + "theoretical_loss": 3.783039581374915, + "tokens_seen": 693501952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039889669007021065, + "loss": 1.5706, + "theoretical_loss": 3.7829668705465975, + "tokens_seen": 693633024 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398876629889669, + "loss": 1.3513, + "theoretical_loss": 3.7828941773030507, + "tokens_seen": 693764096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039885656970912737, + "loss": 1.3129, + "theoretical_loss": 3.782821501636701, + "tokens_seen": 693895168 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039883650952858573, + "loss": 1.5587, + "theoretical_loss": 3.782748843539978, + "tokens_seen": 694026240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988164493480441, + "loss": 1.4957, + "theoretical_loss": 3.7826762030053196, + "tokens_seen": 694157312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987963891675025, + "loss": 1.342, + "theoretical_loss": 3.782603580025164, + "tokens_seen": 694288384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987763289869609, + "loss": 1.5353, + "theoretical_loss": 3.782530974591956, + "tokens_seen": 694419456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039875626880641924, + "loss": 1.3362, + "theoretical_loss": 3.7824583866981465, + "tokens_seen": 694550528 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.28189149498939514, + "objective/train/avg_token_score": 0.025410253554582596, + "objective/train/docs_used": 402587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9674853682518005, + "objective/train/original_loss": 4.519311428070068, + "objective/train/theoretical_loss": 3.782385816336189, + "objective/train/tokens_used": 694681600, + "objective/train/unlikelihood_loss": 0.11868737637996674, + "theoretical_loss": 3.782385816336189, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987362086258776, + "loss": 1.4546, + "theoretical_loss": 3.782385816336189, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398716148445336, + "loss": 1.2963, + "theoretical_loss": 3.7823132634985424, + "tokens_seen": 694812672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986960882647944, + "loss": 1.381, + "theoretical_loss": 3.78224072817767, + "tokens_seen": 694943744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039867602808425274, + "loss": 1.464, + "theoretical_loss": 3.782168210366041, + "tokens_seen": 695074816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986559679037111, + "loss": 1.4982, + "theoretical_loss": 3.7820957100561277, + "tokens_seen": 695205888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986359077231695, + "loss": 1.4137, + "theoretical_loss": 3.782023227240407, + "tokens_seen": 695336960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039861584754262794, + "loss": 1.427, + "theoretical_loss": 3.781950761911362, + "tokens_seen": 695468032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985957873620863, + "loss": 1.5499, + "theoretical_loss": 3.7818783140614793, + "tokens_seen": 695599104 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039857572718154466, + "loss": 1.4892, + "theoretical_loss": 3.7818058836832504, + "tokens_seen": 695730176 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398555667001003, + "loss": 1.4531, + "theoretical_loss": 3.781733470769171, + "tokens_seen": 695861248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039853560682046144, + "loss": 1.5128, + "theoretical_loss": 3.7816610753117432, + "tokens_seen": 695992320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985155466399198, + "loss": 1.3835, + "theoretical_loss": 3.7815886973034707, + "tokens_seen": 696123392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039849548645937816, + "loss": 1.4532, + "theoretical_loss": 3.781516336736864, + "tokens_seen": 696254464 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.37066227197647095, + "objective/train/avg_token_score": 0.009341365657746792, + "objective/train/docs_used": 403173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2960797548294067, + "objective/train/original_loss": 3.9722912311553955, + "objective/train/theoretical_loss": 3.781480162991847, + "objective/train/tokens_used": 696320000, + "objective/train/unlikelihood_loss": 0.13534922897815704, + "theoretical_loss": 3.781480162991847, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984754262788365, + "loss": 1.3637, + "theoretical_loss": 3.7814439936044386, + "tokens_seen": 696385536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984553660982949, + "loss": 1.1348, + "theoretical_loss": 3.7813716678987133, + "tokens_seen": 696516608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984353059177533, + "loss": 1.3888, + "theoretical_loss": 3.781299359612211, + "tokens_seen": 696647680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039841524573721167, + "loss": 1.2831, + "theoretical_loss": 3.7812270687374614, + "tokens_seen": 696778752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039839518555667003, + "loss": 1.3507, + "theoretical_loss": 3.7811547952669975, + "tokens_seen": 696909824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983751253761284, + "loss": 1.5016, + "theoretical_loss": 3.7810825391933562, + "tokens_seen": 697040896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983550651955868, + "loss": 1.337, + "theoretical_loss": 3.781010300509081, + "tokens_seen": 697171968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039833500501504517, + "loss": 1.4295, + "theoretical_loss": 3.780938079206718, + "tokens_seen": 697303040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039831494483450353, + "loss": 1.2459, + "theoretical_loss": 3.7808658752788187, + "tokens_seen": 697434112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982948846539619, + "loss": 1.4012, + "theoretical_loss": 3.7807936887179396, + "tokens_seen": 697565184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039827482447342026, + "loss": 1.4646, + "theoretical_loss": 3.780721519516641, + "tokens_seen": 697696256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039825476429287867, + "loss": 1.4062, + "theoretical_loss": 3.780649367667488, + "tokens_seen": 697827328 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.36247557401657104, + "objective/train/avg_token_score": 0.042056549340486526, + "objective/train/docs_used": 404489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2269220352172852, + "objective/train/original_loss": 4.341649055480957, + "objective/train/theoretical_loss": 3.7805772331630516, + "objective/train/tokens_used": 697958400, + "objective/train/unlikelihood_loss": 0.10854997485876083, + "theoretical_loss": 3.7805772331630516, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039823470411233703, + "loss": 1.5285, + "theoretical_loss": 3.7805772331630516, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982146439317954, + "loss": 1.4666, + "theoretical_loss": 3.780505115995904, + "tokens_seen": 698089472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039819458375125376, + "loss": 1.4054, + "theoretical_loss": 3.780433016158627, + "tokens_seen": 698220544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981745235707122, + "loss": 1.5778, + "theoretical_loss": 3.780360933643802, + "tokens_seen": 698351616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039815446339017054, + "loss": 1.3997, + "theoretical_loss": 3.780288868444018, + "tokens_seen": 698482688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981344032096289, + "loss": 1.2889, + "theoretical_loss": 3.7802168205518676, + "tokens_seen": 698613760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039811434302908726, + "loss": 1.395, + "theoretical_loss": 3.780144789959948, + "tokens_seen": 698744832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980942828485457, + "loss": 1.4899, + "theoretical_loss": 3.7800727766608615, + "tokens_seen": 698875904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039807422266800404, + "loss": 1.3067, + "theoretical_loss": 3.7800007806472133, + "tokens_seen": 699006976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980541624874624, + "loss": 1.2805, + "theoretical_loss": 3.7799288019116153, + "tokens_seen": 699138048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039803410230692077, + "loss": 1.3285, + "theoretical_loss": 3.7798568404466826, + "tokens_seen": 699269120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039801404212637913, + "loss": 1.5841, + "theoretical_loss": 3.779784896245035, + "tokens_seen": 699400192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039799398194583754, + "loss": 1.4457, + "theoretical_loss": 3.7797129692992977, + "tokens_seen": 699531264 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.37316715717315674, + "objective/train/avg_token_score": 0.009445698000490665, + "objective/train/docs_used": 405172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1694257259368896, + "objective/train/original_loss": 3.339930772781372, + "objective/train/theoretical_loss": 3.779677012295091, + "objective/train/tokens_used": 699596800, + "objective/train/unlikelihood_loss": 0.25555452704429626, + "theoretical_loss": 3.779677012295091, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979739217652959, + "loss": 1.4541, + "theoretical_loss": 3.7796410596020986, + "tokens_seen": 699662336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039795386158475427, + "loss": 1.5309, + "theoretical_loss": 3.7795691671460725, + "tokens_seen": 699793408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039793380140421263, + "loss": 1.4089, + "theoretical_loss": 3.7794972919238563, + "tokens_seen": 699924480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039791374122367105, + "loss": 1.3934, + "theoretical_loss": 3.7794254339280937, + "tokens_seen": 700055552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978936810431294, + "loss": 1.2738, + "theoretical_loss": 3.7793535931514315, + "tokens_seen": 700186624 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039787362086258777, + "loss": 1.38, + "theoretical_loss": 3.779281769586521, + "tokens_seen": 700317696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039785356068204613, + "loss": 1.5598, + "theoretical_loss": 3.7792099632260188, + "tokens_seen": 700448768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978335005015045, + "loss": 1.4078, + "theoretical_loss": 3.7791381740625853, + "tokens_seen": 700579840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978134403209629, + "loss": 1.4722, + "theoretical_loss": 3.779066402088886, + "tokens_seen": 700710912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977933801404213, + "loss": 1.2497, + "theoretical_loss": 3.7789946472975906, + "tokens_seen": 700841984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039777331995987964, + "loss": 1.5198, + "theoretical_loss": 3.7789229096813726, + "tokens_seen": 700973056 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397753259779338, + "loss": 1.485, + "theoretical_loss": 3.7788511892329115, + "tokens_seen": 701104128 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.43811094760894775, + "objective/train/avg_token_score": 0.02133597433567047, + "objective/train/docs_used": 406046, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5590468645095825, + "objective/train/original_loss": 4.188094615936279, + "objective/train/theoretical_loss": 3.7787794859448898, + "objective/train/tokens_used": 701235200, + "objective/train/unlikelihood_loss": 0.10562697052955627, + "theoretical_loss": 3.7787794859448898, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977331995987964, + "loss": 1.4971, + "theoretical_loss": 3.7787794859448898, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977131394182548, + "loss": 1.482, + "theoretical_loss": 3.7787077998099954, + "tokens_seen": 701366272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039769307923771314, + "loss": 1.5684, + "theoretical_loss": 3.77863613082092, + "tokens_seen": 701497344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976730190571715, + "loss": 1.3678, + "theoretical_loss": 3.7785644789703614, + "tokens_seen": 701628416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039765295887662987, + "loss": 1.6044, + "theoretical_loss": 3.7784928442510193, + "tokens_seen": 701759488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976328986960883, + "loss": 1.4995, + "theoretical_loss": 3.7784212266556008, + "tokens_seen": 701890560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039761283851554664, + "loss": 1.4306, + "theoretical_loss": 3.778349626176814, + "tokens_seen": 702021632 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397592778335005, + "loss": 1.4616, + "theoretical_loss": 3.778278042807375, + "tokens_seen": 702152704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039757271815446337, + "loss": 1.5438, + "theoretical_loss": 3.778206476540001, + "tokens_seen": 702283776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975526579739218, + "loss": 1.4624, + "theoretical_loss": 3.778134927367417, + "tokens_seen": 702414848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039753259779338015, + "loss": 1.4003, + "theoretical_loss": 3.7780633952823512, + "tokens_seen": 702545920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975125376128385, + "loss": 1.3533, + "theoretical_loss": 3.7779918802775345, + "tokens_seen": 702676992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039749247743229687, + "loss": 1.5902, + "theoretical_loss": 3.777920382345704, + "tokens_seen": 702808064 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.5230938196182251, + "objective/train/avg_token_score": 0.008132471702992916, + "objective/train/docs_used": 406800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7050210237503052, + "objective/train/original_loss": 3.6363258361816406, + "objective/train/theoretical_loss": 3.7778846397798898, + "objective/train/tokens_used": 702873600, + "objective/train/unlikelihood_loss": 0.1861809641122818, + "theoretical_loss": 3.7778846397798898, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039747241725175523, + "loss": 1.4757, + "theoretical_loss": 3.777848901479601, + "tokens_seen": 702939136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039745235707121365, + "loss": 1.3984, + "theoretical_loss": 3.777777437671971, + "tokens_seen": 703070208 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397432296890672, + "loss": 1.6991, + "theoretical_loss": 3.777705990915565, + "tokens_seen": 703201280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974122367101304, + "loss": 1.4025, + "theoretical_loss": 3.777634561203136, + "tokens_seen": 703332352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039739217652958874, + "loss": 1.4983, + "theoretical_loss": 3.7775631485274435, + "tokens_seen": 703463424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039737211634904715, + "loss": 1.4083, + "theoretical_loss": 3.7774917528812515, + "tokens_seen": 703594496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973520561685055, + "loss": 1.5957, + "theoretical_loss": 3.777420374257327, + "tokens_seen": 703725568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973319959879639, + "loss": 1.4341, + "theoretical_loss": 3.777349012648443, + "tokens_seen": 703856640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039731193580742224, + "loss": 1.3375, + "theoretical_loss": 3.7772776680473745, + "tokens_seen": 703987712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972918756268806, + "loss": 1.5625, + "theoretical_loss": 3.777206340446904, + "tokens_seen": 704118784 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397271815446339, + "loss": 1.499, + "theoretical_loss": 3.777135029839817, + "tokens_seen": 704249856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972517552657974, + "loss": 1.5525, + "theoretical_loss": 3.777063736218902, + "tokens_seen": 704380928 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.3191593289375305, + "objective/train/avg_token_score": 0.023736054077744484, + "objective/train/docs_used": 408057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.142675757408142, + "objective/train/original_loss": 4.020290851593018, + "objective/train/theoretical_loss": 3.7769924595769546, + "objective/train/tokens_used": 704512000, + "objective/train/unlikelihood_loss": 0.2041831910610199, + "theoretical_loss": 3.7769924595769546, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039723169508525574, + "loss": 1.4643, + "theoretical_loss": 3.7769924595769546, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972116349047141, + "loss": 1.4275, + "theoretical_loss": 3.7769211999067727, + "tokens_seen": 704643072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971915747241725, + "loss": 1.6245, + "theoretical_loss": 3.776849957201159, + "tokens_seen": 704774144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971715145436309, + "loss": 1.3021, + "theoretical_loss": 3.7767787314529215, + "tokens_seen": 704905216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039715145436308925, + "loss": 1.4288, + "theoretical_loss": 3.7767075226548723, + "tokens_seen": 705036288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039713139418254766, + "loss": 1.2985, + "theoretical_loss": 3.7766363307998265, + "tokens_seen": 705167360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971113340020061, + "loss": 1.5866, + "theoretical_loss": 3.7765651558806055, + "tokens_seen": 705298432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039709127382146444, + "loss": 1.4421, + "theoretical_loss": 3.776493997890034, + "tokens_seen": 705429504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970712136409228, + "loss": 1.542, + "theoretical_loss": 3.776422856820941, + "tokens_seen": 705560576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039705115346038117, + "loss": 1.6065, + "theoretical_loss": 3.7763517326661606, + "tokens_seen": 705691648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039703109327983953, + "loss": 1.3834, + "theoretical_loss": 3.776280625418531, + "tokens_seen": 705822720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039701103309929794, + "loss": 1.5074, + "theoretical_loss": 3.7762095350708935, + "tokens_seen": 705953792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969909729187563, + "loss": 1.4618, + "theoretical_loss": 3.776138461616096, + "tokens_seen": 706084864 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.4511852264404297, + "objective/train/avg_token_score": 0.03026348166167736, + "objective/train/docs_used": 408602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5793392658233643, + "objective/train/original_loss": 4.025704860687256, + "objective/train/theoretical_loss": 3.7761029312212777, + "objective/train/tokens_used": 706150400, + "objective/train/unlikelihood_loss": 0.13033060729503632, + "theoretical_loss": 3.7761029312212777, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039697091273821467, + "loss": 1.5814, + "theoretical_loss": 3.776067405046989, + "tokens_seen": 706215936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039695085255767303, + "loss": 1.3611, + "theoretical_loss": 3.775996365356428, + "tokens_seen": 706347008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039693079237713145, + "loss": 1.374, + "theoretical_loss": 3.7759253425372736, + "tokens_seen": 706478080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969107321965898, + "loss": 1.479, + "theoretical_loss": 3.7758543365823884, + "tokens_seen": 706609152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039689067201604817, + "loss": 1.6716, + "theoretical_loss": 3.7757833474846425, + "tokens_seen": 706740224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039687061183550653, + "loss": 1.4328, + "theoretical_loss": 3.7757123752369077, + "tokens_seen": 706871296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968505516549649, + "loss": 1.3239, + "theoretical_loss": 3.775641419832062, + "tokens_seen": 707002368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968304914744233, + "loss": 1.4179, + "theoretical_loss": 3.7755704812629856, + "tokens_seen": 707133440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968104312938817, + "loss": 1.4862, + "theoretical_loss": 3.7754995595225656, + "tokens_seen": 707264512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039679037111334004, + "loss": 1.4879, + "theoretical_loss": 3.775428654603691, + "tokens_seen": 707395584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967703109327984, + "loss": 1.2324, + "theoretical_loss": 3.7753577664992575, + "tokens_seen": 707526656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967502507522568, + "loss": 1.3985, + "theoretical_loss": 3.7752868952021634, + "tokens_seen": 707657728 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.2740102708339691, + "objective/train/avg_token_score": 0.023776978254318237, + "objective/train/docs_used": 409727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0112217664718628, + "objective/train/original_loss": 4.027196884155273, + "objective/train/theoretical_loss": 3.7752160407053115, + "objective/train/tokens_used": 707788800, + "objective/train/unlikelihood_loss": 0.15683040022850037, + "theoretical_loss": 3.7752160407053115, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967301905717152, + "loss": 1.4565, + "theoretical_loss": 3.7752160407053115, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039671013039117354, + "loss": 1.5837, + "theoretical_loss": 3.775145203001609, + "tokens_seen": 707919872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966900702106319, + "loss": 1.4068, + "theoretical_loss": 3.775074382083968, + "tokens_seen": 708050944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039667001003009027, + "loss": 1.5325, + "theoretical_loss": 3.775003577945305, + "tokens_seen": 708182016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966499498495487, + "loss": 1.4816, + "theoretical_loss": 3.7749327905785393, + "tokens_seen": 708313088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039662988966900704, + "loss": 1.4361, + "theoretical_loss": 3.774862019976596, + "tokens_seen": 708444160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966098294884654, + "loss": 1.4435, + "theoretical_loss": 3.7747912661324046, + "tokens_seen": 708575232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039658976930792377, + "loss": 1.4666, + "theoretical_loss": 3.7747205290388974, + "tokens_seen": 708706304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965697091273822, + "loss": 1.433, + "theoretical_loss": 3.774649808689013, + "tokens_seen": 708837376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039654964894684055, + "loss": 1.3161, + "theoretical_loss": 3.7745791050756914, + "tokens_seen": 708968448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965295887662989, + "loss": 1.3019, + "theoretical_loss": 3.7745084181918793, + "tokens_seen": 709099520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039650952858575727, + "loss": 1.3835, + "theoretical_loss": 3.774437748030528, + "tokens_seen": 709230592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039648946840521563, + "loss": 1.5765, + "theoretical_loss": 3.7743670945845915, + "tokens_seen": 709361664 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.4159335196018219, + "objective/train/avg_token_score": 0.007403578143566847, + "objective/train/docs_used": 410190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4387859106063843, + "objective/train/original_loss": 4.027052402496338, + "objective/train/theoretical_loss": 3.774331774127703, + "objective/train/tokens_used": 709427200, + "objective/train/unlikelihood_loss": 0.12076714634895325, + "theoretical_loss": 3.774331774127703, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039646940822467405, + "loss": 1.5127, + "theoretical_loss": 3.7742964578470284, + "tokens_seen": 709492736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964493480441324, + "loss": 1.3593, + "theoretical_loss": 3.7742258378108016, + "tokens_seen": 709623808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964292878635908, + "loss": 1.5113, + "theoretical_loss": 3.7741552344688793, + "tokens_seen": 709754880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039640922768304914, + "loss": 1.3705, + "theoretical_loss": 3.7740846478142327, + "tokens_seen": 709885952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039638916750250755, + "loss": 1.5118, + "theoretical_loss": 3.774014077839838, + "tokens_seen": 710017024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963691073219659, + "loss": 1.4813, + "theoretical_loss": 3.773943524538675, + "tokens_seen": 710148096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963490471414243, + "loss": 1.3631, + "theoretical_loss": 3.7738729879037285, + "tokens_seen": 710279168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039632898696088264, + "loss": 1.3963, + "theoretical_loss": 3.773802467927986, + "tokens_seen": 710410240 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396308926780341, + "loss": 1.3957, + "theoretical_loss": 3.7737319646044423, + "tokens_seen": 710541312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962888665997994, + "loss": 1.566, + "theoretical_loss": 3.773661477926093, + "tokens_seen": 710672384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962688064192578, + "loss": 1.2884, + "theoretical_loss": 3.7735910078859396, + "tokens_seen": 710803456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039624874623871614, + "loss": 1.4653, + "theoretical_loss": 3.773520554476989, + "tokens_seen": 710934528 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.43316227197647095, + "objective/train/avg_token_score": 0.007836335338652134, + "objective/train/docs_used": 411414, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5592094659805298, + "objective/train/original_loss": 3.8661837577819824, + "objective/train/theoretical_loss": 3.7734501176922493, + "objective/train/tokens_used": 711065600, + "objective/train/unlikelihood_loss": 0.17112526297569275, + "theoretical_loss": 3.7734501176922493, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962286860581745, + "loss": 1.4909, + "theoretical_loss": 3.7734501176922493, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962086258776329, + "loss": 1.44, + "theoretical_loss": 3.773379697524736, + "tokens_seen": 711196672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961885656970913, + "loss": 1.5634, + "theoretical_loss": 3.7733092939674666, + "tokens_seen": 711327744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039616850551654965, + "loss": 1.3417, + "theoretical_loss": 3.7732389070134635, + "tokens_seen": 711458816 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396148445336008, + "loss": 1.1947, + "theoretical_loss": 3.7731685366557537, + "tokens_seen": 711589888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039612838515546637, + "loss": 1.4477, + "theoretical_loss": 3.7730981828873684, + "tokens_seen": 711720960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961083249749248, + "loss": 1.4099, + "theoretical_loss": 3.773027845701342, + "tokens_seen": 711852032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039608826479438315, + "loss": 1.4776, + "theoretical_loss": 3.772957525090715, + "tokens_seen": 711983104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960682046138415, + "loss": 1.4901, + "theoretical_loss": 3.7728872210485296, + "tokens_seen": 712114176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960481444332999, + "loss": 1.5321, + "theoretical_loss": 3.7728169335678343, + "tokens_seen": 712245248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960280842527583, + "loss": 1.4535, + "theoretical_loss": 3.772746662641681, + "tokens_seen": 712376320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039600802407221665, + "loss": 1.2382, + "theoretical_loss": 3.772676408263126, + "tokens_seen": 712507392 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395987963891675, + "loss": 1.5707, + "theoretical_loss": 3.772606170425229, + "tokens_seen": 712638464 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.4084188640117645, + "objective/train/avg_token_score": 0.013639979995787144, + "objective/train/docs_used": 412148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4703177213668823, + "objective/train/original_loss": 4.104379653930664, + "objective/train/theoretical_loss": 3.77257105770686, + "objective/train/tokens_used": 712704000, + "objective/train/unlikelihood_loss": 0.14685511589050293, + "theoretical_loss": 3.77257105770686, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959679037111334, + "loss": 1.3501, + "theoretical_loss": 3.772535949121055, + "tokens_seen": 712769536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959478435305918, + "loss": 1.6008, + "theoretical_loss": 3.7724657443436724, + "tokens_seen": 712900608 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039592778335005016, + "loss": 1.3244, + "theoretical_loss": 3.7723955560861544, + "tokens_seen": 713031680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959077231695085, + "loss": 1.4234, + "theoretical_loss": 3.7723253843415785, + "tokens_seen": 713162752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958876629889669, + "loss": 1.4388, + "theoretical_loss": 3.772255229103025, + "tokens_seen": 713293824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039586760280842524, + "loss": 1.3096, + "theoretical_loss": 3.7721850903635787, + "tokens_seen": 713424896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039584754262788366, + "loss": 1.4692, + "theoretical_loss": 3.772114968116331, + "tokens_seen": 713555968 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395827482447342, + "loss": 1.3513, + "theoretical_loss": 3.7720448623543743, + "tokens_seen": 713687040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958074222668004, + "loss": 1.485, + "theoretical_loss": 3.771974773070807, + "tokens_seen": 713818112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039578736208625875, + "loss": 1.5266, + "theoretical_loss": 3.771904700258731, + "tokens_seen": 713949184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039576730190571716, + "loss": 1.3047, + "theoretical_loss": 3.7718346439112524, + "tokens_seen": 714080256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957472417251755, + "loss": 1.3666, + "theoretical_loss": 3.771764604021482, + "tokens_seen": 714211328 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.3321114480495453, + "objective/train/avg_token_score": 0.025234047323465347, + "objective/train/docs_used": 413348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0405142307281494, + "objective/train/original_loss": 3.703433036804199, + "objective/train/theoretical_loss": 3.7716945805825337, + "objective/train/tokens_used": 714342400, + "objective/train/unlikelihood_loss": 0.25580325722694397, + "theoretical_loss": 3.7716945805825337, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957271815446339, + "loss": 1.5155, + "theoretical_loss": 3.7716945805825337, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039570712136409225, + "loss": 1.4421, + "theoretical_loss": 3.771624573587526, + "tokens_seen": 714473472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956870611835506, + "loss": 1.484, + "theoretical_loss": 3.7715545830295825, + "tokens_seen": 714604544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039566700100300903, + "loss": 1.3397, + "theoretical_loss": 3.7714846089018295, + "tokens_seen": 714735616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956469408224674, + "loss": 1.3847, + "theoretical_loss": 3.7714146511973983, + "tokens_seen": 714866688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956268806419258, + "loss": 1.5151, + "theoretical_loss": 3.7713447099094233, + "tokens_seen": 714997760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039560682046138417, + "loss": 1.5149, + "theoretical_loss": 3.7712747850310446, + "tokens_seen": 715128832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955867602808426, + "loss": 1.5224, + "theoretical_loss": 3.771204876555406, + "tokens_seen": 715259904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039556670010030095, + "loss": 1.4265, + "theoretical_loss": 3.7711349844756543, + "tokens_seen": 715390976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955466399197593, + "loss": 1.4006, + "theoretical_loss": 3.771065108784941, + "tokens_seen": 715522048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039552657973921767, + "loss": 1.3549, + "theoretical_loss": 3.7709952494764227, + "tokens_seen": 715653120 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039550651955867603, + "loss": 1.5052, + "theoretical_loss": 3.7709254065432583, + "tokens_seen": 715784192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039548645937813445, + "loss": 1.3445, + "theoretical_loss": 3.7708555799786128, + "tokens_seen": 715915264 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.43542277812957764, + "objective/train/avg_token_score": 0.008922696113586426, + "objective/train/docs_used": 413910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.389076590538025, + "objective/train/original_loss": 3.6932613849639893, + "objective/train/theoretical_loss": 3.7708206728323495, + "objective/train/tokens_used": 715980800, + "objective/train/unlikelihood_loss": 0.18138150870800018, + "theoretical_loss": 3.7708206728323495, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954663991975928, + "loss": 1.5682, + "theoretical_loss": 3.770785769775654, + "tokens_seen": 716046336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954463390170512, + "loss": 1.4396, + "theoretical_loss": 3.770715975927553, + "tokens_seen": 716177408 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039542627883650954, + "loss": 1.3733, + "theoretical_loss": 3.7706461984274875, + "tokens_seen": 716308480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039540621865596795, + "loss": 1.3308, + "theoretical_loss": 3.7705764372686374, + "tokens_seen": 716439552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953861584754263, + "loss": 1.3821, + "theoretical_loss": 3.770506692444187, + "tokens_seen": 716570624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953660982948847, + "loss": 1.5726, + "theoretical_loss": 3.770436963947325, + "tokens_seen": 716701696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039534603811434304, + "loss": 1.4631, + "theoretical_loss": 3.770367251771245, + "tokens_seen": 716832768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953259779338014, + "loss": 1.4369, + "theoretical_loss": 3.7702975559091416, + "tokens_seen": 716963840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953059177532598, + "loss": 1.3834, + "theoretical_loss": 3.7702278763542174, + "tokens_seen": 717094912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952858575727182, + "loss": 1.3416, + "theoretical_loss": 3.770158213099677, + "tokens_seen": 717225984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039526579739217654, + "loss": 1.5406, + "theoretical_loss": 3.7700885661387287, + "tokens_seen": 717357056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952457372116349, + "loss": 1.4358, + "theoretical_loss": 3.770018935464586, + "tokens_seen": 717488128 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.4139784872531891, + "objective/train/avg_token_score": 0.013032732531428337, + "objective/train/docs_used": 414484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2717984914779663, + "objective/train/original_loss": 3.776548147201538, + "objective/train/theoretical_loss": 3.7699493210704667, + "objective/train/tokens_used": 717619200, + "objective/train/unlikelihood_loss": 0.15414294600486755, + "theoretical_loss": 3.7699493210704667, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952256770310933, + "loss": 1.4229, + "theoretical_loss": 3.7699493210704667, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952056168505517, + "loss": 1.3912, + "theoretical_loss": 3.7698797229495904, + "tokens_seen": 717750272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039518555667001005, + "loss": 1.3891, + "theoretical_loss": 3.769810141095183, + "tokens_seen": 717881344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951654964894684, + "loss": 1.4815, + "theoretical_loss": 3.769740575500475, + "tokens_seen": 718012416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039514543630892677, + "loss": 1.551, + "theoretical_loss": 3.769671026158698, + "tokens_seen": 718143488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951253761283852, + "loss": 1.2395, + "theoretical_loss": 3.76960149306309, + "tokens_seen": 718274560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039510531594784355, + "loss": 1.3502, + "theoretical_loss": 3.769531976206893, + "tokens_seen": 718405632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950852557673019, + "loss": 1.3703, + "theoretical_loss": 3.7694624755833512, + "tokens_seen": 718536704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950651955867603, + "loss": 1.3038, + "theoretical_loss": 3.7693929911857156, + "tokens_seen": 718667776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950451354062187, + "loss": 1.3797, + "theoretical_loss": 3.7693235230072384, + "tokens_seen": 718798848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039502507522567705, + "loss": 1.4026, + "theoretical_loss": 3.7692540710411784, + "tokens_seen": 718929920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950050150451354, + "loss": 1.4331, + "theoretical_loss": 3.769184635280797, + "tokens_seen": 719060992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949849548645938, + "loss": 1.5568, + "theoretical_loss": 3.7691152157193586, + "tokens_seen": 719192064 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.2873900234699249, + "objective/train/avg_token_score": 0.03997653350234032, + "objective/train/docs_used": 415542, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.038185715675354, + "objective/train/original_loss": 4.579502582550049, + "objective/train/theoretical_loss": 3.76908051201114, + "objective/train/tokens_used": 719257600, + "objective/train/unlikelihood_loss": 0.1975022554397583, + "theoretical_loss": 3.76908051201114, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949648946840522, + "loss": 1.1941, + "theoretical_loss": 3.769045812350134, + "tokens_seen": 719323136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039494483450351056, + "loss": 1.4565, + "theoretical_loss": 3.768976425166397, + "tokens_seen": 719454208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949247743229689, + "loss": 1.3448, + "theoretical_loss": 3.768907054161425, + "tokens_seen": 719585280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949047141424273, + "loss": 1.5484, + "theoretical_loss": 3.7688376993284995, + "tokens_seen": 719716352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039488465396188564, + "loss": 1.3142, + "theoretical_loss": 3.768768360660907, + "tokens_seen": 719847424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039486459378134406, + "loss": 1.1802, + "theoretical_loss": 3.7686990381519365, + "tokens_seen": 719978496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948445336008024, + "loss": 1.3436, + "theoretical_loss": 3.768629731794882, + "tokens_seen": 720109568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948244734202608, + "loss": 1.4103, + "theoretical_loss": 3.768560441583041, + "tokens_seen": 720240640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039480441323971915, + "loss": 1.3625, + "theoretical_loss": 3.768491167509716, + "tokens_seen": 720371712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039478435305917756, + "loss": 1.4679, + "theoretical_loss": 3.768421909568212, + "tokens_seen": 720502784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947642928786359, + "loss": 1.4789, + "theoretical_loss": 3.768352667751839, + "tokens_seen": 720633856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947442326980943, + "loss": 1.3368, + "theoretical_loss": 3.768283442053911, + "tokens_seen": 720764928 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.4169110357761383, + "objective/train/avg_token_score": 0.02430744841694832, + "objective/train/docs_used": 416165, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5107887983322144, + "objective/train/original_loss": 4.104403018951416, + "objective/train/theoretical_loss": 3.7682142324677455, + "objective/train/tokens_used": 720896000, + "objective/train/unlikelihood_loss": 0.14461186528205872, + "theoretical_loss": 3.7682142324677455, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039472417251755265, + "loss": 1.3675, + "theoretical_loss": 3.7682142324677455, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394704112337011, + "loss": 1.3197, + "theoretical_loss": 3.768145038986664, + "tokens_seen": 721027072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039468405215646943, + "loss": 1.4227, + "theoretical_loss": 3.768075861603993, + "tokens_seen": 721158144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946639919759278, + "loss": 1.4297, + "theoretical_loss": 3.7680067003130615, + "tokens_seen": 721289216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039464393179538615, + "loss": 1.3914, + "theoretical_loss": 3.767937555107203, + "tokens_seen": 721420288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946238716148445, + "loss": 1.5007, + "theoretical_loss": 3.7678684259797555, + "tokens_seen": 721551360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039460381143430293, + "loss": 1.278, + "theoretical_loss": 3.767799312924061, + "tokens_seen": 721682432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945837512537613, + "loss": 1.3656, + "theoretical_loss": 3.7677302159334642, + "tokens_seen": 721813504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039456369107321966, + "loss": 1.3838, + "theoretical_loss": 3.7676611350013145, + "tokens_seen": 721944576 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394543630892678, + "loss": 1.265, + "theoretical_loss": 3.7675920701209664, + "tokens_seen": 722075648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945235707121364, + "loss": 1.3656, + "theoretical_loss": 3.767523021285777, + "tokens_seen": 722206720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945035105315948, + "loss": 1.4427, + "theoretical_loss": 3.767453988489107, + "tokens_seen": 722337792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039448345035105316, + "loss": 1.5385, + "theoretical_loss": 3.7673849717243226, + "tokens_seen": 722468864 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.3990102708339691, + "objective/train/avg_token_score": 0.013219342567026615, + "objective/train/docs_used": 417606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.416273832321167, + "objective/train/original_loss": 3.709404468536377, + "objective/train/theoretical_loss": 3.7673504693518156, + "objective/train/tokens_used": 722534400, + "objective/train/unlikelihood_loss": 0.1947861611843109, + "theoretical_loss": 3.7673504693518156, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944633901705115, + "loss": 1.3598, + "theoretical_loss": 3.767315970984793, + "tokens_seen": 722599936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944433299899699, + "loss": 1.348, + "theoretical_loss": 3.767246986263891, + "tokens_seen": 722731008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944232698094283, + "loss": 1.3321, + "theoretical_loss": 3.767178017554994, + "tokens_seen": 722862080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039440320962888666, + "loss": 1.3725, + "theoretical_loss": 3.767109064851483, + "tokens_seen": 722993152 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394383149448345, + "loss": 1.4208, + "theoretical_loss": 3.7670401281467427, + "tokens_seen": 723124224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943630892678034, + "loss": 1.5043, + "theoretical_loss": 3.766971207434163, + "tokens_seen": 723255296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039434302908726175, + "loss": 1.3117, + "theoretical_loss": 3.7669023027071367, + "tokens_seen": 723386368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039432296890672017, + "loss": 1.5145, + "theoretical_loss": 3.76683341395906, + "tokens_seen": 723517440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039430290872617853, + "loss": 1.386, + "theoretical_loss": 3.766764541183334, + "tokens_seen": 723648512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942828485456369, + "loss": 1.458, + "theoretical_loss": 3.766695684373363, + "tokens_seen": 723779584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039426278836509525, + "loss": 1.6662, + "theoretical_loss": 3.7666268435225563, + "tokens_seen": 723910656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039424272818455367, + "loss": 1.3722, + "theoretical_loss": 3.766558018624326, + "tokens_seen": 724041728 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.4300464391708374, + "objective/train/avg_token_score": 0.014578673988580704, + "objective/train/docs_used": 418215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6266533136367798, + "objective/train/original_loss": 4.25946569442749, + "objective/train/theoretical_loss": 3.7664892096720886, + "objective/train/tokens_used": 724172800, + "objective/train/unlikelihood_loss": 0.12976768612861633, + "theoretical_loss": 3.7664892096720886, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039422266800401203, + "loss": 1.5332, + "theoretical_loss": 3.7664892096720886, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942026078234704, + "loss": 1.4557, + "theoretical_loss": 3.7664204166592645, + "tokens_seen": 724303872 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039418254764292876, + "loss": 1.314, + "theoretical_loss": 3.7663516395792778, + "tokens_seen": 724434944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941624874623871, + "loss": 1.3049, + "theoretical_loss": 3.7662828784255566, + "tokens_seen": 724566016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941424272818456, + "loss": 1.395, + "theoretical_loss": 3.766214133191533, + "tokens_seen": 724697088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039412236710130395, + "loss": 1.4503, + "theoretical_loss": 3.766145403870643, + "tokens_seen": 724828160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941023069207623, + "loss": 1.2929, + "theoretical_loss": 3.7660766904563268, + "tokens_seen": 724959232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940822467402207, + "loss": 1.3205, + "theoretical_loss": 3.7660079929420274, + "tokens_seen": 725090304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940621865596791, + "loss": 1.4272, + "theoretical_loss": 3.7659393113211923, + "tokens_seen": 725221376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039404212637913745, + "loss": 1.3942, + "theoretical_loss": 3.765870645587274, + "tokens_seen": 725352448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940220661985958, + "loss": 1.4624, + "theoretical_loss": 3.765801995733727, + "tokens_seen": 725483520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940020060180542, + "loss": 1.4069, + "theoretical_loss": 3.7657333617540107, + "tokens_seen": 725614592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939819458375126, + "loss": 1.4183, + "theoretical_loss": 3.765664743641589, + "tokens_seen": 725745664 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.38526391983032227, + "objective/train/avg_token_score": 0.010517455637454987, + "objective/train/docs_used": 419550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3706176280975342, + "objective/train/original_loss": 3.945437431335449, + "objective/train/theoretical_loss": 3.765630440533571, + "objective/train/tokens_used": 725811200, + "objective/train/unlikelihood_loss": 0.14202526211738586, + "theoretical_loss": 3.765630440533571, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039396188565697096, + "loss": 1.396, + "theoretical_loss": 3.7655961413899277, + "tokens_seen": 725876736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939418254764293, + "loss": 1.3026, + "theoretical_loss": 3.7655275549924987, + "tokens_seen": 726007808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939217652958877, + "loss": 1.3096, + "theoretical_loss": 3.765458984442776, + "tokens_seen": 726138880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039390170511534604, + "loss": 1.3314, + "theoretical_loss": 3.765390429734238, + "tokens_seen": 726269952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039388164493480446, + "loss": 1.4553, + "theoretical_loss": 3.765321890860368, + "tokens_seen": 726401024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938615847542628, + "loss": 1.4392, + "theoretical_loss": 3.765253367814652, + "tokens_seen": 726532096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938415245737212, + "loss": 1.3652, + "theoretical_loss": 3.7651848605905798, + "tokens_seen": 726663168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039382146439317955, + "loss": 1.4962, + "theoretical_loss": 3.765116369181646, + "tokens_seen": 726794240 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039380140421263796, + "loss": 1.346, + "theoretical_loss": 3.7650478935813476, + "tokens_seen": 726925312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937813440320963, + "loss": 1.3955, + "theoretical_loss": 3.7649794337831874, + "tokens_seen": 727056384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937612838515547, + "loss": 1.3725, + "theoretical_loss": 3.7649109897806703, + "tokens_seen": 727187456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039374122367101305, + "loss": 1.456, + "theoretical_loss": 3.7648425615673053, + "tokens_seen": 727318528 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.5380620956420898, + "objective/train/avg_token_score": 0.015358886681497097, + "objective/train/docs_used": 420047, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8991224765777588, + "objective/train/original_loss": 3.960615396499634, + "objective/train/theoretical_loss": 3.7647741491366067, + "objective/train/tokens_used": 727449600, + "objective/train/unlikelihood_loss": 0.1288902461528778, + "theoretical_loss": 3.7647741491366067, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937211634904714, + "loss": 1.4423, + "theoretical_loss": 3.7647741491366067, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039370110330992983, + "loss": 1.5228, + "theoretical_loss": 3.764705752482091, + "tokens_seen": 727580672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936810431293882, + "loss": 1.3713, + "theoretical_loss": 3.7646373715972787, + "tokens_seen": 727711744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039366098294884655, + "loss": 1.2685, + "theoretical_loss": 3.764569006475695, + "tokens_seen": 727842816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936409227683049, + "loss": 1.3923, + "theoretical_loss": 3.7645006571108683, + "tokens_seen": 727973888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039362086258776333, + "loss": 1.4292, + "theoretical_loss": 3.764432323496331, + "tokens_seen": 728104960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936008024072217, + "loss": 1.505, + "theoretical_loss": 3.764364005625619, + "tokens_seen": 728236032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039358074222668006, + "loss": 1.3654, + "theoretical_loss": 3.764295703492273, + "tokens_seen": 728367104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935606820461384, + "loss": 1.539, + "theoretical_loss": 3.764227417089836, + "tokens_seen": 728498176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935406218655968, + "loss": 1.3583, + "theoretical_loss": 3.7641591464118562, + "tokens_seen": 728629248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935205616850552, + "loss": 1.333, + "theoretical_loss": 3.7640908914518847, + "tokens_seen": 728760320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039350050150451356, + "loss": 1.3693, + "theoretical_loss": 3.7640226522034768, + "tokens_seen": 728891392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934804413239719, + "loss": 1.4396, + "theoretical_loss": 3.763954428660192, + "tokens_seen": 729022464 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.5046432018280029, + "objective/train/avg_token_score": 0.008104084059596062, + "objective/train/docs_used": 421294, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7709553241729736, + "objective/train/original_loss": 3.8750438690185547, + "objective/train/theoretical_loss": 3.7639203227759586, + "objective/train/tokens_used": 729088000, + "objective/train/unlikelihood_loss": 0.1357206553220749, + "theoretical_loss": 3.7639203227759586, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934603811434303, + "loss": 1.3609, + "theoretical_loss": 3.763886220815592, + "tokens_seen": 729153536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934403209628887, + "loss": 1.3919, + "theoretical_loss": 3.7638180286632443, + "tokens_seen": 729284608 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039342026078234706, + "loss": 1.3994, + "theoretical_loss": 3.763749852196719, + "tokens_seen": 729415680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934002006018054, + "loss": 1.424, + "theoretical_loss": 3.763681691409591, + "tokens_seen": 729546752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933801404212638, + "loss": 1.5318, + "theoretical_loss": 3.7636135462954368, + "tokens_seen": 729677824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039336008024072215, + "loss": 1.2981, + "theoretical_loss": 3.76354541684784, + "tokens_seen": 729808896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039334002006018057, + "loss": 1.5175, + "theoretical_loss": 3.7634773030603847, + "tokens_seen": 729939968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039331995987963893, + "loss": 1.4275, + "theoretical_loss": 3.7634092049266608, + "tokens_seen": 730071040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932998996990973, + "loss": 1.5105, + "theoretical_loss": 3.7633411224402615, + "tokens_seen": 730202112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039327983951855565, + "loss": 1.4328, + "theoretical_loss": 3.7632730555947833, + "tokens_seen": 730333184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039325977933801407, + "loss": 1.3948, + "theoretical_loss": 3.763205004383827, + "tokens_seen": 730464256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039323971915747243, + "loss": 1.4311, + "theoretical_loss": 3.763136968800998, + "tokens_seen": 730595328 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.49615103006362915, + "objective/train/avg_token_score": 0.018022378906607628, + "objective/train/docs_used": 422072, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8124078512191772, + "objective/train/original_loss": 3.9566221237182617, + "objective/train/theoretical_loss": 3.7630689488399027, + "objective/train/tokens_used": 730726400, + "objective/train/unlikelihood_loss": 0.2253096103668213, + "theoretical_loss": 3.7630689488399027, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932196589769308, + "loss": 1.2057, + "theoretical_loss": 3.7630689488399027, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039319959879638916, + "loss": 1.4383, + "theoretical_loss": 3.763000944494154, + "tokens_seen": 730857472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931795386158475, + "loss": 1.4071, + "theoretical_loss": 3.7629329557573676, + "tokens_seen": 730988544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039315947843530593, + "loss": 1.3743, + "theoretical_loss": 3.762864982623163, + "tokens_seen": 731119616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931394182547643, + "loss": 1.4819, + "theoretical_loss": 3.7627970250851632, + "tokens_seen": 731250688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311935807422266, + "loss": 1.5305, + "theoretical_loss": 3.7627290831369953, + "tokens_seen": 731381760 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393099297893681, + "loss": 1.379, + "theoretical_loss": 3.7626611567722894, + "tokens_seen": 731512832 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039307923771313944, + "loss": 1.3891, + "theoretical_loss": 3.7625932459846805, + "tokens_seen": 731643904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930591775325978, + "loss": 1.4535, + "theoretical_loss": 3.762525350767807, + "tokens_seen": 731774976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039303911735205616, + "loss": 1.4437, + "theoretical_loss": 3.7624574711153107, + "tokens_seen": 731906048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930190571715145, + "loss": 1.3948, + "theoretical_loss": 3.7623896070208365, + "tokens_seen": 732037120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929989969909729, + "loss": 1.4662, + "theoretical_loss": 3.762321758478034, + "tokens_seen": 732168192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929789368104313, + "loss": 1.5163, + "theoretical_loss": 3.7622539254805574, + "tokens_seen": 732299264 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.5039711594581604, + "objective/train/avg_token_score": 0.012472734786570072, + "objective/train/docs_used": 423458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7452107667922974, + "objective/train/original_loss": 3.785001754760742, + "objective/train/theoretical_loss": 3.7622200148093334, + "objective/train/tokens_used": 732364800, + "objective/train/unlikelihood_loss": 0.133365660905838, + "theoretical_loss": 3.7622200148093334, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039295887662988967, + "loss": 1.4857, + "theoretical_loss": 3.7621861080220627, + "tokens_seen": 732430336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039293881644934803, + "loss": 1.3086, + "theoretical_loss": 3.7621183060962102, + "tokens_seen": 732561408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929187562688064, + "loss": 1.3428, + "theoretical_loss": 3.7620505196966647, + "tokens_seen": 732692480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928986960882648, + "loss": 1.4514, + "theoretical_loss": 3.761982748817094, + "tokens_seen": 732823552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039287863590772317, + "loss": 1.5364, + "theoretical_loss": 3.7619149934511693, + "tokens_seen": 732954624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039285857572718153, + "loss": 1.4475, + "theoretical_loss": 3.761847253592567, + "tokens_seen": 733085696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928385155466399, + "loss": 1.291, + "theoretical_loss": 3.761779529234966, + "tokens_seen": 733216768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039281845536609825, + "loss": 1.3587, + "theoretical_loss": 3.7617118203720485, + "tokens_seen": 733347840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039279839518555667, + "loss": 1.3847, + "theoretical_loss": 3.761644126997501, + "tokens_seen": 733478912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039277833500501503, + "loss": 1.4809, + "theoretical_loss": 3.7615764491050143, + "tokens_seen": 733609984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927582748244734, + "loss": 1.4966, + "theoretical_loss": 3.7615087866882826, + "tokens_seen": 733741056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039273821464393176, + "loss": 1.3114, + "theoretical_loss": 3.761441139741003, + "tokens_seen": 733872128 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.35105082392692566, + "objective/train/avg_token_score": 0.029971016570925713, + "objective/train/docs_used": 424116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.268815279006958, + "objective/train/original_loss": 4.389547348022461, + "objective/train/theoretical_loss": 3.7613735082568764, + "objective/train/tokens_used": 734003200, + "objective/train/unlikelihood_loss": 0.138614684343338, + "theoretical_loss": 3.7613735082568764, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927181544633902, + "loss": 1.2742, + "theoretical_loss": 3.7613735082568764, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039269809428284854, + "loss": 1.3775, + "theoretical_loss": 3.761305892229609, + "tokens_seen": 734134272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926780341023069, + "loss": 1.2466, + "theoretical_loss": 3.7612382916529086, + "tokens_seen": 734265344 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039265797392176526, + "loss": 1.4391, + "theoretical_loss": 3.7611707065204874, + "tokens_seen": 734396416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039263791374122373, + "loss": 1.404, + "theoretical_loss": 3.761103136826062, + "tokens_seen": 734527488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926178535606821, + "loss": 1.3214, + "theoretical_loss": 3.761035582563352, + "tokens_seen": 734658560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039259779338014046, + "loss": 1.2948, + "theoretical_loss": 3.760968043726081, + "tokens_seen": 734789632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925777331995988, + "loss": 1.4141, + "theoretical_loss": 3.7609005203079757, + "tokens_seen": 734920704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925576730190572, + "loss": 1.3605, + "theoretical_loss": 3.7608330123027667, + "tokens_seen": 735051776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925376128385156, + "loss": 1.292, + "theoretical_loss": 3.760765519704189, + "tokens_seen": 735182848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039251755265797396, + "loss": 1.5685, + "theoretical_loss": 3.7606980425059806, + "tokens_seen": 735313920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924974924774323, + "loss": 1.2658, + "theoretical_loss": 3.760630580701883, + "tokens_seen": 735444992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924774322968907, + "loss": 1.3252, + "theoretical_loss": 3.7605631342856416, + "tokens_seen": 735576064 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.2995479106903076, + "objective/train/avg_token_score": 0.029663361608982086, + "objective/train/docs_used": 424834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9933853149414062, + "objective/train/original_loss": 4.269293785095215, + "objective/train/theoretical_loss": 3.7605294168460133, + "objective/train/tokens_used": 735641600, + "objective/train/unlikelihood_loss": 0.17289303243160248, + "theoretical_loss": 3.7605294168460133, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924573721163491, + "loss": 1.3734, + "theoretical_loss": 3.7604957032510056, + "tokens_seen": 735707136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039243731193580746, + "loss": 1.3465, + "theoretical_loss": 3.7604282875917274, + "tokens_seen": 735838208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924172517552658, + "loss": 1.412, + "theoretical_loss": 3.7603608873015633, + "tokens_seen": 735969280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923971915747242, + "loss": 1.3697, + "theoretical_loss": 3.7602935023742736, + "tokens_seen": 736100352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039237713139418255, + "loss": 1.2509, + "theoretical_loss": 3.7602261328036226, + "tokens_seen": 736231424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039235707121364097, + "loss": 1.5705, + "theoretical_loss": 3.760158778583376, + "tokens_seen": 736362496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039233701103309933, + "loss": 1.4261, + "theoretical_loss": 3.760091439707306, + "tokens_seen": 736493568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923169508525577, + "loss": 1.4259, + "theoretical_loss": 3.7600241161691876, + "tokens_seen": 736624640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039229689067201605, + "loss": 1.5219, + "theoretical_loss": 3.759956807962797, + "tokens_seen": 736755712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039227683049147447, + "loss": 1.263, + "theoretical_loss": 3.759889515081918, + "tokens_seen": 736886784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039225677031093283, + "loss": 1.5659, + "theoretical_loss": 3.7598222375203347, + "tokens_seen": 737017856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922367101303912, + "loss": 1.3973, + "theoretical_loss": 3.7597549752718367, + "tokens_seen": 737148928 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.4684750735759735, + "objective/train/avg_token_score": 0.03102070651948452, + "objective/train/docs_used": 426204, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7906992435455322, + "objective/train/original_loss": 4.015564441680908, + "objective/train/theoretical_loss": 3.759687728330217, + "objective/train/tokens_used": 737280000, + "objective/train/unlikelihood_loss": 0.1498943716287613, + "theoretical_loss": 3.759687728330217, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039221664994984956, + "loss": 1.4579, + "theoretical_loss": 3.759687728330217, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921965897693079, + "loss": 1.4052, + "theoretical_loss": 3.759620496689272, + "tokens_seen": 737411072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039217652958876633, + "loss": 1.3134, + "theoretical_loss": 3.759553280342801, + "tokens_seen": 737542144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921564694082247, + "loss": 1.4192, + "theoretical_loss": 3.759486079284607, + "tokens_seen": 737673216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039213640922768306, + "loss": 1.4059, + "theoretical_loss": 3.7594188935084993, + "tokens_seen": 737804288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921163490471414, + "loss": 1.2566, + "theoretical_loss": 3.7593517230082867, + "tokens_seen": 737935360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039209628886659984, + "loss": 1.4255, + "theoretical_loss": 3.759284567777784, + "tokens_seen": 738066432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920762286860582, + "loss": 1.4255, + "theoretical_loss": 3.75921742781081, + "tokens_seen": 738197504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039205616850551656, + "loss": 1.3369, + "theoretical_loss": 3.759150303101185, + "tokens_seen": 738328576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920361083249749, + "loss": 1.5894, + "theoretical_loss": 3.7590831936427356, + "tokens_seen": 738459648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920160481444333, + "loss": 1.4871, + "theoretical_loss": 3.7590160994292896, + "tokens_seen": 738590720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919959879638917, + "loss": 1.5143, + "theoretical_loss": 3.758949020454679, + "tokens_seen": 738721792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039197592778335007, + "loss": 1.5769, + "theoretical_loss": 3.7588819567127407, + "tokens_seen": 738852864 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.3075513243675232, + "objective/train/avg_token_score": 0.0339408703148365, + "objective/train/docs_used": 426747, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9843421578407288, + "objective/train/original_loss": 3.6582820415496826, + "objective/train/theoretical_loss": 3.7588484305520984, + "objective/train/tokens_used": 738918400, + "objective/train/unlikelihood_loss": 0.21093273162841797, + "theoretical_loss": 3.7588484305520984, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039195586760280843, + "loss": 1.4109, + "theoretical_loss": 3.7588149081973143, + "tokens_seen": 738983936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919358074222668, + "loss": 1.4695, + "theoretical_loss": 3.758747874902242, + "tokens_seen": 739115008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919157472417252, + "loss": 1.3337, + "theoretical_loss": 3.758680856821371, + "tokens_seen": 739246080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039189568706118357, + "loss": 1.3948, + "theoretical_loss": 3.7586138539485514, + "tokens_seen": 739377152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039187562688064193, + "loss": 1.4513, + "theoretical_loss": 3.7585468662776376, + "tokens_seen": 739508224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918555667001003, + "loss": 1.4646, + "theoretical_loss": 3.7584798938024866, + "tokens_seen": 739639296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039183550651955866, + "loss": 1.558, + "theoretical_loss": 3.758412936516959, + "tokens_seen": 739770368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039181544633901707, + "loss": 1.6329, + "theoretical_loss": 3.7583459944149205, + "tokens_seen": 739901440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039179538615847543, + "loss": 1.4624, + "theoretical_loss": 3.7582790674902378, + "tokens_seen": 740032512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917753259779338, + "loss": 1.5375, + "theoretical_loss": 3.758212155736784, + "tokens_seen": 740163584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039175526579739216, + "loss": 1.4223, + "theoretical_loss": 3.7581452591484323, + "tokens_seen": 740294656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917352056168506, + "loss": 1.4265, + "theoretical_loss": 3.7580783777190643, + "tokens_seen": 740425728 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.29600438475608826, + "objective/train/avg_token_score": 0.018239440396428108, + "objective/train/docs_used": 427723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9114974737167358, + "objective/train/original_loss": 3.8528389930725098, + "objective/train/theoretical_loss": 3.75801151144256, + "objective/train/tokens_used": 740556800, + "objective/train/unlikelihood_loss": 0.17719443142414093, + "theoretical_loss": 3.75801151144256, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171514543630894, + "loss": 1.3034, + "theoretical_loss": 3.75801151144256, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916950852557673, + "loss": 1.3945, + "theoretical_loss": 3.7579446603128064, + "tokens_seen": 740687872 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039167502507522566, + "loss": 1.4223, + "theoretical_loss": 3.7578778243236934, + "tokens_seen": 740818944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916549648946841, + "loss": 1.4037, + "theoretical_loss": 3.7578110034691123, + "tokens_seen": 740950016 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163490471414244, + "loss": 1.4941, + "theoretical_loss": 3.7577441977429613, + "tokens_seen": 741081088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916148445336008, + "loss": 1.4633, + "theoretical_loss": 3.7576774071391403, + "tokens_seen": 741212160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039159478435305916, + "loss": 1.5192, + "theoretical_loss": 3.757610631651552, + "tokens_seen": 741343232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915747241725175, + "loss": 1.4978, + "theoretical_loss": 3.7575438712741036, + "tokens_seen": 741474304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039155466399197594, + "loss": 1.3962, + "theoretical_loss": 3.7574771260007074, + "tokens_seen": 741605376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915346038114343, + "loss": 1.2108, + "theoretical_loss": 3.757410395825276, + "tokens_seen": 741736448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039151454363089267, + "loss": 1.4905, + "theoretical_loss": 3.7573436807417275, + "tokens_seen": 741867520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039149448345035103, + "loss": 1.4698, + "theoretical_loss": 3.757276980743983, + "tokens_seen": 741998592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039147442326980945, + "loss": 1.4283, + "theoretical_loss": 3.757210295825968, + "tokens_seen": 742129664 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.37683284282684326, + "objective/train/avg_token_score": 0.008415204472839832, + "objective/train/docs_used": 429091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2889798879623413, + "objective/train/original_loss": 4.006934642791748, + "objective/train/theoretical_loss": 3.7571769590199615, + "objective/train/tokens_used": 742195200, + "objective/train/unlikelihood_loss": 0.16776272654533386, + "theoretical_loss": 3.7571769590199615, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914543630892678, + "loss": 1.5838, + "theoretical_loss": 3.7571436259816107, + "tokens_seen": 742260736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039143430290872617, + "loss": 1.4727, + "theoretical_loss": 3.7570769712048424, + "tokens_seen": 742391808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039141424272818453, + "loss": 1.4207, + "theoretical_loss": 3.7570103314895986, + "tokens_seen": 742522880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913941825476429, + "loss": 1.3775, + "theoretical_loss": 3.7569437068298184, + "tokens_seen": 742653952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913741223671013, + "loss": 1.3082, + "theoretical_loss": 3.7568770972194434, + "tokens_seen": 742785024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913540621865597, + "loss": 1.3519, + "theoretical_loss": 3.756810502652421, + "tokens_seen": 742916096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039133400200601804, + "loss": 1.2282, + "theoretical_loss": 3.756743923122699, + "tokens_seen": 743047168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913139418254764, + "loss": 1.4169, + "theoretical_loss": 3.756677358624231, + "tokens_seen": 743178240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912938816449348, + "loss": 1.3794, + "theoretical_loss": 3.756610809150973, + "tokens_seen": 743309312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912738214643932, + "loss": 1.5191, + "theoretical_loss": 3.7565442746968856, + "tokens_seen": 743440384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039125376128385154, + "loss": 1.3677, + "theoretical_loss": 3.756477755255932, + "tokens_seen": 743571456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912337011033099, + "loss": 1.455, + "theoretical_loss": 3.7564112508220777, + "tokens_seen": 743702528 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.23851417005062103, + "objective/train/avg_token_score": 0.013841371051967144, + "objective/train/docs_used": 429832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7659555673599243, + "objective/train/original_loss": 3.666262149810791, + "objective/train/theoretical_loss": 3.756344761389295, + "objective/train/tokens_used": 743833600, + "objective/train/unlikelihood_loss": 0.2967706620693207, + "theoretical_loss": 3.756344761389295, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039121364092276826, + "loss": 1.3107, + "theoretical_loss": 3.756344761389295, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911935807422267, + "loss": 1.6564, + "theoretical_loss": 3.7562782869515567, + "tokens_seen": 743964672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039117352056168504, + "loss": 1.384, + "theoretical_loss": 3.75621182750284, + "tokens_seen": 744095744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911534603811434, + "loss": 1.3945, + "theoretical_loss": 3.7561453830371256, + "tokens_seen": 744226816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911334002006018, + "loss": 1.3845, + "theoretical_loss": 3.756078953548398, + "tokens_seen": 744357888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039111334002006024, + "loss": 1.3614, + "theoretical_loss": 3.7560125390306456, + "tokens_seen": 744488960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910932798395186, + "loss": 1.5538, + "theoretical_loss": 3.7559461394778584, + "tokens_seen": 744620032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039107321965897696, + "loss": 1.4752, + "theoretical_loss": 3.755879754884032, + "tokens_seen": 744751104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910531594784353, + "loss": 1.6084, + "theoretical_loss": 3.755813385243164, + "tokens_seen": 744882176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910330992978937, + "loss": 1.6859, + "theoretical_loss": 3.755747030549256, + "tokens_seen": 745013248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910130391173521, + "loss": 1.3839, + "theoretical_loss": 3.7556806907963134, + "tokens_seen": 745144320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039099297893681047, + "loss": 1.4966, + "theoretical_loss": 3.7556143659783445, + "tokens_seen": 745275392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039097291875626883, + "loss": 1.4122, + "theoretical_loss": 3.7555480560893617, + "tokens_seen": 745406464 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.4448924660682678, + "objective/train/avg_token_score": 0.01363709382712841, + "objective/train/docs_used": 430489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.563394546508789, + "objective/train/original_loss": 4.073812484741211, + "objective/train/theoretical_loss": 3.7555149067413693, + "objective/train/tokens_used": 745472000, + "objective/train/unlikelihood_loss": 0.14159449934959412, + "theoretical_loss": 3.7555149067413693, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909528585757272, + "loss": 1.4459, + "theoretical_loss": 3.755481761123379, + "tokens_seen": 745537536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909327983951856, + "loss": 1.3834, + "theoretical_loss": 3.7554154810744174, + "tokens_seen": 745668608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039091273821464397, + "loss": 1.4545, + "theoretical_loss": 3.755349215936498, + "tokens_seen": 745799680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039089267803410233, + "loss": 1.471, + "theoretical_loss": 3.755282965703646, + "tokens_seen": 745930752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908726178535607, + "loss": 1.3406, + "theoretical_loss": 3.7552167303698925, + "tokens_seen": 746061824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039085255767301906, + "loss": 1.6518, + "theoretical_loss": 3.755150509929268, + "tokens_seen": 746192896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039083249749247747, + "loss": 1.31, + "theoretical_loss": 3.75508430437581, + "tokens_seen": 746323968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039081243731193583, + "loss": 1.4741, + "theoretical_loss": 3.7550181137035574, + "tokens_seen": 746455040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907923771313942, + "loss": 1.5464, + "theoretical_loss": 3.7549519379065535, + "tokens_seen": 746586112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039077231695085256, + "loss": 1.4542, + "theoretical_loss": 3.7548857769788446, + "tokens_seen": 746717184 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390752256770311, + "loss": 1.3685, + "theoretical_loss": 3.754819630914481, + "tokens_seen": 746848256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039073219658976934, + "loss": 1.3512, + "theoretical_loss": 3.7547534997075145, + "tokens_seen": 746979328 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.5000610947608948, + "objective/train/avg_token_score": 0.015293581411242485, + "objective/train/docs_used": 431396, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.660507082939148, + "objective/train/original_loss": 3.4781405925750732, + "objective/train/theoretical_loss": 3.754687383352003, + "objective/train/tokens_used": 747110400, + "objective/train/unlikelihood_loss": 0.19964474439620972, + "theoretical_loss": 3.754687383352003, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907121364092277, + "loss": 1.6739, + "theoretical_loss": 3.754687383352003, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039069207622868606, + "loss": 1.4193, + "theoretical_loss": 3.754621281842007, + "tokens_seen": 747241472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906720160481445, + "loss": 1.4406, + "theoretical_loss": 3.7545551951715885, + "tokens_seen": 747372544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039065195586760284, + "loss": 1.4892, + "theoretical_loss": 3.7544891233348157, + "tokens_seen": 747503616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906318956870612, + "loss": 1.4621, + "theoretical_loss": 3.7544230663257583, + "tokens_seen": 747634688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039061183550651957, + "loss": 1.467, + "theoretical_loss": 3.75435702413849, + "tokens_seen": 747765760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039059177532597793, + "loss": 1.432, + "theoretical_loss": 3.7542909967670877, + "tokens_seen": 747896832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039057171514543634, + "loss": 1.5256, + "theoretical_loss": 3.754224984205633, + "tokens_seen": 748027904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905516549648947, + "loss": 1.5174, + "theoretical_loss": 3.754158986448209, + "tokens_seen": 748158976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039053159478435307, + "loss": 1.4307, + "theoretical_loss": 3.7540930034889035, + "tokens_seen": 748290048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039051153460381143, + "loss": 1.312, + "theoretical_loss": 3.7540270353218066, + "tokens_seen": 748421120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039049147442326985, + "loss": 1.5499, + "theoretical_loss": 3.753961081941013, + "tokens_seen": 748552192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904714142427282, + "loss": 1.4279, + "theoretical_loss": 3.7538951433406202, + "tokens_seen": 748683264 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.24780058860778809, + "objective/train/avg_token_score": 0.0343734510242939, + "objective/train/docs_used": 432599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7997082471847534, + "objective/train/original_loss": 4.377548694610596, + "objective/train/theoretical_loss": 3.7538621795812306, + "objective/train/tokens_used": 748748800, + "objective/train/unlikelihood_loss": 0.12654738128185272, + "theoretical_loss": 3.7538621795812306, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039045135406218657, + "loss": 1.4201, + "theoretical_loss": 3.7538292195147287, + "tokens_seen": 748814336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039043129388164493, + "loss": 1.5655, + "theoretical_loss": 3.753763310457444, + "tokens_seen": 748945408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904112337011033, + "loss": 1.3015, + "theoretical_loss": 3.753697416162872, + "tokens_seen": 749076480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903911735205617, + "loss": 1.5175, + "theoretical_loss": 3.753631536625125, + "tokens_seen": 749207552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903711133400201, + "loss": 1.3919, + "theoretical_loss": 3.753565671838317, + "tokens_seen": 749338624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039035105315947844, + "loss": 1.6054, + "theoretical_loss": 3.753499821796566, + "tokens_seen": 749469696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903309929789368, + "loss": 1.4777, + "theoretical_loss": 3.7534339864939934, + "tokens_seen": 749600768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903109327983952, + "loss": 1.4856, + "theoretical_loss": 3.7533681659247233, + "tokens_seen": 749731840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902908726178536, + "loss": 1.4702, + "theoretical_loss": 3.753302360082884, + "tokens_seen": 749862912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039027081243731194, + "loss": 1.4873, + "theoretical_loss": 3.753236568962606, + "tokens_seen": 749993984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902507522567703, + "loss": 1.6224, + "theoretical_loss": 3.7531707925580253, + "tokens_seen": 750125056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039023069207622866, + "loss": 1.4225, + "theoretical_loss": 3.7531050308632796, + "tokens_seen": 750256128 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.5135019421577454, + "objective/train/avg_token_score": 0.004611321724951267, + "objective/train/docs_used": 433353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7639703750610352, + "objective/train/original_loss": 3.8232853412628174, + "objective/train/theoretical_loss": 3.7530392838725097, + "objective/train/tokens_used": 750387200, + "objective/train/unlikelihood_loss": 0.15716251730918884, + "theoretical_loss": 3.7530392838725097, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902106318956871, + "loss": 1.6701, + "theoretical_loss": 3.7530392838725097, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039019057171514544, + "loss": 1.4417, + "theoretical_loss": 3.7529735515798603, + "tokens_seen": 750518272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901705115346038, + "loss": 1.4648, + "theoretical_loss": 3.7529078339794806, + "tokens_seen": 750649344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039015045135406217, + "loss": 1.2315, + "theoretical_loss": 3.7528421310655204, + "tokens_seen": 750780416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901303911735206, + "loss": 1.4212, + "theoretical_loss": 3.7527764428321357, + "tokens_seen": 750911488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039011033099297895, + "loss": 1.3217, + "theoretical_loss": 3.7527107692734845, + "tokens_seen": 751042560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900902708124373, + "loss": 1.378, + "theoretical_loss": 3.752645110383728, + "tokens_seen": 751173632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039007021063189567, + "loss": 1.4025, + "theoretical_loss": 3.752579466157031, + "tokens_seen": 751304704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039005015045135403, + "loss": 1.4049, + "theoretical_loss": 3.7525138365875623, + "tokens_seen": 751435776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039003009027081245, + "loss": 1.6542, + "theoretical_loss": 3.7524482216694923, + "tokens_seen": 751566848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900100300902708, + "loss": 1.4771, + "theoretical_loss": 3.752382621396997, + "tokens_seen": 751697920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899899699097292, + "loss": 1.7524, + "theoretical_loss": 3.7523170357642535, + "tokens_seen": 751828992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038996990972918754, + "loss": 1.4132, + "theoretical_loss": 3.752251464765444, + "tokens_seen": 751960064 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.45717254281044006, + "objective/train/avg_token_score": 0.011227022856473923, + "objective/train/docs_used": 434593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6856797933578491, + "objective/train/original_loss": 4.228733062744141, + "objective/train/theoretical_loss": 3.7522186847519476, + "objective/train/tokens_used": 752025600, + "objective/train/unlikelihood_loss": 0.1167980507016182, + "theoretical_loss": 3.7522186847519476, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038994984954864595, + "loss": 1.4942, + "theoretical_loss": 3.752185908394754, + "tokens_seen": 752091136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899297893681043, + "loss": 1.4649, + "theoretical_loss": 3.75212036664637, + "tokens_seen": 752222208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899097291875627, + "loss": 1.4285, + "theoretical_loss": 3.7520548395144844, + "tokens_seen": 752353280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038988966900702104, + "loss": 1.2977, + "theoretical_loss": 3.751989326993292, + "tokens_seen": 752484352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898696088264794, + "loss": 1.5903, + "theoretical_loss": 3.7519238290769907, + "tokens_seen": 752615424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898495486459378, + "loss": 1.3916, + "theoretical_loss": 3.751858345759782, + "tokens_seen": 752746496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898294884653962, + "loss": 1.4929, + "theoretical_loss": 3.7517928770358706, + "tokens_seen": 752877568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038980942828485454, + "loss": 1.4823, + "theoretical_loss": 3.7517274228994646, + "tokens_seen": 753008640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897893681043129, + "loss": 1.3659, + "theoretical_loss": 3.7516619833447757, + "tokens_seen": 753139712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897693079237713, + "loss": 1.4114, + "theoretical_loss": 3.7515965583660176, + "tokens_seen": 753270784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897492477432297, + "loss": 1.4444, + "theoretical_loss": 3.7515311479574094, + "tokens_seen": 753401856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038972918756268805, + "loss": 1.4159, + "theoretical_loss": 3.7514657521131713, + "tokens_seen": 753532928 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.502993643283844, + "objective/train/avg_token_score": 0.0060858107171952724, + "objective/train/docs_used": 435245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7495417594909668, + "objective/train/original_loss": 3.4987025260925293, + "objective/train/theoretical_loss": 3.751400370827529, + "objective/train/tokens_used": 753664000, + "objective/train/unlikelihood_loss": 0.18288074433803558, + "theoretical_loss": 3.751400370827529, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897091273821464, + "loss": 1.4847, + "theoretical_loss": 3.751400370827529, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038968906720160477, + "loss": 1.2712, + "theoretical_loss": 3.751335004094709, + "tokens_seen": 753795072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896690070210632, + "loss": 1.4421, + "theoretical_loss": 3.7512696519089435, + "tokens_seen": 753926144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038964894684052155, + "loss": 1.5137, + "theoretical_loss": 3.7512043142644664, + "tokens_seen": 754057216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038962888665997997, + "loss": 1.347, + "theoretical_loss": 3.7511389911555155, + "tokens_seen": 754188288 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038960882647943833, + "loss": 1.5096, + "theoretical_loss": 3.7510736825763322, + "tokens_seen": 754319360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038958876629889674, + "loss": 1.2872, + "theoretical_loss": 3.75100838852116, + "tokens_seen": 754450432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895687061183551, + "loss": 1.4156, + "theoretical_loss": 3.750943108984247, + "tokens_seen": 754581504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038954864593781347, + "loss": 1.5146, + "theoretical_loss": 3.7508778439598442, + "tokens_seen": 754712576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038952858575727183, + "loss": 1.4055, + "theoretical_loss": 3.7508125934422054, + "tokens_seen": 754843648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038950852557673025, + "loss": 1.3589, + "theoretical_loss": 3.7507473574255874, + "tokens_seen": 754974720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894884653961886, + "loss": 1.2663, + "theoretical_loss": 3.7506821359042517, + "tokens_seen": 755105792 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038946840521564697, + "loss": 1.5921, + "theoretical_loss": 3.7506169288724625, + "tokens_seen": 755236864 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.4199657738208771, + "objective/train/avg_token_score": 0.018268246203660965, + "objective/train/docs_used": 436337, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4811499118804932, + "objective/train/original_loss": 3.924743175506592, + "objective/train/theoretical_loss": 3.7505843307883557, + "objective/train/tokens_used": 755302400, + "objective/train/unlikelihood_loss": 0.16431909799575806, + "theoretical_loss": 3.7505843307883557, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038944834503510533, + "loss": 1.4489, + "theoretical_loss": 3.750551736324486, + "tokens_seen": 755367936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894282848545637, + "loss": 1.2682, + "theoretical_loss": 3.750486558254593, + "tokens_seen": 755499008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894082246740221, + "loss": 1.4079, + "theoretical_loss": 3.7504213946570575, + "tokens_seen": 755630080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893881644934805, + "loss": 1.5093, + "theoretical_loss": 3.7503562455261568, + "tokens_seen": 755761152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038936810431293884, + "loss": 1.4008, + "theoretical_loss": 3.7502911108561694, + "tokens_seen": 755892224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893480441323972, + "loss": 1.6043, + "theoretical_loss": 3.750225990641381, + "tokens_seen": 756023296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893279839518556, + "loss": 1.246, + "theoretical_loss": 3.7501608848760766, + "tokens_seen": 756154368 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389307923771314, + "loss": 1.5534, + "theoretical_loss": 3.7500957935545474, + "tokens_seen": 756285440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038928786359077234, + "loss": 1.4016, + "theoretical_loss": 3.750030716671086, + "tokens_seen": 756416512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892678034102307, + "loss": 1.49, + "theoretical_loss": 3.7499656542199884, + "tokens_seen": 756547584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038924774322968906, + "loss": 1.5503, + "theoretical_loss": 3.749900606195555, + "tokens_seen": 756678656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892276830491475, + "loss": 1.6831, + "theoretical_loss": 3.7498355725920884, + "tokens_seen": 756809728 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.32209187746047974, + "objective/train/avg_token_score": 0.007704250048846006, + "objective/train/docs_used": 436904, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1317243576049805, + "objective/train/original_loss": 3.57211971282959, + "objective/train/theoretical_loss": 3.749770553403895, + "objective/train/tokens_used": 756940800, + "objective/train/unlikelihood_loss": 0.25332245230674744, + "theoretical_loss": 3.749770553403895, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038920762286860584, + "loss": 1.259, + "theoretical_loss": 3.749770553403895, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891875626880642, + "loss": 1.4307, + "theoretical_loss": 3.7497055486252844, + "tokens_seen": 757071872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038916750250752257, + "loss": 1.4855, + "theoretical_loss": 3.7496405582505687, + "tokens_seen": 757202944 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389147442326981, + "loss": 1.5926, + "theoretical_loss": 3.7495755822740646, + "tokens_seen": 757334016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038912738214643935, + "loss": 1.6839, + "theoretical_loss": 3.7495106206900903, + "tokens_seen": 757465088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891073219658977, + "loss": 1.4186, + "theoretical_loss": 3.749445673492968, + "tokens_seen": 757596160 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038908726178535607, + "loss": 1.4306, + "theoretical_loss": 3.749380740677024, + "tokens_seen": 757727232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038906720160481443, + "loss": 1.4441, + "theoretical_loss": 3.749315822236587, + "tokens_seen": 757858304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038904714142427285, + "loss": 1.5755, + "theoretical_loss": 3.7492509181659885, + "tokens_seen": 757989376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890270812437312, + "loss": 1.3564, + "theoretical_loss": 3.749186028459564, + "tokens_seen": 758120448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890070210631896, + "loss": 1.3389, + "theoretical_loss": 3.7491211531116515, + "tokens_seen": 758251520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038898696088264794, + "loss": 1.4864, + "theoretical_loss": 3.749056292116593, + "tokens_seen": 758382592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038896690070210635, + "loss": 1.4697, + "theoretical_loss": 3.748991445468733, + "tokens_seen": 758513664 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.42668622732162476, + "objective/train/avg_token_score": 0.014906138181686401, + "objective/train/docs_used": 438157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.48072350025177, + "objective/train/original_loss": 4.132023334503174, + "objective/train/theoretical_loss": 3.7489590275232367, + "objective/train/tokens_used": 758579200, + "objective/train/unlikelihood_loss": 0.12642821669578552, + "theoretical_loss": 3.7489590275232367, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889468405215647, + "loss": 1.3844, + "theoretical_loss": 3.7489266131624204, + "tokens_seen": 758644736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889267803410231, + "loss": 1.4823, + "theoretical_loss": 3.7488617951920054, + "tokens_seen": 758775808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038890672016048144, + "loss": 1.2582, + "theoretical_loss": 3.748796991551843, + "tokens_seen": 758906880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888866599799398, + "loss": 1.4727, + "theoretical_loss": 3.7487322022362903, + "tokens_seen": 759037952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888665997993982, + "loss": 1.3445, + "theoretical_loss": 3.7486674272397082, + "tokens_seen": 759169024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888465396188566, + "loss": 1.4026, + "theoretical_loss": 3.7486026665564616, + "tokens_seen": 759300096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038882647943831494, + "loss": 1.501, + "theoretical_loss": 3.7485379201809166, + "tokens_seen": 759431168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888064192577733, + "loss": 1.2764, + "theoretical_loss": 3.748473188107444, + "tokens_seen": 759562240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887863590772317, + "loss": 1.4743, + "theoretical_loss": 3.7484084703304177, + "tokens_seen": 759693312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887662988966901, + "loss": 1.4562, + "theoretical_loss": 3.748343766844214, + "tokens_seen": 759824384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038874623871614845, + "loss": 1.5622, + "theoretical_loss": 3.748279077643213, + "tokens_seen": 759955456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887261785356068, + "loss": 1.4005, + "theoretical_loss": 3.748214402721798, + "tokens_seen": 760086528 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.3283235430717468, + "objective/train/avg_token_score": 0.027465643361210823, + "objective/train/docs_used": 438864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0259320735931396, + "objective/train/original_loss": 3.8068859577178955, + "objective/train/theoretical_loss": 3.748149742074355, + "objective/train/tokens_used": 760217600, + "objective/train/unlikelihood_loss": 0.19544579088687897, + "theoretical_loss": 3.748149742074355, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038870611835506517, + "loss": 1.4535, + "theoretical_loss": 3.748149742074355, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886860581745236, + "loss": 1.5279, + "theoretical_loss": 3.7480850956952736, + "tokens_seen": 760348672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866599799398195, + "loss": 1.3934, + "theoretical_loss": 3.7480204635789462, + "tokens_seen": 760479744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886459378134403, + "loss": 1.3507, + "theoretical_loss": 3.74795584571977, + "tokens_seen": 760610816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886258776328987, + "loss": 1.6235, + "theoretical_loss": 3.747891242112142, + "tokens_seen": 760741888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886058174523571, + "loss": 1.3677, + "theoretical_loss": 3.747826652750465, + "tokens_seen": 760872960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038858575727181545, + "loss": 1.4706, + "theoretical_loss": 3.7477620776291447, + "tokens_seen": 761004032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885656970912738, + "loss": 1.3825, + "theoretical_loss": 3.747697516742589, + "tokens_seen": 761135104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885456369107322, + "loss": 1.4516, + "theoretical_loss": 3.7476329700852107, + "tokens_seen": 761266176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885255767301906, + "loss": 1.477, + "theoretical_loss": 3.747568437651423, + "tokens_seen": 761397248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038850551654964896, + "loss": 1.344, + "theoretical_loss": 3.747503919435645, + "tokens_seen": 761528320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884854563691073, + "loss": 1.2744, + "theoretical_loss": 3.7474394154322974, + "tokens_seen": 761659392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884653961885657, + "loss": 1.3868, + "theoretical_loss": 3.747374925635804, + "tokens_seen": 761790464 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.38037633895874023, + "objective/train/avg_token_score": 0.028013383969664574, + "objective/train/docs_used": 439620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3942844867706299, + "objective/train/original_loss": 3.7951772212982178, + "objective/train/theoretical_loss": 3.7473426860633863, + "objective/train/tokens_used": 761856000, + "objective/train/unlikelihood_loss": 0.1631736010313034, + "theoretical_loss": 3.7473426860633863, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038844533600802404, + "loss": 1.5605, + "theoretical_loss": 3.747310450040593, + "tokens_seen": 761921536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842527582748246, + "loss": 1.2781, + "theoretical_loss": 3.7472459886410947, + "tokens_seen": 762052608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884052156469408, + "loss": 1.5646, + "theoretical_loss": 3.7471815414317424, + "tokens_seen": 762183680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883851554663992, + "loss": 1.4635, + "theoretical_loss": 3.747117108406973, + "tokens_seen": 762314752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038836509528585755, + "loss": 1.3558, + "theoretical_loss": 3.747052689561227, + "tokens_seen": 762445824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834503510531596, + "loss": 1.5161, + "theoretical_loss": 3.746988284888946, + "tokens_seen": 762576896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883249749247743, + "loss": 1.461, + "theoretical_loss": 3.746923894384578, + "tokens_seen": 762707968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883049147442327, + "loss": 1.4004, + "theoretical_loss": 3.7468595180425712, + "tokens_seen": 762839040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038828485456369105, + "loss": 1.5129, + "theoretical_loss": 3.746795155857379, + "tokens_seen": 762970112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882647943831494, + "loss": 1.4178, + "theoretical_loss": 3.7467308078234565, + "tokens_seen": 763101184 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038824473420260783, + "loss": 1.5011, + "theoretical_loss": 3.746666473935262, + "tokens_seen": 763232256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882246740220662, + "loss": 1.5124, + "theoretical_loss": 3.7466021541872574, + "tokens_seen": 763363328 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.44611436128616333, + "objective/train/avg_token_score": 0.02311655506491661, + "objective/train/docs_used": 440741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6470202207565308, + "objective/train/original_loss": 4.1252923011779785, + "objective/train/theoretical_loss": 3.746537848573908, + "objective/train/tokens_used": 763494400, + "objective/train/unlikelihood_loss": 0.1503082662820816, + "theoretical_loss": 3.746537848573908, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038820461384152455, + "loss": 1.3367, + "theoretical_loss": 3.746537848573908, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881845536609829, + "loss": 1.3185, + "theoretical_loss": 3.7464735570896823, + "tokens_seen": 763625472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038816449348044133, + "loss": 1.4691, + "theoretical_loss": 3.7464092797290505, + "tokens_seen": 763756544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881444332998997, + "loss": 1.3554, + "theoretical_loss": 3.746345016486488, + "tokens_seen": 763887616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881243731193581, + "loss": 1.3566, + "theoretical_loss": 3.7462807673564713, + "tokens_seen": 764018688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038810431293881647, + "loss": 1.4177, + "theoretical_loss": 3.7462165323334813, + "tokens_seen": 764149760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038808425275827483, + "loss": 1.5714, + "theoretical_loss": 3.7461523114120014, + "tokens_seen": 764280832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038806419257773325, + "loss": 1.3686, + "theoretical_loss": 3.7460881045865184, + "tokens_seen": 764411904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880441323971916, + "loss": 1.5264, + "theoretical_loss": 3.7460239118515224, + "tokens_seen": 764542976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038802407221665, + "loss": 1.4346, + "theoretical_loss": 3.745959733201506, + "tokens_seen": 764674048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038800401203610834, + "loss": 1.4248, + "theoretical_loss": 3.7458955686309654, + "tokens_seen": 764805120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038798395185556675, + "loss": 1.5523, + "theoretical_loss": 3.7458314181344, + "tokens_seen": 764936192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879638916750251, + "loss": 1.5398, + "theoretical_loss": 3.7457672817063115, + "tokens_seen": 765067264 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.37212854623794556, + "objective/train/avg_token_score": 0.02284182980656624, + "objective/train/docs_used": 441342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.416490912437439, + "objective/train/original_loss": 4.022951126098633, + "objective/train/theoretical_loss": 3.745735218766229, + "objective/train/tokens_used": 765132800, + "objective/train/unlikelihood_loss": 0.18392714858055115, + "theoretical_loss": 3.745735218766229, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879438314944835, + "loss": 1.3791, + "theoretical_loss": 3.745703159341205, + "tokens_seen": 765198336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038792377131394184, + "loss": 1.2568, + "theoretical_loss": 3.7456390510335895, + "tokens_seen": 765329408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879037111334002, + "loss": 1.5514, + "theoretical_loss": 3.7455749567779764, + "tokens_seen": 765460480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878836509528586, + "loss": 1.3346, + "theoretical_loss": 3.74551087656888, + "tokens_seen": 765591552 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387863590772317, + "loss": 1.2877, + "theoretical_loss": 3.7454468104008174, + "tokens_seen": 765722624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038784353059177534, + "loss": 1.3251, + "theoretical_loss": 3.74538275826831, + "tokens_seen": 765853696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878234704112337, + "loss": 1.3729, + "theoretical_loss": 3.7453187201658817, + "tokens_seen": 765984768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878034102306921, + "loss": 1.4415, + "theoretical_loss": 3.7452546960880593, + "tokens_seen": 766115840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877833500501505, + "loss": 1.3859, + "theoretical_loss": 3.745190686029372, + "tokens_seen": 766246912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038776328986960885, + "loss": 1.2924, + "theoretical_loss": 3.7451266899843536, + "tokens_seen": 766377984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877432296890672, + "loss": 1.4323, + "theoretical_loss": 3.7450627079475396, + "tokens_seen": 766509056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038772316950852557, + "loss": 1.3762, + "theoretical_loss": 3.74499873991347, + "tokens_seen": 766640128 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.42442572116851807, + "objective/train/avg_token_score": 0.011302497237920761, + "objective/train/docs_used": 442736, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5058423280715942, + "objective/train/original_loss": 3.8860366344451904, + "objective/train/theoretical_loss": 3.744934785876686, + "objective/train/tokens_used": 766771200, + "objective/train/unlikelihood_loss": 0.14501526951789856, + "theoretical_loss": 3.744934785876686, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387703109327984, + "loss": 1.306, + "theoretical_loss": 3.744934785876686, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038768304914744235, + "loss": 1.3238, + "theoretical_loss": 3.7448708458317332, + "tokens_seen": 766902272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876629889669007, + "loss": 1.3272, + "theoretical_loss": 3.7448069197731595, + "tokens_seen": 767033344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876429287863591, + "loss": 1.4696, + "theoretical_loss": 3.7447430076955177, + "tokens_seen": 767164416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876228686058175, + "loss": 1.3658, + "theoretical_loss": 3.7446791095933603, + "tokens_seen": 767295488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038760280842527585, + "loss": 1.4286, + "theoretical_loss": 3.744615225461246, + "tokens_seen": 767426560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875827482447342, + "loss": 1.4236, + "theoretical_loss": 3.7445513552937353, + "tokens_seen": 767557632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875626880641926, + "loss": 1.3858, + "theoretical_loss": 3.7444874990853907, + "tokens_seen": 767688704 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387542627883651, + "loss": 1.3155, + "theoretical_loss": 3.7444236568307803, + "tokens_seen": 767819776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038752256770310936, + "loss": 1.3835, + "theoretical_loss": 3.7443598285244724, + "tokens_seen": 767950848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875025075225677, + "loss": 1.4252, + "theoretical_loss": 3.74429601416104, + "tokens_seen": 768081920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874824473420261, + "loss": 1.2402, + "theoretical_loss": 3.7442322137350597, + "tokens_seen": 768212992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038746238716148444, + "loss": 1.3906, + "theoretical_loss": 3.7441684272411093, + "tokens_seen": 768344064 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.3087121248245239, + "objective/train/avg_token_score": 0.016747141256928444, + "objective/train/docs_used": 443303, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1123443841934204, + "objective/train/original_loss": 3.807309865951538, + "objective/train/theoretical_loss": 3.744136539216952, + "objective/train/tokens_used": 768409600, + "objective/train/unlikelihood_loss": 0.23338647186756134, + "theoretical_loss": 3.744136539216952, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038744232698094286, + "loss": 1.331, + "theoretical_loss": 3.744104654673771, + "tokens_seen": 768475136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874222668004012, + "loss": 1.4252, + "theoretical_loss": 3.744040896027629, + "tokens_seen": 768606208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874022066198596, + "loss": 1.2675, + "theoretical_loss": 3.7439771512972726, + "tokens_seen": 768737280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038738214643931795, + "loss": 1.3953, + "theoretical_loss": 3.7439134204772913, + "tokens_seen": 768868352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038736208625877636, + "loss": 1.448, + "theoretical_loss": 3.743849703562279, + "tokens_seen": 768999424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873420260782347, + "loss": 1.3438, + "theoretical_loss": 3.7437860005468337, + "tokens_seen": 769130496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873219658976931, + "loss": 1.2093, + "theoretical_loss": 3.7437223114255542, + "tokens_seen": 769261568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038730190571715145, + "loss": 1.2425, + "theoretical_loss": 3.7436586361930444, + "tokens_seen": 769392640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872818455366098, + "loss": 1.5094, + "theoretical_loss": 3.7435949748439095, + "tokens_seen": 769523712 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038726178535606823, + "loss": 1.4416, + "theoretical_loss": 3.743531327372759, + "tokens_seen": 769654784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872417251755266, + "loss": 1.3111, + "theoretical_loss": 3.7434676937742046, + "tokens_seen": 769785856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038722166499498495, + "loss": 1.4532, + "theoretical_loss": 3.7434040740428616, + "tokens_seen": 769916928 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.32105326652526855, + "objective/train/avg_token_score": 0.026793792843818665, + "objective/train/docs_used": 444724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1371012926101685, + "objective/train/original_loss": 3.8675286769866943, + "objective/train/theoretical_loss": 3.7433404681733475, + "objective/train/tokens_used": 770048000, + "objective/train/unlikelihood_loss": 0.2552289664745331, + "theoretical_loss": 3.7433404681733475, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872016048144433, + "loss": 1.2757, + "theoretical_loss": 3.7433404681733475, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038718154463390173, + "loss": 1.1978, + "theoretical_loss": 3.7432768761602837, + "tokens_seen": 770179072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871614844533601, + "loss": 1.4035, + "theoretical_loss": 3.7432132979982944, + "tokens_seen": 770310144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038714142427281846, + "loss": 1.4709, + "theoretical_loss": 3.7431497336820065, + "tokens_seen": 770441216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871213640922768, + "loss": 1.4886, + "theoretical_loss": 3.7430861832060494, + "tokens_seen": 770572288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871013039117352, + "loss": 1.3581, + "theoretical_loss": 3.7430226465650573, + "tokens_seen": 770703360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870812437311936, + "loss": 1.6077, + "theoretical_loss": 3.742959123753665, + "tokens_seen": 770834432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038706118355065196, + "loss": 1.5473, + "theoretical_loss": 3.742895614766512, + "tokens_seen": 770965504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870411233701103, + "loss": 1.4347, + "theoretical_loss": 3.74283211959824, + "tokens_seen": 771096576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870210631895687, + "loss": 1.4231, + "theoretical_loss": 3.7427686382434953, + "tokens_seen": 771227648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870010030090271, + "loss": 1.5377, + "theoretical_loss": 3.742705170696924, + "tokens_seen": 771358720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038698094282848546, + "loss": 1.3054, + "theoretical_loss": 3.7426417169531776, + "tokens_seen": 771489792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869608826479438, + "loss": 1.5421, + "theoretical_loss": 3.742578277006911, + "tokens_seen": 771620864 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.3636363744735718, + "objective/train/avg_token_score": 0.021078167483210564, + "objective/train/docs_used": 445347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0308493375778198, + "objective/train/original_loss": 3.8079183101654053, + "objective/train/theoretical_loss": 3.7425465622061624, + "objective/train/tokens_used": 771686400, + "objective/train/unlikelihood_loss": 0.1504705250263214, + "theoretical_loss": 3.7425465622061624, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869408224674022, + "loss": 1.2899, + "theoretical_loss": 3.74251485085278, + "tokens_seen": 771751936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038692076228686055, + "loss": 1.5257, + "theoretical_loss": 3.7424514384854453, + "tokens_seen": 771883008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038690070210631896, + "loss": 1.4523, + "theoretical_loss": 3.742388039899569, + "tokens_seen": 772014080 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038688064192577733, + "loss": 1.2852, + "theoretical_loss": 3.742324655089817, + "tokens_seen": 772145152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868605817452357, + "loss": 1.2857, + "theoretical_loss": 3.742261284050859, + "tokens_seen": 772276224 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038684052156469405, + "loss": 1.3765, + "theoretical_loss": 3.7421979267773655, + "tokens_seen": 772407296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038682046138415247, + "loss": 1.4636, + "theoretical_loss": 3.7421345832640123, + "tokens_seen": 772538368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038680040120361083, + "loss": 1.3313, + "theoretical_loss": 3.7420712535054763, + "tokens_seen": 772669440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867803410230692, + "loss": 1.4163, + "theoretical_loss": 3.742007937496438, + "tokens_seen": 772800512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038676028084252755, + "loss": 1.2165, + "theoretical_loss": 3.741944635231582, + "tokens_seen": 772931584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867402206619859, + "loss": 1.6005, + "theoretical_loss": 3.7418813467055942, + "tokens_seen": 773062656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038672016048144433, + "loss": 1.3839, + "theoretical_loss": 3.741818071913164, + "tokens_seen": 773193728 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.2763318717479706, + "objective/train/avg_token_score": 0.0201827734708786, + "objective/train/docs_used": 446623, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9796729683876038, + "objective/train/original_loss": 3.6378111839294434, + "objective/train/theoretical_loss": 3.7417548108489846, + "objective/train/tokens_used": 773324800, + "objective/train/unlikelihood_loss": 0.24409928917884827, + "theoretical_loss": 3.7417548108489846, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867001003009027, + "loss": 1.4414, + "theoretical_loss": 3.7417548108489846, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038668004012036106, + "loss": 1.3974, + "theoretical_loss": 3.7416915635077506, + "tokens_seen": 773455872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866599799398194, + "loss": 1.3644, + "theoretical_loss": 3.7416283298841604, + "tokens_seen": 773586944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038663991975927784, + "loss": 1.3381, + "theoretical_loss": 3.7415651099729166, + "tokens_seen": 773718016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038661985957873625, + "loss": 1.4587, + "theoretical_loss": 3.7415019037687216, + "tokens_seen": 773849088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865997993981946, + "loss": 1.3284, + "theoretical_loss": 3.7414387112662837, + "tokens_seen": 773980160 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386579739217653, + "loss": 1.524, + "theoretical_loss": 3.741375532460313, + "tokens_seen": 774111232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865596790371114, + "loss": 1.3336, + "theoretical_loss": 3.741312367345522, + "tokens_seen": 774242304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038653961885656976, + "loss": 1.57, + "theoretical_loss": 3.7412492159166275, + "tokens_seen": 774373376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865195586760281, + "loss": 1.646, + "theoretical_loss": 3.7411860781683477, + "tokens_seen": 774504448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864994984954865, + "loss": 1.4486, + "theoretical_loss": 3.741122954095405, + "tokens_seen": 774635520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038647943831494484, + "loss": 1.3589, + "theoretical_loss": 3.7410598436925238, + "tokens_seen": 774766592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038645937813440326, + "loss": 1.3411, + "theoretical_loss": 3.7409967469544325, + "tokens_seen": 774897664 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.5264540314674377, + "objective/train/avg_token_score": 0.00822264514863491, + "objective/train/docs_used": 447277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.874933123588562, + "objective/train/original_loss": 3.633820056915283, + "objective/train/theoretical_loss": 3.7409652037080363, + "objective/train/tokens_used": 774963200, + "objective/train/unlikelihood_loss": 0.1731468290090561, + "theoretical_loss": 3.7409652037080363, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864393179538616, + "loss": 1.4087, + "theoretical_loss": 3.740933663875861, + "tokens_seen": 775028736 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038641925777332, + "loss": 1.5203, + "theoretical_loss": 3.7408705944515432, + "tokens_seen": 775159808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038639919759277835, + "loss": 1.239, + "theoretical_loss": 3.740807538676216, + "tokens_seen": 775290880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038637913741223676, + "loss": 1.4048, + "theoretical_loss": 3.740744496544618, + "tokens_seen": 775421952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863590772316951, + "loss": 1.4134, + "theoretical_loss": 3.740681468051492, + "tokens_seen": 775553024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863390170511535, + "loss": 1.5433, + "theoretical_loss": 3.7406184531915834, + "tokens_seen": 775684096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038631895687061185, + "loss": 1.4334, + "theoretical_loss": 3.74055545195964, + "tokens_seen": 775815168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862988966900702, + "loss": 1.2573, + "theoretical_loss": 3.7404924643504134, + "tokens_seen": 775946240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038627883650952863, + "loss": 1.392, + "theoretical_loss": 3.740429490358657, + "tokens_seen": 776077312 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386258776328987, + "loss": 1.4617, + "theoretical_loss": 3.740366529979128, + "tokens_seen": 776208384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038623871614844535, + "loss": 1.4336, + "theoretical_loss": 3.740303583206586, + "tokens_seen": 776339456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862186559679037, + "loss": 1.5071, + "theoretical_loss": 3.7402406500357936, + "tokens_seen": 776470528 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.32630741596221924, + "objective/train/avg_token_score": 0.010929838754236698, + "objective/train/docs_used": 447858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1377559900283813, + "objective/train/original_loss": 3.5949301719665527, + "objective/train/theoretical_loss": 3.740177730461517, + "objective/train/tokens_used": 776601600, + "objective/train/unlikelihood_loss": 0.3136020004749298, + "theoretical_loss": 3.740177730461517, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038619859578736213, + "loss": 1.3311, + "theoretical_loss": 3.740177730461517, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861785356068205, + "loss": 1.4052, + "theoretical_loss": 3.740114824478524, + "tokens_seen": 776732672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038615847542627886, + "loss": 1.4444, + "theoretical_loss": 3.7400519320815864, + "tokens_seen": 776863744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861384152457372, + "loss": 1.3743, + "theoretical_loss": 3.7399890532654783, + "tokens_seen": 776994816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861183550651956, + "loss": 1.4911, + "theoretical_loss": 3.7399261880249766, + "tokens_seen": 777125888 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386098294884654, + "loss": 1.5227, + "theoretical_loss": 3.739863336354862, + "tokens_seen": 777256960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038607823470411236, + "loss": 1.4423, + "theoretical_loss": 3.739800498249917, + "tokens_seen": 777388032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860581745235707, + "loss": 1.4401, + "theoretical_loss": 3.7397376737049277, + "tokens_seen": 777519104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860381143430291, + "loss": 1.5275, + "theoretical_loss": 3.7396748627146827, + "tokens_seen": 777650176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860180541624875, + "loss": 1.6137, + "theoretical_loss": 3.739612065273973, + "tokens_seen": 777781248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038599799398194586, + "loss": 1.4577, + "theoretical_loss": 3.7395492813775943, + "tokens_seen": 777912320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859779338014042, + "loss": 1.312, + "theoretical_loss": 3.739486511020343, + "tokens_seen": 778043392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859578736208626, + "loss": 1.3607, + "theoretical_loss": 3.739423754197019, + "tokens_seen": 778174464 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4747067391872406, + "objective/train/avg_token_score": 0.006585919763892889, + "objective/train/docs_used": 448758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.646989107131958, + "objective/train/original_loss": 3.845280885696411, + "objective/train/theoretical_loss": 3.7393923808589564, + "objective/train/tokens_used": 778240000, + "objective/train/unlikelihood_loss": 0.1784931719303131, + "theoretical_loss": 3.7393923808589564, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593781344032095, + "loss": 1.5075, + "theoretical_loss": 3.7393610109024267, + "tokens_seen": 778305536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038591775325977937, + "loss": 1.3941, + "theoretical_loss": 3.7392982811313713, + "tokens_seen": 778436608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038589769307923773, + "loss": 1.4607, + "theoretical_loss": 3.739235564878661, + "tokens_seen": 778567680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858776328986961, + "loss": 1.5403, + "theoretical_loss": 3.739172862139109, + "tokens_seen": 778698752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585757271815445, + "loss": 1.4656, + "theoretical_loss": 3.7391101729075285, + "tokens_seen": 778829824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038583751253761287, + "loss": 1.5421, + "theoretical_loss": 3.7390474971787375, + "tokens_seen": 778960896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581745235707123, + "loss": 1.5205, + "theoretical_loss": 3.7389848349475567, + "tokens_seen": 779091968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857973921765296, + "loss": 1.3997, + "theoretical_loss": 3.7389221862088085, + "tokens_seen": 779223040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038577733199598796, + "loss": 1.4383, + "theoretical_loss": 3.738859550957319, + "tokens_seen": 779354112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857572718154463, + "loss": 1.4256, + "theoretical_loss": 3.7387969291879175, + "tokens_seen": 779485184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038573721163490473, + "loss": 1.6094, + "theoretical_loss": 3.7387343208954356, + "tokens_seen": 779616256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857171514543631, + "loss": 1.4508, + "theoretical_loss": 3.738671726074707, + "tokens_seen": 779747328 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.38037633895874023, + "objective/train/avg_token_score": 0.01707608811557293, + "objective/train/docs_used": 449318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3804527521133423, + "objective/train/original_loss": 4.424176216125488, + "objective/train/theoretical_loss": 3.73860914472057, + "objective/train/tokens_used": 779878400, + "objective/train/unlikelihood_loss": 0.10725219547748566, + "theoretical_loss": 3.73860914472057, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038569709127382146, + "loss": 1.5523, + "theoretical_loss": 3.73860914472057, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856770310932798, + "loss": 1.6122, + "theoretical_loss": 3.738546576827865, + "tokens_seen": 780009472 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038565697091273824, + "loss": 1.5617, + "theoretical_loss": 3.7384840223914346, + "tokens_seen": 780140544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856369107321966, + "loss": 1.305, + "theoretical_loss": 3.738421481406125, + "tokens_seen": 780271616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038561685055165496, + "loss": 1.3146, + "theoretical_loss": 3.7383589538667845, + "tokens_seen": 780402688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855967903711133, + "loss": 1.4887, + "theoretical_loss": 3.7382964397682654, + "tokens_seen": 780533760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855767301905717, + "loss": 1.3431, + "theoretical_loss": 3.7382339391054216, + "tokens_seen": 780664832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855566700100301, + "loss": 1.4263, + "theoretical_loss": 3.738171451873111, + "tokens_seen": 780795904 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038553660982948846, + "loss": 1.4549, + "theoretical_loss": 3.7381089780661925, + "tokens_seen": 780926976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855165496489468, + "loss": 1.3453, + "theoretical_loss": 3.73804651767953, + "tokens_seen": 781058048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854964894684052, + "loss": 1.4233, + "theoretical_loss": 3.7379840707079897, + "tokens_seen": 781189120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854764292878636, + "loss": 1.399, + "theoretical_loss": 3.7379216371464388, + "tokens_seen": 781320192 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038545636910732197, + "loss": 1.4778, + "theoretical_loss": 3.73785921698975, + "tokens_seen": 781451264 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4082355797290802, + "objective/train/avg_token_score": 0.016537794843316078, + "objective/train/docs_used": 450726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3863980770111084, + "objective/train/original_loss": 4.011595249176025, + "objective/train/theoretical_loss": 3.7378280119366267, + "objective/train/tokens_used": 781516800, + "objective/train/unlikelihood_loss": 0.15198664367198944, + "theoretical_loss": 3.7378280119366267, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038543630892678033, + "loss": 1.4996, + "theoretical_loss": 3.7377968102327968, + "tokens_seen": 781582336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854162487462387, + "loss": 1.4697, + "theoretical_loss": 3.737734416870457, + "tokens_seen": 781713408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038539618856569705, + "loss": 1.4401, + "theoretical_loss": 3.7376720368976093, + "tokens_seen": 781844480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537612838515547, + "loss": 1.5488, + "theoretical_loss": 3.7376096703091375, + "tokens_seen": 781975552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038535606820461383, + "loss": 1.423, + "theoretical_loss": 3.7375473170999265, + "tokens_seen": 782106624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853360080240722, + "loss": 1.474, + "theoretical_loss": 3.737484977264865, + "tokens_seen": 782237696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038531594784353056, + "loss": 1.5602, + "theoretical_loss": 3.7374226507988437, + "tokens_seen": 782368768 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385295887662989, + "loss": 1.5098, + "theoretical_loss": 3.7373603376967566, + "tokens_seen": 782499840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038527582748244734, + "loss": 1.3475, + "theoretical_loss": 3.7372980379535004, + "tokens_seen": 782630912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852557673019057, + "loss": 1.4533, + "theoretical_loss": 3.7372357515639747, + "tokens_seen": 782761984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038523570712136406, + "loss": 1.5165, + "theoretical_loss": 3.7371734785230823, + "tokens_seen": 782893056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852156469408225, + "loss": 1.4711, + "theoretical_loss": 3.737111218825728, + "tokens_seen": 783024128 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4901026487350464, + "objective/train/avg_token_score": 0.0032358053140342236, + "objective/train/docs_used": 451239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6045764684677124, + "objective/train/original_loss": 3.5582377910614014, + "objective/train/theoretical_loss": 3.7370489724668197, + "objective/train/tokens_used": 783155200, + "objective/train/unlikelihood_loss": 0.24653176963329315, + "theoretical_loss": 3.7370489724668197, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038519558676028084, + "loss": 1.4989, + "theoretical_loss": 3.7370489724668197, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851755265797392, + "loss": 1.2782, + "theoretical_loss": 3.7369867394412677, + "tokens_seen": 783286272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038515546639919756, + "loss": 1.6008, + "theoretical_loss": 3.736924519743986, + "tokens_seen": 783417344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851354062186559, + "loss": 1.5741, + "theoretical_loss": 3.7368623133698913, + "tokens_seen": 783548416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851153460381144, + "loss": 1.4529, + "theoretical_loss": 3.7368001203139016, + "tokens_seen": 783679488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038509528585757276, + "loss": 1.4992, + "theoretical_loss": 3.73673794057094, + "tokens_seen": 783810560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850752256770311, + "loss": 1.3342, + "theoretical_loss": 3.7366757741359304, + "tokens_seen": 783941632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850551654964895, + "loss": 1.4348, + "theoretical_loss": 3.7366136210038006, + "tokens_seen": 784072704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850351053159479, + "loss": 1.5168, + "theoretical_loss": 3.73655148116948, + "tokens_seen": 784203776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501504513540626, + "loss": 1.4606, + "theoretical_loss": 3.736489354627903, + "tokens_seen": 784334848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849949849548646, + "loss": 1.4214, + "theoretical_loss": 3.7364272413740047, + "tokens_seen": 784465920 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384974924774323, + "loss": 1.5889, + "theoretical_loss": 3.7363651414027235, + "tokens_seen": 784596992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038495486459378135, + "loss": 1.3696, + "theoretical_loss": 3.7363030547090013, + "tokens_seen": 784728064 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4292522072792053, + "objective/train/avg_token_score": 0.010085192508995533, + "objective/train/docs_used": 452482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2175325155258179, + "objective/train/original_loss": 3.369680166244507, + "objective/train/theoretical_loss": 3.736272016339644, + "objective/train/tokens_used": 784793600, + "objective/train/unlikelihood_loss": 0.1560431718826294, + "theoretical_loss": 3.736272016339644, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038493480441323977, + "loss": 1.2764, + "theoretical_loss": 3.7362409812877813, + "tokens_seen": 784859136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038491474423269813, + "loss": 1.4882, + "theoretical_loss": 3.736178921134011, + "tokens_seen": 784990208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848946840521565, + "loss": 1.5061, + "theoretical_loss": 3.73611687424264, + "tokens_seen": 785121280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038487462387161485, + "loss": 1.4549, + "theoretical_loss": 3.736054840608621, + "tokens_seen": 785252352 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038485456369107327, + "loss": 1.3974, + "theoretical_loss": 3.7359928202269086, + "tokens_seen": 785383424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038483450351053163, + "loss": 1.3578, + "theoretical_loss": 3.735930813092461, + "tokens_seen": 785514496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038481444332999, + "loss": 1.4857, + "theoretical_loss": 3.735868819200239, + "tokens_seen": 785645568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038479438314944836, + "loss": 1.4016, + "theoretical_loss": 3.735806838545206, + "tokens_seen": 785776640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847743229689067, + "loss": 1.3586, + "theoretical_loss": 3.735744871122328, + "tokens_seen": 785907712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038475426278836513, + "loss": 1.315, + "theoretical_loss": 3.7356829169265744, + "tokens_seen": 786038784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847342026078235, + "loss": 1.3594, + "theoretical_loss": 3.7356209759529158, + "tokens_seen": 786169856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038471414242728186, + "loss": 1.4424, + "theoretical_loss": 3.735559048196328, + "tokens_seen": 786300928 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4347507357597351, + "objective/train/avg_token_score": 0.007102999836206436, + "objective/train/docs_used": 453141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.515565037727356, + "objective/train/original_loss": 3.775101661682129, + "objective/train/theoretical_loss": 3.735497133651788, + "objective/train/tokens_used": 786432000, + "objective/train/unlikelihood_loss": 0.12996675074100494, + "theoretical_loss": 3.735497133651788, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846940822467402, + "loss": 1.4365, + "theoretical_loss": 3.735497133651788, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038467402206619864, + "loss": 1.6741, + "theoretical_loss": 3.735435232314275, + "tokens_seen": 786563072 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384653961885657, + "loss": 1.6669, + "theoretical_loss": 3.7353733441787726, + "tokens_seen": 786694144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038463390170511536, + "loss": 1.2925, + "theoretical_loss": 3.7353114692402656, + "tokens_seen": 786825216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846138415245737, + "loss": 1.3978, + "theoretical_loss": 3.7352496074937425, + "tokens_seen": 786956288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845937813440321, + "loss": 1.3188, + "theoretical_loss": 3.7351877589341944, + "tokens_seen": 787087360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845737211634905, + "loss": 1.438, + "theoretical_loss": 3.7351259235566143, + "tokens_seen": 787218432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038455366098294886, + "loss": 1.5509, + "theoretical_loss": 3.7350641013559986, + "tokens_seen": 787349504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038453360080240723, + "loss": 1.3934, + "theoretical_loss": 3.7350022923273474, + "tokens_seen": 787480576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845135406218656, + "loss": 1.4008, + "theoretical_loss": 3.7349404964656614, + "tokens_seen": 787611648 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384493480441324, + "loss": 1.3394, + "theoretical_loss": 3.734878713765946, + "tokens_seen": 787742720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038447342026078237, + "loss": 1.5279, + "theoretical_loss": 3.7348169442232084, + "tokens_seen": 787873792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038445336008024073, + "loss": 1.3557, + "theoretical_loss": 3.7347551878324587, + "tokens_seen": 788004864 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.44727516174316406, + "objective/train/avg_token_score": 0.00864383950829506, + "objective/train/docs_used": 453953, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5525766611099243, + "objective/train/original_loss": 3.659341812133789, + "objective/train/theoretical_loss": 3.73472431456752, + "objective/train/tokens_used": 788070400, + "objective/train/unlikelihood_loss": 0.17406408488750458, + "theoretical_loss": 3.73472431456752, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844332998996991, + "loss": 1.4251, + "theoretical_loss": 3.7346934445887086, + "tokens_seen": 788135936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038441323971915745, + "loss": 1.2741, + "theoretical_loss": 3.7346317144869747, + "tokens_seen": 788267008 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038439317953861587, + "loss": 1.4061, + "theoretical_loss": 3.734569997522275, + "tokens_seen": 788398080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038437311935807423, + "loss": 1.3781, + "theoretical_loss": 3.73450829368963, + "tokens_seen": 788529152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843530591775326, + "loss": 1.5308, + "theoretical_loss": 3.7344466029840637, + "tokens_seen": 788660224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038433299899699096, + "loss": 1.5306, + "theoretical_loss": 3.7343849254006027, + "tokens_seen": 788791296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843129388164494, + "loss": 1.4465, + "theoretical_loss": 3.734323260934275, + "tokens_seen": 788922368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038429287863590774, + "loss": 1.5448, + "theoretical_loss": 3.7342616095801136, + "tokens_seen": 789053440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842728184553661, + "loss": 1.5827, + "theoretical_loss": 3.734199971333152, + "tokens_seen": 789184512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038425275827482446, + "loss": 1.4759, + "theoretical_loss": 3.734138346188428, + "tokens_seen": 789315584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842326980942829, + "loss": 1.4533, + "theoretical_loss": 3.734076734140981, + "tokens_seen": 789446656 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038421263791374124, + "loss": 1.5141, + "theoretical_loss": 3.7340151351858535, + "tokens_seen": 789577728 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4547898471355438, + "objective/train/avg_token_score": 0.01577068865299225, + "objective/train/docs_used": 455438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5932849645614624, + "objective/train/original_loss": 3.8855347633361816, + "objective/train/theoretical_loss": 3.733953549318091, + "objective/train/tokens_used": 789708800, + "objective/train/unlikelihood_loss": 0.19570748507976532, + "theoretical_loss": 3.733953549318091, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841925777331996, + "loss": 1.5437, + "theoretical_loss": 3.733953549318091, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038417251755265796, + "loss": 1.3321, + "theoretical_loss": 3.733891976532742, + "tokens_seen": 789839872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841524573721163, + "loss": 1.4373, + "theoretical_loss": 3.7338304168248566, + "tokens_seen": 789970944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038413239719157474, + "loss": 1.6041, + "theoretical_loss": 3.733768870189488, + "tokens_seen": 790102016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841123370110331, + "loss": 1.4182, + "theoretical_loss": 3.733707336621692, + "tokens_seen": 790233088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038409227683049147, + "loss": 1.3995, + "theoretical_loss": 3.7336458161165287, + "tokens_seen": 790364160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038407221664994983, + "loss": 1.5121, + "theoretical_loss": 3.7335843086690583, + "tokens_seen": 790495232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038405215646940825, + "loss": 1.4567, + "theoretical_loss": 3.733522814274345, + "tokens_seen": 790626304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840320962888666, + "loss": 1.5182, + "theoretical_loss": 3.733461332927456, + "tokens_seen": 790757376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038401203610832497, + "loss": 1.544, + "theoretical_loss": 3.7333998646234603, + "tokens_seen": 790888448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038399197592778333, + "loss": 1.3909, + "theoretical_loss": 3.7333384093574304, + "tokens_seen": 791019520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839719157472417, + "loss": 1.4824, + "theoretical_loss": 3.733276967124441, + "tokens_seen": 791150592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839518555667001, + "loss": 1.209, + "theoretical_loss": 3.7332155379195697, + "tokens_seen": 791281664 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4156891405582428, + "objective/train/avg_token_score": 0.020018110051751137, + "objective/train/docs_used": 456184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4158743619918823, + "objective/train/original_loss": 3.812213182449341, + "objective/train/theoretical_loss": 3.733184828201141, + "objective/train/tokens_used": 791347200, + "objective/train/unlikelihood_loss": 0.18388943374156952, + "theoretical_loss": 3.733184828201141, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839317953861585, + "loss": 1.2755, + "theoretical_loss": 3.733154121737897, + "tokens_seen": 791412736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038391173520561684, + "loss": 1.3489, + "theoretical_loss": 3.733092718574505, + "tokens_seen": 791543808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838916750250752, + "loss": 1.5518, + "theoretical_loss": 3.7330313284244796, + "tokens_seen": 791674880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838716148445336, + "loss": 1.3141, + "theoretical_loss": 3.732969951282909, + "tokens_seen": 791805952 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383851554663992, + "loss": 1.3247, + "theoretical_loss": 3.732908587144884, + "tokens_seen": 791937024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038383149448345034, + "loss": 1.3224, + "theoretical_loss": 3.7328472360054983, + "tokens_seen": 792068096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838114343029087, + "loss": 1.5871, + "theoretical_loss": 3.7327858978598476, + "tokens_seen": 792199168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038379137412236706, + "loss": 1.3346, + "theoretical_loss": 3.732724572703032, + "tokens_seen": 792330240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837713139418255, + "loss": 1.4265, + "theoretical_loss": 3.732663260530151, + "tokens_seen": 792461312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038375125376128384, + "loss": 1.5549, + "theoretical_loss": 3.7326019613363104, + "tokens_seen": 792592384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837311935807422, + "loss": 1.5179, + "theoretical_loss": 3.7325406751166166, + "tokens_seen": 792723456 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038371113340020057, + "loss": 1.5086, + "theoretical_loss": 3.7324794018661787, + "tokens_seen": 792854528 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.423081636428833, + "objective/train/avg_token_score": 0.021916478872299194, + "objective/train/docs_used": 457386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5908643007278442, + "objective/train/original_loss": 3.9085986614227295, + "objective/train/theoretical_loss": 3.7324181415801094, + "objective/train/tokens_used": 792985600, + "objective/train/unlikelihood_loss": 0.1857742965221405, + "theoretical_loss": 3.7324181415801094, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383691073219659, + "loss": 1.4757, + "theoretical_loss": 3.7324181415801094, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038367101303911735, + "loss": 1.4025, + "theoretical_loss": 3.7323568942535226, + "tokens_seen": 793116672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836509528585757, + "loss": 1.4426, + "theoretical_loss": 3.7322956598815367, + "tokens_seen": 793247744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836308926780341, + "loss": 1.5116, + "theoretical_loss": 3.7322344384592707, + "tokens_seen": 793378816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836108324974925, + "loss": 1.508, + "theoretical_loss": 3.7321732299818486, + "tokens_seen": 793509888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835907723169509, + "loss": 1.5377, + "theoretical_loss": 3.7321120344443948, + "tokens_seen": 793640960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038357071213640927, + "loss": 1.4118, + "theoretical_loss": 3.732050851842038, + "tokens_seen": 793772032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038355065195586763, + "loss": 1.4403, + "theoretical_loss": 3.7319896821699077, + "tokens_seen": 793903104 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383530591775326, + "loss": 1.4266, + "theoretical_loss": 3.7319285254231382, + "tokens_seen": 794034176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835105315947844, + "loss": 1.414, + "theoretical_loss": 3.7318673815968655, + "tokens_seen": 794165248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038349047141424277, + "loss": 1.4088, + "theoretical_loss": 3.731806250686227, + "tokens_seen": 794296320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038347041123370113, + "loss": 1.474, + "theoretical_loss": 3.731745132686365, + "tokens_seen": 794427392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834503510531595, + "loss": 1.3775, + "theoretical_loss": 3.7316840275924226, + "tokens_seen": 794558464 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.43016862869262695, + "objective/train/avg_token_score": 0.009460315108299255, + "objective/train/docs_used": 458050, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4603745937347412, + "objective/train/original_loss": 3.892380475997925, + "objective/train/theoretical_loss": 3.7316534798836543, + "objective/train/tokens_used": 794624000, + "objective/train/unlikelihood_loss": 0.15715791285037994, + "theoretical_loss": 3.7316534798836543, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038343029087261786, + "loss": 1.2388, + "theoretical_loss": 3.731622935399547, + "tokens_seen": 794689536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038341023069207627, + "loss": 1.5007, + "theoretical_loss": 3.7315618561028865, + "tokens_seen": 794820608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038339017051153463, + "loss": 1.425, + "theoretical_loss": 3.7315007896975927, + "tokens_seen": 794951680 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383370110330993, + "loss": 1.3353, + "theoretical_loss": 3.73143973617882, + "tokens_seen": 795082752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038335005015045136, + "loss": 1.4108, + "theoretical_loss": 3.731378695541726, + "tokens_seen": 795213824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833299899699098, + "loss": 1.4819, + "theoretical_loss": 3.73131766778147, + "tokens_seen": 795344896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038330992978936814, + "loss": 1.52, + "theoretical_loss": 3.7312566528932134, + "tokens_seen": 795475968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832898696088265, + "loss": 1.4325, + "theoretical_loss": 3.731195650872121, + "tokens_seen": 795607040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038326980942828486, + "loss": 1.3616, + "theoretical_loss": 3.7311346617133614, + "tokens_seen": 795738112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832497492477433, + "loss": 1.5254, + "theoretical_loss": 3.731073685412104, + "tokens_seen": 795869184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038322968906720164, + "loss": 1.4147, + "theoretical_loss": 3.7310127219635207, + "tokens_seen": 796000256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038320962888666, + "loss": 1.3189, + "theoretical_loss": 3.730951771362787, + "tokens_seen": 796131328 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.49120235443115234, + "objective/train/avg_token_score": 0.004814146086573601, + "objective/train/docs_used": 459137, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7349704504013062, + "objective/train/original_loss": 3.7150371074676514, + "objective/train/theoretical_loss": 3.7308908336050814, + "objective/train/tokens_used": 796262400, + "objective/train/unlikelihood_loss": 0.1845131814479828, + "theoretical_loss": 3.7308908336050814, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038318956870611836, + "loss": 1.4777, + "theoretical_loss": 3.7308908336050814, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831695085255767, + "loss": 1.3352, + "theoretical_loss": 3.7308299086855836, + "tokens_seen": 796393472 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038314944834503514, + "loss": 1.4122, + "theoretical_loss": 3.730768996599477, + "tokens_seen": 796524544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831293881644935, + "loss": 1.3427, + "theoretical_loss": 3.730708097341947, + "tokens_seen": 796655616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038310932798395187, + "loss": 1.3911, + "theoretical_loss": 3.730647210908182, + "tokens_seen": 796786688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038308926780341023, + "loss": 1.3942, + "theoretical_loss": 3.7305863372933725, + "tokens_seen": 796917760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038306920762286865, + "loss": 1.5169, + "theoretical_loss": 3.730525476492712, + "tokens_seen": 797048832 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383049147442327, + "loss": 1.491, + "theoretical_loss": 3.7304646285013963, + "tokens_seen": 797179904 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038302908726178537, + "loss": 1.4388, + "theoretical_loss": 3.7304037933146246, + "tokens_seen": 797310976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038300902708124373, + "loss": 1.3891, + "theoretical_loss": 3.7303429709275973, + "tokens_seen": 797442048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829889669007021, + "loss": 1.3085, + "theoretical_loss": 3.7302821613355186, + "tokens_seen": 797573120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829689067201605, + "loss": 1.4203, + "theoretical_loss": 3.7302213645335947, + "tokens_seen": 797704192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829488465396189, + "loss": 1.453, + "theoretical_loss": 3.730160580517035, + "tokens_seen": 797835264 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4134897291660309, + "objective/train/avg_token_score": 0.02109762839972973, + "objective/train/docs_used": 459607, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4717297554016113, + "objective/train/original_loss": 4.313312530517578, + "objective/train/theoretical_loss": 3.73013019330177, + "objective/train/tokens_used": 797900800, + "objective/train/unlikelihood_loss": 0.13565292954444885, + "theoretical_loss": 3.73013019330177, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292878635907724, + "loss": 1.476, + "theoretical_loss": 3.7300998092810502, + "tokens_seen": 797966336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829087261785356, + "loss": 1.3511, + "theoretical_loss": 3.730039050820855, + "tokens_seen": 798097408 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382888665997994, + "loss": 1.5404, + "theoretical_loss": 3.729978305131666, + "tokens_seen": 798228480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828686058174524, + "loss": 1.4093, + "theoretical_loss": 3.7299175722087026, + "tokens_seen": 798359552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284854563691074, + "loss": 1.5101, + "theoretical_loss": 3.7298568520471855, + "tokens_seen": 798490624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828284854563691, + "loss": 1.5057, + "theoretical_loss": 3.72979614464234, + "tokens_seen": 798621696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038280842527582746, + "loss": 1.356, + "theoretical_loss": 3.729735449989394, + "tokens_seen": 798752768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827883650952859, + "loss": 1.3104, + "theoretical_loss": 3.7296747680835747, + "tokens_seen": 798883840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038276830491474424, + "loss": 1.4527, + "theoretical_loss": 3.7296140989201163, + "tokens_seen": 799014912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827482447342026, + "loss": 1.4236, + "theoretical_loss": 3.729553442494252, + "tokens_seen": 799145984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038272818455366097, + "loss": 1.4974, + "theoretical_loss": 3.7294927988012203, + "tokens_seen": 799277056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827081243731194, + "loss": 1.3163, + "theoretical_loss": 3.7294321678362596, + "tokens_seen": 799408128 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.36100929975509644, + "objective/train/avg_token_score": 0.008971652016043663, + "objective/train/docs_used": 460885, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2825909852981567, + "objective/train/original_loss": 3.9460840225219727, + "objective/train/theoretical_loss": 3.729371549594614, + "objective/train/tokens_used": 799539200, + "objective/train/unlikelihood_loss": 0.1946743130683899, + "theoretical_loss": 3.729371549594614, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038268806419257775, + "loss": 1.3964, + "theoretical_loss": 3.729371549594614, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826680040120361, + "loss": 1.5455, + "theoretical_loss": 3.7293109440715266, + "tokens_seen": 799670272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038264794383149447, + "loss": 1.333, + "theoretical_loss": 3.7292503512622455, + "tokens_seen": 799801344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038262788365095283, + "loss": 1.4536, + "theoretical_loss": 3.729189771162021, + "tokens_seen": 799932416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038260782347041125, + "loss": 1.4315, + "theoretical_loss": 3.7291292037661057, + "tokens_seen": 800063488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825877632898696, + "loss": 1.3332, + "theoretical_loss": 3.729068649069754, + "tokens_seen": 800194560 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382567703109328, + "loss": 1.3829, + "theoretical_loss": 3.729008107068225, + "tokens_seen": 800325632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038254764292878634, + "loss": 1.3806, + "theoretical_loss": 3.7289475777567773, + "tokens_seen": 800456704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038252758274824475, + "loss": 1.3906, + "theoretical_loss": 3.7288870611306746, + "tokens_seen": 800587776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825075225677031, + "loss": 1.3171, + "theoretical_loss": 3.7288265571851817, + "tokens_seen": 800718848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824874623871615, + "loss": 1.5165, + "theoretical_loss": 3.7287660659155666, + "tokens_seen": 800849920 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038246740220661984, + "loss": 1.3781, + "theoretical_loss": 3.7287055873171, + "tokens_seen": 800980992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824473420260782, + "loss": 1.3812, + "theoretical_loss": 3.7286451213850547, + "tokens_seen": 801112064 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.37212854623794556, + "objective/train/avg_token_score": 0.011794453486800194, + "objective/train/docs_used": 461399, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2390127182006836, + "objective/train/original_loss": 3.6054813861846924, + "objective/train/theoretical_loss": 3.7286148931674634, + "objective/train/tokens_used": 801177600, + "objective/train/unlikelihood_loss": 0.1948477327823639, + "theoretical_loss": 3.7286148931674634, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824272818455366, + "loss": 1.3996, + "theoretical_loss": 3.728584668114706, + "tokens_seen": 801243136 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382407221664995, + "loss": 1.1398, + "theoretical_loss": 3.7285242275013317, + "tokens_seen": 801374208 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038238716148445334, + "loss": 1.5253, + "theoretical_loss": 3.7284637995402123, + "tokens_seen": 801505280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823671013039117, + "loss": 1.3172, + "theoretical_loss": 3.728403384226632, + "tokens_seen": 801636352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823470411233701, + "loss": 1.3018, + "theoretical_loss": 3.728342981555875, + "tokens_seen": 801767424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823269809428285, + "loss": 1.5004, + "theoretical_loss": 3.7282825915232296, + "tokens_seen": 801898496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038230692076228685, + "loss": 1.4799, + "theoretical_loss": 3.7282222141239867, + "tokens_seen": 802029568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822868605817452, + "loss": 1.2514, + "theoretical_loss": 3.72816184935344, + "tokens_seen": 802160640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038226680040120357, + "loss": 1.3638, + "theoretical_loss": 3.728101497206884, + "tokens_seen": 802291712 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382246740220662, + "loss": 1.3546, + "theoretical_loss": 3.7280411576796184, + "tokens_seen": 802422784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038222668004012035, + "loss": 1.3688, + "theoretical_loss": 3.727980830766943, + "tokens_seen": 802553856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822066198595787, + "loss": 1.5452, + "theoretical_loss": 3.7279205164641604, + "tokens_seen": 802684928 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.3245967626571655, + "objective/train/avg_token_score": 0.0183734018355608, + "objective/train/docs_used": 462084, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.175994634628296, + "objective/train/original_loss": 4.296977996826172, + "objective/train/theoretical_loss": 3.7278602147665776, + "objective/train/tokens_used": 802816000, + "objective/train/unlikelihood_loss": 0.15977226197719574, + "theoretical_loss": 3.7278602147665776, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821865596790371, + "loss": 1.3075, + "theoretical_loss": 3.7278602147665776, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821664994984955, + "loss": 1.5751, + "theoretical_loss": 3.7277999256695025, + "tokens_seen": 802947072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038214643931795385, + "loss": 1.4762, + "theoretical_loss": 3.727739649168245, + "tokens_seen": 803078144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212637913741227, + "loss": 1.4331, + "theoretical_loss": 3.7276793852581203, + "tokens_seen": 803209216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038210631895687063, + "loss": 1.3274, + "theoretical_loss": 3.7276191339344424, + "tokens_seen": 803340288 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038208625877632905, + "loss": 1.3923, + "theoretical_loss": 3.72755889519253, + "tokens_seen": 803471360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820661985957874, + "loss": 1.4168, + "theoretical_loss": 3.7274986690277045, + "tokens_seen": 803602432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204613841524577, + "loss": 1.5216, + "theoretical_loss": 3.727438455435289, + "tokens_seen": 803733504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038202607823470413, + "loss": 1.4053, + "theoretical_loss": 3.727378254410609, + "tokens_seen": 803864576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820060180541625, + "loss": 1.2178, + "theoretical_loss": 3.7273180659489933, + "tokens_seen": 803995648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819859578736209, + "loss": 1.5903, + "theoretical_loss": 3.727257890045772, + "tokens_seen": 804126720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819658976930793, + "loss": 1.3042, + "theoretical_loss": 3.7271977266962786, + "tokens_seen": 804257792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038194583751253764, + "loss": 1.5218, + "theoretical_loss": 3.7271375758958496, + "tokens_seen": 804388864 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.492118775844574, + "objective/train/avg_token_score": 0.009769570082426071, + "objective/train/docs_used": 463313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8922295570373535, + "objective/train/original_loss": 3.8713629245758057, + "objective/train/theoretical_loss": 3.7271075052000775, + "objective/train/tokens_used": 804454400, + "objective/train/unlikelihood_loss": 0.21652424335479736, + "theoretical_loss": 3.7271075052000775, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381925777331996, + "loss": 1.4401, + "theoretical_loss": 3.7270774376398226, + "tokens_seen": 804519936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819057171514544, + "loss": 1.3122, + "theoretical_loss": 3.727017311923539, + "tokens_seen": 804651008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818856569709128, + "loss": 1.2962, + "theoretical_loss": 3.7269571987423413, + "tokens_seen": 804782080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038186559679037114, + "loss": 1.3543, + "theoretical_loss": 3.726897098091576, + "tokens_seen": 804913152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818455366098295, + "loss": 1.2946, + "theoretical_loss": 3.7268370099665904, + "tokens_seen": 805044224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038182547642928786, + "loss": 1.5173, + "theoretical_loss": 3.7267769343627357, + "tokens_seen": 805175296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818054162487463, + "loss": 1.3396, + "theoretical_loss": 3.726716871275366, + "tokens_seen": 805306368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038178535606820464, + "loss": 1.4702, + "theoretical_loss": 3.726656820699836, + "tokens_seen": 805437440 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381765295887663, + "loss": 1.3396, + "theoretical_loss": 3.726596782631504, + "tokens_seen": 805568512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038174523570712137, + "loss": 1.498, + "theoretical_loss": 3.7265367570657304, + "tokens_seen": 805699584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817251755265798, + "loss": 1.306, + "theoretical_loss": 3.726476743997879, + "tokens_seen": 805830656 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038170511534603815, + "loss": 1.4445, + "theoretical_loss": 3.7264167434233153, + "tokens_seen": 805961728 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4856427311897278, + "objective/train/avg_token_score": 0.013956292532384396, + "objective/train/docs_used": 464591, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4471896886825562, + "objective/train/original_loss": 3.5620546340942383, + "objective/train/theoretical_loss": 3.726356755337407, + "objective/train/tokens_used": 806092800, + "objective/train/unlikelihood_loss": 0.16739284992218018, + "theoretical_loss": 3.726356755337407, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816850551654965, + "loss": 1.4348, + "theoretical_loss": 3.726356755337407, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038166499498495487, + "loss": 1.4298, + "theoretical_loss": 3.7262967797355246, + "tokens_seen": 806223872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038164493480441323, + "loss": 1.5744, + "theoretical_loss": 3.7262368166130413, + "tokens_seen": 806354944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038162487462387165, + "loss": 1.3223, + "theoretical_loss": 3.726176865965333, + "tokens_seen": 806486016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038160481444333, + "loss": 1.3632, + "theoretical_loss": 3.7261169277877766, + "tokens_seen": 806617088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815847542627884, + "loss": 1.3174, + "theoretical_loss": 3.7260570020757537, + "tokens_seen": 806748160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038156469408224674, + "loss": 1.4876, + "theoretical_loss": 3.7259970888246468, + "tokens_seen": 806879232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038154463390170515, + "loss": 1.3061, + "theoretical_loss": 3.7259371880298406, + "tokens_seen": 807010304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815245737211635, + "loss": 1.4325, + "theoretical_loss": 3.725877299686723, + "tokens_seen": 807141376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815045135406219, + "loss": 1.4063, + "theoretical_loss": 3.725817423790685, + "tokens_seen": 807272448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038148445336008024, + "loss": 1.3214, + "theoretical_loss": 3.7257575603371187, + "tokens_seen": 807403520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814643931795386, + "loss": 1.5166, + "theoretical_loss": 3.725697709321419, + "tokens_seen": 807534592 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381444332998997, + "loss": 1.5269, + "theoretical_loss": 3.7256378707389843, + "tokens_seen": 807665664 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.46340420842170715, + "objective/train/avg_token_score": 0.012106151320040226, + "objective/train/docs_used": 465345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6766234636306763, + "objective/train/original_loss": 3.999908208847046, + "objective/train/theoretical_loss": 3.725607956108804, + "objective/train/tokens_used": 807731200, + "objective/train/unlikelihood_loss": 0.19688478112220764, + "theoretical_loss": 3.725607956108804, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814242728184554, + "loss": 1.3714, + "theoretical_loss": 3.7255780445852142, + "tokens_seen": 807796736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038140421263791374, + "loss": 1.4905, + "theoretical_loss": 3.725518230855511, + "tokens_seen": 807927808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813841524573721, + "loss": 1.3918, + "theoretical_loss": 3.72545842954528, + "tokens_seen": 808058880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813640922768305, + "loss": 1.3092, + "theoretical_loss": 3.725398640649929, + "tokens_seen": 808189952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813440320962889, + "loss": 1.2439, + "theoretical_loss": 3.7253388641648666, + "tokens_seen": 808321024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038132397191574725, + "loss": 1.5, + "theoretical_loss": 3.7252791000855057, + "tokens_seen": 808452096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813039117352056, + "loss": 1.4178, + "theoretical_loss": 3.7252193484072613, + "tokens_seen": 808583168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038128385155466397, + "loss": 1.4165, + "theoretical_loss": 3.72515960912555, + "tokens_seen": 808714240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812637913741224, + "loss": 1.4801, + "theoretical_loss": 3.725099882235792, + "tokens_seen": 808845312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038124373119358075, + "loss": 1.3538, + "theoretical_loss": 3.7250401677334093, + "tokens_seen": 808976384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812236710130391, + "loss": 1.4561, + "theoretical_loss": 3.7249804656138257, + "tokens_seen": 809107456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812036108324975, + "loss": 1.3322, + "theoretical_loss": 3.7249207758724685, + "tokens_seen": 809238528 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.5436217188835144, + "objective/train/avg_token_score": 0.005821581464260817, + "objective/train/docs_used": 466409, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.86619234085083, + "objective/train/original_loss": 3.83589768409729, + "objective/train/theoretical_loss": 3.724861098504767, + "objective/train/tokens_used": 809369600, + "objective/train/unlikelihood_loss": 0.13873565196990967, + "theoretical_loss": 3.724861098504767, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811835506519559, + "loss": 1.3247, + "theoretical_loss": 3.724861098504767, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038116349047141425, + "loss": 1.4625, + "theoretical_loss": 3.724801433506153, + "tokens_seen": 809500672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811434302908726, + "loss": 1.5138, + "theoretical_loss": 3.7247417808720606, + "tokens_seen": 809631744 + }, + { + "epoch": 0.25, + "learning_rate": 0.000381123370110331, + "loss": 1.3015, + "theoretical_loss": 3.7246821405979267, + "tokens_seen": 809762816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038110330992978934, + "loss": 1.3544, + "theoretical_loss": 3.724622512679189, + "tokens_seen": 809893888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038108324974924776, + "loss": 1.4899, + "theoretical_loss": 3.7245628971112907, + "tokens_seen": 810024960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810631895687061, + "loss": 1.3084, + "theoretical_loss": 3.7245032938896743, + "tokens_seen": 810156032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810431293881645, + "loss": 1.3926, + "theoretical_loss": 3.724443703009787, + "tokens_seen": 810287104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038102306920762284, + "loss": 1.3782, + "theoretical_loss": 3.7243841244670772, + "tokens_seen": 810418176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038100300902708126, + "loss": 1.3412, + "theoretical_loss": 3.7243245582569955, + "tokens_seen": 810549248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809829488465396, + "loss": 1.6528, + "theoretical_loss": 3.7242650043749963, + "tokens_seen": 810680320 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380962888665998, + "loss": 1.4059, + "theoretical_loss": 3.7242054628165344, + "tokens_seen": 810811392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038094282848545635, + "loss": 1.3871, + "theoretical_loss": 3.724145933577069, + "tokens_seen": 810942464 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.4060361683368683, + "objective/train/avg_token_score": 0.017316017299890518, + "objective/train/docs_used": 466961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3099706172943115, + "objective/train/original_loss": 3.5451700687408447, + "objective/train/theoretical_loss": 3.724116173575542, + "objective/train/tokens_used": 811008000, + "objective/train/unlikelihood_loss": 0.20387853682041168, + "theoretical_loss": 3.724116173575542, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038092276830491476, + "loss": 1.3292, + "theoretical_loss": 3.724086416652061, + "tokens_seen": 811073536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809027081243731, + "loss": 1.4599, + "theoretical_loss": 3.7240269120369733, + "tokens_seen": 811204608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808826479438315, + "loss": 1.4299, + "theoretical_loss": 3.7239674197272707, + "tokens_seen": 811335680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038086258776328985, + "loss": 1.6177, + "theoretical_loss": 3.723907939718422, + "tokens_seen": 811466752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808425275827482, + "loss": 1.3204, + "theoretical_loss": 3.7238484720058973, + "tokens_seen": 811597824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808224674022066, + "loss": 1.3467, + "theoretical_loss": 3.7237890165851693, + "tokens_seen": 811728896 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380802407221665, + "loss": 1.3454, + "theoretical_loss": 3.7237295734517133, + "tokens_seen": 811859968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038078234704112335, + "loss": 1.3753, + "theoretical_loss": 3.723670142601007, + "tokens_seen": 811991040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807622868605817, + "loss": 1.4679, + "theoretical_loss": 3.7236107240285294, + "tokens_seen": 812122112 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038074222668004013, + "loss": 1.4012, + "theoretical_loss": 3.723551317729764, + "tokens_seen": 812253184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807221664994985, + "loss": 1.4764, + "theoretical_loss": 3.7234919237001947, + "tokens_seen": 812384256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038070210631895685, + "loss": 1.5589, + "theoretical_loss": 3.72343254193531, + "tokens_seen": 812515328 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.3621090054512024, + "objective/train/avg_token_score": 0.01322729978710413, + "objective/train/docs_used": 468484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1689765453338623, + "objective/train/original_loss": 3.74946928024292, + "objective/train/theoretical_loss": 3.7233731724305974, + "objective/train/tokens_used": 812646400, + "objective/train/unlikelihood_loss": 0.23567281663417816, + "theoretical_loss": 3.7233731724305974, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806820461384152, + "loss": 1.2581, + "theoretical_loss": 3.7233731724305974, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806619859578736, + "loss": 1.6081, + "theoretical_loss": 3.72331381518155, + "tokens_seen": 812777472 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380641925777332, + "loss": 1.4535, + "theoretical_loss": 3.7232544701836616, + "tokens_seen": 812908544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806218655967904, + "loss": 1.3905, + "theoretical_loss": 3.723195137432429, + "tokens_seen": 813039616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806018054162488, + "loss": 1.5762, + "theoretical_loss": 3.723135816923352, + "tokens_seen": 813170688 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038058174523570714, + "loss": 1.3216, + "theoretical_loss": 3.7230765086519306, + "tokens_seen": 813301760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038056168505516555, + "loss": 1.5828, + "theoretical_loss": 3.7230172126136694, + "tokens_seen": 813432832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805416248746239, + "loss": 1.5325, + "theoretical_loss": 3.7229579288040746, + "tokens_seen": 813563904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805215646940823, + "loss": 1.5038, + "theoretical_loss": 3.7228986572186553, + "tokens_seen": 813694976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038050150451354064, + "loss": 1.6257, + "theoretical_loss": 3.722839397852921, + "tokens_seen": 813826048 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380481444332999, + "loss": 1.4985, + "theoretical_loss": 3.722780150702386, + "tokens_seen": 813957120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804613841524574, + "loss": 1.4007, + "theoretical_loss": 3.722720915762565, + "tokens_seen": 814088192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804413239719158, + "loss": 1.4558, + "theoretical_loss": 3.7226616930289773, + "tokens_seen": 814219264 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.5113025307655334, + "objective/train/avg_token_score": 0.006819791626185179, + "objective/train/docs_used": 469177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7805976867675781, + "objective/train/original_loss": 3.9676969051361084, + "objective/train/theoretical_loss": 3.7226320862381206, + "objective/train/tokens_used": 814284800, + "objective/train/unlikelihood_loss": 0.11741922795772552, + "theoretical_loss": 3.7226320862381206, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042126379137414, + "loss": 1.3662, + "theoretical_loss": 3.722602482497143, + "tokens_seen": 814350336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804012036108325, + "loss": 1.3793, + "theoretical_loss": 3.7225432841625836, + "tokens_seen": 814481408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803811434302909, + "loss": 1.2445, + "theoretical_loss": 3.722484098020826, + "tokens_seen": 814612480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803610832497493, + "loss": 1.5113, + "theoretical_loss": 3.722424924067396, + "tokens_seen": 814743552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038034102306920765, + "loss": 1.418, + "theoretical_loss": 3.722365762297825, + "tokens_seen": 814874624 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380320962888666, + "loss": 1.2143, + "theoretical_loss": 3.7223066127076443, + "tokens_seen": 815005696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038030090270812437, + "loss": 1.4198, + "theoretical_loss": 3.722247475292388, + "tokens_seen": 815136768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802808425275828, + "loss": 1.4376, + "theoretical_loss": 3.7221883500475945, + "tokens_seen": 815267840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038026078234704115, + "loss": 1.3912, + "theoretical_loss": 3.7221292369688017, + "tokens_seen": 815398912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802407221664995, + "loss": 1.4005, + "theoretical_loss": 3.722070136051552, + "tokens_seen": 815529984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802206619859579, + "loss": 1.4376, + "theoretical_loss": 3.7220110472913888, + "tokens_seen": 815661056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802006018054163, + "loss": 1.2676, + "theoretical_loss": 3.7219519706838584, + "tokens_seen": 815792128 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.3723118305206299, + "objective/train/avg_token_score": 0.011704596690833569, + "objective/train/docs_used": 469961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2581655979156494, + "objective/train/original_loss": 3.6174638271331787, + "objective/train/theoretical_loss": 3.7218929062245105, + "objective/train/tokens_used": 815923200, + "objective/train/unlikelihood_loss": 0.20311835408210754, + "theoretical_loss": 3.7218929062245105, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038018054162487465, + "loss": 1.4865, + "theoretical_loss": 3.7218929062245105, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380160481444333, + "loss": 1.3383, + "theoretical_loss": 3.721833853908895, + "tokens_seen": 816054272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801404212637914, + "loss": 1.4329, + "theoretical_loss": 3.7217748137325652, + "tokens_seen": 816185344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038012036108324974, + "loss": 1.4291, + "theoretical_loss": 3.7217157856910776, + "tokens_seen": 816316416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038010030090270816, + "loss": 1.3422, + "theoretical_loss": 3.72165676977999, + "tokens_seen": 816447488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800802407221665, + "loss": 1.528, + "theoretical_loss": 3.721597765994862, + "tokens_seen": 816578560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800601805416249, + "loss": 1.4114, + "theoretical_loss": 3.7215387743312567, + "tokens_seen": 816709632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038004012036108324, + "loss": 1.3804, + "theoretical_loss": 3.72147979478474, + "tokens_seen": 816840704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038002006018054166, + "loss": 1.5583, + "theoretical_loss": 3.721420827350878, + "tokens_seen": 816971776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038, + "loss": 1.5808, + "theoretical_loss": 3.7213618720252413, + "tokens_seen": 817102848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799799398194584, + "loss": 1.4494, + "theoretical_loss": 3.721302928803402, + "tokens_seen": 817233920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995987963891675, + "loss": 1.3854, + "theoretical_loss": 3.7212439976809337, + "tokens_seen": 817364992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037993981945837516, + "loss": 1.476, + "theoretical_loss": 3.721185078653413, + "tokens_seen": 817496064 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.39125123620033264, + "objective/train/avg_token_score": 0.007498208433389664, + "objective/train/docs_used": 471203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2736456394195557, + "objective/train/original_loss": 4.151216506958008, + "objective/train/theoretical_loss": 3.7211556236738765, + "objective/train/tokens_used": 817561600, + "objective/train/unlikelihood_loss": 0.12514013051986694, + "theoretical_loss": 3.7211556236738765, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799197592778335, + "loss": 1.3359, + "theoretical_loss": 3.7211261717164197, + "tokens_seen": 817627136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798996990972919, + "loss": 1.3552, + "theoretical_loss": 3.7210672768655355, + "tokens_seen": 817758208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987963891675025, + "loss": 1.5373, + "theoretical_loss": 3.7210083940963425, + "tokens_seen": 817889280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798595787362086, + "loss": 1.3457, + "theoretical_loss": 3.7209495234044283, + "tokens_seen": 818020352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037983951855566703, + "loss": 1.3138, + "theoretical_loss": 3.7208906647853803, + "tokens_seen": 818151424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798194583751254, + "loss": 1.5323, + "theoretical_loss": 3.720831818234789, + "tokens_seen": 818282496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037979939819458375, + "loss": 1.3978, + "theoretical_loss": 3.7207729837482484, + "tokens_seen": 818413568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797793380140421, + "loss": 1.4256, + "theoretical_loss": 3.720714161321352, + "tokens_seen": 818544640 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037975927783350053, + "loss": 1.3266, + "theoretical_loss": 3.7206553509496993, + "tokens_seen": 818675712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797392176529589, + "loss": 1.4117, + "theoretical_loss": 3.720596552628889, + "tokens_seen": 818806784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037971915747241725, + "loss": 1.3436, + "theoretical_loss": 3.7205377663545236, + "tokens_seen": 818937856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796990972918756, + "loss": 1.2263, + "theoretical_loss": 3.7204789921222075, + "tokens_seen": 819068928 + }, + { + "debugging/length": 467.75, + "debugging/num_segments": 8, + "epoch": 0.25, + "objective/train/aligned_token_count": 0.41434505581855774, + "objective/train/avg_token_score": 0.014345275238156319, + "objective/train/docs_used": 471797, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4222508668899536, + "objective/train/original_loss": 3.699415445327759, + "objective/train/theoretical_loss": 3.7204202299275475, + "objective/train/tokens_used": 819200000, + "objective/train/unlikelihood_loss": 0.15635277330875397, + "theoretical_loss": 3.7204202299275475, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379679037111334, + "loss": 1.427, + "theoretical_loss": 3.7204202299275475, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796589769307924, + "loss": 1.3714, + "theoretical_loss": 3.7203614797661535, + "tokens_seen": 819331072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037963891675025076, + "loss": 1.5278, + "theoretical_loss": 3.7203027416336356, + "tokens_seen": 819462144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796188565697091, + "loss": 1.4275, + "theoretical_loss": 3.7202440155256076, + "tokens_seen": 819593216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795987963891675, + "loss": 1.3728, + "theoretical_loss": 3.720185301437687, + "tokens_seen": 819724288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795787362086259, + "loss": 1.3885, + "theoretical_loss": 3.720126599365491, + "tokens_seen": 819855360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037955867602808426, + "loss": 1.3436, + "theoretical_loss": 3.72006790930464, + "tokens_seen": 819986432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795386158475426, + "loss": 1.384, + "theoretical_loss": 3.7200092312507573, + "tokens_seen": 820117504 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379518555667001, + "loss": 1.2604, + "theoretical_loss": 3.7199505651994684, + "tokens_seen": 820248576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037949849548645935, + "loss": 1.3717, + "theoretical_loss": 3.7198919111464006, + "tokens_seen": 820379648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037947843530591776, + "loss": 1.3023, + "theoretical_loss": 3.719833269087183, + "tokens_seen": 820510720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794583751253761, + "loss": 1.435, + "theoretical_loss": 3.719774639017449, + "tokens_seen": 820641792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794383149448345, + "loss": 1.2524, + "theoretical_loss": 3.719716020932832, + "tokens_seen": 820772864 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.31830400228500366, + "objective/train/avg_token_score": 0.011386780068278313, + "objective/train/docs_used": 473281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1417335271835327, + "objective/train/original_loss": 3.751159191131592, + "objective/train/theoretical_loss": 3.719686716383579, + "objective/train/tokens_used": 820838400, + "objective/train/unlikelihood_loss": 0.20238111913204193, + "theoretical_loss": 3.719686716383579, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037941825476429285, + "loss": 1.2856, + "theoretical_loss": 3.719657414828969, + "tokens_seen": 820903936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037939819458375127, + "loss": 1.2442, + "theoretical_loss": 3.7195988207014983, + "tokens_seen": 821035008 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037937813440320963, + "loss": 1.3953, + "theoretical_loss": 3.7195402385460623, + "tokens_seen": 821166080 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379358074222668, + "loss": 1.4281, + "theoretical_loss": 3.719481668358304, + "tokens_seen": 821297152 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037933801404212635, + "loss": 1.5417, + "theoretical_loss": 3.719423110133868, + "tokens_seen": 821428224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793179538615847, + "loss": 1.4119, + "theoretical_loss": 3.719364563868404, + "tokens_seen": 821559296 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037929789368104313, + "loss": 1.4065, + "theoretical_loss": 3.719306029557562, + "tokens_seen": 821690368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792778335005015, + "loss": 1.7282, + "theoretical_loss": 3.719247507196994, + "tokens_seen": 821821440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037925777331995986, + "loss": 1.4876, + "theoretical_loss": 3.7191889967823557, + "tokens_seen": 821952512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792377131394182, + "loss": 1.3604, + "theoretical_loss": 3.7191304983093034, + "tokens_seen": 822083584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037921765295887664, + "loss": 1.445, + "theoretical_loss": 3.719072011773497, + "tokens_seen": 822214656 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379197592778335, + "loss": 1.213, + "theoretical_loss": 3.719013537170598, + "tokens_seen": 822345728 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.5243157148361206, + "objective/train/avg_token_score": 0.04041815176606178, + "objective/train/docs_used": 473825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8869761228561401, + "objective/train/original_loss": 3.7598676681518555, + "objective/train/theoretical_loss": 3.7189550744962707, + "objective/train/tokens_used": 822476800, + "objective/train/unlikelihood_loss": 0.17111940681934357, + "theoretical_loss": 3.7189550744962707, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037917753259779336, + "loss": 1.4596, + "theoretical_loss": 3.7189550744962707, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791574724172517, + "loss": 1.1405, + "theoretical_loss": 3.7188966237461814, + "tokens_seen": 822607872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791374122367101, + "loss": 1.4641, + "theoretical_loss": 3.718838184915998, + "tokens_seen": 822738944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911735205616856, + "loss": 1.3301, + "theoretical_loss": 3.718779758001392, + "tokens_seen": 822870016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790972918756269, + "loss": 1.3547, + "theoretical_loss": 3.7187213429980357, + "tokens_seen": 823001088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790772316950853, + "loss": 1.5042, + "theoretical_loss": 3.7186629399016047, + "tokens_seen": 823132160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037905717151454364, + "loss": 1.3382, + "theoretical_loss": 3.7186045487077766, + "tokens_seen": 823263232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037903711133400206, + "loss": 1.4433, + "theoretical_loss": 3.7185461694122313, + "tokens_seen": 823394304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790170511534604, + "loss": 1.4193, + "theoretical_loss": 3.7184878020106504, + "tokens_seen": 823525376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789969909729188, + "loss": 1.3668, + "theoretical_loss": 3.718429446498719, + "tokens_seen": 823656448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037897693079237715, + "loss": 1.4431, + "theoretical_loss": 3.7183711028721227, + "tokens_seen": 823787520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037895687061183556, + "loss": 1.5993, + "theoretical_loss": 3.718312771126551, + "tokens_seen": 823918592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789368104312939, + "loss": 1.6528, + "theoretical_loss": 3.718254451257695, + "tokens_seen": 824049664 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.4927908182144165, + "objective/train/avg_token_score": 0.009343437850475311, + "objective/train/docs_used": 475209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.68759024143219, + "objective/train/original_loss": 3.708533763885498, + "objective/train/theoretical_loss": 3.7182252957756896, + "objective/train/tokens_used": 824115200, + "objective/train/unlikelihood_loss": 0.16122353076934814, + "theoretical_loss": 3.7182252957756896, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789167502507523, + "loss": 1.4694, + "theoretical_loss": 3.718196143261248, + "tokens_seen": 824180736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037889669007021065, + "loss": 1.3992, + "theoretical_loss": 3.7181378471329047, + "tokens_seen": 824311808 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378876629889669, + "loss": 1.4572, + "theoretical_loss": 3.718079562868364, + "tokens_seen": 824442880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037885656970912743, + "loss": 1.3337, + "theoretical_loss": 3.7180212904633256, + "tokens_seen": 824573952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788365095285858, + "loss": 1.4281, + "theoretical_loss": 3.7179630299134914, + "tokens_seen": 824705024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037881644934804415, + "loss": 1.5416, + "theoretical_loss": 3.7179047812145662, + "tokens_seen": 824836096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787963891675025, + "loss": 1.3787, + "theoretical_loss": 3.717846544362257, + "tokens_seen": 824967168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037877632898696093, + "loss": 1.4132, + "theoretical_loss": 3.7177883193522723, + "tokens_seen": 825098240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787562688064193, + "loss": 1.2989, + "theoretical_loss": 3.7177301061803236, + "tokens_seen": 825229312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037873620862587766, + "loss": 1.3668, + "theoretical_loss": 3.7176719048421245, + "tokens_seen": 825360384 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378716148445336, + "loss": 1.5319, + "theoretical_loss": 3.717613715333391, + "tokens_seen": 825491456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786960882647944, + "loss": 1.4202, + "theoretical_loss": 3.7175555376498397, + "tokens_seen": 825622528 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.3739613890647888, + "objective/train/avg_token_score": 0.005932169500738382, + "objective/train/docs_used": 475848, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2623873949050903, + "objective/train/original_loss": 3.478762149810791, + "objective/train/theoretical_loss": 3.717497371787192, + "objective/train/tokens_used": 825753600, + "objective/train/unlikelihood_loss": 0.21234969794750214, + "theoretical_loss": 3.717497371787192, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786760280842528, + "loss": 1.4066, + "theoretical_loss": 3.717497371787192, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037865596790371116, + "loss": 1.4632, + "theoretical_loss": 3.7174392177411706, + "tokens_seen": 825884672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786359077231695, + "loss": 1.2759, + "theoretical_loss": 3.717381075507499, + "tokens_seen": 826015744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786158475426279, + "loss": 1.4941, + "theoretical_loss": 3.7173229450819045, + "tokens_seen": 826146816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785957873620863, + "loss": 1.4833, + "theoretical_loss": 3.7172648264601165, + "tokens_seen": 826277888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037857572718154466, + "loss": 1.3309, + "theoretical_loss": 3.7172067196378658, + "tokens_seen": 826408960 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378555667001003, + "loss": 1.3584, + "theoretical_loss": 3.717148624610886, + "tokens_seen": 826540032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785356068204614, + "loss": 1.4915, + "theoretical_loss": 3.717090541374913, + "tokens_seen": 826671104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037851554663991975, + "loss": 1.3334, + "theoretical_loss": 3.7170324699256847, + "tokens_seen": 826802176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037849548645937816, + "loss": 1.3501, + "theoretical_loss": 3.7169744102589415, + "tokens_seen": 826933248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784754262788365, + "loss": 1.2571, + "theoretical_loss": 3.7169163623704256, + "tokens_seen": 827064320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784553660982949, + "loss": 1.3592, + "theoretical_loss": 3.716858326255881, + "tokens_seen": 827195392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037843530591775325, + "loss": 1.5011, + "theoretical_loss": 3.7168003019110554, + "tokens_seen": 827326464 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.30565738677978516, + "objective/train/avg_token_score": 0.05732998251914978, + "objective/train/docs_used": 477214, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9969731569290161, + "objective/train/original_loss": 4.136387825012207, + "objective/train/theoretical_loss": 3.7167712941509587, + "objective/train/tokens_used": 827392000, + "objective/train/unlikelihood_loss": 0.12062285840511322, + "theoretical_loss": 3.7167712941509587, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037841524573721167, + "loss": 1.401, + "theoretical_loss": 3.7167422893316977, + "tokens_seen": 827457536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037839518555667003, + "loss": 1.2944, + "theoretical_loss": 3.7166842885135583, + "tokens_seen": 827588608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783751253761284, + "loss": 1.2305, + "theoretical_loss": 3.7166262994523915, + "tokens_seen": 827719680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037835506519558675, + "loss": 1.4953, + "theoretical_loss": 3.7165683221439525, + "tokens_seen": 827850752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783350050150451, + "loss": 1.4268, + "theoretical_loss": 3.716510356583999, + "tokens_seen": 827981824 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037831494483450353, + "loss": 1.3947, + "theoretical_loss": 3.716452402768292, + "tokens_seen": 828112896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782948846539619, + "loss": 1.6423, + "theoretical_loss": 3.7163944606925927, + "tokens_seen": 828243968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037827482447342026, + "loss": 1.3211, + "theoretical_loss": 3.716336530352666, + "tokens_seen": 828375040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782547642928786, + "loss": 1.6566, + "theoretical_loss": 3.716278611744278, + "tokens_seen": 828506112 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037823470411233704, + "loss": 1.2684, + "theoretical_loss": 3.716220704863198, + "tokens_seen": 828637184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782146439317954, + "loss": 1.1974, + "theoretical_loss": 3.716162809705197, + "tokens_seen": 828768256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037819458375125376, + "loss": 1.3824, + "theoretical_loss": 3.7161049262660484, + "tokens_seen": 828899328 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.375, + "objective/train/avg_token_score": 0.00957283191382885, + "objective/train/docs_used": 478014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2828823328018188, + "objective/train/original_loss": 3.6685211658477783, + "objective/train/theoretical_loss": 3.7160470545415274, + "objective/train/tokens_used": 829030400, + "objective/train/unlikelihood_loss": 0.19550453126430511, + "theoretical_loss": 3.7160470545415274, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781745235707121, + "loss": 1.3896, + "theoretical_loss": 3.7160470545415274, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781544633901705, + "loss": 1.4898, + "theoretical_loss": 3.7159891945274115, + "tokens_seen": 829161472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781344032096289, + "loss": 1.5047, + "theoretical_loss": 3.715931346219481, + "tokens_seen": 829292544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037811434302908726, + "loss": 1.4396, + "theoretical_loss": 3.715873509613517, + "tokens_seen": 829423616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780942828485456, + "loss": 1.4532, + "theoretical_loss": 3.715815684705304, + "tokens_seen": 829554688 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378074222668004, + "loss": 1.3857, + "theoretical_loss": 3.715757871490629, + "tokens_seen": 829685760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780541624874624, + "loss": 1.3689, + "theoretical_loss": 3.71570006996528, + "tokens_seen": 829816832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037803410230692077, + "loss": 1.3175, + "theoretical_loss": 3.7156422801250475, + "tokens_seen": 829947904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037801404212637913, + "loss": 1.35, + "theoretical_loss": 3.7155845019657248, + "tokens_seen": 830078976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779939819458375, + "loss": 1.5512, + "theoretical_loss": 3.715526735483107, + "tokens_seen": 830210048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037797392176529585, + "loss": 1.3632, + "theoretical_loss": 3.715468980672991, + "tokens_seen": 830341120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037795386158475427, + "loss": 1.3396, + "theoretical_loss": 3.7154112375311765, + "tokens_seen": 830472192 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037793380140421263, + "loss": 1.5547, + "theoretical_loss": 3.7153535060534644, + "tokens_seen": 830603264 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.3288734257221222, + "objective/train/avg_token_score": 0.019831860437989235, + "objective/train/docs_used": 479315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0720207691192627, + "objective/train/original_loss": 4.243991851806641, + "objective/train/theoretical_loss": 3.7153246446873363, + "objective/train/tokens_used": 830668800, + "objective/train/unlikelihood_loss": 0.1461578756570816, + "theoretical_loss": 3.7153246446873363, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377913741223671, + "loss": 1.4329, + "theoretical_loss": 3.71529578623566, + "tokens_seen": 830734336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037789368104312936, + "loss": 1.3825, + "theoretical_loss": 3.715238078073568, + "tokens_seen": 830865408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778736208625878, + "loss": 1.3844, + "theoretical_loss": 3.715180381562997, + "tokens_seen": 830996480 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037785356068204614, + "loss": 1.3975, + "theoretical_loss": 3.7151226966997566, + "tokens_seen": 831127552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778335005015045, + "loss": 1.4224, + "theoretical_loss": 3.71506502347966, + "tokens_seen": 831258624 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037781344032096286, + "loss": 1.6055, + "theoretical_loss": 3.715007361898522, + "tokens_seen": 831389696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777933801404213, + "loss": 1.3399, + "theoretical_loss": 3.7149497119521593, + "tokens_seen": 831520768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037777331995987964, + "loss": 1.4271, + "theoretical_loss": 3.71489207363639, + "tokens_seen": 831651840 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377753259779338, + "loss": 1.2951, + "theoretical_loss": 3.7148344469470356, + "tokens_seen": 831782912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037773319959879636, + "loss": 1.5184, + "theoretical_loss": 3.71477683187992, + "tokens_seen": 831913984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777131394182547, + "loss": 1.3934, + "theoretical_loss": 3.7147192284308677, + "tokens_seen": 832045056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037769307923771314, + "loss": 1.4524, + "theoretical_loss": 3.7146616365957073, + "tokens_seen": 832176128 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.4261363744735718, + "objective/train/avg_token_score": 0.013025101274251938, + "objective/train/docs_used": 479893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4979459047317505, + "objective/train/original_loss": 3.7806217670440674, + "objective/train/theoretical_loss": 3.714604056370267, + "objective/train/tokens_used": 832307200, + "objective/train/unlikelihood_loss": 0.18211011588573456, + "theoretical_loss": 3.714604056370267, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776730190571715, + "loss": 1.6195, + "theoretical_loss": 3.714604056370267, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037765295887662987, + "loss": 1.3013, + "theoretical_loss": 3.7145464877503804, + "tokens_seen": 832438272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037763289869608823, + "loss": 1.3596, + "theoretical_loss": 3.714488930731881, + "tokens_seen": 832569344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776128385155467, + "loss": 1.2122, + "theoretical_loss": 3.714431385310604, + "tokens_seen": 832700416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037759277833500506, + "loss": 1.5059, + "theoretical_loss": 3.714373851482389, + "tokens_seen": 832831488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775727181544634, + "loss": 1.3741, + "theoretical_loss": 3.7143163292430765, + "tokens_seen": 832962560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775526579739218, + "loss": 1.4406, + "theoretical_loss": 3.714258818588508, + "tokens_seen": 833093632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037753259779338015, + "loss": 1.3878, + "theoretical_loss": 3.7142013195145287, + "tokens_seen": 833224704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037751253761283857, + "loss": 1.2612, + "theoretical_loss": 3.714143832016987, + "tokens_seen": 833355776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037749247743229693, + "loss": 1.4767, + "theoretical_loss": 3.7140863560917294, + "tokens_seen": 833486848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774724172517553, + "loss": 1.3387, + "theoretical_loss": 3.714028891734609, + "tokens_seen": 833617920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037745235707121365, + "loss": 1.5716, + "theoretical_loss": 3.7139714389414786, + "tokens_seen": 833748992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037743229689067207, + "loss": 1.2769, + "theoretical_loss": 3.713913997708194, + "tokens_seen": 833880064 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.4627321660518646, + "objective/train/avg_token_score": 0.007378584239631891, + "objective/train/docs_used": 480416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5222768783569336, + "objective/train/original_loss": 3.4868133068084717, + "objective/train/theoretical_loss": 3.713885281425199, + "objective/train/tokens_used": 833945600, + "objective/train/unlikelihood_loss": 0.23750236630439758, + "theoretical_loss": 3.713885281425199, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037741223671013043, + "loss": 1.37, + "theoretical_loss": 3.7138565680306117, + "tokens_seen": 834011136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773921765295888, + "loss": 1.5261, + "theoretical_loss": 3.7137991499045926, + "tokens_seen": 834142208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037737211634904715, + "loss": 1.3438, + "theoretical_loss": 3.7137417433259987, + "tokens_seen": 834273280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773520561685055, + "loss": 1.2531, + "theoretical_loss": 3.713684348290693, + "tokens_seen": 834404352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037733199598796393, + "loss": 1.3156, + "theoretical_loss": 3.713626964794543, + "tokens_seen": 834535424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773119358074223, + "loss": 1.3972, + "theoretical_loss": 3.713569592833416, + "tokens_seen": 834666496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037729187562688066, + "loss": 1.2783, + "theoretical_loss": 3.713512232403182, + "tokens_seen": 834797568 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377271815446339, + "loss": 1.5663, + "theoretical_loss": 3.7134548834997148, + "tokens_seen": 834928640 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037725175526579744, + "loss": 1.382, + "theoretical_loss": 3.7133975461188884, + "tokens_seen": 835059712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772316950852558, + "loss": 1.578, + "theoretical_loss": 3.7133402202565793, + "tokens_seen": 835190784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037721163490471416, + "loss": 1.514, + "theoretical_loss": 3.7132829059086667, + "tokens_seen": 835321856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771915747241725, + "loss": 1.3903, + "theoretical_loss": 3.7132256030710318, + "tokens_seen": 835452928 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.5246822834014893, + "objective/train/avg_token_score": 0.01583530567586422, + "objective/train/docs_used": 481156, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5511996746063232, + "objective/train/original_loss": 3.6733298301696777, + "objective/train/theoretical_loss": 3.713168311739558, + "objective/train/tokens_used": 835584000, + "objective/train/unlikelihood_loss": 0.12975269556045532, + "theoretical_loss": 3.713168311739558, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771715145436309, + "loss": 1.4133, + "theoretical_loss": 3.713168311739558, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771514543630893, + "loss": 1.4018, + "theoretical_loss": 3.71311103191013, + "tokens_seen": 835715072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037713139418254766, + "loss": 1.1831, + "theoretical_loss": 3.7130537635786354, + "tokens_seen": 835846144 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377111334002006, + "loss": 1.5655, + "theoretical_loss": 3.7129965067409634, + "tokens_seen": 835977216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770912738214644, + "loss": 1.4086, + "theoretical_loss": 3.7129392613930063, + "tokens_seen": 836108288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770712136409228, + "loss": 1.4405, + "theoretical_loss": 3.712882027530657, + "tokens_seen": 836239360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037705115346038117, + "loss": 1.4356, + "theoretical_loss": 3.712824805149812, + "tokens_seen": 836370432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037703109327983953, + "loss": 1.5595, + "theoretical_loss": 3.712767594246369, + "tokens_seen": 836501504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770110330992979, + "loss": 1.3482, + "theoretical_loss": 3.712710394816228, + "tokens_seen": 836632576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037699097291875625, + "loss": 1.1248, + "theoretical_loss": 3.7126532068552915, + "tokens_seen": 836763648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037697091273821467, + "loss": 1.5212, + "theoretical_loss": 3.712596030359463, + "tokens_seen": 836894720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037695085255767303, + "loss": 1.4544, + "theoretical_loss": 3.7125388653246496, + "tokens_seen": 837025792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769307923771314, + "loss": 1.2609, + "theoretical_loss": 3.7124817117467597, + "tokens_seen": 837156864 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.4461754560470581, + "objective/train/avg_token_score": 0.006035082042217255, + "objective/train/docs_used": 482509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5078374147415161, + "objective/train/original_loss": 3.8577752113342285, + "objective/train/theoretical_loss": 3.712453139252883, + "objective/train/tokens_used": 837222400, + "objective/train/unlikelihood_loss": 0.17890168726444244, + "theoretical_loss": 3.712453139252883, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037691073219658976, + "loss": 1.5227, + "theoretical_loss": 3.712424569621703, + "tokens_seen": 837287936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768906720160482, + "loss": 1.3561, + "theoretical_loss": 3.712367438945394, + "tokens_seen": 837419008 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037687061183550654, + "loss": 1.3481, + "theoretical_loss": 3.7123103197137457, + "tokens_seen": 837550080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768505516549649, + "loss": 1.4355, + "theoretical_loss": 3.7122532119226754, + "tokens_seen": 837681152 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037683049147442326, + "loss": 1.5449, + "theoretical_loss": 3.7121961155681027, + "tokens_seen": 837812224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768104312938817, + "loss": 1.4074, + "theoretical_loss": 3.712139030645948, + "tokens_seen": 837943296 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037679037111334004, + "loss": 1.4172, + "theoretical_loss": 3.712081957152135, + "tokens_seen": 838074368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767703109327984, + "loss": 1.3335, + "theoretical_loss": 3.712024895082588, + "tokens_seen": 838205440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037675025075225676, + "loss": 1.4389, + "theoretical_loss": 3.7119678444332354, + "tokens_seen": 838336512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767301905717151, + "loss": 1.3121, + "theoretical_loss": 3.711910805200006, + "tokens_seen": 838467584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037671013039117354, + "loss": 1.3738, + "theoretical_loss": 3.711853777378832, + "tokens_seen": 838598656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766900702106319, + "loss": 1.4111, + "theoretical_loss": 3.7117967609656457, + "tokens_seen": 838729728 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.3952834904193878, + "objective/train/avg_token_score": 0.015426667407155037, + "objective/train/docs_used": 483094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4050980806350708, + "objective/train/original_loss": 4.066989898681641, + "objective/train/theoretical_loss": 3.7117397559563843, + "objective/train/tokens_used": 838860800, + "objective/train/unlikelihood_loss": 0.1427735835313797, + "theoretical_loss": 3.7117397559563843, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037667001003009027, + "loss": 1.4967, + "theoretical_loss": 3.7117397559563843, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037664994984954863, + "loss": 1.4498, + "theoretical_loss": 3.7116827623469844, + "tokens_seen": 838991872 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662988966900705, + "loss": 1.5299, + "theoretical_loss": 3.711625780133386, + "tokens_seen": 839122944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766098294884654, + "loss": 1.494, + "theoretical_loss": 3.7115688093115313, + "tokens_seen": 839254016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037658976930792377, + "loss": 1.4282, + "theoretical_loss": 3.7115118498773647, + "tokens_seen": 839385088 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037656970912738213, + "loss": 1.3543, + "theoretical_loss": 3.7114549018268317, + "tokens_seen": 839516160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765496489468405, + "loss": 1.3447, + "theoretical_loss": 3.7113979651558804, + "tokens_seen": 839647232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765295887662989, + "loss": 1.27, + "theoretical_loss": 3.7113410398604616, + "tokens_seen": 839778304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765095285857573, + "loss": 1.4955, + "theoretical_loss": 3.7112841259365266, + "tokens_seen": 839909376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037648946840521564, + "loss": 1.3424, + "theoretical_loss": 3.711227223380031, + "tokens_seen": 840040448 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376469408224674, + "loss": 1.2245, + "theoretical_loss": 3.71117033218693, + "tokens_seen": 840171520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764493480441324, + "loss": 1.3451, + "theoretical_loss": 3.7111134523531835, + "tokens_seen": 840302592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764292878635908, + "loss": 1.4084, + "theoretical_loss": 3.7110565838747505, + "tokens_seen": 840433664 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.3588709533214569, + "objective/train/avg_token_score": 0.0050110542215406895, + "objective/train/docs_used": 483926, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1891098022460938, + "objective/train/original_loss": 3.480327606201172, + "objective/train/theoretical_loss": 3.7110281538925154, + "objective/train/tokens_used": 840499200, + "objective/train/unlikelihood_loss": 0.21507981419563293, + "theoretical_loss": 3.7110281538925154, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037640922768304914, + "loss": 1.3768, + "theoretical_loss": 3.7109997267475947, + "tokens_seen": 840564736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763891675025075, + "loss": 1.276, + "theoretical_loss": 3.710942880967681, + "tokens_seen": 840695808 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037636910732196586, + "loss": 1.4002, + "theoretical_loss": 3.710886046530975, + "tokens_seen": 840826880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763490471414243, + "loss": 1.4177, + "theoretical_loss": 3.7108292234334472, + "tokens_seen": 840957952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037632898696088264, + "loss": 1.4772, + "theoretical_loss": 3.7107724116710665, + "tokens_seen": 841089024 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376308926780341, + "loss": 1.4019, + "theoretical_loss": 3.7107156112398076, + "tokens_seen": 841220096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037628886659979937, + "loss": 1.2848, + "theoretical_loss": 3.7106588221356445, + "tokens_seen": 841351168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003762688064192578, + "loss": 1.3472, + "theoretical_loss": 3.7106020443545553, + "tokens_seen": 841482240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037624874623871615, + "loss": 1.4646, + "theoretical_loss": 3.7105452778925176, + "tokens_seen": 841613312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762286860581745, + "loss": 1.4799, + "theoretical_loss": 3.7104885227455138, + "tokens_seen": 841744384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037620862587763287, + "loss": 1.5105, + "theoretical_loss": 3.7104317789095265, + "tokens_seen": 841875456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037618856569709123, + "loss": 1.2909, + "theoretical_loss": 3.710375046380541, + "tokens_seen": 842006528 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.3477517068386078, + "objective/train/avg_token_score": 0.011885336600244045, + "objective/train/docs_used": 484283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1525378227233887, + "objective/train/original_loss": 3.8259310722351074, + "objective/train/theoretical_loss": 3.710318325154545, + "objective/train/tokens_used": 842137600, + "objective/train/unlikelihood_loss": 0.13511060178279877, + "theoretical_loss": 3.710318325154545, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037616850551654965, + "loss": 1.2556, + "theoretical_loss": 3.710318325154545, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376148445336008, + "loss": 1.2842, + "theoretical_loss": 3.710261615227528, + "tokens_seen": 842268672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037612838515546637, + "loss": 1.2628, + "theoretical_loss": 3.7102049165954814, + "tokens_seen": 842399744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761083249749248, + "loss": 1.2348, + "theoretical_loss": 3.710148229254398, + "tokens_seen": 842530816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760882647943832, + "loss": 1.5289, + "theoretical_loss": 3.7100915532002734, + "tokens_seen": 842661888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606820461384157, + "loss": 1.2358, + "theoretical_loss": 3.7100348884291057, + "tokens_seen": 842792960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037604814443329993, + "loss": 1.3357, + "theoretical_loss": 3.7099782349368944, + "tokens_seen": 842924032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760280842527583, + "loss": 1.2584, + "theoretical_loss": 3.709921592719641, + "tokens_seen": 843055104 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037600802407221665, + "loss": 1.5061, + "theoretical_loss": 3.7098649617733486, + "tokens_seen": 843186176 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037598796389167507, + "loss": 1.3164, + "theoretical_loss": 3.709808342094024, + "tokens_seen": 843317248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037596790371113343, + "loss": 1.4844, + "theoretical_loss": 3.7097517336776744, + "tokens_seen": 843448320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759478435305918, + "loss": 1.4285, + "theoretical_loss": 3.7096951365203097, + "tokens_seen": 843579392 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037592778335005016, + "loss": 1.4242, + "theoretical_loss": 3.7096385506179415, + "tokens_seen": 843710464 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.36412513256073, + "objective/train/avg_token_score": 0.015121008269488811, + "objective/train/docs_used": 485567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.238376498222351, + "objective/train/original_loss": 3.959873676300049, + "objective/train/theoretical_loss": 3.7096102618861355, + "objective/train/tokens_used": 843776000, + "objective/train/unlikelihood_loss": 0.18654192984104156, + "theoretical_loss": 3.7096102618861355, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759077231695086, + "loss": 1.3115, + "theoretical_loss": 3.709581975966584, + "tokens_seen": 843841536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037588766298896694, + "loss": 1.3864, + "theoretical_loss": 3.709525412562253, + "tokens_seen": 843972608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758676028084253, + "loss": 1.4233, + "theoretical_loss": 3.709468860400966, + "tokens_seen": 844103680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037584754262788366, + "loss": 1.4341, + "theoretical_loss": 3.7094123194787434, + "tokens_seen": 844234752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758274824473421, + "loss": 1.3758, + "theoretical_loss": 3.7093557897916067, + "tokens_seen": 844365824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037580742226680044, + "loss": 1.435, + "theoretical_loss": 3.7092992713355804, + "tokens_seen": 844496896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757873620862588, + "loss": 1.412, + "theoretical_loss": 3.7092427641066905, + "tokens_seen": 844627968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037576730190571716, + "loss": 1.288, + "theoretical_loss": 3.709186268100965, + "tokens_seen": 844759040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757472417251755, + "loss": 1.5348, + "theoretical_loss": 3.709129783314433, + "tokens_seen": 844890112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037572718154463394, + "loss": 1.3866, + "theoretical_loss": 3.709073309743128, + "tokens_seen": 845021184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757071213640923, + "loss": 1.3176, + "theoretical_loss": 3.7090168473830833, + "tokens_seen": 845152256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037568706118355067, + "loss": 1.3053, + "theoretical_loss": 3.708960396230335, + "tokens_seen": 845283328 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.39167889952659607, + "objective/train/avg_token_score": 0.012442297302186489, + "objective/train/docs_used": 486205, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.328487515449524, + "objective/train/original_loss": 4.165731430053711, + "objective/train/theoretical_loss": 3.7089039562809223, + "objective/train/tokens_used": 845414400, + "objective/train/unlikelihood_loss": 0.1550619900226593, + "theoretical_loss": 3.7089039562809223, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037566700100300903, + "loss": 1.3514, + "theoretical_loss": 3.7089039562809223, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037564694082246745, + "loss": 1.2932, + "theoretical_loss": 3.7088475275308834, + "tokens_seen": 845545472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756268806419258, + "loss": 1.4705, + "theoretical_loss": 3.708791109976262, + "tokens_seen": 845676544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037560682046138417, + "loss": 1.3957, + "theoretical_loss": 3.7087347036131018, + "tokens_seen": 845807616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037558676028084253, + "loss": 1.5719, + "theoretical_loss": 3.708678308437449, + "tokens_seen": 845938688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755667001003009, + "loss": 1.2896, + "theoretical_loss": 3.7086219244453518, + "tokens_seen": 846069760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755466399197593, + "loss": 1.3925, + "theoretical_loss": 3.7085655516328604, + "tokens_seen": 846200832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755265797392177, + "loss": 1.2223, + "theoretical_loss": 3.7085091899960263, + "tokens_seen": 846331904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037550651955867604, + "loss": 1.3669, + "theoretical_loss": 3.708452839530905, + "tokens_seen": 846462976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754864593781344, + "loss": 1.5085, + "theoretical_loss": 3.708396500233552, + "tokens_seen": 846594048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754663991975928, + "loss": 1.4478, + "theoretical_loss": 3.7083401721000255, + "tokens_seen": 846725120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754463390170512, + "loss": 1.5123, + "theoretical_loss": 3.708283855126386, + "tokens_seen": 846856192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037542627883650954, + "loss": 1.6322, + "theoretical_loss": 3.7082275493086954, + "tokens_seen": 846987264 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.5196725130081177, + "objective/train/avg_token_score": 0.022891972213983536, + "objective/train/docs_used": 487507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8703233003616333, + "objective/train/original_loss": 4.225620269775391, + "objective/train/theoretical_loss": 3.708199400582101, + "objective/train/tokens_used": 847052800, + "objective/train/unlikelihood_loss": 0.1102483719587326, + "theoretical_loss": 3.708199400582101, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754062186559679, + "loss": 1.5343, + "theoretical_loss": 3.708171254643018, + "tokens_seen": 847118336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037538615847542626, + "loss": 1.4113, + "theoretical_loss": 3.70811497112542, + "tokens_seen": 847249408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753660982948847, + "loss": 1.4904, + "theoretical_loss": 3.70805869875197, + "tokens_seen": 847380480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037534603811434304, + "loss": 1.3858, + "theoretical_loss": 3.7080024375187373, + "tokens_seen": 847511552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753259779338014, + "loss": 1.4946, + "theoretical_loss": 3.707946187421795, + "tokens_seen": 847642624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037530591775325977, + "loss": 1.4309, + "theoretical_loss": 3.707889948457217, + "tokens_seen": 847773696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752858575727182, + "loss": 1.3008, + "theoretical_loss": 3.7078337206210796, + "tokens_seen": 847904768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037526579739217655, + "loss": 1.2482, + "theoretical_loss": 3.7077775039094605, + "tokens_seen": 848035840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752457372116349, + "loss": 1.2723, + "theoretical_loss": 3.7077212983184404, + "tokens_seen": 848166912 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037522567703109327, + "loss": 1.3721, + "theoretical_loss": 3.707665103844101, + "tokens_seen": 848297984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037520561685055163, + "loss": 1.324, + "theoretical_loss": 3.7076089204825267, + "tokens_seen": 848429056 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037518555667001005, + "loss": 1.3878, + "theoretical_loss": 3.707552748229803, + "tokens_seen": 848560128 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.33742669224739075, + "objective/train/avg_token_score": 0.026813289150595665, + "objective/train/docs_used": 487917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1936206817626953, + "objective/train/original_loss": 4.122153282165527, + "objective/train/theoretical_loss": 3.7074965870820193, + "objective/train/tokens_used": 848691200, + "objective/train/unlikelihood_loss": 0.159932479262352, + "theoretical_loss": 3.7074965870820193, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751654964894684, + "loss": 1.4008, + "theoretical_loss": 3.7074965870820193, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751454363089268, + "loss": 1.459, + "theoretical_loss": 3.7074404370352645, + "tokens_seen": 848822272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037512537612838514, + "loss": 1.3837, + "theoretical_loss": 3.707384298085631, + "tokens_seen": 848953344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037510531594784355, + "loss": 1.4725, + "theoretical_loss": 3.7073281702292133, + "tokens_seen": 849084416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750852557673019, + "loss": 1.5225, + "theoretical_loss": 3.707272053462107, + "tokens_seen": 849215488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750651955867603, + "loss": 1.1284, + "theoretical_loss": 3.7072159477804094, + "tokens_seen": 849346560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037504513540621864, + "loss": 1.4783, + "theoretical_loss": 3.707159853180222, + "tokens_seen": 849477632 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375025075225677, + "loss": 1.3591, + "theoretical_loss": 3.707103769657645, + "tokens_seen": 849608704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750050150451354, + "loss": 1.5539, + "theoretical_loss": 3.7070476972087842, + "tokens_seen": 849739776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749849548645938, + "loss": 1.2545, + "theoretical_loss": 3.706991635829744, + "tokens_seen": 849870848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037496489468405214, + "loss": 1.3255, + "theoretical_loss": 3.706935585516632, + "tokens_seen": 850001920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749448345035105, + "loss": 1.5198, + "theoretical_loss": 3.7068795462655597, + "tokens_seen": 850132992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749247743229689, + "loss": 1.4202, + "theoretical_loss": 3.706823518072638, + "tokens_seen": 850264064 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.26576244831085205, + "objective/train/avg_token_score": 0.02938208170235157, + "objective/train/docs_used": 489068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8557625412940979, + "objective/train/original_loss": 4.089486122131348, + "objective/train/theoretical_loss": 3.706795508121769, + "objective/train/tokens_used": 850329600, + "objective/train/unlikelihood_loss": 0.1921813040971756, + "theoretical_loss": 3.706795508121769, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749047141424273, + "loss": 1.2287, + "theoretical_loss": 3.7067675009339807, + "tokens_seen": 850395136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037488465396188564, + "loss": 1.5074, + "theoretical_loss": 3.7067114948457025, + "tokens_seen": 850526208 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374864593781344, + "loss": 1.4091, + "theoretical_loss": 3.7066554998039227, + "tokens_seen": 850657280 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037484453360080237, + "loss": 1.3843, + "theoretical_loss": 3.70659951580476, + "tokens_seen": 850788352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748244734202608, + "loss": 1.2876, + "theoretical_loss": 3.7065435428443365, + "tokens_seen": 850919424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037480441323971915, + "loss": 1.4978, + "theoretical_loss": 3.706487580918775, + "tokens_seen": 851050496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747843530591775, + "loss": 1.3365, + "theoretical_loss": 3.706431630024202, + "tokens_seen": 851181568 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037476429287863587, + "loss": 1.3003, + "theoretical_loss": 3.7063756901567437, + "tokens_seen": 851312640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747442326980943, + "loss": 1.2662, + "theoretical_loss": 3.706319761312531, + "tokens_seen": 851443712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037472417251755265, + "loss": 1.2286, + "theoretical_loss": 3.7062638434876947, + "tokens_seen": 851574784 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374704112337011, + "loss": 1.391, + "theoretical_loss": 3.706207936678368, + "tokens_seen": 851705856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746840521564694, + "loss": 1.419, + "theoretical_loss": 3.706152040880686, + "tokens_seen": 851836928 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.3321114480495453, + "objective/train/avg_token_score": 0.017706777900457382, + "objective/train/docs_used": 489617, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1683518886566162, + "objective/train/original_loss": 3.9676032066345215, + "objective/train/theoretical_loss": 3.7060961560907857, + "objective/train/tokens_used": 851968000, + "objective/train/unlikelihood_loss": 0.16615766286849976, + "theoretical_loss": 3.7060961560907857, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037466399197592774, + "loss": 1.44, + "theoretical_loss": 3.7060961560907857, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464393179538615, + "loss": 1.2472, + "theoretical_loss": 3.7060402823048073, + "tokens_seen": 852099072 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037462387161484457, + "loss": 1.4608, + "theoretical_loss": 3.705984419518891, + "tokens_seen": 852230144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037460381143430293, + "loss": 1.3966, + "theoretical_loss": 3.7059285677291807, + "tokens_seen": 852361216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745837512537613, + "loss": 1.502, + "theoretical_loss": 3.705872726931821, + "tokens_seen": 852492288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745636910732197, + "loss": 1.3725, + "theoretical_loss": 3.7058168971229586, + "tokens_seen": 852623360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745436308926781, + "loss": 1.2673, + "theoretical_loss": 3.7057610782987433, + "tokens_seen": 852754432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037452357071213644, + "loss": 1.4178, + "theoretical_loss": 3.705705270455325, + "tokens_seen": 852885504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745035105315948, + "loss": 1.4085, + "theoretical_loss": 3.7056494735888563, + "tokens_seen": 853016576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744834503510532, + "loss": 1.3409, + "theoretical_loss": 3.705593687695493, + "tokens_seen": 853147648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744633901705116, + "loss": 1.3477, + "theoretical_loss": 3.705537912771391, + "tokens_seen": 853278720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037444332998996994, + "loss": 1.4839, + "theoretical_loss": 3.7054821488127097, + "tokens_seen": 853409792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744232698094283, + "loss": 1.2615, + "theoretical_loss": 3.7054263958156084, + "tokens_seen": 853540864 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.4416544437408447, + "objective/train/avg_token_score": 0.025568164885044098, + "objective/train/docs_used": 491051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5035914182662964, + "objective/train/original_loss": 3.6911628246307373, + "objective/train/theoretical_loss": 3.7053985234264513, + "objective/train/tokens_used": 853606400, + "objective/train/unlikelihood_loss": 0.269401490688324, + "theoretical_loss": 3.7053985234264513, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037440320962888666, + "loss": 1.3793, + "theoretical_loss": 3.7053706537762503, + "tokens_seen": 853671936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743831494483451, + "loss": 1.4548, + "theoretical_loss": 3.7053149226908, + "tokens_seen": 853803008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037436308926780344, + "loss": 1.2737, + "theoretical_loss": 3.705259202555424, + "tokens_seen": 853934080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743430290872618, + "loss": 1.3662, + "theoretical_loss": 3.70520349336629, + "tokens_seen": 854065152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037432296890672017, + "loss": 1.5419, + "theoretical_loss": 3.7051477951195677, + "tokens_seen": 854196224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743029087261786, + "loss": 1.3069, + "theoretical_loss": 3.705092107811431, + "tokens_seen": 854327296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037428284854563695, + "loss": 1.4932, + "theoretical_loss": 3.705036431438052, + "tokens_seen": 854458368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742627883650953, + "loss": 1.4053, + "theoretical_loss": 3.704980765995608, + "tokens_seen": 854589440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037424272818455367, + "loss": 1.3191, + "theoretical_loss": 3.704925111480277, + "tokens_seen": 854720512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037422266800401203, + "loss": 1.4521, + "theoretical_loss": 3.704869467888237, + "tokens_seen": 854851584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037420260782347045, + "loss": 1.3455, + "theoretical_loss": 3.7048138352156723, + "tokens_seen": 854982656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741825476429288, + "loss": 1.504, + "theoretical_loss": 3.7047582134587644, + "tokens_seen": 855113728 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.464809387922287, + "objective/train/avg_token_score": 0.013745383359491825, + "objective/train/docs_used": 491675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.248916506767273, + "objective/train/original_loss": 3.5991764068603516, + "objective/train/theoretical_loss": 3.7047026026137, + "objective/train/tokens_used": 855244800, + "objective/train/unlikelihood_loss": 0.1444965898990631, + "theoretical_loss": 3.7047026026137, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741624874623872, + "loss": 1.5803, + "theoretical_loss": 3.7047026026137, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037414242728184554, + "loss": 1.3369, + "theoretical_loss": 3.7046470026766665, + "tokens_seen": 855375872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037412236710130395, + "loss": 1.4171, + "theoretical_loss": 3.704591413643853, + "tokens_seen": 855506944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741023069207623, + "loss": 1.398, + "theoretical_loss": 3.704535835511451, + "tokens_seen": 855638016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740822467402207, + "loss": 1.2593, + "theoretical_loss": 3.7044802682756535, + "tokens_seen": 855769088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037406218655967904, + "loss": 1.4864, + "theoretical_loss": 3.7044247119326568, + "tokens_seen": 855900160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740421263791374, + "loss": 1.3132, + "theoretical_loss": 3.7043691664786564, + "tokens_seen": 856031232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740220661985958, + "loss": 1.5225, + "theoretical_loss": 3.7043136319098524, + "tokens_seen": 856162304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740020060180542, + "loss": 1.4152, + "theoretical_loss": 3.7042581082224446, + "tokens_seen": 856293376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037398194583751254, + "loss": 1.3278, + "theoretical_loss": 3.704202595412637, + "tokens_seen": 856424448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739618856569709, + "loss": 1.5429, + "theoretical_loss": 3.7041470934766334, + "tokens_seen": 856555520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739418254764293, + "loss": 1.3143, + "theoretical_loss": 3.704091602410641, + "tokens_seen": 856686592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739217652958877, + "loss": 1.5079, + "theoretical_loss": 3.704036122210868, + "tokens_seen": 856817664 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.25207722187042236, + "objective/train/avg_token_score": 0.019671550020575523, + "objective/train/docs_used": 492132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8664920926094055, + "objective/train/original_loss": 4.571430206298828, + "objective/train/theoretical_loss": 3.704008386184629, + "objective/train/tokens_used": 856883200, + "objective/train/unlikelihood_loss": 0.11070846021175385, + "theoretical_loss": 3.704008386184629, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037390170511534605, + "loss": 1.4153, + "theoretical_loss": 3.7039806528735246, + "tokens_seen": 856948736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738816449348044, + "loss": 1.1698, + "theoretical_loss": 3.7039251943948237, + "tokens_seen": 857079808 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037386158475426277, + "loss": 1.4296, + "theoretical_loss": 3.7038697467709794, + "tokens_seen": 857210880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738415245737212, + "loss": 1.4337, + "theoretical_loss": 3.703814309998207, + "tokens_seen": 857341952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037382146439317955, + "loss": 1.4717, + "theoretical_loss": 3.703758884072725, + "tokens_seen": 857473024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738014042126379, + "loss": 1.3095, + "theoretical_loss": 3.7037034689907538, + "tokens_seen": 857604096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037378134403209627, + "loss": 1.372, + "theoretical_loss": 3.7036480647485153, + "tokens_seen": 857735168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737612838515547, + "loss": 1.2612, + "theoretical_loss": 3.703592671342232, + "tokens_seen": 857866240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037374122367101305, + "loss": 1.4276, + "theoretical_loss": 3.70353728876813, + "tokens_seen": 857997312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737211634904714, + "loss": 1.3834, + "theoretical_loss": 3.703481917022437, + "tokens_seen": 858128384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737011033099298, + "loss": 1.2975, + "theoretical_loss": 3.7034265561013826, + "tokens_seen": 858259456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037368104312938814, + "loss": 1.3587, + "theoretical_loss": 3.703371206001198, + "tokens_seen": 858390528 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.4613269865512848, + "objective/train/avg_token_score": 0.012780630961060524, + "objective/train/docs_used": 493247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5662318468093872, + "objective/train/original_loss": 4.086010932922363, + "objective/train/theoretical_loss": 3.7033158667181154, + "objective/train/tokens_used": 858521600, + "objective/train/unlikelihood_loss": 0.14914512634277344, + "theoretical_loss": 3.7033158667181154, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037366098294884655, + "loss": 1.4723, + "theoretical_loss": 3.7033158667181154, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736409227683049, + "loss": 1.2673, + "theoretical_loss": 3.703260538248371, + "tokens_seen": 858652672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736208625877633, + "loss": 1.312, + "theoretical_loss": 3.7032052205882007, + "tokens_seen": 858783744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037360080240722164, + "loss": 1.4342, + "theoretical_loss": 3.7031499137338444, + "tokens_seen": 858914816 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037358074222668006, + "loss": 1.3737, + "theoretical_loss": 3.7030946176815425, + "tokens_seen": 859045888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735606820461384, + "loss": 1.2742, + "theoretical_loss": 3.7030393324275366, + "tokens_seen": 859176960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735406218655968, + "loss": 1.3559, + "theoretical_loss": 3.702984057968072, + "tokens_seen": 859308032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037352056168505514, + "loss": 1.3341, + "theoretical_loss": 3.7029287942993943, + "tokens_seen": 859439104 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037350050150451356, + "loss": 1.3409, + "theoretical_loss": 3.7028735414177527, + "tokens_seen": 859570176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734804413239719, + "loss": 1.39, + "theoretical_loss": 3.7028182993193965, + "tokens_seen": 859701248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734603811434303, + "loss": 1.5354, + "theoretical_loss": 3.7027630680005785, + "tokens_seen": 859832320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037344032096288865, + "loss": 1.3173, + "theoretical_loss": 3.7027078474575514, + "tokens_seen": 859963392 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373420260782347, + "loss": 1.2805, + "theoretical_loss": 3.7026526376865716, + "tokens_seen": 860094464 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.36223119497299194, + "objective/train/avg_token_score": 0.014901701360940933, + "objective/train/docs_used": 493898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3064836263656616, + "objective/train/original_loss": 3.9059553146362305, + "objective/train/theoretical_loss": 3.7026250368394296, + "objective/train/tokens_used": 860160000, + "objective/train/unlikelihood_loss": 0.13297466933727264, + "theoretical_loss": 3.7026250368394296, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734002006018054, + "loss": 1.4934, + "theoretical_loss": 3.7025974386838962, + "tokens_seen": 860225536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733801404212638, + "loss": 1.3481, + "theoretical_loss": 3.7025422504457857, + "tokens_seen": 860356608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037336008024072215, + "loss": 1.5134, + "theoretical_loss": 3.7024870729684998, + "tokens_seen": 860487680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733400200601805, + "loss": 1.194, + "theoretical_loss": 3.702431906248303, + "tokens_seen": 860618752 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037331995987963893, + "loss": 1.3861, + "theoretical_loss": 3.7023767502814593, + "tokens_seen": 860749824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732998996990973, + "loss": 1.3731, + "theoretical_loss": 3.7023216050642365, + "tokens_seen": 860880896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037327983951855565, + "loss": 1.3899, + "theoretical_loss": 3.702266470592903, + "tokens_seen": 861011968 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373259779338014, + "loss": 1.2695, + "theoretical_loss": 3.70221134686373, + "tokens_seen": 861143040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732397191574724, + "loss": 1.2822, + "theoretical_loss": 3.7021562338729885, + "tokens_seen": 861274112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732196589769308, + "loss": 1.5589, + "theoretical_loss": 3.702101131616954, + "tokens_seen": 861405184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037319959879638916, + "loss": 1.3539, + "theoretical_loss": 3.7020460400919024, + "tokens_seen": 861536256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731795386158475, + "loss": 1.3389, + "theoretical_loss": 3.701990959294112, + "tokens_seen": 861667328 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.3090786933898926, + "objective/train/avg_token_score": 0.07331834733486176, + "objective/train/docs_used": 495406, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9482772946357727, + "objective/train/original_loss": 3.555332899093628, + "objective/train/theoretical_loss": 3.701935889219863, + "objective/train/tokens_used": 861798400, + "objective/train/unlikelihood_loss": 0.17151597142219543, + "theoretical_loss": 3.701935889219863, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731594784353059, + "loss": 1.3685, + "theoretical_loss": 3.701935889219863, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731394182547643, + "loss": 1.3839, + "theoretical_loss": 3.701880829865436, + "tokens_seen": 861929472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731193580742227, + "loss": 1.4607, + "theoretical_loss": 3.7018257812271154, + "tokens_seen": 862060544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730992978936811, + "loss": 1.3856, + "theoretical_loss": 3.7017707433011875, + "tokens_seen": 862191616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037307923771313944, + "loss": 1.2612, + "theoretical_loss": 3.701715716083938, + "tokens_seen": 862322688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730591775325978, + "loss": 1.4123, + "theoretical_loss": 3.701660699571657, + "tokens_seen": 862453760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730391173520562, + "loss": 1.3262, + "theoretical_loss": 3.701605693760636, + "tokens_seen": 862584832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730190571715146, + "loss": 1.3575, + "theoretical_loss": 3.701550698647167, + "tokens_seen": 862715904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037299899699097294, + "loss": 1.2906, + "theoretical_loss": 3.7014957142275446, + "tokens_seen": 862846976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729789368104313, + "loss": 1.348, + "theoretical_loss": 3.7014407404980663, + "tokens_seen": 862978048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729588766298897, + "loss": 1.4147, + "theoretical_loss": 3.7013857774550303, + "tokens_seen": 863109120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729388164493481, + "loss": 1.2737, + "theoretical_loss": 3.701330825094736, + "tokens_seen": 863240192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037291875626880645, + "loss": 1.563, + "theoretical_loss": 3.701275883413487, + "tokens_seen": 863371264 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.4000488817691803, + "objective/train/avg_token_score": 0.011403161101043224, + "objective/train/docs_used": 496136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4209589958190918, + "objective/train/original_loss": 3.7444279193878174, + "objective/train/theoretical_loss": 3.7012484165763486, + "objective/train/tokens_used": 863436800, + "objective/train/unlikelihood_loss": 0.2535700500011444, + "theoretical_loss": 3.7012484165763486, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728986960882648, + "loss": 1.6552, + "theoretical_loss": 3.701220952407586, + "tokens_seen": 863502336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037287863590772317, + "loss": 1.4376, + "theoretical_loss": 3.7011660320733393, + "tokens_seen": 863633408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728585757271816, + "loss": 1.4337, + "theoretical_loss": 3.7011111224070543, + "tokens_seen": 863764480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037283851554663995, + "loss": 1.2309, + "theoretical_loss": 3.7010562234050406, + "tokens_seen": 863895552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728184553660983, + "loss": 1.3512, + "theoretical_loss": 3.7010013350636095, + "tokens_seen": 864026624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727983951855567, + "loss": 1.3406, + "theoretical_loss": 3.7009464573790742, + "tokens_seen": 864157696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727783350050151, + "loss": 1.4882, + "theoretical_loss": 3.7008915903477497, + "tokens_seen": 864288768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037275827482447345, + "loss": 1.3782, + "theoretical_loss": 3.7008367339659523, + "tokens_seen": 864419840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727382146439318, + "loss": 1.4566, + "theoretical_loss": 3.7007818882300016, + "tokens_seen": 864550912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727181544633902, + "loss": 1.4354, + "theoretical_loss": 3.7007270531362173, + "tokens_seen": 864681984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037269809428284854, + "loss": 1.4733, + "theoretical_loss": 3.700672228680922, + "tokens_seen": 864813056 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037267803410230696, + "loss": 1.2466, + "theoretical_loss": 3.70061741486044, + "tokens_seen": 864944128 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.4685361683368683, + "objective/train/avg_token_score": 0.005407474935054779, + "objective/train/docs_used": 497267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5093052387237549, + "objective/train/original_loss": 3.6359965801239014, + "objective/train/theoretical_loss": 3.7005626116710966, + "objective/train/tokens_used": 865075200, + "objective/train/unlikelihood_loss": 0.1423846334218979, + "theoretical_loss": 3.7005626116710966, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726579739217653, + "loss": 1.3352, + "theoretical_loss": 3.7005626116710966, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726379137412237, + "loss": 1.4276, + "theoretical_loss": 3.7005078191092204, + "tokens_seen": 865206272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037261785356068204, + "loss": 1.3456, + "theoretical_loss": 3.700453037171141, + "tokens_seen": 865337344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037259779338014046, + "loss": 1.3809, + "theoretical_loss": 3.7003982658531887, + "tokens_seen": 865468416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725777331995988, + "loss": 1.4759, + "theoretical_loss": 3.7003435051516975, + "tokens_seen": 865599488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725576730190572, + "loss": 1.5283, + "theoretical_loss": 3.700288755063003, + "tokens_seen": 865730560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037253761283851554, + "loss": 1.538, + "theoretical_loss": 3.7002340155834412, + "tokens_seen": 865861632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037251755265797396, + "loss": 1.3806, + "theoretical_loss": 3.7001792867093517, + "tokens_seen": 865992704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724974924774323, + "loss": 1.4212, + "theoretical_loss": 3.700124568437074, + "tokens_seen": 866123776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724774322968907, + "loss": 1.4803, + "theoretical_loss": 3.7000698607629507, + "tokens_seen": 866254848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037245737211634905, + "loss": 1.4764, + "theoretical_loss": 3.700015163683327, + "tokens_seen": 866385920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724373119358074, + "loss": 1.3578, + "theoretical_loss": 3.6999604771945473, + "tokens_seen": 866516992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724172517552658, + "loss": 1.328, + "theoretical_loss": 3.6999058012929606, + "tokens_seen": 866648064 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.39748290181159973, + "objective/train/avg_token_score": 0.01396061759442091, + "objective/train/docs_used": 497986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2560821771621704, + "objective/train/original_loss": 3.4823272228240967, + "objective/train/theoretical_loss": 3.6998784673112235, + "objective/train/tokens_used": 866713600, + "objective/train/unlikelihood_loss": 0.23144493997097015, + "theoretical_loss": 3.6998784673112235, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723971915747242, + "loss": 1.35, + "theoretical_loss": 3.6998511359749164, + "tokens_seen": 866779136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037237713139418255, + "loss": 1.5217, + "theoretical_loss": 3.699796481236765, + "tokens_seen": 866910208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723570712136409, + "loss": 1.3233, + "theoretical_loss": 3.6997418370748605, + "tokens_seen": 867041280 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037233701103309933, + "loss": 1.4654, + "theoretical_loss": 3.699687203485558, + "tokens_seen": 867172352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723169508525577, + "loss": 1.2703, + "theoretical_loss": 3.699632580465215, + "tokens_seen": 867303424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037229689067201605, + "loss": 1.4951, + "theoretical_loss": 3.699577968010188, + "tokens_seen": 867434496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722768304914744, + "loss": 1.2537, + "theoretical_loss": 3.699523366116839, + "tokens_seen": 867565568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722567703109328, + "loss": 1.4116, + "theoretical_loss": 3.6994687747815305, + "tokens_seen": 867696640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722367101303912, + "loss": 1.4425, + "theoretical_loss": 3.6994141940006253, + "tokens_seen": 867827712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037221664994984956, + "loss": 1.4123, + "theoretical_loss": 3.6993596237704907, + "tokens_seen": 867958784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721965897693079, + "loss": 1.2621, + "theoretical_loss": 3.6993050640874934, + "tokens_seen": 868089856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721765295887663, + "loss": 1.5005, + "theoretical_loss": 3.6992505149480026, + "tokens_seen": 868220928 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.39345064759254456, + "objective/train/avg_token_score": 0.008443309925496578, + "objective/train/docs_used": 499247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2602639198303223, + "objective/train/original_loss": 3.548039197921753, + "objective/train/theoretical_loss": 3.69919597634839, + "objective/train/tokens_used": 868352000, + "objective/train/unlikelihood_loss": 0.25408735871315, + "theoretical_loss": 3.69919597634839, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721564694082247, + "loss": 1.4121, + "theoretical_loss": 3.69919597634839, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037213640922768306, + "loss": 1.3104, + "theoretical_loss": 3.6991414482850296, + "tokens_seen": 868483072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721163490471414, + "loss": 1.4091, + "theoretical_loss": 3.6990869307542944, + "tokens_seen": 868614144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720962888665998, + "loss": 1.3233, + "theoretical_loss": 3.6990324237525622, + "tokens_seen": 868745216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037207622868605815, + "loss": 1.4512, + "theoretical_loss": 3.698977927276211, + "tokens_seen": 868876288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037205616850551656, + "loss": 1.384, + "theoretical_loss": 3.6989234413216217, + "tokens_seen": 869007360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720361083249749, + "loss": 1.4707, + "theoretical_loss": 3.698868965885175, + "tokens_seen": 869138432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720160481444333, + "loss": 1.3837, + "theoretical_loss": 3.6988145009632563, + "tokens_seen": 869269504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037199598796389165, + "loss": 1.501, + "theoretical_loss": 3.698760046552249, + "tokens_seen": 869400576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037197592778335007, + "loss": 1.2308, + "theoretical_loss": 3.698705602648543, + "tokens_seen": 869531648 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037195586760280843, + "loss": 1.3402, + "theoretical_loss": 3.698651169248526, + "tokens_seen": 869662720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719358074222668, + "loss": 1.4384, + "theoretical_loss": 3.6985967463485894, + "tokens_seen": 869793792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037191574724172515, + "loss": 1.4371, + "theoretical_loss": 3.698542333945125, + "tokens_seen": 869924864 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.46187683939933777, + "objective/train/avg_token_score": 0.018257692456245422, + "objective/train/docs_used": 499921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6712664365768433, + "objective/train/original_loss": 3.997692108154297, + "objective/train/theoretical_loss": 3.6985151316784437, + "objective/train/tokens_used": 869990400, + "objective/train/unlikelihood_loss": 0.17598630487918854, + "theoretical_loss": 3.6985151316784437, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718956870611835, + "loss": 1.5048, + "theoretical_loss": 3.6984879320345287, + "tokens_seen": 870055936 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037187562688064193, + "loss": 1.379, + "theoretical_loss": 3.6984335406131956, + "tokens_seen": 870187008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718555667001003, + "loss": 1.3564, + "theoretical_loss": 3.698379159677525, + "tokens_seen": 870318080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037183550651955866, + "loss": 1.6163, + "theoretical_loss": 3.6983247892239155, + "tokens_seen": 870449152 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371815446339017, + "loss": 1.337, + "theoretical_loss": 3.69827042924877, + "tokens_seen": 870580224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037179538615847544, + "loss": 1.4197, + "theoretical_loss": 3.69821607974849, + "tokens_seen": 870711296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717753259779338, + "loss": 1.4484, + "theoretical_loss": 3.698161740719483, + "tokens_seen": 870842368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037175526579739216, + "loss": 1.586, + "theoretical_loss": 3.698107412158154, + "tokens_seen": 870973440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717352056168505, + "loss": 1.5828, + "theoretical_loss": 3.6980530940609126, + "tokens_seen": 871104512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717151454363089, + "loss": 1.2986, + "theoretical_loss": 3.6979987864241695, + "tokens_seen": 871235584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716950852557673, + "loss": 1.4425, + "theoretical_loss": 3.697944489244337, + "tokens_seen": 871366656 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037167502507522566, + "loss": 1.3235, + "theoretical_loss": 3.6978902025178284, + "tokens_seen": 871497728 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.4692693054676056, + "objective/train/avg_token_score": 0.03786318376660347, + "objective/train/docs_used": 501092, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6135823726654053, + "objective/train/original_loss": 4.459938049316406, + "objective/train/theoretical_loss": 3.6978359262410603, + "objective/train/tokens_used": 871628800, + "objective/train/unlikelihood_loss": 0.11413759738206863, + "theoretical_loss": 3.6978359262410603, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371654964894684, + "loss": 1.4739, + "theoretical_loss": 3.6978359262410603, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716349047141424, + "loss": 1.308, + "theoretical_loss": 3.6977816604104494, + "tokens_seen": 871759872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037161484453360086, + "loss": 1.5044, + "theoretical_loss": 3.697727405022416, + "tokens_seen": 871890944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715947843530592, + "loss": 1.488, + "theoretical_loss": 3.6976731600733808, + "tokens_seen": 872022016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715747241725176, + "loss": 1.4658, + "theoretical_loss": 3.6976189255597665, + "tokens_seen": 872153088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037155466399197595, + "loss": 1.3303, + "theoretical_loss": 3.697564701477998, + "tokens_seen": 872284160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037153460381143436, + "loss": 1.2618, + "theoretical_loss": 3.6975104878245015, + "tokens_seen": 872415232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715145436308927, + "loss": 1.444, + "theoretical_loss": 3.6974562845957055, + "tokens_seen": 872546304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714944834503511, + "loss": 1.3919, + "theoretical_loss": 3.6974020917880397, + "tokens_seen": 872677376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037147442326980945, + "loss": 1.5372, + "theoretical_loss": 3.6973479093979353, + "tokens_seen": 872808448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714543630892678, + "loss": 1.4086, + "theoretical_loss": 3.697293737421826, + "tokens_seen": 872939520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037143430290872623, + "loss": 1.3701, + "theoretical_loss": 3.6972395758561483, + "tokens_seen": 873070592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714142427281846, + "loss": 1.4026, + "theoretical_loss": 3.697185424697337, + "tokens_seen": 873201664 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.29802054166793823, + "objective/train/avg_token_score": 0.02961268648505211, + "objective/train/docs_used": 501627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.057908296585083, + "objective/train/original_loss": 4.1086602210998535, + "objective/train/theoretical_loss": 3.6971583530193937, + "objective/train/tokens_used": 873267200, + "objective/train/unlikelihood_loss": 0.17612986266613007, + "theoretical_loss": 3.6971583530193937, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037139418254764295, + "loss": 1.3047, + "theoretical_loss": 3.697131283941832, + "tokens_seen": 873332736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713741223671013, + "loss": 1.3749, + "theoretical_loss": 3.697077153586074, + "tokens_seen": 873463808 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037135406218655973, + "loss": 1.4628, + "theoretical_loss": 3.697023033626504, + "tokens_seen": 873594880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713340020060181, + "loss": 1.2401, + "theoretical_loss": 3.6969689240595676, + "tokens_seen": 873725952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037131394182547645, + "loss": 1.4965, + "theoretical_loss": 3.696914824881709, + "tokens_seen": 873857024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712938816449348, + "loss": 1.3674, + "theoretical_loss": 3.6968607360893766, + "tokens_seen": 873988096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712738214643932, + "loss": 1.458, + "theoretical_loss": 3.6968066576790193, + "tokens_seen": 874119168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712537612838516, + "loss": 1.4406, + "theoretical_loss": 3.6967525896470876, + "tokens_seen": 874250240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037123370110330996, + "loss": 1.4542, + "theoretical_loss": 3.6966985319900356, + "tokens_seen": 874381312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712136409227683, + "loss": 1.2803, + "theoretical_loss": 3.696644484704316, + "tokens_seen": 874512384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711935807422267, + "loss": 1.3453, + "theoretical_loss": 3.696590447786386, + "tokens_seen": 874643456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711735205616851, + "loss": 1.4329, + "theoretical_loss": 3.696536421232703, + "tokens_seen": 874774528 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.44409823417663574, + "objective/train/avg_token_score": 0.009593969210982323, + "objective/train/docs_used": 502963, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.322481393814087, + "objective/train/original_loss": 3.6682956218719482, + "objective/train/theoretical_loss": 3.6964824050397276, + "objective/train/tokens_used": 874905600, + "objective/train/unlikelihood_loss": 0.15543308854103088, + "theoretical_loss": 3.6964824050397276, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037115346038114346, + "loss": 1.3912, + "theoretical_loss": 3.6964824050397276, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711334002006018, + "loss": 1.5926, + "theoretical_loss": 3.6964283992039197, + "tokens_seen": 875036672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711133400200602, + "loss": 1.3741, + "theoretical_loss": 3.6963744037217436, + "tokens_seen": 875167744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037109327983951855, + "loss": 1.5263, + "theoretical_loss": 3.6963204185896643, + "tokens_seen": 875298816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037107321965897696, + "loss": 1.4279, + "theoretical_loss": 3.6962664438041473, + "tokens_seen": 875429888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710531594784353, + "loss": 1.3338, + "theoretical_loss": 3.6962124793616615, + "tokens_seen": 875560960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710330992978937, + "loss": 1.1809, + "theoretical_loss": 3.696158525258678, + "tokens_seen": 875692032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037101303911735205, + "loss": 1.599, + "theoretical_loss": 3.696104581491667, + "tokens_seen": 875823104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037099297893681047, + "loss": 1.6311, + "theoretical_loss": 3.6960506480571027, + "tokens_seen": 875954176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037097291875626883, + "loss": 1.3996, + "theoretical_loss": 3.6959967249514607, + "tokens_seen": 876085248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709528585757272, + "loss": 1.4681, + "theoretical_loss": 3.695942812171218, + "tokens_seen": 876216320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037093279839518555, + "loss": 1.325, + "theoretical_loss": 3.6958889097128527, + "tokens_seen": 876347392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709127382146439, + "loss": 1.4382, + "theoretical_loss": 3.6958350175728456, + "tokens_seen": 876478464 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.4307795763015747, + "objective/train/avg_token_score": 0.013570542447268963, + "objective/train/docs_used": 503455, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4876363277435303, + "objective/train/original_loss": 3.998917579650879, + "objective/train/theoretical_loss": 3.695808075371127, + "objective/train/tokens_used": 876544000, + "objective/train/unlikelihood_loss": 0.15190862119197845, + "theoretical_loss": 3.695808075371127, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037089267803410233, + "loss": 1.3914, + "theoretical_loss": 3.695781135747679, + "tokens_seen": 876609536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708726178535607, + "loss": 1.3723, + "theoretical_loss": 3.6957272642338372, + "tokens_seen": 876740608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037085255767301906, + "loss": 1.4367, + "theoretical_loss": 3.6956734030278047, + "tokens_seen": 876871680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708324974924774, + "loss": 1.5063, + "theoretical_loss": 3.69561955212607, + "tokens_seen": 877002752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037081243731193584, + "loss": 1.6357, + "theoretical_loss": 3.6955657115251213, + "tokens_seen": 877133824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707923771313942, + "loss": 1.3419, + "theoretical_loss": 3.69551188122145, + "tokens_seen": 877264896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037077231695085256, + "loss": 1.639, + "theoretical_loss": 3.695458061211548, + "tokens_seen": 877395968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707522567703109, + "loss": 1.6526, + "theoretical_loss": 3.6954042514919108, + "tokens_seen": 877527040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707321965897693, + "loss": 1.3699, + "theoretical_loss": 3.6953504520590332, + "tokens_seen": 877658112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707121364092277, + "loss": 1.3843, + "theoretical_loss": 3.6952966629094135, + "tokens_seen": 877789184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037069207622868606, + "loss": 1.478, + "theoretical_loss": 3.6952428840395504, + "tokens_seen": 877920256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706720160481444, + "loss": 1.3987, + "theoretical_loss": 3.6951891154459453, + "tokens_seen": 878051328 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.45197945833206177, + "objective/train/avg_token_score": 0.021213049069046974, + "objective/train/docs_used": 504719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4116363525390625, + "objective/train/original_loss": 3.7862746715545654, + "objective/train/theoretical_loss": 3.6951353571251015, + "objective/train/tokens_used": 878182400, + "objective/train/unlikelihood_loss": 0.17004284262657166, + "theoretical_loss": 3.6951353571251015, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706519558676028, + "loss": 1.3835, + "theoretical_loss": 3.6951353571251015, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706318956870612, + "loss": 1.4956, + "theoretical_loss": 3.6950816090735232, + "tokens_seen": 878313472 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037061183550651957, + "loss": 1.4983, + "theoretical_loss": 3.6950278712877163, + "tokens_seen": 878444544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037059177532597793, + "loss": 1.5109, + "theoretical_loss": 3.694974143764189, + "tokens_seen": 878575616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705717151454363, + "loss": 1.4131, + "theoretical_loss": 3.6949204264994515, + "tokens_seen": 878706688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037055165496489465, + "loss": 1.357, + "theoretical_loss": 3.6948667194900144, + "tokens_seen": 878837760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037053159478435307, + "loss": 1.4958, + "theoretical_loss": 3.694813022732391, + "tokens_seen": 878968832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037051153460381143, + "loss": 1.4501, + "theoretical_loss": 3.694759336223096, + "tokens_seen": 879099904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704914744232698, + "loss": 1.2939, + "theoretical_loss": 3.694705659958646, + "tokens_seen": 879230976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037047141424272816, + "loss": 1.3783, + "theoretical_loss": 3.694651993935559, + "tokens_seen": 879362048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704513540621866, + "loss": 1.4975, + "theoretical_loss": 3.6945983381503558, + "tokens_seen": 879493120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037043129388164494, + "loss": 1.5105, + "theoretical_loss": 3.694544692599557, + "tokens_seen": 879624192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704112337011033, + "loss": 1.2877, + "theoretical_loss": 3.6944910572796856, + "tokens_seen": 879755264 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.5306695699691772, + "objective/train/avg_token_score": 0.012853487394750118, + "objective/train/docs_used": 505361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7927442789077759, + "objective/train/original_loss": 3.953162908554077, + "objective/train/theoretical_loss": 3.694464243455262, + "objective/train/tokens_used": 879820800, + "objective/train/unlikelihood_loss": 0.1250007599592209, + "theoretical_loss": 3.694464243455262, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037039117352056166, + "loss": 1.5117, + "theoretical_loss": 3.694437432187268, + "tokens_seen": 879886336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703711133400201, + "loss": 1.3159, + "theoretical_loss": 3.694383817318829, + "tokens_seen": 880017408 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037035105315947844, + "loss": 1.461, + "theoretical_loss": 3.6943302126708986, + "tokens_seen": 880148480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703309929789368, + "loss": 1.2549, + "theoretical_loss": 3.694276618240006, + "tokens_seen": 880279552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037031093279839516, + "loss": 1.4831, + "theoretical_loss": 3.6942230340226834, + "tokens_seen": 880410624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702908726178535, + "loss": 1.4676, + "theoretical_loss": 3.6941694600154644, + "tokens_seen": 880541696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037027081243731194, + "loss": 1.2813, + "theoretical_loss": 3.6941158962148837, + "tokens_seen": 880672768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702507522567703, + "loss": 1.3422, + "theoretical_loss": 3.694062342617478, + "tokens_seen": 880803840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037023069207622867, + "loss": 1.4461, + "theoretical_loss": 3.6940087992197865, + "tokens_seen": 880934912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037021063189568703, + "loss": 1.4236, + "theoretical_loss": 3.6939552660183486, + "tokens_seen": 881065984 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037019057171514544, + "loss": 1.3715, + "theoretical_loss": 3.6939017430097074, + "tokens_seen": 881197056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701705115346038, + "loss": 1.6596, + "theoretical_loss": 3.6938482301904054, + "tokens_seen": 881328128 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.43957722187042236, + "objective/train/avg_token_score": 0.01441223081201315, + "objective/train/docs_used": 505856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3785372972488403, + "objective/train/original_loss": 3.7081758975982666, + "objective/train/theoretical_loss": 3.693794727556988, + "objective/train/tokens_used": 881459200, + "objective/train/unlikelihood_loss": 0.20880700647830963, + "theoretical_loss": 3.693794727556988, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037015045135406217, + "loss": 1.3662, + "theoretical_loss": 3.693794727556988, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037013039117352053, + "loss": 1.5085, + "theoretical_loss": 3.693741235106003, + "tokens_seen": 881590272 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037011033099297895, + "loss": 1.3848, + "theoretical_loss": 3.693687752833998, + "tokens_seen": 881721344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037009027081243736, + "loss": 1.3185, + "theoretical_loss": 3.693634280737524, + "tokens_seen": 881852416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700702106318957, + "loss": 1.3883, + "theoretical_loss": 3.6935808188131327, + "tokens_seen": 881983488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700501504513541, + "loss": 1.4393, + "theoretical_loss": 3.6935273670573787, + "tokens_seen": 882114560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037003009027081245, + "loss": 1.5155, + "theoretical_loss": 3.693473925466816, + "tokens_seen": 882245632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037001003009027087, + "loss": 1.4406, + "theoretical_loss": 3.693420494038003, + "tokens_seen": 882376704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036998996990972923, + "loss": 1.2849, + "theoretical_loss": 3.693367072767497, + "tokens_seen": 882507776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699699097291876, + "loss": 1.4675, + "theoretical_loss": 3.6933136616518594, + "tokens_seen": 882638848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036994984954864595, + "loss": 1.3103, + "theoretical_loss": 3.6932602606876523, + "tokens_seen": 882769920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699297893681043, + "loss": 1.3286, + "theoretical_loss": 3.693206869871439, + "tokens_seen": 882900992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036990972918756273, + "loss": 1.3763, + "theoretical_loss": 3.693153489199785, + "tokens_seen": 883032064 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.4285801649093628, + "objective/train/avg_token_score": 0.01727200485765934, + "objective/train/docs_used": 506939, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2683100700378418, + "objective/train/original_loss": 3.5251686573028564, + "objective/train/theoretical_loss": 3.693126802667095, + "objective/train/tokens_used": 883097600, + "objective/train/unlikelihood_loss": 0.22674039006233215, + "theoretical_loss": 3.693126802667095, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698896690070211, + "loss": 1.4285, + "theoretical_loss": 3.693100118669258, + "tokens_seen": 883163136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036986960882647946, + "loss": 1.3448, + "theoretical_loss": 3.6930467582764255, + "tokens_seen": 883294208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698495486459378, + "loss": 1.3953, + "theoretical_loss": 3.6929934080178595, + "tokens_seen": 883425280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036982948846539624, + "loss": 1.3294, + "theoretical_loss": 3.692940067890131, + "tokens_seen": 883556352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698094282848546, + "loss": 1.4758, + "theoretical_loss": 3.6928867378898147, + "tokens_seen": 883687424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036978936810431296, + "loss": 1.4042, + "theoretical_loss": 3.6928334180134854, + "tokens_seen": 883818496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697693079237713, + "loss": 1.5407, + "theoretical_loss": 3.69278010825772, + "tokens_seen": 883949568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697492477432297, + "loss": 1.3502, + "theoretical_loss": 3.692726808619098, + "tokens_seen": 884080640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697291875626881, + "loss": 1.5157, + "theoretical_loss": 3.692673519094199, + "tokens_seen": 884211712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036970912738214646, + "loss": 1.5004, + "theoretical_loss": 3.6926202396796057, + "tokens_seen": 884342784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696890672016048, + "loss": 1.3174, + "theoretical_loss": 3.692566970371902, + "tokens_seen": 884473856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696690070210632, + "loss": 1.3918, + "theoretical_loss": 3.692513711167673, + "tokens_seen": 884604928 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.3708455562591553, + "objective/train/avg_token_score": 0.006593772675842047, + "objective/train/docs_used": 507654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3380584716796875, + "objective/train/original_loss": 4.2020649909973145, + "objective/train/theoretical_loss": 3.692460462063506, + "objective/train/tokens_used": 884736000, + "objective/train/unlikelihood_loss": 0.13321493566036224, + "theoretical_loss": 3.692460462063506, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696489468405216, + "loss": 1.4179, + "theoretical_loss": 3.692460462063506, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036962888665997997, + "loss": 1.4161, + "theoretical_loss": 3.6924072230559895, + "tokens_seen": 884867072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036960882647943833, + "loss": 1.4742, + "theoretical_loss": 3.692353994141714, + "tokens_seen": 884998144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695887662988967, + "loss": 1.6075, + "theoretical_loss": 3.6923007753172716, + "tokens_seen": 885129216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036956870611835505, + "loss": 1.2747, + "theoretical_loss": 3.692247566579256, + "tokens_seen": 885260288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036954864593781347, + "loss": 1.3417, + "theoretical_loss": 3.6921943679242624, + "tokens_seen": 885391360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036952858575727183, + "loss": 1.3794, + "theoretical_loss": 3.692141179348888, + "tokens_seen": 885522432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695085255767302, + "loss": 1.4444, + "theoretical_loss": 3.692088000849732, + "tokens_seen": 885653504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036948846539618856, + "loss": 1.2293, + "theoretical_loss": 3.692034832423394, + "tokens_seen": 885784576 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369468405215647, + "loss": 1.4118, + "theoretical_loss": 3.6919816740664757, + "tokens_seen": 885915648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036944834503510534, + "loss": 1.5117, + "theoretical_loss": 3.6919285257755816, + "tokens_seen": 886046720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694282848545637, + "loss": 1.3565, + "theoretical_loss": 3.6918753875473165, + "tokens_seen": 886177792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036940822467402206, + "loss": 1.3458, + "theoretical_loss": 3.691822259378287, + "tokens_seen": 886308864 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.41055718064308167, + "objective/train/avg_token_score": 0.010015953332185745, + "objective/train/docs_used": 509172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.442650556564331, + "objective/train/original_loss": 3.678863763809204, + "objective/train/theoretical_loss": 3.6917956990649263, + "objective/train/tokens_used": 886374400, + "objective/train/unlikelihood_loss": 0.2298004925251007, + "theoretical_loss": 3.6917956990649263, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693881644934805, + "loss": 1.4075, + "theoretical_loss": 3.6917691412651026, + "tokens_seen": 886439936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036936810431293884, + "loss": 1.4947, + "theoretical_loss": 3.691716033204373, + "tokens_seen": 886571008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693480441323972, + "loss": 1.4056, + "theoretical_loss": 3.6916629351927095, + "tokens_seen": 886702080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036932798395185556, + "loss": 1.6169, + "theoretical_loss": 3.6916098472267267, + "tokens_seen": 886833152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693079237713139, + "loss": 1.3758, + "theoretical_loss": 3.6915567693030384, + "tokens_seen": 886964224 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036928786359077234, + "loss": 1.3003, + "theoretical_loss": 3.6915037014182626, + "tokens_seen": 887095296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692678034102307, + "loss": 1.3861, + "theoretical_loss": 3.6914506435690173, + "tokens_seen": 887226368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036924774322968907, + "loss": 1.4166, + "theoretical_loss": 3.691397595751922, + "tokens_seen": 887357440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036922768304914743, + "loss": 1.5073, + "theoretical_loss": 3.6913445579635997, + "tokens_seen": 887488512 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036920762286860585, + "loss": 1.4704, + "theoretical_loss": 3.691291530200672, + "tokens_seen": 887619584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691875626880642, + "loss": 1.4181, + "theoretical_loss": 3.6912385124597655, + "tokens_seen": 887750656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036916750250752257, + "loss": 1.5202, + "theoretical_loss": 3.6911855047375055, + "tokens_seen": 887881728 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.5295698642730713, + "objective/train/avg_token_score": 0.008101345039904118, + "objective/train/docs_used": 509747, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7344613075256348, + "objective/train/original_loss": 3.812901258468628, + "objective/train/theoretical_loss": 3.691132507030521, + "objective/train/tokens_used": 888012800, + "objective/train/unlikelihood_loss": 0.12133732438087463, + "theoretical_loss": 3.691132507030521, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036914744232698093, + "loss": 1.5297, + "theoretical_loss": 3.691132507030521, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691273821464393, + "loss": 1.412, + "theoretical_loss": 3.6910795193354415, + "tokens_seen": 888143872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691073219658977, + "loss": 1.4424, + "theoretical_loss": 3.691026541648899, + "tokens_seen": 888274944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690872617853561, + "loss": 1.4075, + "theoretical_loss": 3.6909735739675265, + "tokens_seen": 888406016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036906720160481444, + "loss": 1.4873, + "theoretical_loss": 3.690920616287958, + "tokens_seen": 888537088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690471414242728, + "loss": 1.3195, + "theoretical_loss": 3.6908676686068307, + "tokens_seen": 888668160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690270812437312, + "loss": 1.4376, + "theoretical_loss": 3.6908147309207826, + "tokens_seen": 888799232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690070210631896, + "loss": 1.4631, + "theoretical_loss": 3.6907618032264526, + "tokens_seen": 888930304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036898696088264794, + "loss": 1.5327, + "theoretical_loss": 3.6907088855204826, + "tokens_seen": 889061376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689669007021063, + "loss": 1.4121, + "theoretical_loss": 3.6906559777995156, + "tokens_seen": 889192448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036894684052156466, + "loss": 1.4565, + "theoretical_loss": 3.690603080060195, + "tokens_seen": 889323520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689267803410231, + "loss": 1.4551, + "theoretical_loss": 3.6905501922991686, + "tokens_seen": 889454592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036890672016048144, + "loss": 1.4029, + "theoretical_loss": 3.690497314513083, + "tokens_seen": 889585664 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.22971652448177338, + "objective/train/avg_token_score": 0.015085110440850258, + "objective/train/docs_used": 511200, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.6907581686973572, + "objective/train/original_loss": 4.484684467315674, + "objective/train/theoretical_loss": 3.690470879359596, + "objective/train/tokens_used": 889651200, + "objective/train/unlikelihood_loss": 0.13602155447006226, + "theoretical_loss": 3.690470879359596, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688866599799398, + "loss": 1.2411, + "theoretical_loss": 3.6904444466985877, + "tokens_seen": 889716736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036886659979939817, + "loss": 1.6495, + "theoretical_loss": 3.6903915888523344, + "tokens_seen": 889847808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688465396188566, + "loss": 1.3313, + "theoretical_loss": 3.690338740970975, + "tokens_seen": 889978880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036882647943831494, + "loss": 1.5239, + "theoretical_loss": 3.6902859030511634, + "tokens_seen": 890109952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688064192577733, + "loss": 1.3813, + "theoretical_loss": 3.6902330750895564, + "tokens_seen": 890241024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036878635907723167, + "loss": 1.49, + "theoretical_loss": 3.690180257082811, + "tokens_seen": 890372096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036876629889669003, + "loss": 1.3969, + "theoretical_loss": 3.690127449027586, + "tokens_seen": 890503168 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036874623871614845, + "loss": 1.5152, + "theoretical_loss": 3.6900746509205424, + "tokens_seen": 890634240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687261785356068, + "loss": 1.5478, + "theoretical_loss": 3.6900218627583428, + "tokens_seen": 890765312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036870611835506517, + "loss": 1.4159, + "theoretical_loss": 3.6899690845376503, + "tokens_seen": 890896384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036868605817452353, + "loss": 1.4613, + "theoretical_loss": 3.6899163162551307, + "tokens_seen": 891027456 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036866599799398195, + "loss": 1.4328, + "theoretical_loss": 3.6898635579074517, + "tokens_seen": 891158528 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.3825146555900574, + "objective/train/avg_token_score": 0.01421495620161295, + "objective/train/docs_used": 511885, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3332901000976562, + "objective/train/original_loss": 4.279423236846924, + "objective/train/theoretical_loss": 3.6898108094912816, + "objective/train/tokens_used": 891289600, + "objective/train/unlikelihood_loss": 0.11614857614040375, + "theoretical_loss": 3.6898108094912816, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686459378134403, + "loss": 1.4206, + "theoretical_loss": 3.6898108094912816, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686258776328987, + "loss": 1.4744, + "theoretical_loss": 3.6897580710032907, + "tokens_seen": 891420672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686058174523571, + "loss": 1.4648, + "theoretical_loss": 3.689705342440151, + "tokens_seen": 891551744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036858575727181545, + "loss": 1.4598, + "theoretical_loss": 3.689652623798536, + "tokens_seen": 891682816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036856569709127387, + "loss": 1.4565, + "theoretical_loss": 3.689599915075121, + "tokens_seen": 891813888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036854563691073223, + "loss": 1.3381, + "theoretical_loss": 3.6895472162665826, + "tokens_seen": 891944960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685255767301906, + "loss": 1.5222, + "theoretical_loss": 3.6894945273695985, + "tokens_seen": 892076032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036850551654964896, + "loss": 1.5862, + "theoretical_loss": 3.6894418483808504, + "tokens_seen": 892207104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684854563691074, + "loss": 1.7396, + "theoretical_loss": 3.6893891792970184, + "tokens_seen": 892338176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036846539618856574, + "loss": 1.4386, + "theoretical_loss": 3.689336520114786, + "tokens_seen": 892469248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684453360080241, + "loss": 1.5298, + "theoretical_loss": 3.6892838708308378, + "tokens_seen": 892600320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036842527582748246, + "loss": 1.2879, + "theoretical_loss": 3.6892312314418607, + "tokens_seen": 892731392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684052156469409, + "loss": 1.4585, + "theoretical_loss": 3.689178601944542, + "tokens_seen": 892862464 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.41501709818840027, + "objective/train/avg_token_score": 0.012059747241437435, + "objective/train/docs_used": 512575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3720542192459106, + "objective/train/original_loss": 3.398897409439087, + "objective/train/theoretical_loss": 3.68915229090422, + "objective/train/tokens_used": 892928000, + "objective/train/unlikelihood_loss": 0.267887681722641, + "theoretical_loss": 3.68915229090422, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036838515546639924, + "loss": 1.481, + "theoretical_loss": 3.6891259823355718, + "tokens_seen": 892993536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683650952858576, + "loss": 1.4599, + "theoretical_loss": 3.6890733726116407, + "tokens_seen": 893124608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036834503510531596, + "loss": 1.4219, + "theoretical_loss": 3.6890207727694415, + "tokens_seen": 893255680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683249749247743, + "loss": 1.3508, + "theoretical_loss": 3.6889681828056693, + "tokens_seen": 893386752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036830491474423274, + "loss": 1.3175, + "theoretical_loss": 3.688915602717019, + "tokens_seen": 893517824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682848545636911, + "loss": 1.3841, + "theoretical_loss": 3.688863032500189, + "tokens_seen": 893648896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036826479438314947, + "loss": 1.3229, + "theoretical_loss": 3.6888104721518777, + "tokens_seen": 893779968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036824473420260783, + "loss": 1.3707, + "theoretical_loss": 3.688757921668786, + "tokens_seen": 893911040 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036822467402206625, + "loss": 1.5595, + "theoretical_loss": 3.6887053810476162, + "tokens_seen": 894042112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682046138415246, + "loss": 1.1536, + "theoretical_loss": 3.688652850285072, + "tokens_seen": 894173184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036818455366098297, + "loss": 1.3591, + "theoretical_loss": 3.6886003293778584, + "tokens_seen": 894304256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036816449348044133, + "loss": 1.4135, + "theoretical_loss": 3.6885478183226836, + "tokens_seen": 894435328 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.39540567994117737, + "objective/train/avg_token_score": 0.012886048294603825, + "objective/train/docs_used": 513630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3885366916656494, + "objective/train/original_loss": 4.303807735443115, + "objective/train/theoretical_loss": 3.6884953171162556, + "objective/train/tokens_used": 894566400, + "objective/train/unlikelihood_loss": 0.1156373843550682, + "theoretical_loss": 3.6884953171162556, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681444332998997, + "loss": 1.3762, + "theoretical_loss": 3.6884953171162556, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681243731193581, + "loss": 1.3534, + "theoretical_loss": 3.688442825755284, + "tokens_seen": 894697472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681043129388165, + "loss": 1.3354, + "theoretical_loss": 3.688390344236481, + "tokens_seen": 894828544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036808425275827484, + "loss": 1.387, + "theoretical_loss": 3.6883378725565605, + "tokens_seen": 894959616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680641925777332, + "loss": 1.4612, + "theoretical_loss": 3.6882854107122363, + "tokens_seen": 895090688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680441323971916, + "loss": 1.4442, + "theoretical_loss": 3.688232958700226, + "tokens_seen": 895221760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036802407221665, + "loss": 1.536, + "theoretical_loss": 3.688180516517247, + "tokens_seen": 895352832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036800401203610834, + "loss": 1.3747, + "theoretical_loss": 3.6881280841600192, + "tokens_seen": 895483904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679839518555667, + "loss": 1.4673, + "theoretical_loss": 3.688075661625264, + "tokens_seen": 895614976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036796389167502506, + "loss": 1.5658, + "theoretical_loss": 3.688023248909704, + "tokens_seen": 895746048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679438314944835, + "loss": 1.449, + "theoretical_loss": 3.687970846010063, + "tokens_seen": 895877120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036792377131394184, + "loss": 1.363, + "theoretical_loss": 3.6879184529230673, + "tokens_seen": 896008192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679037111334002, + "loss": 1.4728, + "theoretical_loss": 3.6878660696454446, + "tokens_seen": 896139264 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.4114736020565033, + "objective/train/avg_token_score": 0.014809527434408665, + "objective/train/docs_used": 514378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3640265464782715, + "objective/train/original_loss": 3.6368560791015625, + "objective/train/theoretical_loss": 3.6878398816841265, + "objective/train/tokens_used": 896204800, + "objective/train/unlikelihood_loss": 0.19905586540699005, + "theoretical_loss": 3.6878398816841265, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036788365095285857, + "loss": 1.3923, + "theoretical_loss": 3.6878136961739245, + "tokens_seen": 896270336 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367863590772317, + "loss": 1.4159, + "theoretical_loss": 3.6877613325052367, + "tokens_seen": 896401408 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036784353059177535, + "loss": 1.5209, + "theoretical_loss": 3.687708978636114, + "tokens_seen": 896532480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678234704112337, + "loss": 1.4303, + "theoretical_loss": 3.68765663456329, + "tokens_seen": 896663552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036780341023069207, + "loss": 1.4784, + "theoretical_loss": 3.6876043002834997, + "tokens_seen": 896794624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036778335005015043, + "loss": 1.438, + "theoretical_loss": 3.6875519757934807, + "tokens_seen": 896925696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036776328986960885, + "loss": 1.2726, + "theoretical_loss": 3.687499661089971, + "tokens_seen": 897056768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677432296890672, + "loss": 1.4542, + "theoretical_loss": 3.687447356169711, + "tokens_seen": 897187840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036772316950852557, + "loss": 1.4326, + "theoretical_loss": 3.6873950610294415, + "tokens_seen": 897318912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036770310932798393, + "loss": 1.2781, + "theoretical_loss": 3.6873427756659067, + "tokens_seen": 897449984 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036768304914744235, + "loss": 1.3362, + "theoretical_loss": 3.687290500075851, + "tokens_seen": 897581056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676629889669007, + "loss": 1.2986, + "theoretical_loss": 3.6872382342560206, + "tokens_seen": 897712128 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.41031280159950256, + "objective/train/avg_token_score": 0.009382237680256367, + "objective/train/docs_used": 515835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.192286729812622, + "objective/train/original_loss": 3.765613079071045, + "objective/train/theoretical_loss": 3.6871859782031624, + "objective/train/tokens_used": 897843200, + "objective/train/unlikelihood_loss": 0.1680060476064682, + "theoretical_loss": 3.6871859782031624, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676429287863591, + "loss": 1.2313, + "theoretical_loss": 3.6871859782031624, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036762286860581744, + "loss": 1.489, + "theoretical_loss": 3.6871337319140274, + "tokens_seen": 897974272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676028084252758, + "loss": 1.415, + "theoretical_loss": 3.6870814953853657, + "tokens_seen": 898105344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675827482447342, + "loss": 1.5738, + "theoretical_loss": 3.68702926861393, + "tokens_seen": 898236416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675626880641926, + "loss": 1.3073, + "theoretical_loss": 3.6869770515964744, + "tokens_seen": 898367488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036754262788365094, + "loss": 1.4549, + "theoretical_loss": 3.686924844329755, + "tokens_seen": 898498560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675225677031093, + "loss": 1.3477, + "theoretical_loss": 3.6868726468105275, + "tokens_seen": 898629632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675025075225677, + "loss": 1.4256, + "theoretical_loss": 3.686820459035552, + "tokens_seen": 898760704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674824473420261, + "loss": 1.4993, + "theoretical_loss": 3.686768281001588, + "tokens_seen": 898891776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036746238716148444, + "loss": 1.4966, + "theoretical_loss": 3.6867161127053976, + "tokens_seen": 899022848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674423269809428, + "loss": 1.4447, + "theoretical_loss": 3.6866639541437447, + "tokens_seen": 899153920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036742226680040117, + "loss": 1.4297, + "theoretical_loss": 3.686611805313393, + "tokens_seen": 899284992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674022066198596, + "loss": 1.3744, + "theoretical_loss": 3.68655966621111, + "tokens_seen": 899416064 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.37591642141342163, + "objective/train/avg_token_score": 0.02023055963218212, + "objective/train/docs_used": 516375, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.303239107131958, + "objective/train/original_loss": 3.9634275436401367, + "objective/train/theoretical_loss": 3.6865336003069844, + "objective/train/tokens_used": 899481600, + "objective/train/unlikelihood_loss": 0.15485051274299622, + "theoretical_loss": 3.6865336003069844, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036738214643931795, + "loss": 1.3553, + "theoretical_loss": 3.6865075368336635, + "tokens_seen": 899547136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673620862587763, + "loss": 1.2734, + "theoretical_loss": 3.686455417177823, + "tokens_seen": 899678208 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036734202607823467, + "loss": 1.2157, + "theoretical_loss": 3.6864033072403593, + "tokens_seen": 899809280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673219658976931, + "loss": 1.4882, + "theoretical_loss": 3.6863512070180455, + "tokens_seen": 899940352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036730190571715145, + "loss": 1.3988, + "theoretical_loss": 3.686299116507655, + "tokens_seen": 900071424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672818455366098, + "loss": 1.4214, + "theoretical_loss": 3.6862470357059642, + "tokens_seen": 900202496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672617853560682, + "loss": 1.441, + "theoretical_loss": 3.6861949646097507, + "tokens_seen": 900333568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036724172517552654, + "loss": 1.5636, + "theoretical_loss": 3.6861429032157926, + "tokens_seen": 900464640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036722166499498495, + "loss": 1.3268, + "theoretical_loss": 3.6860908515208703, + "tokens_seen": 900595712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672016048144433, + "loss": 1.2911, + "theoretical_loss": 3.686038809521766, + "tokens_seen": 900726784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671815446339017, + "loss": 1.5481, + "theoretical_loss": 3.685986777215263, + "tokens_seen": 900857856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036716148445336004, + "loss": 1.2527, + "theoretical_loss": 3.6859347545981462, + "tokens_seen": 900988928 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.34011486172676086, + "objective/train/avg_token_score": 0.01983329840004444, + "objective/train/docs_used": 517599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1051846742630005, + "objective/train/original_loss": 4.3793134689331055, + "objective/train/theoretical_loss": 3.685882741667202, + "objective/train/tokens_used": 901120000, + "objective/train/unlikelihood_loss": 0.176338791847229, + "theoretical_loss": 3.685882741667202, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036714142427281846, + "loss": 1.3236, + "theoretical_loss": 3.685882741667202, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671213640922768, + "loss": 1.3387, + "theoretical_loss": 3.6858307384192184, + "tokens_seen": 901251072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036710130391173524, + "loss": 1.5136, + "theoretical_loss": 3.6857787448509853, + "tokens_seen": 901382144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670812437311936, + "loss": 1.3401, + "theoretical_loss": 3.685726760959293, + "tokens_seen": 901513216 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367061183550652, + "loss": 1.4146, + "theoretical_loss": 3.685674786740935, + "tokens_seen": 901644288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670411233701104, + "loss": 1.3162, + "theoretical_loss": 3.6856228221927054, + "tokens_seen": 901775360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036702106318956874, + "loss": 1.372, + "theoretical_loss": 3.685570867311399, + "tokens_seen": 901906432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670010030090271, + "loss": 1.4588, + "theoretical_loss": 3.6855189220938134, + "tokens_seen": 902037504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036698094282848546, + "loss": 1.3679, + "theoretical_loss": 3.6854669865367473, + "tokens_seen": 902168576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669608826479439, + "loss": 1.5033, + "theoretical_loss": 3.685415060637002, + "tokens_seen": 902299648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036694082246740224, + "loss": 1.4213, + "theoretical_loss": 3.6853631443913772, + "tokens_seen": 902430720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669207622868606, + "loss": 1.3596, + "theoretical_loss": 3.685311237796678, + "tokens_seen": 902561792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036690070210631897, + "loss": 1.4151, + "theoretical_loss": 3.685259340849708, + "tokens_seen": 902692864 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.32923999428749084, + "objective/train/avg_token_score": 0.007819044403731823, + "objective/train/docs_used": 518319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1362106800079346, + "objective/train/original_loss": 3.916128158569336, + "objective/train/theoretical_loss": 3.685233395993124, + "objective/train/tokens_used": 902758400, + "objective/train/unlikelihood_loss": 0.20320723950862885, + "theoretical_loss": 3.685233395993124, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668806419257774, + "loss": 1.3142, + "theoretical_loss": 3.6852074535472745, + "tokens_seen": 902823936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036686058174523575, + "loss": 1.466, + "theoretical_loss": 3.685155575886185, + "tokens_seen": 902955008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668405215646941, + "loss": 1.3675, + "theoretical_loss": 3.6851037078632483, + "tokens_seen": 903086080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036682046138415247, + "loss": 1.311, + "theoretical_loss": 3.685051849475276, + "tokens_seen": 903217152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036680040120361083, + "loss": 1.408, + "theoretical_loss": 3.6850000007190804, + "tokens_seen": 903348224 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036678034102306925, + "loss": 1.4158, + "theoretical_loss": 3.684948161591475, + "tokens_seen": 903479296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667602808425276, + "loss": 1.2925, + "theoretical_loss": 3.684896332089276, + "tokens_seen": 903610368 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366740220661986, + "loss": 1.5069, + "theoretical_loss": 3.6848445122093, + "tokens_seen": 903741440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036672016048144434, + "loss": 1.4295, + "theoretical_loss": 3.6847927019483646, + "tokens_seen": 903872512 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036670010030090275, + "loss": 1.2953, + "theoretical_loss": 3.6847409013032912, + "tokens_seen": 904003584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666800401203611, + "loss": 1.268, + "theoretical_loss": 3.6846891102709005, + "tokens_seen": 904134656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666599799398195, + "loss": 1.3328, + "theoretical_loss": 3.684637328848016, + "tokens_seen": 904265728 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.4965175986289978, + "objective/train/avg_token_score": 0.01438069436699152, + "objective/train/docs_used": 519704, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6696712970733643, + "objective/train/original_loss": 3.9789063930511475, + "objective/train/theoretical_loss": 3.684585557031461, + "objective/train/tokens_used": 904396800, + "objective/train/unlikelihood_loss": 0.19375556707382202, + "theoretical_loss": 3.684585557031461, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036663991975927784, + "loss": 1.3658, + "theoretical_loss": 3.684585557031461, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666198595787362, + "loss": 1.2461, + "theoretical_loss": 3.684533794818063, + "tokens_seen": 904527872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665997993981946, + "loss": 1.5455, + "theoretical_loss": 3.684482042204649, + "tokens_seen": 904658944 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366579739217653, + "loss": 1.3851, + "theoretical_loss": 3.6844302991880484, + "tokens_seen": 904790016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036655967903711134, + "loss": 1.4233, + "theoretical_loss": 3.6843785657650905, + "tokens_seen": 904921088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665396188565697, + "loss": 1.3615, + "theoretical_loss": 3.684326841932609, + "tokens_seen": 905052160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665195586760281, + "loss": 1.1474, + "theoretical_loss": 3.684275127687436, + "tokens_seen": 905183232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664994984954865, + "loss": 1.4166, + "theoretical_loss": 3.684223423026408, + "tokens_seen": 905314304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036647943831494484, + "loss": 1.6052, + "theoretical_loss": 3.68417172794636, + "tokens_seen": 905445376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664593781344032, + "loss": 1.4899, + "theoretical_loss": 3.684120042444132, + "tokens_seen": 905576448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036643931795386157, + "loss": 1.4149, + "theoretical_loss": 3.6840683665165614, + "tokens_seen": 905707520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036641925777332, + "loss": 1.211, + "theoretical_loss": 3.6840167001604907, + "tokens_seen": 905838592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036639919759277835, + "loss": 1.3265, + "theoretical_loss": 3.683965043372762, + "tokens_seen": 905969664 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.4700024425983429, + "objective/train/avg_token_score": 0.024333622306585312, + "objective/train/docs_used": 520483, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.445951223373413, + "objective/train/original_loss": 3.5988359451293945, + "objective/train/theoretical_loss": 3.6839392185660396, + "objective/train/tokens_used": 906035200, + "objective/train/unlikelihood_loss": 0.16731397807598114, + "theoretical_loss": 3.6839392185660396, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663791374122367, + "loss": 1.4668, + "theoretical_loss": 3.6839133961502197, + "tokens_seen": 906100736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036635907723169507, + "loss": 1.4015, + "theoretical_loss": 3.6838617584897086, + "tokens_seen": 906231808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663390170511535, + "loss": 1.35, + "theoretical_loss": 3.6838101303880766, + "tokens_seen": 906362880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036631895687061185, + "loss": 1.3259, + "theoretical_loss": 3.6837585118421723, + "tokens_seen": 906493952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662988966900702, + "loss": 1.3912, + "theoretical_loss": 3.6837069028488454, + "tokens_seen": 906625024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662788365095286, + "loss": 1.3489, + "theoretical_loss": 3.683655303404947, + "tokens_seen": 906756096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036625877632898694, + "loss": 1.333, + "theoretical_loss": 3.6836037135073307, + "tokens_seen": 906887168 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036623871614844535, + "loss": 1.357, + "theoretical_loss": 3.683552133152851, + "tokens_seen": 907018240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662186559679037, + "loss": 1.315, + "theoretical_loss": 3.683500562338364, + "tokens_seen": 907149312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003661985957873621, + "loss": 1.5374, + "theoretical_loss": 3.6834490010607266, + "tokens_seen": 907280384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036617853560682044, + "loss": 1.3291, + "theoretical_loss": 3.6833974493167982, + "tokens_seen": 907411456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036615847542627886, + "loss": 1.4503, + "theoretical_loss": 3.6833459071034396, + "tokens_seen": 907542528 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4754398763179779, + "objective/train/avg_token_score": 0.017755771055817604, + "objective/train/docs_used": 521740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6788755655288696, + "objective/train/original_loss": 3.7448816299438477, + "objective/train/theoretical_loss": 3.6832943744175126, + "objective/train/tokens_used": 907673600, + "objective/train/unlikelihood_loss": 0.16688300669193268, + "theoretical_loss": 3.6832943744175126, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661384152457372, + "loss": 1.4405, + "theoretical_loss": 3.6832943744175126, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661183550651956, + "loss": 1.4465, + "theoretical_loss": 3.683242851255881, + "tokens_seen": 907804672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036609829488465394, + "loss": 1.4747, + "theoretical_loss": 3.6831913376154084, + "tokens_seen": 907935744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036607823470411236, + "loss": 1.2997, + "theoretical_loss": 3.6831398334929624, + "tokens_seen": 908066816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660581745235707, + "loss": 1.4876, + "theoretical_loss": 3.683088338885411, + "tokens_seen": 908197888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660381143430291, + "loss": 1.4118, + "theoretical_loss": 3.6830368537896225, + "tokens_seen": 908328960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036601805416248745, + "loss": 1.2578, + "theoretical_loss": 3.6829853782024697, + "tokens_seen": 908460032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659979939819458, + "loss": 1.4569, + "theoretical_loss": 3.6829339121208227, + "tokens_seen": 908591104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659779338014042, + "loss": 1.4635, + "theoretical_loss": 3.682882455541557, + "tokens_seen": 908722176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659578736208626, + "loss": 1.2033, + "theoretical_loss": 3.682831008461547, + "tokens_seen": 908853248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036593781344032095, + "loss": 1.277, + "theoretical_loss": 3.6827795708776705, + "tokens_seen": 908984320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659177532597793, + "loss": 1.614, + "theoretical_loss": 3.6827281427868046, + "tokens_seen": 909115392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036589769307923773, + "loss": 1.41, + "theoretical_loss": 3.6826767241858294, + "tokens_seen": 909246464 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4004765450954437, + "objective/train/avg_token_score": 0.015282467007637024, + "objective/train/docs_used": 522301, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4406872987747192, + "objective/train/original_loss": 3.9599339962005615, + "objective/train/theoretical_loss": 3.6826510184430763, + "objective/train/tokens_used": 909312000, + "objective/train/unlikelihood_loss": 0.12224063277244568, + "theoretical_loss": 3.6826510184430763, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658776328986961, + "loss": 1.5165, + "theoretical_loss": 3.6826253150716264, + "tokens_seen": 909377536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036585757271815445, + "loss": 1.3413, + "theoretical_loss": 3.682573915441078, + "tokens_seen": 909508608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658375125376128, + "loss": 1.4452, + "theoretical_loss": 3.682522525291069, + "tokens_seen": 909639680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658174523570712, + "loss": 1.3746, + "theoretical_loss": 3.6824711446184843, + "tokens_seen": 909770752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657973921765296, + "loss": 1.4758, + "theoretical_loss": 3.6824197734202118, + "tokens_seen": 909901824 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036577733199598796, + "loss": 1.5073, + "theoretical_loss": 3.6823684116931386, + "tokens_seen": 910032896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657572718154463, + "loss": 1.4527, + "theoretical_loss": 3.6823170594341565, + "tokens_seen": 910163968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657372116349047, + "loss": 1.4664, + "theoretical_loss": 3.6822657166401562, + "tokens_seen": 910295040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657171514543631, + "loss": 1.4736, + "theoretical_loss": 3.6822143833080307, + "tokens_seen": 910426112 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036569709127382146, + "loss": 1.6285, + "theoretical_loss": 3.682163059434674, + "tokens_seen": 910557184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656770310932798, + "loss": 1.3849, + "theoretical_loss": 3.6821117450169836, + "tokens_seen": 910688256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656569709127382, + "loss": 1.4939, + "theoretical_loss": 3.6820604400518553, + "tokens_seen": 910819328 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4665811359882355, + "objective/train/avg_token_score": 0.019231745973229408, + "objective/train/docs_used": 523753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5149257183074951, + "objective/train/original_loss": 3.5024781227111816, + "objective/train/theoretical_loss": 3.682009144536188, + "objective/train/tokens_used": 910950400, + "objective/train/unlikelihood_loss": 0.1984291970729828, + "theoretical_loss": 3.682009144536188, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563691073219655, + "loss": 1.444, + "theoretical_loss": 3.682009144536188, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036561685055165496, + "loss": 1.4421, + "theoretical_loss": 3.681957858466883, + "tokens_seen": 911081472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655967903711134, + "loss": 1.436, + "theoretical_loss": 3.681906581840842, + "tokens_seen": 911212544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036557673019057174, + "loss": 1.5163, + "theoretical_loss": 3.681855314654967, + "tokens_seen": 911343616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655566700100301, + "loss": 1.3416, + "theoretical_loss": 3.681804056906164, + "tokens_seen": 911474688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655366098294885, + "loss": 1.4586, + "theoretical_loss": 3.6817528085913382, + "tokens_seen": 911605760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655165496489469, + "loss": 1.3613, + "theoretical_loss": 3.6817015697073985, + "tokens_seen": 911736832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036549648946840525, + "loss": 1.2993, + "theoretical_loss": 3.6816503402512533, + "tokens_seen": 911867904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654764292878636, + "loss": 1.3053, + "theoretical_loss": 3.681599120219812, + "tokens_seen": 911998976 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036545636910732197, + "loss": 1.4122, + "theoretical_loss": 3.6815479096099883, + "tokens_seen": 912130048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654363089267804, + "loss": 1.5037, + "theoretical_loss": 3.6814967084186945, + "tokens_seen": 912261120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036541624874623875, + "loss": 1.5294, + "theoretical_loss": 3.681445516642847, + "tokens_seen": 912392192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653961885656971, + "loss": 1.3504, + "theoretical_loss": 3.68139433427936, + "tokens_seen": 912523264 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.5026881694793701, + "objective/train/avg_token_score": 0.0120518384501338, + "objective/train/docs_used": 524575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7356411218643188, + "objective/train/original_loss": 4.013809680938721, + "objective/train/theoretical_loss": 3.681368746626289, + "objective/train/tokens_used": 912588800, + "objective/train/unlikelihood_loss": 0.11147240549325943, + "theoretical_loss": 3.681368746626289, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036537612838515547, + "loss": 1.3834, + "theoretical_loss": 3.6813431613251533, + "tokens_seen": 912654336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653560682046139, + "loss": 1.4133, + "theoretical_loss": 3.6812919977771448, + "tokens_seen": 912785408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036533600802407225, + "loss": 1.3576, + "theoretical_loss": 3.6812408436322555, + "tokens_seen": 912916480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653159478435306, + "loss": 1.4012, + "theoretical_loss": 3.681189698887408, + "tokens_seen": 913047552 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365295887662989, + "loss": 1.3104, + "theoretical_loss": 3.681138563539526, + "tokens_seen": 913178624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036527582748244734, + "loss": 1.5394, + "theoretical_loss": 3.6810874375855342, + "tokens_seen": 913309696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036525576730190575, + "loss": 1.4473, + "theoretical_loss": 3.681036321022359, + "tokens_seen": 913440768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652357071213641, + "loss": 1.4352, + "theoretical_loss": 3.680985213846929, + "tokens_seen": 913571840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652156469408225, + "loss": 1.4146, + "theoretical_loss": 3.6809341160561724, + "tokens_seen": 913702912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036519558676028084, + "loss": 1.5235, + "theoretical_loss": 3.6808830276470212, + "tokens_seen": 913833984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036517552657973926, + "loss": 1.5282, + "theoretical_loss": 3.6808319486164067, + "tokens_seen": 913965056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651554663991976, + "loss": 1.4958, + "theoretical_loss": 3.6807808789612633, + "tokens_seen": 914096128 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.43957722187042236, + "objective/train/avg_token_score": 0.007488319184631109, + "objective/train/docs_used": 525722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4323551654815674, + "objective/train/original_loss": 3.336665630340576, + "objective/train/theoretical_loss": 3.680729818678526, + "objective/train/tokens_used": 914227200, + "objective/train/unlikelihood_loss": 0.25074630975723267, + "theoretical_loss": 3.680729818678526, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365135406218656, + "loss": 1.4257, + "theoretical_loss": 3.680729818678526, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036511534603811434, + "loss": 1.5125, + "theoretical_loss": 3.6806787677651323, + "tokens_seen": 914358272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036509528585757276, + "loss": 1.4302, + "theoretical_loss": 3.680627726218019, + "tokens_seen": 914489344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650752256770311, + "loss": 1.3394, + "theoretical_loss": 3.6805766940341256, + "tokens_seen": 914620416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650551654964895, + "loss": 1.3574, + "theoretical_loss": 3.6805256712103938, + "tokens_seen": 914751488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036503510531594785, + "loss": 1.4032, + "theoretical_loss": 3.6804746577437655, + "tokens_seen": 914882560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650150451354062, + "loss": 1.5081, + "theoretical_loss": 3.680423653631184, + "tokens_seen": 915013632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649949849548646, + "loss": 1.4209, + "theoretical_loss": 3.680372658869596, + "tokens_seen": 915144704 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364974924774323, + "loss": 1.3068, + "theoretical_loss": 3.680321673455947, + "tokens_seen": 915275776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036495486459378135, + "loss": 1.4659, + "theoretical_loss": 3.6802706973871855, + "tokens_seen": 915406848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649348044132397, + "loss": 1.3211, + "theoretical_loss": 3.680219730660261, + "tokens_seen": 915537920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036491474423269813, + "loss": 1.2844, + "theoretical_loss": 3.680168773272124, + "tokens_seen": 915668992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648946840521565, + "loss": 1.4376, + "theoretical_loss": 3.680117825219728, + "tokens_seen": 915800064 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.46560361981391907, + "objective/train/avg_token_score": 0.014752034097909927, + "objective/train/docs_used": 526330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5809056758880615, + "objective/train/original_loss": 3.7449471950531006, + "objective/train/theoretical_loss": 3.6800923546934805, + "objective/train/tokens_used": 915865600, + "objective/train/unlikelihood_loss": 0.24874065816402435, + "theoretical_loss": 3.6800923546934805, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036487462387161485, + "loss": 1.4686, + "theoretical_loss": 3.680066886500026, + "tokens_seen": 915931136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648545636910732, + "loss": 1.515, + "theoretical_loss": 3.6800159571099735, + "tokens_seen": 916062208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648345035105316, + "loss": 1.3529, + "theoretical_loss": 3.679965037046527, + "tokens_seen": 916193280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036481444332999, + "loss": 1.4091, + "theoretical_loss": 3.6799141263066444, + "tokens_seen": 916324352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036479438314944836, + "loss": 1.4572, + "theoretical_loss": 3.6798632248872862, + "tokens_seen": 916455424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647743229689067, + "loss": 1.3951, + "theoretical_loss": 3.6798123327854126, + "tokens_seen": 916586496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647542627883651, + "loss": 1.3825, + "theoretical_loss": 3.6797614499979865, + "tokens_seen": 916717568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647342026078235, + "loss": 1.3257, + "theoretical_loss": 3.679710576521971, + "tokens_seen": 916848640 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036471414242728186, + "loss": 1.7206, + "theoretical_loss": 3.679659712354332, + "tokens_seen": 916979712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646940822467402, + "loss": 1.5312, + "theoretical_loss": 3.6796088574920356, + "tokens_seen": 917110784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646740220661986, + "loss": 1.5271, + "theoretical_loss": 3.679558011932051, + "tokens_seen": 917241856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036465396188565695, + "loss": 1.3848, + "theoretical_loss": 3.6795071756713464, + "tokens_seen": 917372928 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.3484237492084503, + "objective/train/avg_token_score": 0.024387724697589874, + "objective/train/docs_used": 527026, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1906174421310425, + "objective/train/original_loss": 3.6521694660186768, + "objective/train/theoretical_loss": 3.6794563487068936, + "objective/train/tokens_used": 917504000, + "objective/train/unlikelihood_loss": 0.20489788055419922, + "theoretical_loss": 3.6794563487068936, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036463390170511536, + "loss": 1.4198, + "theoretical_loss": 3.6794563487068936, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646138415245737, + "loss": 1.3374, + "theoretical_loss": 3.6794055310356644, + "tokens_seen": 917635072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645937813440321, + "loss": 1.3722, + "theoretical_loss": 3.679354722654633, + "tokens_seen": 917766144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036457372116349045, + "loss": 1.3713, + "theoretical_loss": 3.6793039235607745, + "tokens_seen": 917897216 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036455366098294887, + "loss": 1.5907, + "theoretical_loss": 3.6792531337510654, + "tokens_seen": 918028288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036453360080240723, + "loss": 1.5397, + "theoretical_loss": 3.6792023532224833, + "tokens_seen": 918159360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645135406218656, + "loss": 1.2746, + "theoretical_loss": 3.6791515819720084, + "tokens_seen": 918290432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036449348044132395, + "loss": 1.3545, + "theoretical_loss": 3.6791008199966218, + "tokens_seen": 918421504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644734202607823, + "loss": 1.3488, + "theoretical_loss": 3.6790500672933044, + "tokens_seen": 918552576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036445336008024073, + "loss": 1.4063, + "theoretical_loss": 3.678999323859041, + "tokens_seen": 918683648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644332998996991, + "loss": 1.336, + "theoretical_loss": 3.6789485896908163, + "tokens_seen": 918814720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036441323971915746, + "loss": 1.4644, + "theoretical_loss": 3.6788978647856174, + "tokens_seen": 918945792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643931795386158, + "loss": 1.3478, + "theoretical_loss": 3.678847149140431, + "tokens_seen": 919076864 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4171554148197174, + "objective/train/avg_token_score": 0.0110766077414155, + "objective/train/docs_used": 528220, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.495461106300354, + "objective/train/original_loss": 3.8933098316192627, + "objective/train/theoretical_loss": 3.678821794789402, + "objective/train/tokens_used": 919142400, + "objective/train/unlikelihood_loss": 0.19146324694156647, + "theoretical_loss": 3.678821794789402, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036437311935807424, + "loss": 1.4004, + "theoretical_loss": 3.6787964427522475, + "tokens_seen": 919207936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643530591775326, + "loss": 1.4487, + "theoretical_loss": 3.678745745618057, + "tokens_seen": 919339008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036433299899699096, + "loss": 1.4017, + "theoretical_loss": 3.678695057734852, + "tokens_seen": 919470080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643129388164493, + "loss": 1.4275, + "theoretical_loss": 3.6786443790996257, + "tokens_seen": 919601152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642928786359077, + "loss": 1.589, + "theoretical_loss": 3.6785937097093733, + "tokens_seen": 919732224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642728184553661, + "loss": 1.278, + "theoretical_loss": 3.678543049561091, + "tokens_seen": 919863296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036425275827482446, + "loss": 1.3989, + "theoretical_loss": 3.678492398651777, + "tokens_seen": 919994368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642326980942828, + "loss": 1.51, + "theoretical_loss": 3.6784417569784296, + "tokens_seen": 920125440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642126379137412, + "loss": 1.3204, + "theoretical_loss": 3.6783911245380505, + "tokens_seen": 920256512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641925777331996, + "loss": 1.38, + "theoretical_loss": 3.6783405013276402, + "tokens_seen": 920387584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036417251755265797, + "loss": 1.4022, + "theoretical_loss": 3.678289887344204, + "tokens_seen": 920518656 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036415245737211633, + "loss": 1.2561, + "theoretical_loss": 3.6782392825847445, + "tokens_seen": 920649728 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.3925342261791229, + "objective/train/avg_token_score": 0.0071686734445393085, + "objective/train/docs_used": 528886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.263667106628418, + "objective/train/original_loss": 3.8773934841156006, + "objective/train/theoretical_loss": 3.6781886870462692, + "objective/train/tokens_used": 920780800, + "objective/train/unlikelihood_loss": 0.1516854614019394, + "theoretical_loss": 3.6781886870462692, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641323971915747, + "loss": 1.3657, + "theoretical_loss": 3.6781886870462692, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036411233701103316, + "loss": 1.3336, + "theoretical_loss": 3.6781381007257856, + "tokens_seen": 920911872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640922768304915, + "loss": 1.4774, + "theoretical_loss": 3.6780875236203023, + "tokens_seen": 921042944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640722166499499, + "loss": 1.2788, + "theoretical_loss": 3.6780369557268298, + "tokens_seen": 921174016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036405215646940825, + "loss": 1.5636, + "theoretical_loss": 3.6779863970423796, + "tokens_seen": 921305088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640320962888666, + "loss": 1.4095, + "theoretical_loss": 3.677935847563965, + "tokens_seen": 921436160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364012036108325, + "loss": 1.4448, + "theoretical_loss": 3.6778853072886006, + "tokens_seen": 921567232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639919759277834, + "loss": 1.3299, + "theoretical_loss": 3.6778347762133023, + "tokens_seen": 921698304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036397191574724175, + "loss": 1.3249, + "theoretical_loss": 3.677784254335087, + "tokens_seen": 921829376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639518555667001, + "loss": 1.3842, + "theoretical_loss": 3.6777337416509743, + "tokens_seen": 921960448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036393179538615853, + "loss": 1.4383, + "theoretical_loss": 3.677683238157984, + "tokens_seen": 922091520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639117352056169, + "loss": 1.3453, + "theoretical_loss": 3.6776327438531373, + "tokens_seen": 922222592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036389167502507525, + "loss": 1.5541, + "theoretical_loss": 3.6775822587334566, + "tokens_seen": 922353664 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4955400824546814, + "objective/train/avg_token_score": 0.008413422852754593, + "objective/train/docs_used": 530064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5333219766616821, + "objective/train/original_loss": 3.174649953842163, + "objective/train/theoretical_loss": 3.677557019617124, + "objective/train/tokens_used": 922419200, + "objective/train/unlikelihood_loss": 0.2531989514827728, + "theoretical_loss": 3.677557019617124, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638716148445336, + "loss": 1.3604, + "theoretical_loss": 3.6775317827959673, + "tokens_seen": 922484736 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363851554663992, + "loss": 1.2959, + "theoretical_loss": 3.677481316037694, + "tokens_seen": 922615808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638314944834504, + "loss": 1.2969, + "theoretical_loss": 3.6774308584556645, + "tokens_seen": 922746880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036381143430290876, + "loss": 1.3554, + "theoretical_loss": 3.677380410046907, + "tokens_seen": 922877952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637913741223671, + "loss": 1.4652, + "theoretical_loss": 3.677329970808451, + "tokens_seen": 923009024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637713139418255, + "loss": 1.6296, + "theoretical_loss": 3.677279540737328, + "tokens_seen": 923140096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637512537612839, + "loss": 1.4082, + "theoretical_loss": 3.67722911983057, + "tokens_seen": 923271168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036373119358074226, + "loss": 1.4872, + "theoretical_loss": 3.6771787080852123, + "tokens_seen": 923402240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637111334002006, + "loss": 1.2982, + "theoretical_loss": 3.6771283054982886, + "tokens_seen": 923533312 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363691073219659, + "loss": 1.6098, + "theoretical_loss": 3.677077912066837, + "tokens_seen": 923664384 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036367101303911735, + "loss": 1.3882, + "theoretical_loss": 3.6770275277878945, + "tokens_seen": 923795456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036365095285857576, + "loss": 1.377, + "theoretical_loss": 3.676977152658501, + "tokens_seen": 923926528 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.42124876379966736, + "objective/train/avg_token_score": 0.011774392798542976, + "objective/train/docs_used": 530632, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5075197219848633, + "objective/train/original_loss": 3.5660994052886963, + "objective/train/theoretical_loss": 3.676926786675698, + "objective/train/tokens_used": 924057600, + "objective/train/unlikelihood_loss": 0.19304485619068146, + "theoretical_loss": 3.676926786675698, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636308926780341, + "loss": 1.3734, + "theoretical_loss": 3.676926786675698, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636108324974925, + "loss": 1.3719, + "theoretical_loss": 3.6768764298365264, + "tokens_seen": 924188672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036359077231695085, + "loss": 1.4656, + "theoretical_loss": 3.6768260821380307, + "tokens_seen": 924319744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036357071213640927, + "loss": 1.4986, + "theoretical_loss": 3.6767757435772555, + "tokens_seen": 924450816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036355065195586763, + "loss": 1.5116, + "theoretical_loss": 3.6767254141512478, + "tokens_seen": 924581888 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363530591775326, + "loss": 1.3521, + "theoretical_loss": 3.676675093857054, + "tokens_seen": 924712960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036351053159478435, + "loss": 1.4292, + "theoretical_loss": 3.6766247826917247, + "tokens_seen": 924844032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634904714142427, + "loss": 1.4754, + "theoretical_loss": 3.6765744806523095, + "tokens_seen": 924975104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036347041123370113, + "loss": 1.3813, + "theoretical_loss": 3.6765241877358603, + "tokens_seen": 925106176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634503510531595, + "loss": 1.4882, + "theoretical_loss": 3.6764739039394305, + "tokens_seen": 925237248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036343029087261786, + "loss": 1.3224, + "theoretical_loss": 3.676423629260075, + "tokens_seen": 925368320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634102306920762, + "loss": 1.6143, + "theoretical_loss": 3.676373363694849, + "tokens_seen": 925499392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036339017051153464, + "loss": 1.4381, + "theoretical_loss": 3.67632310724081, + "tokens_seen": 925630464 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.33290567994117737, + "objective/train/avg_token_score": 0.016993219032883644, + "objective/train/docs_used": 532017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.070075273513794, + "objective/train/original_loss": 3.990078926086426, + "objective/train/theoretical_loss": 3.6762979824295665, + "objective/train/tokens_used": 925696000, + "objective/train/unlikelihood_loss": 0.149410218000412, + "theoretical_loss": 3.6762979824295665, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363370110330993, + "loss": 1.5136, + "theoretical_loss": 3.6762728598950174, + "tokens_seen": 925761536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036335005015045136, + "loss": 1.4262, + "theoretical_loss": 3.6762226216545297, + "tokens_seen": 925892608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633299899699097, + "loss": 1.3022, + "theoretical_loss": 3.67617239251641, + "tokens_seen": 926023680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633099297893681, + "loss": 1.3757, + "theoretical_loss": 3.6761221724777204, + "tokens_seen": 926154752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632898696088265, + "loss": 1.3988, + "theoretical_loss": 3.6760719615355244, + "tokens_seen": 926285824 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036326980942828486, + "loss": 1.4277, + "theoretical_loss": 3.676021759686889, + "tokens_seen": 926416896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632497492477432, + "loss": 1.45, + "theoretical_loss": 3.6759715669288795, + "tokens_seen": 926547968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632296890672016, + "loss": 1.2156, + "theoretical_loss": 3.6759213832585647, + "tokens_seen": 926679040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036320962888666, + "loss": 1.5234, + "theoretical_loss": 3.6758712086730148, + "tokens_seen": 926810112 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036318956870611837, + "loss": 1.3449, + "theoretical_loss": 3.6758210431692993, + "tokens_seen": 926941184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036316950852557673, + "loss": 1.4607, + "theoretical_loss": 3.675770886744492, + "tokens_seen": 927072256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631494483450351, + "loss": 1.259, + "theoretical_loss": 3.6757207393956657, + "tokens_seen": 927203328 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.33815982937812805, + "objective/train/avg_token_score": 0.008303831331431866, + "objective/train/docs_used": 532753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.272034764289856, + "objective/train/original_loss": 3.9469377994537354, + "objective/train/theoretical_loss": 3.6756706011198963, + "objective/train/tokens_used": 927334400, + "objective/train/unlikelihood_loss": 0.2563776969909668, + "theoretical_loss": 3.6756706011198963, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036312938816449345, + "loss": 1.3041, + "theoretical_loss": 3.6756706011198963, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036310932798395187, + "loss": 1.3601, + "theoretical_loss": 3.675620471914259, + "tokens_seen": 927465472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036308926780341023, + "loss": 1.4568, + "theoretical_loss": 3.675570351775832, + "tokens_seen": 927596544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630692076228686, + "loss": 1.4514, + "theoretical_loss": 3.6755202407016943, + "tokens_seen": 927727616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036304914744232696, + "loss": 1.1919, + "theoretical_loss": 3.675470138688927, + "tokens_seen": 927858688 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036302908726178537, + "loss": 1.426, + "theoretical_loss": 3.6754200457346107, + "tokens_seen": 927989760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036300902708124374, + "loss": 1.4105, + "theoretical_loss": 3.6753699618358295, + "tokens_seen": 928120832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629889669007021, + "loss": 1.5292, + "theoretical_loss": 3.6753198869896675, + "tokens_seen": 928251904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036296890672016046, + "loss": 1.3343, + "theoretical_loss": 3.6752698211932104, + "tokens_seen": 928382976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629488465396188, + "loss": 1.2812, + "theoretical_loss": 3.675219764443546, + "tokens_seen": 928514048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036292878635907724, + "loss": 1.3467, + "theoretical_loss": 3.675169716737762, + "tokens_seen": 928645120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629087261785356, + "loss": 1.2943, + "theoretical_loss": 3.6751196780729485, + "tokens_seen": 928776192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036288866599799396, + "loss": 1.3846, + "theoretical_loss": 3.6750696484461978, + "tokens_seen": 928907264 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.44086021184921265, + "objective/train/avg_token_score": 0.01040811836719513, + "objective/train/docs_used": 534166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4418307542800903, + "objective/train/original_loss": 3.6829819679260254, + "objective/train/theoretical_loss": 3.6750446370211867, + "objective/train/tokens_used": 928972800, + "objective/train/unlikelihood_loss": 0.15786431729793549, + "theoretical_loss": 3.6750446370211867, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628686058174523, + "loss": 1.4228, + "theoretical_loss": 3.675019627854601, + "tokens_seen": 929038336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036284854563691074, + "loss": 1.4584, + "theoretical_loss": 3.6749696162952525, + "tokens_seen": 929169408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628284854563691, + "loss": 1.3367, + "theoretical_loss": 3.674919613765248, + "tokens_seen": 929300480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036280842527582747, + "loss": 1.3202, + "theoretical_loss": 3.6748696202616835, + "tokens_seen": 929431552 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036278836509528583, + "loss": 1.3551, + "theoretical_loss": 3.674819635781658, + "tokens_seen": 929562624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036276830491474424, + "loss": 1.3624, + "theoretical_loss": 3.674769660322269, + "tokens_seen": 929693696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627482447342026, + "loss": 1.1669, + "theoretical_loss": 3.6747196938806184, + "tokens_seen": 929824768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036272818455366097, + "loss": 1.4714, + "theoretical_loss": 3.6746697364538083, + "tokens_seen": 929955840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036270812437311933, + "loss": 1.3716, + "theoretical_loss": 3.6746197880389415, + "tokens_seen": 930086912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626880641925777, + "loss": 1.3665, + "theoretical_loss": 3.674569848633123, + "tokens_seen": 930217984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626680040120361, + "loss": 1.5955, + "theoretical_loss": 3.6745199182334582, + "tokens_seen": 930349056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036264794383149447, + "loss": 1.4058, + "theoretical_loss": 3.674469996837055, + "tokens_seen": 930480128 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4736681282520294, + "objective/train/avg_token_score": 0.014163702726364136, + "objective/train/docs_used": 534638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5139890909194946, + "objective/train/original_loss": 3.3884758949279785, + "objective/train/theoretical_loss": 3.6744200844410217, + "objective/train/tokens_used": 930611200, + "objective/train/unlikelihood_loss": 0.15940915048122406, + "theoretical_loss": 3.6744200844410217, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036262788365095283, + "loss": 1.5254, + "theoretical_loss": 3.6744200844410217, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036260782347041125, + "loss": 1.4889, + "theoretical_loss": 3.674370181042469, + "tokens_seen": 930742272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036258776328986967, + "loss": 1.3047, + "theoretical_loss": 3.6743202866385074, + "tokens_seen": 930873344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036256770310932803, + "loss": 1.4197, + "theoretical_loss": 3.6742704012262504, + "tokens_seen": 931004416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625476429287864, + "loss": 1.4748, + "theoretical_loss": 3.674220524802811, + "tokens_seen": 931135488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036252758274824475, + "loss": 1.627, + "theoretical_loss": 3.674170657365305, + "tokens_seen": 931266560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625075225677031, + "loss": 1.4705, + "theoretical_loss": 3.6741207989108493, + "tokens_seen": 931397632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036248746238716153, + "loss": 1.429, + "theoretical_loss": 3.6740709494365618, + "tokens_seen": 931528704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624674022066199, + "loss": 1.1689, + "theoretical_loss": 3.6740211089395616, + "tokens_seen": 931659776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036244734202607826, + "loss": 1.3755, + "theoretical_loss": 3.6739712774169693, + "tokens_seen": 931790848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624272818455366, + "loss": 1.4209, + "theoretical_loss": 3.6739214548659076, + "tokens_seen": 931921920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036240722166499504, + "loss": 1.3317, + "theoretical_loss": 3.6738716412834984, + "tokens_seen": 932052992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623871614844534, + "loss": 1.289, + "theoretical_loss": 3.6738218366668685, + "tokens_seen": 932184064 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.5211387872695923, + "objective/train/avg_token_score": 0.027682365849614143, + "objective/train/docs_used": 535899, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7249022722244263, + "objective/train/original_loss": 3.63735032081604, + "objective/train/theoretical_loss": 3.673796937719821, + "objective/train/tokens_used": 932249600, + "objective/train/unlikelihood_loss": 0.16856487095355988, + "theoretical_loss": 3.673796937719821, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036236710130391176, + "loss": 1.5121, + "theoretical_loss": 3.673772041013142, + "tokens_seen": 932315136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623470411233701, + "loss": 1.3872, + "theoretical_loss": 3.673722254319446, + "tokens_seen": 932446208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623269809428285, + "loss": 1.4107, + "theoretical_loss": 3.6736724765829107, + "tokens_seen": 932577280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623069207622869, + "loss": 1.3352, + "theoretical_loss": 3.673622707800665, + "tokens_seen": 932708352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036228686058174526, + "loss": 1.4026, + "theoretical_loss": 3.6735729479698405, + "tokens_seen": 932839424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622668004012036, + "loss": 1.2742, + "theoretical_loss": 3.6735231970875697, + "tokens_seen": 932970496 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362246740220662, + "loss": 1.3438, + "theoretical_loss": 3.6734734551509867, + "tokens_seen": 933101568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622266800401204, + "loss": 1.3017, + "theoretical_loss": 3.673423722157226, + "tokens_seen": 933232640 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036220661985957877, + "loss": 1.3801, + "theoretical_loss": 3.673373998103425, + "tokens_seen": 933363712 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036218655967903713, + "loss": 1.2845, + "theoretical_loss": 3.673324282986721, + "tokens_seen": 933494784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621664994984955, + "loss": 1.3568, + "theoretical_loss": 3.673274576804254, + "tokens_seen": 933625856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214643931795385, + "loss": 1.2666, + "theoretical_loss": 3.673224879553163, + "tokens_seen": 933756928 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.3636363744735718, + "objective/train/avg_token_score": 0.0114415492862463, + "objective/train/docs_used": 536516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1716868877410889, + "objective/train/original_loss": 4.167331218719482, + "objective/train/theoretical_loss": 3.6731751912305914, + "objective/train/tokens_used": 933888000, + "objective/train/unlikelihood_loss": 0.14656978845596313, + "theoretical_loss": 3.6731751912305914, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036212637913741227, + "loss": 1.2869, + "theoretical_loss": 3.6731751912305914, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036210631895687063, + "loss": 1.4532, + "theoretical_loss": 3.673125511833682, + "tokens_seen": 934019072 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362086258776329, + "loss": 1.5492, + "theoretical_loss": 3.673075841359578, + "tokens_seen": 934150144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036206619859578736, + "loss": 1.3709, + "theoretical_loss": 3.6730261798054262, + "tokens_seen": 934281216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620461384152458, + "loss": 1.4321, + "theoretical_loss": 3.6729765271683736, + "tokens_seen": 934412288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036202607823470414, + "loss": 1.3982, + "theoretical_loss": 3.6729268834455686, + "tokens_seen": 934543360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620060180541625, + "loss": 1.4704, + "theoretical_loss": 3.672877248634161, + "tokens_seen": 934674432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036198595787362086, + "loss": 1.3945, + "theoretical_loss": 3.672827622731301, + "tokens_seen": 934805504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619658976930792, + "loss": 1.4295, + "theoretical_loss": 3.672778005734142, + "tokens_seen": 934936576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036194583751253764, + "loss": 1.2398, + "theoretical_loss": 3.672728397639837, + "tokens_seen": 935067648 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361925777331996, + "loss": 1.3717, + "theoretical_loss": 3.672678798445541, + "tokens_seen": 935198720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036190571715145436, + "loss": 1.3579, + "theoretical_loss": 3.672629208148411, + "tokens_seen": 935329792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618856569709127, + "loss": 1.5405, + "theoretical_loss": 3.6725796267456032, + "tokens_seen": 935460864 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.37939882278442383, + "objective/train/avg_token_score": 0.05874001234769821, + "objective/train/docs_used": 537462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2182044982910156, + "objective/train/original_loss": 3.779721975326538, + "objective/train/theoretical_loss": 3.672554839378683, + "objective/train/tokens_used": 935526400, + "objective/train/unlikelihood_loss": 0.16419436037540436, + "theoretical_loss": 3.672554839378683, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036186559679037114, + "loss": 1.44, + "theoretical_loss": 3.6725300542342776, + "tokens_seen": 935591936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618455366098295, + "loss": 1.253, + "theoretical_loss": 3.6724804906115933, + "tokens_seen": 935723008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036182547642928787, + "loss": 1.3851, + "theoretical_loss": 3.672430935874713, + "tokens_seen": 935854080 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036180541624874623, + "loss": 1.3661, + "theoretical_loss": 3.6723813900207984, + "tokens_seen": 935985152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036178535606820464, + "loss": 1.5423, + "theoretical_loss": 3.672331853047014, + "tokens_seen": 936116224 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361765295887663, + "loss": 1.2643, + "theoretical_loss": 3.6722823249505256, + "tokens_seen": 936247296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036174523570712137, + "loss": 1.2678, + "theoretical_loss": 3.672232805728499, + "tokens_seen": 936378368 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036172517552657973, + "loss": 1.4265, + "theoretical_loss": 3.672183295378103, + "tokens_seen": 936509440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617051153460381, + "loss": 1.3961, + "theoretical_loss": 3.6721337938965064, + "tokens_seen": 936640512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616850551654965, + "loss": 1.5047, + "theoretical_loss": 3.67208430128088, + "tokens_seen": 936771584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036166499498495487, + "loss": 1.4069, + "theoretical_loss": 3.672034817528396, + "tokens_seen": 936902656 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036164493480441323, + "loss": 1.2935, + "theoretical_loss": 3.6719853426362263, + "tokens_seen": 937033728 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.36638563871383667, + "objective/train/avg_token_score": 0.0227508544921875, + "objective/train/docs_used": 538123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2613353729248047, + "objective/train/original_loss": 3.9611504077911377, + "objective/train/theoretical_loss": 3.671935876601547, + "objective/train/tokens_used": 937164800, + "objective/train/unlikelihood_loss": 0.16599014401435852, + "theoretical_loss": 3.671935876601547, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616248746238716, + "loss": 1.3573, + "theoretical_loss": 3.671935876601547, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036160481444333, + "loss": 1.3769, + "theoretical_loss": 3.671886419421533, + "tokens_seen": 937295872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615847542627884, + "loss": 1.1926, + "theoretical_loss": 3.6718369710933607, + "tokens_seen": 937426944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036156469408224674, + "loss": 1.4, + "theoretical_loss": 3.6717875316142097, + "tokens_seen": 937558016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615446339017051, + "loss": 1.362, + "theoretical_loss": 3.6717381009812593, + "tokens_seen": 937689088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036152457372116346, + "loss": 1.3618, + "theoretical_loss": 3.67168867919169, + "tokens_seen": 937820160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615045135406219, + "loss": 1.2799, + "theoretical_loss": 3.6716392662426847, + "tokens_seen": 937951232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036148445336008024, + "loss": 1.2745, + "theoretical_loss": 3.6715898621314262, + "tokens_seen": 938082304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614643931795386, + "loss": 1.3213, + "theoretical_loss": 3.6715404668550997, + "tokens_seen": 938213376 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036144433299899697, + "loss": 1.4298, + "theoretical_loss": 3.671491080410892, + "tokens_seen": 938344448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614242728184554, + "loss": 1.326, + "theoretical_loss": 3.6714417027959887, + "tokens_seen": 938475520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036140421263791374, + "loss": 1.4222, + "theoretical_loss": 3.6713923340075802, + "tokens_seen": 938606592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613841524573721, + "loss": 1.3581, + "theoretical_loss": 3.671342974042856, + "tokens_seen": 938737664 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.35673263669013977, + "objective/train/avg_token_score": 0.005345341749489307, + "objective/train/docs_used": 539283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1492719650268555, + "objective/train/original_loss": 4.046452522277832, + "objective/train/theoretical_loss": 3.6713182973684972, + "objective/train/tokens_used": 938803200, + "objective/train/unlikelihood_loss": 0.1200408861041069, + "theoretical_loss": 3.6713182973684972, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036136409227683047, + "loss": 1.5519, + "theoretical_loss": 3.6712936228990065, + "tokens_seen": 938868736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036134403209628883, + "loss": 1.3457, + "theoretical_loss": 3.6712442805732257, + "tokens_seen": 938999808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036132397191574725, + "loss": 1.4196, + "theoretical_loss": 3.6711949470627063, + "tokens_seen": 939130880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613039117352056, + "loss": 1.3338, + "theoretical_loss": 3.671145622364644, + "tokens_seen": 939261952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036128385155466397, + "loss": 1.2794, + "theoretical_loss": 3.671096306476235, + "tokens_seen": 939393024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036126379137412233, + "loss": 1.5866, + "theoretical_loss": 3.671046999394677, + "tokens_seen": 939524096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036124373119358075, + "loss": 1.3855, + "theoretical_loss": 3.6709977011171686, + "tokens_seen": 939655168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612236710130391, + "loss": 1.4325, + "theoretical_loss": 3.670948411640911, + "tokens_seen": 939786240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612036108324975, + "loss": 1.4197, + "theoretical_loss": 3.6708991309631043, + "tokens_seen": 939917312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036118355065195584, + "loss": 1.5139, + "theoretical_loss": 3.6708498590809526, + "tokens_seen": 940048384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611634904714142, + "loss": 1.3353, + "theoretical_loss": 3.6708005959916594, + "tokens_seen": 940179456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611434302908726, + "loss": 1.4471, + "theoretical_loss": 3.67075134169243, + "tokens_seen": 940310528 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4704301059246063, + "objective/train/avg_token_score": 0.01721164584159851, + "objective/train/docs_used": 539932, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6431195735931396, + "objective/train/original_loss": 3.913891077041626, + "objective/train/theoretical_loss": 3.6707020961804715, + "objective/train/tokens_used": 940441600, + "objective/train/unlikelihood_loss": 0.13901834189891815, + "theoretical_loss": 3.6707020961804715, + "tokens_seen": 940441600 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361123370110331, + "loss": 1.1843, + "theoretical_loss": 3.6707020961804715, + "tokens_seen": 940441600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611033099297894, + "loss": 1.3724, + "theoretical_loss": 3.6706528594529915, + "tokens_seen": 940572672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036108324974924776, + "loss": 1.4621, + "theoretical_loss": 3.6706036315071993, + "tokens_seen": 940703744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610631895687062, + "loss": 1.3429, + "theoretical_loss": 3.670554412340305, + "tokens_seen": 940834816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036104312938816454, + "loss": 1.4178, + "theoretical_loss": 3.6705052019495206, + "tokens_seen": 940965888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610230692076229, + "loss": 1.173, + "theoretical_loss": 3.6704560003320594, + "tokens_seen": 941096960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036100300902708126, + "loss": 1.3789, + "theoretical_loss": 3.6704068074851355, + "tokens_seen": 941228032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609829488465396, + "loss": 1.4672, + "theoretical_loss": 3.6703576234059643, + "tokens_seen": 941359104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036096288866599804, + "loss": 1.3166, + "theoretical_loss": 3.6703084480917623, + "tokens_seen": 941490176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609428284854564, + "loss": 1.3746, + "theoretical_loss": 3.6702592815397486, + "tokens_seen": 941621248 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036092276830491476, + "loss": 1.3981, + "theoretical_loss": 3.670210123747142, + "tokens_seen": 941752320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609027081243731, + "loss": 1.4672, + "theoretical_loss": 3.6701609747111625, + "tokens_seen": 941883392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036088264794383154, + "loss": 1.3785, + "theoretical_loss": 3.6701118344290333, + "tokens_seen": 942014464 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.6168743968009949, + "objective/train/avg_token_score": 0.006595643237233162, + "objective/train/docs_used": 541311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.20288348197937, + "objective/train/original_loss": 3.9145617485046387, + "objective/train/theoretical_loss": 3.6700872675697944, + "objective/train/tokens_used": 942080000, + "objective/train/unlikelihood_loss": 0.12248560786247253, + "theoretical_loss": 3.6700872675697944, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608625877632899, + "loss": 1.4297, + "theoretical_loss": 3.670062702897977, + "tokens_seen": 942145536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036084252758274827, + "loss": 1.3517, + "theoretical_loss": 3.6700135801152176, + "tokens_seen": 942276608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036082246740220663, + "loss": 1.3407, + "theoretical_loss": 3.6699644660779818, + "tokens_seen": 942407680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036080240722166505, + "loss": 1.3697, + "theoretical_loss": 3.669915360783495, + "tokens_seen": 942538752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607823470411234, + "loss": 1.553, + "theoretical_loss": 3.6698662642289874, + "tokens_seen": 942669824 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036076228686058177, + "loss": 1.3953, + "theoretical_loss": 3.669817176411687, + "tokens_seen": 942800896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036074222668004013, + "loss": 1.4227, + "theoretical_loss": 3.6697680973288254, + "tokens_seen": 942931968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607221664994985, + "loss": 1.3336, + "theoretical_loss": 3.6697190269776345, + "tokens_seen": 943063040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607021063189569, + "loss": 1.412, + "theoretical_loss": 3.6696699653553475, + "tokens_seen": 943194112 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036068204613841527, + "loss": 1.4349, + "theoretical_loss": 3.6696209124591985, + "tokens_seen": 943325184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036066198595787364, + "loss": 1.5027, + "theoretical_loss": 3.6695718682864236, + "tokens_seen": 943456256 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360641925777332, + "loss": 1.4613, + "theoretical_loss": 3.669522832834261, + "tokens_seen": 943587328 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.5349462628364563, + "objective/train/avg_token_score": 0.01020575687289238, + "objective/train/docs_used": 542104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7980324029922485, + "objective/train/original_loss": 3.495116949081421, + "objective/train/theoretical_loss": 3.6694738060999468, + "objective/train/tokens_used": 943718400, + "objective/train/unlikelihood_loss": 0.17911632359027863, + "theoretical_loss": 3.6694738060999468, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606218655967904, + "loss": 1.4216, + "theoretical_loss": 3.6694738060999468, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606018054162488, + "loss": 1.2576, + "theoretical_loss": 3.6694247880807223, + "tokens_seen": 943849472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036058174523570714, + "loss": 1.3839, + "theoretical_loss": 3.669375778773828, + "tokens_seen": 943980544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605616850551655, + "loss": 1.3656, + "theoretical_loss": 3.6693267781765058, + "tokens_seen": 944111616 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036054162487462386, + "loss": 1.3374, + "theoretical_loss": 3.669277786285999, + "tokens_seen": 944242688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605215646940823, + "loss": 1.5175, + "theoretical_loss": 3.6692288030995526, + "tokens_seen": 944373760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036050150451354064, + "loss": 1.3805, + "theoretical_loss": 3.6691798286144124, + "tokens_seen": 944504832 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360481444332999, + "loss": 1.3534, + "theoretical_loss": 3.6691308628278247, + "tokens_seen": 944635904 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036046138415245737, + "loss": 1.3409, + "theoretical_loss": 3.669081905737039, + "tokens_seen": 944766976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604413239719158, + "loss": 1.4956, + "theoretical_loss": 3.6690329573393043, + "tokens_seen": 944898048 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036042126379137414, + "loss": 1.3591, + "theoretical_loss": 3.668984017631872, + "tokens_seen": 945029120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604012036108325, + "loss": 1.3791, + "theoretical_loss": 3.6689350866119934, + "tokens_seen": 945160192 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036038114343029087, + "loss": 1.4253, + "theoretical_loss": 3.668886164276923, + "tokens_seen": 945291264 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.3261241316795349, + "objective/train/avg_token_score": 0.010203108191490173, + "objective/train/docs_used": 543471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.009924054145813, + "objective/train/original_loss": 3.9562549591064453, + "objective/train/theoretical_loss": 3.668861706365332, + "objective/train/tokens_used": 945356800, + "objective/train/unlikelihood_loss": 0.17419351637363434, + "theoretical_loss": 3.668861706365332, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036036108324974923, + "loss": 1.3079, + "theoretical_loss": 3.668837250623914, + "tokens_seen": 945422336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036034102306920765, + "loss": 1.3695, + "theoretical_loss": 3.6687883456502233, + "tokens_seen": 945553408 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360320962888666, + "loss": 1.5004, + "theoretical_loss": 3.668739449353108, + "tokens_seen": 945684480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036030090270812437, + "loss": 1.3311, + "theoretical_loss": 3.6686905617298264, + "tokens_seen": 945815552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036028084252758273, + "loss": 1.2694, + "theoretical_loss": 3.668641682777638, + "tokens_seen": 945946624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036026078234704115, + "loss": 1.4606, + "theoretical_loss": 3.668592812493803, + "tokens_seen": 946077696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602407221664995, + "loss": 1.4479, + "theoretical_loss": 3.668543950875585, + "tokens_seen": 946208768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602206619859579, + "loss": 1.4183, + "theoretical_loss": 3.668495097920246, + "tokens_seen": 946339840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036020060180541624, + "loss": 1.4586, + "theoretical_loss": 3.668446253625051, + "tokens_seen": 946470912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601805416248746, + "loss": 1.3131, + "theoretical_loss": 3.6683974179872667, + "tokens_seen": 946601984 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360160481444333, + "loss": 1.5883, + "theoretical_loss": 3.668348591004159, + "tokens_seen": 946733056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601404212637914, + "loss": 1.4038, + "theoretical_loss": 3.6682997726729965, + "tokens_seen": 946864128 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.30296921730041504, + "objective/train/avg_token_score": 0.014760198071599007, + "objective/train/docs_used": 544140, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9535803198814392, + "objective/train/original_loss": 3.894678831100464, + "objective/train/theoretical_loss": 3.668250962991049, + "objective/train/tokens_used": 946995200, + "objective/train/unlikelihood_loss": 0.26928845047950745, + "theoretical_loss": 3.668250962991049, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036012036108324974, + "loss": 1.3217, + "theoretical_loss": 3.668250962991049, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601003009027081, + "loss": 1.5096, + "theoretical_loss": 3.668202161955587, + "tokens_seen": 947126272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600802407221665, + "loss": 1.4963, + "theoretical_loss": 3.668153369563883, + "tokens_seen": 947257344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600601805416249, + "loss": 1.1883, + "theoretical_loss": 3.6681045858132104, + "tokens_seen": 947388416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036004012036108324, + "loss": 1.5731, + "theoretical_loss": 3.668055810700843, + "tokens_seen": 947519488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600200601805416, + "loss": 1.423, + "theoretical_loss": 3.6680070442240567, + "tokens_seen": 947650560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035999999999999997, + "loss": 1.177, + "theoretical_loss": 3.667958286380129, + "tokens_seen": 947781632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599799398194584, + "loss": 1.3722, + "theoretical_loss": 3.667909537166338, + "tokens_seen": 947912704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035995987963891675, + "loss": 1.3469, + "theoretical_loss": 3.667860796579962, + "tokens_seen": 948043776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599398194583751, + "loss": 1.5053, + "theoretical_loss": 3.6678120646182837, + "tokens_seen": 948174848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035991975927783347, + "loss": 1.3781, + "theoretical_loss": 3.6677633412785835, + "tokens_seen": 948305920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598996990972919, + "loss": 1.3341, + "theoretical_loss": 3.667714626558145, + "tokens_seen": 948436992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035987963891675025, + "loss": 1.3311, + "theoretical_loss": 3.6676659204542528, + "tokens_seen": 948568064 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.4374389052391052, + "objective/train/avg_token_score": 0.013001547195017338, + "objective/train/docs_used": 544543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4771860837936401, + "objective/train/original_loss": 3.5811593532562256, + "objective/train/theoretical_loss": 3.6676415706326635, + "objective/train/tokens_used": 948633600, + "objective/train/unlikelihood_loss": 0.19514577090740204, + "theoretical_loss": 3.6676415706326635, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598595787362086, + "loss": 1.5741, + "theoretical_loss": 3.6676172229641923, + "tokens_seen": 948699136 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359839518555667, + "loss": 1.3602, + "theoretical_loss": 3.66756853408525, + "tokens_seen": 948830208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035981945837512534, + "loss": 1.3795, + "theoretical_loss": 3.6675198538147153, + "tokens_seen": 948961280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035979939819458375, + "loss": 1.3958, + "theoretical_loss": 3.6674711821498756, + "tokens_seen": 949092352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597793380140421, + "loss": 1.3584, + "theoretical_loss": 3.667422519088023, + "tokens_seen": 949223424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597592778335005, + "loss": 1.2126, + "theoretical_loss": 3.667373864626448, + "tokens_seen": 949354496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035973921765295884, + "loss": 1.365, + "theoretical_loss": 3.667325218762445, + "tokens_seen": 949485568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035971915747241726, + "loss": 1.4609, + "theoretical_loss": 3.667276581493307, + "tokens_seen": 949616640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596990972918756, + "loss": 1.3631, + "theoretical_loss": 3.6672279528163303, + "tokens_seen": 949747712 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359679037111334, + "loss": 1.2611, + "theoretical_loss": 3.6671793327288107, + "tokens_seen": 949878784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965897693079234, + "loss": 1.4998, + "theoretical_loss": 3.6671307212280473, + "tokens_seen": 950009856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035963891675025076, + "loss": 1.5053, + "theoretical_loss": 3.6670821183113373, + "tokens_seen": 950140928 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.30217498540878296, + "objective/train/avg_token_score": 0.012087201699614525, + "objective/train/docs_used": 545942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0214953422546387, + "objective/train/original_loss": 4.339548587799072, + "objective/train/theoretical_loss": 3.667033523975983, + "objective/train/tokens_used": 950272000, + "objective/train/unlikelihood_loss": 0.12828905880451202, + "theoretical_loss": 3.667033523975983, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596188565697091, + "loss": 1.3231, + "theoretical_loss": 3.667033523975983, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035959879638916754, + "loss": 1.3589, + "theoretical_loss": 3.6669849382192847, + "tokens_seen": 950403072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595787362086259, + "loss": 1.321, + "theoretical_loss": 3.6669363610385464, + "tokens_seen": 950534144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035955867602808426, + "loss": 1.5352, + "theoretical_loss": 3.66688779243107, + "tokens_seen": 950665216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595386158475427, + "loss": 1.4601, + "theoretical_loss": 3.666839232394163, + "tokens_seen": 950796288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035951855566700104, + "loss": 1.5747, + "theoretical_loss": 3.6667906809251303, + "tokens_seen": 950927360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594984954864594, + "loss": 1.2789, + "theoretical_loss": 3.666742138021281, + "tokens_seen": 951058432 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035947843530591777, + "loss": 1.4886, + "theoretical_loss": 3.6666936036799216, + "tokens_seen": 951189504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594583751253762, + "loss": 1.4403, + "theoretical_loss": 3.666645077898364, + "tokens_seen": 951320576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035943831494483454, + "loss": 1.2969, + "theoretical_loss": 3.66659656067392, + "tokens_seen": 951451648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594182547642929, + "loss": 1.428, + "theoretical_loss": 3.6665480520039004, + "tokens_seen": 951582720 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035939819458375127, + "loss": 1.5113, + "theoretical_loss": 3.66649955188562, + "tokens_seen": 951713792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035937813440320963, + "loss": 1.5979, + "theoretical_loss": 3.666451060316393, + "tokens_seen": 951844864 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.5238880515098572, + "objective/train/avg_token_score": 0.021302634850144386, + "objective/train/docs_used": 546639, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7860549688339233, + "objective/train/original_loss": 3.881726026535034, + "objective/train/theoretical_loss": 3.6664268177368364, + "objective/train/tokens_used": 951910400, + "objective/train/unlikelihood_loss": 0.13393664360046387, + "theoretical_loss": 3.6664268177368364, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035935807422266805, + "loss": 1.3869, + "theoretical_loss": 3.666402577293537, + "tokens_seen": 951975936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593380140421264, + "loss": 1.3779, + "theoretical_loss": 3.666354102814368, + "tokens_seen": 952107008 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035931795386158477, + "loss": 1.2248, + "theoretical_loss": 3.6663056368762046, + "tokens_seen": 952238080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035929789368104313, + "loss": 1.3454, + "theoretical_loss": 3.666257179476367, + "tokens_seen": 952369152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035927783350050155, + "loss": 1.4426, + "theoretical_loss": 3.6662087306121762, + "tokens_seen": 952500224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592577733199599, + "loss": 1.5249, + "theoretical_loss": 3.6661602902809545, + "tokens_seen": 952631296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592377131394183, + "loss": 1.4749, + "theoretical_loss": 3.6661118584800247, + "tokens_seen": 952762368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035921765295887664, + "loss": 1.3279, + "theoretical_loss": 3.6660634352067123, + "tokens_seen": 952893440 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359197592778335, + "loss": 1.3663, + "theoretical_loss": 3.6660150204583424, + "tokens_seen": 953024512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591775325977934, + "loss": 1.3057, + "theoretical_loss": 3.665966614232243, + "tokens_seen": 953155584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591574724172518, + "loss": 1.4411, + "theoretical_loss": 3.665918216525741, + "tokens_seen": 953286656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035913741223671014, + "loss": 1.3874, + "theoretical_loss": 3.6658698273361665, + "tokens_seen": 953417728 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.4087243378162384, + "objective/train/avg_token_score": 0.008824365213513374, + "objective/train/docs_used": 547886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4714593887329102, + "objective/train/original_loss": 3.8361740112304688, + "objective/train/theoretical_loss": 3.66582144666085, + "objective/train/tokens_used": 953548800, + "objective/train/unlikelihood_loss": 0.19405020773410797, + "theoretical_loss": 3.66582144666085, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591173520561685, + "loss": 1.4147, + "theoretical_loss": 3.66582144666085, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590972918756269, + "loss": 1.1682, + "theoretical_loss": 3.6657730744971238, + "tokens_seen": 953679872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590772316950853, + "loss": 1.2985, + "theoretical_loss": 3.665724710842321, + "tokens_seen": 953810944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035905717151454364, + "loss": 1.4041, + "theoretical_loss": 3.6656763556937744, + "tokens_seen": 953942016 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359037111334002, + "loss": 1.3637, + "theoretical_loss": 3.6656280090488216, + "tokens_seen": 954073088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035901705115346037, + "loss": 1.5767, + "theoretical_loss": 3.6655796709047976, + "tokens_seen": 954204160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589969909729188, + "loss": 1.3506, + "theoretical_loss": 3.665531341259041, + "tokens_seen": 954335232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035897693079237715, + "loss": 1.4197, + "theoretical_loss": 3.6654830201088906, + "tokens_seen": 954466304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589568706118355, + "loss": 1.4613, + "theoretical_loss": 3.6654347074516873, + "tokens_seen": 954597376 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035893681043129387, + "loss": 1.3964, + "theoretical_loss": 3.6653864032847716, + "tokens_seen": 954728448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589167502507523, + "loss": 1.538, + "theoretical_loss": 3.6653381076054865, + "tokens_seen": 954859520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035889669007021065, + "loss": 1.3638, + "theoretical_loss": 3.6652898204111763, + "tokens_seen": 954990592 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358876629889669, + "loss": 1.3817, + "theoretical_loss": 3.665241541699185, + "tokens_seen": 955121664 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.41006842255592346, + "objective/train/avg_token_score": 0.0027227371465414762, + "objective/train/docs_used": 548540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4417232275009155, + "objective/train/original_loss": 3.8670425415039062, + "objective/train/theoretical_loss": 3.6652174055232303, + "objective/train/tokens_used": 955187200, + "objective/train/unlikelihood_loss": 0.19085726141929626, + "theoretical_loss": 3.6652174055232303, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588565697091274, + "loss": 1.3756, + "theoretical_loss": 3.66519327146686, + "tokens_seen": 955252736 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035883650952858574, + "loss": 1.4215, + "theoretical_loss": 3.665145009711548, + "tokens_seen": 955383808 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035881644934804415, + "loss": 1.4151, + "theoretical_loss": 3.6650967564305983, + "tokens_seen": 955514880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587963891675025, + "loss": 1.4473, + "theoretical_loss": 3.6650485116213596, + "tokens_seen": 955645952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587763289869609, + "loss": 1.4043, + "theoretical_loss": 3.6650002752811837, + "tokens_seen": 955777024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035875626880641924, + "loss": 1.3622, + "theoretical_loss": 3.664952047407423, + "tokens_seen": 955908096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035873620862587766, + "loss": 1.3278, + "theoretical_loss": 3.66490382799743, + "tokens_seen": 956039168 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358716148445336, + "loss": 1.603, + "theoretical_loss": 3.66485561704856, + "tokens_seen": 956170240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586960882647944, + "loss": 1.4986, + "theoretical_loss": 3.664807414558169, + "tokens_seen": 956301312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035867602808425274, + "loss": 1.4092, + "theoretical_loss": 3.664759220523613, + "tokens_seen": 956432384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035865596790371116, + "loss": 1.3175, + "theoretical_loss": 3.6647110349422505, + "tokens_seen": 956563456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586359077231695, + "loss": 1.5611, + "theoretical_loss": 3.6646628578114417, + "tokens_seen": 956694528 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.423081636428833, + "objective/train/avg_token_score": 0.01869446597993374, + "objective/train/docs_used": 549880, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4246835708618164, + "objective/train/original_loss": 3.8789021968841553, + "objective/train/theoretical_loss": 3.664614689128546, + "objective/train/tokens_used": 956825600, + "objective/train/unlikelihood_loss": 0.15293046832084656, + "theoretical_loss": 3.664614689128546, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586158475426279, + "loss": 1.4468, + "theoretical_loss": 3.664614689128546, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035859578736208625, + "loss": 1.4844, + "theoretical_loss": 3.6645665288909255, + "tokens_seen": 956956672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585757271815446, + "loss": 1.4343, + "theoretical_loss": 3.664518377095944, + "tokens_seen": 957087744 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358555667001003, + "loss": 1.4554, + "theoretical_loss": 3.6644702337409636, + "tokens_seen": 957218816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585356068204614, + "loss": 1.4351, + "theoretical_loss": 3.6644220988233513, + "tokens_seen": 957349888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035851554663991975, + "loss": 1.2733, + "theoretical_loss": 3.6643739723404725, + "tokens_seen": 957480960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584954864593781, + "loss": 1.3624, + "theoretical_loss": 3.6643258542896957, + "tokens_seen": 957612032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035847542627883653, + "loss": 1.3252, + "theoretical_loss": 3.664277744668389, + "tokens_seen": 957743104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584553660982949, + "loss": 1.4139, + "theoretical_loss": 3.6642296434739228, + "tokens_seen": 957874176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035843530591775325, + "loss": 1.5173, + "theoretical_loss": 3.6641815507036677, + "tokens_seen": 958005248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584152457372116, + "loss": 1.4629, + "theoretical_loss": 3.664133466354997, + "tokens_seen": 958136320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035839518555667, + "loss": 1.412, + "theoretical_loss": 3.664085390425283, + "tokens_seen": 958267392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583751253761284, + "loss": 1.264, + "theoretical_loss": 3.664037322911901, + "tokens_seen": 958398464 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.4647482931613922, + "objective/train/avg_token_score": 0.04183371365070343, + "objective/train/docs_used": 550706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5117523670196533, + "objective/train/original_loss": 3.8492813110351562, + "objective/train/theoretical_loss": 3.664013292310515, + "objective/train/tokens_used": 958464000, + "objective/train/unlikelihood_loss": 0.13987703621387482, + "theoretical_loss": 3.664013292310515, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035835506519558676, + "loss": 1.3949, + "theoretical_loss": 3.6639892638122276, + "tokens_seen": 958529536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583350050150451, + "loss": 1.4312, + "theoretical_loss": 3.663941213123639, + "tokens_seen": 958660608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583149448345035, + "loss": 1.3087, + "theoretical_loss": 3.6638931708435134, + "tokens_seen": 958791680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582948846539619, + "loss": 1.4852, + "theoretical_loss": 3.663845136969231, + "tokens_seen": 958922752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035827482447342026, + "loss": 1.4914, + "theoretical_loss": 3.6637971114981713, + "tokens_seen": 959053824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582547642928786, + "loss": 1.285, + "theoretical_loss": 3.6637490944277165, + "tokens_seen": 959184896 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358234704112337, + "loss": 1.5443, + "theoretical_loss": 3.6637010857552497, + "tokens_seen": 959315968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035821464393179535, + "loss": 1.4576, + "theoretical_loss": 3.6636530854781553, + "tokens_seen": 959447040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035819458375125376, + "loss": 1.2595, + "theoretical_loss": 3.663605093593818, + "tokens_seen": 959578112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581745235707121, + "loss": 1.3928, + "theoretical_loss": 3.6635571100996245, + "tokens_seen": 959709184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581544633901705, + "loss": 1.3326, + "theoretical_loss": 3.663509134992962, + "tokens_seen": 959840256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035813440320962885, + "loss": 1.4657, + "theoretical_loss": 3.66346116827122, + "tokens_seen": 959971328 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.38361436128616333, + "objective/train/avg_token_score": 0.01017189584672451, + "objective/train/docs_used": 551132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2384686470031738, + "objective/train/original_loss": 3.5551533699035645, + "objective/train/theoretical_loss": 3.6634132099317886, + "objective/train/tokens_used": 960102400, + "objective/train/unlikelihood_loss": 0.19672907888889313, + "theoretical_loss": 3.6634132099317886, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035811434302908727, + "loss": 1.5085, + "theoretical_loss": 3.6634132099317886, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580942828485457, + "loss": 1.3765, + "theoretical_loss": 3.663365259972058, + "tokens_seen": 960233472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035807422266800404, + "loss": 1.3259, + "theoretical_loss": 3.6633173183894208, + "tokens_seen": 960364544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580541624874624, + "loss": 1.4953, + "theoretical_loss": 3.6632693851812705, + "tokens_seen": 960495616 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035803410230692077, + "loss": 1.3929, + "theoretical_loss": 3.663221460345002, + "tokens_seen": 960626688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580140421263792, + "loss": 1.3147, + "theoretical_loss": 3.6631735438780106, + "tokens_seen": 960757760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035799398194583755, + "loss": 1.3079, + "theoretical_loss": 3.6631256357776945, + "tokens_seen": 960888832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579739217652959, + "loss": 1.3376, + "theoretical_loss": 3.6630777360414504, + "tokens_seen": 961019904 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035795386158475427, + "loss": 1.3904, + "theoretical_loss": 3.663029844666678, + "tokens_seen": 961150976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579338014042127, + "loss": 1.4542, + "theoretical_loss": 3.6629819616507775, + "tokens_seen": 961282048 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035791374122367105, + "loss": 1.4085, + "theoretical_loss": 3.662934086991151, + "tokens_seen": 961413120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578936810431294, + "loss": 1.2497, + "theoretical_loss": 3.662886220685202, + "tokens_seen": 961544192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578736208625878, + "loss": 1.4866, + "theoretical_loss": 3.6628383627303327, + "tokens_seen": 961675264 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.39742180705070496, + "objective/train/avg_token_score": 0.009587054140865803, + "objective/train/docs_used": 552317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.321476697921753, + "objective/train/original_loss": 3.955338954925537, + "objective/train/theoretical_loss": 3.6628144368837425, + "objective/train/tokens_used": 961740800, + "objective/train/unlikelihood_loss": 0.14273592829704285, + "theoretical_loss": 3.6628144368837425, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035785356068204614, + "loss": 1.2257, + "theoretical_loss": 3.662790513123949, + "tokens_seen": 961806336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035783350050150455, + "loss": 1.4934, + "theoretical_loss": 3.662742671863458, + "tokens_seen": 961937408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578134403209629, + "loss": 1.5383, + "theoretical_loss": 3.6626948389462655, + "tokens_seen": 962068480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577933801404213, + "loss": 1.3965, + "theoretical_loss": 3.6626470143697816, + "tokens_seen": 962199552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035777331995987964, + "loss": 1.4446, + "theoretical_loss": 3.6625991981314145, + "tokens_seen": 962330624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035775325977933806, + "loss": 1.4201, + "theoretical_loss": 3.6625513902285767, + "tokens_seen": 962461696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577331995987964, + "loss": 1.4069, + "theoretical_loss": 3.6625035906586794, + "tokens_seen": 962592768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577131394182548, + "loss": 1.2076, + "theoretical_loss": 3.662455799419136, + "tokens_seen": 962723840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035769307923771314, + "loss": 1.3967, + "theoretical_loss": 3.66240801650736, + "tokens_seen": 962854912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035767301905717156, + "loss": 1.3425, + "theoretical_loss": 3.6623602419207684, + "tokens_seen": 962985984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576529588766299, + "loss": 1.4551, + "theoretical_loss": 3.662312475656777, + "tokens_seen": 963117056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576328986960883, + "loss": 1.4428, + "theoretical_loss": 3.6622647177128034, + "tokens_seen": 963248128 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.15518084168434143, + "objective/train/avg_token_score": 0.011443916708230972, + "objective/train/docs_used": 552873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.4723800718784332, + "objective/train/original_loss": 4.026332855224609, + "objective/train/theoretical_loss": 3.662216968086267, + "objective/train/tokens_used": 963379200, + "objective/train/unlikelihood_loss": 0.19881534576416016, + "theoretical_loss": 3.662216968086267, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035761283851554665, + "loss": 1.305, + "theoretical_loss": 3.662216968086267, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357592778335005, + "loss": 1.3834, + "theoretical_loss": 3.662169226774588, + "tokens_seen": 963510272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575727181544634, + "loss": 1.4042, + "theoretical_loss": 3.6621214937751874, + "tokens_seen": 963641344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575526579739218, + "loss": 1.4152, + "theoretical_loss": 3.662073769085488, + "tokens_seen": 963772416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035753259779338015, + "loss": 1.4755, + "theoretical_loss": 3.6620260527029127, + "tokens_seen": 963903488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575125376128385, + "loss": 1.2916, + "theoretical_loss": 3.661978344624887, + "tokens_seen": 964034560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035749247743229693, + "loss": 1.3873, + "theoretical_loss": 3.661930644848836, + "tokens_seen": 964165632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574724172517553, + "loss": 1.4644, + "theoretical_loss": 3.6618829533721873, + "tokens_seen": 964296704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035745235707121365, + "loss": 1.2889, + "theoretical_loss": 3.661835270192369, + "tokens_seen": 964427776 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357432296890672, + "loss": 1.32, + "theoretical_loss": 3.6617875953068095, + "tokens_seen": 964558848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574122367101304, + "loss": 1.2765, + "theoretical_loss": 3.661739928712941, + "tokens_seen": 964689920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573921765295888, + "loss": 1.2349, + "theoretical_loss": 3.661692270408194, + "tokens_seen": 964820992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035737211634904716, + "loss": 1.281, + "theoretical_loss": 3.6616446203900006, + "tokens_seen": 964952064 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.40518084168434143, + "objective/train/avg_token_score": 0.019017038866877556, + "objective/train/docs_used": 554313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4109416007995605, + "objective/train/original_loss": 4.264377117156982, + "objective/train/theoretical_loss": 3.6616207984875606, + "objective/train/tokens_used": 965017600, + "objective/train/unlikelihood_loss": 0.1246563270688057, + "theoretical_loss": 3.6616207984875606, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573520561685055, + "loss": 1.4015, + "theoretical_loss": 3.6615969786557967, + "tokens_seen": 965083136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573319959879639, + "loss": 1.4018, + "theoretical_loss": 3.6615493452030154, + "tokens_seen": 965214208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573119358074223, + "loss": 1.3093, + "theoretical_loss": 3.6615017200290936, + "tokens_seen": 965345280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035729187562688066, + "loss": 1.3674, + "theoretical_loss": 3.6614541031314687, + "tokens_seen": 965476352 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357271815446339, + "loss": 1.2561, + "theoretical_loss": 3.6614064945075784, + "tokens_seen": 965607424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572517552657974, + "loss": 1.3227, + "theoretical_loss": 3.661358894154864, + "tokens_seen": 965738496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035723169508525575, + "loss": 1.4176, + "theoretical_loss": 3.6613113020707644, + "tokens_seen": 965869568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035721163490471416, + "loss": 1.3676, + "theoretical_loss": 3.661263718252723, + "tokens_seen": 966000640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571915747241725, + "loss": 1.2262, + "theoretical_loss": 3.6612161426981817, + "tokens_seen": 966131712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571715145436309, + "loss": 1.4442, + "theoretical_loss": 3.661168575404585, + "tokens_seen": 966262784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035715145436308925, + "loss": 1.4583, + "theoretical_loss": 3.6611210163693784, + "tokens_seen": 966393856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035713139418254767, + "loss": 1.5321, + "theoretical_loss": 3.661073465590008, + "tokens_seen": 966524928 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.3427419364452362, + "objective/train/avg_token_score": 0.015760909765958786, + "objective/train/docs_used": 554946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.039088487625122, + "objective/train/original_loss": 3.839756965637207, + "objective/train/theoretical_loss": 3.6610259230639217, + "objective/train/tokens_used": 966656000, + "objective/train/unlikelihood_loss": 0.16241952776908875, + "theoretical_loss": 3.6610259230639217, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035711133400200603, + "loss": 1.3356, + "theoretical_loss": 3.6610259230639217, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570912738214644, + "loss": 1.3812, + "theoretical_loss": 3.6609783887885676, + "tokens_seen": 966787072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035707121364092275, + "loss": 1.4599, + "theoretical_loss": 3.660930862761396, + "tokens_seen": 966918144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570511534603811, + "loss": 1.3088, + "theoretical_loss": 3.660883344979858, + "tokens_seen": 967049216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035703109327983953, + "loss": 1.3409, + "theoretical_loss": 3.660835835441406, + "tokens_seen": 967180288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570110330992979, + "loss": 1.2812, + "theoretical_loss": 3.660788334143492, + "tokens_seen": 967311360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035699097291875626, + "loss": 1.2849, + "theoretical_loss": 3.6607408410835713, + "tokens_seen": 967442432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569709127382146, + "loss": 1.4523, + "theoretical_loss": 3.6606933562590998, + "tokens_seen": 967573504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035695085255767303, + "loss": 1.2365, + "theoretical_loss": 3.660645879667533, + "tokens_seen": 967704576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569307923771314, + "loss": 1.4152, + "theoretical_loss": 3.6605984113063292, + "tokens_seen": 967835648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035691073219658976, + "loss": 1.4801, + "theoretical_loss": 3.6605509511729473, + "tokens_seen": 967966720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568906720160481, + "loss": 1.3573, + "theoretical_loss": 3.6605034992648475, + "tokens_seen": 968097792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568706118355065, + "loss": 1.6273, + "theoretical_loss": 3.6604560555794903, + "tokens_seen": 968228864 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.38233137130737305, + "objective/train/avg_token_score": 0.03409533575177193, + "objective/train/docs_used": 556500, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1578701734542847, + "objective/train/original_loss": 4.5173468589782715, + "objective/train/theoretical_loss": 3.6604323368195475, + "objective/train/tokens_used": 968294400, + "objective/train/unlikelihood_loss": 0.09163815528154373, + "theoretical_loss": 3.6604323368195475, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568505516549649, + "loss": 1.4404, + "theoretical_loss": 3.660408620114339, + "tokens_seen": 968359936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035683049147442326, + "loss": 1.3872, + "theoretical_loss": 3.660361192866856, + "tokens_seen": 968491008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568104312938816, + "loss": 1.3946, + "theoretical_loss": 3.660313773834506, + "tokens_seen": 968622080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035679037111334, + "loss": 1.4318, + "theoretical_loss": 3.660266363014755, + "tokens_seen": 968753152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567703109327984, + "loss": 1.4538, + "theoretical_loss": 3.6602189604050697, + "tokens_seen": 968884224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035675025075225677, + "loss": 1.4093, + "theoretical_loss": 3.660171566002918, + "tokens_seen": 969015296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035673019057171513, + "loss": 1.3833, + "theoretical_loss": 3.660124179805769, + "tokens_seen": 969146368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567101303911735, + "loss": 1.3517, + "theoretical_loss": 3.6600768018110923, + "tokens_seen": 969277440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035669007021063185, + "loss": 1.3262, + "theoretical_loss": 3.6600294320163593, + "tokens_seen": 969408512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035667001003009027, + "loss": 1.3028, + "theoretical_loss": 3.6599820704190433, + "tokens_seen": 969539584 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035664994984954863, + "loss": 1.5977, + "theoretical_loss": 3.6599347170166165, + "tokens_seen": 969670656 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356629889669007, + "loss": 1.4433, + "theoretical_loss": 3.6598873718065548, + "tokens_seen": 969801728 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.3360826075077057, + "objective/train/avg_token_score": 0.02158435620367527, + "objective/train/docs_used": 557201, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.072542667388916, + "objective/train/original_loss": 3.965404510498047, + "objective/train/theoretical_loss": 3.659840034786333, + "objective/train/tokens_used": 969932800, + "objective/train/unlikelihood_loss": 0.14196471869945526, + "theoretical_loss": 3.659840034786333, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035660982948846536, + "loss": 1.4385, + "theoretical_loss": 3.659840034786333, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565897693079238, + "loss": 1.424, + "theoretical_loss": 3.6597927059534276, + "tokens_seen": 970063872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565697091273822, + "loss": 1.4797, + "theoretical_loss": 3.659745385305318, + "tokens_seen": 970194944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035654964894684055, + "loss": 1.3424, + "theoretical_loss": 3.659698072839482, + "tokens_seen": 970326016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565295887662989, + "loss": 1.5506, + "theoretical_loss": 3.6596507685534005, + "tokens_seen": 970457088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035650952858575733, + "loss": 1.2629, + "theoretical_loss": 3.659603472444555, + "tokens_seen": 970588160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564894684052157, + "loss": 1.2345, + "theoretical_loss": 3.6595561845104276, + "tokens_seen": 970719232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035646940822467405, + "loss": 1.3563, + "theoretical_loss": 3.6595089047485017, + "tokens_seen": 970850304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564493480441324, + "loss": 1.4386, + "theoretical_loss": 3.659461633156262, + "tokens_seen": 970981376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564292878635908, + "loss": 1.4433, + "theoretical_loss": 3.659414369731195, + "tokens_seen": 971112448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564092276830492, + "loss": 1.3992, + "theoretical_loss": 3.659367114470786, + "tokens_seen": 971243520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035638916750250756, + "loss": 1.4837, + "theoretical_loss": 3.659319867372525, + "tokens_seen": 971374592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563691073219659, + "loss": 1.404, + "theoretical_loss": 3.6592726284339, + "tokens_seen": 971505664 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.2927052676677704, + "objective/train/avg_token_score": 0.02205132693052292, + "objective/train/docs_used": 558372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0991299152374268, + "objective/train/original_loss": 4.443863868713379, + "objective/train/theoretical_loss": 3.6592490120236665, + "objective/train/tokens_used": 971571200, + "objective/train/unlikelihood_loss": 0.12972882390022278, + "theoretical_loss": 3.6592490120236665, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563490471414243, + "loss": 1.4629, + "theoretical_loss": 3.6592253976524014, + "tokens_seen": 971636736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563289869608827, + "loss": 1.3716, + "theoretical_loss": 3.65917817502552, + "tokens_seen": 971767808 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035630892678034106, + "loss": 1.3775, + "theoretical_loss": 3.6591309605507494, + "tokens_seen": 971898880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562888665997994, + "loss": 1.3613, + "theoretical_loss": 3.6590837542255827, + "tokens_seen": 972029952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562688064192578, + "loss": 1.4809, + "theoretical_loss": 3.659036556047514, + "tokens_seen": 972161024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035624874623871615, + "loss": 1.4282, + "theoretical_loss": 3.65898936601404, + "tokens_seen": 972292096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035622868605817456, + "loss": 1.4007, + "theoretical_loss": 3.658942184122657, + "tokens_seen": 972423168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562086258776329, + "loss": 1.285, + "theoretical_loss": 3.658895010370863, + "tokens_seen": 972554240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561885656970913, + "loss": 1.4577, + "theoretical_loss": 3.658847844756157, + "tokens_seen": 972685312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035616850551654965, + "loss": 1.4883, + "theoretical_loss": 3.65880068727604, + "tokens_seen": 972816384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035614844533600807, + "loss": 1.1596, + "theoretical_loss": 3.6587535379280123, + "tokens_seen": 972947456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612838515546643, + "loss": 1.3531, + "theoretical_loss": 3.6587063967095776, + "tokens_seen": 973078528 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.3252077102661133, + "objective/train/avg_token_score": 0.012795456685125828, + "objective/train/docs_used": 558840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9325836300849915, + "objective/train/original_loss": 3.7907679080963135, + "objective/train/theoretical_loss": 3.6586592636182376, + "objective/train/tokens_used": 973209600, + "objective/train/unlikelihood_loss": 0.16836106777191162, + "theoretical_loss": 3.6586592636182376, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561083249749248, + "loss": 1.4233, + "theoretical_loss": 3.6586592636182376, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035608826479438315, + "loss": 1.3146, + "theoretical_loss": 3.658612138651498, + "tokens_seen": 973340672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003560682046138415, + "loss": 1.4269, + "theoretical_loss": 3.658565021806865, + "tokens_seen": 973471744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035604814443329993, + "loss": 1.4009, + "theoretical_loss": 3.6585179130818446, + "tokens_seen": 973602816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560280842527583, + "loss": 1.3078, + "theoretical_loss": 3.6584708124739453, + "tokens_seen": 973733888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035600802407221666, + "loss": 1.4618, + "theoretical_loss": 3.658423719980675, + "tokens_seen": 973864960 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355987963891675, + "loss": 1.503, + "theoretical_loss": 3.658376635599545, + "tokens_seen": 973996032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035596790371113344, + "loss": 1.307, + "theoretical_loss": 3.6583295593280667, + "tokens_seen": 974127104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559478435305918, + "loss": 1.4605, + "theoretical_loss": 3.6582824911637513, + "tokens_seen": 974258176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035592778335005016, + "loss": 1.4446, + "theoretical_loss": 3.6582354311041128, + "tokens_seen": 974389248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559077231695085, + "loss": 1.3981, + "theoretical_loss": 3.658188379146666, + "tokens_seen": 974520320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558876629889669, + "loss": 1.4775, + "theoretical_loss": 3.658141335288926, + "tokens_seen": 974651392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558676028084253, + "loss": 1.3826, + "theoretical_loss": 3.6580942995284103, + "tokens_seen": 974782464 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.4689638316631317, + "objective/train/avg_token_score": 0.029502632096409798, + "objective/train/docs_used": 560118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4518026113510132, + "objective/train/original_loss": 3.527937889099121, + "objective/train/theoretical_loss": 3.6580707846838356, + "objective/train/tokens_used": 974848000, + "objective/train/unlikelihood_loss": 0.1697324812412262, + "theoretical_loss": 3.6580707846838356, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035584754262788366, + "loss": 1.4059, + "theoretical_loss": 3.6580472718626353, + "tokens_seen": 974913536 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355827482447342, + "loss": 1.4666, + "theoretical_loss": 3.6580002522891215, + "tokens_seen": 975044608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558074222668004, + "loss": 1.2906, + "theoretical_loss": 3.657953240805388, + "tokens_seen": 975175680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557873620862588, + "loss": 1.4498, + "theoretical_loss": 3.657906237408956, + "tokens_seen": 975306752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035576730190571717, + "loss": 1.3404, + "theoretical_loss": 3.6578592420973477, + "tokens_seen": 975437824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035574724172517553, + "loss": 1.3351, + "theoretical_loss": 3.6578122548680865, + "tokens_seen": 975568896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557271815446339, + "loss": 1.408, + "theoretical_loss": 3.6577652757186963, + "tokens_seen": 975699968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035570712136409225, + "loss": 1.3243, + "theoretical_loss": 3.657718304646703, + "tokens_seen": 975831040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035568706118355067, + "loss": 1.229, + "theoretical_loss": 3.657671341649633, + "tokens_seen": 975962112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035566700100300903, + "loss": 1.3295, + "theoretical_loss": 3.657624386725014, + "tokens_seen": 976093184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556469408224674, + "loss": 1.4004, + "theoretical_loss": 3.657577439870375, + "tokens_seen": 976224256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035562688064192576, + "loss": 1.3905, + "theoretical_loss": 3.6575305010832455, + "tokens_seen": 976355328 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.5413612127304077, + "objective/train/avg_token_score": 0.01300020981580019, + "objective/train/docs_used": 560789, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.758821964263916, + "objective/train/original_loss": 3.3474416732788086, + "objective/train/theoretical_loss": 3.6574835703611566, + "objective/train/tokens_used": 976486400, + "objective/train/unlikelihood_loss": 0.20596948266029358, + "theoretical_loss": 3.6574835703611566, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035560682046138417, + "loss": 1.3688, + "theoretical_loss": 3.6574835703611566, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035558676028084253, + "loss": 1.5052, + "theoretical_loss": 3.6574366477016396, + "tokens_seen": 976617472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555667001003009, + "loss": 1.4818, + "theoretical_loss": 3.657389733102228, + "tokens_seen": 976748544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035554663991975926, + "loss": 1.2898, + "theoretical_loss": 3.6573428265604564, + "tokens_seen": 976879616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555265797392176, + "loss": 1.6261, + "theoretical_loss": 3.6572959280738595, + "tokens_seen": 977010688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035550651955867604, + "loss": 1.4376, + "theoretical_loss": 3.657249037639974, + "tokens_seen": 977141760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554864593781344, + "loss": 1.3637, + "theoretical_loss": 3.6572021552563365, + "tokens_seen": 977272832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035546639919759276, + "loss": 1.2846, + "theoretical_loss": 3.657155280920487, + "tokens_seen": 977403904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554463390170511, + "loss": 1.4252, + "theoretical_loss": 3.6571084146299633, + "tokens_seen": 977534976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035542627883650954, + "loss": 1.5418, + "theoretical_loss": 3.6570615563823075, + "tokens_seen": 977666048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554062186559679, + "loss": 1.3106, + "theoretical_loss": 3.657014706175061, + "tokens_seen": 977797120 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035538615847542627, + "loss": 1.2886, + "theoretical_loss": 3.6569678640057655, + "tokens_seen": 977928192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035536609829488463, + "loss": 1.2883, + "theoretical_loss": 3.6569210298719668, + "tokens_seen": 978059264 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.3669965863227844, + "objective/train/avg_token_score": 0.027926605194807053, + "objective/train/docs_used": 561934, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2823001146316528, + "objective/train/original_loss": 3.914296865463257, + "objective/train/theoretical_loss": 3.656897615817611, + "objective/train/tokens_used": 978124800, + "objective/train/unlikelihood_loss": 0.13823816180229187, + "theoretical_loss": 3.656897615817611, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035534603811434304, + "loss": 1.3695, + "theoretical_loss": 3.656874203771208, + "tokens_seen": 978190336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553259779338014, + "loss": 1.2921, + "theoretical_loss": 3.6568273857010363, + "tokens_seen": 978321408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035530591775325977, + "loss": 1.4758, + "theoretical_loss": 3.656780575658999, + "tokens_seen": 978452480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035528585757271813, + "loss": 1.3068, + "theoretical_loss": 3.6567337736426433, + "tokens_seen": 978583552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552657973921765, + "loss": 1.4066, + "theoretical_loss": 3.6566869796495194, + "tokens_seen": 978714624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552457372116349, + "loss": 1.3173, + "theoretical_loss": 3.6566401936771773, + "tokens_seen": 978845696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035522567703109327, + "loss": 1.4187, + "theoretical_loss": 3.6565934157231688, + "tokens_seen": 978976768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035520561685055163, + "loss": 1.393, + "theoretical_loss": 3.6565466457850455, + "tokens_seen": 979107840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035518555667001, + "loss": 1.2189, + "theoretical_loss": 3.656499883860362, + "tokens_seen": 979238912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551654964894684, + "loss": 1.3858, + "theoretical_loss": 3.6564531299466725, + "tokens_seen": 979369984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551454363089268, + "loss": 1.2855, + "theoretical_loss": 3.656406384041533, + "tokens_seen": 979501056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035512537612838514, + "loss": 1.457, + "theoretical_loss": 3.6563596461424996, + "tokens_seen": 979632128 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.4206378161907196, + "objective/train/avg_token_score": 0.0169062577188015, + "objective/train/docs_used": 562605, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4026451110839844, + "objective/train/original_loss": 3.5622217655181885, + "objective/train/theoretical_loss": 3.6563129162471313, + "objective/train/tokens_used": 979763200, + "objective/train/unlikelihood_loss": 0.23778095841407776, + "theoretical_loss": 3.6563129162471313, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551053159478435, + "loss": 1.3656, + "theoretical_loss": 3.6563129162471313, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550852557673019, + "loss": 1.4093, + "theoretical_loss": 3.656266194352986, + "tokens_seen": 979894272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035506519558676033, + "loss": 1.1524, + "theoretical_loss": 3.6562194804576245, + "tokens_seen": 980025344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550451354062187, + "loss": 1.4261, + "theoretical_loss": 3.6561727745586072, + "tokens_seen": 980156416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035502507522567706, + "loss": 1.3937, + "theoretical_loss": 3.6561260766534973, + "tokens_seen": 980287488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550050150451354, + "loss": 1.2463, + "theoretical_loss": 3.656079386739857, + "tokens_seen": 980418560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035498495486459384, + "loss": 1.2397, + "theoretical_loss": 3.656032704815251, + "tokens_seen": 980549632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549648946840522, + "loss": 1.4042, + "theoretical_loss": 3.6559860308772447, + "tokens_seen": 980680704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035494483450351056, + "loss": 1.4068, + "theoretical_loss": 3.655939364923405, + "tokens_seen": 980811776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549247743229689, + "loss": 1.3336, + "theoretical_loss": 3.655892706951298, + "tokens_seen": 980942848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549047141424273, + "loss": 1.3143, + "theoretical_loss": 3.6558460569584943, + "tokens_seen": 981073920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548846539618857, + "loss": 1.3478, + "theoretical_loss": 3.655799414942561, + "tokens_seen": 981204992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035486459378134406, + "loss": 1.253, + "theoretical_loss": 3.6557527809010715, + "tokens_seen": 981336064 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.30272483825683594, + "objective/train/avg_token_score": 0.013817708007991314, + "objective/train/docs_used": 563033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0947778224945068, + "objective/train/original_loss": 3.8225607872009277, + "objective/train/theoretical_loss": 3.6557294668699836, + "objective/train/tokens_used": 981401600, + "objective/train/unlikelihood_loss": 0.18993601202964783, + "theoretical_loss": 3.6557294668699836, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548445336008024, + "loss": 1.3334, + "theoretical_loss": 3.655706154831596, + "tokens_seen": 981467136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548244734202608, + "loss": 1.4409, + "theoretical_loss": 3.6556595367317075, + "tokens_seen": 981598208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548044132397192, + "loss": 1.4032, + "theoretical_loss": 3.6556129265989803, + "tokens_seen": 981729280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035478435305917757, + "loss": 1.3828, + "theoretical_loss": 3.655566324430989, + "tokens_seen": 981860352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035476429287863593, + "loss": 1.2849, + "theoretical_loss": 3.6555197302253095, + "tokens_seen": 981991424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547442326980943, + "loss": 1.4979, + "theoretical_loss": 3.6554731439795187, + "tokens_seen": 982122496 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035472417251755265, + "loss": 1.2859, + "theoretical_loss": 3.655426565691196, + "tokens_seen": 982253568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035470411233701107, + "loss": 1.348, + "theoretical_loss": 3.6553799953579196, + "tokens_seen": 982384640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035468405215646943, + "loss": 1.4571, + "theoretical_loss": 3.6553334329772693, + "tokens_seen": 982515712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546639919759278, + "loss": 1.3368, + "theoretical_loss": 3.6552868785468275, + "tokens_seen": 982646784 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035464393179538616, + "loss": 1.3102, + "theoretical_loss": 3.655240332064176, + "tokens_seen": 982777856 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035462387161484457, + "loss": 1.4931, + "theoretical_loss": 3.655193793526898, + "tokens_seen": 982908928 + }, + { + "debugging/length": 532.2, + "debugging/num_segments": 5, + "epoch": 0.3, + "objective/train/aligned_token_count": 0.44507575035095215, + "objective/train/avg_token_score": 0.02262255735695362, + "objective/train/docs_used": 564216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.418688178062439, + "objective/train/original_loss": 3.5507466793060303, + "objective/train/theoretical_loss": 3.6551472629325787, + "objective/train/tokens_used": 983040000, + "objective/train/unlikelihood_loss": 0.2308674454689026, + "theoretical_loss": 3.6551472629325787, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035460381143430293, + "loss": 1.4681, + "theoretical_loss": 3.6551472629325787, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545837512537613, + "loss": 1.3194, + "theoretical_loss": 3.655100740278803, + "tokens_seen": 983171072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035456369107321966, + "loss": 1.4251, + "theoretical_loss": 3.655054225563158, + "tokens_seen": 983302144 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354543630892678, + "loss": 1.4711, + "theoretical_loss": 3.655007718783231, + "tokens_seen": 983433216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035452357071213644, + "loss": 1.3482, + "theoretical_loss": 3.654961219936611, + "tokens_seen": 983564288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545035105315948, + "loss": 1.3376, + "theoretical_loss": 3.654914729020887, + "tokens_seen": 983695360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035448345035105316, + "loss": 1.3219, + "theoretical_loss": 3.6548682460336512, + "tokens_seen": 983826432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544633901705115, + "loss": 1.4918, + "theoretical_loss": 3.6548217709724944, + "tokens_seen": 983957504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035444332998996994, + "loss": 1.4087, + "theoretical_loss": 3.6547753038350095, + "tokens_seen": 984088576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544232698094283, + "loss": 1.4375, + "theoretical_loss": 3.6547288446187913, + "tokens_seen": 984219648 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035440320962888667, + "loss": 1.4951, + "theoretical_loss": 3.654682393321434, + "tokens_seen": 984350720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035438314944834503, + "loss": 1.4026, + "theoretical_loss": 3.654635949940534, + "tokens_seen": 984481792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035436308926780344, + "loss": 1.4373, + "theoretical_loss": 3.654589514473688, + "tokens_seen": 984612864 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.39320626854896545, + "objective/train/avg_token_score": 0.010986763052642345, + "objective/train/docs_used": 564602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3506890535354614, + "objective/train/original_loss": 3.8798861503601074, + "objective/train/theoretical_loss": 3.654566299707285, + "objective/train/tokens_used": 984678400, + "objective/train/unlikelihood_loss": 0.1373981088399887, + "theoretical_loss": 3.654566299707285, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543430290872618, + "loss": 1.2881, + "theoretical_loss": 3.6545430869184954, + "tokens_seen": 984743936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035432296890672017, + "loss": 1.4335, + "theoretical_loss": 3.6544966672725545, + "tokens_seen": 984875008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035430290872617853, + "loss": 1.5621, + "theoretical_loss": 3.6544502555334653, + "tokens_seen": 985006080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542828485456369, + "loss": 1.2623, + "theoretical_loss": 3.6544038516988295, + "tokens_seen": 985137152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542627883650953, + "loss": 1.2649, + "theoretical_loss": 3.6543574557662497, + "tokens_seen": 985268224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035424272818455367, + "loss": 1.3717, + "theoretical_loss": 3.654311067733329, + "tokens_seen": 985399296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035422266800401203, + "loss": 1.3067, + "theoretical_loss": 3.6542646875976716, + "tokens_seen": 985530368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542026078234704, + "loss": 1.2263, + "theoretical_loss": 3.654218315356884, + "tokens_seen": 985661440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541825476429288, + "loss": 1.3883, + "theoretical_loss": 3.6541719510085713, + "tokens_seen": 985792512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541624874623872, + "loss": 1.4713, + "theoretical_loss": 3.654125594550343, + "tokens_seen": 985923584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035414242728184554, + "loss": 1.5074, + "theoretical_loss": 3.6540792459798057, + "tokens_seen": 986054656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541223671013039, + "loss": 1.4182, + "theoretical_loss": 3.65403290529457, + "tokens_seen": 986185728 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.4630987346172333, + "objective/train/avg_token_score": 0.01241773460060358, + "objective/train/docs_used": 565772, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4464741945266724, + "objective/train/original_loss": 3.666053295135498, + "objective/train/theoretical_loss": 3.653986572492247, + "objective/train/tokens_used": 986316800, + "objective/train/unlikelihood_loss": 0.21595467627048492, + "theoretical_loss": 3.653986572492247, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035410230692076226, + "loss": 1.3563, + "theoretical_loss": 3.653986572492247, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540822467402207, + "loss": 1.3485, + "theoretical_loss": 3.653940247570448, + "tokens_seen": 986447872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035406218655967904, + "loss": 1.3982, + "theoretical_loss": 3.6538939305267855, + "tokens_seen": 986578944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540421263791374, + "loss": 1.199, + "theoretical_loss": 3.653847621358874, + "tokens_seen": 986710016 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035402206619859577, + "loss": 1.4425, + "theoretical_loss": 3.653801320064328, + "tokens_seen": 986841088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540020060180542, + "loss": 1.3989, + "theoretical_loss": 3.6537550266407637, + "tokens_seen": 986972160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035398194583751254, + "loss": 1.4257, + "theoretical_loss": 3.6537087410857976, + "tokens_seen": 987103232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539618856569709, + "loss": 1.2971, + "theoretical_loss": 3.653662463397048, + "tokens_seen": 987234304 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035394182547642927, + "loss": 1.4911, + "theoretical_loss": 3.653616193572134, + "tokens_seen": 987365376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035392176529588763, + "loss": 1.3159, + "theoretical_loss": 3.6535699316086756, + "tokens_seen": 987496448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035390170511534605, + "loss": 1.4359, + "theoretical_loss": 3.6535236775042934, + "tokens_seen": 987627520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538816449348044, + "loss": 1.4109, + "theoretical_loss": 3.6534774312566105, + "tokens_seen": 987758592 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035386158475426277, + "loss": 1.3772, + "theoretical_loss": 3.6534311928632497, + "tokens_seen": 987889664 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.5131353735923767, + "objective/train/avg_token_score": 0.009819666855037212, + "objective/train/docs_used": 566398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6271960735321045, + "objective/train/original_loss": 3.5202207565307617, + "objective/train/theoretical_loss": 3.653408076611197, + "objective/train/tokens_used": 987955200, + "objective/train/unlikelihood_loss": 0.15294396877288818, + "theoretical_loss": 3.653408076611197, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035384152457372113, + "loss": 1.5056, + "theoretical_loss": 3.6533849623218346, + "tokens_seen": 988020736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035382146439317955, + "loss": 1.2033, + "theoretical_loss": 3.6533387396299912, + "tokens_seen": 988151808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538014042126379, + "loss": 1.4141, + "theoretical_loss": 3.6532925247853454, + "tokens_seen": 988282880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537813440320963, + "loss": 1.4001, + "theoretical_loss": 3.6532463177855248, + "tokens_seen": 988413952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035376128385155464, + "loss": 1.3908, + "theoretical_loss": 3.653200118628157, + "tokens_seen": 988545024 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353741223671013, + "loss": 1.4621, + "theoretical_loss": 3.653153927310873, + "tokens_seen": 988676096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537211634904714, + "loss": 1.3822, + "theoretical_loss": 3.653107743831301, + "tokens_seen": 988807168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537011033099298, + "loss": 1.5152, + "theoretical_loss": 3.6530615681870744, + "tokens_seen": 988938240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035368104312938814, + "loss": 1.4451, + "theoretical_loss": 3.653015400375824, + "tokens_seen": 989069312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536609829488465, + "loss": 1.3273, + "theoretical_loss": 3.652969240395185, + "tokens_seen": 989200384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536409227683049, + "loss": 1.3629, + "theoretical_loss": 3.6529230882427903, + "tokens_seen": 989331456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536208625877633, + "loss": 1.3134, + "theoretical_loss": 3.6528769439162767, + "tokens_seen": 989462528 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.43566715717315674, + "objective/train/avg_token_score": 0.004809169098734856, + "objective/train/docs_used": 567574, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4599754810333252, + "objective/train/original_loss": 3.5639615058898926, + "objective/train/theoretical_loss": 3.65283080741328, + "objective/train/tokens_used": 989593600, + "objective/train/unlikelihood_loss": 0.2056824266910553, + "theoretical_loss": 3.65283080741328, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536008024072217, + "loss": 1.4705, + "theoretical_loss": 3.65283080741328, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035358074222668006, + "loss": 1.1901, + "theoretical_loss": 3.652784678731438, + "tokens_seen": 989724672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535606820461384, + "loss": 1.5224, + "theoretical_loss": 3.65273855786839, + "tokens_seen": 989855744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035354062186559684, + "loss": 1.4191, + "theoretical_loss": 3.652692444821774, + "tokens_seen": 989986816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535205616850552, + "loss": 1.516, + "theoretical_loss": 3.652646339589232, + "tokens_seen": 990117888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035350050150451356, + "loss": 1.3617, + "theoretical_loss": 3.6526002421684063, + "tokens_seen": 990248960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534804413239719, + "loss": 1.4038, + "theoretical_loss": 3.652554152556938, + "tokens_seen": 990380032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035346038114343034, + "loss": 1.3331, + "theoretical_loss": 3.6525080707524715, + "tokens_seen": 990511104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534403209628887, + "loss": 1.4997, + "theoretical_loss": 3.652461996752652, + "tokens_seen": 990642176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035342026078234707, + "loss": 1.503, + "theoretical_loss": 3.652415930555125, + "tokens_seen": 990773248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035340020060180543, + "loss": 1.285, + "theoretical_loss": 3.652369872157537, + "tokens_seen": 990904320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035338014042126384, + "loss": 1.4287, + "theoretical_loss": 3.6523238215575367, + "tokens_seen": 991035392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533600802407222, + "loss": 1.3083, + "theoretical_loss": 3.6522777787527714, + "tokens_seen": 991166464 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.4090298116207123, + "objective/train/avg_token_score": 0.02059776335954666, + "objective/train/docs_used": 568180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.470354676246643, + "objective/train/original_loss": 3.8700811862945557, + "objective/train/theoretical_loss": 3.6522547602728683, + "objective/train/tokens_used": 991232000, + "objective/train/unlikelihood_loss": 0.16083025932312012, + "theoretical_loss": 3.6522547602728683, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035334002006018057, + "loss": 1.4681, + "theoretical_loss": 3.652231743740893, + "tokens_seen": 991297536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035331995987963893, + "loss": 1.3436, + "theoretical_loss": 3.6521857165195506, + "tokens_seen": 991428608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532998996990973, + "loss": 1.2919, + "theoretical_loss": 3.652139697086397, + "tokens_seen": 991559680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532798395185557, + "loss": 1.3618, + "theoretical_loss": 3.652093685439085, + "tokens_seen": 991690752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035325977933801407, + "loss": 1.3632, + "theoretical_loss": 3.6520476815752687, + "tokens_seen": 991821824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035323971915747243, + "loss": 1.3469, + "theoretical_loss": 3.652001685492603, + "tokens_seen": 991952896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532196589769308, + "loss": 1.4643, + "theoretical_loss": 3.6519556971887437, + "tokens_seen": 992083968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531995987963892, + "loss": 1.4688, + "theoretical_loss": 3.651909716661348, + "tokens_seen": 992215040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531795386158476, + "loss": 1.4835, + "theoretical_loss": 3.6518637439080734, + "tokens_seen": 992346112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315947843530594, + "loss": 1.3667, + "theoretical_loss": 3.6518177789265795, + "tokens_seen": 992477184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531394182547643, + "loss": 1.3285, + "theoretical_loss": 3.651771821714526, + "tokens_seen": 992608256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035311935807422266, + "loss": 1.2963, + "theoretical_loss": 3.6517258722695747, + "tokens_seen": 992739328 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.5208333134651184, + "objective/train/avg_token_score": 0.006660131737589836, + "objective/train/docs_used": 569645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7138742208480835, + "objective/train/original_loss": 3.498253583908081, + "objective/train/theoretical_loss": 3.6516799305893866, + "objective/train/tokens_used": 992870400, + "objective/train/unlikelihood_loss": 0.19257555902004242, + "theoretical_loss": 3.6516799305893866, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530992978936811, + "loss": 1.31, + "theoretical_loss": 3.6516799305893866, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035307923771313944, + "loss": 1.4021, + "theoretical_loss": 3.6516339966716256, + "tokens_seen": 993001472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530591775325978, + "loss": 1.552, + "theoretical_loss": 3.651588070513955, + "tokens_seen": 993132544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035303911735205617, + "loss": 1.4162, + "theoretical_loss": 3.6515421521140405, + "tokens_seen": 993263616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530190571715146, + "loss": 1.4761, + "theoretical_loss": 3.651496241469548, + "tokens_seen": 993394688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035299899699097294, + "loss": 1.5041, + "theoretical_loss": 3.6514503385781456, + "tokens_seen": 993525760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529789368104313, + "loss": 1.4932, + "theoretical_loss": 3.6514044434374995, + "tokens_seen": 993656832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035295887662988967, + "loss": 1.2621, + "theoretical_loss": 3.65135855604528, + "tokens_seen": 993787904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035293881644934803, + "loss": 1.4231, + "theoretical_loss": 3.651312676399158, + "tokens_seen": 993918976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035291875626880645, + "loss": 1.493, + "theoretical_loss": 3.651266804496803, + "tokens_seen": 994050048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528986960882648, + "loss": 1.3174, + "theoretical_loss": 3.651220940335888, + "tokens_seen": 994181120 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035287863590772317, + "loss": 1.3782, + "theoretical_loss": 3.6511750839140866, + "tokens_seen": 994312192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035285857572718153, + "loss": 1.3643, + "theoretical_loss": 3.651129235229072, + "tokens_seen": 994443264 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.36070382595062256, + "objective/train/avg_token_score": 0.01513746939599514, + "objective/train/docs_used": 570064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1956753730773926, + "objective/train/original_loss": 3.717043876647949, + "objective/train/theoretical_loss": 3.6511063137871336, + "objective/train/tokens_used": 994508800, + "objective/train/unlikelihood_loss": 0.2787242531776428, + "theoretical_loss": 3.6511063137871336, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035283851554663995, + "loss": 1.4578, + "theoretical_loss": 3.65108339427852, + "tokens_seen": 994574336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528184553660983, + "loss": 1.3113, + "theoretical_loss": 3.6510375610601065, + "tokens_seen": 994705408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527983951855567, + "loss": 1.4662, + "theoretical_loss": 3.6509917355715094, + "tokens_seen": 994836480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035277833500501504, + "loss": 1.2984, + "theoretical_loss": 3.650945917810406, + "tokens_seen": 994967552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527582748244734, + "loss": 1.3558, + "theoretical_loss": 3.6509001077744756, + "tokens_seen": 995098624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527382146439318, + "loss": 1.4162, + "theoretical_loss": 3.6508543054613996, + "tokens_seen": 995229696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527181544633902, + "loss": 1.334, + "theoretical_loss": 3.650808510868858, + "tokens_seen": 995360768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035269809428284854, + "loss": 1.4138, + "theoretical_loss": 3.6507627239945326, + "tokens_seen": 995491840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526780341023069, + "loss": 1.565, + "theoretical_loss": 3.650716944836107, + "tokens_seen": 995622912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526579739217653, + "loss": 1.4543, + "theoretical_loss": 3.6506711733912667, + "tokens_seen": 995753984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526379137412237, + "loss": 1.3753, + "theoretical_loss": 3.6506254096576956, + "tokens_seen": 995885056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035261785356068204, + "loss": 1.3043, + "theoretical_loss": 3.65057965363308, + "tokens_seen": 996016128 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.3966275751590729, + "objective/train/avg_token_score": 0.016575051471590996, + "objective/train/docs_used": 571551, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2609350681304932, + "objective/train/original_loss": 4.093544006347656, + "objective/train/theoretical_loss": 3.6505339053151076, + "objective/train/tokens_used": 996147200, + "objective/train/unlikelihood_loss": 0.114774689078331, + "theoretical_loss": 3.6505339053151076, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525977933801404, + "loss": 1.4598, + "theoretical_loss": 3.6505339053151076, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035257773319959877, + "loss": 1.5782, + "theoretical_loss": 3.650488164701466, + "tokens_seen": 996278272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525576730190572, + "loss": 1.3161, + "theoretical_loss": 3.6504424317898447, + "tokens_seen": 996409344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035253761283851555, + "loss": 1.4033, + "theoretical_loss": 3.650396706577934, + "tokens_seen": 996540416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525175526579739, + "loss": 1.4777, + "theoretical_loss": 3.6503509890634254, + "tokens_seen": 996671488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035249749247743227, + "loss": 1.487, + "theoretical_loss": 3.65030527924401, + "tokens_seen": 996802560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524774322968907, + "loss": 1.3503, + "theoretical_loss": 3.650259577117382, + "tokens_seen": 996933632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035245737211634905, + "loss": 1.5736, + "theoretical_loss": 3.650213882681235, + "tokens_seen": 997064704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524373119358074, + "loss": 1.4786, + "theoretical_loss": 3.6501681959332646, + "tokens_seen": 997195776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524172517552658, + "loss": 1.3544, + "theoretical_loss": 3.650122516871167, + "tokens_seen": 997326848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035239719157472414, + "loss": 1.4541, + "theoretical_loss": 3.650076845492639, + "tokens_seen": 997457920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035237713139418255, + "loss": 1.4976, + "theoretical_loss": 3.6500311817953786, + "tokens_seen": 997588992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523570712136409, + "loss": 1.4358, + "theoretical_loss": 3.6499855257770855, + "tokens_seen": 997720064 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.4836876690387726, + "objective/train/avg_token_score": 0.007558201905339956, + "objective/train/docs_used": 572250, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6696515083312988, + "objective/train/original_loss": 3.8328704833984375, + "objective/train/theoretical_loss": 3.6499627006468325, + "objective/train/tokens_used": 997785600, + "objective/train/unlikelihood_loss": 0.14731846749782562, + "theoretical_loss": 3.6499627006468325, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523370110330993, + "loss": 1.5496, + "theoretical_loss": 3.6499398774354592, + "tokens_seen": 997851136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035231695085255764, + "loss": 1.3751, + "theoretical_loss": 3.6498942367682017, + "tokens_seen": 997982208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035229689067201606, + "loss": 1.3682, + "theoretical_loss": 3.6498486037730142, + "tokens_seen": 998113280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522768304914744, + "loss": 1.2942, + "theoretical_loss": 3.6498029784476005, + "tokens_seen": 998244352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522567703109328, + "loss": 1.3897, + "theoretical_loss": 3.649757360789664, + "tokens_seen": 998375424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035223671013039114, + "loss": 1.4728, + "theoretical_loss": 3.649711750796911, + "tokens_seen": 998506496 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035221664994984956, + "loss": 1.59, + "theoretical_loss": 3.6496661484670465, + "tokens_seen": 998637568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521965897693079, + "loss": 1.4133, + "theoretical_loss": 3.649620553797777, + "tokens_seen": 998768640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521765295887663, + "loss": 1.3863, + "theoretical_loss": 3.6495749667868123, + "tokens_seen": 998899712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035215646940822465, + "loss": 1.3708, + "theoretical_loss": 3.64952938743186, + "tokens_seen": 999030784 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352136409227683, + "loss": 1.3249, + "theoretical_loss": 3.6494838157306306, + "tokens_seen": 999161856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521163490471414, + "loss": 1.5187, + "theoretical_loss": 3.6494382516808357, + "tokens_seen": 999292928 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.33516618609428406, + "objective/train/avg_token_score": 0.016869336366653442, + "objective/train/docs_used": 573370, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0737072229385376, + "objective/train/original_loss": 3.9573769569396973, + "objective/train/theoretical_loss": 3.649392695280186, + "objective/train/tokens_used": 999424000, + "objective/train/unlikelihood_loss": 0.21445997059345245, + "theoretical_loss": 3.649392695280186, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035209628886659984, + "loss": 1.3691, + "theoretical_loss": 3.649392695280186, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520762286860582, + "loss": 1.4697, + "theoretical_loss": 3.649347146526396, + "tokens_seen": 999555072 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035205616850551657, + "loss": 1.3607, + "theoretical_loss": 3.649301605417178, + "tokens_seen": 999686144 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352036108324975, + "loss": 1.3809, + "theoretical_loss": 3.6492560719502483, + "tokens_seen": 999817216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035201604814443334, + "loss": 1.4106, + "theoretical_loss": 3.649210546123322, + "tokens_seen": 999948288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519959879638917, + "loss": 1.3823, + "theoretical_loss": 3.6491650279341163, + "tokens_seen": 1000079360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035197592778335007, + "loss": 1.402, + "theoretical_loss": 3.6491195173803495, + "tokens_seen": 1000210432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035195586760280843, + "loss": 1.383, + "theoretical_loss": 3.64907401445974, + "tokens_seen": 1000341504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035193580742226685, + "loss": 1.4538, + "theoretical_loss": 3.6490285191700074, + "tokens_seen": 1000472576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519157472417252, + "loss": 1.5932, + "theoretical_loss": 3.6489830315088727, + "tokens_seen": 1000603648 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035189568706118357, + "loss": 1.4367, + "theoretical_loss": 3.6489375514740576, + "tokens_seen": 1000734720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035187562688064193, + "loss": 1.2726, + "theoretical_loss": 3.6488920790632857, + "tokens_seen": 1000865792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035185556670010035, + "loss": 1.5615, + "theoretical_loss": 3.64884661427428, + "tokens_seen": 1000996864 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.46535924077033997, + "objective/train/avg_token_score": 0.00691384868696332, + "objective/train/docs_used": 574030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6538045406341553, + "objective/train/original_loss": 4.0024213790893555, + "objective/train/theoretical_loss": 3.6488238847372285, + "objective/train/tokens_used": 1001062400, + "objective/train/unlikelihood_loss": 0.1327609419822693, + "theoretical_loss": 3.6488238847372285, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518355065195587, + "loss": 1.3809, + "theoretical_loss": 3.6488011571047654, + "tokens_seen": 1001127936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518154463390171, + "loss": 1.3337, + "theoretical_loss": 3.6487557075524677, + "tokens_seen": 1001259008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035179538615847544, + "loss": 1.5324, + "theoretical_loss": 3.6487102656151134, + "tokens_seen": 1001390080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517753259779338, + "loss": 1.4379, + "theoretical_loss": 3.6486648312904304, + "tokens_seen": 1001521152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517552657973922, + "loss": 1.4136, + "theoretical_loss": 3.6486194045761473, + "tokens_seen": 1001652224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517352056168506, + "loss": 1.4819, + "theoretical_loss": 3.6485739854699935, + "tokens_seen": 1001783296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035171514543630894, + "loss": 1.4511, + "theoretical_loss": 3.6485285739696995, + "tokens_seen": 1001914368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516950852557673, + "loss": 1.5802, + "theoretical_loss": 3.648483170072997, + "tokens_seen": 1002045440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516750250752257, + "loss": 1.3837, + "theoretical_loss": 3.6484377737776192, + "tokens_seen": 1002176512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516549648946841, + "loss": 1.3181, + "theoretical_loss": 3.6483923850812987, + "tokens_seen": 1002307584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035163490471414244, + "loss": 1.441, + "theoretical_loss": 3.6483470039817707, + "tokens_seen": 1002438656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516148445336008, + "loss": 1.3677, + "theoretical_loss": 3.6483016304767704, + "tokens_seen": 1002569728 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.3579545319080353, + "objective/train/avg_token_score": 0.015897734090685844, + "objective/train/docs_used": 574442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.232710838317871, + "objective/train/original_loss": 3.901421308517456, + "objective/train/theoretical_loss": 3.6482562645640337, + "objective/train/tokens_used": 1002700800, + "objective/train/unlikelihood_loss": 0.14255332946777344, + "theoretical_loss": 3.6482562645640337, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035159478435305917, + "loss": 1.403, + "theoretical_loss": 3.6482562645640337, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515747241725176, + "loss": 1.3995, + "theoretical_loss": 3.6482109062412986, + "tokens_seen": 1002831872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035155466399197595, + "loss": 1.3129, + "theoretical_loss": 3.648165555506303, + "tokens_seen": 1002962944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515346038114343, + "loss": 1.4665, + "theoretical_loss": 3.6481202123567864, + "tokens_seen": 1003094016 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035151454363089267, + "loss": 1.3284, + "theoretical_loss": 3.6480748767904894, + "tokens_seen": 1003225088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514944834503511, + "loss": 1.3856, + "theoretical_loss": 3.6480295488051535, + "tokens_seen": 1003356160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035147442326980945, + "loss": 1.419, + "theoretical_loss": 3.64798422839852, + "tokens_seen": 1003487232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514543630892678, + "loss": 1.367, + "theoretical_loss": 3.647938915568333, + "tokens_seen": 1003618304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514343029087262, + "loss": 1.3688, + "theoretical_loss": 3.6478936103123365, + "tokens_seen": 1003749376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035141424272818454, + "loss": 1.3298, + "theoretical_loss": 3.647848312628275, + "tokens_seen": 1003880448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035139418254764295, + "loss": 1.3666, + "theoretical_loss": 3.6478030225138953, + "tokens_seen": 1004011520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513741223671013, + "loss": 1.3124, + "theoretical_loss": 3.647757739966944, + "tokens_seen": 1004142592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513540621865597, + "loss": 1.476, + "theoretical_loss": 3.6477124649851698, + "tokens_seen": 1004273664 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.4769061505794525, + "objective/train/avg_token_score": 0.019502347335219383, + "objective/train/docs_used": 575683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5307387113571167, + "objective/train/original_loss": 4.083261966705322, + "objective/train/theoretical_loss": 3.64768983033052, + "objective/train/tokens_used": 1004339200, + "objective/train/unlikelihood_loss": 0.13358524441719055, + "theoretical_loss": 3.64768983033052, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035133400200601804, + "loss": 1.4487, + "theoretical_loss": 3.647667197566321, + "tokens_seen": 1004404736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035131394182547646, + "loss": 1.3929, + "theoretical_loss": 3.6476219377081476, + "tokens_seen": 1004535808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512938816449348, + "loss": 1.2836, + "theoretical_loss": 3.647576685408401, + "tokens_seen": 1004666880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512738214643932, + "loss": 1.3646, + "theoretical_loss": 3.6475314406648334, + "tokens_seen": 1004797952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035125376128385154, + "loss": 1.4293, + "theoretical_loss": 3.6474862034751965, + "tokens_seen": 1004929024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035123370110330996, + "loss": 1.3834, + "theoretical_loss": 3.6474409738372446, + "tokens_seen": 1005060096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512136409227683, + "loss": 1.2971, + "theoretical_loss": 3.647395751748733, + "tokens_seen": 1005191168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511935807422267, + "loss": 1.4844, + "theoretical_loss": 3.6473505372074166, + "tokens_seen": 1005322240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035117352056168505, + "loss": 1.3377, + "theoretical_loss": 3.647305330211053, + "tokens_seen": 1005453312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511534603811434, + "loss": 1.3465, + "theoretical_loss": 3.647260130757399, + "tokens_seen": 1005584384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511334002006018, + "loss": 1.2369, + "theoretical_loss": 3.647214938844214, + "tokens_seen": 1005715456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511133400200602, + "loss": 1.3712, + "theoretical_loss": 3.6471697544692567, + "tokens_seen": 1005846528 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.43682795763015747, + "objective/train/avg_token_score": 0.010027443058788776, + "objective/train/docs_used": 576356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5100964307785034, + "objective/train/original_loss": 3.784379482269287, + "objective/train/theoretical_loss": 3.6471245776302883, + "objective/train/tokens_used": 1005977600, + "objective/train/unlikelihood_loss": 0.14172670245170593, + "theoretical_loss": 3.6471245776302883, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035109327983951855, + "loss": 1.3394, + "theoretical_loss": 3.6471245776302883, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510732196589769, + "loss": 1.2953, + "theoretical_loss": 3.6470794083250695, + "tokens_seen": 1006108672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035105315947843533, + "loss": 1.2844, + "theoretical_loss": 3.647034246551364, + "tokens_seen": 1006239744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510330992978937, + "loss": 1.3195, + "theoretical_loss": 3.646989092306934, + "tokens_seen": 1006370816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035101303911735205, + "loss": 1.5523, + "theoretical_loss": 3.6469439455895447, + "tokens_seen": 1006501888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509929789368104, + "loss": 1.3438, + "theoretical_loss": 3.646898806396961, + "tokens_seen": 1006632960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509729187562688, + "loss": 1.1806, + "theoretical_loss": 3.646853674726949, + "tokens_seen": 1006764032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509528585757272, + "loss": 1.5419, + "theoretical_loss": 3.6468085505772763, + "tokens_seen": 1006895104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035093279839518556, + "loss": 1.4088, + "theoretical_loss": 3.646763433945711, + "tokens_seen": 1007026176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509127382146439, + "loss": 1.3821, + "theoretical_loss": 3.6467183248300215, + "tokens_seen": 1007157248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508926780341023, + "loss": 1.322, + "theoretical_loss": 3.646673223227979, + "tokens_seen": 1007288320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508726178535607, + "loss": 1.4138, + "theoretical_loss": 3.646628129137354, + "tokens_seen": 1007419392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035085255767301906, + "loss": 1.4809, + "theoretical_loss": 3.6465830425559185, + "tokens_seen": 1007550464 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.4149560034275055, + "objective/train/avg_token_score": 0.010837187990546227, + "objective/train/docs_used": 577861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3405011892318726, + "objective/train/original_loss": 3.7844693660736084, + "objective/train/theoretical_loss": 3.6465605020804506, + "objective/train/tokens_used": 1007616000, + "objective/train/unlikelihood_loss": 0.19078943133354187, + "theoretical_loss": 3.6465605020804506, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508324974924774, + "loss": 1.3426, + "theoretical_loss": 3.646537963481445, + "tokens_seen": 1007681536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508124373119358, + "loss": 1.2685, + "theoretical_loss": 3.646492891911708, + "tokens_seen": 1007812608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035079237713139415, + "loss": 1.2731, + "theoretical_loss": 3.646447827844482, + "tokens_seen": 1007943680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035077231695085256, + "loss": 1.4583, + "theoretical_loss": 3.6464027712775433, + "tokens_seen": 1008074752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507522567703109, + "loss": 1.5625, + "theoretical_loss": 3.6463577222086676, + "tokens_seen": 1008205824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507321965897693, + "loss": 1.3419, + "theoretical_loss": 3.646312680635633, + "tokens_seen": 1008336896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035071213640922765, + "loss": 1.4165, + "theoretical_loss": 3.646267646556219, + "tokens_seen": 1008467968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035069207622868607, + "loss": 1.5083, + "theoretical_loss": 3.6462226199682046, + "tokens_seen": 1008599040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035067201604814443, + "loss": 1.413, + "theoretical_loss": 3.646177600869369, + "tokens_seen": 1008730112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506519558676028, + "loss": 1.485, + "theoretical_loss": 3.646132589257496, + "tokens_seen": 1008861184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035063189568706115, + "loss": 1.3924, + "theoretical_loss": 3.646087585130366, + "tokens_seen": 1008992256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506118355065195, + "loss": 1.4616, + "theoretical_loss": 3.6460425884857637, + "tokens_seen": 1009123328 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.34341397881507874, + "objective/train/avg_token_score": 0.007534741889685392, + "objective/train/docs_used": 578450, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.258749008178711, + "objective/train/original_loss": 4.296646595001221, + "objective/train/theoretical_loss": 3.6459975993214724, + "objective/train/tokens_used": 1009254400, + "objective/train/unlikelihood_loss": 0.15291565656661987, + "theoretical_loss": 3.6459975993214724, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350591775325978, + "loss": 1.4079, + "theoretical_loss": 3.6459975993214724, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035057171514543635, + "loss": 1.2309, + "theoretical_loss": 3.645952617635278, + "tokens_seen": 1009385472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505516549648947, + "loss": 1.4333, + "theoretical_loss": 3.645907643424966, + "tokens_seen": 1009516544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035053159478435307, + "loss": 1.3065, + "theoretical_loss": 3.645862676688325, + "tokens_seen": 1009647616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505115346038115, + "loss": 1.4091, + "theoretical_loss": 3.6458177174231414, + "tokens_seen": 1009778688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035049147442326985, + "loss": 1.3537, + "theoretical_loss": 3.6457727656272043, + "tokens_seen": 1009909760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504714142427282, + "loss": 1.2421, + "theoretical_loss": 3.6457278212983053, + "tokens_seen": 1010040832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504513540621866, + "loss": 1.3307, + "theoretical_loss": 3.6456828844342337, + "tokens_seen": 1010171904 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035043129388164494, + "loss": 1.5459, + "theoretical_loss": 3.6456379550327815, + "tokens_seen": 1010302976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035041123370110335, + "loss": 1.3311, + "theoretical_loss": 3.6455930330917425, + "tokens_seen": 1010434048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503911735205617, + "loss": 1.3016, + "theoretical_loss": 3.645548118608909, + "tokens_seen": 1010565120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503711133400201, + "loss": 1.3019, + "theoretical_loss": 3.6455032115820765, + "tokens_seen": 1010696192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035035105315947844, + "loss": 1.3343, + "theoretical_loss": 3.6454583120090405, + "tokens_seen": 1010827264 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.4002321660518646, + "objective/train/avg_token_score": 0.002780243521556258, + "objective/train/docs_used": 579792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.158745288848877, + "objective/train/original_loss": 3.414778709411621, + "objective/train/theoretical_loss": 3.6454358650170073, + "objective/train/tokens_used": 1010892800, + "objective/train/unlikelihood_loss": 0.21636781096458435, + "theoretical_loss": 3.6454358650170073, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035033099297893686, + "loss": 1.4011, + "theoretical_loss": 3.645413419887597, + "tokens_seen": 1010958336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503109327983952, + "loss": 1.5224, + "theoretical_loss": 3.6453685352155443, + "tokens_seen": 1011089408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502908726178536, + "loss": 1.3793, + "theoretical_loss": 3.6453236579906805, + "tokens_seen": 1011220480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035027081243731194, + "loss": 1.3878, + "theoretical_loss": 3.6452787882108044, + "tokens_seen": 1011351552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035025075225677036, + "loss": 1.4288, + "theoretical_loss": 3.645233925873717, + "tokens_seen": 1011482624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502306920762287, + "loss": 1.2878, + "theoretical_loss": 3.645189070977219, + "tokens_seen": 1011613696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502106318956871, + "loss": 1.4074, + "theoretical_loss": 3.6451442235191127, + "tokens_seen": 1011744768 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035019057171514545, + "loss": 1.244, + "theoretical_loss": 3.645099383497201, + "tokens_seen": 1011875840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501705115346038, + "loss": 1.3708, + "theoretical_loss": 3.6450545509092884, + "tokens_seen": 1012006912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501504513540622, + "loss": 1.4617, + "theoretical_loss": 3.6450097257531793, + "tokens_seen": 1012137984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501303911735206, + "loss": 1.2183, + "theoretical_loss": 3.64496490802668, + "tokens_seen": 1012269056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035011033099297895, + "loss": 1.253, + "theoretical_loss": 3.6449200977275966, + "tokens_seen": 1012400128 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.3552052676677704, + "objective/train/avg_token_score": 0.00677725812420249, + "objective/train/docs_used": 580179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.118126392364502, + "objective/train/original_loss": 4.062026023864746, + "objective/train/theoretical_loss": 3.6448752948537377, + "objective/train/tokens_used": 1012531200, + "objective/train/unlikelihood_loss": 0.1604587882757187, + "theoretical_loss": 3.6448752948537377, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500902708124373, + "loss": 1.3152, + "theoretical_loss": 3.6448752948537377, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035007021063189573, + "loss": 1.4685, + "theoretical_loss": 3.6448304994029113, + "tokens_seen": 1012662272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500501504513541, + "loss": 1.3288, + "theoretical_loss": 3.6447857113729274, + "tokens_seen": 1012793344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035003009027081245, + "loss": 1.2116, + "theoretical_loss": 3.6447409307615963, + "tokens_seen": 1012924416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500100300902708, + "loss": 1.2846, + "theoretical_loss": 3.644696157566729, + "tokens_seen": 1013055488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499899699097292, + "loss": 1.4553, + "theoretical_loss": 3.644651391786139, + "tokens_seen": 1013186560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499699097291876, + "loss": 1.352, + "theoretical_loss": 3.644606633417639, + "tokens_seen": 1013317632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034994984954864596, + "loss": 1.2838, + "theoretical_loss": 3.644561882459043, + "tokens_seen": 1013448704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499297893681043, + "loss": 1.2639, + "theoretical_loss": 3.6445171389081663, + "tokens_seen": 1013579776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499097291875627, + "loss": 1.5675, + "theoretical_loss": 3.6444724027628252, + "tokens_seen": 1013710848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498896690070211, + "loss": 1.2858, + "theoretical_loss": 3.6444276740208363, + "tokens_seen": 1013841920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034986960882647946, + "loss": 1.2943, + "theoretical_loss": 3.644382952680018, + "tokens_seen": 1013972992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498495486459378, + "loss": 1.4733, + "theoretical_loss": 3.644338238738189, + "tokens_seen": 1014104064 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.4313294291496277, + "objective/train/avg_token_score": 0.015423484146595001, + "objective/train/docs_used": 581599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6097753047943115, + "objective/train/original_loss": 3.6197855472564697, + "objective/train/theoretical_loss": 3.6443158845412142, + "objective/train/tokens_used": 1014169600, + "objective/train/unlikelihood_loss": 0.27291402220726013, + "theoretical_loss": 3.6443158845412142, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498294884653962, + "loss": 1.4708, + "theoretical_loss": 3.6442935321931693, + "tokens_seen": 1014235136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034980942828485455, + "loss": 1.4029, + "theoretical_loss": 3.644248833042779, + "tokens_seen": 1014366208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034978936810431296, + "loss": 1.3666, + "theoretical_loss": 3.64420414128484, + "tokens_seen": 1014497280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497693079237713, + "loss": 1.407, + "theoretical_loss": 3.6441594569171754, + "tokens_seen": 1014628352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497492477432297, + "loss": 1.4114, + "theoretical_loss": 3.644114779937608, + "tokens_seen": 1014759424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034972918756268805, + "loss": 1.4009, + "theoretical_loss": 3.6440701103439626, + "tokens_seen": 1014890496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034970912738214647, + "loss": 1.4196, + "theoretical_loss": 3.644025448134064, + "tokens_seen": 1015021568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034968906720160483, + "loss": 1.2731, + "theoretical_loss": 3.6439807933057393, + "tokens_seen": 1015152640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496690070210632, + "loss": 1.3781, + "theoretical_loss": 3.6439361458568147, + "tokens_seen": 1015283712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034964894684052155, + "loss": 1.4365, + "theoretical_loss": 3.6438915057851196, + "tokens_seen": 1015414784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496288866599799, + "loss": 1.4548, + "theoretical_loss": 3.6438468730884814, + "tokens_seen": 1015545856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034960882647943833, + "loss": 1.5522, + "theoretical_loss": 3.6438022477647314, + "tokens_seen": 1015676928 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.5857771039009094, + "objective/train/avg_token_score": 0.009331127628684044, + "objective/train/docs_used": 582330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0380396842956543, + "objective/train/original_loss": 3.8352012634277344, + "objective/train/theoretical_loss": 3.6437576298116996, + "objective/train/tokens_used": 1015808000, + "objective/train/unlikelihood_loss": 0.11454664170742035, + "theoretical_loss": 3.6437576298116996, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495887662988967, + "loss": 1.6195, + "theoretical_loss": 3.6437576298116996, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034956870611835506, + "loss": 1.5187, + "theoretical_loss": 3.6437130192272176, + "tokens_seen": 1015939072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495486459378134, + "loss": 1.5116, + "theoretical_loss": 3.6436684160091186, + "tokens_seen": 1016070144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034952858575727183, + "loss": 1.4105, + "theoretical_loss": 3.643623820155237, + "tokens_seen": 1016201216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495085255767302, + "loss": 1.3507, + "theoretical_loss": 3.6435792316634057, + "tokens_seen": 1016332288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034948846539618856, + "loss": 1.5424, + "theoretical_loss": 3.6435346505314605, + "tokens_seen": 1016463360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494684052156469, + "loss": 1.4133, + "theoretical_loss": 3.6434900767572382, + "tokens_seen": 1016594432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494483450351053, + "loss": 1.2113, + "theoretical_loss": 3.6434455103385766, + "tokens_seen": 1016725504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494282848545637, + "loss": 1.4535, + "theoretical_loss": 3.6434009512733123, + "tokens_seen": 1016856576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034940822467402206, + "loss": 1.2557, + "theoretical_loss": 3.643356399559286, + "tokens_seen": 1016987648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493881644934804, + "loss": 1.3886, + "theoretical_loss": 3.6433118551943364, + "tokens_seen": 1017118720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493681043129388, + "loss": 1.5137, + "theoretical_loss": 3.6432673181763056, + "tokens_seen": 1017249792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493480441323972, + "loss": 1.361, + "theoretical_loss": 3.643222788503034, + "tokens_seen": 1017380864 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.3973607122898102, + "objective/train/avg_token_score": 0.014590024016797543, + "objective/train/docs_used": 582938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4243128299713135, + "objective/train/original_loss": 3.6094589233398438, + "objective/train/theoretical_loss": 3.64320052642001, + "objective/train/tokens_used": 1017446400, + "objective/train/unlikelihood_loss": 0.1775514781475067, + "theoretical_loss": 3.64320052642001, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034932798395185557, + "loss": 1.5002, + "theoretical_loss": 3.643178266172366, + "tokens_seen": 1017511936 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034930792377131393, + "loss": 1.4128, + "theoretical_loss": 3.6431337511821438, + "tokens_seen": 1017643008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492878635907723, + "loss": 1.4265, + "theoretical_loss": 3.6430892435302136, + "tokens_seen": 1017774080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034926780341023065, + "loss": 1.439, + "theoretical_loss": 3.6430447432144186, + "tokens_seen": 1017905152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034924774322968907, + "loss": 1.4352, + "theoretical_loss": 3.643000250232607, + "tokens_seen": 1018036224 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034922768304914743, + "loss": 1.3985, + "theoretical_loss": 3.6429557645826254, + "tokens_seen": 1018167296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492076228686058, + "loss": 1.3261, + "theoretical_loss": 3.642911286262322, + "tokens_seen": 1018298368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034918756268806416, + "loss": 1.4912, + "theoretical_loss": 3.6428668152695467, + "tokens_seen": 1018429440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034916750250752257, + "loss": 1.4406, + "theoretical_loss": 3.642822351602148, + "tokens_seen": 1018560512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034914744232698093, + "loss": 1.427, + "theoretical_loss": 3.642777895257978, + "tokens_seen": 1018691584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491273821464393, + "loss": 1.3068, + "theoretical_loss": 3.6427334462348884, + "tokens_seen": 1018822656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034910732196589766, + "loss": 1.4349, + "theoretical_loss": 3.642689004530731, + "tokens_seen": 1018953728 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.3799486756324768, + "objective/train/avg_token_score": 0.011530186980962753, + "objective/train/docs_used": 584109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.064042091369629, + "objective/train/original_loss": 3.7128713130950928, + "objective/train/theoretical_loss": 3.6426445701433607, + "objective/train/tokens_used": 1019084800, + "objective/train/unlikelihood_loss": 0.18346698582172394, + "theoretical_loss": 3.6426445701433607, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034908726178535613, + "loss": 1.311, + "theoretical_loss": 3.6426445701433607, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490672016048145, + "loss": 1.4584, + "theoretical_loss": 3.6426001430706316, + "tokens_seen": 1019215872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034904714142427285, + "loss": 1.3903, + "theoretical_loss": 3.642555723310399, + "tokens_seen": 1019346944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490270812437312, + "loss": 1.2791, + "theoretical_loss": 3.6425113108605194, + "tokens_seen": 1019478016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490070210631896, + "loss": 1.4687, + "theoretical_loss": 3.6424669057188495, + "tokens_seen": 1019609088 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348986960882648, + "loss": 1.3015, + "theoretical_loss": 3.642422507883248, + "tokens_seen": 1019740160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034896690070210636, + "loss": 1.3666, + "theoretical_loss": 3.6423781173515746, + "tokens_seen": 1019871232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489468405215647, + "loss": 1.4729, + "theoretical_loss": 3.6423337341216877, + "tokens_seen": 1020002304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489267803410231, + "loss": 1.5528, + "theoretical_loss": 3.6422893581914497, + "tokens_seen": 1020133376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489067201604815, + "loss": 1.3966, + "theoretical_loss": 3.642244989558721, + "tokens_seen": 1020264448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034888665997993986, + "loss": 1.361, + "theoretical_loss": 3.6422006282213655, + "tokens_seen": 1020395520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488665997993982, + "loss": 1.4664, + "theoretical_loss": 3.642156274177246, + "tokens_seen": 1020526592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488465396188566, + "loss": 1.3218, + "theoretical_loss": 3.642111927424228, + "tokens_seen": 1020657664 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.31805962324142456, + "objective/train/avg_token_score": 0.01613454334437847, + "objective/train/docs_used": 584789, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0935728549957275, + "objective/train/original_loss": 4.037130355834961, + "objective/train/theoretical_loss": 3.642089756781214, + "objective/train/tokens_used": 1020723200, + "objective/train/unlikelihood_loss": 0.15320149064064026, + "theoretical_loss": 3.642089756781214, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034882647943831495, + "loss": 1.2226, + "theoretical_loss": 3.642067587960175, + "tokens_seen": 1020788736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034880641925777336, + "loss": 1.4204, + "theoretical_loss": 3.6420232557829553, + "tokens_seen": 1020919808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487863590772317, + "loss": 1.4269, + "theoretical_loss": 3.6419789308904345, + "tokens_seen": 1021050880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487662988966901, + "loss": 1.4167, + "theoretical_loss": 3.6419346132804815, + "tokens_seen": 1021181952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034874623871614845, + "loss": 1.3712, + "theoretical_loss": 3.6418903029509653, + "tokens_seen": 1021313024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034872617853560687, + "loss": 1.2262, + "theoretical_loss": 3.641845999899755, + "tokens_seen": 1021444096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034870611835506523, + "loss": 1.3951, + "theoretical_loss": 3.6418017041247226, + "tokens_seen": 1021575168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486860581745236, + "loss": 1.206, + "theoretical_loss": 3.6417574156237387, + "tokens_seen": 1021706240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034866599799398195, + "loss": 1.4469, + "theoretical_loss": 3.641713134394676, + "tokens_seen": 1021837312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486459378134403, + "loss": 1.5736, + "theoretical_loss": 3.6416688604354084, + "tokens_seen": 1021968384 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034862587763289873, + "loss": 1.2889, + "theoretical_loss": 3.64162459374381, + "tokens_seen": 1022099456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486058174523571, + "loss": 1.3978, + "theoretical_loss": 3.641580334317756, + "tokens_seen": 1022230528 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.36192572116851807, + "objective/train/avg_token_score": 0.011094314977526665, + "objective/train/docs_used": 586155, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1584200859069824, + "objective/train/original_loss": 3.634643077850342, + "objective/train/theoretical_loss": 3.6415360821551226, + "objective/train/tokens_used": 1022361600, + "objective/train/unlikelihood_loss": 0.1882539987564087, + "theoretical_loss": 3.6415360821551226, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034858575727181546, + "loss": 1.2332, + "theoretical_loss": 3.6415360821551226, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485656970912738, + "loss": 1.3629, + "theoretical_loss": 3.6414918372537866, + "tokens_seen": 1022492672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034854563691073223, + "loss": 1.3787, + "theoretical_loss": 3.6414475996116265, + "tokens_seen": 1022623744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485255767301906, + "loss": 1.4235, + "theoretical_loss": 3.6414033692265204, + "tokens_seen": 1022754816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034850551654964896, + "loss": 1.323, + "theoretical_loss": 3.641359146096348, + "tokens_seen": 1022885888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484854563691073, + "loss": 1.5149, + "theoretical_loss": 3.6413149302189907, + "tokens_seen": 1023016960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484653961885657, + "loss": 1.2879, + "theoretical_loss": 3.6412707215923295, + "tokens_seen": 1023148032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484453360080241, + "loss": 1.394, + "theoretical_loss": 3.641226520214246, + "tokens_seen": 1023279104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034842527582748246, + "loss": 1.2977, + "theoretical_loss": 3.6411823260826246, + "tokens_seen": 1023410176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484052156469408, + "loss": 1.4071, + "theoretical_loss": 3.6411381391953492, + "tokens_seen": 1023541248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483851554663992, + "loss": 1.3542, + "theoretical_loss": 3.6410939595503047, + "tokens_seen": 1023672320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483650952858576, + "loss": 1.4056, + "theoretical_loss": 3.641049787145376, + "tokens_seen": 1023803392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034834503510531597, + "loss": 1.5626, + "theoretical_loss": 3.6410056219784526, + "tokens_seen": 1023934464 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.304618775844574, + "objective/train/avg_token_score": 0.045001786202192307, + "objective/train/docs_used": 586672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9293109178543091, + "objective/train/original_loss": 4.2454609870910645, + "objective/train/theoretical_loss": 3.6409835421085814, + "objective/train/tokens_used": 1024000000, + "objective/train/unlikelihood_loss": 0.14533387124538422, + "theoretical_loss": 3.6409835421085814, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034832497492477433, + "loss": 1.3169, + "theoretical_loss": 3.640961464047419, + "tokens_seen": 1024065536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483049147442327, + "loss": 1.3597, + "theoretical_loss": 3.640917313350166, + "tokens_seen": 1024196608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034828485456369105, + "loss": 1.3128, + "theoretical_loss": 3.640873169884582, + "tokens_seen": 1024327680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034826479438314947, + "loss": 1.5027, + "theoretical_loss": 3.6408290336485583, + "tokens_seen": 1024458752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034824473420260783, + "loss": 1.3523, + "theoretical_loss": 3.640784904639985, + "tokens_seen": 1024589824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482246740220662, + "loss": 1.3947, + "theoretical_loss": 3.6407407828567555, + "tokens_seen": 1024720896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034820461384152456, + "loss": 1.4068, + "theoretical_loss": 3.640696668296761, + "tokens_seen": 1024851968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034818455366098297, + "loss": 1.4624, + "theoretical_loss": 3.6406525609578972, + "tokens_seen": 1024983040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034816449348044133, + "loss": 1.3513, + "theoretical_loss": 3.6406084608380582, + "tokens_seen": 1025114112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481444332998997, + "loss": 1.4692, + "theoretical_loss": 3.6405643679351396, + "tokens_seen": 1025245184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034812437311935806, + "loss": 1.4193, + "theoretical_loss": 3.6405202822470377, + "tokens_seen": 1025376256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481043129388164, + "loss": 1.2513, + "theoretical_loss": 3.6404762037716503, + "tokens_seen": 1025507328 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.47776147723197937, + "objective/train/avg_token_score": 0.007456145249307156, + "objective/train/docs_used": 587795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6344645023345947, + "objective/train/original_loss": 3.631981611251831, + "objective/train/theoretical_loss": 3.6404321325068754, + "objective/train/tokens_used": 1025638400, + "objective/train/unlikelihood_loss": 0.2005711942911148, + "theoretical_loss": 3.6404321325068754, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034808425275827484, + "loss": 1.4637, + "theoretical_loss": 3.6404321325068754, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480641925777332, + "loss": 1.3946, + "theoretical_loss": 3.640388068450613, + "tokens_seen": 1025769472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034804413239719156, + "loss": 1.3804, + "theoretical_loss": 3.640344011600762, + "tokens_seen": 1025900544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480240722166499, + "loss": 1.4152, + "theoretical_loss": 3.6402999619552245, + "tokens_seen": 1026031616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034800401203610834, + "loss": 1.3953, + "theoretical_loss": 3.640255919511901, + "tokens_seen": 1026162688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479839518555667, + "loss": 1.5444, + "theoretical_loss": 3.6402118842686955, + "tokens_seen": 1026293760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034796389167502506, + "loss": 1.2597, + "theoretical_loss": 3.6401678562235107, + "tokens_seen": 1026424832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034794383149448343, + "loss": 1.3421, + "theoretical_loss": 3.6401238353742515, + "tokens_seen": 1026555904 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034792377131394184, + "loss": 1.3678, + "theoretical_loss": 3.6400798217188233, + "tokens_seen": 1026686976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479037111334002, + "loss": 1.2828, + "theoretical_loss": 3.640035815255132, + "tokens_seen": 1026818048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034788365095285857, + "loss": 1.4239, + "theoretical_loss": 3.6399918159810847, + "tokens_seen": 1026949120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034786359077231693, + "loss": 1.358, + "theoretical_loss": 3.63994782389459, + "tokens_seen": 1027080192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478435305917753, + "loss": 1.3776, + "theoretical_loss": 3.639903838993556, + "tokens_seen": 1027211264 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.38874632120132446, + "objective/train/avg_token_score": 0.016582434996962547, + "objective/train/docs_used": 588595, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3833376169204712, + "objective/train/original_loss": 3.6898438930511475, + "objective/train/theoretical_loss": 3.6398818492369336, + "objective/train/tokens_used": 1027276800, + "objective/train/unlikelihood_loss": 0.24722784757614136, + "theoretical_loss": 3.6398818492369336, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478234704112337, + "loss": 1.4512, + "theoretical_loss": 3.639859861275893, + "tokens_seen": 1027342336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034780341023069207, + "loss": 1.2691, + "theoretical_loss": 3.6398158907395106, + "tokens_seen": 1027473408 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034778335005015043, + "loss": 1.2687, + "theoretical_loss": 3.6397719273823212, + "tokens_seen": 1027604480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477632898696088, + "loss": 1.2902, + "theoretical_loss": 3.639727971202237, + "tokens_seen": 1027735552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477432296890672, + "loss": 1.4122, + "theoretical_loss": 3.639684022197171, + "tokens_seen": 1027866624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477231695085256, + "loss": 1.4438, + "theoretical_loss": 3.6396400803650377, + "tokens_seen": 1027997696 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034770310932798394, + "loss": 1.4805, + "theoretical_loss": 3.639596145703752, + "tokens_seen": 1028128768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476830491474423, + "loss": 1.4118, + "theoretical_loss": 3.6395522182112288, + "tokens_seen": 1028259840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034766298896690066, + "loss": 1.4989, + "theoretical_loss": 3.639508297885386, + "tokens_seen": 1028390912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476429287863591, + "loss": 1.1661, + "theoretical_loss": 3.6394643847241404, + "tokens_seen": 1028521984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034762286860581744, + "loss": 1.4303, + "theoretical_loss": 3.6394204787254107, + "tokens_seen": 1028653056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476028084252758, + "loss": 1.3691, + "theoretical_loss": 3.6393765798871165, + "tokens_seen": 1028784128 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.4570503532886505, + "objective/train/avg_token_score": 0.009330947883427143, + "objective/train/docs_used": 589927, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4876413345336914, + "objective/train/original_loss": 3.6532764434814453, + "objective/train/theoretical_loss": 3.639332688207178, + "objective/train/tokens_used": 1028915200, + "objective/train/unlikelihood_loss": 0.17469806969165802, + "theoretical_loss": 3.639332688207178, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475827482447342, + "loss": 1.3863, + "theoretical_loss": 3.639332688207178, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034756268806419263, + "loss": 1.3937, + "theoretical_loss": 3.639288803683515, + "tokens_seen": 1029046272 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347542627883651, + "loss": 1.298, + "theoretical_loss": 3.639244926314051, + "tokens_seen": 1029177344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034752256770310936, + "loss": 1.4048, + "theoretical_loss": 3.6392010560967085, + "tokens_seen": 1029308416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475025075225677, + "loss": 1.4178, + "theoretical_loss": 3.63915719302941, + "tokens_seen": 1029439488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474824473420261, + "loss": 1.3197, + "theoretical_loss": 3.6391133371100817, + "tokens_seen": 1029570560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474623871614845, + "loss": 1.4013, + "theoretical_loss": 3.639069488336648, + "tokens_seen": 1029701632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034744232698094286, + "loss": 1.4546, + "theoretical_loss": 3.639025646707035, + "tokens_seen": 1029832704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474222668004012, + "loss": 1.3809, + "theoretical_loss": 3.638981812219171, + "tokens_seen": 1029963776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474022066198596, + "loss": 1.4249, + "theoretical_loss": 3.6389379848709824, + "tokens_seen": 1030094848 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347382146439318, + "loss": 1.3254, + "theoretical_loss": 3.638894164660399, + "tokens_seen": 1030225920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034736208625877637, + "loss": 1.4309, + "theoretical_loss": 3.6388503515853503, + "tokens_seen": 1030356992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034734202607823473, + "loss": 1.262, + "theoretical_loss": 3.6388065456437673, + "tokens_seen": 1030488064 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.49517351388931274, + "objective/train/avg_token_score": 0.0043596127070486546, + "objective/train/docs_used": 590720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7783817052841187, + "objective/train/original_loss": 3.6100010871887207, + "objective/train/theoretical_loss": 3.6387846453473784, + "objective/train/tokens_used": 1030553600, + "objective/train/unlikelihood_loss": 0.16239966452121735, + "theoretical_loss": 3.6387846453473784, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473219658976931, + "loss": 1.4237, + "theoretical_loss": 3.6387627468335806, + "tokens_seen": 1030619136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034730190571715145, + "loss": 1.2545, + "theoretical_loss": 3.6387189551527226, + "tokens_seen": 1030750208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034728184553660987, + "loss": 1.4702, + "theoretical_loss": 3.6386751705991274, + "tokens_seen": 1030881280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034726178535606823, + "loss": 1.2928, + "theoretical_loss": 3.638631393170728, + "tokens_seen": 1031012352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472417251755266, + "loss": 1.3503, + "theoretical_loss": 3.6385876228654603, + "tokens_seen": 1031143424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034722166499498496, + "loss": 1.3008, + "theoretical_loss": 3.6385438596812594, + "tokens_seen": 1031274496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034720160481444337, + "loss": 1.2832, + "theoretical_loss": 3.638500103616062, + "tokens_seen": 1031405568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034718154463390173, + "loss": 1.3789, + "theoretical_loss": 3.6384563546678055, + "tokens_seen": 1031536640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471614844533601, + "loss": 1.5017, + "theoretical_loss": 3.6384126128344283, + "tokens_seen": 1031667712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034714142427281846, + "loss": 1.2902, + "theoretical_loss": 3.6383688781138694, + "tokens_seen": 1031798784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471213640922768, + "loss": 1.404, + "theoretical_loss": 3.6383251505040697, + "tokens_seen": 1031929856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034710130391173524, + "loss": 1.2998, + "theoretical_loss": 3.6382814300029693, + "tokens_seen": 1032060928 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.43114614486694336, + "objective/train/avg_token_score": 0.00578828901052475, + "objective/train/docs_used": 591991, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4023683071136475, + "objective/train/original_loss": 3.749821901321411, + "objective/train/theoretical_loss": 3.6382377166085096, + "objective/train/tokens_used": 1032192000, + "objective/train/unlikelihood_loss": 0.17787593603134155, + "theoretical_loss": 3.6382377166085096, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470812437311936, + "loss": 1.4151, + "theoretical_loss": 3.6382377166085096, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034706118355065196, + "loss": 1.4258, + "theoretical_loss": 3.6381940103186343, + "tokens_seen": 1032323072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470411233701103, + "loss": 1.2602, + "theoretical_loss": 3.6381503111312865, + "tokens_seen": 1032454144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034702106318956874, + "loss": 1.1889, + "theoretical_loss": 3.63810661904441, + "tokens_seen": 1032585216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470010030090271, + "loss": 1.4185, + "theoretical_loss": 3.6380629340559505, + "tokens_seen": 1032716288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034698094282848547, + "loss": 1.4349, + "theoretical_loss": 3.6380192561638545, + "tokens_seen": 1032847360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034696088264794383, + "loss": 1.3969, + "theoretical_loss": 3.6379755853660676, + "tokens_seen": 1032978432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034694082246740224, + "loss": 1.385, + "theoretical_loss": 3.6379319216605386, + "tokens_seen": 1033109504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469207622868606, + "loss": 1.4372, + "theoretical_loss": 3.637888265045216, + "tokens_seen": 1033240576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034690070210631897, + "loss": 1.3413, + "theoretical_loss": 3.6378446155180484, + "tokens_seen": 1033371648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034688064192577733, + "loss": 1.6347, + "theoretical_loss": 3.6378009730769874, + "tokens_seen": 1033502720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468605817452357, + "loss": 1.2665, + "theoretical_loss": 3.6377573377199837, + "tokens_seen": 1033633792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468405215646941, + "loss": 1.2604, + "theoretical_loss": 3.6377137094449887, + "tokens_seen": 1033764864 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.47531768679618835, + "objective/train/avg_token_score": 0.01228614803403616, + "objective/train/docs_used": 592462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.411664366722107, + "objective/train/original_loss": 3.505551338195801, + "objective/train/theoretical_loss": 3.6376918979626054, + "objective/train/tokens_used": 1033830400, + "objective/train/unlikelihood_loss": 0.14863863587379456, + "theoretical_loss": 3.6376918979626054, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034682046138415247, + "loss": 1.4251, + "theoretical_loss": 3.6376700882499566, + "tokens_seen": 1033895936 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034680040120361083, + "loss": 1.5131, + "theoretical_loss": 3.6376264741328397, + "tokens_seen": 1034027008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467803410230692, + "loss": 1.3009, + "theoretical_loss": 3.6375828670915933, + "tokens_seen": 1034158080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467602808425276, + "loss": 1.2162, + "theoretical_loss": 3.637539267124173, + "tokens_seen": 1034289152 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346740220661986, + "loss": 1.3338, + "theoretical_loss": 3.6374956742285343, + "tokens_seen": 1034420224 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034672016048144434, + "loss": 1.3705, + "theoretical_loss": 3.6374520884026347, + "tokens_seen": 1034551296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467001003009027, + "loss": 1.3105, + "theoretical_loss": 3.6374085096444326, + "tokens_seen": 1034682368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034668004012036106, + "loss": 1.3131, + "theoretical_loss": 3.6373649379518866, + "tokens_seen": 1034813440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466599799398195, + "loss": 1.2474, + "theoretical_loss": 3.637321373322956, + "tokens_seen": 1034944512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034663991975927784, + "loss": 1.2206, + "theoretical_loss": 3.637277815755602, + "tokens_seen": 1035075584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466198595787362, + "loss": 1.4881, + "theoretical_loss": 3.6372342652477854, + "tokens_seen": 1035206656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034659979939819456, + "loss": 1.4385, + "theoretical_loss": 3.6371907217974684, + "tokens_seen": 1035337728 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.3526392877101898, + "objective/train/avg_token_score": 0.013029717840254307, + "objective/train/docs_used": 593828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1165224313735962, + "objective/train/original_loss": 3.4852232933044434, + "objective/train/theoretical_loss": 3.6371471854026147, + "objective/train/tokens_used": 1035468800, + "objective/train/unlikelihood_loss": 0.26969805359840393, + "theoretical_loss": 3.6371471854026147, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346579739217653, + "loss": 1.3687, + "theoretical_loss": 3.6371471854026147, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034655967903711134, + "loss": 1.4663, + "theoretical_loss": 3.6371036560611874, + "tokens_seen": 1035599872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465396188565697, + "loss": 1.2308, + "theoretical_loss": 3.6370601337711514, + "tokens_seen": 1035730944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034651955867602807, + "loss": 1.5651, + "theoretical_loss": 3.637016618530473, + "tokens_seen": 1035862016 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034649949849548643, + "loss": 1.2887, + "theoretical_loss": 3.6369731103371183, + "tokens_seen": 1035993088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034647943831494485, + "loss": 1.3538, + "theoretical_loss": 3.6369296091890537, + "tokens_seen": 1036124160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464593781344032, + "loss": 1.4826, + "theoretical_loss": 3.6368861150842484, + "tokens_seen": 1036255232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034643931795386157, + "loss": 1.4139, + "theoretical_loss": 3.636842628020671, + "tokens_seen": 1036386304 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034641925777331993, + "loss": 1.3958, + "theoretical_loss": 3.6367991479962916, + "tokens_seen": 1036517376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034639919759277835, + "loss": 1.3853, + "theoretical_loss": 3.6367556750090797, + "tokens_seen": 1036648448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463791374122367, + "loss": 1.3386, + "theoretical_loss": 3.6367122090570088, + "tokens_seen": 1036779520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463590772316951, + "loss": 1.3995, + "theoretical_loss": 3.6366687501380492, + "tokens_seen": 1036910592 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034633901705115344, + "loss": 1.4215, + "theoretical_loss": 3.6366252982501757, + "tokens_seen": 1037041664 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.4178885519504547, + "objective/train/avg_token_score": 0.024844851344823837, + "objective/train/docs_used": 594513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5084189176559448, + "objective/train/original_loss": 3.815505027770996, + "objective/train/theoretical_loss": 3.6366035749422627, + "objective/train/tokens_used": 1037107200, + "objective/train/unlikelihood_loss": 0.15977101027965546, + "theoretical_loss": 3.6366035749422627, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463189568706118, + "loss": 1.409, + "theoretical_loss": 3.6365818533913608, + "tokens_seen": 1037172736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462988966900702, + "loss": 1.3899, + "theoretical_loss": 3.636538415559581, + "tokens_seen": 1037303808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462788365095286, + "loss": 1.513, + "theoretical_loss": 3.6364949847528107, + "tokens_seen": 1037434880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034625877632898694, + "loss": 1.3865, + "theoretical_loss": 3.636451560969027, + "tokens_seen": 1037565952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462387161484453, + "loss": 1.2239, + "theoretical_loss": 3.6364081442062073, + "tokens_seen": 1037697024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462186559679037, + "loss": 1.5296, + "theoretical_loss": 3.6363647344623296, + "tokens_seen": 1037828096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461985957873621, + "loss": 1.4807, + "theoretical_loss": 3.6363213317353726, + "tokens_seen": 1037959168 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034617853560682044, + "loss": 1.3281, + "theoretical_loss": 3.6362779360233173, + "tokens_seen": 1038090240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461584754262788, + "loss": 1.322, + "theoretical_loss": 3.6362345473241433, + "tokens_seen": 1038221312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034613841524573717, + "loss": 1.3022, + "theoretical_loss": 3.6361911656358323, + "tokens_seen": 1038352384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461183550651956, + "loss": 1.4187, + "theoretical_loss": 3.6361477909563673, + "tokens_seen": 1038483456 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034609829488465395, + "loss": 1.4205, + "theoretical_loss": 3.636104423283731, + "tokens_seen": 1038614528 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.34408602118492126, + "objective/train/avg_token_score": 0.02930273674428463, + "objective/train/docs_used": 595196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0360416173934937, + "objective/train/original_loss": 3.8812637329101562, + "objective/train/theoretical_loss": 3.6360610626159087, + "objective/train/tokens_used": 1038745600, + "objective/train/unlikelihood_loss": 0.22696544229984283, + "theoretical_loss": 3.6360610626159087, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034607823470411236, + "loss": 1.2076, + "theoretical_loss": 3.6360610626159087, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460581745235707, + "loss": 1.3977, + "theoretical_loss": 3.636017708950883, + "tokens_seen": 1038876672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034603811434302914, + "loss": 1.3588, + "theoretical_loss": 3.6359743622866416, + "tokens_seen": 1039007744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460180541624875, + "loss": 1.4335, + "theoretical_loss": 3.63593102262117, + "tokens_seen": 1039138816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034599799398194587, + "loss": 1.1914, + "theoretical_loss": 3.6358876899524564, + "tokens_seen": 1039269888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034597793380140423, + "loss": 1.5832, + "theoretical_loss": 3.6358443642784883, + "tokens_seen": 1039400960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034595787362086264, + "loss": 1.365, + "theoretical_loss": 3.635801045597255, + "tokens_seen": 1039532032 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345937813440321, + "loss": 1.459, + "theoretical_loss": 3.635757733906747, + "tokens_seen": 1039663104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034591775325977937, + "loss": 1.0733, + "theoretical_loss": 3.635714429204954, + "tokens_seen": 1039794176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034589769307923773, + "loss": 1.2872, + "theoretical_loss": 3.6356711314898678, + "tokens_seen": 1039925248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458776328986961, + "loss": 1.3131, + "theoretical_loss": 3.635627840759481, + "tokens_seen": 1040056320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458575727181545, + "loss": 1.2283, + "theoretical_loss": 3.6355845570117875, + "tokens_seen": 1040187392 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034583751253761287, + "loss": 1.3346, + "theoretical_loss": 3.63554128024478, + "tokens_seen": 1040318464 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.5335410833358765, + "objective/train/avg_token_score": 0.015850786119699478, + "objective/train/docs_used": 596146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.858044981956482, + "objective/train/original_loss": 3.782604455947876, + "objective/train/theoretical_loss": 3.635519644478408, + "objective/train/tokens_used": 1040384000, + "objective/train/unlikelihood_loss": 0.17008520662784576, + "theoretical_loss": 3.635519644478408, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034581745235707123, + "loss": 1.41, + "theoretical_loss": 3.6354980104564545, + "tokens_seen": 1040449536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457973921765296, + "loss": 1.4838, + "theoretical_loss": 3.6354547476448067, + "tokens_seen": 1040580608 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345777331995988, + "loss": 1.3367, + "theoretical_loss": 3.6354114918078317, + "tokens_seen": 1040711680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457572718154464, + "loss": 1.2494, + "theoretical_loss": 3.6353682429435286, + "tokens_seen": 1040842752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034573721163490474, + "loss": 1.3609, + "theoretical_loss": 3.6353250010498943, + "tokens_seen": 1040973824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457171514543631, + "loss": 1.3707, + "theoretical_loss": 3.635281766124929, + "tokens_seen": 1041104896 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034569709127382146, + "loss": 1.416, + "theoretical_loss": 3.6352385381666315, + "tokens_seen": 1041235968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456770310932799, + "loss": 1.2837, + "theoretical_loss": 3.6351953171730025, + "tokens_seen": 1041367040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034565697091273824, + "loss": 1.3116, + "theoretical_loss": 3.635152103142044, + "tokens_seen": 1041498112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456369107321966, + "loss": 1.4211, + "theoretical_loss": 3.635108896071759, + "tokens_seen": 1041629184 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034561685055165496, + "loss": 1.2439, + "theoretical_loss": 3.6350656959601486, + "tokens_seen": 1041760256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455967903711134, + "loss": 1.418, + "theoretical_loss": 3.6350225028052185, + "tokens_seen": 1041891328 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.557612419128418, + "objective/train/avg_token_score": 0.03179457411170006, + "objective/train/docs_used": 596810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7983969449996948, + "objective/train/original_loss": 3.637775182723999, + "objective/train/theoretical_loss": 3.634979316604973, + "objective/train/tokens_used": 1042022400, + "objective/train/unlikelihood_loss": 0.1405249685049057, + "theoretical_loss": 3.634979316604973, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034557673019057174, + "loss": 1.3063, + "theoretical_loss": 3.634979316604973, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455566700100301, + "loss": 1.3623, + "theoretical_loss": 3.6349361373574176, + "tokens_seen": 1042153472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034553660982948847, + "loss": 1.3357, + "theoretical_loss": 3.6348929650605584, + "tokens_seen": 1042284544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034551654964894683, + "loss": 1.3648, + "theoretical_loss": 3.634849799712404, + "tokens_seen": 1042415616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034549648946840525, + "loss": 1.299, + "theoretical_loss": 3.63480664131096, + "tokens_seen": 1042546688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454764292878636, + "loss": 1.2149, + "theoretical_loss": 3.6347634898542385, + "tokens_seen": 1042677760 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034545636910732197, + "loss": 1.3777, + "theoretical_loss": 3.6347203453402464, + "tokens_seen": 1042808832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034543630892678033, + "loss": 1.2236, + "theoretical_loss": 3.6346772077669955, + "tokens_seen": 1042939904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034541624874623875, + "loss": 1.3542, + "theoretical_loss": 3.6346340771324974, + "tokens_seen": 1043070976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453961885656971, + "loss": 1.4216, + "theoretical_loss": 3.634590953434764, + "tokens_seen": 1043202048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453761283851555, + "loss": 1.3432, + "theoretical_loss": 3.634547836671808, + "tokens_seen": 1043333120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034535606820461384, + "loss": 1.2515, + "theoretical_loss": 3.6345047268416435, + "tokens_seen": 1043464192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453360080240722, + "loss": 1.2274, + "theoretical_loss": 3.6344616239422844, + "tokens_seen": 1043595264 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.5276148319244385, + "objective/train/avg_token_score": 0.03986110910773277, + "objective/train/docs_used": 598197, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5360949039459229, + "objective/train/original_loss": 3.704511880874634, + "objective/train/theoretical_loss": 3.634440075091037, + "objective/train/tokens_used": 1043660800, + "objective/train/unlikelihood_loss": 0.15461306273937225, + "theoretical_loss": 3.634440075091037, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453159478435306, + "loss": 1.3997, + "theoretical_loss": 3.6344185279717474, + "tokens_seen": 1043726336 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345295887662989, + "loss": 1.4418, + "theoretical_loss": 3.634375438928048, + "tokens_seen": 1043857408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034527582748244734, + "loss": 1.299, + "theoretical_loss": 3.634332356809204, + "tokens_seen": 1043988480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452557673019057, + "loss": 1.4294, + "theoretical_loss": 3.634289281613232, + "tokens_seen": 1044119552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452357071213641, + "loss": 1.3623, + "theoretical_loss": 3.634246213338152, + "tokens_seen": 1044250624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452156469408225, + "loss": 1.4439, + "theoretical_loss": 3.634203151981983, + "tokens_seen": 1044381696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034519558676028084, + "loss": 1.2897, + "theoretical_loss": 3.634160097542745, + "tokens_seen": 1044512768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451755265797392, + "loss": 1.1791, + "theoretical_loss": 3.6341170500184603, + "tokens_seen": 1044643840 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034515546639919757, + "loss": 1.6201, + "theoretical_loss": 3.6340740094071493, + "tokens_seen": 1044774912 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345135406218656, + "loss": 1.4161, + "theoretical_loss": 3.6340309757068354, + "tokens_seen": 1044905984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034511534603811435, + "loss": 1.3379, + "theoretical_loss": 3.6339879489155433, + "tokens_seen": 1045037056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450952858575727, + "loss": 1.3494, + "theoretical_loss": 3.633944929031296, + "tokens_seen": 1045168128 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.3000977635383606, + "objective/train/avg_token_score": 0.016476191580295563, + "objective/train/docs_used": 598880, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0701817274093628, + "objective/train/original_loss": 3.506985664367676, + "objective/train/theoretical_loss": 3.6339019160521198, + "objective/train/tokens_used": 1045299200, + "objective/train/unlikelihood_loss": 0.21717678010463715, + "theoretical_loss": 3.6339019160521198, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034507522567703107, + "loss": 1.3165, + "theoretical_loss": 3.6339019160521198, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450551654964895, + "loss": 1.3046, + "theoretical_loss": 3.6338589099760394, + "tokens_seen": 1045430272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034503510531594785, + "loss": 1.4273, + "theoretical_loss": 3.6338159108010832, + "tokens_seen": 1045561344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450150451354062, + "loss": 1.3468, + "theoretical_loss": 3.6337729185252776, + "tokens_seen": 1045692416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449949849548646, + "loss": 1.445, + "theoretical_loss": 3.633729933146652, + "tokens_seen": 1045823488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034497492477432294, + "loss": 1.3321, + "theoretical_loss": 3.633686954663235, + "tokens_seen": 1045954560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034495486459378135, + "loss": 1.4158, + "theoretical_loss": 3.6336439830730574, + "tokens_seen": 1046085632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449348044132397, + "loss": 1.278, + "theoretical_loss": 3.6336010183741494, + "tokens_seen": 1046216704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449147442326981, + "loss": 1.5745, + "theoretical_loss": 3.6335580605645434, + "tokens_seen": 1046347776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034489468405215644, + "loss": 1.34, + "theoretical_loss": 3.6335151096422713, + "tokens_seen": 1046478848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034487462387161486, + "loss": 1.2457, + "theoretical_loss": 3.633472165605366, + "tokens_seen": 1046609920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448545636910732, + "loss": 1.1494, + "theoretical_loss": 3.6334292284518632, + "tokens_seen": 1046740992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448345035105316, + "loss": 1.3533, + "theoretical_loss": 3.633386298179797, + "tokens_seen": 1046872064 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.4690249264240265, + "objective/train/avg_token_score": 0.010628259740769863, + "objective/train/docs_used": 599993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4770283699035645, + "objective/train/original_loss": 3.289506196975708, + "objective/train/theoretical_loss": 3.6333648356236883, + "objective/train/tokens_used": 1046937600, + "objective/train/unlikelihood_loss": 0.24652303755283356, + "theoretical_loss": 3.6333648356236883, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034481444332998994, + "loss": 1.4529, + "theoretical_loss": 3.6333433747872026, + "tokens_seen": 1047003136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034479438314944836, + "loss": 1.5562, + "theoretical_loss": 3.6333004582721173, + "tokens_seen": 1047134208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447743229689067, + "loss": 1.4002, + "theoretical_loss": 3.6332575486325784, + "tokens_seen": 1047265280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447542627883651, + "loss": 1.3682, + "theoretical_loss": 3.633214645866624, + "tokens_seen": 1047396352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034473420260782345, + "loss": 1.4218, + "theoretical_loss": 3.6331717499722926, + "tokens_seen": 1047527424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447141424272818, + "loss": 1.4121, + "theoretical_loss": 3.6331288609476244, + "tokens_seen": 1047658496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446940822467402, + "loss": 1.5015, + "theoretical_loss": 3.6330859787906604, + "tokens_seen": 1047789568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446740220661986, + "loss": 1.4527, + "theoretical_loss": 3.6330431034994417, + "tokens_seen": 1047920640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034465396188565695, + "loss": 1.3086, + "theoretical_loss": 3.63300023507201, + "tokens_seen": 1048051712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446339017051153, + "loss": 1.2406, + "theoretical_loss": 3.6329573735064087, + "tokens_seen": 1048182784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034461384152457373, + "loss": 1.5516, + "theoretical_loss": 3.6329145188006815, + "tokens_seen": 1048313856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034459378134403214, + "loss": 1.2353, + "theoretical_loss": 3.6328716709528734, + "tokens_seen": 1048444928 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.36992913484573364, + "objective/train/avg_token_score": 0.00892473105341196, + "objective/train/docs_used": 601088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1939748525619507, + "objective/train/original_loss": 4.2273359298706055, + "objective/train/theoretical_loss": 3.632828829961029, + "objective/train/tokens_used": 1048576000, + "objective/train/unlikelihood_loss": 0.13863138854503632, + "theoretical_loss": 3.632828829961029, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445737211634905, + "loss": 1.2199, + "theoretical_loss": 3.632828829961029, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034455366098294887, + "loss": 1.3458, + "theoretical_loss": 3.6327859958231956, + "tokens_seen": 1048707072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034453360080240723, + "loss": 1.4605, + "theoretical_loss": 3.632743168537419, + "tokens_seen": 1048838144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034451354062186565, + "loss": 1.2929, + "theoretical_loss": 3.632700348101748, + "tokens_seen": 1048969216 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344493480441324, + "loss": 1.2953, + "theoretical_loss": 3.6326575345142307, + "tokens_seen": 1049100288 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034447342026078237, + "loss": 1.362, + "theoretical_loss": 3.632614727772917, + "tokens_seen": 1049231360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034445336008024073, + "loss": 1.3092, + "theoretical_loss": 3.6325719278758557, + "tokens_seen": 1049362432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034443329989969915, + "loss": 1.3296, + "theoretical_loss": 3.6325291348211, + "tokens_seen": 1049493504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444132397191575, + "loss": 1.454, + "theoretical_loss": 3.6324863486067, + "tokens_seen": 1049624576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443931795386159, + "loss": 1.5365, + "theoretical_loss": 3.6324435692307087, + "tokens_seen": 1049755648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034437311935807424, + "loss": 1.1725, + "theoretical_loss": 3.6324007966911798, + "tokens_seen": 1049886720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443530591775326, + "loss": 1.3764, + "theoretical_loss": 3.6323580309861674, + "tokens_seen": 1050017792 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344332998996991, + "loss": 1.5223, + "theoretical_loss": 3.6323152721137264, + "tokens_seen": 1050148864 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.46041056513786316, + "objective/train/avg_token_score": 0.013329225592315197, + "objective/train/docs_used": 601638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.508044958114624, + "objective/train/original_loss": 3.5703628063201904, + "objective/train/theoretical_loss": 3.6322938952391124, + "objective/train/tokens_used": 1050214400, + "objective/train/unlikelihood_loss": 0.19386138021945953, + "theoretical_loss": 3.6322938952391124, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443129388164494, + "loss": 1.2695, + "theoretical_loss": 3.632272520071912, + "tokens_seen": 1050279936 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034429287863590774, + "loss": 1.4187, + "theoretical_loss": 3.632229774858782, + "tokens_seen": 1050411008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442728184553661, + "loss": 1.3806, + "theoretical_loss": 3.632187036472393, + "tokens_seen": 1050542080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442527582748245, + "loss": 1.3556, + "theoretical_loss": 3.632144304910804, + "tokens_seen": 1050673152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442326980942829, + "loss": 1.3397, + "theoretical_loss": 3.6321015801720726, + "tokens_seen": 1050804224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034421263791374124, + "loss": 1.3776, + "theoretical_loss": 3.63205886225426, + "tokens_seen": 1050935296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441925777331996, + "loss": 1.4162, + "theoretical_loss": 3.632016151155425, + "tokens_seen": 1051066368 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034417251755265797, + "loss": 1.4316, + "theoretical_loss": 3.631973446873631, + "tokens_seen": 1051197440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441524573721164, + "loss": 1.4186, + "theoretical_loss": 3.6319307494069384, + "tokens_seen": 1051328512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034413239719157475, + "loss": 1.4346, + "theoretical_loss": 3.631888058753411, + "tokens_seen": 1051459584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441123370110331, + "loss": 1.4528, + "theoretical_loss": 3.631845374911113, + "tokens_seen": 1051590656 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034409227683049147, + "loss": 1.4901, + "theoretical_loss": 3.6318026978781077, + "tokens_seen": 1051721728 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.42595309019088745, + "objective/train/avg_token_score": 0.02684997208416462, + "objective/train/docs_used": 602388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2941303253173828, + "objective/train/original_loss": 3.713991165161133, + "objective/train/theoretical_loss": 3.631760027652461, + "objective/train/tokens_used": 1051852800, + "objective/train/unlikelihood_loss": 0.14472883939743042, + "theoretical_loss": 3.631760027652461, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440722166499499, + "loss": 1.3932, + "theoretical_loss": 3.631760027652461, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034405215646940825, + "loss": 1.4742, + "theoretical_loss": 3.631717364232239, + "tokens_seen": 1051983872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440320962888666, + "loss": 1.2689, + "theoretical_loss": 3.631674707615509, + "tokens_seen": 1052114944 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344012036108325, + "loss": 1.5226, + "theoretical_loss": 3.6316320578003385, + "tokens_seen": 1052246016 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034399197592778334, + "loss": 1.4915, + "theoretical_loss": 3.631589414784795, + "tokens_seen": 1052377088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034397191574724175, + "loss": 1.2615, + "theoretical_loss": 3.631546778566949, + "tokens_seen": 1052508160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439518555667001, + "loss": 1.3336, + "theoretical_loss": 3.6315041491448703, + "tokens_seen": 1052639232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439317953861585, + "loss": 1.3339, + "theoretical_loss": 3.6314615265166292, + "tokens_seen": 1052770304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034391173520561684, + "loss": 1.3529, + "theoretical_loss": 3.6314189106802974, + "tokens_seen": 1052901376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034389167502507526, + "loss": 1.2882, + "theoretical_loss": 3.6313763016339484, + "tokens_seen": 1053032448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438716148445336, + "loss": 1.4215, + "theoretical_loss": 3.631333699375654, + "tokens_seen": 1053163520 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343851554663992, + "loss": 1.5308, + "theoretical_loss": 3.6312911039034885, + "tokens_seen": 1053294592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034383149448345034, + "loss": 1.5041, + "theoretical_loss": 3.6312485152155274, + "tokens_seen": 1053425664 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.5117301940917969, + "objective/train/avg_token_score": 0.005069754086434841, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8015470504760742, + "objective/train/original_loss": 3.598889112472534, + "objective/train/theoretical_loss": 3.6312272234150216, + "objective/train/tokens_used": 1053491200, + "objective/train/unlikelihood_loss": 0.1789616495370865, + "theoretical_loss": 3.6312272234150216, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034381143430290876, + "loss": 1.3975, + "theoretical_loss": 3.6312059333098454, + "tokens_seen": 1053556736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437913741223671, + "loss": 1.371, + "theoretical_loss": 3.6311633581845193, + "tokens_seen": 1053687808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437713139418255, + "loss": 1.3972, + "theoretical_loss": 3.631120789837626, + "tokens_seen": 1053818880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034375125376128385, + "loss": 1.3306, + "theoretical_loss": 3.631078228267244, + "tokens_seen": 1053949952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437311935807422, + "loss": 1.4166, + "theoretical_loss": 3.631035673471451, + "tokens_seen": 1054081024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437111334002006, + "loss": 1.2348, + "theoretical_loss": 3.6309931254483274, + "tokens_seen": 1054212096 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343691073219659, + "loss": 1.3322, + "theoretical_loss": 3.630950584195953, + "tokens_seen": 1054343168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034367101303911735, + "loss": 1.2542, + "theoretical_loss": 3.6309080497124087, + "tokens_seen": 1054474240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436509528585757, + "loss": 1.166, + "theoretical_loss": 3.6308655219957764, + "tokens_seen": 1054605312 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034363089267803413, + "loss": 1.4829, + "theoretical_loss": 3.6308230010441394, + "tokens_seen": 1054736384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436108324974925, + "loss": 1.3987, + "theoretical_loss": 3.6307804868555804, + "tokens_seen": 1054867456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034359077231695085, + "loss": 1.2896, + "theoretical_loss": 3.6307379794281838, + "tokens_seen": 1054998528 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.4736681282520294, + "objective/train/avg_token_score": 0.020433956757187843, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.514562726020813, + "objective/train/original_loss": 3.539909601211548, + "objective/train/theoretical_loss": 3.630695478760034, + "objective/train/tokens_used": 1055129600, + "objective/train/unlikelihood_loss": 0.17292724549770355, + "theoretical_loss": 3.630695478760034, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435707121364092, + "loss": 1.3483, + "theoretical_loss": 3.630695478760034, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435506519558676, + "loss": 1.4239, + "theoretical_loss": 3.6306529848492177, + "tokens_seen": 1055260672 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343530591775326, + "loss": 1.3666, + "theoretical_loss": 3.6306104976938203, + "tokens_seen": 1055391744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034351053159478436, + "loss": 1.4003, + "theoretical_loss": 3.6305680172919303, + "tokens_seen": 1055522816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434904714142427, + "loss": 1.297, + "theoretical_loss": 3.6305255436416353, + "tokens_seen": 1055653888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434704112337011, + "loss": 1.4212, + "theoretical_loss": 3.630483076741023, + "tokens_seen": 1055784960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434503510531595, + "loss": 1.3941, + "theoretical_loss": 3.630440616588185, + "tokens_seen": 1055916032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034343029087261786, + "loss": 1.2957, + "theoretical_loss": 3.6303981631812103, + "tokens_seen": 1056047104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434102306920762, + "loss": 1.3436, + "theoretical_loss": 3.6303557165181912, + "tokens_seen": 1056178176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433901705115346, + "loss": 1.2, + "theoretical_loss": 3.630313276597218, + "tokens_seen": 1056309248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034337011033099295, + "loss": 1.2442, + "theoretical_loss": 3.630270843416385, + "tokens_seen": 1056440320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034335005015045136, + "loss": 1.2531, + "theoretical_loss": 3.6302284169737855, + "tokens_seen": 1056571392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433299899699097, + "loss": 1.3774, + "theoretical_loss": 3.630185997267513, + "tokens_seen": 1056702464 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.36919599771499634, + "objective/train/avg_token_score": 0.010494265705347061, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3709325790405273, + "objective/train/original_loss": 4.190634727478027, + "objective/train/theoretical_loss": 3.6301647899399043, + "objective/train/tokens_used": 1056768000, + "objective/train/unlikelihood_loss": 0.1270867884159088, + "theoretical_loss": 3.6301647899399043, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433099297893681, + "loss": 1.2524, + "theoretical_loss": 3.630143584295663, + "tokens_seen": 1056833536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034328986960882645, + "loss": 1.298, + "theoretical_loss": 3.6301011780563313, + "tokens_seen": 1056964608 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034326980942828487, + "loss": 1.2461, + "theoretical_loss": 3.630058778547615, + "tokens_seen": 1057095680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034324974924774323, + "loss": 1.3012, + "theoretical_loss": 3.63001638576761, + "tokens_seen": 1057226752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432296890672016, + "loss": 1.3052, + "theoretical_loss": 3.6299739997144163, + "tokens_seen": 1057357824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034320962888665995, + "loss": 1.2708, + "theoretical_loss": 3.629931620386132, + "tokens_seen": 1057488896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431895687061183, + "loss": 1.2604, + "theoretical_loss": 3.6298892477808566, + "tokens_seen": 1057619968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034316950852557673, + "loss": 1.2323, + "theoretical_loss": 3.6298468818966905, + "tokens_seen": 1057751040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431494483450351, + "loss": 1.3143, + "theoretical_loss": 3.6298045227317357, + "tokens_seen": 1057882112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034312938816449345, + "loss": 1.1681, + "theoretical_loss": 3.629762170284094, + "tokens_seen": 1058013184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431093279839518, + "loss": 1.2414, + "theoretical_loss": 3.6297198245518674, + "tokens_seen": 1058144256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430892678034103, + "loss": 1.3928, + "theoretical_loss": 3.6296774855331604, + "tokens_seen": 1058275328 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.4632820188999176, + "objective/train/avg_token_score": 0.017699383199214935, + "objective/train/docs_used": 604503, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4413363933563232, + "objective/train/original_loss": 4.030559539794922, + "objective/train/theoretical_loss": 3.6296351532260767, + "objective/train/tokens_used": 1058406400, + "objective/train/unlikelihood_loss": 0.12413002550601959, + "theoretical_loss": 3.6296351532260767, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034306920762286865, + "loss": 1.3385, + "theoretical_loss": 3.6296351532260767, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343049147442327, + "loss": 1.2994, + "theoretical_loss": 3.6295928276287217, + "tokens_seen": 1058537472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430290872617854, + "loss": 1.2567, + "theoretical_loss": 3.629550508739201, + "tokens_seen": 1058668544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034300902708124374, + "loss": 1.3677, + "theoretical_loss": 3.629508196555622, + "tokens_seen": 1058799616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034298896690070215, + "loss": 1.2551, + "theoretical_loss": 3.6294658910760913, + "tokens_seen": 1058930688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429689067201605, + "loss": 1.0953, + "theoretical_loss": 3.6294235922987177, + "tokens_seen": 1059061760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429488465396189, + "loss": 1.3689, + "theoretical_loss": 3.6293813002216093, + "tokens_seen": 1059192832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034292878635907724, + "loss": 1.143, + "theoretical_loss": 3.629339014842876, + "tokens_seen": 1059323904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034290872617853566, + "loss": 1.3785, + "theoretical_loss": 3.6292967361606294, + "tokens_seen": 1059454976 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342888665997994, + "loss": 1.506, + "theoretical_loss": 3.62925446417298, + "tokens_seen": 1059586048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428686058174524, + "loss": 1.2672, + "theoretical_loss": 3.6292121988780393, + "tokens_seen": 1059717120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034284854563691074, + "loss": 1.2053, + "theoretical_loss": 3.629169940273921, + "tokens_seen": 1059848192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428284854563691, + "loss": 1.261, + "theoretical_loss": 3.629127688358738, + "tokens_seen": 1059979264 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.371456503868103, + "objective/train/avg_token_score": 0.018903860822319984, + "objective/train/docs_used": 605021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1593444347381592, + "objective/train/original_loss": 3.860377788543701, + "objective/train/theoretical_loss": 3.6291065649089083, + "objective/train/tokens_used": 1060044800, + "objective/train/unlikelihood_loss": 0.1958257555961609, + "theoretical_loss": 3.6291065649089083, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428084252758275, + "loss": 1.3924, + "theoretical_loss": 3.629085443130605, + "tokens_seen": 1060110336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427883650952859, + "loss": 1.3287, + "theoretical_loss": 3.6290432045876364, + "tokens_seen": 1060241408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034276830491474425, + "loss": 1.3278, + "theoretical_loss": 3.629000972727949, + "tokens_seen": 1060372480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427482447342026, + "loss": 1.3295, + "theoretical_loss": 3.6289587475496585, + "tokens_seen": 1060503552 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342728184553661, + "loss": 1.2089, + "theoretical_loss": 3.628916529050883, + "tokens_seen": 1060634624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427081243731194, + "loss": 1.3321, + "theoretical_loss": 3.6288743172297404, + "tokens_seen": 1060765696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034268806419257775, + "loss": 1.2827, + "theoretical_loss": 3.6288321120843494, + "tokens_seen": 1060896768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426680040120361, + "loss": 1.2644, + "theoretical_loss": 3.6287899136128297, + "tokens_seen": 1061027840 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034264794383149453, + "loss": 1.2574, + "theoretical_loss": 3.6287477218133017, + "tokens_seen": 1061158912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426278836509529, + "loss": 1.431, + "theoretical_loss": 3.6287055366838867, + "tokens_seen": 1061289984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034260782347041125, + "loss": 1.3437, + "theoretical_loss": 3.628663358222707, + "tokens_seen": 1061421056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425877632898696, + "loss": 1.351, + "theoretical_loss": 3.6286211864278846, + "tokens_seen": 1061552128 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.39619991183280945, + "objective/train/avg_token_score": 0.012817149981856346, + "objective/train/docs_used": 606310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4630290269851685, + "objective/train/original_loss": 4.182855129241943, + "objective/train/theoretical_loss": 3.6285790212975435, + "objective/train/tokens_used": 1061683200, + "objective/train/unlikelihood_loss": 0.12338002026081085, + "theoretical_loss": 3.6285790212975435, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342567703109328, + "loss": 1.3689, + "theoretical_loss": 3.6285790212975435, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425476429287864, + "loss": 1.2915, + "theoretical_loss": 3.628536862829807, + "tokens_seen": 1061814272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034252758274824476, + "loss": 1.2887, + "theoretical_loss": 3.6284947110228014, + "tokens_seen": 1061945344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425075225677031, + "loss": 1.3961, + "theoretical_loss": 3.6284525658746523, + "tokens_seen": 1062076416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424874623871615, + "loss": 1.2877, + "theoretical_loss": 3.6284104273834847, + "tokens_seen": 1062207488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424674022066199, + "loss": 1.3243, + "theoretical_loss": 3.6283682955474275, + "tokens_seen": 1062338560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034244734202607826, + "loss": 1.4166, + "theoretical_loss": 3.628326170364608, + "tokens_seen": 1062469632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424272818455366, + "loss": 1.2609, + "theoretical_loss": 3.6282840518331545, + "tokens_seen": 1062600704 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342407221664995, + "loss": 1.324, + "theoretical_loss": 3.628241939951198, + "tokens_seen": 1062731776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034238716148445335, + "loss": 1.3242, + "theoretical_loss": 3.628199834716867, + "tokens_seen": 1062862848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034236710130391176, + "loss": 1.3649, + "theoretical_loss": 3.628157736128294, + "tokens_seen": 1062993920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423470411233701, + "loss": 1.4604, + "theoretical_loss": 3.6281156441836098, + "tokens_seen": 1063124992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423269809428285, + "loss": 1.1677, + "theoretical_loss": 3.6280735588809474, + "tokens_seen": 1063256064 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.4382331371307373, + "objective/train/avg_token_score": 0.004980679135769606, + "objective/train/docs_used": 606853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.544931411743164, + "objective/train/original_loss": 3.9982669353485107, + "objective/train/theoretical_loss": 3.628052518719791, + "objective/train/tokens_used": 1063321600, + "objective/train/unlikelihood_loss": 0.134706512093544, + "theoretical_loss": 3.628052518719791, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034230692076228685, + "loss": 1.4286, + "theoretical_loss": 3.62803148021844, + "tokens_seen": 1063387136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034228686058174527, + "loss": 1.2864, + "theoretical_loss": 3.6279894081942223, + "tokens_seen": 1063518208 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034226680040120363, + "loss": 1.4649, + "theoretical_loss": 3.627947342806428, + "tokens_seen": 1063649280 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342246740220662, + "loss": 1.1347, + "theoretical_loss": 3.6279052840531936, + "tokens_seen": 1063780352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034222668004012035, + "loss": 1.3128, + "theoretical_loss": 3.6278632319326545, + "tokens_seen": 1063911424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422066198595787, + "loss": 1.313, + "theoretical_loss": 3.627821186442948, + "tokens_seen": 1064042496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034218655967903713, + "loss": 1.1639, + "theoretical_loss": 3.6277791475822134, + "tokens_seen": 1064173568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421664994984955, + "loss": 1.3709, + "theoretical_loss": 3.6277371153485873, + "tokens_seen": 1064304640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034214643931795386, + "loss": 1.4656, + "theoretical_loss": 3.62769508974021, + "tokens_seen": 1064435712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421263791374122, + "loss": 1.2404, + "theoretical_loss": 3.6276530707552217, + "tokens_seen": 1064566784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034210631895687063, + "loss": 1.3473, + "theoretical_loss": 3.6276110583917625, + "tokens_seen": 1064697856 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342086258776329, + "loss": 1.2891, + "theoretical_loss": 3.6275690526479747, + "tokens_seen": 1064828928 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.4305351972579956, + "objective/train/avg_token_score": 0.012417568825185299, + "objective/train/docs_used": 607789, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4594976902008057, + "objective/train/original_loss": 3.6953115463256836, + "objective/train/theoretical_loss": 3.6275270535220008, + "objective/train/tokens_used": 1064960000, + "objective/train/unlikelihood_loss": 0.20986884832382202, + "theoretical_loss": 3.6275270535220008, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034206619859578736, + "loss": 1.3137, + "theoretical_loss": 3.6275270535220008, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420461384152457, + "loss": 1.3313, + "theoretical_loss": 3.627485061011983, + "tokens_seen": 1065091072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420260782347041, + "loss": 1.4104, + "theoretical_loss": 3.6274430751160653, + "tokens_seen": 1065222144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420060180541625, + "loss": 1.3456, + "theoretical_loss": 3.627401095832393, + "tokens_seen": 1065353216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034198595787362086, + "loss": 1.2389, + "theoretical_loss": 3.6273591231591107, + "tokens_seen": 1065484288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419658976930792, + "loss": 1.3682, + "theoretical_loss": 3.627317157094365, + "tokens_seen": 1065615360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419458375125376, + "loss": 1.3818, + "theoretical_loss": 3.627275197636303, + "tokens_seen": 1065746432 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341925777331996, + "loss": 1.3116, + "theoretical_loss": 3.6272332447830715, + "tokens_seen": 1065877504 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034190571715145436, + "loss": 1.4793, + "theoretical_loss": 3.6271912985328187, + "tokens_seen": 1066008576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418856569709127, + "loss": 1.3057, + "theoretical_loss": 3.6271493588836945, + "tokens_seen": 1066139648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418655967903711, + "loss": 1.4223, + "theoretical_loss": 3.6271074258338487, + "tokens_seen": 1066270720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034184553660982945, + "loss": 1.3274, + "theoretical_loss": 3.627065499381431, + "tokens_seen": 1066401792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034182547642928787, + "loss": 1.3985, + "theoretical_loss": 3.627023579524593, + "tokens_seen": 1066532864 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.49046921730041504, + "objective/train/avg_token_score": 0.013046039268374443, + "objective/train/docs_used": 608475, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6403714418411255, + "objective/train/original_loss": 3.616307020187378, + "objective/train/theoretical_loss": 3.627002622068939, + "objective/train/tokens_used": 1066598400, + "objective/train/unlikelihood_loss": 0.15455709397792816, + "theoretical_loss": 3.627002622068939, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034180541624874623, + "loss": 1.4079, + "theoretical_loss": 3.6269816662614875, + "tokens_seen": 1066663936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417853560682046, + "loss": 1.2935, + "theoretical_loss": 3.6269397595902664, + "tokens_seen": 1066795008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034176529588766295, + "loss": 1.3336, + "theoretical_loss": 3.6268978595090835, + "tokens_seen": 1066926080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034174523570712137, + "loss": 1.5132, + "theoretical_loss": 3.6268559660160933, + "tokens_seen": 1067057152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034172517552657973, + "loss": 1.3659, + "theoretical_loss": 3.626814079109451, + "tokens_seen": 1067188224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417051153460381, + "loss": 1.3145, + "theoretical_loss": 3.626772198787312, + "tokens_seen": 1067319296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034168505516549646, + "loss": 1.3636, + "theoretical_loss": 3.6267303250478324, + "tokens_seen": 1067450368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416649949849548, + "loss": 1.2877, + "theoretical_loss": 3.6266884578891703, + "tokens_seen": 1067581440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034164493480441324, + "loss": 1.3045, + "theoretical_loss": 3.6266465973094832, + "tokens_seen": 1067712512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416248746238716, + "loss": 1.1361, + "theoretical_loss": 3.62660474330693, + "tokens_seen": 1067843584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034160481444332996, + "loss": 1.4392, + "theoretical_loss": 3.6265628958796707, + "tokens_seen": 1067974656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415847542627884, + "loss": 1.2803, + "theoretical_loss": 3.626521055025864, + "tokens_seen": 1068105728 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.3489736020565033, + "objective/train/avg_token_score": 0.03230096772313118, + "objective/train/docs_used": 609657, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1583484411239624, + "objective/train/original_loss": 4.849977970123291, + "objective/train/theoretical_loss": 3.626479220743673, + "objective/train/tokens_used": 1068236800, + "objective/train/unlikelihood_loss": 0.07369246333837509, + "theoretical_loss": 3.626479220743673, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415646940822468, + "loss": 1.4129, + "theoretical_loss": 3.626479220743673, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034154463390170516, + "loss": 1.2253, + "theoretical_loss": 3.6264373930312575, + "tokens_seen": 1068367872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415245737211635, + "loss": 1.3984, + "theoretical_loss": 3.626395571886781, + "tokens_seen": 1068498944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415045135406219, + "loss": 1.4128, + "theoretical_loss": 3.6263537573084066, + "tokens_seen": 1068630016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414844533600803, + "loss": 1.2828, + "theoretical_loss": 3.6263119492942977, + "tokens_seen": 1068761088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034146439317953866, + "loss": 1.4296, + "theoretical_loss": 3.62627014784262, + "tokens_seen": 1068892160 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341444332998997, + "loss": 1.3545, + "theoretical_loss": 3.6262283529515376, + "tokens_seen": 1069023232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414242728184554, + "loss": 1.4797, + "theoretical_loss": 3.626186564619217, + "tokens_seen": 1069154304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034140421263791375, + "loss": 1.3997, + "theoretical_loss": 3.6261447828438254, + "tokens_seen": 1069285376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034138415245737216, + "loss": 1.3062, + "theoretical_loss": 3.6261030076235303, + "tokens_seen": 1069416448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413640922768305, + "loss": 1.3766, + "theoretical_loss": 3.6260612389565, + "tokens_seen": 1069547520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413440320962889, + "loss": 1.3502, + "theoretical_loss": 3.626019476840904, + "tokens_seen": 1069678592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034132397191574725, + "loss": 1.3616, + "theoretical_loss": 3.6259777212749116, + "tokens_seen": 1069809664 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.3779325485229492, + "objective/train/avg_token_score": 0.017528358846902847, + "objective/train/docs_used": 610408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3788695335388184, + "objective/train/original_loss": 4.042832851409912, + "objective/train/theoretical_loss": 3.625956845947445, + "objective/train/tokens_used": 1069875200, + "objective/train/unlikelihood_loss": 0.1505182832479477, + "theoretical_loss": 3.625956845947445, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034130391173520567, + "loss": 1.434, + "theoretical_loss": 3.625935972256694, + "tokens_seen": 1069940736 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034128385155466403, + "loss": 1.2665, + "theoretical_loss": 3.6258942297844214, + "tokens_seen": 1070071808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412637913741224, + "loss": 1.3202, + "theoretical_loss": 3.6258524938562666, + "tokens_seen": 1070202880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034124373119358075, + "loss": 1.4834, + "theoretical_loss": 3.625810764470402, + "tokens_seen": 1070333952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412236710130391, + "loss": 1.1739, + "theoretical_loss": 3.6257690416250012, + "tokens_seen": 1070465024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034120361083249753, + "loss": 1.2467, + "theoretical_loss": 3.6257273253182394, + "tokens_seen": 1070596096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411835506519559, + "loss": 1.2685, + "theoretical_loss": 3.62568561554829, + "tokens_seen": 1070727168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034116349047141426, + "loss": 1.3961, + "theoretical_loss": 3.625643912313329, + "tokens_seen": 1070858240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411434302908726, + "loss": 1.2636, + "theoretical_loss": 3.6256022156115337, + "tokens_seen": 1070989312 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034112337011033103, + "loss": 1.3885, + "theoretical_loss": 3.625560525441081, + "tokens_seen": 1071120384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411033099297894, + "loss": 1.2601, + "theoretical_loss": 3.6255188418001483, + "tokens_seen": 1071251456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034108324974924776, + "loss": 1.4105, + "theoretical_loss": 3.625477164686914, + "tokens_seen": 1071382528 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.4709799587726593, + "objective/train/avg_token_score": 0.010310190729796886, + "objective/train/docs_used": 611059, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4338877201080322, + "objective/train/original_loss": 3.4106380939483643, + "objective/train/theoretical_loss": 3.6254354940995586, + "objective/train/tokens_used": 1071513600, + "objective/train/unlikelihood_loss": 0.21617424488067627, + "theoretical_loss": 3.6254354940995586, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410631895687061, + "loss": 1.3881, + "theoretical_loss": 3.6254354940995586, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410431293881645, + "loss": 1.2884, + "theoretical_loss": 3.625393830036261, + "tokens_seen": 1071644672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410230692076229, + "loss": 1.2777, + "theoretical_loss": 3.6253521724952025, + "tokens_seen": 1071775744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034100300902708126, + "loss": 1.4866, + "theoretical_loss": 3.6253105214745647, + "tokens_seen": 1071906816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409829488465396, + "loss": 1.4, + "theoretical_loss": 3.6252688769725303, + "tokens_seen": 1072037888 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340962888665998, + "loss": 1.2163, + "theoretical_loss": 3.6252272389872813, + "tokens_seen": 1072168960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409428284854564, + "loss": 1.3219, + "theoretical_loss": 3.625185607517002, + "tokens_seen": 1072300032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034092276830491477, + "loss": 1.2709, + "theoretical_loss": 3.625143982559877, + "tokens_seen": 1072431104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034090270812437313, + "loss": 1.2956, + "theoretical_loss": 3.625102364114091, + "tokens_seen": 1072562176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408826479438315, + "loss": 1.2335, + "theoretical_loss": 3.6250607521778306, + "tokens_seen": 1072693248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034086258776328985, + "loss": 1.4779, + "theoretical_loss": 3.6250191467492816, + "tokens_seen": 1072824320 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034084252758274827, + "loss": 1.3085, + "theoretical_loss": 3.624977547826632, + "tokens_seen": 1072955392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034082246740220663, + "loss": 1.3841, + "theoretical_loss": 3.62493595540807, + "tokens_seen": 1073086464 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.4900415539741516, + "objective/train/avg_token_score": 0.012406887486577034, + "objective/train/docs_used": 612309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7033065557479858, + "objective/train/original_loss": 3.674712657928467, + "objective/train/theoretical_loss": 3.6249151616372552, + "objective/train/tokens_used": 1073152000, + "objective/train/unlikelihood_loss": 0.21300873160362244, + "theoretical_loss": 3.6249151616372552, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340802407221665, + "loss": 1.4206, + "theoretical_loss": 3.6248943694917832, + "tokens_seen": 1073217536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034078234704112335, + "loss": 1.3298, + "theoretical_loss": 3.6248527900759626, + "tokens_seen": 1073348608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034076228686058177, + "loss": 1.2991, + "theoretical_loss": 3.624811217158798, + "tokens_seen": 1073479680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034074222668004013, + "loss": 1.2799, + "theoretical_loss": 3.6247696507384806, + "tokens_seen": 1073610752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407221664994985, + "loss": 1.3688, + "theoretical_loss": 3.6247280908132007, + "tokens_seen": 1073741824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034070210631895686, + "loss": 1.3564, + "theoretical_loss": 3.624686537381153, + "tokens_seen": 1073872896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406820461384152, + "loss": 1.261, + "theoretical_loss": 3.6246449904405287, + "tokens_seen": 1074003968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034066198595787364, + "loss": 1.337, + "theoretical_loss": 3.6246034499895234, + "tokens_seen": 1074135040 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340641925777332, + "loss": 1.3186, + "theoretical_loss": 3.6245619160263303, + "tokens_seen": 1074266112 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034062186559679036, + "loss": 1.2792, + "theoretical_loss": 3.6245203885491453, + "tokens_seen": 1074397184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406018054162487, + "loss": 1.3373, + "theoretical_loss": 3.624478867556164, + "tokens_seen": 1074528256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034058174523570714, + "loss": 1.4733, + "theoretical_loss": 3.6244373530455842, + "tokens_seen": 1074659328 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.3489736020565033, + "objective/train/avg_token_score": 0.03156447410583496, + "objective/train/docs_used": 613082, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0295233726501465, + "objective/train/original_loss": 3.5219712257385254, + "objective/train/theoretical_loss": 3.624395845015602, + "objective/train/tokens_used": 1074790400, + "objective/train/unlikelihood_loss": 0.19533295929431915, + "theoretical_loss": 3.624395845015602, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405616850551655, + "loss": 1.1956, + "theoretical_loss": 3.624395845015602, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034054162487462386, + "loss": 1.346, + "theoretical_loss": 3.624354343464417, + "tokens_seen": 1074921472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405215646940822, + "loss": 1.2393, + "theoretical_loss": 3.6243128483902276, + "tokens_seen": 1075052544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034050150451354064, + "loss": 1.3691, + "theoretical_loss": 3.6242713597912335, + "tokens_seen": 1075183616 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340481444332999, + "loss": 1.3909, + "theoretical_loss": 3.624229877665634, + "tokens_seen": 1075314688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034046138415245737, + "loss": 1.3563, + "theoretical_loss": 3.624188402011632, + "tokens_seen": 1075445760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034044132397191573, + "loss": 1.3252, + "theoretical_loss": 3.6241469328274283, + "tokens_seen": 1075576832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404212637913741, + "loss": 1.293, + "theoretical_loss": 3.6241054701112256, + "tokens_seen": 1075707904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404012036108325, + "loss": 1.391, + "theoretical_loss": 3.6240640138612275, + "tokens_seen": 1075838976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034038114343029087, + "loss": 1.4075, + "theoretical_loss": 3.6240225640756374, + "tokens_seen": 1075970048 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034036108324974923, + "loss": 1.1787, + "theoretical_loss": 3.62398112075266, + "tokens_seen": 1076101120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403410230692076, + "loss": 1.4571, + "theoretical_loss": 3.6239396838905016, + "tokens_seen": 1076232192 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340320962888666, + "loss": 1.4021, + "theoretical_loss": 3.6238982534873667, + "tokens_seen": 1076363264 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.36412513256073, + "objective/train/avg_token_score": 0.04303035885095596, + "objective/train/docs_used": 614240, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1610873937606812, + "objective/train/original_loss": 3.6557626724243164, + "objective/train/theoretical_loss": 3.6238775407073733, + "objective/train/tokens_used": 1076428800, + "objective/train/unlikelihood_loss": 0.1986236423254013, + "theoretical_loss": 3.6238775407073733, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403009027081244, + "loss": 1.2664, + "theoretical_loss": 3.623856829541464, + "tokens_seen": 1076494336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034028084252758274, + "loss": 1.5008, + "theoretical_loss": 3.6238154120509996, + "tokens_seen": 1076625408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402607823470411, + "loss": 1.3842, + "theoretical_loss": 3.6237740010141826, + "tokens_seen": 1076756480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034024072216649946, + "loss": 1.4296, + "theoretical_loss": 3.6237325964292215, + "tokens_seen": 1076887552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402206619859579, + "loss": 1.3481, + "theoretical_loss": 3.6236911982943267, + "tokens_seen": 1077018624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034020060180541624, + "loss": 1.4151, + "theoretical_loss": 3.623649806607707, + "tokens_seen": 1077149696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401805416248746, + "loss": 1.2646, + "theoretical_loss": 3.623608421367576, + "tokens_seen": 1077280768 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034016048144433296, + "loss": 1.3451, + "theoretical_loss": 3.6235670425721436, + "tokens_seen": 1077411840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401404212637914, + "loss": 1.4463, + "theoretical_loss": 3.623525670219623, + "tokens_seen": 1077542912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034012036108324974, + "loss": 1.5708, + "theoretical_loss": 3.623484304308228, + "tokens_seen": 1077673984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401003009027081, + "loss": 1.4357, + "theoretical_loss": 3.623442944836171, + "tokens_seen": 1077805056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400802407221665, + "loss": 1.3262, + "theoretical_loss": 3.6234015918016684, + "tokens_seen": 1077936128 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.3006476163864136, + "objective/train/avg_token_score": 0.014853325672447681, + "objective/train/docs_used": 615029, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.091731071472168, + "objective/train/original_loss": 3.5971333980560303, + "objective/train/theoretical_loss": 3.6233602452029348, + "objective/train/tokens_used": 1078067200, + "objective/train/unlikelihood_loss": 0.2735164761543274, + "theoretical_loss": 3.6233602452029348, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400601805416249, + "loss": 1.4663, + "theoretical_loss": 3.6233602452029348, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400401203610833, + "loss": 1.3347, + "theoretical_loss": 3.623318905038187, + "tokens_seen": 1078198272 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034002006018054166, + "loss": 1.4328, + "theoretical_loss": 3.6232775713056404, + "tokens_seen": 1078329344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034, + "loss": 1.5252, + "theoretical_loss": 3.6232362440035137, + "tokens_seen": 1078460416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399799398194584, + "loss": 1.3211, + "theoretical_loss": 3.623194923130025, + "tokens_seen": 1078591488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399598796389168, + "loss": 1.375, + "theoretical_loss": 3.623153608683393, + "tokens_seen": 1078722560 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033993981945837517, + "loss": 1.4509, + "theoretical_loss": 3.6231123006618375, + "tokens_seen": 1078853632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033991975927783353, + "loss": 1.3794, + "theoretical_loss": 3.623070999063579, + "tokens_seen": 1078984704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398996990972919, + "loss": 1.3029, + "theoretical_loss": 3.623029703886839, + "tokens_seen": 1079115776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033987963891675025, + "loss": 1.3083, + "theoretical_loss": 3.622988415129838, + "tokens_seen": 1079246848 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033985957873620867, + "loss": 1.3717, + "theoretical_loss": 3.6229471327907996, + "tokens_seen": 1079377920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033983951855566703, + "loss": 1.3361, + "theoretical_loss": 3.6229058568679466, + "tokens_seen": 1079508992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398194583751254, + "loss": 1.4436, + "theoretical_loss": 3.622864587359503, + "tokens_seen": 1079640064 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.43908846378326416, + "objective/train/avg_token_score": 0.006012861616909504, + "objective/train/docs_used": 616160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2385116815567017, + "objective/train/original_loss": 3.701709270477295, + "objective/train/theoretical_loss": 3.62284395501013, + "objective/train/tokens_used": 1079705600, + "objective/train/unlikelihood_loss": 0.1386016309261322, + "theoretical_loss": 3.62284395501013, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033979939819458376, + "loss": 1.4263, + "theoretical_loss": 3.622823324263694, + "tokens_seen": 1079771136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033977933801404217, + "loss": 1.3934, + "theoretical_loss": 3.6227820675787443, + "tokens_seen": 1079902208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033975927783350053, + "loss": 1.3926, + "theoretical_loss": 3.6227408173028794, + "tokens_seen": 1080033280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397392176529589, + "loss": 1.2855, + "theoretical_loss": 3.6226995734343275, + "tokens_seen": 1080164352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033971915747241726, + "loss": 1.3761, + "theoretical_loss": 3.6226583359713147, + "tokens_seen": 1080295424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396990972918756, + "loss": 1.5393, + "theoretical_loss": 3.6226171049120697, + "tokens_seen": 1080426496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033967903711133404, + "loss": 1.2828, + "theoretical_loss": 3.6225758802548214, + "tokens_seen": 1080557568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396589769307924, + "loss": 1.3341, + "theoretical_loss": 3.6225346619977996, + "tokens_seen": 1080688640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033963891675025076, + "loss": 1.413, + "theoretical_loss": 3.6224934501392343, + "tokens_seen": 1080819712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396188565697091, + "loss": 1.391, + "theoretical_loss": 3.6224522446773566, + "tokens_seen": 1080950784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033959879638916754, + "loss": 1.3104, + "theoretical_loss": 3.6224110456103977, + "tokens_seen": 1081081856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395787362086259, + "loss": 1.33, + "theoretical_loss": 3.6223698529365906, + "tokens_seen": 1081212928 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.5687927603721619, + "objective/train/avg_token_score": 0.008178889751434326, + "objective/train/docs_used": 616786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8403513431549072, + "objective/train/original_loss": 3.59763240814209, + "objective/train/theoretical_loss": 3.6223286666541683, + "objective/train/tokens_used": 1081344000, + "objective/train/unlikelihood_loss": 0.14923860132694244, + "theoretical_loss": 3.6223286666541683, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033955867602808426, + "loss": 1.4021, + "theoretical_loss": 3.6223286666541683, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395386158475426, + "loss": 1.4334, + "theoretical_loss": 3.6222874867613646, + "tokens_seen": 1081475072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033951855566700104, + "loss": 1.3276, + "theoretical_loss": 3.6222463132564133, + "tokens_seen": 1081606144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394984954864594, + "loss": 1.3732, + "theoretical_loss": 3.62220514613755, + "tokens_seen": 1081737216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033947843530591777, + "loss": 1.3216, + "theoretical_loss": 3.6221639854030117, + "tokens_seen": 1081868288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033945837512537613, + "loss": 1.4189, + "theoretical_loss": 3.6221228310510334, + "tokens_seen": 1081999360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394383149448345, + "loss": 1.187, + "theoretical_loss": 3.6220816830798532, + "tokens_seen": 1082130432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394182547642929, + "loss": 1.3783, + "theoretical_loss": 3.622040541487709, + "tokens_seen": 1082261504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033939819458375127, + "loss": 1.376, + "theoretical_loss": 3.621999406272839, + "tokens_seen": 1082392576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033937813440320963, + "loss": 1.2511, + "theoretical_loss": 3.6219582774334835, + "tokens_seen": 1082523648 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339358074222668, + "loss": 1.3608, + "theoretical_loss": 3.621917154967882, + "tokens_seen": 1082654720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393380140421264, + "loss": 1.3689, + "theoretical_loss": 3.621876038874275, + "tokens_seen": 1082785792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393179538615848, + "loss": 1.2836, + "theoretical_loss": 3.621834929150905, + "tokens_seen": 1082916864 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.42741936445236206, + "objective/train/avg_token_score": 0.01599992625415325, + "objective/train/docs_used": 617451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3952101469039917, + "objective/train/original_loss": 3.9377992153167725, + "objective/train/theoretical_loss": 3.6218143766775093, + "objective/train/tokens_used": 1082982400, + "objective/train/unlikelihood_loss": 0.13472191989421844, + "theoretical_loss": 3.6218143766775093, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033929789368104314, + "loss": 1.4094, + "theoretical_loss": 3.621793825796013, + "tokens_seen": 1083047936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392778335005015, + "loss": 1.3489, + "theoretical_loss": 3.621752728807843, + "tokens_seen": 1083179008 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033925777331995986, + "loss": 1.5519, + "theoretical_loss": 3.621711638184638, + "tokens_seen": 1083310080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392377131394183, + "loss": 1.2863, + "theoretical_loss": 3.621670553924642, + "tokens_seen": 1083441152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033921765295887664, + "loss": 1.3473, + "theoretical_loss": 3.6216294760261007, + "tokens_seen": 1083572224 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339197592778335, + "loss": 1.4847, + "theoretical_loss": 3.6215884044872593, + "tokens_seen": 1083703296 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033917753259779336, + "loss": 1.3366, + "theoretical_loss": 3.6215473393063644, + "tokens_seen": 1083834368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391574724172518, + "loss": 1.3076, + "theoretical_loss": 3.621506280481663, + "tokens_seen": 1083965440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033913741223671014, + "loss": 1.2592, + "theoretical_loss": 3.6214652280114024, + "tokens_seen": 1084096512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391173520561685, + "loss": 1.3039, + "theoretical_loss": 3.6214241818938318, + "tokens_seen": 1084227584 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033909729187562687, + "loss": 1.4071, + "theoretical_loss": 3.6213831421272005, + "tokens_seen": 1084358656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033907723169508523, + "loss": 1.5266, + "theoretical_loss": 3.621342108709757, + "tokens_seen": 1084489728 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.3249022364616394, + "objective/train/avg_token_score": 0.00504777068272233, + "objective/train/docs_used": 618647, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9597523808479309, + "objective/train/original_loss": 3.926823616027832, + "objective/train/theoretical_loss": 3.621301081639753, + "objective/train/tokens_used": 1084620800, + "objective/train/unlikelihood_loss": 0.09516184777021408, + "theoretical_loss": 3.621301081639753, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033905717151454365, + "loss": 1.2593, + "theoretical_loss": 3.621301081639753, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339037111334002, + "loss": 1.2492, + "theoretical_loss": 3.6212600609154393, + "tokens_seen": 1084751872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033901705115346037, + "loss": 1.3708, + "theoretical_loss": 3.621219046535068, + "tokens_seen": 1084882944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033899699097291873, + "loss": 1.2584, + "theoretical_loss": 3.6211780384968923, + "tokens_seen": 1085014016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033897693079237715, + "loss": 1.2898, + "theoretical_loss": 3.6211370367991647, + "tokens_seen": 1085145088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389568706118355, + "loss": 1.317, + "theoretical_loss": 3.621096041440139, + "tokens_seen": 1085276160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389368104312939, + "loss": 1.2965, + "theoretical_loss": 3.621055052418071, + "tokens_seen": 1085407232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033891675025075224, + "loss": 1.2751, + "theoretical_loss": 3.6210140697312143, + "tokens_seen": 1085538304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388966900702106, + "loss": 1.2638, + "theoretical_loss": 3.620973093377827, + "tokens_seen": 1085669376 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338876629889669, + "loss": 1.4237, + "theoretical_loss": 3.6209321233561647, + "tokens_seen": 1085800448 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338876629889669, + "loss": 1.3959, + "theoretical_loss": 3.620891159664485, + "tokens_seen": 1085931520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388565697091274, + "loss": 1.4063, + "theoretical_loss": 3.620850202301046, + "tokens_seen": 1086062592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033883650952858574, + "loss": 1.3171, + "theoretical_loss": 3.620809251264107, + "tokens_seen": 1086193664 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.3181818127632141, + "objective/train/avg_token_score": 0.03115883283317089, + "objective/train/docs_used": 619458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.073768138885498, + "objective/train/original_loss": 3.859205484390259, + "objective/train/theoretical_loss": 3.6207887781175305, + "objective/train/tokens_used": 1086259200, + "objective/train/unlikelihood_loss": 0.19596649706363678, + "theoretical_loss": 3.6207887781175305, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388164493480441, + "loss": 1.3389, + "theoretical_loss": 3.620768306551927, + "tokens_seen": 1086324736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387963891675025, + "loss": 1.3547, + "theoretical_loss": 3.6207273681627665, + "tokens_seen": 1086455808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387763289869609, + "loss": 1.3367, + "theoretical_loss": 3.6206864360948865, + "tokens_seen": 1086586880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033875626880641924, + "loss": 1.2713, + "theoretical_loss": 3.620645510346548, + "tokens_seen": 1086717952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387362086258776, + "loss": 1.5305, + "theoretical_loss": 3.620604590916014, + "tokens_seen": 1086849024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033871614844533597, + "loss": 1.497, + "theoretical_loss": 3.620563677801547, + "tokens_seen": 1086980096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386960882647944, + "loss": 1.2514, + "theoretical_loss": 3.620522771001411, + "tokens_seen": 1087111168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033867602808425275, + "loss": 1.4287, + "theoretical_loss": 3.6204818705138697, + "tokens_seen": 1087242240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386559679037111, + "loss": 1.4406, + "theoretical_loss": 3.620440976337189, + "tokens_seen": 1087373312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033863590772316947, + "loss": 1.3033, + "theoretical_loss": 3.6204000884696343, + "tokens_seen": 1087504384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386158475426279, + "loss": 1.1373, + "theoretical_loss": 3.6203592069094714, + "tokens_seen": 1087635456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033859578736208625, + "loss": 1.333, + "theoretical_loss": 3.6203183316549685, + "tokens_seen": 1087766528 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.41898825764656067, + "objective/train/avg_token_score": 0.0180879645049572, + "objective/train/docs_used": 620460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4624638557434082, + "objective/train/original_loss": 4.034957408905029, + "objective/train/theoretical_loss": 3.6202774627043923, + "objective/train/tokens_used": 1087897600, + "objective/train/unlikelihood_loss": 0.14046654105186462, + "theoretical_loss": 3.6202774627043923, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033857572718154467, + "loss": 1.257, + "theoretical_loss": 3.6202774627043923, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033855566700100303, + "loss": 1.3066, + "theoretical_loss": 3.6202366000560118, + "tokens_seen": 1088028672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033853560682046144, + "loss": 1.3526, + "theoretical_loss": 3.620195743708096, + "tokens_seen": 1088159744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385155466399198, + "loss": 1.2837, + "theoretical_loss": 3.620154893658915, + "tokens_seen": 1088290816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033849548645937817, + "loss": 1.2386, + "theoretical_loss": 3.620114049906739, + "tokens_seen": 1088421888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033847542627883653, + "loss": 1.4272, + "theoretical_loss": 3.6200732124498387, + "tokens_seen": 1088552960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384553660982949, + "loss": 1.4194, + "theoretical_loss": 3.620032381286487, + "tokens_seen": 1088684032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384353059177533, + "loss": 1.271, + "theoretical_loss": 3.6199915564149556, + "tokens_seen": 1088815104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033841524573721167, + "loss": 1.4222, + "theoretical_loss": 3.6199507378335185, + "tokens_seen": 1088946176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033839518555667003, + "loss": 1.371, + "theoretical_loss": 3.6199099255404494, + "tokens_seen": 1089077248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383751253761284, + "loss": 1.2811, + "theoretical_loss": 3.6198691195340222, + "tokens_seen": 1089208320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383550651955868, + "loss": 1.5101, + "theoretical_loss": 3.619828319812513, + "tokens_seen": 1089339392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383350050150452, + "loss": 1.4073, + "theoretical_loss": 3.619787526374197, + "tokens_seen": 1089470464 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.42442572116851807, + "objective/train/avg_token_score": 0.03458975628018379, + "objective/train/docs_used": 620976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5144895315170288, + "objective/train/original_loss": 4.124884128570557, + "objective/train/theoretical_loss": 3.6197671320106983, + "objective/train/tokens_used": 1089536000, + "objective/train/unlikelihood_loss": 0.1613459587097168, + "theoretical_loss": 3.6197671320106983, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033831494483450354, + "loss": 1.4338, + "theoretical_loss": 3.6197467392173515, + "tokens_seen": 1089601536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382948846539619, + "loss": 1.2328, + "theoretical_loss": 3.619705958340254, + "tokens_seen": 1089732608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033827482447342026, + "loss": 1.3435, + "theoretical_loss": 3.6196651837411817, + "tokens_seen": 1089863680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382547642928787, + "loss": 1.233, + "theoretical_loss": 3.619624415418414, + "tokens_seen": 1089994752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033823470411233704, + "loss": 1.3541, + "theoretical_loss": 3.6195836533702295, + "tokens_seen": 1090125824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382146439317954, + "loss": 1.3231, + "theoretical_loss": 3.619542897594909, + "tokens_seen": 1090256896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033819458375125376, + "loss": 1.3466, + "theoretical_loss": 3.619502148090733, + "tokens_seen": 1090387968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381745235707122, + "loss": 1.3616, + "theoretical_loss": 3.6194614048559828, + "tokens_seen": 1090519040 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033815446339017054, + "loss": 1.4552, + "theoretical_loss": 3.6194206678889396, + "tokens_seen": 1090650112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381344032096289, + "loss": 1.3834, + "theoretical_loss": 3.6193799371878876, + "tokens_seen": 1090781184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033811434302908727, + "loss": 1.3202, + "theoretical_loss": 3.6193392127511097, + "tokens_seen": 1090912256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033809428284854563, + "loss": 1.3078, + "theoretical_loss": 3.61929849457689, + "tokens_seen": 1091043328 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.25287145376205444, + "objective/train/avg_token_score": 0.028310926631093025, + "objective/train/docs_used": 621707, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8041895031929016, + "objective/train/original_loss": 4.141907691955566, + "objective/train/theoretical_loss": 3.619257782663513, + "objective/train/tokens_used": 1091174400, + "objective/train/unlikelihood_loss": 0.18120165169239044, + "theoretical_loss": 3.619257782663513, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033807422266800405, + "loss": 1.1881, + "theoretical_loss": 3.619257782663513, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380541624874624, + "loss": 1.2884, + "theoretical_loss": 3.619217077009264, + "tokens_seen": 1091305472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033803410230692077, + "loss": 1.2636, + "theoretical_loss": 3.6191763776124297, + "tokens_seen": 1091436544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033801404212637913, + "loss": 1.3568, + "theoretical_loss": 3.619135684471297, + "tokens_seen": 1091567616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033799398194583755, + "loss": 1.3662, + "theoretical_loss": 3.6190949975841527, + "tokens_seen": 1091698688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379739217652959, + "loss": 1.4307, + "theoretical_loss": 3.6190543169492857, + "tokens_seen": 1091829760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379538615847543, + "loss": 1.4041, + "theoretical_loss": 3.6190136425649837, + "tokens_seen": 1091960832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033793380140421264, + "loss": 1.2241, + "theoretical_loss": 3.6189729744295374, + "tokens_seen": 1092091904 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337913741223671, + "loss": 1.2912, + "theoretical_loss": 3.6189323125412365, + "tokens_seen": 1092222976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378936810431294, + "loss": 1.3128, + "theoretical_loss": 3.618891656898372, + "tokens_seen": 1092354048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378736208625878, + "loss": 1.3653, + "theoretical_loss": 3.6188510074992344, + "tokens_seen": 1092485120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033785356068204614, + "loss": 1.3812, + "theoretical_loss": 3.6188103643421172, + "tokens_seen": 1092616192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378335005015045, + "loss": 1.2615, + "theoretical_loss": 3.618769727425313, + "tokens_seen": 1092747264 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.31286656856536865, + "objective/train/avg_token_score": 0.02689872868359089, + "objective/train/docs_used": 622698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1303755044937134, + "objective/train/original_loss": 4.247082233428955, + "objective/train/theoretical_loss": 3.618749411306495, + "objective/train/tokens_used": 1092812800, + "objective/train/unlikelihood_loss": 0.15665476024150848, + "theoretical_loss": 3.618749411306495, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378134403209629, + "loss": 1.2965, + "theoretical_loss": 3.618729096747115, + "tokens_seen": 1092878336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377933801404213, + "loss": 1.2293, + "theoretical_loss": 3.6186884723058172, + "tokens_seen": 1093009408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033777331995987964, + "loss": 1.3578, + "theoretical_loss": 3.6186478540997156, + "tokens_seen": 1093140480 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337753259779338, + "loss": 1.3031, + "theoretical_loss": 3.618607242127104, + "tokens_seen": 1093271552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033773319959879637, + "loss": 1.3308, + "theoretical_loss": 3.61856663638628, + "tokens_seen": 1093402624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377131394182548, + "loss": 1.2849, + "theoretical_loss": 3.61852603687554, + "tokens_seen": 1093533696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033769307923771315, + "loss": 1.4807, + "theoretical_loss": 3.618485443593182, + "tokens_seen": 1093664768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376730190571715, + "loss": 1.3833, + "theoretical_loss": 3.6184448565375034, + "tokens_seen": 1093795840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033765295887662987, + "loss": 1.3044, + "theoretical_loss": 3.6184042757068036, + "tokens_seen": 1093926912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376328986960883, + "loss": 1.2501, + "theoretical_loss": 3.618363701099382, + "tokens_seen": 1094057984 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033761283851554665, + "loss": 1.3966, + "theoretical_loss": 3.618323132713539, + "tokens_seen": 1094189056 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337592778335005, + "loss": 1.3744, + "theoretical_loss": 3.618282570547575, + "tokens_seen": 1094320128 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.3688294291496277, + "objective/train/avg_token_score": 0.014695006422698498, + "objective/train/docs_used": 623416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3113912343978882, + "objective/train/original_loss": 4.256772041320801, + "objective/train/theoretical_loss": 3.618242014599793, + "objective/train/tokens_used": 1094451200, + "objective/train/unlikelihood_loss": 0.10496540367603302, + "theoretical_loss": 3.618242014599793, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375727181544634, + "loss": 1.4407, + "theoretical_loss": 3.618242014599793, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033755265797392174, + "loss": 1.3777, + "theoretical_loss": 3.618201464868493, + "tokens_seen": 1094582272 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033753259779338015, + "loss": 1.1352, + "theoretical_loss": 3.61816092135198, + "tokens_seen": 1094713344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375125376128385, + "loss": 1.4587, + "theoretical_loss": 3.618120384048556, + "tokens_seen": 1094844416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374924774322969, + "loss": 1.4218, + "theoretical_loss": 3.6180798529565266, + "tokens_seen": 1094975488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033747241725175524, + "loss": 1.1808, + "theoretical_loss": 3.6180393280741954, + "tokens_seen": 1095106560 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033745235707121366, + "loss": 1.4355, + "theoretical_loss": 3.617998809399869, + "tokens_seen": 1095237632 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337432296890672, + "loss": 1.2876, + "theoretical_loss": 3.6179582969318527, + "tokens_seen": 1095368704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374122367101304, + "loss": 1.2664, + "theoretical_loss": 3.6179177906684536, + "tokens_seen": 1095499776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033739217652958874, + "loss": 1.2002, + "theoretical_loss": 3.6178772906079804, + "tokens_seen": 1095630848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373721163490471, + "loss": 1.4124, + "theoretical_loss": 3.6178367967487404, + "tokens_seen": 1095761920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373520561685055, + "loss": 1.2626, + "theoretical_loss": 3.6177963090890417, + "tokens_seen": 1095892992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373319959879639, + "loss": 1.331, + "theoretical_loss": 3.617755827627195, + "tokens_seen": 1096024064 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.5296920537948608, + "objective/train/avg_token_score": 0.012129692360758781, + "objective/train/docs_used": 624708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8187059164047241, + "objective/train/original_loss": 3.6660492420196533, + "objective/train/theoretical_loss": 3.6177355892199383, + "objective/train/tokens_used": 1096089600, + "objective/train/unlikelihood_loss": 0.15639354288578033, + "theoretical_loss": 3.6177355892199383, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033731193580742225, + "loss": 1.286, + "theoretical_loss": 3.6177153523615107, + "tokens_seen": 1096155136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372918756268806, + "loss": 1.3916, + "theoretical_loss": 3.6176748832902987, + "tokens_seen": 1096286208 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337271815446339, + "loss": 1.2695, + "theoretical_loss": 3.6176344204118713, + "tokens_seen": 1096417280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372517552657974, + "loss": 1.202, + "theoretical_loss": 3.6175939637245405, + "tokens_seen": 1096548352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033723169508525575, + "loss": 1.3118, + "theoretical_loss": 3.6175535132266186, + "tokens_seen": 1096679424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372116349047141, + "loss": 1.4544, + "theoretical_loss": 3.6175130689164194, + "tokens_seen": 1096810496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371915747241725, + "loss": 1.3534, + "theoretical_loss": 3.617472630792258, + "tokens_seen": 1096941568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371715145436309, + "loss": 1.3887, + "theoretical_loss": 3.617432198852448, + "tokens_seen": 1097072640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033715145436308925, + "loss": 1.3184, + "theoretical_loss": 3.6173917730953056, + "tokens_seen": 1097203712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371313941825476, + "loss": 1.3018, + "theoretical_loss": 3.6173513535191466, + "tokens_seen": 1097334784 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337111334002006, + "loss": 1.3904, + "theoretical_loss": 3.6173109401222883, + "tokens_seen": 1097465856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370912738214644, + "loss": 1.3114, + "theoretical_loss": 3.617270532903047, + "tokens_seen": 1097596928 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.4127565920352936, + "objective/train/avg_token_score": 0.051338497549295425, + "objective/train/docs_used": 625328, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3618358373641968, + "objective/train/original_loss": 3.767439365386963, + "objective/train/theoretical_loss": 3.617230131859743, + "objective/train/tokens_used": 1097728000, + "objective/train/unlikelihood_loss": 0.18313588201999664, + "theoretical_loss": 3.617230131859743, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370712136409228, + "loss": 1.3254, + "theoretical_loss": 3.617230131859743, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033705115346038117, + "loss": 1.3862, + "theoretical_loss": 3.6171897369906922, + "tokens_seen": 1097859072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033703109327983953, + "loss": 1.4037, + "theoretical_loss": 3.617149348294217, + "tokens_seen": 1097990144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033701103309929795, + "loss": 1.4577, + "theoretical_loss": 3.6171089657686357, + "tokens_seen": 1098121216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369909729187563, + "loss": 1.3973, + "theoretical_loss": 3.61706858941227, + "tokens_seen": 1098252288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369709127382147, + "loss": 1.2607, + "theoretical_loss": 3.61702821922344, + "tokens_seen": 1098383360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033695085255767304, + "loss": 1.3545, + "theoretical_loss": 3.6169878552004695, + "tokens_seen": 1098514432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369307923771314, + "loss": 1.3185, + "theoretical_loss": 3.6169474973416804, + "tokens_seen": 1098645504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369107321965898, + "loss": 1.2884, + "theoretical_loss": 3.6169071456453956, + "tokens_seen": 1098776576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368906720160482, + "loss": 1.3194, + "theoretical_loss": 3.61686680010994, + "tokens_seen": 1098907648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033687061183550654, + "loss": 1.3279, + "theoretical_loss": 3.6168264607336376, + "tokens_seen": 1099038720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368505516549649, + "loss": 1.3334, + "theoretical_loss": 3.6167861275148145, + "tokens_seen": 1099169792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368304914744233, + "loss": 1.3937, + "theoretical_loss": 3.6167458004517963, + "tokens_seen": 1099300864 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.38581377267837524, + "objective/train/avg_token_score": 0.009347434155642986, + "objective/train/docs_used": 626712, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3463314771652222, + "objective/train/original_loss": 3.8716883659362793, + "objective/train/theoretical_loss": 3.616725639228191, + "objective/train/tokens_used": 1099366400, + "objective/train/unlikelihood_loss": 0.15615318715572357, + "theoretical_loss": 3.616725639228191, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368104312938817, + "loss": 1.398, + "theoretical_loss": 3.61670547954291, + "tokens_seen": 1099431936 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033679037111334004, + "loss": 1.4071, + "theoretical_loss": 3.616665164786482, + "tokens_seen": 1099563008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367703109327984, + "loss": 1.3357, + "theoretical_loss": 3.6166248561808416, + "tokens_seen": 1099694080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033675025075225677, + "loss": 1.4304, + "theoretical_loss": 3.616584553724316, + "tokens_seen": 1099825152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367301905717152, + "loss": 1.321, + "theoretical_loss": 3.616544257415236, + "tokens_seen": 1099956224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033671013039117355, + "loss": 1.2777, + "theoretical_loss": 3.616503967251931, + "tokens_seen": 1100087296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366900702106319, + "loss": 1.4183, + "theoretical_loss": 3.6164636832327313, + "tokens_seen": 1100218368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033667001003009027, + "loss": 1.255, + "theoretical_loss": 3.6164234053559676, + "tokens_seen": 1100349440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366499498495487, + "loss": 1.3759, + "theoretical_loss": 3.616383133619973, + "tokens_seen": 1100480512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033662988966900705, + "loss": 1.4536, + "theoretical_loss": 3.6163428680230796, + "tokens_seen": 1100611584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366098294884654, + "loss": 1.2896, + "theoretical_loss": 3.6163026085636205, + "tokens_seen": 1100742656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365897693079238, + "loss": 1.3112, + "theoretical_loss": 3.61626235523993, + "tokens_seen": 1100873728 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.36821848154067993, + "objective/train/avg_token_score": 0.009577035903930664, + "objective/train/docs_used": 627263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2547622919082642, + "objective/train/original_loss": 3.7617404460906982, + "objective/train/theoretical_loss": 3.6162221080503416, + "objective/train/tokens_used": 1101004800, + "objective/train/unlikelihood_loss": 0.14662618935108185, + "theoretical_loss": 3.6162221080503416, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033656970912738214, + "loss": 1.4666, + "theoretical_loss": 3.6162221080503416, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033654964894684055, + "loss": 1.4981, + "theoretical_loss": 3.616181866993191, + "tokens_seen": 1101135872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365295887662989, + "loss": 1.1038, + "theoretical_loss": 3.6161416320668147, + "tokens_seen": 1101266944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365095285857573, + "loss": 1.3647, + "theoretical_loss": 3.616101403269548, + "tokens_seen": 1101398016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033648946840521564, + "loss": 1.275, + "theoretical_loss": 3.6160611805997283, + "tokens_seen": 1101529088 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033646940822467406, + "loss": 1.4161, + "theoretical_loss": 3.616020964055694, + "tokens_seen": 1101660160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364493480441324, + "loss": 1.4699, + "theoretical_loss": 3.615980753635783, + "tokens_seen": 1101791232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364292878635908, + "loss": 1.2891, + "theoretical_loss": 3.6159405493383345, + "tokens_seen": 1101922304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033640922768304914, + "loss": 1.3533, + "theoretical_loss": 3.615900351161688, + "tokens_seen": 1102053376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363891675025075, + "loss": 1.3817, + "theoretical_loss": 3.615860159104184, + "tokens_seen": 1102184448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363691073219659, + "loss": 1.4334, + "theoretical_loss": 3.6158199731641636, + "tokens_seen": 1102315520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363490471414243, + "loss": 1.404, + "theoretical_loss": 3.6157797933399682, + "tokens_seen": 1102446592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033632898696088265, + "loss": 1.3271, + "theoretical_loss": 3.6157396196299403, + "tokens_seen": 1102577664 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.31683772802352905, + "objective/train/avg_token_score": 0.020152246579527855, + "objective/train/docs_used": 628470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1545360088348389, + "objective/train/original_loss": 4.006408214569092, + "objective/train/theoretical_loss": 3.615719535067221, + "objective/train/tokens_used": 1102643200, + "objective/train/unlikelihood_loss": 0.1712460219860077, + "theoretical_loss": 3.615719535067221, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336308926780341, + "loss": 1.4004, + "theoretical_loss": 3.6156994520324224, + "tokens_seen": 1102708736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362888665997994, + "loss": 1.3183, + "theoretical_loss": 3.615659290545759, + "tokens_seen": 1102839808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362688064192578, + "loss": 1.157, + "theoretical_loss": 3.6156191351682936, + "tokens_seen": 1102970880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033624874623871615, + "loss": 1.1923, + "theoretical_loss": 3.615578985898371, + "tokens_seen": 1103101952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362286860581745, + "loss": 1.2603, + "theoretical_loss": 3.615538842734337, + "tokens_seen": 1103233024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033620862587763293, + "loss": 1.3, + "theoretical_loss": 3.615498705674537, + "tokens_seen": 1103364096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361885656970913, + "loss": 1.5284, + "theoretical_loss": 3.6154585747173194, + "tokens_seen": 1103495168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033616850551654965, + "loss": 1.4671, + "theoretical_loss": 3.6154184498610307, + "tokens_seen": 1103626240 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336148445336008, + "loss": 1.3761, + "theoretical_loss": 3.6153783311040186, + "tokens_seen": 1103757312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361283851554664, + "loss": 1.4228, + "theoretical_loss": 3.6153382184446325, + "tokens_seen": 1103888384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361083249749248, + "loss": 1.461, + "theoretical_loss": 3.6152981118812217, + "tokens_seen": 1104019456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033608826479438316, + "loss": 1.4375, + "theoretical_loss": 3.615258011412136, + "tokens_seen": 1104150528 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.45173507928848267, + "objective/train/avg_token_score": 0.02246457152068615, + "objective/train/docs_used": 629051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4576631784439087, + "objective/train/original_loss": 3.8835036754608154, + "objective/train/theoretical_loss": 3.615217917035726, + "objective/train/tokens_used": 1104281600, + "objective/train/unlikelihood_loss": 0.13549575209617615, + "theoretical_loss": 3.615217917035726, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360682046138415, + "loss": 1.2522, + "theoretical_loss": 3.615217917035726, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360481444332999, + "loss": 1.3689, + "theoretical_loss": 3.615177828750343, + "tokens_seen": 1104412672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360280842527583, + "loss": 1.3465, + "theoretical_loss": 3.615137746554339, + "tokens_seen": 1104543744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033600802407221666, + "loss": 1.5594, + "theoretical_loss": 3.615097670446067, + "tokens_seen": 1104674816 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335987963891675, + "loss": 1.4321, + "theoretical_loss": 3.6150576004238797, + "tokens_seen": 1104805888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359679037111334, + "loss": 1.3812, + "theoretical_loss": 3.6150175364861314, + "tokens_seen": 1104936960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033594784353059174, + "loss": 1.3706, + "theoretical_loss": 3.614977478631176, + "tokens_seen": 1105068032 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033592778335005016, + "loss": 1.2999, + "theoretical_loss": 3.6149374268573697, + "tokens_seen": 1105199104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359077231695085, + "loss": 1.5506, + "theoretical_loss": 3.6148973811630674, + "tokens_seen": 1105330176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003358876629889669, + "loss": 1.4552, + "theoretical_loss": 3.6148573415466254, + "tokens_seen": 1105461248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033586760280842525, + "loss": 1.282, + "theoretical_loss": 3.614817308006401, + "tokens_seen": 1105592320 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033584754262788366, + "loss": 1.2659, + "theoretical_loss": 3.614777280540753, + "tokens_seen": 1105723392 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335827482447342, + "loss": 1.4715, + "theoretical_loss": 3.614737259148038, + "tokens_seen": 1105854464 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.3852028250694275, + "objective/train/avg_token_score": 0.022245066240429878, + "objective/train/docs_used": 630354, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.247960090637207, + "objective/train/original_loss": 3.9521636962890625, + "objective/train/theoretical_loss": 3.614717250728518, + "objective/train/tokens_used": 1105920000, + "objective/train/unlikelihood_loss": 0.17557135224342346, + "theoretical_loss": 3.614717250728518, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358074222668004, + "loss": 1.387, + "theoretical_loss": 3.6146972438266163, + "tokens_seen": 1105985536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033578736208625875, + "loss": 1.2588, + "theoretical_loss": 3.6146572345748464, + "tokens_seen": 1106116608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357673019057171, + "loss": 1.4639, + "theoretical_loss": 3.6146172313910894, + "tokens_seen": 1106247680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033574724172517553, + "loss": 1.3097, + "theoretical_loss": 3.614577234273706, + "tokens_seen": 1106378752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357271815446339, + "loss": 1.3326, + "theoretical_loss": 3.614537243221058, + "tokens_seen": 1106509824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033570712136409225, + "loss": 1.3896, + "theoretical_loss": 3.6144972582315065, + "tokens_seen": 1106640896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356870611835506, + "loss": 1.3831, + "theoretical_loss": 3.6144572793034158, + "tokens_seen": 1106771968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033566700100300903, + "loss": 1.3517, + "theoretical_loss": 3.614417306435148, + "tokens_seen": 1106903040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356469408224674, + "loss": 1.3372, + "theoretical_loss": 3.6143773396250687, + "tokens_seen": 1107034112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033562688064192576, + "loss": 1.3335, + "theoretical_loss": 3.614337378871541, + "tokens_seen": 1107165184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356068204613841, + "loss": 1.1703, + "theoretical_loss": 3.614297424172931, + "tokens_seen": 1107296256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355867602808425, + "loss": 1.2479, + "theoretical_loss": 3.6142574755276047, + "tokens_seen": 1107427328 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.41654446721076965, + "objective/train/avg_token_score": 0.03537094220519066, + "objective/train/docs_used": 630918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3929799795150757, + "objective/train/original_loss": 3.7367806434631348, + "objective/train/theoretical_loss": 3.614217532933929, + "objective/train/tokens_used": 1107558400, + "objective/train/unlikelihood_loss": 0.16754809021949768, + "theoretical_loss": 3.614217532933929, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033556670010030095, + "loss": 1.4107, + "theoretical_loss": 3.614217532933929, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355466399197593, + "loss": 1.4719, + "theoretical_loss": 3.6141775963902703, + "tokens_seen": 1107689472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355265797392177, + "loss": 1.4497, + "theoretical_loss": 3.6141376658949973, + "tokens_seen": 1107820544 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033550651955867604, + "loss": 1.2572, + "theoretical_loss": 3.614097741446478, + "tokens_seen": 1107951616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033548645937813446, + "loss": 1.313, + "theoretical_loss": 3.6140578230430815, + "tokens_seen": 1108082688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354663991975928, + "loss": 1.5947, + "theoretical_loss": 3.6140179106831787, + "tokens_seen": 1108213760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354463390170512, + "loss": 1.3022, + "theoretical_loss": 3.613978004365139, + "tokens_seen": 1108344832 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033542627883650954, + "loss": 1.4962, + "theoretical_loss": 3.613938104087333, + "tokens_seen": 1108475904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354062186559679, + "loss": 1.2911, + "theoretical_loss": 3.6138982098481334, + "tokens_seen": 1108606976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353861584754263, + "loss": 1.3687, + "theoretical_loss": 3.613858321645912, + "tokens_seen": 1108738048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353660982948847, + "loss": 1.1936, + "theoretical_loss": 3.613818439479042, + "tokens_seen": 1108869120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033534603811434305, + "loss": 1.2737, + "theoretical_loss": 3.613778563345897, + "tokens_seen": 1109000192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353259779338014, + "loss": 1.3943, + "theoretical_loss": 3.6137386932448505, + "tokens_seen": 1109131264 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.5479594469070435, + "objective/train/avg_token_score": 0.006020635366439819, + "objective/train/docs_used": 632203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7893400192260742, + "objective/train/original_loss": 3.661371946334839, + "objective/train/theoretical_loss": 3.613718760455857, + "objective/train/tokens_used": 1109196800, + "objective/train/unlikelihood_loss": 0.12863227725028992, + "theoretical_loss": 3.613718760455857, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353059177532598, + "loss": 1.4071, + "theoretical_loss": 3.613698829174278, + "tokens_seen": 1109262336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352858575727182, + "loss": 1.3267, + "theoretical_loss": 3.6136589711325557, + "tokens_seen": 1109393408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033526579739217655, + "loss": 1.4703, + "theoretical_loss": 3.613619119118058, + "tokens_seen": 1109524480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352457372116349, + "loss": 1.2731, + "theoretical_loss": 3.6135792731291625, + "tokens_seen": 1109655552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033522567703109333, + "loss": 1.2506, + "theoretical_loss": 3.613539433164247, + "tokens_seen": 1109786624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352056168505517, + "loss": 1.4399, + "theoretical_loss": 3.6134995992216883, + "tokens_seen": 1109917696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033518555667001005, + "loss": 1.4055, + "theoretical_loss": 3.613459771299866, + "tokens_seen": 1110048768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351654964894684, + "loss": 1.3845, + "theoretical_loss": 3.613419949397159, + "tokens_seen": 1110179840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351454363089268, + "loss": 1.3502, + "theoretical_loss": 3.6133801335119475, + "tokens_seen": 1110310912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351253761283852, + "loss": 1.3801, + "theoretical_loss": 3.6133403236426114, + "tokens_seen": 1110441984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033510531594784356, + "loss": 1.3582, + "theoretical_loss": 3.613300519787532, + "tokens_seen": 1110573056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350852557673019, + "loss": 1.3717, + "theoretical_loss": 3.6132607219450916, + "tokens_seen": 1110704128 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.5791788697242737, + "objective/train/avg_token_score": 0.012251202948391438, + "objective/train/docs_used": 632721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8967013359069824, + "objective/train/original_loss": 3.5206515789031982, + "objective/train/theoretical_loss": 3.6132209301136715, + "objective/train/tokens_used": 1110835200, + "objective/train/unlikelihood_loss": 0.16137850284576416, + "theoretical_loss": 3.6132209301136715, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350651955867603, + "loss": 1.444, + "theoretical_loss": 3.6132209301136715, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350451354062187, + "loss": 1.2899, + "theoretical_loss": 3.613181144291656, + "tokens_seen": 1110966272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033502507522567706, + "loss": 1.4499, + "theoretical_loss": 3.6131413644774275, + "tokens_seen": 1111097344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350050150451354, + "loss": 1.4835, + "theoretical_loss": 3.613101590669371, + "tokens_seen": 1111228416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349849548645938, + "loss": 1.4521, + "theoretical_loss": 3.6130618228658715, + "tokens_seen": 1111359488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033496489468405215, + "loss": 1.3921, + "theoretical_loss": 3.6130220610653137, + "tokens_seen": 1111490560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033494483450351056, + "loss": 1.3933, + "theoretical_loss": 3.6129823052660845, + "tokens_seen": 1111621632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349247743229689, + "loss": 1.4454, + "theoretical_loss": 3.61294255546657, + "tokens_seen": 1111752704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349047141424273, + "loss": 1.4249, + "theoretical_loss": 3.6129028116651583, + "tokens_seen": 1111883776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033488465396188565, + "loss": 1.419, + "theoretical_loss": 3.612863073860237, + "tokens_seen": 1112014848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033486459378134406, + "loss": 1.2965, + "theoretical_loss": 3.6128233420501945, + "tokens_seen": 1112145920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348445336008024, + "loss": 1.5979, + "theoretical_loss": 3.6127836162334206, + "tokens_seen": 1112276992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348244734202608, + "loss": 1.4901, + "theoretical_loss": 3.6127438964083045, + "tokens_seen": 1112408064 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.3507453501224518, + "objective/train/avg_token_score": 0.010616501793265343, + "objective/train/docs_used": 634025, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1486400365829468, + "objective/train/original_loss": 3.876255750656128, + "objective/train/theoretical_loss": 3.6127240387421153, + "objective/train/tokens_used": 1112473600, + "objective/train/unlikelihood_loss": 0.20414204895496368, + "theoretical_loss": 3.6127240387421153, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033480441323971915, + "loss": 1.2186, + "theoretical_loss": 3.6127041825732373, + "tokens_seen": 1112539136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347843530591775, + "loss": 1.5373, + "theoretical_loss": 3.6126644747266097, + "tokens_seen": 1112670208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033476429287863593, + "loss": 1.1705, + "theoretical_loss": 3.612624772866813, + "tokens_seen": 1112801280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347442326980943, + "loss": 1.2557, + "theoretical_loss": 3.6125850769922407, + "tokens_seen": 1112932352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033472417251755265, + "loss": 1.3987, + "theoretical_loss": 3.612545387101285, + "tokens_seen": 1113063424 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334704112337011, + "loss": 1.4517, + "theoretical_loss": 3.61250570319234, + "tokens_seen": 1113194496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033468405215646943, + "loss": 1.3174, + "theoretical_loss": 3.612466025263799, + "tokens_seen": 1113325568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346639919759278, + "loss": 1.3564, + "theoretical_loss": 3.612426353314058, + "tokens_seen": 1113456640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033464393179538616, + "loss": 1.2701, + "theoretical_loss": 3.6123866873415116, + "tokens_seen": 1113587712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346238716148445, + "loss": 1.3895, + "theoretical_loss": 3.612347027344556, + "tokens_seen": 1113718784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346038114343029, + "loss": 1.3721, + "theoretical_loss": 3.6123073733215882, + "tokens_seen": 1113849856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345837512537613, + "loss": 1.3479, + "theoretical_loss": 3.612267725271005, + "tokens_seen": 1113980928 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.294721394777298, + "objective/train/avg_token_score": 0.04145824909210205, + "objective/train/docs_used": 634689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8377479314804077, + "objective/train/original_loss": 3.8602466583251953, + "objective/train/theoretical_loss": 3.612228083191205, + "objective/train/tokens_used": 1114112000, + "objective/train/unlikelihood_loss": 0.2820836007595062, + "theoretical_loss": 3.612228083191205, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033456369107321966, + "loss": 1.4193, + "theoretical_loss": 3.612228083191205, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334543630892678, + "loss": 1.3925, + "theoretical_loss": 3.6121884470805865, + "tokens_seen": 1114243072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345235707121364, + "loss": 1.3039, + "theoretical_loss": 3.612148816937548, + "tokens_seen": 1114374144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345035105315948, + "loss": 1.2549, + "theoretical_loss": 3.6121091927604905, + "tokens_seen": 1114505216 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033448345035105316, + "loss": 1.3582, + "theoretical_loss": 3.6120695745478137, + "tokens_seen": 1114636288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344633901705115, + "loss": 1.4202, + "theoretical_loss": 3.6120299622979184, + "tokens_seen": 1114767360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344433299899699, + "loss": 1.6335, + "theoretical_loss": 3.6119903560092066, + "tokens_seen": 1114898432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033442326980942825, + "loss": 1.3914, + "theoretical_loss": 3.6119507556800805, + "tokens_seen": 1115029504 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033440320962888667, + "loss": 1.4239, + "theoretical_loss": 3.611911161308943, + "tokens_seen": 1115160576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033438314944834503, + "loss": 1.2513, + "theoretical_loss": 3.6118715728941972, + "tokens_seen": 1115291648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343630892678034, + "loss": 1.2888, + "theoretical_loss": 3.611831990434248, + "tokens_seen": 1115422720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033434302908726175, + "loss": 1.1959, + "theoretical_loss": 3.611792413927499, + "tokens_seen": 1115553792 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033432296890672017, + "loss": 1.3388, + "theoretical_loss": 3.611752843372356, + "tokens_seen": 1115684864 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.442937433719635, + "objective/train/avg_token_score": 0.012557187117636204, + "objective/train/docs_used": 636068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4559035301208496, + "objective/train/original_loss": 3.8122951984405518, + "objective/train/theoretical_loss": 3.611733060326139, + "objective/train/tokens_used": 1115750400, + "objective/train/unlikelihood_loss": 0.18507951498031616, + "theoretical_loss": 3.611733060326139, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033430290872617853, + "loss": 1.2632, + "theoretical_loss": 3.6117132787672257, + "tokens_seen": 1115815936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342828485456369, + "loss": 1.4263, + "theoretical_loss": 3.6116737201105136, + "tokens_seen": 1115947008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033426278836509526, + "loss": 1.3788, + "theoretical_loss": 3.611634167400627, + "tokens_seen": 1116078080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342427281845536, + "loss": 1.295, + "theoretical_loss": 3.611594620635975, + "tokens_seen": 1116209152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033422266800401204, + "loss": 1.4382, + "theoretical_loss": 3.6115550798149645, + "tokens_seen": 1116340224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342026078234704, + "loss": 1.4281, + "theoretical_loss": 3.6115155449360046, + "tokens_seen": 1116471296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033418254764292876, + "loss": 1.2635, + "theoretical_loss": 3.6114760159975052, + "tokens_seen": 1116602368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341624874623871, + "loss": 1.3258, + "theoretical_loss": 3.611436492997877, + "tokens_seen": 1116733440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033414242728184554, + "loss": 1.3514, + "theoretical_loss": 3.6113969759355307, + "tokens_seen": 1116864512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341223671013039, + "loss": 1.3056, + "theoretical_loss": 3.6113574648088766, + "tokens_seen": 1116995584 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033410230692076226, + "loss": 1.3656, + "theoretical_loss": 3.6113179596163283, + "tokens_seen": 1117126656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340822467402207, + "loss": 1.4188, + "theoretical_loss": 3.611278460356298, + "tokens_seen": 1117257728 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.5503421425819397, + "objective/train/avg_token_score": 0.017878206446766853, + "objective/train/docs_used": 636687, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9106156826019287, + "objective/train/original_loss": 3.8704416751861572, + "objective/train/theoretical_loss": 3.611238967027199, + "objective/train/tokens_used": 1117388800, + "objective/train/unlikelihood_loss": 0.13337932527065277, + "theoretical_loss": 3.611238967027199, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340621865596791, + "loss": 1.4619, + "theoretical_loss": 3.611238967027199, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033404212637913746, + "loss": 1.4005, + "theoretical_loss": 3.6111994796274445, + "tokens_seen": 1117519872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340220661985958, + "loss": 1.3207, + "theoretical_loss": 3.6111599981554496, + "tokens_seen": 1117650944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340020060180542, + "loss": 1.5142, + "theoretical_loss": 3.61112052260963, + "tokens_seen": 1117782016 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033398194583751255, + "loss": 1.4311, + "theoretical_loss": 3.6110810529884003, + "tokens_seen": 1117913088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033396188565697096, + "loss": 1.3071, + "theoretical_loss": 3.6110415892901773, + "tokens_seen": 1118044160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339418254764293, + "loss": 1.452, + "theoretical_loss": 3.611002131513378, + "tokens_seen": 1118175232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339217652958877, + "loss": 1.3651, + "theoretical_loss": 3.6109626796564203, + "tokens_seen": 1118306304 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033390170511534605, + "loss": 1.5606, + "theoretical_loss": 3.6109232337177217, + "tokens_seen": 1118437376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033388164493480447, + "loss": 1.2516, + "theoretical_loss": 3.6108837936957014, + "tokens_seen": 1118568448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033386158475426283, + "loss": 1.4595, + "theoretical_loss": 3.6108443595887785, + "tokens_seen": 1118699520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338415245737212, + "loss": 1.2628, + "theoretical_loss": 3.610804931395373, + "tokens_seen": 1118830592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033382146439317955, + "loss": 1.4303, + "theoretical_loss": 3.6107655091139064, + "tokens_seen": 1118961664 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.4530791640281677, + "objective/train/avg_token_score": 0.014708121307194233, + "objective/train/docs_used": 638066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.467583179473877, + "objective/train/original_loss": 3.8466379642486572, + "objective/train/theoretical_loss": 3.6107458001896564, + "objective/train/tokens_used": 1119027200, + "objective/train/unlikelihood_loss": 0.17593598365783691, + "theoretical_loss": 3.6107458001896564, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338014042126379, + "loss": 1.4011, + "theoretical_loss": 3.6107260927427984, + "tokens_seen": 1119092736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033378134403209633, + "loss": 1.4445, + "theoretical_loss": 3.610686682280472, + "tokens_seen": 1119223808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337612838515547, + "loss": 1.2919, + "theoretical_loss": 3.610647277725349, + "tokens_seen": 1119354880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033374122367101306, + "loss": 1.2293, + "theoretical_loss": 3.610607879075853, + "tokens_seen": 1119485952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337211634904714, + "loss": 1.2719, + "theoretical_loss": 3.6105684863304064, + "tokens_seen": 1119617024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033370110330992983, + "loss": 1.3228, + "theoretical_loss": 3.610529099487435, + "tokens_seen": 1119748096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336810431293882, + "loss": 1.4874, + "theoretical_loss": 3.610489718545362, + "tokens_seen": 1119879168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033366098294884656, + "loss": 1.3188, + "theoretical_loss": 3.610450343502614, + "tokens_seen": 1120010240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336409227683049, + "loss": 1.5029, + "theoretical_loss": 3.6104109743576176, + "tokens_seen": 1120141312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336208625877633, + "loss": 1.4716, + "theoretical_loss": 3.610371611108798, + "tokens_seen": 1120272384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336008024072217, + "loss": 1.2769, + "theoretical_loss": 3.6103322537545828, + "tokens_seen": 1120403456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033358074222668006, + "loss": 1.2424, + "theoretical_loss": 3.6102929022934003, + "tokens_seen": 1120534528 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.20851661264896393, + "objective/train/avg_token_score": 0.029175754636526108, + "objective/train/docs_used": 638756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.695441484451294, + "objective/train/original_loss": 4.805039882659912, + "objective/train/theoretical_loss": 3.610253556723679, + "objective/train/tokens_used": 1120665600, + "objective/train/unlikelihood_loss": 0.0902491956949234, + "theoretical_loss": 3.610253556723679, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335606820461384, + "loss": 1.3666, + "theoretical_loss": 3.610253556723679, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335406218655968, + "loss": 1.2734, + "theoretical_loss": 3.6102142170438474, + "tokens_seen": 1120796672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335205616850552, + "loss": 1.3324, + "theoretical_loss": 3.610174883252336, + "tokens_seen": 1120927744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033350050150451356, + "loss": 1.4886, + "theoretical_loss": 3.610135555347574, + "tokens_seen": 1121058816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334804413239719, + "loss": 1.4013, + "theoretical_loss": 3.610096233327993, + "tokens_seen": 1121189888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334603811434303, + "loss": 1.4189, + "theoretical_loss": 3.6100569171920247, + "tokens_seen": 1121320960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033344032096288865, + "loss": 1.2714, + "theoretical_loss": 3.6100176069381003, + "tokens_seen": 1121452032 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033342026078234707, + "loss": 1.4718, + "theoretical_loss": 3.6099783025646532, + "tokens_seen": 1121583104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033340020060180543, + "loss": 1.2597, + "theoretical_loss": 3.6099390040701165, + "tokens_seen": 1121714176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333801404212638, + "loss": 1.4896, + "theoretical_loss": 3.6098997114529237, + "tokens_seen": 1121845248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033336008024072215, + "loss": 1.3679, + "theoretical_loss": 3.60986042471151, + "tokens_seen": 1121976320 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033334002006018057, + "loss": 1.3377, + "theoretical_loss": 3.609821143844309, + "tokens_seen": 1122107392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033331995987963893, + "loss": 1.3536, + "theoretical_loss": 3.6097818688497583, + "tokens_seen": 1122238464 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.41758307814598083, + "objective/train/avg_token_score": 0.017736399546265602, + "objective/train/docs_used": 639983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4126509428024292, + "objective/train/original_loss": 3.8375165462493896, + "objective/train/theoretical_loss": 3.609762233554238, + "objective/train/tokens_used": 1122304000, + "objective/train/unlikelihood_loss": 0.1596277505159378, + "theoretical_loss": 3.609762233554238, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332998996990973, + "loss": 1.3017, + "theoretical_loss": 3.6097425997262933, + "tokens_seen": 1122369536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033327983951855566, + "loss": 1.385, + "theoretical_loss": 3.6097033364723505, + "tokens_seen": 1122500608 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333259779338014, + "loss": 1.5178, + "theoretical_loss": 3.609664079086368, + "tokens_seen": 1122631680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033323971915747244, + "loss": 1.2876, + "theoretical_loss": 3.6096248275667833, + "tokens_seen": 1122762752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332196589769308, + "loss": 1.5027, + "theoretical_loss": 3.609585581912036, + "tokens_seen": 1122893824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033319959879638916, + "loss": 1.3879, + "theoretical_loss": 3.6095463421205642, + "tokens_seen": 1123024896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331795386158475, + "loss": 1.3541, + "theoretical_loss": 3.609507108190808, + "tokens_seen": 1123155968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033315947843530594, + "loss": 1.4137, + "theoretical_loss": 3.609467880121209, + "tokens_seen": 1123287040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331394182547643, + "loss": 1.3198, + "theoretical_loss": 3.6094286579102066, + "tokens_seen": 1123418112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033311935807422266, + "loss": 1.3443, + "theoretical_loss": 3.6093894415562437, + "tokens_seen": 1123549184 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333099297893681, + "loss": 1.3758, + "theoretical_loss": 3.609350231057762, + "tokens_seen": 1123680256 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033307923771313944, + "loss": 1.547, + "theoretical_loss": 3.609311026413204, + "tokens_seen": 1123811328 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.3256964683532715, + "objective/train/avg_token_score": 0.02644532360136509, + "objective/train/docs_used": 640598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1054093837738037, + "objective/train/original_loss": 3.448108673095703, + "objective/train/theoretical_loss": 3.609271827621014, + "objective/train/tokens_used": 1123942400, + "objective/train/unlikelihood_loss": 0.23476353287696838, + "theoretical_loss": 3.609271827621014, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330591775325978, + "loss": 1.4191, + "theoretical_loss": 3.609271827621014, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033303911735205617, + "loss": 1.4639, + "theoretical_loss": 3.6092326346796355, + "tokens_seen": 1124073472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033301905717151453, + "loss": 1.3827, + "theoretical_loss": 3.6091934475875136, + "tokens_seen": 1124204544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329989969909729, + "loss": 1.5321, + "theoretical_loss": 3.6091542663430927, + "tokens_seen": 1124335616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329789368104313, + "loss": 1.3459, + "theoretical_loss": 3.6091150909448193, + "tokens_seen": 1124466688 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033295887662988967, + "loss": 1.2979, + "theoretical_loss": 3.6090759213911396, + "tokens_seen": 1124597760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033293881644934803, + "loss": 1.3889, + "theoretical_loss": 3.6090367576805007, + "tokens_seen": 1124728832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329187562688064, + "loss": 1.2795, + "theoretical_loss": 3.60899759981135, + "tokens_seen": 1124859904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328986960882648, + "loss": 1.2685, + "theoretical_loss": 3.6089584477821357, + "tokens_seen": 1124990976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328786359077232, + "loss": 1.3102, + "theoretical_loss": 3.608919301591307, + "tokens_seen": 1125122048 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033285857572718154, + "loss": 1.3751, + "theoretical_loss": 3.6088801612373125, + "tokens_seen": 1125253120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328385155466399, + "loss": 1.4986, + "theoretical_loss": 3.608841026718603, + "tokens_seen": 1125384192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033281845536609826, + "loss": 1.3864, + "theoretical_loss": 3.6088018980336285, + "tokens_seen": 1125515264 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.41660556197166443, + "objective/train/avg_token_score": 0.00774718914180994, + "objective/train/docs_used": 641290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.422176480293274, + "objective/train/original_loss": 3.895298480987549, + "objective/train/theoretical_loss": 3.608782335878308, + "objective/train/tokens_used": 1125580800, + "objective/train/unlikelihood_loss": 0.15792806446552277, + "theoretical_loss": 3.608782335878308, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327983951855567, + "loss": 1.4113, + "theoretical_loss": 3.6087627751808404, + "tokens_seen": 1125646336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033277833500501504, + "loss": 1.251, + "theoretical_loss": 3.6087236581586906, + "tokens_seen": 1125777408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327582748244734, + "loss": 1.2804, + "theoretical_loss": 3.608684546965631, + "tokens_seen": 1125908480 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033273821464393176, + "loss": 1.2607, + "theoretical_loss": 3.608645441600115, + "tokens_seen": 1126039552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327181544633902, + "loss": 1.5661, + "theoretical_loss": 3.6086063420605954, + "tokens_seen": 1126170624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033269809428284854, + "loss": 1.4063, + "theoretical_loss": 3.608567248345527, + "tokens_seen": 1126301696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326780341023069, + "loss": 1.4223, + "theoretical_loss": 3.6085281604533646, + "tokens_seen": 1126432768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033265797392176527, + "loss": 1.3791, + "theoretical_loss": 3.608489078382563, + "tokens_seen": 1126563840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033263791374122363, + "loss": 1.4633, + "theoretical_loss": 3.6084500021315784, + "tokens_seen": 1126694912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033261785356068205, + "loss": 1.3777, + "theoretical_loss": 3.6084109316988666, + "tokens_seen": 1126825984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325977933801404, + "loss": 1.4018, + "theoretical_loss": 3.6083718670828864, + "tokens_seen": 1126957056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325777331995988, + "loss": 1.7493, + "theoretical_loss": 3.608332808282093, + "tokens_seen": 1127088128 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.44483137130737305, + "objective/train/avg_token_score": 0.0343501977622509, + "objective/train/docs_used": 642585, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3754688501358032, + "objective/train/original_loss": 3.781877279281616, + "objective/train/theoretical_loss": 3.6082937552949463, + "objective/train/tokens_used": 1127219200, + "objective/train/unlikelihood_loss": 0.11242921650409698, + "theoretical_loss": 3.6082937552949463, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325576730190572, + "loss": 1.4788, + "theoretical_loss": 3.6082937552949463, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325376128385156, + "loss": 1.4012, + "theoretical_loss": 3.608254708119905, + "tokens_seen": 1127350272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033251755265797396, + "loss": 1.4345, + "theoretical_loss": 3.608215666755428, + "tokens_seen": 1127481344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033249749247743233, + "loss": 1.4507, + "theoretical_loss": 3.6081766311999757, + "tokens_seen": 1127612416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324774322968907, + "loss": 1.2792, + "theoretical_loss": 3.6081376014520083, + "tokens_seen": 1127743488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033245737211634905, + "loss": 1.4626, + "theoretical_loss": 3.6080985775099874, + "tokens_seen": 1127874560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033243731193580747, + "loss": 1.4328, + "theoretical_loss": 3.608059559372374, + "tokens_seen": 1128005632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033241725175526583, + "loss": 1.3916, + "theoretical_loss": 3.6080205470376314, + "tokens_seen": 1128136704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323971915747242, + "loss": 1.3522, + "theoretical_loss": 3.607981540504222, + "tokens_seen": 1128267776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033237713139418255, + "loss": 1.3986, + "theoretical_loss": 3.607942539770609, + "tokens_seen": 1128398848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033235707121364097, + "loss": 1.5451, + "theoretical_loss": 3.6079035448352577, + "tokens_seen": 1128529920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033233701103309933, + "loss": 1.367, + "theoretical_loss": 3.6078645556966316, + "tokens_seen": 1128660992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323169508525577, + "loss": 1.3911, + "theoretical_loss": 3.6078255723531965, + "tokens_seen": 1128792064 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.3469574749469757, + "objective/train/avg_token_score": 0.015191564336419106, + "objective/train/docs_used": 643131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0879076719284058, + "objective/train/original_loss": 4.15885591506958, + "objective/train/theoretical_loss": 3.6078060828541956, + "objective/train/tokens_used": 1128857600, + "objective/train/unlikelihood_loss": 0.12423203885555267, + "theoretical_loss": 3.6078060828541956, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033229689067201606, + "loss": 1.4031, + "theoretical_loss": 3.607786594803418, + "tokens_seen": 1128923136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322768304914744, + "loss": 1.429, + "theoretical_loss": 3.6077476230457624, + "tokens_seen": 1129054208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033225677031093284, + "loss": 1.3456, + "theoretical_loss": 3.6077086570786974, + "tokens_seen": 1129185280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322367101303912, + "loss": 1.43, + "theoretical_loss": 3.6076696969006905, + "tokens_seen": 1129316352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033221664994984956, + "loss": 1.5671, + "theoretical_loss": 3.6076307425102088, + "tokens_seen": 1129447424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321965897693079, + "loss": 1.4159, + "theoretical_loss": 3.6075917939057223, + "tokens_seen": 1129578496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033217652958876634, + "loss": 1.3639, + "theoretical_loss": 3.6075528510857, + "tokens_seen": 1129709568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321564694082247, + "loss": 1.3132, + "theoretical_loss": 3.6075139140486114, + "tokens_seen": 1129840640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033213640922768306, + "loss": 1.3442, + "theoretical_loss": 3.6074749827929278, + "tokens_seen": 1129971712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321163490471414, + "loss": 1.333, + "theoretical_loss": 3.6074360573171194, + "tokens_seen": 1130102784 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033209628886659984, + "loss": 1.5145, + "theoretical_loss": 3.6073971376196585, + "tokens_seen": 1130233856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320762286860582, + "loss": 1.4028, + "theoretical_loss": 3.6073582236990176, + "tokens_seen": 1130364928 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.46114370226860046, + "objective/train/avg_token_score": 0.02730853110551834, + "objective/train/docs_used": 644333, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5368561744689941, + "objective/train/original_loss": 3.7062885761260986, + "objective/train/theoretical_loss": 3.607319315553669, + "objective/train/tokens_used": 1130496000, + "objective/train/unlikelihood_loss": 0.20675647258758545, + "theoretical_loss": 3.607319315553669, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033205616850551657, + "loss": 1.2908, + "theoretical_loss": 3.607319315553669, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033203610832497493, + "loss": 1.4627, + "theoretical_loss": 3.607280413182086, + "tokens_seen": 1130627072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320160481444333, + "loss": 1.3168, + "theoretical_loss": 3.607241516582743, + "tokens_seen": 1130758144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319959879638917, + "loss": 1.3252, + "theoretical_loss": 3.607202625754115, + "tokens_seen": 1130889216 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033197592778335007, + "loss": 1.5711, + "theoretical_loss": 3.607163740694676, + "tokens_seen": 1131020288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033195586760280843, + "loss": 1.2618, + "theoretical_loss": 3.6071248614029026, + "tokens_seen": 1131151360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319358074222668, + "loss": 1.4819, + "theoretical_loss": 3.607085987877271, + "tokens_seen": 1131282432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319157472417252, + "loss": 1.3975, + "theoretical_loss": 3.607047120116259, + "tokens_seen": 1131413504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318956870611836, + "loss": 1.388, + "theoretical_loss": 3.607008258118342, + "tokens_seen": 1131544576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033187562688064194, + "loss": 1.4321, + "theoretical_loss": 3.606969401881999, + "tokens_seen": 1131675648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318555667001003, + "loss": 1.279, + "theoretical_loss": 3.6069305514057097, + "tokens_seen": 1131806720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033183550651955866, + "loss": 1.4563, + "theoretical_loss": 3.606891706687952, + "tokens_seen": 1131937792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318154463390171, + "loss": 1.3115, + "theoretical_loss": 3.6068528677272065, + "tokens_seen": 1132068864 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.36589688062667847, + "objective/train/avg_token_score": 0.030686097219586372, + "objective/train/docs_used": 644979, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2991795539855957, + "objective/train/original_loss": 3.922508955001831, + "objective/train/theoretical_loss": 3.6068334504052375, + "objective/train/tokens_used": 1132134400, + "objective/train/unlikelihood_loss": 0.1431983858346939, + "theoretical_loss": 3.6068334504052375, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033179538615847544, + "loss": 1.3516, + "theoretical_loss": 3.6068140345219524, + "tokens_seen": 1132199936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317753259779338, + "loss": 1.3938, + "theoretical_loss": 3.6067752070706725, + "tokens_seen": 1132331008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033175526579739216, + "loss": 1.5087, + "theoretical_loss": 3.606736385371846, + "tokens_seen": 1132462080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317352056168506, + "loss": 1.3337, + "theoretical_loss": 3.6066975694239574, + "tokens_seen": 1132593152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033171514543630894, + "loss": 1.3651, + "theoretical_loss": 3.606658759225488, + "tokens_seen": 1132724224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316950852557673, + "loss": 1.4251, + "theoretical_loss": 3.6066199547749216, + "tokens_seen": 1132855296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033167502507522567, + "loss": 1.3466, + "theoretical_loss": 3.6065811560707415, + "tokens_seen": 1132986368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033165496489468403, + "loss": 1.3023, + "theoretical_loss": 3.606542363111432, + "tokens_seen": 1133117440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033163490471414245, + "loss": 1.2998, + "theoretical_loss": 3.606503575895479, + "tokens_seen": 1133248512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316148445336008, + "loss": 1.311, + "theoretical_loss": 3.6064647944213672, + "tokens_seen": 1133379584 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033159478435305917, + "loss": 1.3473, + "theoretical_loss": 3.6064260186875834, + "tokens_seen": 1133510656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033157472417251753, + "loss": 1.3202, + "theoretical_loss": 3.606387248692614, + "tokens_seen": 1133641728 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.48472627997398376, + "objective/train/avg_token_score": 0.007526305038481951, + "objective/train/docs_used": 646384, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4453260898590088, + "objective/train/original_loss": 3.463900566101074, + "objective/train/theoretical_loss": 3.6063484844349456, + "objective/train/tokens_used": 1133772800, + "objective/train/unlikelihood_loss": 0.17139916121959686, + "theoretical_loss": 3.6063484844349456, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033155466399197595, + "loss": 1.3797, + "theoretical_loss": 3.6063484844349456, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315346038114343, + "loss": 1.373, + "theoretical_loss": 3.606309725913067, + "tokens_seen": 1133903872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315145436308927, + "loss": 1.2913, + "theoretical_loss": 3.6062709731254663, + "tokens_seen": 1134034944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033149448345035104, + "loss": 1.4341, + "theoretical_loss": 3.6062322260706328, + "tokens_seen": 1134166016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314744232698094, + "loss": 1.3335, + "theoretical_loss": 3.606193484747055, + "tokens_seen": 1134297088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314543630892678, + "loss": 1.3007, + "theoretical_loss": 3.606154749153225, + "tokens_seen": 1134428160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314343029087262, + "loss": 1.3161, + "theoretical_loss": 3.606116019287631, + "tokens_seen": 1134559232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033141424272818454, + "loss": 1.4938, + "theoretical_loss": 3.6060772951487667, + "tokens_seen": 1134690304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313941825476429, + "loss": 1.3076, + "theoretical_loss": 3.6060385767351226, + "tokens_seen": 1134821376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313741223671013, + "loss": 1.3373, + "theoretical_loss": 3.6059998640451907, + "tokens_seen": 1134952448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313540621865597, + "loss": 1.2862, + "theoretical_loss": 3.6059611570774655, + "tokens_seen": 1135083520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033133400200601804, + "loss": 1.4025, + "theoretical_loss": 3.6059224558304397, + "tokens_seen": 1135214592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313139418254764, + "loss": 1.2878, + "theoretical_loss": 3.6058837603026075, + "tokens_seen": 1135345664 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.4075024425983429, + "objective/train/avg_token_score": 0.03563220053911209, + "objective/train/docs_used": 647116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.412953495979309, + "objective/train/original_loss": 4.424417018890381, + "objective/train/theoretical_loss": 3.6058644146829186, + "objective/train/tokens_used": 1135411200, + "objective/train/unlikelihood_loss": 0.12008237093687057, + "theoretical_loss": 3.6058644146829186, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033129388164493477, + "loss": 1.5162, + "theoretical_loss": 3.605845070492464, + "tokens_seen": 1135476736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312738214643932, + "loss": 1.3797, + "theoretical_loss": 3.6058063863985033, + "tokens_seen": 1135607808 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033125376128385155, + "loss": 1.2622, + "theoretical_loss": 3.605767708019222, + "tokens_seen": 1135738880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312337011033099, + "loss": 1.2384, + "theoretical_loss": 3.605729035353117, + "tokens_seen": 1135869952 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033121364092276827, + "loss": 1.5037, + "theoretical_loss": 3.6056903683986854, + "tokens_seen": 1136001024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311935807422267, + "loss": 1.3947, + "theoretical_loss": 3.605651707154424, + "tokens_seen": 1136132096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033117352056168505, + "loss": 1.3058, + "theoretical_loss": 3.6056130516188305, + "tokens_seen": 1136263168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311534603811434, + "loss": 1.2929, + "theoretical_loss": 3.605574401790405, + "tokens_seen": 1136394240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033113340020060177, + "loss": 1.3507, + "theoretical_loss": 3.6055357576676457, + "tokens_seen": 1136525312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033111334002006013, + "loss": 1.374, + "theoretical_loss": 3.6054971192490526, + "tokens_seen": 1136656384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033109327983951855, + "loss": 1.4419, + "theoretical_loss": 3.6054584865331263, + "tokens_seen": 1136787456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033107321965897697, + "loss": 1.3611, + "theoretical_loss": 3.605419859518368, + "tokens_seen": 1136918528 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.34439149498939514, + "objective/train/avg_token_score": 0.009137037210166454, + "objective/train/docs_used": 648555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.226346492767334, + "objective/train/original_loss": 4.0563836097717285, + "objective/train/theoretical_loss": 3.605381238203279, + "objective/train/tokens_used": 1137049600, + "objective/train/unlikelihood_loss": 0.14451530575752258, + "theoretical_loss": 3.605381238203279, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033105315947843533, + "loss": 1.3178, + "theoretical_loss": 3.605381238203279, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310330992978937, + "loss": 1.4838, + "theoretical_loss": 3.6053426225863614, + "tokens_seen": 1137180672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310130391173521, + "loss": 1.5533, + "theoretical_loss": 3.6053040126661173, + "tokens_seen": 1137311744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033099297893681047, + "loss": 1.4919, + "theoretical_loss": 3.605265408441051, + "tokens_seen": 1137442816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033097291875626883, + "loss": 1.4413, + "theoretical_loss": 3.6052268099096656, + "tokens_seen": 1137573888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309528585757272, + "loss": 1.3401, + "theoretical_loss": 3.6051882170704657, + "tokens_seen": 1137704960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309327983951856, + "loss": 1.4705, + "theoretical_loss": 3.605149629921956, + "tokens_seen": 1137836032 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330912738214644, + "loss": 1.4627, + "theoretical_loss": 3.6051110484626423, + "tokens_seen": 1137967104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033089267803410234, + "loss": 1.3472, + "theoretical_loss": 3.60507247269103, + "tokens_seen": 1138098176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308726178535607, + "loss": 1.3397, + "theoretical_loss": 3.605033902605627, + "tokens_seen": 1138229248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033085255767301906, + "loss": 1.4746, + "theoretical_loss": 3.6049953382049393, + "tokens_seen": 1138360320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308324974924775, + "loss": 1.5242, + "theoretical_loss": 3.604956779487475, + "tokens_seen": 1138491392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033081243731193584, + "loss": 1.5015, + "theoretical_loss": 3.6049182264517423, + "tokens_seen": 1138622464 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.36265885829925537, + "objective/train/avg_token_score": 0.017149945721030235, + "objective/train/docs_used": 649005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2543026208877563, + "objective/train/original_loss": 4.160501003265381, + "objective/train/theoretical_loss": 3.60489895206406, + "objective/train/tokens_used": 1138688000, + "objective/train/unlikelihood_loss": 0.16913659870624542, + "theoretical_loss": 3.60489895206406, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307923771313942, + "loss": 1.364, + "theoretical_loss": 3.6048796790962507, + "tokens_seen": 1138753536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033077231695085256, + "loss": 1.3649, + "theoretical_loss": 3.604841137419509, + "tokens_seen": 1138884608 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330752256770311, + "loss": 1.3283, + "theoretical_loss": 3.6048026014200274, + "tokens_seen": 1139015680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033073219658976934, + "loss": 1.5893, + "theoretical_loss": 3.6047640710963162, + "tokens_seen": 1139146752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307121364092277, + "loss": 1.5232, + "theoretical_loss": 3.604725546446887, + "tokens_seen": 1139277824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033069207622868607, + "loss": 1.3563, + "theoretical_loss": 3.604687027470251, + "tokens_seen": 1139408896 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033067201604814443, + "loss": 1.4285, + "theoretical_loss": 3.6046485141649214, + "tokens_seen": 1139539968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033065195586760285, + "loss": 1.5382, + "theoretical_loss": 3.60461000652941, + "tokens_seen": 1139671040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306318956870612, + "loss": 1.507, + "theoretical_loss": 3.60457150456223, + "tokens_seen": 1139802112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033061183550651957, + "loss": 1.2929, + "theoretical_loss": 3.604533008261896, + "tokens_seen": 1139933184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033059177532597793, + "loss": 1.3168, + "theoretical_loss": 3.6044945176269225, + "tokens_seen": 1140064256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033057171514543635, + "loss": 1.553, + "theoretical_loss": 3.6044560326558246, + "tokens_seen": 1140195328 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.4087854325771332, + "objective/train/avg_token_score": 0.024077000096440315, + "objective/train/docs_used": 649993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3185713291168213, + "objective/train/original_loss": 3.918933868408203, + "objective/train/theoretical_loss": 3.604417553347117, + "objective/train/tokens_used": 1140326400, + "objective/train/unlikelihood_loss": 0.18836532533168793, + "theoretical_loss": 3.604417553347117, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305516549648947, + "loss": 1.4608, + "theoretical_loss": 3.604417553347117, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305315947843531, + "loss": 1.2414, + "theoretical_loss": 3.6043790796993176, + "tokens_seen": 1140457472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033051153460381144, + "loss": 1.2748, + "theoretical_loss": 3.604340611710941, + "tokens_seen": 1140588544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304914744232698, + "loss": 1.3318, + "theoretical_loss": 3.6043021493805067, + "tokens_seen": 1140719616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304714142427282, + "loss": 1.3325, + "theoretical_loss": 3.604263692706531, + "tokens_seen": 1140850688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304513540621866, + "loss": 1.2664, + "theoretical_loss": 3.604225241687532, + "tokens_seen": 1140981760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033043129388164494, + "loss": 1.258, + "theoretical_loss": 3.60418679632203, + "tokens_seen": 1141112832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304112337011033, + "loss": 1.5374, + "theoretical_loss": 3.604148356608544, + "tokens_seen": 1141243904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303911735205617, + "loss": 1.3098, + "theoretical_loss": 3.604109922545594, + "tokens_seen": 1141374976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303711133400201, + "loss": 1.4003, + "theoretical_loss": 3.6040714941317002, + "tokens_seen": 1141506048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033035105315947844, + "loss": 1.4544, + "theoretical_loss": 3.6040330713653845, + "tokens_seen": 1141637120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303309929789368, + "loss": 1.3765, + "theoretical_loss": 3.6039946542451684, + "tokens_seen": 1141768192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033031093279839517, + "loss": 1.4986, + "theoretical_loss": 3.6039562427695744, + "tokens_seen": 1141899264 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.39400050044059753, + "objective/train/avg_token_score": 0.006121420301496983, + "objective/train/docs_used": 650682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2908718585968018, + "objective/train/original_loss": 3.7027134895324707, + "objective/train/theoretical_loss": 3.603937039148049, + "objective/train/tokens_used": 1141964800, + "objective/train/unlikelihood_loss": 0.21558697521686554, + "theoretical_loss": 3.603937039148049, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302908726178536, + "loss": 1.3852, + "theoretical_loss": 3.603917836937125, + "tokens_seen": 1142030336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033027081243731195, + "loss": 1.408, + "theoretical_loss": 3.603879436746344, + "tokens_seen": 1142161408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302507522567703, + "loss": 1.3544, + "theoretical_loss": 3.603841042195755, + "tokens_seen": 1142292480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033023069207622867, + "loss": 1.487, + "theoretical_loss": 3.603802653283883, + "tokens_seen": 1142423552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302106318956871, + "loss": 1.4398, + "theoretical_loss": 3.6037642700092523, + "tokens_seen": 1142554624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033019057171514545, + "loss": 1.2718, + "theoretical_loss": 3.603725892370389, + "tokens_seen": 1142685696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301705115346038, + "loss": 1.344, + "theoretical_loss": 3.6036875203658196, + "tokens_seen": 1142816768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301504513540622, + "loss": 1.422, + "theoretical_loss": 3.6036491539940707, + "tokens_seen": 1142947840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033013039117352054, + "loss": 1.4985, + "theoretical_loss": 3.603610793253669, + "tokens_seen": 1143078912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033011033099297895, + "loss": 1.3151, + "theoretical_loss": 3.6035724381431433, + "tokens_seen": 1143209984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300902708124373, + "loss": 1.4742, + "theoretical_loss": 3.6035340886610214, + "tokens_seen": 1143341056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300702106318957, + "loss": 1.2421, + "theoretical_loss": 3.6034957448058327, + "tokens_seen": 1143472128 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.27266618609428406, + "objective/train/avg_token_score": 0.019020676612854004, + "objective/train/docs_used": 652177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8491913080215454, + "objective/train/original_loss": 4.16588830947876, + "objective/train/theoretical_loss": 3.603457406576106, + "objective/train/tokens_used": 1143603200, + "objective/train/unlikelihood_loss": 0.19579684734344482, + "theoretical_loss": 3.603457406576106, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033005015045135404, + "loss": 1.2819, + "theoretical_loss": 3.603457406576106, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033003009027081245, + "loss": 1.3306, + "theoretical_loss": 3.6034190739703718, + "tokens_seen": 1143734272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300100300902708, + "loss": 1.4993, + "theoretical_loss": 3.6033807469871615, + "tokens_seen": 1143865344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299899699097292, + "loss": 1.3539, + "theoretical_loss": 3.603342425625005, + "tokens_seen": 1143996416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032996990972918754, + "loss": 1.3742, + "theoretical_loss": 3.6033041098824343, + "tokens_seen": 1144127488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299498495486459, + "loss": 1.4222, + "theoretical_loss": 3.6032657997579824, + "tokens_seen": 1144258560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299297893681043, + "loss": 1.2215, + "theoretical_loss": 3.6032274952501817, + "tokens_seen": 1144389632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299097291875627, + "loss": 1.36, + "theoretical_loss": 3.6031891963575653, + "tokens_seen": 1144520704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032988966900702104, + "loss": 1.5195, + "theoretical_loss": 3.6031509030786673, + "tokens_seen": 1144651776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298696088264794, + "loss": 1.2894, + "theoretical_loss": 3.603112615412022, + "tokens_seen": 1144782848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298495486459378, + "loss": 1.4149, + "theoretical_loss": 3.6030743333561652, + "tokens_seen": 1144913920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298294884653962, + "loss": 1.338, + "theoretical_loss": 3.603036056909632, + "tokens_seen": 1145044992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032980942828485455, + "loss": 1.2002, + "theoretical_loss": 3.6029977860709588, + "tokens_seen": 1145176064 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.5810117125511169, + "objective/train/avg_token_score": 0.003155706450343132, + "objective/train/docs_used": 652765, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7120308876037598, + "objective/train/original_loss": 3.1839566230773926, + "objective/train/theoretical_loss": 3.6029786527541123, + "objective/train/tokens_used": 1145241600, + "objective/train/unlikelihood_loss": 0.16666950285434723, + "theoretical_loss": 3.6029786527541123, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297893681043129, + "loss": 1.4873, + "theoretical_loss": 3.6029595208386818, + "tokens_seen": 1145307136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297693079237713, + "loss": 1.4267, + "theoretical_loss": 3.602921261211338, + "tokens_seen": 1145438208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297492477432297, + "loss": 1.4317, + "theoretical_loss": 3.602883007187466, + "tokens_seen": 1145569280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032972918756268805, + "loss": 1.4727, + "theoretical_loss": 3.6028447587656034, + "tokens_seen": 1145700352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297091273821464, + "loss": 1.4463, + "theoretical_loss": 3.60280651594429, + "tokens_seen": 1145831424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296890672016048, + "loss": 1.5297, + "theoretical_loss": 3.6027682787220643, + "tokens_seen": 1145962496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296690070210632, + "loss": 1.4982, + "theoretical_loss": 3.602730047097467, + "tokens_seen": 1146093568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032964894684052155, + "loss": 1.3373, + "theoretical_loss": 3.602691821069038, + "tokens_seen": 1146224640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296288866599799, + "loss": 1.4221, + "theoretical_loss": 3.602653600635319, + "tokens_seen": 1146355712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296088264794383, + "loss": 1.416, + "theoretical_loss": 3.602615385794851, + "tokens_seen": 1146486784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295887662988967, + "loss": 1.3415, + "theoretical_loss": 3.6025771765461765, + "tokens_seen": 1146617856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295687061183551, + "loss": 1.3726, + "theoretical_loss": 3.602538972887838, + "tokens_seen": 1146748928 + }, + { + "debugging/length": 560.2222222222222, + "debugging/num_segments": 9, + "epoch": 0.35, + "objective/train/aligned_token_count": 0.5686705708503723, + "objective/train/avg_token_score": 0.003827182576060295, + "objective/train/docs_used": 652965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0544469356536865, + "objective/train/original_loss": 4.11958646774292, + "objective/train/theoretical_loss": 3.602500774818379, + "objective/train/tokens_used": 1146880000, + "objective/train/unlikelihood_loss": 0.10806860029697418, + "theoretical_loss": 3.602500774818379, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295486459378135, + "loss": 1.4812, + "theoretical_loss": 3.602500774818379, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032952858575727184, + "loss": 1.3569, + "theoretical_loss": 3.6024625823363428, + "tokens_seen": 1147011072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295085255767302, + "loss": 1.383, + "theoretical_loss": 3.602424395440275, + "tokens_seen": 1147142144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294884653961886, + "loss": 1.5622, + "theoretical_loss": 3.6023862141287193, + "tokens_seen": 1147273216 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329468405215647, + "loss": 1.5259, + "theoretical_loss": 3.602348038400221, + "tokens_seen": 1147404288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032944834503510534, + "loss": 1.4537, + "theoretical_loss": 3.602309868253327, + "tokens_seen": 1147535360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294282848545637, + "loss": 1.5375, + "theoretical_loss": 3.6022717036865832, + "tokens_seen": 1147666432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294082246740221, + "loss": 1.4867, + "theoretical_loss": 3.602233544698537, + "tokens_seen": 1147797504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293881644934805, + "loss": 1.5231, + "theoretical_loss": 3.602195391287736, + "tokens_seen": 1147928576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032936810431293884, + "loss": 1.4683, + "theoretical_loss": 3.6021572434527283, + "tokens_seen": 1148059648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293480441323972, + "loss": 1.4991, + "theoretical_loss": 3.6021191011920624, + "tokens_seen": 1148190720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032932798395185557, + "loss": 1.3873, + "theoretical_loss": 3.602080964504288, + "tokens_seen": 1148321792 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329307923771314, + "loss": 1.4828, + "theoretical_loss": 3.602042833387954, + "tokens_seen": 1148452864 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.5603005886077881, + "objective/train/avg_token_score": 0.011395683512091637, + "objective/train/docs_used": 654364, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.828805923461914, + "objective/train/original_loss": 3.4743967056274414, + "objective/train/theoretical_loss": 3.602023769918625, + "objective/train/tokens_used": 1148518400, + "objective/train/unlikelihood_loss": 0.1928052008152008, + "theoretical_loss": 3.602023769918625, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032928786359077235, + "loss": 1.4438, + "theoretical_loss": 3.602004707841612, + "tokens_seen": 1148583936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292678034102307, + "loss": 1.364, + "theoretical_loss": 3.601966587863812, + "tokens_seen": 1148715008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032924774322968907, + "loss": 1.5645, + "theoretical_loss": 3.6019284734531056, + "tokens_seen": 1148846080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292276830491475, + "loss": 1.252, + "theoretical_loss": 3.6018903646080447, + "tokens_seen": 1148977152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032920762286860585, + "loss": 1.5375, + "theoretical_loss": 3.601852261327182, + "tokens_seen": 1149108224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291875626880642, + "loss": 1.3331, + "theoretical_loss": 3.601814163609071, + "tokens_seen": 1149239296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291675025075226, + "loss": 1.4886, + "theoretical_loss": 3.6017760714522637, + "tokens_seen": 1149370368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032914744232698094, + "loss": 1.4593, + "theoretical_loss": 3.6017379848553164, + "tokens_seen": 1149501440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032912738214643935, + "loss": 1.2852, + "theoretical_loss": 3.6016999038167814, + "tokens_seen": 1149632512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291073219658977, + "loss": 1.2064, + "theoretical_loss": 3.6016618283352164, + "tokens_seen": 1149763584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290872617853561, + "loss": 1.4393, + "theoretical_loss": 3.6016237584091755, + "tokens_seen": 1149894656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032906720160481444, + "loss": 1.5048, + "theoretical_loss": 3.601585694037215, + "tokens_seen": 1150025728 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.4813660681247711, + "objective/train/avg_token_score": 0.008018618449568748, + "objective/train/docs_used": 654951, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3912237882614136, + "objective/train/original_loss": 4.003458499908447, + "objective/train/theoretical_loss": 3.601547635217892, + "objective/train/tokens_used": 1150156800, + "objective/train/unlikelihood_loss": 0.10658945143222809, + "theoretical_loss": 3.601547635217892, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032904714142427286, + "loss": 1.4276, + "theoretical_loss": 3.601547635217892, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290270812437312, + "loss": 1.5137, + "theoretical_loss": 3.601509581949765, + "tokens_seen": 1150287872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290070210631896, + "loss": 1.4893, + "theoretical_loss": 3.60147153423139, + "tokens_seen": 1150418944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032898696088264794, + "loss": 1.3876, + "theoretical_loss": 3.601433492061326, + "tokens_seen": 1150550016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289669007021063, + "loss": 1.3378, + "theoretical_loss": 3.6013954554381336, + "tokens_seen": 1150681088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289468405215647, + "loss": 1.4624, + "theoretical_loss": 3.60135742436037, + "tokens_seen": 1150812160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289267803410231, + "loss": 1.4537, + "theoretical_loss": 3.601319398826597, + "tokens_seen": 1150943232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032890672016048145, + "loss": 1.3592, + "theoretical_loss": 3.601281378835374, + "tokens_seen": 1151074304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288866599799398, + "loss": 1.3648, + "theoretical_loss": 3.6012433643852626, + "tokens_seen": 1151205376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288665997993982, + "loss": 1.3785, + "theoretical_loss": 3.601205355474825, + "tokens_seen": 1151336448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288465396188566, + "loss": 1.2257, + "theoretical_loss": 3.601167352102623, + "tokens_seen": 1151467520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032882647943831495, + "loss": 1.3454, + "theoretical_loss": 3.601129354267219, + "tokens_seen": 1151598592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288064192577733, + "loss": 1.389, + "theoretical_loss": 3.6010913619671765, + "tokens_seen": 1151729664 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.3395650088787079, + "objective/train/avg_token_score": 0.015624585561454296, + "objective/train/docs_used": 656079, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.025367021560669, + "objective/train/original_loss": 3.85591983795166, + "objective/train/theoretical_loss": 3.601072367892467, + "objective/train/tokens_used": 1151795200, + "objective/train/unlikelihood_loss": 0.18545204401016235, + "theoretical_loss": 3.601072367892467, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287863590772317, + "loss": 1.4114, + "theoretical_loss": 3.6010533752010594, + "tokens_seen": 1151860736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287662988966901, + "loss": 1.3927, + "theoretical_loss": 3.6010153939674323, + "tokens_seen": 1151991808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032874623871614845, + "loss": 1.5263, + "theoretical_loss": 3.6009774182648595, + "tokens_seen": 1152122880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287261785356068, + "loss": 1.3304, + "theoretical_loss": 3.600939448091907, + "tokens_seen": 1152253952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287061183550652, + "loss": 1.503, + "theoretical_loss": 3.6009014834471404, + "tokens_seen": 1152385024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286860581745236, + "loss": 1.3751, + "theoretical_loss": 3.6008635243291263, + "tokens_seen": 1152516096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032866599799398195, + "loss": 1.3634, + "theoretical_loss": 3.6008255707364323, + "tokens_seen": 1152647168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286459378134403, + "loss": 1.4007, + "theoretical_loss": 3.6007876226676245, + "tokens_seen": 1152778240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286258776328987, + "loss": 1.504, + "theoretical_loss": 3.6007496801212726, + "tokens_seen": 1152909312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286058174523571, + "loss": 1.4324, + "theoretical_loss": 3.6007117430959443, + "tokens_seen": 1153040384 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032858575727181546, + "loss": 1.5729, + "theoretical_loss": 3.6006738115902097, + "tokens_seen": 1153171456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285656970912738, + "loss": 1.4454, + "theoretical_loss": 3.600635885602637, + "tokens_seen": 1153302528 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.32869014143943787, + "objective/train/avg_token_score": 0.007234439253807068, + "objective/train/docs_used": 656588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1612753868103027, + "objective/train/original_loss": 3.881105422973633, + "objective/train/theoretical_loss": 3.6005979651317976, + "objective/train/tokens_used": 1153433600, + "objective/train/unlikelihood_loss": 0.1826450377702713, + "theoretical_loss": 3.6005979651317976, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285456369107322, + "loss": 1.254, + "theoretical_loss": 3.6005979651317976, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032852557673019054, + "loss": 1.4248, + "theoretical_loss": 3.600560050176262, + "tokens_seen": 1153564672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032850551654964896, + "loss": 1.5175, + "theoretical_loss": 3.6005221407346015, + "tokens_seen": 1153695744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284854563691073, + "loss": 1.2594, + "theoretical_loss": 3.6004842368053875, + "tokens_seen": 1153826816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284653961885657, + "loss": 1.505, + "theoretical_loss": 3.6004463383871927, + "tokens_seen": 1153957888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032844533600802405, + "loss": 1.4441, + "theoretical_loss": 3.6004084454785907, + "tokens_seen": 1154088960 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032842527582748246, + "loss": 1.5432, + "theoretical_loss": 3.600370558078154, + "tokens_seen": 1154220032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284052156469408, + "loss": 1.271, + "theoretical_loss": 3.6003326761844567, + "tokens_seen": 1154351104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283851554663992, + "loss": 1.4043, + "theoretical_loss": 3.6002947997960737, + "tokens_seen": 1154482176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032836509528585755, + "loss": 1.213, + "theoretical_loss": 3.6002569289115796, + "tokens_seen": 1154613248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283450351053159, + "loss": 1.268, + "theoretical_loss": 3.6002190635295497, + "tokens_seen": 1154744320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032832497492477433, + "loss": 1.4685, + "theoretical_loss": 3.6001812036485608, + "tokens_seen": 1154875392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283049147442327, + "loss": 1.2009, + "theoretical_loss": 3.6001433492671895, + "tokens_seen": 1155006464 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.40163734555244446, + "objective/train/avg_token_score": 0.03616270795464516, + "objective/train/docs_used": 657761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.444759726524353, + "objective/train/original_loss": 4.009532451629639, + "objective/train/theoretical_loss": 3.600124424138415, + "objective/train/tokens_used": 1155072000, + "objective/train/unlikelihood_loss": 0.16049516201019287, + "theoretical_loss": 3.600124424138415, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032828485456369105, + "loss": 1.289, + "theoretical_loss": 3.600105500384012, + "tokens_seen": 1155137536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282647943831494, + "loss": 1.4874, + "theoretical_loss": 3.600067656997607, + "tokens_seen": 1155268608 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032824473420260783, + "loss": 1.2626, + "theoretical_loss": 3.6000298191065525, + "tokens_seen": 1155399680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282246740220662, + "loss": 1.582, + "theoretical_loss": 3.5999919867094268, + "tokens_seen": 1155530752 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032820461384152456, + "loss": 1.4808, + "theoretical_loss": 3.5999541598048097, + "tokens_seen": 1155661824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281845536609829, + "loss": 1.3898, + "theoretical_loss": 3.59991633839128, + "tokens_seen": 1155792896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281644934804413, + "loss": 1.5141, + "theoretical_loss": 3.599878522467419, + "tokens_seen": 1155923968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281444332998997, + "loss": 1.4076, + "theoretical_loss": 3.599840712031807, + "tokens_seen": 1156055040 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032812437311935806, + "loss": 1.3184, + "theoretical_loss": 3.5998029070830264, + "tokens_seen": 1156186112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281043129388164, + "loss": 1.4192, + "theoretical_loss": 3.5997651076196577, + "tokens_seen": 1156317184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280842527582748, + "loss": 1.3869, + "theoretical_loss": 3.599727313640284, + "tokens_seen": 1156448256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032806419257773326, + "loss": 1.4881, + "theoretical_loss": 3.599689525143489, + "tokens_seen": 1156579328 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.28934505581855774, + "objective/train/avg_token_score": 0.008428278379142284, + "objective/train/docs_used": 658495, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9790442585945129, + "objective/train/original_loss": 3.81870436668396, + "objective/train/theoretical_loss": 3.599651742127855, + "objective/train/tokens_used": 1156710400, + "objective/train/unlikelihood_loss": 0.2005680501461029, + "theoretical_loss": 3.599651742127855, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280441323971916, + "loss": 1.3213, + "theoretical_loss": 3.599651742127855, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032802407221665, + "loss": 1.3618, + "theoretical_loss": 3.5996139645919656, + "tokens_seen": 1156841472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032800401203610834, + "loss": 1.5197, + "theoretical_loss": 3.599576192534407, + "tokens_seen": 1156972544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279839518555667, + "loss": 1.3433, + "theoretical_loss": 3.599538425953763, + "tokens_seen": 1157103616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279638916750251, + "loss": 1.3392, + "theoretical_loss": 3.5995006648486196, + "tokens_seen": 1157234688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279438314944835, + "loss": 1.4527, + "theoretical_loss": 3.5994629092175625, + "tokens_seen": 1157365760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032792377131394185, + "loss": 1.3217, + "theoretical_loss": 3.5994251590591793, + "tokens_seen": 1157496832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279037111334002, + "loss": 1.3915, + "theoretical_loss": 3.5993874143720563, + "tokens_seen": 1157627904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278836509528586, + "loss": 1.2616, + "theoretical_loss": 3.5993496751547815, + "tokens_seen": 1157758976 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327863590772317, + "loss": 1.368, + "theoretical_loss": 3.5993119414059427, + "tokens_seen": 1157890048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032784353059177535, + "loss": 1.3548, + "theoretical_loss": 3.5992742131241293, + "tokens_seen": 1158021120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278234704112337, + "loss": 1.372, + "theoretical_loss": 3.5992364903079306, + "tokens_seen": 1158152192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032780341023069213, + "loss": 1.3702, + "theoretical_loss": 3.599198772955936, + "tokens_seen": 1158283264 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.4720185697078705, + "objective/train/avg_token_score": 0.006147913169115782, + "objective/train/docs_used": 659118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4008657932281494, + "objective/train/original_loss": 3.365294933319092, + "objective/train/theoretical_loss": 3.5991799163285743, + "objective/train/tokens_used": 1158348800, + "objective/train/unlikelihood_loss": 0.18335799872875214, + "theoretical_loss": 3.5991799163285743, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277833500501505, + "loss": 1.2946, + "theoretical_loss": 3.5991610610667353, + "tokens_seen": 1158414336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032776328986960885, + "loss": 1.3357, + "theoretical_loss": 3.5991233546389205, + "tokens_seen": 1158545408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277432296890672, + "loss": 1.3164, + "theoretical_loss": 3.599085653671082, + "tokens_seen": 1158676480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277231695085256, + "loss": 1.3532, + "theoretical_loss": 3.5990479581618118, + "tokens_seen": 1158807552 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327703109327984, + "loss": 1.3735, + "theoretical_loss": 3.5990102681097027, + "tokens_seen": 1158938624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032768304914744235, + "loss": 1.3046, + "theoretical_loss": 3.5989725835133473, + "tokens_seen": 1159069696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276629889669007, + "loss": 1.4004, + "theoretical_loss": 3.5989349043713394, + "tokens_seen": 1159200768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276429287863591, + "loss": 1.3792, + "theoretical_loss": 3.5988972306822724, + "tokens_seen": 1159331840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276228686058175, + "loss": 1.4462, + "theoretical_loss": 3.598859562444741, + "tokens_seen": 1159462912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032760280842527586, + "loss": 1.3668, + "theoretical_loss": 3.5988218996573407, + "tokens_seen": 1159593984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275827482447342, + "loss": 1.3707, + "theoretical_loss": 3.5987842423186662, + "tokens_seen": 1159725056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275626880641926, + "loss": 1.3438, + "theoretical_loss": 3.5987465904273144, + "tokens_seen": 1159856128 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.43933284282684326, + "objective/train/avg_token_score": 0.03916909545660019, + "objective/train/docs_used": 660623, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.47878098487854, + "objective/train/original_loss": 4.079814434051514, + "objective/train/theoretical_loss": 3.5987089439818805, + "objective/train/tokens_used": 1159987200, + "objective/train/unlikelihood_loss": 0.11045675724744797, + "theoretical_loss": 3.5987089439818805, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032754262788365094, + "loss": 1.1975, + "theoretical_loss": 3.5987089439818805, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032752256770310936, + "loss": 1.3786, + "theoretical_loss": 3.5986713029809634, + "tokens_seen": 1160118272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275025075225677, + "loss": 1.1812, + "theoretical_loss": 3.59863366742316, + "tokens_seen": 1160249344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274824473420261, + "loss": 1.391, + "theoretical_loss": 3.5985960373070673, + "tokens_seen": 1160380416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032746238716148445, + "loss": 1.1809, + "theoretical_loss": 3.5985584126312853, + "tokens_seen": 1160511488 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032744232698094286, + "loss": 1.2908, + "theoretical_loss": 3.598520793394413, + "tokens_seen": 1160642560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274222668004012, + "loss": 1.397, + "theoretical_loss": 3.59848317959505, + "tokens_seen": 1160773632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274022066198596, + "loss": 1.2586, + "theoretical_loss": 3.5984455712317955, + "tokens_seen": 1160904704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032738214643931795, + "loss": 1.4198, + "theoretical_loss": 3.5984079683032517, + "tokens_seen": 1161035776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273620862587763, + "loss": 1.3735, + "theoretical_loss": 3.598370370808019, + "tokens_seen": 1161166848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032734202607823473, + "loss": 1.3227, + "theoretical_loss": 3.598332778744699, + "tokens_seen": 1161297920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273219658976931, + "loss": 1.3128, + "theoretical_loss": 3.5982951921118946, + "tokens_seen": 1161428992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032730190571715145, + "loss": 1.1574, + "theoretical_loss": 3.5982576109082083, + "tokens_seen": 1161560064 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.47006353735923767, + "objective/train/avg_token_score": 0.01880132220685482, + "objective/train/docs_used": 661405, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5821563005447388, + "objective/train/original_loss": 3.9629170894622803, + "objective/train/theoretical_loss": 3.598238822341848, + "objective/train/tokens_used": 1161625600, + "objective/train/unlikelihood_loss": 0.12642356753349304, + "theoretical_loss": 3.598238822341848, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272818455366098, + "loss": 1.492, + "theoretical_loss": 3.5982200351322433, + "tokens_seen": 1161691136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032726178535606823, + "loss": 1.3377, + "theoretical_loss": 3.598182464782604, + "tokens_seen": 1161822208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272417251755266, + "loss": 1.4637, + "theoretical_loss": 3.5981448998578935, + "tokens_seen": 1161953280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032722166499498496, + "loss": 1.4001, + "theoretical_loss": 3.5981073403567176, + "tokens_seen": 1162084352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272016048144433, + "loss": 1.4742, + "theoretical_loss": 3.598069786277682, + "tokens_seen": 1162215424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271815446339017, + "loss": 1.5337, + "theoretical_loss": 3.5980322376193916, + "tokens_seen": 1162346496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271614844533601, + "loss": 1.3945, + "theoretical_loss": 3.5979946943804535, + "tokens_seen": 1162477568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032714142427281846, + "loss": 1.3481, + "theoretical_loss": 3.597957156559475, + "tokens_seen": 1162608640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271213640922768, + "loss": 1.6015, + "theoretical_loss": 3.597919624155062, + "tokens_seen": 1162739712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271013039117352, + "loss": 1.5089, + "theoretical_loss": 3.5978820971658236, + "tokens_seen": 1162870784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270812437311936, + "loss": 1.4797, + "theoretical_loss": 3.5978445755903685, + "tokens_seen": 1163001856 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032706118355065196, + "loss": 1.4358, + "theoretical_loss": 3.5978070594273044, + "tokens_seen": 1163132928 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.5210165977478027, + "objective/train/avg_token_score": 0.011428765021264553, + "objective/train/docs_used": 662532, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.658143401145935, + "objective/train/original_loss": 3.818161964416504, + "objective/train/theoretical_loss": 3.5977695486752426, + "objective/train/tokens_used": 1163264000, + "objective/train/unlikelihood_loss": 0.11492922157049179, + "theoretical_loss": 3.5977695486752426, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270411233701103, + "loss": 1.4927, + "theoretical_loss": 3.5977695486752426, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270210631895687, + "loss": 1.3985, + "theoretical_loss": 3.5977320433327913, + "tokens_seen": 1163395072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032700100300902705, + "loss": 1.4435, + "theoretical_loss": 3.5976945433985623, + "tokens_seen": 1163526144 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032698094282848547, + "loss": 1.5127, + "theoretical_loss": 3.597657048871166, + "tokens_seen": 1163657216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032696088264794383, + "loss": 1.4045, + "theoretical_loss": 3.597619559749214, + "tokens_seen": 1163788288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269408224674022, + "loss": 1.6285, + "theoretical_loss": 3.597582076031319, + "tokens_seen": 1163919360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032692076228686055, + "loss": 1.4272, + "theoretical_loss": 3.5975445977160927, + "tokens_seen": 1164050432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032690070210631897, + "loss": 1.4396, + "theoretical_loss": 3.597507124802149, + "tokens_seen": 1164181504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032688064192577733, + "loss": 1.4871, + "theoretical_loss": 3.5974696572881006, + "tokens_seen": 1164312576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268605817452357, + "loss": 1.3434, + "theoretical_loss": 3.597432195172562, + "tokens_seen": 1164443648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032684052156469406, + "loss": 1.4663, + "theoretical_loss": 3.597394738454148, + "tokens_seen": 1164574720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268204613841524, + "loss": 1.606, + "theoretical_loss": 3.597357287131474, + "tokens_seen": 1164705792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032680040120361084, + "loss": 1.4317, + "theoretical_loss": 3.5973198412031544, + "tokens_seen": 1164836864 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.48051074147224426, + "objective/train/avg_token_score": 0.005870673805475235, + "objective/train/docs_used": 663185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.602812647819519, + "objective/train/original_loss": 3.9566640853881836, + "objective/train/theoretical_loss": 3.5973011202614456, + "objective/train/tokens_used": 1164902400, + "objective/train/unlikelihood_loss": 0.15959294140338898, + "theoretical_loss": 3.5973011202614456, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267803410230692, + "loss": 1.6522, + "theoretical_loss": 3.5972824006678064, + "tokens_seen": 1164967936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032676028084252756, + "loss": 1.4825, + "theoretical_loss": 3.5972449655240473, + "tokens_seen": 1165099008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267402206619859, + "loss": 1.4224, + "theoretical_loss": 3.597207535770493, + "tokens_seen": 1165230080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032672016048144434, + "loss": 1.4185, + "theoretical_loss": 3.5971701114057613, + "tokens_seen": 1165361152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267001003009027, + "loss": 1.4216, + "theoretical_loss": 3.5971326924284703, + "tokens_seen": 1165492224 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032668004012036106, + "loss": 1.3732, + "theoretical_loss": 3.59709527883724, + "tokens_seen": 1165623296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266599799398194, + "loss": 1.5118, + "theoretical_loss": 3.5970578706306884, + "tokens_seen": 1165754368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032663991975927784, + "loss": 1.444, + "theoretical_loss": 3.597020467807436, + "tokens_seen": 1165885440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266198595787362, + "loss": 1.3431, + "theoretical_loss": 3.596983070366102, + "tokens_seen": 1166016512 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032659979939819457, + "loss": 1.4424, + "theoretical_loss": 3.5969456783053078, + "tokens_seen": 1166147584 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032657973921765293, + "loss": 1.3055, + "theoretical_loss": 3.5969082916236745, + "tokens_seen": 1166278656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032655967903711135, + "loss": 1.3283, + "theoretical_loss": 3.596870910319824, + "tokens_seen": 1166409728 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.33791545033454895, + "objective/train/avg_token_score": 0.048525091260671616, + "objective/train/docs_used": 664585, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0477665662765503, + "objective/train/original_loss": 4.1057939529418945, + "objective/train/theoretical_loss": 3.596833534392379, + "objective/train/tokens_used": 1166540800, + "objective/train/unlikelihood_loss": 0.19251859188079834, + "theoretical_loss": 3.596833534392379, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032653961885656976, + "loss": 1.4171, + "theoretical_loss": 3.596833534392379, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265195586760281, + "loss": 1.5871, + "theoretical_loss": 3.5967961638399615, + "tokens_seen": 1166671872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264994984954865, + "loss": 1.2915, + "theoretical_loss": 3.596758798661195, + "tokens_seen": 1166802944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032647943831494485, + "loss": 1.32, + "theoretical_loss": 3.5967214388547033, + "tokens_seen": 1166934016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032645937813440326, + "loss": 1.5388, + "theoretical_loss": 3.5966840844191106, + "tokens_seen": 1167065088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264393179538616, + "loss": 1.416, + "theoretical_loss": 3.5966467353530422, + "tokens_seen": 1167196160 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032641925777332, + "loss": 1.3806, + "theoretical_loss": 3.5966093916551225, + "tokens_seen": 1167327232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032639919759277835, + "loss": 1.2759, + "theoretical_loss": 3.5965720533239782, + "tokens_seen": 1167458304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263791374122367, + "loss": 1.3356, + "theoretical_loss": 3.5965347203582354, + "tokens_seen": 1167589376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032635907723169513, + "loss": 1.3701, + "theoretical_loss": 3.596497392756521, + "tokens_seen": 1167720448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263390170511535, + "loss": 1.2621, + "theoretical_loss": 3.5964600705174616, + "tokens_seen": 1167851520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032631895687061185, + "loss": 1.312, + "theoretical_loss": 3.5964227536396858, + "tokens_seen": 1167982592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262988966900702, + "loss": 1.4051, + "theoretical_loss": 3.5963854421218215, + "tokens_seen": 1168113664 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.4045087993144989, + "objective/train/avg_token_score": 0.012721749022603035, + "objective/train/docs_used": 665171, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.426487922668457, + "objective/train/original_loss": 3.873084783554077, + "objective/train/theoretical_loss": 3.596366788372428, + "objective/train/tokens_used": 1168179200, + "objective/train/unlikelihood_loss": 0.16045056283473969, + "theoretical_loss": 3.596366788372428, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032627883650952863, + "loss": 1.3704, + "theoretical_loss": 3.596348135962498, + "tokens_seen": 1168244736 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326258776328987, + "loss": 1.4697, + "theoretical_loss": 3.5963108351603443, + "tokens_seen": 1168375808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032623871614844536, + "loss": 1.4187, + "theoretical_loss": 3.596273539713991, + "tokens_seen": 1168506880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262186559679037, + "loss": 1.3305, + "theoretical_loss": 3.5962362496220672, + "tokens_seen": 1168637952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261985957873621, + "loss": 1.4781, + "theoretical_loss": 3.5961989648832042, + "tokens_seen": 1168769024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261785356068205, + "loss": 1.4248, + "theoretical_loss": 3.5961616854960337, + "tokens_seen": 1168900096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032615847542627886, + "loss": 1.3436, + "theoretical_loss": 3.5961244114591877, + "tokens_seen": 1169031168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261384152457372, + "loss": 1.3994, + "theoretical_loss": 3.596087142771298, + "tokens_seen": 1169162240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261183550651956, + "loss": 1.1759, + "theoretical_loss": 3.596049879430998, + "tokens_seen": 1169293312 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326098294884654, + "loss": 1.3932, + "theoretical_loss": 3.5960126214369206, + "tokens_seen": 1169424384 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032607823470411236, + "loss": 1.3501, + "theoretical_loss": 3.5959753687877005, + "tokens_seen": 1169555456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260581745235707, + "loss": 1.44, + "theoretical_loss": 3.595938121481971, + "tokens_seen": 1169686528 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.36693549156188965, + "objective/train/avg_token_score": 0.03444541245698929, + "objective/train/docs_used": 666471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2079639434814453, + "objective/train/original_loss": 4.231860637664795, + "objective/train/theoretical_loss": 3.595900879518368, + "objective/train/tokens_used": 1169817600, + "objective/train/unlikelihood_loss": 0.13814839720726013, + "theoretical_loss": 3.595900879518368, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260381143430291, + "loss": 1.4271, + "theoretical_loss": 3.595900879518368, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032601805416248745, + "loss": 1.2721, + "theoretical_loss": 3.5958636428955257, + "tokens_seen": 1169948672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032599799398194587, + "loss": 1.3356, + "theoretical_loss": 3.595826411612081, + "tokens_seen": 1170079744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032597793380140423, + "loss": 1.4391, + "theoretical_loss": 3.59578918566667, + "tokens_seen": 1170210816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259578736208626, + "loss": 1.528, + "theoretical_loss": 3.5957519650579295, + "tokens_seen": 1170341888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032593781344032095, + "loss": 1.5294, + "theoretical_loss": 3.595714749784497, + "tokens_seen": 1170472960 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032591775325977937, + "loss": 1.4962, + "theoretical_loss": 3.5956775398450103, + "tokens_seen": 1170604032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032589769307923773, + "loss": 1.4469, + "theoretical_loss": 3.595640335238108, + "tokens_seen": 1170735104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258776328986961, + "loss": 1.5969, + "theoretical_loss": 3.5956031359624285, + "tokens_seen": 1170866176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032585757271815446, + "loss": 1.3846, + "theoretical_loss": 3.595565942016612, + "tokens_seen": 1170997248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258375125376128, + "loss": 1.3143, + "theoretical_loss": 3.5955287533992975, + "tokens_seen": 1171128320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032581745235707124, + "loss": 1.3443, + "theoretical_loss": 3.5954915701091257, + "tokens_seen": 1171259392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003257973921765296, + "loss": 1.2534, + "theoretical_loss": 3.5954543921447377, + "tokens_seen": 1171390464 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.45668378472328186, + "objective/train/avg_token_score": 0.01738378405570984, + "objective/train/docs_used": 666886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4930630922317505, + "objective/train/original_loss": 3.581326484680176, + "objective/train/theoretical_loss": 3.595435805159288, + "objective/train/tokens_used": 1171456000, + "objective/train/unlikelihood_loss": 0.16848526895046234, + "theoretical_loss": 3.595435805159288, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032577733199598796, + "loss": 1.4286, + "theoretical_loss": 3.5954172195047747, + "tokens_seen": 1171521536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257572718154463, + "loss": 1.1774, + "theoretical_loss": 3.5953800521878794, + "tokens_seen": 1171652608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032573721163490474, + "loss": 1.5604, + "theoretical_loss": 3.5953428901926925, + "tokens_seen": 1171783680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257171514543631, + "loss": 1.4142, + "theoretical_loss": 3.595305733517858, + "tokens_seen": 1171914752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032569709127382146, + "loss": 1.2706, + "theoretical_loss": 3.5952685821620194, + "tokens_seen": 1172045824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256770310932798, + "loss": 1.5938, + "theoretical_loss": 3.59523143612382, + "tokens_seen": 1172176896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032565697091273824, + "loss": 1.3794, + "theoretical_loss": 3.595194295401905, + "tokens_seen": 1172307968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256369107321966, + "loss": 1.2502, + "theoretical_loss": 3.595157159994918, + "tokens_seen": 1172439040 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032561685055165497, + "loss": 1.5027, + "theoretical_loss": 3.595120029901506, + "tokens_seen": 1172570112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032559679037111333, + "loss": 1.5543, + "theoretical_loss": 3.595082905120313, + "tokens_seen": 1172701184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255767301905717, + "loss": 1.4475, + "theoretical_loss": 3.5950457856499867, + "tokens_seen": 1172832256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255566700100301, + "loss": 1.5248, + "theoretical_loss": 3.5950086714891736, + "tokens_seen": 1172963328 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.25885874032974243, + "objective/train/avg_token_score": 0.015750322490930557, + "objective/train/docs_used": 667905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8621861934661865, + "objective/train/original_loss": 3.7453269958496094, + "objective/train/theoretical_loss": 3.594971562636521, + "objective/train/tokens_used": 1173094400, + "objective/train/unlikelihood_loss": 0.32057327032089233, + "theoretical_loss": 3.594971562636521, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032553660982948847, + "loss": 1.4262, + "theoretical_loss": 3.594971562636521, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032551654964894683, + "loss": 1.5628, + "theoretical_loss": 3.594934459090677, + "tokens_seen": 1173225472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254964894684052, + "loss": 1.395, + "theoretical_loss": 3.5948973608502897, + "tokens_seen": 1173356544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254764292878636, + "loss": 1.4254, + "theoretical_loss": 3.594860267914008, + "tokens_seen": 1173487616 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325456369107322, + "loss": 1.5642, + "theoretical_loss": 3.594823180280481, + "tokens_seen": 1173618688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032543630892678034, + "loss": 1.4165, + "theoretical_loss": 3.594786097948359, + "tokens_seen": 1173749760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254162487462387, + "loss": 1.4045, + "theoretical_loss": 3.594749020916292, + "tokens_seen": 1173880832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032539618856569706, + "loss": 1.374, + "theoretical_loss": 3.5947119491829307, + "tokens_seen": 1174011904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253761283851555, + "loss": 1.3694, + "theoretical_loss": 3.5946748827469266, + "tokens_seen": 1174142976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032535606820461384, + "loss": 1.4266, + "theoretical_loss": 3.5946378216069315, + "tokens_seen": 1174274048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253360080240722, + "loss": 1.4473, + "theoretical_loss": 3.594600765761598, + "tokens_seen": 1174405120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032531594784353056, + "loss": 1.5008, + "theoretical_loss": 3.5945637152095786, + "tokens_seen": 1174536192 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325295887662989, + "loss": 1.4093, + "theoretical_loss": 3.594526669949526, + "tokens_seen": 1174667264 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.5116690993309021, + "objective/train/avg_token_score": 0.008039632812142372, + "objective/train/docs_used": 668590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.667617678642273, + "objective/train/original_loss": 3.6572980880737305, + "objective/train/theoretical_loss": 3.594508149303567, + "objective/train/tokens_used": 1174732800, + "objective/train/unlikelihood_loss": 0.16716612875461578, + "theoretical_loss": 3.594508149303567, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032527582748244734, + "loss": 1.5302, + "theoretical_loss": 3.594489629980095, + "tokens_seen": 1174798336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252557673019057, + "loss": 1.3946, + "theoretical_loss": 3.5944525952999395, + "tokens_seen": 1174929408 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032523570712136407, + "loss": 1.5471, + "theoretical_loss": 3.594415565907714, + "tokens_seen": 1175060480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032521564694082243, + "loss": 1.5932, + "theoretical_loss": 3.5943785418020746, + "tokens_seen": 1175191552 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032519558676028084, + "loss": 1.4154, + "theoretical_loss": 3.5943415229816758, + "tokens_seen": 1175322624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251755265797392, + "loss": 1.4174, + "theoretical_loss": 3.5943045094451747, + "tokens_seen": 1175453696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032515546639919757, + "loss": 1.2981, + "theoretical_loss": 3.5942675011912275, + "tokens_seen": 1175584768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032513540621865593, + "loss": 1.5508, + "theoretical_loss": 3.594230498218492, + "tokens_seen": 1175715840 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032511534603811435, + "loss": 1.5475, + "theoretical_loss": 3.5941935005256256, + "tokens_seen": 1175846912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250952858575727, + "loss": 1.3605, + "theoretical_loss": 3.5941565081112863, + "tokens_seen": 1175977984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032507522567703107, + "loss": 1.5625, + "theoretical_loss": 3.5941195209741332, + "tokens_seen": 1176109056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250551654964895, + "loss": 1.3587, + "theoretical_loss": 3.5940825391128257, + "tokens_seen": 1176240128 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.3811705708503723, + "objective/train/avg_token_score": 0.02655358612537384, + "objective/train/docs_used": 669752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3693901300430298, + "objective/train/original_loss": 4.12606143951416, + "objective/train/theoretical_loss": 3.5940455625260226, + "objective/train/tokens_used": 1176371200, + "objective/train/unlikelihood_loss": 0.17581036686897278, + "theoretical_loss": 3.5940455625260226, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032503510531594785, + "loss": 1.4106, + "theoretical_loss": 3.5940455625260226, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032501504513540627, + "loss": 1.4722, + "theoretical_loss": 3.5940085912123845, + "tokens_seen": 1176502272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032499498495486463, + "loss": 1.4081, + "theoretical_loss": 3.593971625170572, + "tokens_seen": 1176633344 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324974924774323, + "loss": 1.2878, + "theoretical_loss": 3.5939346643992467, + "tokens_seen": 1176764416 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032495486459378135, + "loss": 1.4187, + "theoretical_loss": 3.59389770889707, + "tokens_seen": 1176895488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032493480441323977, + "loss": 1.4201, + "theoretical_loss": 3.5938607586627036, + "tokens_seen": 1177026560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032491474423269813, + "loss": 1.3221, + "theoretical_loss": 3.5938238136948106, + "tokens_seen": 1177157632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248946840521565, + "loss": 1.3816, + "theoretical_loss": 3.5937868739920535, + "tokens_seen": 1177288704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032487462387161486, + "loss": 1.4684, + "theoretical_loss": 3.593749939553097, + "tokens_seen": 1177419776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248545636910732, + "loss": 1.4932, + "theoretical_loss": 3.5937130103766046, + "tokens_seen": 1177550848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032483450351053164, + "loss": 1.5071, + "theoretical_loss": 3.59367608646124, + "tokens_seen": 1177681920 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032481444332999, + "loss": 1.3803, + "theoretical_loss": 3.5936391678056694, + "tokens_seen": 1177812992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032479438314944836, + "loss": 1.3054, + "theoretical_loss": 3.5936022544085575, + "tokens_seen": 1177944064 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.28757330775260925, + "objective/train/avg_token_score": 0.013449753634631634, + "objective/train/docs_used": 670392, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9581223726272583, + "objective/train/original_loss": 4.1560797691345215, + "objective/train/theoretical_loss": 3.593583799681507, + "objective/train/tokens_used": 1178009600, + "objective/train/unlikelihood_loss": 0.12798520922660828, + "theoretical_loss": 3.593583799681507, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247743229689067, + "loss": 1.305, + "theoretical_loss": 3.593565346268571, + "tokens_seen": 1178075136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032475426278836514, + "loss": 1.4501, + "theoretical_loss": 3.5935284433843764, + "tokens_seen": 1178206208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247342026078235, + "loss": 1.3699, + "theoretical_loss": 3.593491545754641, + "tokens_seen": 1178337280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032471414242728186, + "loss": 1.1804, + "theoretical_loss": 3.5934546533780303, + "tokens_seen": 1178468352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246940822467402, + "loss": 1.6243, + "theoretical_loss": 3.5934177662532147, + "tokens_seen": 1178599424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032467402206619864, + "loss": 1.4085, + "theoretical_loss": 3.5933808843788615, + "tokens_seen": 1178730496 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324653961885657, + "loss": 1.3948, + "theoretical_loss": 3.5933440077536396, + "tokens_seen": 1178861568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032463390170511537, + "loss": 1.2693, + "theoretical_loss": 3.593307136376219, + "tokens_seen": 1178992640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032461384152457373, + "loss": 1.4561, + "theoretical_loss": 3.5932702702452692, + "tokens_seen": 1179123712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245937813440321, + "loss": 1.3707, + "theoretical_loss": 3.5932334093594607, + "tokens_seen": 1179254784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245737211634905, + "loss": 1.3777, + "theoretical_loss": 3.593196553717464, + "tokens_seen": 1179385856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032455366098294887, + "loss": 1.446, + "theoretical_loss": 3.5931597033179514, + "tokens_seen": 1179516928 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.2953934371471405, + "objective/train/avg_token_score": 0.011487065814435482, + "objective/train/docs_used": 671618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8233643770217896, + "objective/train/original_loss": 3.8673088550567627, + "objective/train/theoretical_loss": 3.5931228581595938, + "objective/train/tokens_used": 1179648000, + "objective/train/unlikelihood_loss": 0.20144256949424744, + "theoretical_loss": 3.5931228581595938, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032453360080240723, + "loss": 1.1835, + "theoretical_loss": 3.5931228581595938, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245135406218656, + "loss": 1.2794, + "theoretical_loss": 3.5930860182410638, + "tokens_seen": 1179779072 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324493480441324, + "loss": 1.342, + "theoretical_loss": 3.5930491835610345, + "tokens_seen": 1179910144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244734202607824, + "loss": 1.2508, + "theoretical_loss": 3.593012354118179, + "tokens_seen": 1180041216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032445336008024074, + "loss": 1.2889, + "theoretical_loss": 3.592975529911171, + "tokens_seen": 1180172288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244332998996991, + "loss": 1.3876, + "theoretical_loss": 3.592938710938685, + "tokens_seen": 1180303360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032441323971915746, + "loss": 1.3625, + "theoretical_loss": 3.5929018971993956, + "tokens_seen": 1180434432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243931795386159, + "loss": 1.4698, + "theoretical_loss": 3.592865088691978, + "tokens_seen": 1180565504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032437311935807424, + "loss": 1.4184, + "theoretical_loss": 3.592828285415108, + "tokens_seen": 1180696576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243530591775326, + "loss": 1.4275, + "theoretical_loss": 3.592791487367462, + "tokens_seen": 1180827648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032433299899699096, + "loss": 1.4952, + "theoretical_loss": 3.592754694547716, + "tokens_seen": 1180958720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243129388164494, + "loss": 1.3362, + "theoretical_loss": 3.592717906954548, + "tokens_seen": 1181089792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032429287863590774, + "loss": 1.3994, + "theoretical_loss": 3.592681124586635, + "tokens_seen": 1181220864 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.35300585627555847, + "objective/train/avg_token_score": 0.029674863442778587, + "objective/train/docs_used": 672229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1109164953231812, + "objective/train/original_loss": 3.7770514488220215, + "objective/train/theoretical_loss": 3.592662735361736, + "objective/train/tokens_used": 1181286400, + "objective/train/unlikelihood_loss": 0.18284808099269867, + "theoretical_loss": 3.592662735361736, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242728184553661, + "loss": 1.3871, + "theoretical_loss": 3.5926443474426555, + "tokens_seen": 1181351936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032425275827482447, + "loss": 1.555, + "theoretical_loss": 3.592607575521288, + "tokens_seen": 1181483008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032423269809428283, + "loss": 1.4655, + "theoretical_loss": 3.5925708088212116, + "tokens_seen": 1181614080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032421263791374125, + "loss": 1.3584, + "theoretical_loss": 3.592534047341106, + "tokens_seen": 1181745152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241925777331996, + "loss": 1.4617, + "theoretical_loss": 3.592497291079651, + "tokens_seen": 1181876224 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032417251755265797, + "loss": 1.4786, + "theoretical_loss": 3.592460540035527, + "tokens_seen": 1182007296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032415245737211633, + "loss": 1.5238, + "theoretical_loss": 3.5924237942074155, + "tokens_seen": 1182138368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032413239719157475, + "loss": 1.4317, + "theoretical_loss": 3.5923870535939972, + "tokens_seen": 1182269440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241123370110331, + "loss": 1.3529, + "theoretical_loss": 3.5923503181939553, + "tokens_seen": 1182400512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032409227683049147, + "loss": 1.4055, + "theoretical_loss": 3.592313588005971, + "tokens_seen": 1182531584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032407221664994984, + "loss": 1.4466, + "theoretical_loss": 3.592276863028728, + "tokens_seen": 1182662656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240521564694082, + "loss": 1.4874, + "theoretical_loss": 3.5922401432609092, + "tokens_seen": 1182793728 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.3831256031990051, + "objective/train/avg_token_score": 0.008703947067260742, + "objective/train/docs_used": 673021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1497057676315308, + "objective/train/original_loss": 3.633528709411621, + "objective/train/theoretical_loss": 3.5922034287011995, + "objective/train/tokens_used": 1182924800, + "objective/train/unlikelihood_loss": 0.21875718235969543, + "theoretical_loss": 3.5922034287011995, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240320962888666, + "loss": 1.3203, + "theoretical_loss": 3.5922034287011995, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324012036108325, + "loss": 1.3794, + "theoretical_loss": 3.592166719348282, + "tokens_seen": 1183055872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032399197592778334, + "loss": 1.5266, + "theoretical_loss": 3.592130015200842, + "tokens_seen": 1183186944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239719157472417, + "loss": 1.371, + "theoretical_loss": 3.592093316257565, + "tokens_seen": 1183318016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239518555667001, + "loss": 1.5621, + "theoretical_loss": 3.5920566225171364, + "tokens_seen": 1183449088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239317953861585, + "loss": 1.4738, + "theoretical_loss": 3.592019933978243, + "tokens_seen": 1183580160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032391173520561684, + "loss": 1.4759, + "theoretical_loss": 3.5919832506395712, + "tokens_seen": 1183711232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238916750250752, + "loss": 1.5552, + "theoretical_loss": 3.5919465724998076, + "tokens_seen": 1183842304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032387161484453357, + "loss": 1.3828, + "theoretical_loss": 3.5919098995576415, + "tokens_seen": 1183973376 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323851554663992, + "loss": 1.3296, + "theoretical_loss": 3.59187323181176, + "tokens_seen": 1184104448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032383149448345034, + "loss": 1.4219, + "theoretical_loss": 3.5918365692608516, + "tokens_seen": 1184235520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238114343029087, + "loss": 1.4691, + "theoretical_loss": 3.5917999119036064, + "tokens_seen": 1184366592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032379137412236707, + "loss": 1.2758, + "theoretical_loss": 3.5917632597387126, + "tokens_seen": 1184497664 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.42521995306015015, + "objective/train/avg_token_score": 0.02006668597459793, + "objective/train/docs_used": 674157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2694730758666992, + "objective/train/original_loss": 3.6165823936462402, + "objective/train/theoretical_loss": 3.591744935602989, + "objective/train/tokens_used": 1184563200, + "objective/train/unlikelihood_loss": 0.16473890841007233, + "theoretical_loss": 3.591744935602989, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237713139418255, + "loss": 1.4064, + "theoretical_loss": 3.5917266127648615, + "tokens_seen": 1184628736 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032375125376128385, + "loss": 1.3732, + "theoretical_loss": 3.5916899709807435, + "tokens_seen": 1184759808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237311935807422, + "loss": 1.3475, + "theoretical_loss": 3.591653334385049, + "tokens_seen": 1184890880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032371113340020057, + "loss": 1.3674, + "theoretical_loss": 3.59161670297647, + "tokens_seen": 1185021952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032369107321965893, + "loss": 1.3824, + "theoretical_loss": 3.591580076753698, + "tokens_seen": 1185153024 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032367101303911735, + "loss": 1.3795, + "theoretical_loss": 3.591543455715426, + "tokens_seen": 1185284096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236509528585757, + "loss": 1.4293, + "theoretical_loss": 3.591506839860347, + "tokens_seen": 1185415168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236308926780341, + "loss": 1.4533, + "theoretical_loss": 3.591470229187154, + "tokens_seen": 1185546240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032361083249749244, + "loss": 1.324, + "theoretical_loss": 3.5914336236945408, + "tokens_seen": 1185677312 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032359077231695085, + "loss": 1.4503, + "theoretical_loss": 3.5913970233812016, + "tokens_seen": 1185808384 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032357071213640927, + "loss": 1.3575, + "theoretical_loss": 3.5913604282458325, + "tokens_seen": 1185939456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032355065195586763, + "loss": 1.3487, + "theoretical_loss": 3.5913238382871273, + "tokens_seen": 1186070528 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.36498045921325684, + "objective/train/avg_token_score": 0.025275424122810364, + "objective/train/docs_used": 674881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1912466287612915, + "objective/train/original_loss": 3.8096370697021484, + "objective/train/theoretical_loss": 3.5912872535037828, + "objective/train/tokens_used": 1186201600, + "objective/train/unlikelihood_loss": 0.15047334134578705, + "theoretical_loss": 3.5912872535037828, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323530591775326, + "loss": 1.29, + "theoretical_loss": 3.5912872535037828, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235105315947844, + "loss": 1.45, + "theoretical_loss": 3.591250673894494, + "tokens_seen": 1186332672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234904714142428, + "loss": 1.4226, + "theoretical_loss": 3.591214099457959, + "tokens_seen": 1186463744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032347041123370114, + "loss": 1.4731, + "theoretical_loss": 3.5911775301928746, + "tokens_seen": 1186594816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234503510531595, + "loss": 1.3684, + "theoretical_loss": 3.591140966097938, + "tokens_seen": 1186725888 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032343029087261786, + "loss": 1.3277, + "theoretical_loss": 3.5911044071718474, + "tokens_seen": 1186856960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234102306920763, + "loss": 1.3191, + "theoretical_loss": 3.5910678534133016, + "tokens_seen": 1186988032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032339017051153464, + "loss": 1.3315, + "theoretical_loss": 3.591031304821, + "tokens_seen": 1187119104 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323370110330993, + "loss": 1.4961, + "theoretical_loss": 3.5909947613936417, + "tokens_seen": 1187250176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032335005015045136, + "loss": 1.337, + "theoretical_loss": 3.5909582231299266, + "tokens_seen": 1187381248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233299899699098, + "loss": 1.4731, + "theoretical_loss": 3.590921690028556, + "tokens_seen": 1187512320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032330992978936814, + "loss": 1.372, + "theoretical_loss": 3.59088516208823, + "tokens_seen": 1187643392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232898696088265, + "loss": 1.2951, + "theoretical_loss": 3.5908486393076506, + "tokens_seen": 1187774464 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.4709799587726593, + "objective/train/avg_token_score": 0.018071504309773445, + "objective/train/docs_used": 676005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.551163911819458, + "objective/train/original_loss": 3.6569418907165527, + "objective/train/theoretical_loss": 3.5908303798518597, + "objective/train/tokens_used": 1187840000, + "objective/train/unlikelihood_loss": 0.205780029296875, + "theoretical_loss": 3.5908303798518597, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032326980942828487, + "loss": 1.2018, + "theoretical_loss": 3.590812121685519, + "tokens_seen": 1187905536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032324974924774323, + "loss": 1.5378, + "theoretical_loss": 3.590775609220538, + "tokens_seen": 1188036608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032322968906720165, + "loss": 1.4859, + "theoretical_loss": 3.590739101911411, + "tokens_seen": 1188167680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032320962888666, + "loss": 1.4924, + "theoretical_loss": 3.59070259975684, + "tokens_seen": 1188298752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032318956870611837, + "loss": 1.2538, + "theoretical_loss": 3.59066610275553, + "tokens_seen": 1188429824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032316950852557673, + "loss": 1.4134, + "theoretical_loss": 3.590629610906184, + "tokens_seen": 1188560896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032314944834503515, + "loss": 1.3817, + "theoretical_loss": 3.5905931242075084, + "tokens_seen": 1188691968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231293881644935, + "loss": 1.3642, + "theoretical_loss": 3.5905566426582074, + "tokens_seen": 1188823040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231093279839519, + "loss": 1.2003, + "theoretical_loss": 3.5905201662569866, + "tokens_seen": 1188954112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032308926780341024, + "loss": 1.3415, + "theoretical_loss": 3.590483695002552, + "tokens_seen": 1189085184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230692076228686, + "loss": 1.4714, + "theoretical_loss": 3.5904472288936105, + "tokens_seen": 1189216256 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323049147442327, + "loss": 1.4063, + "theoretical_loss": 3.590410767928869, + "tokens_seen": 1189347328 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.3305840790271759, + "objective/train/avg_token_score": 0.06581297516822815, + "objective/train/docs_used": 676669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9510149955749512, + "objective/train/original_loss": 4.567226409912109, + "objective/train/theoretical_loss": 3.590374312107035, + "objective/train/tokens_used": 1189478400, + "objective/train/unlikelihood_loss": 0.11152733862400055, + "theoretical_loss": 3.590374312107035, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230290872617854, + "loss": 1.3764, + "theoretical_loss": 3.590374312107035, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032300902708124374, + "loss": 1.2387, + "theoretical_loss": 3.590337861426817, + "tokens_seen": 1189609472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229889669007021, + "loss": 1.1674, + "theoretical_loss": 3.5903014158869224, + "tokens_seen": 1189740544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229689067201605, + "loss": 1.4143, + "theoretical_loss": 3.590264975486061, + "tokens_seen": 1189871616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229488465396189, + "loss": 1.3567, + "theoretical_loss": 3.5902285402229426, + "tokens_seen": 1190002688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032292878635907724, + "loss": 1.483, + "theoretical_loss": 3.5901921100962753, + "tokens_seen": 1190133760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229087261785356, + "loss": 1.3188, + "theoretical_loss": 3.5901556851047713, + "tokens_seen": 1190264832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032288866599799397, + "loss": 1.3627, + "theoretical_loss": 3.5901192652471403, + "tokens_seen": 1190395904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228686058174524, + "loss": 1.4999, + "theoretical_loss": 3.5900828505220943, + "tokens_seen": 1190526976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032284854563691074, + "loss": 1.3075, + "theoretical_loss": 3.590046440928344, + "tokens_seen": 1190658048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228284854563691, + "loss": 1.5029, + "theoretical_loss": 3.5900100364646024, + "tokens_seen": 1190789120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032280842527582747, + "loss": 1.4538, + "theoretical_loss": 3.5899736371295825, + "tokens_seen": 1190920192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227883650952859, + "loss": 1.4547, + "theoretical_loss": 3.589937242921996, + "tokens_seen": 1191051264 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.2912389934062958, + "objective/train/avg_token_score": 0.023563530296087265, + "objective/train/docs_used": 677774, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9892024397850037, + "objective/train/original_loss": 4.053339004516602, + "objective/train/theoretical_loss": 3.5899190477405885, + "objective/train/tokens_used": 1191116800, + "objective/train/unlikelihood_loss": 0.1627437174320221, + "theoretical_loss": 3.5899190477405885, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032276830491474425, + "loss": 1.4205, + "theoretical_loss": 3.5899008538405575, + "tokens_seen": 1191182336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227482447342026, + "loss": 1.481, + "theoretical_loss": 3.589864469883981, + "tokens_seen": 1191313408 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032272818455366097, + "loss": 1.5161, + "theoretical_loss": 3.5898280910509808, + "tokens_seen": 1191444480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032270812437311933, + "loss": 1.3447, + "theoretical_loss": 3.5897917173402716, + "tokens_seen": 1191575552 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032268806419257775, + "loss": 1.402, + "theoretical_loss": 3.589755348750569, + "tokens_seen": 1191706624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226680040120361, + "loss": 1.4405, + "theoretical_loss": 3.58971898528059, + "tokens_seen": 1191837696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226479438314945, + "loss": 1.4108, + "theoretical_loss": 3.589682626929049, + "tokens_seen": 1191968768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032262788365095284, + "loss": 1.4568, + "theoretical_loss": 3.5896462736946644, + "tokens_seen": 1192099840 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032260782347041125, + "loss": 1.4169, + "theoretical_loss": 3.589609925576153, + "tokens_seen": 1192230912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225877632898696, + "loss": 1.4957, + "theoretical_loss": 3.589573582572232, + "tokens_seen": 1192361984 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322567703109328, + "loss": 1.2588, + "theoretical_loss": 3.5895372446816203, + "tokens_seen": 1192493056 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032254764292878634, + "loss": 1.3862, + "theoretical_loss": 3.5895009119030368, + "tokens_seen": 1192624128 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.32600194215774536, + "objective/train/avg_token_score": 0.007693300023674965, + "objective/train/docs_used": 678420, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.164167046546936, + "objective/train/original_loss": 3.914943218231201, + "objective/train/theoretical_loss": 3.5894645842351993, + "objective/train/tokens_used": 1192755200, + "objective/train/unlikelihood_loss": 0.1659243404865265, + "theoretical_loss": 3.5894645842351993, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225275827482447, + "loss": 1.4393, + "theoretical_loss": 3.5894645842351993, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225075225677031, + "loss": 1.3274, + "theoretical_loss": 3.5894282616768285, + "tokens_seen": 1192886272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224874623871615, + "loss": 1.1513, + "theoretical_loss": 3.5893919442266444, + "tokens_seen": 1193017344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032246740220661984, + "loss": 1.3687, + "theoretical_loss": 3.5893556318833673, + "tokens_seen": 1193148416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224473420260782, + "loss": 1.2986, + "theoretical_loss": 3.589319324645719, + "tokens_seen": 1193279488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224272818455366, + "loss": 1.4856, + "theoretical_loss": 3.5892830225124186, + "tokens_seen": 1193410560 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322407221664995, + "loss": 1.4234, + "theoretical_loss": 3.589246725482191, + "tokens_seen": 1193541632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032238716148445335, + "loss": 1.4839, + "theoretical_loss": 3.589210433553756, + "tokens_seen": 1193672704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223671013039117, + "loss": 1.4714, + "theoretical_loss": 3.5891741467258385, + "tokens_seen": 1193803776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223470411233701, + "loss": 1.3173, + "theoretical_loss": 3.58913786499716, + "tokens_seen": 1193934848 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223269809428285, + "loss": 1.4237, + "theoretical_loss": 3.5891015883664457, + "tokens_seen": 1194065920 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032230692076228685, + "loss": 1.5562, + "theoretical_loss": 3.5890653168324187, + "tokens_seen": 1194196992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222868605817452, + "loss": 1.3695, + "theoretical_loss": 3.5890290503938047, + "tokens_seen": 1194328064 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.4364613890647888, + "objective/train/avg_token_score": 0.014686062932014465, + "objective/train/docs_used": 679445, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2336450815200806, + "objective/train/original_loss": 3.6997463703155518, + "objective/train/theoretical_loss": 3.5890109190848785, + "objective/train/tokens_used": 1194393600, + "objective/train/unlikelihood_loss": 0.20565827190876007, + "theoretical_loss": 3.5890109190848785, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222668004012036, + "loss": 1.4371, + "theoretical_loss": 3.5889927890493274, + "tokens_seen": 1194459136 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322246740220662, + "loss": 1.324, + "theoretical_loss": 3.588956532797714, + "tokens_seen": 1194590208 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032222668004012035, + "loss": 1.5657, + "theoretical_loss": 3.588920281637689, + "tokens_seen": 1194721280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222066198595787, + "loss": 1.4454, + "theoretical_loss": 3.58888403556798, + "tokens_seen": 1194852352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221865596790371, + "loss": 1.4612, + "theoretical_loss": 3.5888477945873136, + "tokens_seen": 1194983424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221664994984955, + "loss": 1.3241, + "theoretical_loss": 3.5888115586944176, + "tokens_seen": 1195114496 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032214643931795386, + "loss": 1.4749, + "theoretical_loss": 3.588775327888019, + "tokens_seen": 1195245568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221263791374122, + "loss": 1.355, + "theoretical_loss": 3.588739102166847, + "tokens_seen": 1195376640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221063189568706, + "loss": 1.284, + "theoretical_loss": 3.5887028815296294, + "tokens_seen": 1195507712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032208625877632894, + "loss": 1.4483, + "theoretical_loss": 3.5886666659750963, + "tokens_seen": 1195638784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220661985957874, + "loss": 1.1509, + "theoretical_loss": 3.5886304555019772, + "tokens_seen": 1195769856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220461384152458, + "loss": 1.4312, + "theoretical_loss": 3.5885942501090025, + "tokens_seen": 1195900928 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.5045210123062134, + "objective/train/avg_token_score": 0.0054487367160618305, + "objective/train/docs_used": 680084, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.799157738685608, + "objective/train/original_loss": 4.015037536621094, + "objective/train/theoretical_loss": 3.588558049794902, + "objective/train/tokens_used": 1196032000, + "objective/train/unlikelihood_loss": 0.13541606068611145, + "theoretical_loss": 3.588558049794902, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032202607823470414, + "loss": 1.4187, + "theoretical_loss": 3.588558049794902, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220060180541625, + "loss": 1.4412, + "theoretical_loss": 3.588521854558407, + "tokens_seen": 1196163072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219859578736209, + "loss": 1.4562, + "theoretical_loss": 3.5884856643982497, + "tokens_seen": 1196294144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219658976930793, + "loss": 1.1688, + "theoretical_loss": 3.588449479313162, + "tokens_seen": 1196425216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032194583751253764, + "loss": 1.3537, + "theoretical_loss": 3.5884132993018754, + "tokens_seen": 1196556288 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321925777331996, + "loss": 1.3902, + "theoretical_loss": 3.5883771243631237, + "tokens_seen": 1196687360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032190571715145437, + "loss": 1.2715, + "theoretical_loss": 3.5883409544956395, + "tokens_seen": 1196818432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218856569709128, + "loss": 1.3468, + "theoretical_loss": 3.5883047896981575, + "tokens_seen": 1196949504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032186559679037115, + "loss": 1.3561, + "theoretical_loss": 3.588268629969411, + "tokens_seen": 1197080576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218455366098295, + "loss": 1.238, + "theoretical_loss": 3.5882324753081356, + "tokens_seen": 1197211648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032182547642928787, + "loss": 1.4039, + "theoretical_loss": 3.588196325713066, + "tokens_seen": 1197342720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218054162487463, + "loss": 1.4353, + "theoretical_loss": 3.5881601811829373, + "tokens_seen": 1197473792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032178535606820465, + "loss": 1.3339, + "theoretical_loss": 3.588124041716487, + "tokens_seen": 1197604864 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.37335044145584106, + "objective/train/avg_token_score": 0.04167129844427109, + "objective/train/docs_used": 681502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1797363758087158, + "objective/train/original_loss": 4.297612190246582, + "objective/train/theoretical_loss": 3.5881059738817456, + "objective/train/tokens_used": 1197670400, + "objective/train/unlikelihood_loss": 0.1500898003578186, + "theoretical_loss": 3.5881059738817456, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321765295887663, + "loss": 1.3823, + "theoretical_loss": 3.58808790731245, + "tokens_seen": 1197735936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032174523570712137, + "loss": 1.4542, + "theoretical_loss": 3.5880517779695644, + "tokens_seen": 1197867008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032172517552657974, + "loss": 1.4668, + "theoretical_loss": 3.588015653686567, + "tokens_seen": 1197998080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032170511534603815, + "loss": 1.4577, + "theoretical_loss": 3.587979534462196, + "tokens_seen": 1198129152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216850551654965, + "loss": 1.385, + "theoretical_loss": 3.58794342029519, + "tokens_seen": 1198260224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216649949849549, + "loss": 1.3725, + "theoretical_loss": 3.5879073111842876, + "tokens_seen": 1198391296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032164493480441324, + "loss": 1.3857, + "theoretical_loss": 3.5878712071282273, + "tokens_seen": 1198522368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032162487462387165, + "loss": 1.5857, + "theoretical_loss": 3.58783510812575, + "tokens_seen": 1198653440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032160481444333, + "loss": 1.3795, + "theoretical_loss": 3.5877990141755944, + "tokens_seen": 1198784512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215847542627884, + "loss": 1.381, + "theoretical_loss": 3.5877629252765026, + "tokens_seen": 1198915584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032156469408224674, + "loss": 1.3696, + "theoretical_loss": 3.587726841427215, + "tokens_seen": 1199046656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215446339017051, + "loss": 1.4241, + "theoretical_loss": 3.5876907626264734, + "tokens_seen": 1199177728 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.4076857268810272, + "objective/train/avg_token_score": 0.010865685530006886, + "objective/train/docs_used": 682215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3691320419311523, + "objective/train/original_loss": 3.769862413406372, + "objective/train/theoretical_loss": 3.5876546888730187, + "objective/train/tokens_used": 1199308800, + "objective/train/unlikelihood_loss": 0.1661466807126999, + "theoretical_loss": 3.5876546888730187, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215245737211635, + "loss": 1.5126, + "theoretical_loss": 3.5876546888730187, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215045135406219, + "loss": 1.5161, + "theoretical_loss": 3.5876186201655944, + "tokens_seen": 1199439872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032148445336008024, + "loss": 1.3251, + "theoretical_loss": 3.587582556502943, + "tokens_seen": 1199570944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214643931795386, + "loss": 1.4524, + "theoretical_loss": 3.5875464978838076, + "tokens_seen": 1199702016 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321444332998997, + "loss": 1.3989, + "theoretical_loss": 3.5875104443069326, + "tokens_seen": 1199833088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214242728184554, + "loss": 1.3655, + "theoretical_loss": 3.5874743957710615, + "tokens_seen": 1199964160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032140421263791375, + "loss": 1.2439, + "theoretical_loss": 3.5874383522749396, + "tokens_seen": 1200095232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213841524573721, + "loss": 1.3668, + "theoretical_loss": 3.5874023138173117, + "tokens_seen": 1200226304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213640922768305, + "loss": 1.4698, + "theoretical_loss": 3.587366280396923, + "tokens_seen": 1200357376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213440320962889, + "loss": 1.4152, + "theoretical_loss": 3.5873302520125194, + "tokens_seen": 1200488448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032132397191574725, + "loss": 1.4666, + "theoretical_loss": 3.587294228662848, + "tokens_seen": 1200619520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213039117352056, + "loss": 1.4156, + "theoretical_loss": 3.587258210346656, + "tokens_seen": 1200750592 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321283851554664, + "loss": 1.3155, + "theoretical_loss": 3.5872221970626894, + "tokens_seen": 1200881664 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.3702346086502075, + "objective/train/avg_token_score": 0.013883780688047409, + "objective/train/docs_used": 683386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3218061923980713, + "objective/train/original_loss": 4.093341827392578, + "objective/train/theoretical_loss": 3.5872041923074, + "objective/train/tokens_used": 1200947200, + "objective/train/unlikelihood_loss": 0.15901117026805878, + "theoretical_loss": 3.5872041923074, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212637913741224, + "loss": 1.3261, + "theoretical_loss": 3.587186188809697, + "tokens_seen": 1201012736 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032124373119358075, + "loss": 1.448, + "theoretical_loss": 3.5871501855864274, + "tokens_seen": 1201143808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212236710130391, + "loss": 1.2809, + "theoretical_loss": 3.587114187391628, + "tokens_seen": 1201274880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212036108324975, + "loss": 1.3151, + "theoretical_loss": 3.5870781942240493, + "tokens_seen": 1201405952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211835506519559, + "loss": 1.3064, + "theoretical_loss": 3.58704220608244, + "tokens_seen": 1201537024 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032116349047141426, + "loss": 1.3951, + "theoretical_loss": 3.58700622296555, + "tokens_seen": 1201668096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211434302908726, + "loss": 1.4477, + "theoretical_loss": 3.5869702448721306, + "tokens_seen": 1201799168 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321123370110331, + "loss": 1.397, + "theoretical_loss": 3.5869342718009323, + "tokens_seen": 1201930240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032110330992978934, + "loss": 1.3468, + "theoretical_loss": 3.5868983037507065, + "tokens_seen": 1202061312 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032108324974924776, + "loss": 1.5714, + "theoretical_loss": 3.586862340720205, + "tokens_seen": 1202192384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210631895687061, + "loss": 1.3116, + "theoretical_loss": 3.58682638270818, + "tokens_seen": 1202323456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210431293881645, + "loss": 1.5156, + "theoretical_loss": 3.5867904297133846, + "tokens_seen": 1202454528 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.4923020601272583, + "objective/train/avg_token_score": 0.023214025422930717, + "objective/train/docs_used": 683937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5233057737350464, + "objective/train/original_loss": 3.616352081298828, + "objective/train/theoretical_loss": 3.5867544817345713, + "objective/train/tokens_used": 1202585600, + "objective/train/unlikelihood_loss": 0.15156559646129608, + "theoretical_loss": 3.5867544817345713, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032102306920762285, + "loss": 1.2394, + "theoretical_loss": 3.5867544817345713, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032100300902708126, + "loss": 1.2576, + "theoretical_loss": 3.5867185387704943, + "tokens_seen": 1202716672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209829488465396, + "loss": 1.3327, + "theoretical_loss": 3.5866826008199073, + "tokens_seen": 1202847744 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320962888665998, + "loss": 1.6164, + "theoretical_loss": 3.586646667881565, + "tokens_seen": 1202978816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032094282848545635, + "loss": 1.5649, + "theoretical_loss": 3.5866107399542217, + "tokens_seen": 1203109888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209227683049147, + "loss": 1.4596, + "theoretical_loss": 3.5865748170366345, + "tokens_seen": 1203240960 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032090270812437313, + "loss": 1.3553, + "theoretical_loss": 3.5865388991275573, + "tokens_seen": 1203372032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208826479438315, + "loss": 1.5104, + "theoretical_loss": 3.5865029862257476, + "tokens_seen": 1203503104 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032086258776328985, + "loss": 1.3163, + "theoretical_loss": 3.586467078329961, + "tokens_seen": 1203634176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208425275827482, + "loss": 1.5175, + "theoretical_loss": 3.586431175438956, + "tokens_seen": 1203765248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032082246740220663, + "loss": 1.4961, + "theoretical_loss": 3.58639527755149, + "tokens_seen": 1203896320 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320802407221665, + "loss": 1.3763, + "theoretical_loss": 3.58635938466632, + "tokens_seen": 1204027392 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032078234704112336, + "loss": 1.4851, + "theoretical_loss": 3.586323496782205, + "tokens_seen": 1204158464 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.43866080045700073, + "objective/train/avg_token_score": 0.043025922030210495, + "objective/train/docs_used": 684175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4305909872055054, + "objective/train/original_loss": 4.117286205291748, + "objective/train/theoretical_loss": 3.5863055547151554, + "objective/train/tokens_used": 1204224000, + "objective/train/unlikelihood_loss": 0.18875925242900848, + "theoretical_loss": 3.5863055547151554, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207622868605817, + "loss": 1.4098, + "theoretical_loss": 3.586287613897904, + "tokens_seen": 1204289536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207422266800401, + "loss": 1.4518, + "theoretical_loss": 3.5862517360121773, + "tokens_seen": 1204420608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207221664994985, + "loss": 1.4859, + "theoretical_loss": 3.5862158631237833, + "tokens_seen": 1204551680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032070210631895686, + "loss": 1.3504, + "theoretical_loss": 3.5861799952314826, + "tokens_seen": 1204682752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206820461384152, + "loss": 1.3427, + "theoretical_loss": 3.586144132334036, + "tokens_seen": 1204813824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206619859578736, + "loss": 1.3463, + "theoretical_loss": 3.5861082744302055, + "tokens_seen": 1204944896 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320641925777332, + "loss": 1.3908, + "theoretical_loss": 3.586072421518752, + "tokens_seen": 1205075968 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032062186559679036, + "loss": 1.5689, + "theoretical_loss": 3.586036573598437, + "tokens_seen": 1205207040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206018054162487, + "loss": 1.3729, + "theoretical_loss": 3.586000730668024, + "tokens_seen": 1205338112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205817452357071, + "loss": 1.4439, + "theoretical_loss": 3.5859648927262753, + "tokens_seen": 1205469184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205616850551655, + "loss": 1.386, + "theoretical_loss": 3.5859290597719546, + "tokens_seen": 1205600256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205416248746239, + "loss": 1.3944, + "theoretical_loss": 3.5858932318038255, + "tokens_seen": 1205731328 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.3516617715358734, + "objective/train/avg_token_score": 0.03645244613289833, + "objective/train/docs_used": 684684, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1452617645263672, + "objective/train/original_loss": 4.93324613571167, + "objective/train/theoretical_loss": 3.585857408820652, + "objective/train/tokens_used": 1205862400, + "objective/train/unlikelihood_loss": 0.11678414046764374, + "theoretical_loss": 3.585857408820652, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205215646940823, + "loss": 1.3767, + "theoretical_loss": 3.585857408820652, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032050150451354064, + "loss": 1.3607, + "theoretical_loss": 3.585821590821199, + "tokens_seen": 1205993472 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320481444332999, + "loss": 1.2826, + "theoretical_loss": 3.5857857778042312, + "tokens_seen": 1206124544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204613841524574, + "loss": 1.2331, + "theoretical_loss": 3.585749969768515, + "tokens_seen": 1206255616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204413239719158, + "loss": 1.371, + "theoretical_loss": 3.5857141667128163, + "tokens_seen": 1206386688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032042126379137415, + "loss": 1.2877, + "theoretical_loss": 3.5856783686359015, + "tokens_seen": 1206517760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204012036108325, + "loss": 1.3462, + "theoretical_loss": 3.5856425755365366, + "tokens_seen": 1206648832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203811434302909, + "loss": 1.6061, + "theoretical_loss": 3.5856067874134903, + "tokens_seen": 1206779904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203610832497493, + "loss": 1.3048, + "theoretical_loss": 3.585571004265529, + "tokens_seen": 1206910976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032034102306920765, + "loss": 1.187, + "theoretical_loss": 3.585535226091422, + "tokens_seen": 1207042048 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320320962888666, + "loss": 1.5654, + "theoretical_loss": 3.5854994528899375, + "tokens_seen": 1207173120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203009027081244, + "loss": 1.5411, + "theoretical_loss": 3.585463684659844, + "tokens_seen": 1207304192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202808425275828, + "loss": 1.5217, + "theoretical_loss": 3.585427921399912, + "tokens_seen": 1207435264 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.5422776341438293, + "objective/train/avg_token_score": 0.009530531242489815, + "objective/train/docs_used": 686035, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7777564525604248, + "objective/train/original_loss": 3.539638042449951, + "objective/train/theoretical_loss": 3.585410041633372, + "objective/train/tokens_used": 1207500800, + "objective/train/unlikelihood_loss": 0.18544457852840424, + "theoretical_loss": 3.585410041633372, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032026078234704115, + "loss": 1.306, + "theoretical_loss": 3.585392163108911, + "tokens_seen": 1207566336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202407221664995, + "loss": 1.4662, + "theoretical_loss": 3.5853564097856117, + "tokens_seen": 1207697408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202206619859579, + "loss": 1.4017, + "theoretical_loss": 3.585320661428784, + "tokens_seen": 1207828480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202006018054163, + "loss": 1.3387, + "theoretical_loss": 3.5852849180372006, + "tokens_seen": 1207959552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032018054162487466, + "loss": 1.3673, + "theoretical_loss": 3.5852491796096317, + "tokens_seen": 1208090624 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320160481444333, + "loss": 1.4513, + "theoretical_loss": 3.5852134461448504, + "tokens_seen": 1208221696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201404212637914, + "loss": 1.2809, + "theoretical_loss": 3.585177717641629, + "tokens_seen": 1208352768 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032012036108324974, + "loss": 1.4458, + "theoretical_loss": 3.585141994098741, + "tokens_seen": 1208483840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032010030090270816, + "loss": 1.179, + "theoretical_loss": 3.585106275514959, + "tokens_seen": 1208614912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200802407221665, + "loss": 1.3999, + "theoretical_loss": 3.5850705618890575, + "tokens_seen": 1208745984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200601805416249, + "loss": 1.3841, + "theoretical_loss": 3.5850348532198106, + "tokens_seen": 1208877056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032004012036108325, + "loss": 1.3642, + "theoretical_loss": 3.5849991495059927, + "tokens_seen": 1209008128 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.43041300773620605, + "objective/train/avg_token_score": 0.016021080315113068, + "objective/train/docs_used": 686549, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4823147058486938, + "objective/train/original_loss": 3.486070156097412, + "objective/train/theoretical_loss": 3.58496345074638, + "objective/train/tokens_used": 1209139200, + "objective/train/unlikelihood_loss": 0.20333431661128998, + "theoretical_loss": 3.58496345074638, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032002006018054166, + "loss": 1.394, + "theoretical_loss": 3.58496345074638, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032, + "loss": 1.3572, + "theoretical_loss": 3.584927756939747, + "tokens_seen": 1209270272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199799398194584, + "loss": 1.3403, + "theoretical_loss": 3.58489206808487, + "tokens_seen": 1209401344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031995987963891675, + "loss": 1.3614, + "theoretical_loss": 3.5848563841805268, + "tokens_seen": 1209532416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199398194583751, + "loss": 1.306, + "theoretical_loss": 3.584820705225493, + "tokens_seen": 1209663488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031991975927783353, + "loss": 1.3846, + "theoretical_loss": 3.584785031218546, + "tokens_seen": 1209794560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198996990972919, + "loss": 1.4215, + "theoretical_loss": 3.5847493621584636, + "tokens_seen": 1209925632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031987963891675025, + "loss": 1.3639, + "theoretical_loss": 3.584713698044025, + "tokens_seen": 1210056704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198595787362086, + "loss": 1.3529, + "theoretical_loss": 3.5846780388740074, + "tokens_seen": 1210187776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031983951855566703, + "loss": 1.4453, + "theoretical_loss": 3.5846423846471915, + "tokens_seen": 1210318848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198194583751254, + "loss": 1.1514, + "theoretical_loss": 3.584606735362356, + "tokens_seen": 1210449920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031979939819458376, + "loss": 1.4297, + "theoretical_loss": 3.5845710910182804, + "tokens_seen": 1210580992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197793380140421, + "loss": 1.4276, + "theoretical_loss": 3.584535451613746, + "tokens_seen": 1210712064 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.481121689081192, + "objective/train/avg_token_score": 0.014049279503524303, + "objective/train/docs_used": 687794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7139185667037964, + "objective/train/original_loss": 3.8951668739318848, + "objective/train/theoretical_loss": 3.5845176337634257, + "objective/train/tokens_used": 1210777600, + "objective/train/unlikelihood_loss": 0.14530795812606812, + "theoretical_loss": 3.5845176337634257, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197592778335005, + "loss": 1.3361, + "theoretical_loss": 3.5844998171475333, + "tokens_seen": 1210843136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197392176529589, + "loss": 1.4077, + "theoretical_loss": 3.5844641876184236, + "tokens_seen": 1210974208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031971915747241726, + "loss": 1.4424, + "theoretical_loss": 3.5844285630251984, + "tokens_seen": 1211105280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196990972918756, + "loss": 1.2183, + "theoretical_loss": 3.58439294336664, + "tokens_seen": 1211236352 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319679037111334, + "loss": 1.506, + "theoretical_loss": 3.584357328641531, + "tokens_seen": 1211367424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196589769307924, + "loss": 1.3828, + "theoretical_loss": 3.5843217188486545, + "tokens_seen": 1211498496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031963891675025076, + "loss": 1.492, + "theoretical_loss": 3.584286113986794, + "tokens_seen": 1211629568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196188565697091, + "loss": 1.4614, + "theoretical_loss": 3.584250514054732, + "tokens_seen": 1211760640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195987963891675, + "loss": 1.3545, + "theoretical_loss": 3.584214919051255, + "tokens_seen": 1211891712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031957873620862585, + "loss": 1.4523, + "theoretical_loss": 3.5841793289751465, + "tokens_seen": 1212022784 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031955867602808427, + "loss": 1.5559, + "theoretical_loss": 3.584143743825192, + "tokens_seen": 1212153856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031953861584754263, + "loss": 1.498, + "theoretical_loss": 3.584108163600177, + "tokens_seen": 1212284928 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.3048631548881531, + "objective/train/avg_token_score": 0.020269062370061874, + "objective/train/docs_used": 688453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9617087244987488, + "objective/train/original_loss": 3.927225351333618, + "objective/train/theoretical_loss": 3.5840725882988873, + "objective/train/tokens_used": 1212416000, + "objective/train/unlikelihood_loss": 0.19555844366550446, + "theoretical_loss": 3.5840725882988873, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319518555667001, + "loss": 1.2276, + "theoretical_loss": 3.5840725882988873, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031949849548645935, + "loss": 1.3741, + "theoretical_loss": 3.5840370179201093, + "tokens_seen": 1212547072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031947843530591777, + "loss": 1.2568, + "theoretical_loss": 3.5840014524626307, + "tokens_seen": 1212678144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031945837512537613, + "loss": 1.4135, + "theoretical_loss": 3.5839658919252377, + "tokens_seen": 1212809216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194383149448345, + "loss": 1.261, + "theoretical_loss": 3.583930336306719, + "tokens_seen": 1212940288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031941825476429286, + "loss": 1.3002, + "theoretical_loss": 3.583894785605862, + "tokens_seen": 1213071360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193981945837512, + "loss": 1.5668, + "theoretical_loss": 3.5838592398214555, + "tokens_seen": 1213202432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031937813440320964, + "loss": 1.3178, + "theoretical_loss": 3.5838236989522887, + "tokens_seen": 1213333504 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319358074222668, + "loss": 1.3753, + "theoretical_loss": 3.583788162997151, + "tokens_seen": 1213464576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031933801404212636, + "loss": 1.4788, + "theoretical_loss": 3.583752631954833, + "tokens_seen": 1213595648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193179538615847, + "loss": 1.3689, + "theoretical_loss": 3.5837171058241237, + "tokens_seen": 1213726720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031929789368104314, + "loss": 1.2638, + "theoretical_loss": 3.5836815846038146, + "tokens_seen": 1213857792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192778335005015, + "loss": 1.3457, + "theoretical_loss": 3.5836460682926967, + "tokens_seen": 1213988864 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.48057183623313904, + "objective/train/avg_token_score": 0.01698978990316391, + "objective/train/docs_used": 689723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5353022813796997, + "objective/train/original_loss": 3.8551487922668457, + "objective/train/theoretical_loss": 3.583628311977707, + "objective/train/tokens_used": 1214054400, + "objective/train/unlikelihood_loss": 0.17180033028125763, + "theoretical_loss": 3.583628311977707, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031925777331995986, + "loss": 1.2752, + "theoretical_loss": 3.583610556889562, + "tokens_seen": 1214119936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192377131394182, + "loss": 1.3111, + "theoretical_loss": 3.583575050393202, + "tokens_seen": 1214251008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192176529588766, + "loss": 1.4653, + "theoretical_loss": 3.583539548802409, + "tokens_seen": 1214382080 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319197592778335, + "loss": 1.3, + "theoretical_loss": 3.5835040521159764, + "tokens_seen": 1214513152 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031917753259779337, + "loss": 1.4862, + "theoretical_loss": 3.5834685603326975, + "tokens_seen": 1214644224 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031915747241725173, + "loss": 1.3616, + "theoretical_loss": 3.583433073451366, + "tokens_seen": 1214775296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191374122367101, + "loss": 1.31, + "theoretical_loss": 3.5833975914707756, + "tokens_seen": 1214906368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191173520561685, + "loss": 1.4398, + "theoretical_loss": 3.5833621143897214, + "tokens_seen": 1215037440 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031909729187562687, + "loss": 1.458, + "theoretical_loss": 3.5833266422069983, + "tokens_seen": 1215168512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031907723169508523, + "loss": 1.3798, + "theoretical_loss": 3.583291174921402, + "tokens_seen": 1215299584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031905717151454365, + "loss": 1.2745, + "theoretical_loss": 3.5832557125317273, + "tokens_seen": 1215430656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031903711133400206, + "loss": 1.4025, + "theoretical_loss": 3.583220255036772, + "tokens_seen": 1215561728 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.4477028250694275, + "objective/train/avg_token_score": 0.008430707268416882, + "objective/train/docs_used": 690230, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.558432936668396, + "objective/train/original_loss": 3.5454485416412354, + "objective/train/theoretical_loss": 3.5831848024353317, + "objective/train/tokens_used": 1215692800, + "objective/train/unlikelihood_loss": 0.17317147552967072, + "theoretical_loss": 3.5831848024353317, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190170511534604, + "loss": 1.4801, + "theoretical_loss": 3.5831848024353317, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189969909729188, + "loss": 1.4177, + "theoretical_loss": 3.5831493547262037, + "tokens_seen": 1215823872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031897693079237715, + "loss": 1.4931, + "theoretical_loss": 3.5831139119081863, + "tokens_seen": 1215954944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189568706118355, + "loss": 1.4338, + "theoretical_loss": 3.583078473980077, + "tokens_seen": 1216086016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031893681043129393, + "loss": 1.2849, + "theoretical_loss": 3.583043040940674, + "tokens_seen": 1216217088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189167502507523, + "loss": 1.3379, + "theoretical_loss": 3.5830076127887764, + "tokens_seen": 1216348160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031889669007021065, + "loss": 1.3966, + "theoretical_loss": 3.5829721895231836, + "tokens_seen": 1216479232 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318876629889669, + "loss": 1.451, + "theoretical_loss": 3.582936771142695, + "tokens_seen": 1216610304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031885656970912743, + "loss": 1.4837, + "theoretical_loss": 3.582901357646111, + "tokens_seen": 1216741376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188365095285858, + "loss": 1.5201, + "theoretical_loss": 3.5828659490322323, + "tokens_seen": 1216872448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031881644934804416, + "loss": 1.5207, + "theoretical_loss": 3.582830545299859, + "tokens_seen": 1217003520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187963891675025, + "loss": 1.282, + "theoretical_loss": 3.5827951464477934, + "tokens_seen": 1217134592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187763289869609, + "loss": 1.4663, + "theoretical_loss": 3.5827597524748374, + "tokens_seen": 1217265664 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.4456256031990051, + "objective/train/avg_token_score": 0.02469589188694954, + "objective/train/docs_used": 691368, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4652098417282104, + "objective/train/original_loss": 3.81040358543396, + "objective/train/theoretical_loss": 3.5827420573176507, + "objective/train/tokens_used": 1217331200, + "objective/train/unlikelihood_loss": 0.14845961332321167, + "theoretical_loss": 3.5827420573176507, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187562688064193, + "loss": 1.5723, + "theoretical_loss": 3.582724363379792, + "tokens_seen": 1217396736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031873620862587766, + "loss": 1.2889, + "theoretical_loss": 3.5826889791614613, + "tokens_seen": 1217527808 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318716148445336, + "loss": 1.3937, + "theoretical_loss": 3.5826535998186477, + "tokens_seen": 1217658880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186960882647944, + "loss": 1.3304, + "theoretical_loss": 3.5826182253501546, + "tokens_seen": 1217789952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186760280842528, + "loss": 1.3603, + "theoretical_loss": 3.5825828557547865, + "tokens_seen": 1217921024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031865596790371116, + "loss": 1.37, + "theoretical_loss": 3.582547491031347, + "tokens_seen": 1218052096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186359077231695, + "loss": 1.237, + "theoretical_loss": 3.5825121311786416, + "tokens_seen": 1218183168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186158475426279, + "loss": 1.1696, + "theoretical_loss": 3.5824767761954748, + "tokens_seen": 1218314240 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031859578736208625, + "loss": 1.4433, + "theoretical_loss": 3.5824414260806527, + "tokens_seen": 1218445312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031857572718154467, + "loss": 1.3992, + "theoretical_loss": 3.582406080832981, + "tokens_seen": 1218576384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031855566700100303, + "loss": 1.3398, + "theoretical_loss": 3.582370740451267, + "tokens_seen": 1218707456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185356068204614, + "loss": 1.5004, + "theoretical_loss": 3.582335404934317, + "tokens_seen": 1218838528 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.47782257199287415, + "objective/train/avg_token_score": 0.012256566435098648, + "objective/train/docs_used": 692060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5405457019805908, + "objective/train/original_loss": 3.9979941844940186, + "objective/train/theoretical_loss": 3.5823000742809374, + "objective/train/tokens_used": 1218969600, + "objective/train/unlikelihood_loss": 0.14614969491958618, + "theoretical_loss": 3.5823000742809374, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031851554663991975, + "loss": 1.4936, + "theoretical_loss": 3.5823000742809374, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031849548645937817, + "loss": 1.4452, + "theoretical_loss": 3.582264748489937, + "tokens_seen": 1219100672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031847542627883653, + "loss": 1.3123, + "theoretical_loss": 3.582229427560124, + "tokens_seen": 1219231744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184553660982949, + "loss": 1.6765, + "theoretical_loss": 3.582194111490307, + "tokens_seen": 1219362816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031843530591775326, + "loss": 1.3377, + "theoretical_loss": 3.582158800279294, + "tokens_seen": 1219493888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184152457372116, + "loss": 1.3009, + "theoretical_loss": 3.5821234939258955, + "tokens_seen": 1219624960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031839518555667004, + "loss": 1.3248, + "theoretical_loss": 3.5820881924289205, + "tokens_seen": 1219756032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183751253761284, + "loss": 1.275, + "theoretical_loss": 3.58205289578718, + "tokens_seen": 1219887104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031835506519558676, + "loss": 1.3521, + "theoretical_loss": 3.5820176039994838, + "tokens_seen": 1220018176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183350050150451, + "loss": 1.3752, + "theoretical_loss": 3.581982317064644, + "tokens_seen": 1220149248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031831494483450354, + "loss": 1.3589, + "theoretical_loss": 3.5819470349814715, + "tokens_seen": 1220280320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182948846539619, + "loss": 1.4334, + "theoretical_loss": 3.5819117577487782, + "tokens_seen": 1220411392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031827482447342026, + "loss": 1.275, + "theoretical_loss": 3.5818764853653766, + "tokens_seen": 1220542464 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.5813782811164856, + "objective/train/avg_token_score": 0.006016418803483248, + "objective/train/docs_used": 693613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9248859882354736, + "objective/train/original_loss": 3.7803778648376465, + "objective/train/theoretical_loss": 3.581858850991789, + "objective/train/tokens_used": 1220608000, + "objective/train/unlikelihood_loss": 0.13684310019016266, + "theoretical_loss": 3.581858850991789, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182547642928786, + "loss": 1.3452, + "theoretical_loss": 3.581841217830079, + "tokens_seen": 1220673536 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318234704112337, + "loss": 1.2876, + "theoretical_loss": 3.5818059551416996, + "tokens_seen": 1220804608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182146439317954, + "loss": 1.387, + "theoretical_loss": 3.581770697299051, + "tokens_seen": 1220935680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031819458375125377, + "loss": 1.2579, + "theoretical_loss": 3.5817354443009473, + "tokens_seen": 1221066752 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031817452357071213, + "loss": 1.4575, + "theoretical_loss": 3.581700196146204, + "tokens_seen": 1221197824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181544633901705, + "loss": 1.3249, + "theoretical_loss": 3.581664952833634, + "tokens_seen": 1221328896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181344032096289, + "loss": 1.4145, + "theoretical_loss": 3.5816297143620544, + "tokens_seen": 1221459968 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031811434302908727, + "loss": 1.2331, + "theoretical_loss": 3.5815944807302804, + "tokens_seen": 1221591040 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031809428284854563, + "loss": 1.2857, + "theoretical_loss": 3.5815592519371275, + "tokens_seen": 1221722112 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318074222668004, + "loss": 1.2784, + "theoretical_loss": 3.5815240279814127, + "tokens_seen": 1221853184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180541624874624, + "loss": 1.3407, + "theoretical_loss": 3.5814888088619528, + "tokens_seen": 1221984256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031803410230692077, + "loss": 1.3758, + "theoretical_loss": 3.5814535945775647, + "tokens_seen": 1222115328 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.545637845993042, + "objective/train/avg_token_score": 0.007774205878376961, + "objective/train/docs_used": 694027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4959824085235596, + "objective/train/original_loss": 3.3811662197113037, + "objective/train/theoretical_loss": 3.5814183851270673, + "objective/train/tokens_used": 1222246400, + "objective/train/unlikelihood_loss": 0.15404260158538818, + "theoretical_loss": 3.5814183851270673, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031801404212637913, + "loss": 1.3248, + "theoretical_loss": 3.5814183851270673, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179939819458375, + "loss": 1.4243, + "theoretical_loss": 3.5813831805092775, + "tokens_seen": 1222377472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031797392176529586, + "loss": 1.2809, + "theoretical_loss": 3.581347980723015, + "tokens_seen": 1222508544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179538615847543, + "loss": 1.435, + "theoretical_loss": 3.5813127857670977, + "tokens_seen": 1222639616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031793380140421264, + "loss": 1.4202, + "theoretical_loss": 3.581277595640346, + "tokens_seen": 1222770688 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317913741223671, + "loss": 1.4417, + "theoretical_loss": 3.581242410341579, + "tokens_seen": 1222901760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031789368104312936, + "loss": 1.5531, + "theoretical_loss": 3.5812072298696176, + "tokens_seen": 1223032832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178736208625878, + "loss": 1.4016, + "theoretical_loss": 3.581172054223282, + "tokens_seen": 1223163904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031785356068204614, + "loss": 1.3528, + "theoretical_loss": 3.5811368834013937, + "tokens_seen": 1223294976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178335005015045, + "loss": 1.2715, + "theoretical_loss": 3.581101717402773, + "tokens_seen": 1223426048 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031781344032096287, + "loss": 1.2496, + "theoretical_loss": 3.5810665562262436, + "tokens_seen": 1223557120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031779338014042123, + "loss": 1.2765, + "theoretical_loss": 3.581031399870627, + "tokens_seen": 1223688192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031777331995987964, + "loss": 1.3158, + "theoretical_loss": 3.580996248334746, + "tokens_seen": 1223819264 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.32178640365600586, + "objective/train/avg_token_score": 0.02192055620253086, + "objective/train/docs_used": 695222, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0434319972991943, + "objective/train/original_loss": 4.228107929229736, + "objective/train/theoretical_loss": 3.580978674373838, + "objective/train/tokens_used": 1223884800, + "objective/train/unlikelihood_loss": 0.12738944590091705, + "theoretical_loss": 3.580978674373838, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317753259779338, + "loss": 1.2496, + "theoretical_loss": 3.580961101617423, + "tokens_seen": 1223950336 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031773319959879637, + "loss": 1.4469, + "theoretical_loss": 3.580925959717483, + "tokens_seen": 1224081408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031771313941825473, + "loss": 1.3933, + "theoretical_loss": 3.580890822633749, + "tokens_seen": 1224212480 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031769307923771315, + "loss": 1.336, + "theoretical_loss": 3.580855690365045, + "tokens_seen": 1224343552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176730190571715, + "loss": 1.2066, + "theoretical_loss": 3.580820562910197, + "tokens_seen": 1224474624 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031765295887662987, + "loss": 1.4167, + "theoretical_loss": 3.580785440268029, + "tokens_seen": 1224605696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031763289869608823, + "loss": 1.3724, + "theoretical_loss": 3.5807503224373676, + "tokens_seen": 1224736768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176128385155466, + "loss": 1.3863, + "theoretical_loss": 3.5807152094170385, + "tokens_seen": 1224867840 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317592778335005, + "loss": 1.3442, + "theoretical_loss": 3.5806801012058678, + "tokens_seen": 1224998912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175727181544634, + "loss": 1.3185, + "theoretical_loss": 3.580644997802683, + "tokens_seen": 1225129984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175526579739218, + "loss": 1.2019, + "theoretical_loss": 3.580609899206311, + "tokens_seen": 1225261056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031753259779338015, + "loss": 1.3157, + "theoretical_loss": 3.58057480541558, + "tokens_seen": 1225392128 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.34659090638160706, + "objective/train/avg_token_score": 0.014647016301751137, + "objective/train/docs_used": 695595, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1693223714828491, + "objective/train/original_loss": 3.9977846145629883, + "objective/train/theoretical_loss": 3.5805397164293167, + "objective/train/tokens_used": 1225523200, + "objective/train/unlikelihood_loss": 0.137997567653656, + "theoretical_loss": 3.5805397164293167, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031751253761283857, + "loss": 1.4432, + "theoretical_loss": 3.5805397164293167, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031749247743229693, + "loss": 1.2826, + "theoretical_loss": 3.580504632246351, + "tokens_seen": 1225654272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174724172517553, + "loss": 1.4091, + "theoretical_loss": 3.5804695528655115, + "tokens_seen": 1225785344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031745235707121366, + "loss": 1.4625, + "theoretical_loss": 3.5804344782856274, + "tokens_seen": 1225916416 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317432296890672, + "loss": 1.3494, + "theoretical_loss": 3.5803994085055284, + "tokens_seen": 1226047488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031741223671013044, + "loss": 1.387, + "theoretical_loss": 3.5803643435240446, + "tokens_seen": 1226178560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173921765295888, + "loss": 1.52, + "theoretical_loss": 3.5803292833400064, + "tokens_seen": 1226309632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031737211634904716, + "loss": 1.4415, + "theoretical_loss": 3.580294227952246, + "tokens_seen": 1226440704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173520561685055, + "loss": 1.4684, + "theoretical_loss": 3.5802591773595926, + "tokens_seen": 1226571776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031733199598796394, + "loss": 1.3833, + "theoretical_loss": 3.58022413156088, + "tokens_seen": 1226702848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173119358074223, + "loss": 1.427, + "theoretical_loss": 3.5801890905549394, + "tokens_seen": 1226833920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031729187562688066, + "loss": 1.2555, + "theoretical_loss": 3.5801540543406034, + "tokens_seen": 1226964992 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317271815446339, + "loss": 1.3318, + "theoretical_loss": 3.5801190229167057, + "tokens_seen": 1227096064 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.4635874927043915, + "objective/train/avg_token_score": 0.016565006226301193, + "objective/train/docs_used": 696474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5446860790252686, + "objective/train/original_loss": 3.7141802310943604, + "objective/train/theoretical_loss": 3.580101509000806, + "objective/train/tokens_used": 1227161600, + "objective/train/unlikelihood_loss": 0.21671420335769653, + "theoretical_loss": 3.580101509000806, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172517552657974, + "loss": 1.4016, + "theoretical_loss": 3.5800839962820787, + "tokens_seen": 1227227136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172316950852558, + "loss": 1.3519, + "theoretical_loss": 3.5800489744355573, + "tokens_seen": 1227358208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031721163490471417, + "loss": 1.4454, + "theoretical_loss": 3.580013957375975, + "tokens_seen": 1227489280 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031719157472417253, + "loss": 1.3818, + "theoretical_loss": 3.5799789451021664, + "tokens_seen": 1227620352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171715145436309, + "loss": 1.2942, + "theoretical_loss": 3.579943937612967, + "tokens_seen": 1227751424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171514543630893, + "loss": 1.3691, + "theoretical_loss": 3.5799089349072126, + "tokens_seen": 1227882496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031713139418254767, + "loss": 1.3976, + "theoretical_loss": 3.5798739369837382, + "tokens_seen": 1228013568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031711133400200603, + "loss": 1.3812, + "theoretical_loss": 3.5798389438413807, + "tokens_seen": 1228144640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170912738214644, + "loss": 1.3616, + "theoretical_loss": 3.5798039554789765, + "tokens_seen": 1228275712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170712136409228, + "loss": 1.3892, + "theoretical_loss": 3.5797689718953625, + "tokens_seen": 1228406784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170511534603812, + "loss": 1.4795, + "theoretical_loss": 3.579733993089377, + "tokens_seen": 1228537856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031703109327983954, + "loss": 1.2805, + "theoretical_loss": 3.5796990190598565, + "tokens_seen": 1228668928 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.4052419364452362, + "objective/train/avg_token_score": 0.015196052379906178, + "objective/train/docs_used": 696846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3371021747589111, + "objective/train/original_loss": 3.5605454444885254, + "objective/train/theoretical_loss": 3.5796640498056407, + "objective/train/tokens_used": 1228800000, + "objective/train/unlikelihood_loss": 0.1954372227191925, + "theoretical_loss": 3.5796640498056407, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170110330992979, + "loss": 1.3264, + "theoretical_loss": 3.5796640498056407, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031699097291875626, + "loss": 1.4571, + "theoretical_loss": 3.579629085325568, + "tokens_seen": 1228931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169709127382147, + "loss": 1.3918, + "theoretical_loss": 3.5795941256184776, + "tokens_seen": 1229062144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031695085255767304, + "loss": 1.4877, + "theoretical_loss": 3.5795591706832086, + "tokens_seen": 1229193216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169307923771314, + "loss": 1.3201, + "theoretical_loss": 3.579524220518601, + "tokens_seen": 1229324288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031691073219658976, + "loss": 1.5235, + "theoretical_loss": 3.5794892751234952, + "tokens_seen": 1229455360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168906720160482, + "loss": 1.4117, + "theoretical_loss": 3.579454334496732, + "tokens_seen": 1229586432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031687061183550654, + "loss": 1.3553, + "theoretical_loss": 3.5794193986371527, + "tokens_seen": 1229717504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168505516549649, + "loss": 1.2643, + "theoretical_loss": 3.579384467543599, + "tokens_seen": 1229848576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031683049147442327, + "loss": 1.4688, + "theoretical_loss": 3.5793495412149126, + "tokens_seen": 1229979648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031681043129388163, + "loss": 1.4159, + "theoretical_loss": 3.579314619649936, + "tokens_seen": 1230110720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031679037111334004, + "loss": 1.4839, + "theoretical_loss": 3.5792797028475114, + "tokens_seen": 1230241792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167703109327984, + "loss": 1.2108, + "theoretical_loss": 3.5792447908064826, + "tokens_seen": 1230372864 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.2826857268810272, + "objective/train/avg_token_score": 0.03822416812181473, + "objective/train/docs_used": 698095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9177280068397522, + "objective/train/original_loss": 4.414151191711426, + "objective/train/theoretical_loss": 3.5792273365711305, + "objective/train/tokens_used": 1230438400, + "objective/train/unlikelihood_loss": 0.15019060671329498, + "theoretical_loss": 3.5792273365711305, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031675025075225677, + "loss": 1.3128, + "theoretical_loss": 3.5792098835256936, + "tokens_seen": 1230503936 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031673019057171513, + "loss": 1.2633, + "theoretical_loss": 3.5791749810039875, + "tokens_seen": 1230635008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031671013039117355, + "loss": 1.4981, + "theoretical_loss": 3.579140083240209, + "tokens_seen": 1230766080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166900702106319, + "loss": 1.3196, + "theoretical_loss": 3.579105190233203, + "tokens_seen": 1230897152 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031667001003009027, + "loss": 1.2724, + "theoretical_loss": 3.579070301981815, + "tokens_seen": 1231028224 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031664994984954863, + "loss": 1.3873, + "theoretical_loss": 3.5790354184848896, + "tokens_seen": 1231159296 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316629889669007, + "loss": 1.2182, + "theoretical_loss": 3.579000539741274, + "tokens_seen": 1231290368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166098294884654, + "loss": 1.1022, + "theoretical_loss": 3.5789656657498137, + "tokens_seen": 1231421440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165897693079238, + "loss": 1.3577, + "theoretical_loss": 3.5789307965093564, + "tokens_seen": 1231552512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031656970912738214, + "loss": 1.3601, + "theoretical_loss": 3.5788959320187486, + "tokens_seen": 1231683584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165496489468405, + "loss": 1.3703, + "theoretical_loss": 3.578861072276838, + "tokens_seen": 1231814656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165295887662989, + "loss": 1.3943, + "theoretical_loss": 3.578826217282473, + "tokens_seen": 1231945728 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.35227271914482117, + "objective/train/avg_token_score": 0.026345469057559967, + "objective/train/docs_used": 698724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1844558715820312, + "objective/train/original_loss": 4.2838454246521, + "objective/train/theoretical_loss": 3.5787913670345013, + "objective/train/tokens_used": 1232076800, + "objective/train/unlikelihood_loss": 0.10895750671625137, + "theoretical_loss": 3.5787913670345013, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165095285857573, + "loss": 1.4502, + "theoretical_loss": 3.5787913670345013, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031648946840521564, + "loss": 1.3116, + "theoretical_loss": 3.5787565215317727, + "tokens_seen": 1232207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316469408224674, + "loss": 1.3607, + "theoretical_loss": 3.578721680773136, + "tokens_seen": 1232338944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031644934804413237, + "loss": 1.4937, + "theoretical_loss": 3.5786868447574403, + "tokens_seen": 1232470016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164292878635908, + "loss": 1.5632, + "theoretical_loss": 3.578652013483537, + "tokens_seen": 1232601088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031640922768304914, + "loss": 1.2653, + "theoretical_loss": 3.5786171869502743, + "tokens_seen": 1232732160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163891675025075, + "loss": 1.3755, + "theoretical_loss": 3.578582365156505, + "tokens_seen": 1232863232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031636910732196587, + "loss": 1.3329, + "theoretical_loss": 3.5785475481010796, + "tokens_seen": 1232994304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163490471414243, + "loss": 1.3965, + "theoretical_loss": 3.57851273578285, + "tokens_seen": 1233125376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031632898696088265, + "loss": 1.1735, + "theoretical_loss": 3.5784779282006682, + "tokens_seen": 1233256448 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316308926780341, + "loss": 1.4149, + "theoretical_loss": 3.5784431253533864, + "tokens_seen": 1233387520 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031628886659979937, + "loss": 1.4485, + "theoretical_loss": 3.578408327239857, + "tokens_seen": 1233518592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031626880641925773, + "loss": 1.2773, + "theoretical_loss": 3.578373533858935, + "tokens_seen": 1233649664 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.41507819294929504, + "objective/train/avg_token_score": 0.02956448867917061, + "objective/train/docs_used": 699861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3864712715148926, + "objective/train/original_loss": 3.8828225135803223, + "objective/train/theoretical_loss": 3.578356138942842, + "objective/train/tokens_used": 1233715200, + "objective/train/unlikelihood_loss": 0.14931270480155945, + "theoretical_loss": 3.578356138942842, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031624874623871615, + "loss": 1.4208, + "theoretical_loss": 3.5783387452094715, + "tokens_seen": 1233780736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162286860581745, + "loss": 1.4601, + "theoretical_loss": 3.578303961290322, + "tokens_seen": 1233911808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162086258776329, + "loss": 1.2733, + "theoretical_loss": 3.5782691821003407, + "tokens_seen": 1234042880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031618856569709124, + "loss": 1.3652, + "theoretical_loss": 3.578234407638383, + "tokens_seen": 1234173952 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031616850551654965, + "loss": 1.3338, + "theoretical_loss": 3.5781996379033036, + "tokens_seen": 1234305024 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316148445336008, + "loss": 1.2505, + "theoretical_loss": 3.578164872893958, + "tokens_seen": 1234436096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161283851554664, + "loss": 1.3929, + "theoretical_loss": 3.578130112609202, + "tokens_seen": 1234567168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031610832497492474, + "loss": 1.3771, + "theoretical_loss": 3.578095357047893, + "tokens_seen": 1234698240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160882647943831, + "loss": 1.3805, + "theoretical_loss": 3.578060606208887, + "tokens_seen": 1234829312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160682046138415, + "loss": 1.3505, + "theoretical_loss": 3.5780258600910413, + "tokens_seen": 1234960384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031604814443329994, + "loss": 1.4266, + "theoretical_loss": 3.5779911186932134, + "tokens_seen": 1235091456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160280842527583, + "loss": 1.5004, + "theoretical_loss": 3.5779563820142624, + "tokens_seen": 1235222528 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.3961388170719147, + "objective/train/avg_token_score": 0.011691992171108723, + "objective/train/docs_used": 700550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1044576168060303, + "objective/train/original_loss": 3.4253785610198975, + "objective/train/theoretical_loss": 3.577921650053045, + "objective/train/tokens_used": 1235353600, + "objective/train/unlikelihood_loss": 0.19166605174541473, + "theoretical_loss": 3.577921650053045, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031600802407221666, + "loss": 1.5231, + "theoretical_loss": 3.577921650053045, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159879638916751, + "loss": 1.3696, + "theoretical_loss": 3.577886922808421, + "tokens_seen": 1235484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031596790371113344, + "loss": 1.4515, + "theoretical_loss": 3.57785220027925, + "tokens_seen": 1235615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159478435305918, + "loss": 1.4456, + "theoretical_loss": 3.5778174824643907, + "tokens_seen": 1235746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031592778335005016, + "loss": 1.5371, + "theoretical_loss": 3.577782769362704, + "tokens_seen": 1235877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159077231695086, + "loss": 1.373, + "theoretical_loss": 3.5777480609730494, + "tokens_seen": 1236008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031588766298896694, + "loss": 1.4447, + "theoretical_loss": 3.577713357294288, + "tokens_seen": 1236140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158676028084253, + "loss": 1.4461, + "theoretical_loss": 3.577678658325281, + "tokens_seen": 1236271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031584754262788367, + "loss": 1.4647, + "theoretical_loss": 3.5776439640648907, + "tokens_seen": 1236402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031582748244734203, + "loss": 1.3889, + "theoretical_loss": 3.5776092745119783, + "tokens_seen": 1236533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031580742226680044, + "loss": 1.4832, + "theoretical_loss": 3.5775745896654056, + "tokens_seen": 1236664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157873620862588, + "loss": 1.2551, + "theoretical_loss": 3.577539909524037, + "tokens_seen": 1236795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031576730190571717, + "loss": 1.4227, + "theoretical_loss": 3.5775052340867344, + "tokens_seen": 1236926464 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.5745967626571655, + "objective/train/avg_token_score": 0.011720869690179825, + "objective/train/docs_used": 701918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8321902751922607, + "objective/train/original_loss": 3.4837892055511475, + "objective/train/theoretical_loss": 3.577487898131753, + "objective/train/tokens_used": 1236992000, + "objective/train/unlikelihood_loss": 0.1529117375612259, + "theoretical_loss": 3.577487898131753, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031574724172517553, + "loss": 1.5084, + "theoretical_loss": 3.577470563352362, + "tokens_seen": 1237057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031572718154463395, + "loss": 1.4794, + "theoretical_loss": 3.5774358973197833, + "tokens_seen": 1237188608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157071213640923, + "loss": 1.4343, + "theoretical_loss": 3.577401235987863, + "tokens_seen": 1237319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031568706118355067, + "loss": 1.378, + "theoretical_loss": 3.5773665793554654, + "tokens_seen": 1237450752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031566700100300903, + "loss": 1.3279, + "theoretical_loss": 3.577331927421456, + "tokens_seen": 1237581824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156469408224674, + "loss": 1.2451, + "theoretical_loss": 3.5772972801847005, + "tokens_seen": 1237712896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156268806419258, + "loss": 1.289, + "theoretical_loss": 3.577262637644065, + "tokens_seen": 1237843968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156068204613842, + "loss": 1.3421, + "theoretical_loss": 3.5772279997984144, + "tokens_seen": 1237975040 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031558676028084254, + "loss": 1.413, + "theoretical_loss": 3.5771933666466174, + "tokens_seen": 1238106112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155667001003009, + "loss": 1.3986, + "theoretical_loss": 3.57715873818754, + "tokens_seen": 1238237184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155466399197593, + "loss": 1.3904, + "theoretical_loss": 3.5771241144200494, + "tokens_seen": 1238368256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155265797392177, + "loss": 1.4485, + "theoretical_loss": 3.5770894953430146, + "tokens_seen": 1238499328 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.3975439965724945, + "objective/train/avg_token_score": 0.010389717295765877, + "objective/train/docs_used": 702757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2891199588775635, + "objective/train/original_loss": 3.8938634395599365, + "objective/train/theoretical_loss": 3.577054880955303, + "objective/train/tokens_used": 1238630400, + "objective/train/unlikelihood_loss": 0.14993628859519958, + "theoretical_loss": 3.577054880955303, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031550651955867604, + "loss": 1.329, + "theoretical_loss": 3.577054880955303, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154864593781344, + "loss": 1.2343, + "theoretical_loss": 3.5770202712557833, + "tokens_seen": 1238761472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031546639919759277, + "loss": 1.3719, + "theoretical_loss": 3.576985666243325, + "tokens_seen": 1238892544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154463390170512, + "loss": 1.4731, + "theoretical_loss": 3.5769510659167976, + "tokens_seen": 1239023616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031542627883650954, + "loss": 1.3067, + "theoretical_loss": 3.5769164702750706, + "tokens_seen": 1239154688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154062186559679, + "loss": 1.2978, + "theoretical_loss": 3.576881879317015, + "tokens_seen": 1239285760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031538615847542627, + "loss": 1.3401, + "theoretical_loss": 3.5768472930415003, + "tokens_seen": 1239416832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153660982948847, + "loss": 1.36, + "theoretical_loss": 3.576812711447398, + "tokens_seen": 1239547904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031534603811434305, + "loss": 1.3488, + "theoretical_loss": 3.57677813453358, + "tokens_seen": 1239678976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153259779338014, + "loss": 1.273, + "theoretical_loss": 3.576743562298917, + "tokens_seen": 1239810048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031530591775325977, + "loss": 1.3583, + "theoretical_loss": 3.5767089947422828, + "tokens_seen": 1239941120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031528585757271813, + "loss": 1.4173, + "theoretical_loss": 3.576674431862549, + "tokens_seen": 1240072192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031526579739217655, + "loss": 1.2661, + "theoretical_loss": 3.5766398736585883, + "tokens_seen": 1240203264 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.5086143612861633, + "objective/train/avg_token_score": 0.012611065059900284, + "objective/train/docs_used": 703958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5925217866897583, + "objective/train/original_loss": 3.806576728820801, + "objective/train/theoretical_loss": 3.5766225963096714, + "objective/train/tokens_used": 1240268800, + "objective/train/unlikelihood_loss": 0.14473141729831696, + "theoretical_loss": 3.5766225963096714, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152457372116349, + "loss": 1.3522, + "theoretical_loss": 3.576605320129275, + "tokens_seen": 1240334336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152256770310933, + "loss": 1.3617, + "theoretical_loss": 3.576570771273482, + "tokens_seen": 1240465408 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031520561685055164, + "loss": 1.369, + "theoretical_loss": 3.5765362270900845, + "tokens_seen": 1240596480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031518555667001005, + "loss": 1.4694, + "theoretical_loss": 3.576501687577956, + "tokens_seen": 1240727552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151654964894684, + "loss": 1.3493, + "theoretical_loss": 3.576467152735972, + "tokens_seen": 1240858624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151454363089268, + "loss": 1.348, + "theoretical_loss": 3.576432622563008, + "tokens_seen": 1240989696 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031512537612838514, + "loss": 1.2204, + "theoretical_loss": 3.5763980970579388, + "tokens_seen": 1241120768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151053159478435, + "loss": 1.4806, + "theoretical_loss": 3.5763635762196415, + "tokens_seen": 1241251840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150852557673019, + "loss": 1.3602, + "theoretical_loss": 3.5763290600469926, + "tokens_seen": 1241382912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150651955867603, + "loss": 1.2992, + "theoretical_loss": 3.5762945485388675, + "tokens_seen": 1241513984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031504513540621864, + "loss": 1.4454, + "theoretical_loss": 3.5762600416941455, + "tokens_seen": 1241645056 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315025075225677, + "loss": 1.3263, + "theoretical_loss": 3.5762255395117037, + "tokens_seen": 1241776128 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.2611803412437439, + "objective/train/avg_token_score": 0.020229818299412727, + "objective/train/docs_used": 704662, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9152143597602844, + "objective/train/original_loss": 4.399595737457275, + "objective/train/theoretical_loss": 3.5761910419904193, + "objective/train/tokens_used": 1241907200, + "objective/train/unlikelihood_loss": 0.18543991446495056, + "theoretical_loss": 3.5761910419904193, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150050150451354, + "loss": 1.4176, + "theoretical_loss": 3.5761910419904193, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149849548645938, + "loss": 1.3778, + "theoretical_loss": 3.5761565491291716, + "tokens_seen": 1242038272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031496489468405215, + "loss": 1.3219, + "theoretical_loss": 3.5761220609268385, + "tokens_seen": 1242169344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149448345035105, + "loss": 1.4489, + "theoretical_loss": 3.5760875773823004, + "tokens_seen": 1242300416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149247743229689, + "loss": 1.364, + "theoretical_loss": 3.5760530984944365, + "tokens_seen": 1242431488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149047141424273, + "loss": 1.2039, + "theoretical_loss": 3.576018624262127, + "tokens_seen": 1242562560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031488465396188565, + "loss": 1.4509, + "theoretical_loss": 3.5759841546842512, + "tokens_seen": 1242693632 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314864593781344, + "loss": 1.3492, + "theoretical_loss": 3.5759496897596907, + "tokens_seen": 1242824704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148445336008024, + "loss": 1.1937, + "theoretical_loss": 3.5759152294873267, + "tokens_seen": 1242955776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148244734202608, + "loss": 1.2004, + "theoretical_loss": 3.5758807738660403, + "tokens_seen": 1243086848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031480441323971915, + "loss": 1.3824, + "theoretical_loss": 3.575846322894715, + "tokens_seen": 1243217920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147843530591775, + "loss": 1.4253, + "theoretical_loss": 3.5758118765722307, + "tokens_seen": 1243348992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147642928786359, + "loss": 1.3967, + "theoretical_loss": 3.5757774348974714, + "tokens_seen": 1243480064 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.44287633895874023, + "objective/train/avg_token_score": 0.01065880712121725, + "objective/train/docs_used": 705833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.586069941520691, + "objective/train/original_loss": 3.9955692291259766, + "objective/train/theoretical_loss": 3.5757602158026396, + "objective/train/tokens_used": 1243545600, + "objective/train/unlikelihood_loss": 0.1316809356212616, + "theoretical_loss": 3.5757602158026396, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147442326980943, + "loss": 1.3492, + "theoretical_loss": 3.5757429978693205, + "tokens_seen": 1243611136 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031472417251755266, + "loss": 1.41, + "theoretical_loss": 3.5757085654866607, + "tokens_seen": 1243742208 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314704112337011, + "loss": 1.408, + "theoretical_loss": 3.5756741377483765, + "tokens_seen": 1243873280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146840521564694, + "loss": 1.4028, + "theoretical_loss": 3.5756397146533514, + "tokens_seen": 1244004352 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031466399197592774, + "loss": 1.3088, + "theoretical_loss": 3.5756052962004707, + "tokens_seen": 1244135424 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031464393179538616, + "loss": 1.3002, + "theoretical_loss": 3.57557088238862, + "tokens_seen": 1244266496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146238716148445, + "loss": 1.3276, + "theoretical_loss": 3.5755364732166828, + "tokens_seen": 1244397568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146038114343029, + "loss": 1.4286, + "theoretical_loss": 3.5755020686835466, + "tokens_seen": 1244528640 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031458375125376125, + "loss": 1.3413, + "theoretical_loss": 3.575467668788097, + "tokens_seen": 1244659712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145636910732197, + "loss": 1.2627, + "theoretical_loss": 3.575433273529221, + "tokens_seen": 1244790784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145436308926781, + "loss": 1.3912, + "theoretical_loss": 3.5753988829058043, + "tokens_seen": 1244921856 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031452357071213644, + "loss": 1.1927, + "theoretical_loss": 3.5753644969167353, + "tokens_seen": 1245052928 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.4439149498939514, + "objective/train/avg_token_score": 0.012424762360751629, + "objective/train/docs_used": 706471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4641262292861938, + "objective/train/original_loss": 3.5480899810791016, + "objective/train/theoretical_loss": 3.5753301155609014, + "objective/train/tokens_used": 1245184000, + "objective/train/unlikelihood_loss": 0.17369605600833893, + "theoretical_loss": 3.5753301155609014, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145035105315948, + "loss": 1.4515, + "theoretical_loss": 3.5753301155609014, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031448345035105317, + "loss": 1.3526, + "theoretical_loss": 3.575295738837191, + "tokens_seen": 1245315072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144633901705116, + "loss": 1.45, + "theoretical_loss": 3.5752613667444924, + "tokens_seen": 1245446144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031444332998996994, + "loss": 1.438, + "theoretical_loss": 3.5752269992816945, + "tokens_seen": 1245577216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144232698094283, + "loss": 1.4716, + "theoretical_loss": 3.575192636447686, + "tokens_seen": 1245708288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031440320962888667, + "loss": 1.4002, + "theoretical_loss": 3.575158278241357, + "tokens_seen": 1245839360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143831494483451, + "loss": 1.3958, + "theoretical_loss": 3.575123924661598, + "tokens_seen": 1245970432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031436308926780345, + "loss": 1.3446, + "theoretical_loss": 3.575089575707299, + "tokens_seen": 1246101504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143430290872618, + "loss": 1.386, + "theoretical_loss": 3.57505523137735, + "tokens_seen": 1246232576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031432296890672017, + "loss": 1.3877, + "theoretical_loss": 3.5750208916706434, + "tokens_seen": 1246363648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031430290872617853, + "loss": 1.3348, + "theoretical_loss": 3.5749865565860697, + "tokens_seen": 1246494720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031428284854563695, + "loss": 1.3416, + "theoretical_loss": 3.5749522261225217, + "tokens_seen": 1246625792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142627883650953, + "loss": 1.3585, + "theoretical_loss": 3.5749179002788916, + "tokens_seen": 1246756864 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.4417155385017395, + "objective/train/avg_token_score": 0.02173241600394249, + "objective/train/docs_used": 707914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5563459396362305, + "objective/train/original_loss": 4.009653568267822, + "objective/train/theoretical_loss": 3.574900739089199, + "objective/train/tokens_used": 1246822400, + "objective/train/unlikelihood_loss": 0.14615082740783691, + "theoretical_loss": 3.574900739089199, + "tokens_seen": 1246822400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142427281845537, + "loss": 1.5359, + "theoretical_loss": 3.574883579054071, + "tokens_seen": 1246887936 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031422266800401204, + "loss": 1.3513, + "theoretical_loss": 3.5748492624469543, + "tokens_seen": 1247019008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031420260782347045, + "loss": 1.3117, + "theoretical_loss": 3.5748149504564344, + "tokens_seen": 1247150080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141825476429288, + "loss": 1.3367, + "theoretical_loss": 3.5747806430814055, + "tokens_seen": 1247281152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141624874623872, + "loss": 1.2964, + "theoretical_loss": 3.574746340320761, + "tokens_seen": 1247412224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031414242728184554, + "loss": 1.4577, + "theoretical_loss": 3.574712042173396, + "tokens_seen": 1247543296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141223671013039, + "loss": 1.4674, + "theoretical_loss": 3.5746777486382064, + "tokens_seen": 1247674368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141023069207623, + "loss": 1.3431, + "theoretical_loss": 3.574643459714086, + "tokens_seen": 1247805440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140822467402207, + "loss": 1.4089, + "theoretical_loss": 3.574609175399931, + "tokens_seen": 1247936512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031406218655967904, + "loss": 1.2952, + "theoretical_loss": 3.574574895694638, + "tokens_seen": 1248067584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140421263791374, + "loss": 1.2855, + "theoretical_loss": 3.5745406205971033, + "tokens_seen": 1248198656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140220661985958, + "loss": 1.3486, + "theoretical_loss": 3.5745063501062235, + "tokens_seen": 1248329728 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.39937683939933777, + "objective/train/avg_token_score": 0.01781587302684784, + "objective/train/docs_used": 708590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1363708972930908, + "objective/train/original_loss": 4.074802875518799, + "objective/train/theoretical_loss": 3.574472084220896, + "objective/train/tokens_used": 1248460800, + "objective/train/unlikelihood_loss": 0.19421115517616272, + "theoretical_loss": 3.574472084220896, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140020060180542, + "loss": 1.4642, + "theoretical_loss": 3.574472084220896, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031398194583751255, + "loss": 1.5373, + "theoretical_loss": 3.574437822940019, + "tokens_seen": 1248591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139618856569709, + "loss": 1.2406, + "theoretical_loss": 3.5744035662624896, + "tokens_seen": 1248722944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139418254764293, + "loss": 1.3042, + "theoretical_loss": 3.574369314187207, + "tokens_seen": 1248854016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139217652958877, + "loss": 1.3347, + "theoretical_loss": 3.5743350667130698, + "tokens_seen": 1248985088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031390170511534605, + "loss": 1.2952, + "theoretical_loss": 3.5743008238389766, + "tokens_seen": 1249116160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138816449348044, + "loss": 1.4092, + "theoretical_loss": 3.5742665855638274, + "tokens_seen": 1249247232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138615847542628, + "loss": 1.3207, + "theoretical_loss": 3.574232351886522, + "tokens_seen": 1249378304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138415245737212, + "loss": 1.3782, + "theoretical_loss": 3.5741981228059605, + "tokens_seen": 1249509376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031382146439317955, + "loss": 1.4026, + "theoretical_loss": 3.5741638983210438, + "tokens_seen": 1249640448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138014042126379, + "loss": 1.3203, + "theoretical_loss": 3.5741296784306735, + "tokens_seen": 1249771520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137813440320963, + "loss": 1.3566, + "theoretical_loss": 3.57409546313375, + "tokens_seen": 1249902592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137612838515547, + "loss": 1.368, + "theoretical_loss": 3.5740612524291757, + "tokens_seen": 1250033664 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.34164223074913025, + "objective/train/avg_token_score": 0.014163387008011341, + "objective/train/docs_used": 709757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1497161388397217, + "objective/train/original_loss": 3.947155237197876, + "objective/train/theoretical_loss": 3.5740441487986767, + "objective/train/tokens_used": 1250099200, + "objective/train/unlikelihood_loss": 0.14038622379302979, + "theoretical_loss": 3.5740441487986767, + "tokens_seen": 1250099200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031374122367101306, + "loss": 1.4647, + "theoretical_loss": 3.574027046315853, + "tokens_seen": 1250164736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137211634904714, + "loss": 1.1943, + "theoretical_loss": 3.573992844792684, + "tokens_seen": 1250295808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137011033099298, + "loss": 1.5752, + "theoretical_loss": 3.5739586478585714, + "tokens_seen": 1250426880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031368104312938814, + "loss": 1.476, + "theoretical_loss": 3.573924455512419, + "tokens_seen": 1250557952 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031366098294884656, + "loss": 1.4059, + "theoretical_loss": 3.5738902677531312, + "tokens_seen": 1250689024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136409227683049, + "loss": 1.4807, + "theoretical_loss": 3.5738560845796106, + "tokens_seen": 1250820096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136208625877633, + "loss": 1.288, + "theoretical_loss": 3.573821905990762, + "tokens_seen": 1250951168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031360080240722165, + "loss": 1.3011, + "theoretical_loss": 3.573787731985491, + "tokens_seen": 1251082240 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031358074222668006, + "loss": 1.4115, + "theoretical_loss": 3.5737535625627026, + "tokens_seen": 1251213312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135606820461384, + "loss": 1.4315, + "theoretical_loss": 3.5737193977213018, + "tokens_seen": 1251344384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135406218655968, + "loss": 1.4711, + "theoretical_loss": 3.573685237460195, + "tokens_seen": 1251475456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031352056168505515, + "loss": 1.5123, + "theoretical_loss": 3.5736510817782885, + "tokens_seen": 1251606528 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.5279203057289124, + "objective/train/avg_token_score": 0.017824076116085052, + "objective/train/docs_used": 710221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7168092727661133, + "objective/train/original_loss": 3.673182725906372, + "objective/train/theoretical_loss": 3.5736169306744885, + "objective/train/tokens_used": 1251737600, + "objective/train/unlikelihood_loss": 0.14243413507938385, + "theoretical_loss": 3.5736169306744885, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135005015045135, + "loss": 1.5202, + "theoretical_loss": 3.5736169306744885, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031348044132397193, + "loss": 1.3351, + "theoretical_loss": 3.573582784147703, + "tokens_seen": 1251868672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134603811434303, + "loss": 1.4705, + "theoretical_loss": 3.5735486421968385, + "tokens_seen": 1251999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031344032096288865, + "loss": 1.3436, + "theoretical_loss": 3.5735145048208032, + "tokens_seen": 1252130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313420260782347, + "loss": 1.2891, + "theoretical_loss": 3.5734803720185058, + "tokens_seen": 1252261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031340020060180543, + "loss": 1.4354, + "theoretical_loss": 3.5734462437888546, + "tokens_seen": 1252392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133801404212638, + "loss": 1.4223, + "theoretical_loss": 3.573412120130758, + "tokens_seen": 1252524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031336008024072216, + "loss": 1.4608, + "theoretical_loss": 3.573378001043126, + "tokens_seen": 1252655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133400200601805, + "loss": 1.2858, + "theoretical_loss": 3.573343886524868, + "tokens_seen": 1252786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133199598796389, + "loss": 1.2639, + "theoretical_loss": 3.5733097765748942, + "tokens_seen": 1252917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132998996990973, + "loss": 1.5037, + "theoretical_loss": 3.573275671192115, + "tokens_seen": 1253048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031327983951855566, + "loss": 1.3139, + "theoretical_loss": 3.5732415703754405, + "tokens_seen": 1253179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313259779338014, + "loss": 1.237, + "theoretical_loss": 3.5732074741237834, + "tokens_seen": 1253310464 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.35752686858177185, + "objective/train/avg_token_score": 0.025846872478723526, + "objective/train/docs_used": 710893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2280304431915283, + "objective/train/original_loss": 3.825455904006958, + "objective/train/theoretical_loss": 3.5731904277094957, + "objective/train/tokens_used": 1253376000, + "objective/train/unlikelihood_loss": 0.2751415967941284, + "theoretical_loss": 3.5731904277094957, + "tokens_seen": 1253376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132397191574724, + "loss": 1.2519, + "theoretical_loss": 3.573173382436054, + "tokens_seen": 1253441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132196589769308, + "loss": 1.2909, + "theoretical_loss": 3.573139295311165, + "tokens_seen": 1253572608 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031319959879638916, + "loss": 1.3601, + "theoretical_loss": 3.5731052127480285, + "tokens_seen": 1253703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131795386158475, + "loss": 1.4278, + "theoretical_loss": 3.5730711347455575, + "tokens_seen": 1253834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131594784353059, + "loss": 1.4566, + "theoretical_loss": 3.573037061302664, + "tokens_seen": 1253965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031313941825476425, + "loss": 1.2772, + "theoretical_loss": 3.5730029924182625, + "tokens_seen": 1254096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031311935807422267, + "loss": 1.4029, + "theoretical_loss": 3.5729689280912664, + "tokens_seen": 1254227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031309929789368103, + "loss": 1.4303, + "theoretical_loss": 3.5729348683205897, + "tokens_seen": 1254359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130792377131394, + "loss": 1.3692, + "theoretical_loss": 3.5729008131051474, + "tokens_seen": 1254490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130591775325978, + "loss": 1.3681, + "theoretical_loss": 3.572866762443854, + "tokens_seen": 1254621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130391173520562, + "loss": 1.4624, + "theoretical_loss": 3.5728327163356255, + "tokens_seen": 1254752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130190571715146, + "loss": 1.3263, + "theoretical_loss": 3.572798674779377, + "tokens_seen": 1254883328 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.3926564157009125, + "objective/train/avg_token_score": 0.019924413412809372, + "objective/train/docs_used": 712142, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2856049537658691, + "objective/train/original_loss": 3.510986566543579, + "objective/train/theoretical_loss": 3.572764637774024, + "objective/train/tokens_used": 1255014400, + "objective/train/unlikelihood_loss": 0.2521393299102783, + "theoretical_loss": 3.572764637774024, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031299899699097295, + "loss": 1.3917, + "theoretical_loss": 3.572764637774024, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129789368104313, + "loss": 1.2566, + "theoretical_loss": 3.572730605318484, + "tokens_seen": 1255145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129588766298897, + "loss": 1.2589, + "theoretical_loss": 3.5726965774116737, + "tokens_seen": 1255276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129388164493481, + "loss": 1.2941, + "theoretical_loss": 3.572662554052509, + "tokens_seen": 1255407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031291875626880645, + "loss": 1.408, + "theoretical_loss": 3.572628535239909, + "tokens_seen": 1255538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128986960882648, + "loss": 1.1986, + "theoretical_loss": 3.5725945209727907, + "tokens_seen": 1255669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128786359077232, + "loss": 1.4442, + "theoretical_loss": 3.5725605112500727, + "tokens_seen": 1255800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128585757271816, + "loss": 1.4412, + "theoretical_loss": 3.5725265060706732, + "tokens_seen": 1255931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031283851554663995, + "loss": 1.1591, + "theoretical_loss": 3.5724925054335115, + "tokens_seen": 1256062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128184553660983, + "loss": 1.4088, + "theoretical_loss": 3.5724585093375074, + "tokens_seen": 1256194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127983951855567, + "loss": 1.2935, + "theoretical_loss": 3.5724245177815797, + "tokens_seen": 1256325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127783350050151, + "loss": 1.3546, + "theoretical_loss": 3.5723905307646495, + "tokens_seen": 1256456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031275827482447346, + "loss": 1.3405, + "theoretical_loss": 3.5723565482856365, + "tokens_seen": 1256587264 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.44238758087158203, + "objective/train/avg_token_score": 0.016206545755267143, + "objective/train/docs_used": 712613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.407757043838501, + "objective/train/original_loss": 3.746412515640259, + "objective/train/theoretical_loss": 3.5723395587475117, + "objective/train/tokens_used": 1256652800, + "objective/train/unlikelihood_loss": 0.16399602591991425, + "theoretical_loss": 3.5723395587475117, + "tokens_seen": 1256652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127382146439318, + "loss": 1.4366, + "theoretical_loss": 3.572322570343462, + "tokens_seen": 1256718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127181544633902, + "loss": 1.3791, + "theoretical_loss": 3.5722885969370473, + "tokens_seen": 1256849408 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031269809428284854, + "loss": 1.5203, + "theoretical_loss": 3.5722546280653136, + "tokens_seen": 1256980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031267803410230696, + "loss": 1.3986, + "theoretical_loss": 3.572220663727183, + "tokens_seen": 1257111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126579739217653, + "loss": 1.1987, + "theoretical_loss": 3.572186703921578, + "tokens_seen": 1257242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126379137412237, + "loss": 1.3124, + "theoretical_loss": 3.5721527486474214, + "tokens_seen": 1257373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031261785356068205, + "loss": 1.4827, + "theoretical_loss": 3.5721187979036353, + "tokens_seen": 1257504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031259779338014046, + "loss": 1.5482, + "theoretical_loss": 3.572084851689145, + "tokens_seen": 1257635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125777331995988, + "loss": 1.4243, + "theoretical_loss": 3.572050910002872, + "tokens_seen": 1257766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125576730190572, + "loss": 1.5392, + "theoretical_loss": 3.572016972843743, + "tokens_seen": 1257897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031253761283851555, + "loss": 1.3404, + "theoretical_loss": 3.5719830402106805, + "tokens_seen": 1258029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125175526579739, + "loss": 1.2838, + "theoretical_loss": 3.57194911210261, + "tokens_seen": 1258160128 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.3030303120613098, + "objective/train/avg_token_score": 0.00934206135571003, + "objective/train/docs_used": 713934, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0763776302337646, + "objective/train/original_loss": 4.278534412384033, + "objective/train/theoretical_loss": 3.571915188518457, + "objective/train/tokens_used": 1258291200, + "objective/train/unlikelihood_loss": 0.16740037500858307, + "theoretical_loss": 3.571915188518457, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031249749247743233, + "loss": 1.2548, + "theoretical_loss": 3.571915188518457, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124774322968907, + "loss": 1.3106, + "theoretical_loss": 3.571881269457147, + "tokens_seen": 1258422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031245737211634905, + "loss": 1.3285, + "theoretical_loss": 3.5718473549176064, + "tokens_seen": 1258553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124373119358074, + "loss": 1.4812, + "theoretical_loss": 3.571813444898761, + "tokens_seen": 1258684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031241725175526583, + "loss": 1.4072, + "theoretical_loss": 3.5717795393995377, + "tokens_seen": 1258815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123971915747242, + "loss": 1.4181, + "theoretical_loss": 3.5717456384188635, + "tokens_seen": 1258946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031237713139418256, + "loss": 1.3303, + "theoretical_loss": 3.5717117419556663, + "tokens_seen": 1259077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123570712136409, + "loss": 1.3391, + "theoretical_loss": 3.5716778500088737, + "tokens_seen": 1259208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123370110330993, + "loss": 1.3297, + "theoretical_loss": 3.571643962577414, + "tokens_seen": 1259339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123169508525577, + "loss": 1.1883, + "theoretical_loss": 3.571610079660215, + "tokens_seen": 1259470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031229689067201606, + "loss": 1.295, + "theoretical_loss": 3.571576201256207, + "tokens_seen": 1259601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122768304914744, + "loss": 1.2779, + "theoretical_loss": 3.5715423273643188, + "tokens_seen": 1259732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122567703109328, + "loss": 1.2843, + "theoretical_loss": 3.5715084579834793, + "tokens_seen": 1259864064 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.5949413776397705, + "objective/train/avg_token_score": 0.02111905999481678, + "objective/train/docs_used": 714588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9605203866958618, + "objective/train/original_loss": 3.2083277702331543, + "objective/train/theoretical_loss": 3.571491524984369, + "objective/train/tokens_used": 1259929600, + "objective/train/unlikelihood_loss": 0.29471489787101746, + "theoretical_loss": 3.571491524984369, + "tokens_seen": 1259929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122367101303912, + "loss": 1.486, + "theoretical_loss": 3.5714745931126193, + "tokens_seen": 1259995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031221664994984956, + "loss": 1.3546, + "theoretical_loss": 3.571440732750669, + "tokens_seen": 1260126208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121965897693079, + "loss": 1.398, + "theoretical_loss": 3.5714068768965594, + "tokens_seen": 1260257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121765295887663, + "loss": 1.4401, + "theoretical_loss": 3.571373025549221, + "tokens_seen": 1260388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031215646940822465, + "loss": 1.2497, + "theoretical_loss": 3.5713391787075857, + "tokens_seen": 1260519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031213640922768307, + "loss": 1.3026, + "theoretical_loss": 3.571305336370586, + "tokens_seen": 1260650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031211634904714143, + "loss": 1.4728, + "theoretical_loss": 3.5712714985371528, + "tokens_seen": 1260781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120962888665998, + "loss": 1.4717, + "theoretical_loss": 3.57123766520622, + "tokens_seen": 1260912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031207622868605815, + "loss": 1.3757, + "theoretical_loss": 3.571203836376719, + "tokens_seen": 1261043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031205616850551657, + "loss": 1.3259, + "theoretical_loss": 3.5711700120475847, + "tokens_seen": 1261174784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031203610832497493, + "loss": 1.3872, + "theoretical_loss": 3.5711361922177502, + "tokens_seen": 1261305856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120160481444333, + "loss": 1.3605, + "theoretical_loss": 3.5711023768861487, + "tokens_seen": 1261436928 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.345796674489975, + "objective/train/avg_token_score": 0.0102199986577034, + "objective/train/docs_used": 715954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1135615110397339, + "objective/train/original_loss": 3.685187339782715, + "objective/train/theoretical_loss": 3.571068566051716, + "objective/train/tokens_used": 1261568000, + "objective/train/unlikelihood_loss": 0.1403382271528244, + "theoretical_loss": 3.571068566051716, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031199598796389166, + "loss": 1.349, + "theoretical_loss": 3.571068566051716, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031197592778335, + "loss": 1.281, + "theoretical_loss": 3.5710347597133865, + "tokens_seen": 1261699072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031195586760280843, + "loss": 1.3867, + "theoretical_loss": 3.571000957870094, + "tokens_seen": 1261830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119358074222668, + "loss": 1.4022, + "theoretical_loss": 3.5709671605207762, + "tokens_seen": 1261961216 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031191574724172516, + "loss": 1.2973, + "theoretical_loss": 3.5709333676643675, + "tokens_seen": 1262092288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118956870611835, + "loss": 1.2305, + "theoretical_loss": 3.570899579299804, + "tokens_seen": 1262223360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031187562688064194, + "loss": 1.5271, + "theoretical_loss": 3.570865795426023, + "tokens_seen": 1262354432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118555667001003, + "loss": 1.409, + "theoretical_loss": 3.570832016041961, + "tokens_seen": 1262485504 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031183550651955866, + "loss": 1.3391, + "theoretical_loss": 3.5707982411465555, + "tokens_seen": 1262616576 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311815446339017, + "loss": 1.039, + "theoretical_loss": 3.570764470738744, + "tokens_seen": 1262747648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117953861584754, + "loss": 1.3235, + "theoretical_loss": 3.570730704817465, + "tokens_seen": 1262878720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117753259779338, + "loss": 1.3161, + "theoretical_loss": 3.570696943381656, + "tokens_seen": 1263009792 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031175526579739217, + "loss": 1.3657, + "theoretical_loss": 3.5706631864302567, + "tokens_seen": 1263140864 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.3877077102661133, + "objective/train/avg_token_score": 0.01689940132200718, + "objective/train/docs_used": 716702, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3269243240356445, + "objective/train/original_loss": 4.028445243835449, + "objective/train/theoretical_loss": 3.5706463096358787, + "objective/train/tokens_used": 1263206400, + "objective/train/unlikelihood_loss": 0.16066420078277588, + "theoretical_loss": 3.5706463096358787, + "tokens_seen": 1263206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031173520561685053, + "loss": 1.3326, + "theoretical_loss": 3.570629433962206, + "tokens_seen": 1263271936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117151454363089, + "loss": 1.5337, + "theoretical_loss": 3.5705956859764427, + "tokens_seen": 1263403008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116950852557673, + "loss": 1.3131, + "theoretical_loss": 3.5705619424719073, + "tokens_seen": 1263534080 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031167502507522567, + "loss": 1.3062, + "theoretical_loss": 3.5705282034475396, + "tokens_seen": 1263665152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031165496489468403, + "loss": 1.2714, + "theoretical_loss": 3.5704944689022806, + "tokens_seen": 1263796224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116349047141424, + "loss": 1.3671, + "theoretical_loss": 3.570460738835071, + "tokens_seen": 1263927296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116148445336008, + "loss": 1.3484, + "theoretical_loss": 3.570427013244852, + "tokens_seen": 1264058368 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031159478435305917, + "loss": 1.5259, + "theoretical_loss": 3.570393292130565, + "tokens_seen": 1264189440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031157472417251753, + "loss": 1.3368, + "theoretical_loss": 3.5703595754911523, + "tokens_seen": 1264320512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031155466399197595, + "loss": 1.1934, + "theoretical_loss": 3.570325863325557, + "tokens_seen": 1264451584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115346038114343, + "loss": 1.3753, + "theoretical_loss": 3.5702921556327203, + "tokens_seen": 1264582656 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031151454363089273, + "loss": 1.3961, + "theoretical_loss": 3.5702584524115863, + "tokens_seen": 1264713728 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.3732282519340515, + "objective/train/avg_token_score": 0.007788247894495726, + "objective/train/docs_used": 717941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2194559574127197, + "objective/train/original_loss": 3.322699785232544, + "objective/train/theoretical_loss": 3.5702247536610976, + "objective/train/tokens_used": 1264844800, + "objective/train/unlikelihood_loss": 0.26282376050949097, + "theoretical_loss": 3.5702247536610976, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114944834503511, + "loss": 1.1627, + "theoretical_loss": 3.5702247536610976, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031147442326980945, + "loss": 1.3738, + "theoretical_loss": 3.570191059380199, + "tokens_seen": 1264975872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114543630892678, + "loss": 1.4925, + "theoretical_loss": 3.5701573695678346, + "tokens_seen": 1265106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031143430290872623, + "loss": 1.4583, + "theoretical_loss": 3.5701236842229482, + "tokens_seen": 1265238016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114142427281846, + "loss": 1.2399, + "theoretical_loss": 3.570090003344485, + "tokens_seen": 1265369088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031139418254764296, + "loss": 1.4397, + "theoretical_loss": 3.57005632693139, + "tokens_seen": 1265500160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113741223671013, + "loss": 1.4931, + "theoretical_loss": 3.5700226549826093, + "tokens_seen": 1265631232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113540621865597, + "loss": 1.2753, + "theoretical_loss": 3.569988987497088, + "tokens_seen": 1265762304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113340020060181, + "loss": 1.3899, + "theoretical_loss": 3.5699553244737734, + "tokens_seen": 1265893376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031131394182547646, + "loss": 1.4489, + "theoretical_loss": 3.569921665911612, + "tokens_seen": 1266024448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112938816449348, + "loss": 1.3193, + "theoretical_loss": 3.56988801180955, + "tokens_seen": 1266155520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112738214643932, + "loss": 1.3423, + "theoretical_loss": 3.5698543621665353, + "tokens_seen": 1266286592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112537612838516, + "loss": 1.4218, + "theoretical_loss": 3.569820716981516, + "tokens_seen": 1266417664 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.5047042965888977, + "objective/train/avg_token_score": 0.01323844026774168, + "objective/train/docs_used": 718481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5929802656173706, + "objective/train/original_loss": 3.8222198486328125, + "objective/train/theoretical_loss": 3.5698038960604257, + "objective/train/tokens_used": 1266483200, + "objective/train/unlikelihood_loss": 0.14763601124286652, + "theoretical_loss": 3.5698038960604257, + "tokens_seen": 1266483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031123370110330996, + "loss": 1.1256, + "theoretical_loss": 3.56978707625344, + "tokens_seen": 1266548736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112136409227683, + "loss": 1.3925, + "theoretical_loss": 3.5697534399812545, + "tokens_seen": 1266679808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111935807422267, + "loss": 1.3952, + "theoretical_loss": 3.56971980816391, + "tokens_seen": 1266810880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031117352056168505, + "loss": 1.3429, + "theoretical_loss": 3.5696861808003555, + "tokens_seen": 1266941952 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031115346038114347, + "loss": 1.3276, + "theoretical_loss": 3.5696525578895395, + "tokens_seen": 1267073024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031113340020060183, + "loss": 1.3956, + "theoretical_loss": 3.5696189394304128, + "tokens_seen": 1267204096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111133400200602, + "loss": 1.259, + "theoretical_loss": 3.569585325421925, + "tokens_seen": 1267335168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031109327983951855, + "loss": 1.3333, + "theoretical_loss": 3.5695517158630268, + "tokens_seen": 1267466240 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031107321965897697, + "loss": 1.4706, + "theoretical_loss": 3.5695181107526697, + "tokens_seen": 1267597312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031105315947843533, + "loss": 1.3512, + "theoretical_loss": 3.5694845100898043, + "tokens_seen": 1267728384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110330992978937, + "loss": 1.5141, + "theoretical_loss": 3.569450913873382, + "tokens_seen": 1267859456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031101303911735206, + "loss": 1.4778, + "theoretical_loss": 3.5694173221023564, + "tokens_seen": 1267990528 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.5081866979598999, + "objective/train/avg_token_score": 0.006329500116407871, + "objective/train/docs_used": 719716, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8324867486953735, + "objective/train/original_loss": 3.780313491821289, + "objective/train/theoretical_loss": 3.5693837347756783, + "objective/train/tokens_used": 1268121600, + "objective/train/unlikelihood_loss": 0.17310085892677307, + "theoretical_loss": 3.5693837347756783, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109929789368104, + "loss": 1.4585, + "theoretical_loss": 3.5693837347756783, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031097291875626883, + "loss": 1.3625, + "theoretical_loss": 3.5693501518923005, + "tokens_seen": 1268252672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109528585757272, + "loss": 1.3915, + "theoretical_loss": 3.5693165734511774, + "tokens_seen": 1268383744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031093279839518556, + "loss": 1.3458, + "theoretical_loss": 3.5692829994512607, + "tokens_seen": 1268514816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109127382146439, + "loss": 1.3571, + "theoretical_loss": 3.5692494298915056, + "tokens_seen": 1268645888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031089267803410234, + "loss": 1.4234, + "theoretical_loss": 3.5692158647708654, + "tokens_seen": 1268776960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108726178535607, + "loss": 1.415, + "theoretical_loss": 3.5691823040882946, + "tokens_seen": 1268908032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031085255767301906, + "loss": 1.3577, + "theoretical_loss": 3.569148747842749, + "tokens_seen": 1269039104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108324974924774, + "loss": 1.2972, + "theoretical_loss": 3.5691151960331826, + "tokens_seen": 1269170176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108124373119358, + "loss": 1.3252, + "theoretical_loss": 3.5690816486585515, + "tokens_seen": 1269301248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107923771313942, + "loss": 1.4657, + "theoretical_loss": 3.5690481057178114, + "tokens_seen": 1269432320 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031077231695085257, + "loss": 1.3604, + "theoretical_loss": 3.5690145672099187, + "tokens_seen": 1269563392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031075225677031093, + "loss": 1.3907, + "theoretical_loss": 3.56898103313383, + "tokens_seen": 1269694464 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.46041056513786316, + "objective/train/avg_token_score": 0.02471236139535904, + "objective/train/docs_used": 720440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4541411399841309, + "objective/train/original_loss": 3.695448160171509, + "objective/train/theoretical_loss": 3.5689642677573863, + "objective/train/tokens_used": 1269760000, + "objective/train/unlikelihood_loss": 0.17488782107830048, + "theoretical_loss": 3.5689642677573863, + "tokens_seen": 1269760000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107321965897693, + "loss": 1.3652, + "theoretical_loss": 3.568947503488503, + "tokens_seen": 1269825536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107121364092277, + "loss": 1.2792, + "theoretical_loss": 3.568913978272893, + "tokens_seen": 1269956608 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031069207622868607, + "loss": 1.334, + "theoretical_loss": 3.5688804574859603, + "tokens_seen": 1270087680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031067201604814443, + "loss": 1.3953, + "theoretical_loss": 3.568846941126661, + "tokens_seen": 1270218752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106519558676028, + "loss": 1.3987, + "theoretical_loss": 3.568813429193954, + "tokens_seen": 1270349824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106318956870612, + "loss": 1.6009, + "theoretical_loss": 3.568779921686798, + "tokens_seen": 1270480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031061183550651957, + "loss": 1.329, + "theoretical_loss": 3.568746418604153, + "tokens_seen": 1270611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031059177532597793, + "loss": 1.3981, + "theoretical_loss": 3.568712919944977, + "tokens_seen": 1270743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105717151454363, + "loss": 1.4198, + "theoretical_loss": 3.568679425708231, + "tokens_seen": 1270874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031055165496489466, + "loss": 1.4523, + "theoretical_loss": 3.5686459358928744, + "tokens_seen": 1271005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105315947843531, + "loss": 1.2989, + "theoretical_loss": 3.5686124504978673, + "tokens_seen": 1271136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031051153460381144, + "loss": 1.3444, + "theoretical_loss": 3.5685789695221715, + "tokens_seen": 1271267328 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.5382453799247742, + "objective/train/avg_token_score": 0.009136535227298737, + "objective/train/docs_used": 721295, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6398098468780518, + "objective/train/original_loss": 3.5170607566833496, + "objective/train/theoretical_loss": 3.5685454929647475, + "objective/train/tokens_used": 1271398400, + "objective/train/unlikelihood_loss": 0.14550815522670746, + "theoretical_loss": 3.5685454929647475, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104914744232698, + "loss": 1.4126, + "theoretical_loss": 3.5685454929647475, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031047141424272816, + "loss": 1.4392, + "theoretical_loss": 3.5685120208245578, + "tokens_seen": 1271529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104513540621866, + "loss": 1.3121, + "theoretical_loss": 3.5684785531005634, + "tokens_seen": 1271660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031043129388164494, + "loss": 1.5538, + "theoretical_loss": 3.5684450897917266, + "tokens_seen": 1271791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104112337011033, + "loss": 1.3559, + "theoretical_loss": 3.5684116308970104, + "tokens_seen": 1271922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031039117352056167, + "loss": 1.4151, + "theoretical_loss": 3.5683781764153775, + "tokens_seen": 1272053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031037111334002003, + "loss": 1.4626, + "theoretical_loss": 3.5683447263457913, + "tokens_seen": 1272184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031035105315947844, + "loss": 1.3913, + "theoretical_loss": 3.5683112806872157, + "tokens_seen": 1272315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103309929789368, + "loss": 1.4966, + "theoretical_loss": 3.568277839438614, + "tokens_seen": 1272446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031031093279839517, + "loss": 1.5095, + "theoretical_loss": 3.568244402598951, + "tokens_seen": 1272578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031029087261785353, + "loss": 1.5394, + "theoretical_loss": 3.5682109701671916, + "tokens_seen": 1272709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031027081243731195, + "loss": 1.3874, + "theoretical_loss": 3.5681775421423003, + "tokens_seen": 1272840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102507522567703, + "loss": 1.4284, + "theoretical_loss": 3.568144118523243, + "tokens_seen": 1272971264 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.4120234549045563, + "objective/train/avg_token_score": 0.015620015561580658, + "objective/train/docs_used": 722001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3662954568862915, + "objective/train/original_loss": 3.5743255615234375, + "objective/train/theoretical_loss": 3.5681274083655787, + "objective/train/tokens_used": 1273036800, + "objective/train/unlikelihood_loss": 0.1855190545320511, + "theoretical_loss": 3.5681274083655787, + "tokens_seen": 1273036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031023069207622867, + "loss": 1.4225, + "theoretical_loss": 3.5681106993089853, + "tokens_seen": 1273102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031021063189568703, + "loss": 1.5452, + "theoretical_loss": 3.568077284498493, + "tokens_seen": 1273233408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101905717151454, + "loss": 1.4723, + "theoretical_loss": 3.5680438740907334, + "tokens_seen": 1273364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101705115346038, + "loss": 1.2989, + "theoretical_loss": 3.5680104680846725, + "tokens_seen": 1273495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101504513540622, + "loss": 1.407, + "theoretical_loss": 3.567977066479277, + "tokens_seen": 1273626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031013039117352054, + "loss": 1.3754, + "theoretical_loss": 3.5679436692735154, + "tokens_seen": 1273757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101103309929789, + "loss": 1.3362, + "theoretical_loss": 3.5679102764663555, + "tokens_seen": 1273888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100902708124373, + "loss": 1.4053, + "theoretical_loss": 3.567876888056765, + "tokens_seen": 1274019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100702106318957, + "loss": 1.2508, + "theoretical_loss": 3.5678435040437124, + "tokens_seen": 1274150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100501504513541, + "loss": 1.5247, + "theoretical_loss": 3.5678101244261677, + "tokens_seen": 1274281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031003009027081246, + "loss": 1.424, + "theoretical_loss": 3.567776749203098, + "tokens_seen": 1274413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100100300902708, + "loss": 1.4478, + "theoretical_loss": 3.567743378373475, + "tokens_seen": 1274544128 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.40860214829444885, + "objective/train/avg_token_score": 0.06949402391910553, + "objective/train/docs_used": 723183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4152452945709229, + "objective/train/original_loss": 4.066036224365234, + "objective/train/theoretical_loss": 3.5677100119362675, + "objective/train/tokens_used": 1274675200, + "objective/train/unlikelihood_loss": 0.13668085634708405, + "theoretical_loss": 3.5677100119362675, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030998996990972924, + "loss": 1.4389, + "theoretical_loss": 3.5677100119362675, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099699097291876, + "loss": 1.2729, + "theoretical_loss": 3.567676649890446, + "tokens_seen": 1274806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030994984954864596, + "loss": 1.2524, + "theoretical_loss": 3.5676432922349814, + "tokens_seen": 1274937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099297893681043, + "loss": 1.3814, + "theoretical_loss": 3.567609938968844, + "tokens_seen": 1275068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030990972918756274, + "loss": 1.5211, + "theoretical_loss": 3.5675765900910057, + "tokens_seen": 1275199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098896690070211, + "loss": 1.4278, + "theoretical_loss": 3.5675432456004383, + "tokens_seen": 1275330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030986960882647946, + "loss": 1.3237, + "theoretical_loss": 3.567509905496113, + "tokens_seen": 1275461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098495486459378, + "loss": 1.3475, + "theoretical_loss": 3.5674765697770034, + "tokens_seen": 1275592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098294884653962, + "loss": 1.3699, + "theoretical_loss": 3.5674432384420807, + "tokens_seen": 1275723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098094282848546, + "loss": 1.4132, + "theoretical_loss": 3.5674099114903193, + "tokens_seen": 1275854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030978936810431297, + "loss": 1.4973, + "theoretical_loss": 3.567376588920692, + "tokens_seen": 1275985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030976930792377133, + "loss": 1.4459, + "theoretical_loss": 3.567343270732172, + "tokens_seen": 1276116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097492477432297, + "loss": 1.5469, + "theoretical_loss": 3.5673099569237348, + "tokens_seen": 1276248064 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.34384164214134216, + "objective/train/avg_token_score": 0.005261159967631102, + "objective/train/docs_used": 723876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1645642518997192, + "objective/train/original_loss": 3.698387861251831, + "objective/train/theoretical_loss": 3.567293301661726, + "objective/train/tokens_used": 1276313600, + "objective/train/unlikelihood_loss": 0.20447641611099243, + "theoretical_loss": 3.567293301661726, + "tokens_seen": 1276313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097291875626881, + "loss": 1.4203, + "theoretical_loss": 3.567276647494354, + "tokens_seen": 1276379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030970912738214647, + "loss": 1.2919, + "theoretical_loss": 3.5672433424430037, + "tokens_seen": 1276510208 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030968906720160483, + "loss": 1.4128, + "theoretical_loss": 3.56721004176866, + "tokens_seen": 1276641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096690070210632, + "loss": 1.2594, + "theoretical_loss": 3.5671767454702983, + "tokens_seen": 1276772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096489468405216, + "loss": 1.3538, + "theoretical_loss": 3.567143453546894, + "tokens_seen": 1276903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030962888665997997, + "loss": 1.415, + "theoretical_loss": 3.5671101659974234, + "tokens_seen": 1277034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030960882647943833, + "loss": 1.1233, + "theoretical_loss": 3.567076882820863, + "tokens_seen": 1277165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095887662988967, + "loss": 1.4714, + "theoretical_loss": 3.56704360401619, + "tokens_seen": 1277296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030956870611835506, + "loss": 1.4641, + "theoretical_loss": 3.5670103295823816, + "tokens_seen": 1277427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095486459378135, + "loss": 1.5463, + "theoretical_loss": 3.5669770595184143, + "tokens_seen": 1277558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030952858575727184, + "loss": 1.3929, + "theoretical_loss": 3.566943793823267, + "tokens_seen": 1277689856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095085255767302, + "loss": 1.1982, + "theoretical_loss": 3.5669105324959176, + "tokens_seen": 1277820928 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.46212121844291687, + "objective/train/avg_token_score": 0.01487169973552227, + "objective/train/docs_used": 725220, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6353477239608765, + "objective/train/original_loss": 3.98511004447937, + "objective/train/theoretical_loss": 3.566877275535345, + "objective/train/tokens_used": 1277952000, + "objective/train/unlikelihood_loss": 0.13372761011123657, + "theoretical_loss": 3.566877275535345, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030948846539618856, + "loss": 1.4226, + "theoretical_loss": 3.566877275535345, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309468405215647, + "loss": 1.4714, + "theoretical_loss": 3.566844022940528, + "tokens_seen": 1278083072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030944834503510534, + "loss": 1.5258, + "theoretical_loss": 3.5668107747104454, + "tokens_seen": 1278214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094282848545637, + "loss": 1.4053, + "theoretical_loss": 3.566777530844077, + "tokens_seen": 1278345216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030940822467402207, + "loss": 1.4034, + "theoretical_loss": 3.5667442913404024, + "tokens_seen": 1278476288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030938816449348043, + "loss": 1.5157, + "theoretical_loss": 3.566711056198403, + "tokens_seen": 1278607360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030936810431293884, + "loss": 1.4448, + "theoretical_loss": 3.5666778254170586, + "tokens_seen": 1278738432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093480441323972, + "loss": 1.5438, + "theoretical_loss": 3.5666445989953504, + "tokens_seen": 1278869504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030932798395185557, + "loss": 1.3526, + "theoretical_loss": 3.5666113769322596, + "tokens_seen": 1279000576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030930792377131393, + "loss": 1.6237, + "theoretical_loss": 3.5665781592267676, + "tokens_seen": 1279131648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030928786359077235, + "loss": 1.3544, + "theoretical_loss": 3.5665449458778573, + "tokens_seen": 1279262720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092678034102307, + "loss": 1.4939, + "theoretical_loss": 3.56651173688451, + "tokens_seen": 1279393792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030924774322968907, + "loss": 1.4467, + "theoretical_loss": 3.566478532245709, + "tokens_seen": 1279524864 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.33962610363960266, + "objective/train/avg_token_score": 0.022915367037057877, + "objective/train/docs_used": 725568, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1759028434753418, + "objective/train/original_loss": 3.902873992919922, + "objective/train/theoretical_loss": 3.566461931558945, + "objective/train/tokens_used": 1279590400, + "objective/train/unlikelihood_loss": 0.22344884276390076, + "theoretical_loss": 3.566461931558945, + "tokens_seen": 1279590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030922768304914743, + "loss": 1.296, + "theoretical_loss": 3.566445331960437, + "tokens_seen": 1279655936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092076228686058, + "loss": 1.4555, + "theoretical_loss": 3.5664121360276777, + "tokens_seen": 1279787008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091875626880642, + "loss": 1.4883, + "theoretical_loss": 3.566378944446414, + "tokens_seen": 1279918080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091675025075226, + "loss": 1.5248, + "theoretical_loss": 3.5663457572156307, + "tokens_seen": 1280049152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030914744232698094, + "loss": 1.6415, + "theoretical_loss": 3.5663125743343125, + "tokens_seen": 1280180224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091273821464393, + "loss": 1.3082, + "theoretical_loss": 3.566279395801444, + "tokens_seen": 1280311296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091073219658977, + "loss": 1.5049, + "theoretical_loss": 3.566246221616009, + "tokens_seen": 1280442368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090872617853561, + "loss": 1.4744, + "theoretical_loss": 3.5662130517769945, + "tokens_seen": 1280573440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030906720160481444, + "loss": 1.33, + "theoretical_loss": 3.5661798862833853, + "tokens_seen": 1280704512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090471414242728, + "loss": 1.4715, + "theoretical_loss": 3.5661467251341685, + "tokens_seen": 1280835584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030902708124373116, + "loss": 1.5844, + "theoretical_loss": 3.566113568328329, + "tokens_seen": 1280966656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090070210631896, + "loss": 1.2599, + "theoretical_loss": 3.566080415864855, + "tokens_seen": 1281097728 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.40041545033454895, + "objective/train/avg_token_score": 0.00788168702274561, + "objective/train/docs_used": 726917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2657763957977295, + "objective/train/original_loss": 3.5127954483032227, + "objective/train/theoretical_loss": 3.566047267742733, + "objective/train/tokens_used": 1281228800, + "objective/train/unlikelihood_loss": 0.1803327351808548, + "theoretical_loss": 3.566047267742733, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030898696088264794, + "loss": 1.5979, + "theoretical_loss": 3.566047267742733, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089669007021063, + "loss": 1.3726, + "theoretical_loss": 3.56601412396095, + "tokens_seen": 1281359872 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030894684052156467, + "loss": 1.4882, + "theoretical_loss": 3.565980984518495, + "tokens_seen": 1281490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089267803410231, + "loss": 1.4628, + "theoretical_loss": 3.565947849414355, + "tokens_seen": 1281622016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030890672016048145, + "loss": 1.438, + "theoretical_loss": 3.5659147186475186, + "tokens_seen": 1281753088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088866599799398, + "loss": 1.4196, + "theoretical_loss": 3.565881592216975, + "tokens_seen": 1281884160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030886659979939817, + "loss": 1.544, + "theoretical_loss": 3.565848470121714, + "tokens_seen": 1282015232 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030884653961885653, + "loss": 1.5026, + "theoretical_loss": 3.5658153523607234, + "tokens_seen": 1282146304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030882647943831495, + "loss": 1.4324, + "theoretical_loss": 3.5657822389329947, + "tokens_seen": 1282277376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088064192577733, + "loss": 1.2546, + "theoretical_loss": 3.565749129837517, + "tokens_seen": 1282408448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087863590772317, + "loss": 1.3389, + "theoretical_loss": 3.5657160250732813, + "tokens_seen": 1282539520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030876629889669004, + "loss": 1.3817, + "theoretical_loss": 3.565682924639278, + "tokens_seen": 1282670592 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030874623871614845, + "loss": 1.3299, + "theoretical_loss": 3.565649828534499, + "tokens_seen": 1282801664 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.32429128885269165, + "objective/train/avg_token_score": 0.005629888270050287, + "objective/train/docs_used": 727506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0914019346237183, + "objective/train/original_loss": 3.9509170055389404, + "objective/train/theoretical_loss": 3.565633282105253, + "objective/train/tokens_used": 1282867200, + "objective/train/unlikelihood_loss": 0.17031577229499817, + "theoretical_loss": 3.565633282105253, + "tokens_seen": 1282867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087261785356068, + "loss": 1.3479, + "theoretical_loss": 3.565616736757935, + "tokens_seen": 1282932736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087061183550652, + "loss": 1.4571, + "theoretical_loss": 3.5655836493085786, + "tokens_seen": 1283063808 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030868605817452354, + "loss": 1.3245, + "theoretical_loss": 3.5655505661854217, + "tokens_seen": 1283194880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086659979939819, + "loss": 1.3448, + "theoretical_loss": 3.5655174873874564, + "tokens_seen": 1283325952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086459378134403, + "loss": 1.4586, + "theoretical_loss": 3.5654844129136762, + "tokens_seen": 1283457024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086258776328987, + "loss": 1.2927, + "theoretical_loss": 3.5654513427630743, + "tokens_seen": 1283588096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030860581745235704, + "loss": 1.512, + "theoretical_loss": 3.5654182769346434, + "tokens_seen": 1283719168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085857572718154, + "loss": 1.2904, + "theoretical_loss": 3.565385215427378, + "tokens_seen": 1283850240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085656970912738, + "loss": 1.4045, + "theoretical_loss": 3.565352158240273, + "tokens_seen": 1283981312 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030854563691073224, + "loss": 1.3385, + "theoretical_loss": 3.565319105372322, + "tokens_seen": 1284112384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085255767301906, + "loss": 1.4726, + "theoretical_loss": 3.5652860568225204, + "tokens_seen": 1284243456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030850551654964896, + "loss": 1.4022, + "theoretical_loss": 3.5652530125898627, + "tokens_seen": 1284374528 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.48448190093040466, + "objective/train/avg_token_score": 0.003776711178943515, + "objective/train/docs_used": 728764, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5325560569763184, + "objective/train/original_loss": 3.510700225830078, + "objective/train/theoretical_loss": 3.5652199726733453, + "objective/train/tokens_used": 1284505600, + "objective/train/unlikelihood_loss": 0.1880500614643097, + "theoretical_loss": 3.5652199726733453, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084854563691074, + "loss": 1.3353, + "theoretical_loss": 3.5652199726733453, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030846539618856574, + "loss": 1.4526, + "theoretical_loss": 3.5651869370719633, + "tokens_seen": 1284636672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084453360080241, + "loss": 1.4368, + "theoretical_loss": 3.565153905784714, + "tokens_seen": 1284767744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030842527582748247, + "loss": 1.3997, + "theoretical_loss": 3.565120878810593, + "tokens_seen": 1284898816 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030840521564694083, + "loss": 1.3169, + "theoretical_loss": 3.565087856148598, + "tokens_seen": 1285029888 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030838515546639924, + "loss": 1.5256, + "theoretical_loss": 3.5650548377977254, + "tokens_seen": 1285160960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083650952858576, + "loss": 1.4068, + "theoretical_loss": 3.5650218237569735, + "tokens_seen": 1285292032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030834503510531597, + "loss": 1.516, + "theoretical_loss": 3.5649888140253396, + "tokens_seen": 1285423104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030832497492477433, + "loss": 1.4235, + "theoretical_loss": 3.5649558086018223, + "tokens_seen": 1285554176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030830491474423275, + "loss": 1.4207, + "theoretical_loss": 3.564922807485421, + "tokens_seen": 1285685248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082848545636911, + "loss": 1.2794, + "theoretical_loss": 3.5648898106751328, + "tokens_seen": 1285816320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030826479438314947, + "loss": 1.3699, + "theoretical_loss": 3.5648568181699583, + "tokens_seen": 1285947392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030824473420260783, + "loss": 1.3906, + "theoretical_loss": 3.564823829968897, + "tokens_seen": 1286078464 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.4665811359882355, + "objective/train/avg_token_score": 0.019542723894119263, + "objective/train/docs_used": 729512, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3884855508804321, + "objective/train/original_loss": 3.9265525341033936, + "objective/train/theoretical_loss": 3.5648073374820957, + "objective/train/tokens_used": 1286144000, + "objective/train/unlikelihood_loss": 0.14879894256591797, + "theoretical_loss": 3.5648073374820957, + "tokens_seen": 1286144000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082246740220662, + "loss": 1.3596, + "theoretical_loss": 3.564790846070948, + "tokens_seen": 1286209536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082046138415246, + "loss": 1.2556, + "theoretical_loss": 3.5647578664751123, + "tokens_seen": 1286340608 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308184553660983, + "loss": 1.52, + "theoretical_loss": 3.564724891180391, + "tokens_seen": 1286471680 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030816449348044134, + "loss": 1.4755, + "theoretical_loss": 3.5646919201857834, + "tokens_seen": 1286602752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081444332998997, + "loss": 1.3665, + "theoretical_loss": 3.564658953490292, + "tokens_seen": 1286733824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081243731193581, + "loss": 1.221, + "theoretical_loss": 3.5646259910929183, + "tokens_seen": 1286864896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081043129388165, + "loss": 1.1911, + "theoretical_loss": 3.564593032992664, + "tokens_seen": 1286995968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030808425275827484, + "loss": 1.4986, + "theoretical_loss": 3.5645600791885315, + "tokens_seen": 1287127040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080641925777332, + "loss": 1.4196, + "theoretical_loss": 3.564527129679523, + "tokens_seen": 1287258112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030804413239719157, + "loss": 1.466, + "theoretical_loss": 3.5644941844646416, + "tokens_seen": 1287389184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030802407221665, + "loss": 1.4086, + "theoretical_loss": 3.5644612435428913, + "tokens_seen": 1287520256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030800401203610834, + "loss": 1.4173, + "theoretical_loss": 3.5644283069132743, + "tokens_seen": 1287651328 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.44776391983032227, + "objective/train/avg_token_score": 0.0032580653205513954, + "objective/train/docs_used": 730476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.453680396080017, + "objective/train/original_loss": 3.5461320877075195, + "objective/train/theoretical_loss": 3.564395374574796, + "objective/train/tokens_used": 1287782400, + "objective/train/unlikelihood_loss": 0.20559458434581757, + "theoretical_loss": 3.564395374574796, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079839518555667, + "loss": 1.3643, + "theoretical_loss": 3.564395374574796, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030796389167502507, + "loss": 1.2831, + "theoretical_loss": 3.5643624465264594, + "tokens_seen": 1287913472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079438314944835, + "loss": 1.2369, + "theoretical_loss": 3.5643295227672698, + "tokens_seen": 1288044544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030792377131394185, + "loss": 1.414, + "theoretical_loss": 3.564296603296232, + "tokens_seen": 1288175616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079037111334002, + "loss": 1.3611, + "theoretical_loss": 3.5642636881123515, + "tokens_seen": 1288306688 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030788365095285857, + "loss": 1.3884, + "theoretical_loss": 3.5642307772146333, + "tokens_seen": 1288437760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030786359077231693, + "loss": 1.3235, + "theoretical_loss": 3.5641978706020834, + "tokens_seen": 1288568832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030784353059177535, + "loss": 1.3202, + "theoretical_loss": 3.5641649682737087, + "tokens_seen": 1288699904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078234704112337, + "loss": 1.3319, + "theoretical_loss": 3.5641320702285153, + "tokens_seen": 1288830976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078034102306921, + "loss": 1.3275, + "theoretical_loss": 3.5640991764655103, + "tokens_seen": 1288962048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030778335005015044, + "loss": 1.506, + "theoretical_loss": 3.5640662869837003, + "tokens_seen": 1289093120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030776328986960885, + "loss": 1.4511, + "theoretical_loss": 3.564033401782094, + "tokens_seen": 1289224192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077432296890672, + "loss": 1.455, + "theoretical_loss": 3.564000520859698, + "tokens_seen": 1289355264 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.43291789293289185, + "objective/train/avg_token_score": 0.0034780579153448343, + "objective/train/docs_used": 731120, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.482696294784546, + "objective/train/original_loss": 3.57106614112854, + "objective/train/theoretical_loss": 3.5639840820028947, + "objective/train/tokens_used": 1289420800, + "objective/train/unlikelihood_loss": 0.16511254012584686, + "theoretical_loss": 3.5639840820028947, + "tokens_seen": 1289420800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077231695085256, + "loss": 1.4215, + "theoretical_loss": 3.5639676442155217, + "tokens_seen": 1289486336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030770310932798394, + "loss": 1.3182, + "theoretical_loss": 3.563934771848573, + "tokens_seen": 1289617408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076830491474423, + "loss": 1.3978, + "theoretical_loss": 3.5639019037578614, + "tokens_seen": 1289748480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076629889669007, + "loss": 1.3047, + "theoretical_loss": 3.5638690399423956, + "tokens_seen": 1289879552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076429287863591, + "loss": 1.3933, + "theoretical_loss": 3.563836180401185, + "tokens_seen": 1290010624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030762286860581744, + "loss": 1.4588, + "theoretical_loss": 3.56380332513324, + "tokens_seen": 1290141696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076028084252758, + "loss": 1.4967, + "theoretical_loss": 3.563770474137571, + "tokens_seen": 1290272768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075827482447342, + "loss": 1.4998, + "theoretical_loss": 3.5637376274131873, + "tokens_seen": 1290403840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075626880641926, + "loss": 1.4122, + "theoretical_loss": 3.563704784959101, + "tokens_seen": 1290534912 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030754262788365095, + "loss": 1.4344, + "theoretical_loss": 3.563671946774323, + "tokens_seen": 1290665984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075225677031093, + "loss": 1.3181, + "theoretical_loss": 3.563639112857865, + "tokens_seen": 1290797056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075025075225677, + "loss": 1.4924, + "theoretical_loss": 3.5636062832087383, + "tokens_seen": 1290928128 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.45173507928848267, + "objective/train/avg_token_score": 0.033176932483911514, + "objective/train/docs_used": 732356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4072487354278564, + "objective/train/original_loss": 3.9873006343841553, + "objective/train/theoretical_loss": 3.5635734578259557, + "objective/train/tokens_used": 1291059200, + "objective/train/unlikelihood_loss": 0.09515228867530823, + "theoretical_loss": 3.5635734578259557, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074824473420261, + "loss": 1.5611, + "theoretical_loss": 3.5635734578259557, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030746238716148445, + "loss": 1.3604, + "theoretical_loss": 3.563540636708529, + "tokens_seen": 1291190272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074423269809428, + "loss": 1.3811, + "theoretical_loss": 3.563507819855472, + "tokens_seen": 1291321344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074222668004012, + "loss": 1.538, + "theoretical_loss": 3.5634750072657972, + "tokens_seen": 1291452416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074022066198596, + "loss": 1.3537, + "theoretical_loss": 3.5634421989385183, + "tokens_seen": 1291583488 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030738214643931795, + "loss": 1.3575, + "theoretical_loss": 3.5634093948726493, + "tokens_seen": 1291714560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073620862587763, + "loss": 1.2927, + "theoretical_loss": 3.5633765950672043, + "tokens_seen": 1291845632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073420260782347, + "loss": 1.4631, + "theoretical_loss": 3.5633437995211974, + "tokens_seen": 1291976704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073219658976931, + "loss": 1.3941, + "theoretical_loss": 3.563311008233644, + "tokens_seen": 1292107776 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030730190571715146, + "loss": 1.3735, + "theoretical_loss": 3.563278221203559, + "tokens_seen": 1292238848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072818455366098, + "loss": 1.4794, + "theoretical_loss": 3.5632454384299574, + "tokens_seen": 1292369920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072617853560682, + "loss": 1.38, + "theoretical_loss": 3.563212659911856, + "tokens_seen": 1292500992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030724172517552654, + "loss": 1.4487, + "theoretical_loss": 3.56317988564827, + "tokens_seen": 1292632064 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.39668866991996765, + "objective/train/avg_token_score": 0.011018197052180767, + "objective/train/docs_used": 733109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.081235408782959, + "objective/train/original_loss": 3.5113918781280518, + "objective/train/theoretical_loss": 3.5631635001116138, + "objective/train/tokens_used": 1292697600, + "objective/train/unlikelihood_loss": 0.1911245435476303, + "theoretical_loss": 3.5631635001116138, + "tokens_seen": 1292697600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030722166499498496, + "loss": 1.4697, + "theoretical_loss": 3.5631471156382166, + "tokens_seen": 1292763136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072016048144433, + "loss": 1.3995, + "theoretical_loss": 3.5631143498807125, + "tokens_seen": 1292894208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071815446339017, + "loss": 1.3431, + "theoretical_loss": 3.563081588374774, + "tokens_seen": 1293025280 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030716148445336005, + "loss": 1.4452, + "theoretical_loss": 3.5630488311194193, + "tokens_seen": 1293156352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030714142427281846, + "loss": 1.5393, + "theoretical_loss": 3.563016078113666, + "tokens_seen": 1293287424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071213640922768, + "loss": 1.2204, + "theoretical_loss": 3.5629833293565323, + "tokens_seen": 1293418496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071013039117352, + "loss": 1.3788, + "theoretical_loss": 3.562950584847036, + "tokens_seen": 1293549568 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030708124373119355, + "loss": 1.4493, + "theoretical_loss": 3.562917844584197, + "tokens_seen": 1293680640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070611835506519, + "loss": 1.3664, + "theoretical_loss": 3.562885108567033, + "tokens_seen": 1293811712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070411233701104, + "loss": 1.357, + "theoretical_loss": 3.5628523767945643, + "tokens_seen": 1293942784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030702106318956874, + "loss": 1.3957, + "theoretical_loss": 3.562819649265811, + "tokens_seen": 1294073856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070010030090271, + "loss": 1.4159, + "theoretical_loss": 3.5627869259797924, + "tokens_seen": 1294204928 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.5008553266525269, + "objective/train/avg_token_score": 0.006392046343535185, + "objective/train/docs_used": 734261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5740562677383423, + "objective/train/original_loss": 3.559659719467163, + "objective/train/theoretical_loss": 3.5627542069355282, + "objective/train/tokens_used": 1294336000, + "objective/train/unlikelihood_loss": 0.1881576031446457, + "theoretical_loss": 3.5627542069355282, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030698094282848547, + "loss": 1.4227, + "theoretical_loss": 3.5627542069355282, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069608826479439, + "loss": 1.3421, + "theoretical_loss": 3.5627214921320407, + "tokens_seen": 1294467072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030694082246740225, + "loss": 1.3638, + "theoretical_loss": 3.56268878156835, + "tokens_seen": 1294598144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069207622868606, + "loss": 1.4074, + "theoretical_loss": 3.5626560752434777, + "tokens_seen": 1294729216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030690070210631897, + "loss": 1.3221, + "theoretical_loss": 3.5626233731564456, + "tokens_seen": 1294860288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030688064192577733, + "loss": 1.399, + "theoretical_loss": 3.562590675306276, + "tokens_seen": 1294991360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030686058174523575, + "loss": 1.4099, + "theoretical_loss": 3.56255798169199, + "tokens_seen": 1295122432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068405215646941, + "loss": 1.3548, + "theoretical_loss": 3.562525292312611, + "tokens_seen": 1295253504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068204613841525, + "loss": 1.3936, + "theoretical_loss": 3.562492607167162, + "tokens_seen": 1295384576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030680040120361084, + "loss": 1.2458, + "theoretical_loss": 3.5624599262546663, + "tokens_seen": 1295515648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030678034102306925, + "loss": 1.3739, + "theoretical_loss": 3.5624272495741476, + "tokens_seen": 1295646720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067602808425276, + "loss": 1.3599, + "theoretical_loss": 3.5623945771246297, + "tokens_seen": 1295777792 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306740220661986, + "loss": 1.4111, + "theoretical_loss": 3.562361908905137, + "tokens_seen": 1295908864 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.3374877870082855, + "objective/train/avg_token_score": 0.014073585160076618, + "objective/train/docs_used": 734748, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.087935209274292, + "objective/train/original_loss": 3.519335985183716, + "objective/train/theoretical_loss": 3.562345576381345, + "objective/train/tokens_used": 1295974400, + "objective/train/unlikelihood_loss": 0.21748222410678864, + "theoretical_loss": 3.562345576381345, + "tokens_seen": 1295974400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030672016048144434, + "loss": 1.2029, + "theoretical_loss": 3.562329244914694, + "tokens_seen": 1296039936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067001003009027, + "loss": 1.3936, + "theoretical_loss": 3.5622965851523256, + "tokens_seen": 1296171008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066800401203611, + "loss": 1.4914, + "theoretical_loss": 3.562263929617057, + "tokens_seen": 1296302080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066599799398195, + "loss": 1.3702, + "theoretical_loss": 3.5622312783079133, + "tokens_seen": 1296433152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030663991975927784, + "loss": 1.4175, + "theoretical_loss": 3.5621986312239216, + "tokens_seen": 1296564224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066198595787362, + "loss": 1.3963, + "theoretical_loss": 3.5621659883641072, + "tokens_seen": 1296695296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065997993981946, + "loss": 1.404, + "theoretical_loss": 3.562133349727497, + "tokens_seen": 1296826368 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306579739217653, + "loss": 1.3398, + "theoretical_loss": 3.5621007153131172, + "tokens_seen": 1296957440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030655967903711135, + "loss": 1.4817, + "theoretical_loss": 3.5620680851199964, + "tokens_seen": 1297088512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065396188565697, + "loss": 1.2282, + "theoretical_loss": 3.5620354591471606, + "tokens_seen": 1297219584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065195586760281, + "loss": 1.2827, + "theoretical_loss": 3.5620028373936385, + "tokens_seen": 1297350656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064994984954865, + "loss": 1.3558, + "theoretical_loss": 3.5619702198584577, + "tokens_seen": 1297481728 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.4121456444263458, + "objective/train/avg_token_score": 0.01298209186643362, + "objective/train/docs_used": 735602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3153554201126099, + "objective/train/original_loss": 3.4961352348327637, + "objective/train/theoretical_loss": 3.5619376065406474, + "objective/train/tokens_used": 1297612800, + "objective/train/unlikelihood_loss": 0.19728703796863556, + "theoretical_loss": 3.5619376065406474, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030647943831494485, + "loss": 1.3719, + "theoretical_loss": 3.5619376065406474, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064593781344032, + "loss": 1.4417, + "theoretical_loss": 3.5619049974392354, + "tokens_seen": 1297743872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064393179538616, + "loss": 1.339, + "theoretical_loss": 3.5618723925532514, + "tokens_seen": 1297874944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030641925777332, + "loss": 1.3475, + "theoretical_loss": 3.5618397918817246, + "tokens_seen": 1298006016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030639919759277835, + "loss": 1.4776, + "theoretical_loss": 3.561807195423685, + "tokens_seen": 1298137088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063791374122367, + "loss": 1.3532, + "theoretical_loss": 3.561774603178163, + "tokens_seen": 1298268160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063590772316951, + "loss": 1.4686, + "theoretical_loss": 3.5617420151441888, + "tokens_seen": 1298399232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063390170511535, + "loss": 1.3957, + "theoretical_loss": 3.561709431320792, + "tokens_seen": 1298530304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030631895687061186, + "loss": 1.4539, + "theoretical_loss": 3.5616768517070057, + "tokens_seen": 1298661376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062988966900702, + "loss": 1.4064, + "theoretical_loss": 3.561644276301859, + "tokens_seen": 1298792448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062788365095286, + "loss": 1.4853, + "theoretical_loss": 3.5616117051043856, + "tokens_seen": 1298923520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030625877632898694, + "loss": 1.4024, + "theoretical_loss": 3.5615791381136157, + "tokens_seen": 1299054592 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030623871614844536, + "loss": 1.2636, + "theoretical_loss": 3.5615465753285833, + "tokens_seen": 1299185664 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.5162512063980103, + "objective/train/avg_token_score": 0.01830436848104, + "objective/train/docs_used": 736089, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7378356456756592, + "objective/train/original_loss": 4.050041198730469, + "objective/train/theoretical_loss": 3.561530295512916, + "objective/train/tokens_used": 1299251200, + "objective/train/unlikelihood_loss": 0.09324180334806442, + "theoretical_loss": 3.561530295512916, + "tokens_seen": 1299251200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062186559679037, + "loss": 1.5531, + "theoretical_loss": 3.56151401674832, + "tokens_seen": 1299316736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061985957873621, + "loss": 1.552, + "theoretical_loss": 3.5614814623718587, + "tokens_seen": 1299447808 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030617853560682045, + "loss": 1.3214, + "theoretical_loss": 3.5614489121982333, + "tokens_seen": 1299578880 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030615847542627886, + "loss": 1.3598, + "theoretical_loss": 3.561416366226477, + "tokens_seen": 1299709952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061384152457372, + "loss": 1.4293, + "theoretical_loss": 3.561383824455624, + "tokens_seen": 1299841024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061183550651956, + "loss": 1.5027, + "theoretical_loss": 3.561351286884708, + "tokens_seen": 1299972096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030609829488465395, + "loss": 1.3794, + "theoretical_loss": 3.561318753512764, + "tokens_seen": 1300103168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060782347041123, + "loss": 1.4231, + "theoretical_loss": 3.5612862243388275, + "tokens_seen": 1300234240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030605817452357073, + "loss": 1.274, + "theoretical_loss": 3.561253699361932, + "tokens_seen": 1300365312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060381143430291, + "loss": 1.2797, + "theoretical_loss": 3.561221178581114, + "tokens_seen": 1300496384 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030601805416248745, + "loss": 1.4177, + "theoretical_loss": 3.56118866199541, + "tokens_seen": 1300627456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059979939819458, + "loss": 1.5381, + "theoretical_loss": 3.5611561496038555, + "tokens_seen": 1300758528 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.36516374349594116, + "objective/train/avg_token_score": 0.008565003983676434, + "objective/train/docs_used": 737292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.260839819908142, + "objective/train/original_loss": 3.615532636642456, + "objective/train/theoretical_loss": 3.5611236414054868, + "objective/train/tokens_used": 1300889600, + "objective/train/unlikelihood_loss": 0.169486403465271, + "theoretical_loss": 3.5611236414054868, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030597793380140423, + "loss": 1.349, + "theoretical_loss": 3.5611236414054868, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059578736208626, + "loss": 1.3634, + "theoretical_loss": 3.561091137399341, + "tokens_seen": 1301020672 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030593781344032096, + "loss": 1.4607, + "theoretical_loss": 3.5610586375844546, + "tokens_seen": 1301151744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003059177532597793, + "loss": 1.3355, + "theoretical_loss": 3.561026141959866, + "tokens_seen": 1301282816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058976930792377, + "loss": 1.2987, + "theoretical_loss": 3.5609936505246123, + "tokens_seen": 1301413888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058776328986961, + "loss": 1.2775, + "theoretical_loss": 3.5609611632777316, + "tokens_seen": 1301544960 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030585757271815446, + "loss": 1.4226, + "theoretical_loss": 3.5609286802182627, + "tokens_seen": 1301676032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058375125376128, + "loss": 1.2922, + "theoretical_loss": 3.560896201345243, + "tokens_seen": 1301807104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058174523570712, + "loss": 1.3034, + "theoretical_loss": 3.560863726657714, + "tokens_seen": 1301938176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057973921765296, + "loss": 1.2889, + "theoretical_loss": 3.560831256154713, + "tokens_seen": 1302069248 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030577733199598796, + "loss": 1.4819, + "theoretical_loss": 3.5607987898352795, + "tokens_seen": 1302200320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057572718154463, + "loss": 1.4676, + "theoretical_loss": 3.5607663276984547, + "tokens_seen": 1302331392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057372116349047, + "loss": 1.3934, + "theoretical_loss": 3.560733869743278, + "tokens_seen": 1302462464 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.35599949955940247, + "objective/train/avg_token_score": 0.020273858681321144, + "objective/train/docs_used": 737760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1933743953704834, + "objective/train/original_loss": 3.9673187732696533, + "objective/train/theoretical_loss": 3.5607176423335085, + "objective/train/tokens_used": 1302528000, + "objective/train/unlikelihood_loss": 0.16282345354557037, + "theoretical_loss": 3.5607176423335085, + "tokens_seen": 1302528000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030571715145436305, + "loss": 1.3262, + "theoretical_loss": 3.560701415968791, + "tokens_seen": 1302593536 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030569709127382147, + "loss": 1.5048, + "theoretical_loss": 3.560668966374034, + "tokens_seen": 1302724608 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030567703109327983, + "loss": 1.4631, + "theoretical_loss": 3.560636520958047, + "tokens_seen": 1302855680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056569709127382, + "loss": 1.4999, + "theoretical_loss": 3.560604079719874, + "tokens_seen": 1302986752 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030563691073219655, + "loss": 1.4523, + "theoretical_loss": 3.560571642658555, + "tokens_seen": 1303117824 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030561685055165497, + "loss": 1.3759, + "theoretical_loss": 3.560539209773133, + "tokens_seen": 1303248896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030559679037111333, + "loss": 1.4627, + "theoretical_loss": 3.5605067810626503, + "tokens_seen": 1303379968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055767301905717, + "loss": 1.3382, + "theoretical_loss": 3.5604743565261496, + "tokens_seen": 1303511040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030555667001003006, + "loss": 1.5353, + "theoretical_loss": 3.5604419361626745, + "tokens_seen": 1303642112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055366098294885, + "loss": 1.4777, + "theoretical_loss": 3.5604095199712678, + "tokens_seen": 1303773184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055165496489469, + "loss": 1.4961, + "theoretical_loss": 3.5603771079509734, + "tokens_seen": 1303904256 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030549648946840525, + "loss": 1.4406, + "theoretical_loss": 3.5603447001008357, + "tokens_seen": 1304035328 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.4097629487514496, + "objective/train/avg_token_score": 0.01061365008354187, + "objective/train/docs_used": 739145, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3497594594955444, + "objective/train/original_loss": 3.7048537731170654, + "objective/train/theoretical_loss": 3.560312296419899, + "objective/train/tokens_used": 1304166400, + "objective/train/unlikelihood_loss": 0.22053582966327667, + "theoretical_loss": 3.560312296419899, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054764292878636, + "loss": 1.2901, + "theoretical_loss": 3.560312296419899, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305456369107322, + "loss": 1.5343, + "theoretical_loss": 3.5602798969072076, + "tokens_seen": 1304297472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054363089267804, + "loss": 1.3421, + "theoretical_loss": 3.5602475015618067, + "tokens_seen": 1304428544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030541624874623875, + "loss": 1.4728, + "theoretical_loss": 3.5602151103827424, + "tokens_seen": 1304559616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053961885656971, + "loss": 1.1439, + "theoretical_loss": 3.56018272336906, + "tokens_seen": 1304690688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053761283851555, + "loss": 1.5169, + "theoretical_loss": 3.5601503405198045, + "tokens_seen": 1304821760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053560682046139, + "loss": 1.5465, + "theoretical_loss": 3.560117961834023, + "tokens_seen": 1304952832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030533600802407226, + "loss": 1.43, + "theoretical_loss": 3.5600855873107617, + "tokens_seen": 1305083904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053159478435306, + "loss": 1.3205, + "theoretical_loss": 3.5600532169490684, + "tokens_seen": 1305214976 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305295887662989, + "loss": 1.2814, + "theoretical_loss": 3.560020850747989, + "tokens_seen": 1305346048 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030527582748244734, + "loss": 1.2796, + "theoretical_loss": 3.5599884887065727, + "tokens_seen": 1305477120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030525576730190576, + "loss": 1.1815, + "theoretical_loss": 3.5599561308238656, + "tokens_seen": 1305608192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052357071213641, + "loss": 1.3293, + "theoretical_loss": 3.559923777098917, + "tokens_seen": 1305739264 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.3730449676513672, + "objective/train/avg_token_score": 0.02527187578380108, + "objective/train/docs_used": 739729, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0790526866912842, + "objective/train/original_loss": 3.652127742767334, + "objective/train/theoretical_loss": 3.5599076017953046, + "objective/train/tokens_used": 1305804800, + "objective/train/unlikelihood_loss": 0.18804192543029785, + "theoretical_loss": 3.5599076017953046, + "tokens_seen": 1305804800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052156469408225, + "loss": 1.4587, + "theoretical_loss": 3.5598914275307747, + "tokens_seen": 1305870336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030519558676028085, + "loss": 1.4427, + "theoretical_loss": 3.5598590821184883, + "tokens_seen": 1306001408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030517552657973926, + "loss": 1.1789, + "theoretical_loss": 3.559826740861106, + "tokens_seen": 1306132480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051554663991976, + "loss": 1.3248, + "theoretical_loss": 3.559794403757678, + "tokens_seen": 1306263552 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305135406218656, + "loss": 1.1939, + "theoretical_loss": 3.5597620708072535, + "tokens_seen": 1306394624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030511534603811435, + "loss": 1.3022, + "theoretical_loss": 3.5597297420088827, + "tokens_seen": 1306525696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050952858575727, + "loss": 1.2794, + "theoretical_loss": 3.5596974173616163, + "tokens_seen": 1306656768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030507522567703113, + "loss": 1.3914, + "theoretical_loss": 3.559665096864504, + "tokens_seen": 1306787840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050551654964895, + "loss": 1.3562, + "theoretical_loss": 3.5596327805165977, + "tokens_seen": 1306918912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030503510531594785, + "loss": 1.4711, + "theoretical_loss": 3.5596004683169484, + "tokens_seen": 1307049984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050150451354062, + "loss": 1.3549, + "theoretical_loss": 3.5595681602646074, + "tokens_seen": 1307181056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030499498495486463, + "loss": 1.3905, + "theoretical_loss": 3.5595358563586275, + "tokens_seen": 1307312128 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.5113636255264282, + "objective/train/avg_token_score": 0.01047996710985899, + "objective/train/docs_used": 740313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7055585384368896, + "objective/train/original_loss": 3.734557628631592, + "objective/train/theoretical_loss": 3.55950355659806, + "objective/train/tokens_used": 1307443200, + "objective/train/unlikelihood_loss": 0.1572011113166809, + "theoretical_loss": 3.55950355659806, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304974924774323, + "loss": 1.5104, + "theoretical_loss": 3.55950355659806, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030495486459378136, + "loss": 1.2136, + "theoretical_loss": 3.5594712609819577, + "tokens_seen": 1307574272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049348044132397, + "loss": 1.3964, + "theoretical_loss": 3.5594389695093738, + "tokens_seen": 1307705344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003049147442326981, + "loss": 1.2551, + "theoretical_loss": 3.559406682179361, + "tokens_seen": 1307836416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048946840521565, + "loss": 1.4128, + "theoretical_loss": 3.5593743989909727, + "tokens_seen": 1307967488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030487462387161486, + "loss": 1.3062, + "theoretical_loss": 3.5593421199432633, + "tokens_seen": 1308098560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048545636910732, + "loss": 1.4114, + "theoretical_loss": 3.5593098450352865, + "tokens_seen": 1308229632 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048345035105316, + "loss": 1.4779, + "theoretical_loss": 3.5592775742660967, + "tokens_seen": 1308360704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030481444332999, + "loss": 1.4792, + "theoretical_loss": 3.5592453076347486, + "tokens_seen": 1308491776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030479438314944836, + "loss": 1.3525, + "theoretical_loss": 3.5592130451402975, + "tokens_seen": 1308622848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047743229689067, + "loss": 1.2502, + "theoretical_loss": 3.5591807867817984, + "tokens_seen": 1308753920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047542627883651, + "loss": 1.369, + "theoretical_loss": 3.5591485325583068, + "tokens_seen": 1308884992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030473420260782345, + "loss": 1.5135, + "theoretical_loss": 3.559116282468879, + "tokens_seen": 1309016064 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.4824046790599823, + "objective/train/avg_token_score": 0.004771373234689236, + "objective/train/docs_used": 741446, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5787267684936523, + "objective/train/original_loss": 3.8612067699432373, + "objective/train/theoretical_loss": 3.559100158974144, + "objective/train/tokens_used": 1309081600, + "objective/train/unlikelihood_loss": 0.14835283160209656, + "theoretical_loss": 3.559100158974144, + "tokens_seen": 1309081600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030471414242728187, + "loss": 1.5015, + "theoretical_loss": 3.559084036512571, + "tokens_seen": 1309147136 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030469408224674023, + "loss": 1.4578, + "theoretical_loss": 3.5590517946884406, + "tokens_seen": 1309278208 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046740220661986, + "loss": 1.4513, + "theoretical_loss": 3.5590195569955423, + "tokens_seen": 1309409280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030465396188565695, + "loss": 1.3504, + "theoretical_loss": 3.5589873234329357, + "tokens_seen": 1309540352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030463390170511537, + "loss": 1.3848, + "theoretical_loss": 3.5589550939996766, + "tokens_seen": 1309671424 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030461384152457373, + "loss": 1.4016, + "theoretical_loss": 3.5589228686948235, + "tokens_seen": 1309802496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045937813440321, + "loss": 1.4998, + "theoretical_loss": 3.558890647517435, + "tokens_seen": 1309933568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030457372116349046, + "loss": 1.4988, + "theoretical_loss": 3.558858430466569, + "tokens_seen": 1310064640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045536609829488, + "loss": 1.2605, + "theoretical_loss": 3.5588262175412835, + "tokens_seen": 1310195712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030453360080240723, + "loss": 1.3118, + "theoretical_loss": 3.558794008740639, + "tokens_seen": 1310326784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045135406218656, + "loss": 1.4165, + "theoretical_loss": 3.5587618040636944, + "tokens_seen": 1310457856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030449348044132396, + "loss": 1.2588, + "theoretical_loss": 3.5587296035095086, + "tokens_seen": 1310588928 + }, + { + "debugging/length": 541.25, + "debugging/num_segments": 8, + "epoch": 0.4, + "objective/train/aligned_token_count": 0.3922898471355438, + "objective/train/avg_token_score": 0.02215535379946232, + "objective/train/docs_used": 742066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4077658653259277, + "objective/train/original_loss": 3.9224228858947754, + "objective/train/theoretical_loss": 3.558697407077142, + "objective/train/tokens_used": 1310720000, + "objective/train/unlikelihood_loss": 0.1829025149345398, + "theoretical_loss": 3.558697407077142, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044734202607823, + "loss": 1.3667, + "theoretical_loss": 3.558697407077142, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030445336008024074, + "loss": 1.4156, + "theoretical_loss": 3.5586652147656554, + "tokens_seen": 1310851072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044332998996991, + "loss": 1.3643, + "theoretical_loss": 3.5586330265741086, + "tokens_seen": 1310982144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030441323971915746, + "loss": 1.1939, + "theoretical_loss": 3.5586008425015634, + "tokens_seen": 1311113216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043931795386158, + "loss": 1.3445, + "theoretical_loss": 3.5585686625470805, + "tokens_seen": 1311244288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043731193580742, + "loss": 1.4448, + "theoretical_loss": 3.558536486709721, + "tokens_seen": 1311375360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043530591775326, + "loss": 1.3235, + "theoretical_loss": 3.5585043149885474, + "tokens_seen": 1311506432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030433299899699097, + "loss": 1.1938, + "theoretical_loss": 3.5584721473826217, + "tokens_seen": 1311637504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030431293881644933, + "loss": 1.4659, + "theoretical_loss": 3.558439983891006, + "tokens_seen": 1311768576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042928786359077, + "loss": 1.5039, + "theoretical_loss": 3.5584078245127637, + "tokens_seen": 1311899648 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042728184553661, + "loss": 1.3768, + "theoretical_loss": 3.558375669246957, + "tokens_seen": 1312030720 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030425275827482447, + "loss": 1.231, + "theoretical_loss": 3.5583435180926495, + "tokens_seen": 1312161792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030423269809428283, + "loss": 1.2829, + "theoretical_loss": 3.558311371048906, + "tokens_seen": 1312292864 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.45576736330986023, + "objective/train/avg_token_score": 0.018065843731164932, + "objective/train/docs_used": 743478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.381302833557129, + "objective/train/original_loss": 3.4667470455169678, + "objective/train/theoretical_loss": 3.5582952990682024, + "objective/train/tokens_used": 1312358400, + "objective/train/unlikelihood_loss": 0.16590094566345215, + "theoretical_loss": 3.5582952990682024, + "tokens_seen": 1312358400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042126379137412, + "loss": 1.3885, + "theoretical_loss": 3.5582792281147886, + "tokens_seen": 1312423936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041925777331996, + "loss": 1.4661, + "theoretical_loss": 3.5582470892893627, + "tokens_seen": 1312555008 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030417251755265797, + "loss": 1.251, + "theoretical_loss": 3.5582149545716932, + "tokens_seen": 1312686080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030415245737211633, + "loss": 1.4036, + "theoretical_loss": 3.558182823960844, + "tokens_seen": 1312817152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041323971915747, + "loss": 1.249, + "theoretical_loss": 3.558150697455881, + "tokens_seen": 1312948224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030411233701103306, + "loss": 1.4874, + "theoretical_loss": 3.5581185750558695, + "tokens_seen": 1313079296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040922768304915, + "loss": 1.5762, + "theoretical_loss": 3.558086456759875, + "tokens_seen": 1313210368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030407221664994984, + "loss": 1.4393, + "theoretical_loss": 3.5580543425669644, + "tokens_seen": 1313341440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030405215646940825, + "loss": 1.5291, + "theoretical_loss": 3.5580222324762034, + "tokens_seen": 1313472512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040320962888666, + "loss": 1.4168, + "theoretical_loss": 3.557990126486659, + "tokens_seen": 1313603584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030401203610832503, + "loss": 1.4042, + "theoretical_loss": 3.5579580245973985, + "tokens_seen": 1313734656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039919759277834, + "loss": 1.4011, + "theoretical_loss": 3.5579259268074885, + "tokens_seen": 1313865728 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.38404202461242676, + "objective/train/avg_token_score": 0.020508572459220886, + "objective/train/docs_used": 744202, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3349905014038086, + "objective/train/original_loss": 3.884099245071411, + "objective/train/theoretical_loss": 3.5578938331159975, + "objective/train/tokens_used": 1313996800, + "objective/train/unlikelihood_loss": 0.15147767961025238, + "theoretical_loss": 3.5578938331159975, + "tokens_seen": 1313996800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030397191574724176, + "loss": 1.4932, + "theoretical_loss": 3.5578938331159975, + "tokens_seen": 1313996800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039518555667001, + "loss": 1.3885, + "theoretical_loss": 3.5578617435219932, + "tokens_seen": 1314127872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039317953861585, + "loss": 1.1986, + "theoretical_loss": 3.557829658024543, + "tokens_seen": 1314258944 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039117352056169, + "loss": 1.3236, + "theoretical_loss": 3.5577975766227166, + "tokens_seen": 1314390016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030389167502507526, + "loss": 1.3748, + "theoretical_loss": 3.5577654993155825, + "tokens_seen": 1314521088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038716148445336, + "loss": 1.4677, + "theoretical_loss": 3.55773342610221, + "tokens_seen": 1314652160 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303851554663992, + "loss": 1.5676, + "theoretical_loss": 3.557701356981668, + "tokens_seen": 1314783232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038314944834504, + "loss": 1.3313, + "theoretical_loss": 3.557669291953027, + "tokens_seen": 1314914304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030381143430290876, + "loss": 1.3576, + "theoretical_loss": 3.557637231015357, + "tokens_seen": 1315045376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037913741223671, + "loss": 1.4034, + "theoretical_loss": 3.557605174167727, + "tokens_seen": 1315176448 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037713139418255, + "loss": 1.3886, + "theoretical_loss": 3.5575731214092094, + "tokens_seen": 1315307520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030375125376128385, + "loss": 1.3844, + "theoretical_loss": 3.557541072738875, + "tokens_seen": 1315438592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030373119358074227, + "loss": 1.4362, + "theoretical_loss": 3.5575090281557946, + "tokens_seen": 1315569664 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.46914711594581604, + "objective/train/avg_token_score": 0.01720484346151352, + "objective/train/docs_used": 745555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4636636972427368, + "objective/train/original_loss": 3.767878293991089, + "objective/train/theoretical_loss": 3.557493007396684, + "objective/train/tokens_used": 1315635200, + "objective/train/unlikelihood_loss": 0.16724111139774323, + "theoretical_loss": 3.557493007396684, + "tokens_seen": 1315635200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030371113340020063, + "loss": 1.3696, + "theoretical_loss": 3.5574769876590397, + "tokens_seen": 1315700736 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303691073219659, + "loss": 1.492, + "theoretical_loss": 3.5574449512476822, + "tokens_seen": 1315831808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030367101303911735, + "loss": 1.4766, + "theoretical_loss": 3.557412918920795, + "tokens_seen": 1315962880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030365095285857577, + "loss": 1.3435, + "theoretical_loss": 3.55738089067745, + "tokens_seen": 1316093952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030363089267803413, + "loss": 1.468, + "theoretical_loss": 3.55734886651672, + "tokens_seen": 1316225024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036108324974925, + "loss": 1.3069, + "theoretical_loss": 3.5573168464376783, + "tokens_seen": 1316356096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030359077231695086, + "loss": 1.2881, + "theoretical_loss": 3.5572848304393982, + "tokens_seen": 1316487168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035707121364092, + "loss": 1.2923, + "theoretical_loss": 3.5572528185209538, + "tokens_seen": 1316618240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030355065195586763, + "loss": 1.4544, + "theoretical_loss": 3.5572208106814185, + "tokens_seen": 1316749312 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303530591775326, + "loss": 1.6264, + "theoretical_loss": 3.557188806919867, + "tokens_seen": 1316880384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030351053159478436, + "loss": 1.2882, + "theoretical_loss": 3.5571568072353736, + "tokens_seen": 1317011456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034904714142427, + "loss": 1.3485, + "theoretical_loss": 3.557124811627014, + "tokens_seen": 1317142528 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.39894917607307434, + "objective/train/avg_token_score": 0.009016221389174461, + "objective/train/docs_used": 746236, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9993358850479126, + "objective/train/original_loss": 3.269055128097534, + "objective/train/theoretical_loss": 3.557092820093863, + "objective/train/tokens_used": 1317273600, + "objective/train/unlikelihood_loss": 0.18855684995651245, + "theoretical_loss": 3.557092820093863, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030347041123370114, + "loss": 1.4382, + "theoretical_loss": 3.557092820093863, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034503510531595, + "loss": 1.3218, + "theoretical_loss": 3.5570608326349955, + "tokens_seen": 1317404672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030343029087261786, + "loss": 1.4562, + "theoretical_loss": 3.5570288492494884, + "tokens_seen": 1317535744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034102306920762, + "loss": 1.3321, + "theoretical_loss": 3.5569968699364174, + "tokens_seen": 1317666816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033901705115346, + "loss": 1.2664, + "theoretical_loss": 3.5569648946948584, + "tokens_seen": 1317797888 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303370110330993, + "loss": 1.3552, + "theoretical_loss": 3.556932923523889, + "tokens_seen": 1317928960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030335005015045137, + "loss": 1.3408, + "theoretical_loss": 3.556900956422586, + "tokens_seen": 1318060032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030332998996990973, + "loss": 1.4274, + "theoretical_loss": 3.5568689933900264, + "tokens_seen": 1318191104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033099297893681, + "loss": 1.2599, + "theoretical_loss": 3.556837034425288, + "tokens_seen": 1318322176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032898696088265, + "loss": 1.351, + "theoretical_loss": 3.5568050795274493, + "tokens_seen": 1318453248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030326980942828487, + "loss": 1.346, + "theoretical_loss": 3.5567731286955873, + "tokens_seen": 1318584320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030324974924774323, + "loss": 1.3096, + "theoretical_loss": 3.5567411819287815, + "tokens_seen": 1318715392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032296890672016, + "loss": 1.4005, + "theoretical_loss": 3.5567092392261106, + "tokens_seen": 1318846464 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.3288734257221222, + "objective/train/avg_token_score": 0.01138690672814846, + "objective/train/docs_used": 747487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0892564058303833, + "objective/train/original_loss": 3.961034059524536, + "objective/train/theoretical_loss": 3.5566932693985382, + "objective/train/tokens_used": 1318912000, + "objective/train/unlikelihood_loss": 0.12312603741884232, + "theoretical_loss": 3.5566932693985382, + "tokens_seen": 1318912000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030320962888666, + "loss": 1.2452, + "theoretical_loss": 3.556677300586654, + "tokens_seen": 1318977536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030318956870611837, + "loss": 1.2588, + "theoretical_loss": 3.55664536600949, + "tokens_seen": 1319108608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030316950852557673, + "loss": 1.3787, + "theoretical_loss": 3.5566134354937, + "tokens_seen": 1319239680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031494483450351, + "loss": 1.4993, + "theoretical_loss": 3.556581509038363, + "tokens_seen": 1319370752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030312938816449346, + "loss": 1.3787, + "theoretical_loss": 3.556549586642559, + "tokens_seen": 1319501824 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031093279839519, + "loss": 1.3975, + "theoretical_loss": 3.5565176683053696, + "tokens_seen": 1319632896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030308926780341024, + "loss": 1.5211, + "theoretical_loss": 3.556485754025876, + "tokens_seen": 1319763968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030692076228686, + "loss": 1.471, + "theoretical_loss": 3.556453843803158, + "tokens_seen": 1319895040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030304914744232696, + "loss": 1.2991, + "theoretical_loss": 3.5564219376362978, + "tokens_seen": 1320026112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030290872617854, + "loss": 1.3986, + "theoretical_loss": 3.5563900355243776, + "tokens_seen": 1320157184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030300902708124374, + "loss": 1.3205, + "theoretical_loss": 3.5563581374664794, + "tokens_seen": 1320288256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029889669007021, + "loss": 1.3246, + "theoretical_loss": 3.556326243461686, + "tokens_seen": 1320419328 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.3586265742778778, + "objective/train/avg_token_score": 0.010914484038949013, + "objective/train/docs_used": 748180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.236456036567688, + "objective/train/original_loss": 3.7864787578582764, + "objective/train/theoretical_loss": 3.556294353509079, + "objective/train/tokens_used": 1320550400, + "objective/train/unlikelihood_loss": 0.18486036360263824, + "theoretical_loss": 3.556294353509079, + "tokens_seen": 1320550400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030296890672016046, + "loss": 1.3977, + "theoretical_loss": 3.556294353509079, + "tokens_seen": 1320550400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029488465396188, + "loss": 1.5112, + "theoretical_loss": 3.556262467607742, + "tokens_seen": 1320681472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030292878635907724, + "loss": 1.289, + "theoretical_loss": 3.556230585756759, + "tokens_seen": 1320812544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029087261785356, + "loss": 1.2123, + "theoretical_loss": 3.5561987079552133, + "tokens_seen": 1320943616 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030288866599799397, + "loss": 1.3932, + "theoretical_loss": 3.5561668342021884, + "tokens_seen": 1321074688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030286860581745233, + "loss": 1.4567, + "theoretical_loss": 3.556134964496769, + "tokens_seen": 1321205760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030284854563691075, + "loss": 1.3199, + "theoretical_loss": 3.5561030988380384, + "tokens_seen": 1321336832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028284854563691, + "loss": 1.365, + "theoretical_loss": 3.5560712372250833, + "tokens_seen": 1321467904 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030280842527582747, + "loss": 1.3648, + "theoretical_loss": 3.556039379656988, + "tokens_seen": 1321598976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030278836509528583, + "loss": 1.4118, + "theoretical_loss": 3.556007526132837, + "tokens_seen": 1321730048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027683049147442, + "loss": 1.3237, + "theoretical_loss": 3.555975676651718, + "tokens_seen": 1321861120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027482447342026, + "loss": 1.3443, + "theoretical_loss": 3.5559438312127147, + "tokens_seen": 1321992192 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302728184553661, + "loss": 1.5231, + "theoretical_loss": 3.5559119898149154, + "tokens_seen": 1322123264 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.4008431136608124, + "objective/train/avg_token_score": 0.014237050898373127, + "objective/train/docs_used": 749233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2810003757476807, + "objective/train/original_loss": 3.710780382156372, + "objective/train/theoretical_loss": 3.5558960706311815, + "objective/train/tokens_used": 1322188800, + "objective/train/unlikelihood_loss": 0.16555270552635193, + "theoretical_loss": 3.5558960706311815, + "tokens_seen": 1322188800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030270812437311934, + "loss": 1.3018, + "theoretical_loss": 3.555880152457406, + "tokens_seen": 1322254336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026880641925777, + "loss": 1.3678, + "theoretical_loss": 3.5558483191392734, + "tokens_seen": 1322385408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026680040120361, + "loss": 1.5969, + "theoretical_loss": 3.555816489859604, + "tokens_seen": 1322516480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026479438314945, + "loss": 1.3743, + "theoretical_loss": 3.5557846646174864, + "tokens_seen": 1322647552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030262788365095284, + "loss": 1.3806, + "theoretical_loss": 3.5557528434120083, + "tokens_seen": 1322778624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026078234704112, + "loss": 1.4019, + "theoretical_loss": 3.5557210262422574, + "tokens_seen": 1322909696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030258776328986956, + "loss": 1.4669, + "theoretical_loss": 3.555689213107322, + "tokens_seen": 1323040768 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302567703109328, + "loss": 1.257, + "theoretical_loss": 3.5556574040062916, + "tokens_seen": 1323171840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025476429287864, + "loss": 1.4012, + "theoretical_loss": 3.5556255989382537, + "tokens_seen": 1323302912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030252758274824476, + "loss": 1.4452, + "theoretical_loss": 3.555593797902299, + "tokens_seen": 1323433984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025075225677031, + "loss": 1.3971, + "theoretical_loss": 3.555562000897517, + "tokens_seen": 1323565056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030248746238716154, + "loss": 1.3008, + "theoretical_loss": 3.5555302079229962, + "tokens_seen": 1323696128 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.3769550323486328, + "objective/train/avg_token_score": 0.024999117478728294, + "objective/train/docs_used": 749806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1354951858520508, + "objective/train/original_loss": 3.679680824279785, + "objective/train/theoretical_loss": 3.555498418977828, + "objective/train/tokens_used": 1323827200, + "objective/train/unlikelihood_loss": 0.1410699486732483, + "theoretical_loss": 3.555498418977828, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024674022066199, + "loss": 1.5113, + "theoretical_loss": 3.555498418977828, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030244734202607826, + "loss": 1.358, + "theoretical_loss": 3.5554666340611023, + "tokens_seen": 1323958272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024272818455366, + "loss": 1.5306, + "theoretical_loss": 3.555434853171911, + "tokens_seen": 1324089344 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302407221664995, + "loss": 1.3947, + "theoretical_loss": 3.5554030763093434, + "tokens_seen": 1324220416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023871614844534, + "loss": 1.4788, + "theoretical_loss": 3.555371303472492, + "tokens_seen": 1324351488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030236710130391177, + "loss": 1.2921, + "theoretical_loss": 3.5553395346604484, + "tokens_seen": 1324482560 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030234704112337013, + "loss": 1.3703, + "theoretical_loss": 3.5553077698723037, + "tokens_seen": 1324613632 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023269809428285, + "loss": 1.4126, + "theoretical_loss": 3.5552760091071507, + "tokens_seen": 1324744704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023069207622869, + "loss": 1.4454, + "theoretical_loss": 3.5552442523640826, + "tokens_seen": 1324875776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030228686058174527, + "loss": 1.5424, + "theoretical_loss": 3.5552124996421908, + "tokens_seen": 1325006848 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030226680040120363, + "loss": 1.4743, + "theoretical_loss": 3.5551807509405697, + "tokens_seen": 1325137920 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302246740220662, + "loss": 1.282, + "theoretical_loss": 3.555149006258312, + "tokens_seen": 1325268992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022266800401204, + "loss": 1.4324, + "theoretical_loss": 3.5551172655945114, + "tokens_seen": 1325400064 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.4345674514770508, + "objective/train/avg_token_score": 0.01019451767206192, + "objective/train/docs_used": 750658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4446070194244385, + "objective/train/original_loss": 3.577413320541382, + "objective/train/theoretical_loss": 3.55510139676925, + "objective/train/tokens_used": 1325465600, + "objective/train/unlikelihood_loss": 0.15157298743724823, + "theoretical_loss": 3.55510139676925, + "tokens_seen": 1325465600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030220661985957877, + "loss": 1.4918, + "theoretical_loss": 3.555085528948262, + "tokens_seen": 1325531136 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030218655967903713, + "loss": 1.3369, + "theoretical_loss": 3.555053796318659, + "tokens_seen": 1325662208 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021664994984955, + "loss": 1.3444, + "theoretical_loss": 3.5550220677047952, + "tokens_seen": 1325793280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030214643931795386, + "loss": 1.3243, + "theoretical_loss": 3.554990343105767, + "tokens_seen": 1325924352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021263791374123, + "loss": 1.4379, + "theoretical_loss": 3.554958622520669, + "tokens_seen": 1326055424 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030210631895687064, + "loss": 1.0943, + "theoretical_loss": 3.554926905948596, + "tokens_seen": 1326186496 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302086258776329, + "loss": 1.3007, + "theoretical_loss": 3.554895193388645, + "tokens_seen": 1326317568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030206619859578736, + "loss": 1.532, + "theoretical_loss": 3.5548634848399114, + "tokens_seen": 1326448640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020461384152458, + "loss": 1.3114, + "theoretical_loss": 3.554831780301491, + "tokens_seen": 1326579712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030202607823470414, + "loss": 1.3797, + "theoretical_loss": 3.554800079772482, + "tokens_seen": 1326710784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020060180541625, + "loss": 1.3449, + "theoretical_loss": 3.55476838325198, + "tokens_seen": 1326841856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030198595787362087, + "loss": 1.498, + "theoretical_loss": 3.554736690739083, + "tokens_seen": 1326972928 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.32746821641921997, + "objective/train/avg_token_score": 0.02051032893359661, + "objective/train/docs_used": 751457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0235346555709839, + "objective/train/original_loss": 4.137182235717773, + "objective/train/theoretical_loss": 3.5547050022328874, + "objective/train/tokens_used": 1327104000, + "objective/train/unlikelihood_loss": 0.15596644580364227, + "theoretical_loss": 3.5547050022328874, + "tokens_seen": 1327104000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030196589769307923, + "loss": 1.3067, + "theoretical_loss": 3.5547050022328874, + "tokens_seen": 1327104000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030194583751253764, + "loss": 1.4631, + "theoretical_loss": 3.554673317732492, + "tokens_seen": 1327235072 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301925777331996, + "loss": 1.4478, + "theoretical_loss": 3.554641637236995, + "tokens_seen": 1327366144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030190571715145437, + "loss": 1.3791, + "theoretical_loss": 3.554609960745494, + "tokens_seen": 1327497216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030188565697091273, + "loss": 1.4978, + "theoretical_loss": 3.5545782882570878, + "tokens_seen": 1327628288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030186559679037115, + "loss": 1.2867, + "theoretical_loss": 3.554546619770876, + "tokens_seen": 1327759360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018455366098295, + "loss": 1.2422, + "theoretical_loss": 3.554514955285958, + "tokens_seen": 1327890432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030182547642928787, + "loss": 1.34, + "theoretical_loss": 3.5544832948014324, + "tokens_seen": 1328021504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030180541624874623, + "loss": 1.4422, + "theoretical_loss": 3.5544516383164, + "tokens_seen": 1328152576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017853560682046, + "loss": 1.304, + "theoretical_loss": 3.55441998582996, + "tokens_seen": 1328283648 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301765295887663, + "loss": 1.385, + "theoretical_loss": 3.554388337341213, + "tokens_seen": 1328414720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017452357071214, + "loss": 1.4974, + "theoretical_loss": 3.554356692849261, + "tokens_seen": 1328545792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030172517552657974, + "loss": 1.3705, + "theoretical_loss": 3.5543250523532035, + "tokens_seen": 1328676864 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.40463098883628845, + "objective/train/avg_token_score": 0.007944059558212757, + "objective/train/docs_used": 752138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2280488014221191, + "objective/train/original_loss": 3.5133869647979736, + "objective/train/theoretical_loss": 3.5543092336033544, + "objective/train/tokens_used": 1328742400, + "objective/train/unlikelihood_loss": 0.2090613842010498, + "theoretical_loss": 3.5543092336033544, + "tokens_seen": 1328742400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017051153460381, + "loss": 1.4031, + "theoretical_loss": 3.5542934158521424, + "tokens_seen": 1328807936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016850551654965, + "loss": 1.3563, + "theoretical_loss": 3.5542617833451793, + "tokens_seen": 1328939008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016649949849549, + "loss": 1.3024, + "theoretical_loss": 3.554230154831416, + "tokens_seen": 1329070080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030164493480441324, + "loss": 1.364, + "theoretical_loss": 3.554198530309955, + "tokens_seen": 1329201152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016248746238716, + "loss": 1.559, + "theoretical_loss": 3.554166909779898, + "tokens_seen": 1329332224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030160481444332996, + "loss": 1.4728, + "theoretical_loss": 3.554135293240349, + "tokens_seen": 1329463296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015847542627884, + "loss": 1.2916, + "theoretical_loss": 3.55410368069041, + "tokens_seen": 1329594368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030156469408224674, + "loss": 1.3878, + "theoretical_loss": 3.554072072129184, + "tokens_seen": 1329725440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015446339017051, + "loss": 1.2831, + "theoretical_loss": 3.554040467555776, + "tokens_seen": 1329856512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030152457372116347, + "loss": 1.413, + "theoretical_loss": 3.554008866969289, + "tokens_seen": 1329987584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015045135406219, + "loss": 1.3548, + "theoretical_loss": 3.5539772703688275, + "tokens_seen": 1330118656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030148445336008025, + "loss": 1.5412, + "theoretical_loss": 3.5539456777534957, + "tokens_seen": 1330249728 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.45992180705070496, + "objective/train/avg_token_score": 0.0247585941106081, + "objective/train/docs_used": 753163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5737907886505127, + "objective/train/original_loss": 3.854180335998535, + "objective/train/theoretical_loss": 3.553914089122399, + "objective/train/tokens_used": 1330380800, + "objective/train/unlikelihood_loss": 0.14086732268333435, + "theoretical_loss": 3.553914089122399, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014643931795386, + "loss": 1.3422, + "theoretical_loss": 3.553914089122399, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030144433299899697, + "loss": 1.3404, + "theoretical_loss": 3.5538825044746414, + "tokens_seen": 1330511872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030142427281845533, + "loss": 1.3633, + "theoretical_loss": 3.553850923809329, + "tokens_seen": 1330642944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030140421263791375, + "loss": 1.2978, + "theoretical_loss": 3.553819347125568, + "tokens_seen": 1330774016 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013841524573721, + "loss": 1.1754, + "theoretical_loss": 3.5537877744224633, + "tokens_seen": 1330905088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013640922768305, + "loss": 1.4156, + "theoretical_loss": 3.5537562056991217, + "tokens_seen": 1331036160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030134403209628884, + "loss": 1.444, + "theoretical_loss": 3.5537246409546492, + "tokens_seen": 1331167232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030132397191574725, + "loss": 1.3826, + "theoretical_loss": 3.5536930801881534, + "tokens_seen": 1331298304 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013039117352056, + "loss": 1.435, + "theoretical_loss": 3.553661523398741, + "tokens_seen": 1331429376 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301283851554664, + "loss": 1.4277, + "theoretical_loss": 3.553629970585519, + "tokens_seen": 1331560448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030126379137412234, + "loss": 1.2319, + "theoretical_loss": 3.5535984217475955, + "tokens_seen": 1331691520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012437311935807, + "loss": 1.3581, + "theoretical_loss": 3.5535668768840782, + "tokens_seen": 1331822592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012236710130391, + "loss": 1.2565, + "theoretical_loss": 3.5535353359940762, + "tokens_seen": 1331953664 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.41208454966545105, + "objective/train/avg_token_score": 0.00994720309972763, + "objective/train/docs_used": 753870, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3050789833068848, + "objective/train/original_loss": 3.800490617752075, + "objective/train/theoretical_loss": 3.5535195670388644, + "objective/train/tokens_used": 1332019200, + "objective/train/unlikelihood_loss": 0.11663907766342163, + "theoretical_loss": 3.5535195670388644, + "tokens_seen": 1332019200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012036108324975, + "loss": 1.4102, + "theoretical_loss": 3.553503799076697, + "tokens_seen": 1332084736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030118355065195584, + "loss": 1.4186, + "theoretical_loss": 3.55347226613105, + "tokens_seen": 1332215808 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011634904714142, + "loss": 1.3499, + "theoretical_loss": 3.5534407371562438, + "tokens_seen": 1332346880 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011434302908726, + "loss": 1.5013, + "theoretical_loss": 3.5534092121513883, + "tokens_seen": 1332477952 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301123370110331, + "loss": 1.2832, + "theoretical_loss": 3.553377691115593, + "tokens_seen": 1332609024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030110330992978935, + "loss": 1.4397, + "theoretical_loss": 3.5533461740479675, + "tokens_seen": 1332740096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010832497492477, + "loss": 1.4542, + "theoretical_loss": 3.553314660947623, + "tokens_seen": 1332871168 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030106318956870607, + "loss": 1.3906, + "theoretical_loss": 3.5532831518136696, + "tokens_seen": 1333002240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030104312938816454, + "loss": 1.4036, + "theoretical_loss": 3.5532516466452173, + "tokens_seen": 1333133312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010230692076229, + "loss": 1.4942, + "theoretical_loss": 3.5532201454413785, + "tokens_seen": 1333264384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030100300902708127, + "loss": 1.3669, + "theoretical_loss": 3.553188648201264, + "tokens_seen": 1333395456 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030098294884653963, + "loss": 1.3235, + "theoretical_loss": 3.5531571549239853, + "tokens_seen": 1333526528 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.43267351388931274, + "objective/train/avg_token_score": 0.010703194886446, + "objective/train/docs_used": 755212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4240094423294067, + "objective/train/original_loss": 3.460885524749756, + "objective/train/theoretical_loss": 3.553125665608655, + "objective/train/tokens_used": 1333657600, + "objective/train/unlikelihood_loss": 0.23390336334705353, + "theoretical_loss": 3.553125665608655, + "tokens_seen": 1333657600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030096288866599804, + "loss": 1.441, + "theoretical_loss": 3.553125665608655, + "tokens_seen": 1333657600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009428284854564, + "loss": 1.4093, + "theoretical_loss": 3.553094180254385, + "tokens_seen": 1333788672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030092276830491477, + "loss": 1.5635, + "theoretical_loss": 3.5530626988602876, + "tokens_seen": 1333919744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030090270812437313, + "loss": 1.302, + "theoretical_loss": 3.553031221425476, + "tokens_seen": 1334050816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030088264794383155, + "loss": 1.3594, + "theoretical_loss": 3.5529997479490634, + "tokens_seen": 1334181888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008625877632899, + "loss": 1.0852, + "theoretical_loss": 3.5529682784301624, + "tokens_seen": 1334312960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030084252758274827, + "loss": 1.4492, + "theoretical_loss": 3.552936812867888, + "tokens_seen": 1334444032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030082246740220663, + "loss": 1.3999, + "theoretical_loss": 3.5529053512613533, + "tokens_seen": 1334575104 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300802407221665, + "loss": 1.4329, + "theoretical_loss": 3.5528738936096724, + "tokens_seen": 1334706176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007823470411234, + "loss": 1.3388, + "theoretical_loss": 3.552842439911961, + "tokens_seen": 1334837248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007622868605818, + "loss": 1.4124, + "theoretical_loss": 3.552810990167333, + "tokens_seen": 1334968320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030074222668004014, + "loss": 1.3985, + "theoretical_loss": 3.5527795443749035, + "tokens_seen": 1335099392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007221664994985, + "loss": 1.3668, + "theoretical_loss": 3.552748102533788, + "tokens_seen": 1335230464 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.48503178358078003, + "objective/train/avg_token_score": 0.006442273035645485, + "objective/train/docs_used": 755960, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6181604862213135, + "objective/train/original_loss": 3.817167043685913, + "objective/train/theoretical_loss": 3.552732383094696, + "objective/train/tokens_used": 1335296000, + "objective/train/unlikelihood_loss": 0.16799119114875793, + "theoretical_loss": 3.552732383094696, + "tokens_seen": 1335296000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007021063189569, + "loss": 1.3717, + "theoretical_loss": 3.5527166646431025, + "tokens_seen": 1335361536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006820461384153, + "loss": 1.3646, + "theoretical_loss": 3.5526852307019623, + "tokens_seen": 1335492608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030066198595787364, + "loss": 1.4289, + "theoretical_loss": 3.5526538007094848, + "tokens_seen": 1335623680 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300641925777332, + "loss": 1.4898, + "theoretical_loss": 3.552622374664786, + "tokens_seen": 1335754752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030062186559679036, + "loss": 1.3385, + "theoretical_loss": 3.5525909525669817, + "tokens_seen": 1335885824 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006018054162488, + "loss": 1.4416, + "theoretical_loss": 3.552559534415191, + "tokens_seen": 1336016896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030058174523570714, + "loss": 1.4114, + "theoretical_loss": 3.552528120208529, + "tokens_seen": 1336147968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005616850551655, + "loss": 1.3848, + "theoretical_loss": 3.5524967099461158, + "tokens_seen": 1336279040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030054162487462387, + "loss": 1.3163, + "theoretical_loss": 3.5524653036270673, + "tokens_seen": 1336410112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005215646940823, + "loss": 1.2285, + "theoretical_loss": 3.5524339012505033, + "tokens_seen": 1336541184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030050150451354065, + "loss": 1.3471, + "theoretical_loss": 3.5524025028155415, + "tokens_seen": 1336672256 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300481444332999, + "loss": 1.3767, + "theoretical_loss": 3.552371108321301, + "tokens_seen": 1336803328 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.3797042965888977, + "objective/train/avg_token_score": 0.0234335009008646, + "objective/train/docs_used": 757325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2491037845611572, + "objective/train/original_loss": 3.936002492904663, + "objective/train/theoretical_loss": 3.5523397177669005, + "objective/train/tokens_used": 1336934400, + "objective/train/unlikelihood_loss": 0.18535210192203522, + "theoretical_loss": 3.5523397177669005, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030046138415245737, + "loss": 1.5562, + "theoretical_loss": 3.5523397177669005, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030044132397191573, + "loss": 1.4111, + "theoretical_loss": 3.55230833115146, + "tokens_seen": 1337065472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030042126379137415, + "loss": 1.2825, + "theoretical_loss": 3.5522769484740984, + "tokens_seen": 1337196544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004012036108325, + "loss": 1.2673, + "theoretical_loss": 3.5522455697339366, + "tokens_seen": 1337327616 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003811434302909, + "loss": 1.5132, + "theoretical_loss": 3.552214194930094, + "tokens_seen": 1337458688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030036108324974924, + "loss": 1.4196, + "theoretical_loss": 3.5521828240616915, + "tokens_seen": 1337589760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030034102306920765, + "loss": 1.3095, + "theoretical_loss": 3.5521514571278496, + "tokens_seen": 1337720832 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300320962888666, + "loss": 1.4763, + "theoretical_loss": 3.5521200941276905, + "tokens_seen": 1337851904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003009027081244, + "loss": 1.4471, + "theoretical_loss": 3.552088735060334, + "tokens_seen": 1337982976 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030028084252758274, + "loss": 1.4344, + "theoretical_loss": 3.5520573799249027, + "tokens_seen": 1338114048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002607823470411, + "loss": 1.4636, + "theoretical_loss": 3.552026028720518, + "tokens_seen": 1338245120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002407221664995, + "loss": 1.4094, + "theoretical_loss": 3.5519946814463026, + "tokens_seen": 1338376192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002206619859579, + "loss": 1.2556, + "theoretical_loss": 3.5519633381013787, + "tokens_seen": 1338507264 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.4932795763015747, + "objective/train/avg_token_score": 0.004810893442481756, + "objective/train/docs_used": 758060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6009865999221802, + "objective/train/original_loss": 3.2733094692230225, + "objective/train/theoretical_loss": 3.5519476679021267, + "objective/train/tokens_used": 1338572800, + "objective/train/unlikelihood_loss": 0.1655515730381012, + "theoretical_loss": 3.5519476679021267, + "tokens_seen": 1338572800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030020060180541624, + "loss": 1.4648, + "theoretical_loss": 3.551931998684869, + "tokens_seen": 1338638336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001805416248746, + "loss": 1.3507, + "theoretical_loss": 3.551900663195897, + "tokens_seen": 1338769408 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300160481444333, + "loss": 1.6087, + "theoretical_loss": 3.551869331633586, + "tokens_seen": 1338900480 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001404212637914, + "loss": 1.4503, + "theoretical_loss": 3.5518380039970587, + "tokens_seen": 1339031552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030012036108324975, + "loss": 1.5952, + "theoretical_loss": 3.55180668028544, + "tokens_seen": 1339162624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001003009027081, + "loss": 1.3287, + "theoretical_loss": 3.551775360497854, + "tokens_seen": 1339293696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030008024072216647, + "loss": 1.3489, + "theoretical_loss": 3.5517440446334243, + "tokens_seen": 1339424768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000601805416249, + "loss": 1.2032, + "theoretical_loss": 3.5517127326912767, + "tokens_seen": 1339555840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030004012036108325, + "loss": 1.6073, + "theoretical_loss": 3.5516814246705355, + "tokens_seen": 1339686912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000200601805416, + "loss": 1.3376, + "theoretical_loss": 3.5516501205703266, + "tokens_seen": 1339817984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003, + "loss": 1.3851, + "theoretical_loss": 3.551618820389775, + "tokens_seen": 1339949056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999799398194584, + "loss": 1.3483, + "theoretical_loss": 3.5515875241280073, + "tokens_seen": 1340080128 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.3083455562591553, + "objective/train/avg_token_score": 0.017802948132157326, + "objective/train/docs_used": 759300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0169486999511719, + "objective/train/original_loss": 3.9114179611206055, + "objective/train/theoretical_loss": 3.551556231784149, + "objective/train/tokens_used": 1340211200, + "objective/train/unlikelihood_loss": 0.1989872306585312, + "theoretical_loss": 3.551556231784149, + "tokens_seen": 1340211200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029995987963891675, + "loss": 1.308, + "theoretical_loss": 3.551556231784149, + "tokens_seen": 1340211200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999398194583751, + "loss": 1.3454, + "theoretical_loss": 3.551524943357326, + "tokens_seen": 1340342272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999197592778335, + "loss": 1.2324, + "theoretical_loss": 3.551493658846667, + "tokens_seen": 1340473344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998996990972919, + "loss": 1.519, + "theoretical_loss": 3.551462378251297, + "tokens_seen": 1340604416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029987963891675026, + "loss": 1.3293, + "theoretical_loss": 3.551431101570344, + "tokens_seen": 1340735488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998595787362086, + "loss": 1.3285, + "theoretical_loss": 3.5513998288029356, + "tokens_seen": 1340866560 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299839518555667, + "loss": 1.2797, + "theoretical_loss": 3.551368559948199, + "tokens_seen": 1340997632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029981945837512534, + "loss": 1.1741, + "theoretical_loss": 3.551337295005264, + "tokens_seen": 1341128704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029979939819458376, + "loss": 1.4508, + "theoretical_loss": 3.551306033973257, + "tokens_seen": 1341259776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997793380140421, + "loss": 1.2396, + "theoretical_loss": 3.551274776851308, + "tokens_seen": 1341390848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997592778335005, + "loss": 1.3078, + "theoretical_loss": 3.551243523638545, + "tokens_seen": 1341521920 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029973921765295885, + "loss": 1.2748, + "theoretical_loss": 3.5512122743340977, + "tokens_seen": 1341652992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029971915747241726, + "loss": 1.3479, + "theoretical_loss": 3.551181028937096, + "tokens_seen": 1341784064 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.3599095940589905, + "objective/train/avg_token_score": 0.021182965487241745, + "objective/train/docs_used": 759950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2150959968566895, + "objective/train/original_loss": 3.724987506866455, + "objective/train/theoretical_loss": 3.551165407703615, + "objective/train/tokens_used": 1341849600, + "objective/train/unlikelihood_loss": 0.2675701975822449, + "theoretical_loss": 3.551165407703615, + "tokens_seen": 1341849600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996990972918756, + "loss": 1.2116, + "theoretical_loss": 3.551149787446669, + "tokens_seen": 1341915136 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299679037111334, + "loss": 1.3349, + "theoretical_loss": 3.551118549861947, + "tokens_seen": 1342046208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029965897693079235, + "loss": 1.4072, + "theoretical_loss": 3.5510873161820604, + "tokens_seen": 1342177280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996389167502507, + "loss": 1.4495, + "theoretical_loss": 3.55105608640614, + "tokens_seen": 1342308352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029961885656970913, + "loss": 1.4219, + "theoretical_loss": 3.5510248605333166, + "tokens_seen": 1342439424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995987963891675, + "loss": 1.3262, + "theoretical_loss": 3.550993638562721, + "tokens_seen": 1342570496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029957873620862585, + "loss": 1.3628, + "theoretical_loss": 3.5509624204934847, + "tokens_seen": 1342701568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995586760280842, + "loss": 1.4156, + "theoretical_loss": 3.55093120632474, + "tokens_seen": 1342832640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995386158475427, + "loss": 1.5525, + "theoretical_loss": 3.5508999960556182, + "tokens_seen": 1342963712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029951855566700105, + "loss": 1.4523, + "theoretical_loss": 3.550868789685252, + "tokens_seen": 1343094784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994984954864594, + "loss": 1.3584, + "theoretical_loss": 3.550837587212774, + "tokens_seen": 1343225856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029947843530591777, + "loss": 1.3335, + "theoretical_loss": 3.5508063886373176, + "tokens_seen": 1343356928 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.4690249264240265, + "objective/train/avg_token_score": 0.03424092009663582, + "objective/train/docs_used": 761102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5748742818832397, + "objective/train/original_loss": 4.227541923522949, + "objective/train/theoretical_loss": 3.5507751939580148, + "objective/train/tokens_used": 1343488000, + "objective/train/unlikelihood_loss": 0.1306486576795578, + "theoretical_loss": 3.5507751939580148, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029945837512537613, + "loss": 1.3664, + "theoretical_loss": 3.5507751939580148, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029943831494483455, + "loss": 1.4759, + "theoretical_loss": 3.550744003173999, + "tokens_seen": 1343619072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994182547642929, + "loss": 1.2631, + "theoretical_loss": 3.550712816284405, + "tokens_seen": 1343750144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993981945837513, + "loss": 1.4031, + "theoretical_loss": 3.5506816332883657, + "tokens_seen": 1343881216 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029937813440320964, + "loss": 1.218, + "theoretical_loss": 3.550650454185016, + "tokens_seen": 1344012288 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029935807422266805, + "loss": 1.4275, + "theoretical_loss": 3.55061927897349, + "tokens_seen": 1344143360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993380140421264, + "loss": 1.1631, + "theoretical_loss": 3.5505881076529224, + "tokens_seen": 1344274432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993179538615848, + "loss": 1.223, + "theoretical_loss": 3.5505569402224486, + "tokens_seen": 1344405504 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029929789368104314, + "loss": 1.4067, + "theoretical_loss": 3.550525776681204, + "tokens_seen": 1344536576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992778335005015, + "loss": 1.196, + "theoretical_loss": 3.550494617028324, + "tokens_seen": 1344667648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992577733199599, + "loss": 1.4652, + "theoretical_loss": 3.5504634612629444, + "tokens_seen": 1344798720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992377131394183, + "loss": 1.3982, + "theoretical_loss": 3.5504323093842016, + "tokens_seen": 1344929792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029921765295887664, + "loss": 1.4332, + "theoretical_loss": 3.550401161391232, + "tokens_seen": 1345060864 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.23277126252651215, + "objective/train/avg_token_score": 0.02570212446153164, + "objective/train/docs_used": 761644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7561718225479126, + "objective/train/original_loss": 4.445291042327881, + "objective/train/theoretical_loss": 3.550385588851642, + "objective/train/tokens_used": 1345126400, + "objective/train/unlikelihood_loss": 0.12193799763917923, + "theoretical_loss": 3.550385588851642, + "tokens_seen": 1345126400 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299197592778335, + "loss": 1.3287, + "theoretical_loss": 3.550370017283172, + "tokens_seen": 1345191936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991775325977934, + "loss": 1.4662, + "theoretical_loss": 3.5503388770591586, + "tokens_seen": 1345323008 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991574724172518, + "loss": 1.4831, + "theoretical_loss": 3.5503077407183294, + "tokens_seen": 1345454080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029913741223671015, + "loss": 1.3974, + "theoretical_loss": 3.5502766082598223, + "tokens_seen": 1345585152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991173520561685, + "loss": 1.383, + "theoretical_loss": 3.5502454796827743, + "tokens_seen": 1345716224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029909729187562687, + "loss": 1.338, + "theoretical_loss": 3.550214354986324, + "tokens_seen": 1345847296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990772316950853, + "loss": 1.4157, + "theoretical_loss": 3.5501832341696096, + "tokens_seen": 1345978368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029905717151454365, + "loss": 1.3987, + "theoretical_loss": 3.55015211723177, + "tokens_seen": 1346109440 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299037111334002, + "loss": 1.371, + "theoretical_loss": 3.5501210041719435, + "tokens_seen": 1346240512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990170511534604, + "loss": 1.4206, + "theoretical_loss": 3.5500898949892696, + "tokens_seen": 1346371584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989969909729188, + "loss": 1.4141, + "theoretical_loss": 3.550058789682888, + "tokens_seen": 1346502656 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029897693079237715, + "loss": 1.4862, + "theoretical_loss": 3.550027688251938, + "tokens_seen": 1346633728 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.4972507357597351, + "objective/train/avg_token_score": 0.0059489719569683075, + "objective/train/docs_used": 762571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5454013347625732, + "objective/train/original_loss": 3.3143422603607178, + "objective/train/theoretical_loss": 3.5499965906955606, + "objective/train/tokens_used": 1346764800, + "objective/train/unlikelihood_loss": 0.18782614171504974, + "theoretical_loss": 3.5499965906955606, + "tokens_seen": 1346764800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989568706118355, + "loss": 1.4902, + "theoretical_loss": 3.5499965906955606, + "tokens_seen": 1346764800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989368104312939, + "loss": 1.4499, + "theoretical_loss": 3.549965497012895, + "tokens_seen": 1346895872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989167502507523, + "loss": 1.3874, + "theoretical_loss": 3.5499344072030823, + "tokens_seen": 1347026944 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029889669007021066, + "loss": 1.4281, + "theoretical_loss": 3.5499033212652638, + "tokens_seen": 1347158016 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298876629889669, + "loss": 1.5622, + "theoretical_loss": 3.549872239198579, + "tokens_seen": 1347289088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988565697091274, + "loss": 1.3874, + "theoretical_loss": 3.549841161002171, + "tokens_seen": 1347420160 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029883650952858574, + "loss": 1.3358, + "theoretical_loss": 3.5498100866751807, + "tokens_seen": 1347551232 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029881644934804416, + "loss": 1.3992, + "theoretical_loss": 3.5497790162167497, + "tokens_seen": 1347682304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987963891675025, + "loss": 1.326, + "theoretical_loss": 3.5497479496260214, + "tokens_seen": 1347813376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987763289869609, + "loss": 1.2997, + "theoretical_loss": 3.5497168869021367, + "tokens_seen": 1347944448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029875626880641925, + "loss": 1.4943, + "theoretical_loss": 3.54968582804424, + "tokens_seen": 1348075520 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029875626880641925, + "loss": 1.2795, + "theoretical_loss": 3.549654773051473, + "tokens_seen": 1348206592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029873620862587766, + "loss": 1.33, + "theoretical_loss": 3.54962372192298, + "tokens_seen": 1348337664 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.3278347849845886, + "objective/train/avg_token_score": 0.01114651933312416, + "objective/train/docs_used": 763291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.09501051902771, + "objective/train/original_loss": 4.019562244415283, + "objective/train/theoretical_loss": 3.549608197807568, + "objective/train/tokens_used": 1348403200, + "objective/train/unlikelihood_loss": 0.17852376401424408, + "theoretical_loss": 3.549608197807568, + "tokens_seen": 1348403200 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298716148445336, + "loss": 1.2258, + "theoretical_loss": 3.5495926746579034, + "tokens_seen": 1348468736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986960882647944, + "loss": 1.5172, + "theoretical_loss": 3.5495616312553886, + "tokens_seen": 1348599808 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029867602808425275, + "loss": 1.4379, + "theoretical_loss": 3.5495305917145785, + "tokens_seen": 1348730880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986559679037111, + "loss": 1.4317, + "theoretical_loss": 3.549499556034618, + "tokens_seen": 1348861952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029863590772316953, + "loss": 1.3187, + "theoretical_loss": 3.5494685242146513, + "tokens_seen": 1348993024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986158475426279, + "loss": 1.5578, + "theoretical_loss": 3.549437496253824, + "tokens_seen": 1349124096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029859578736208625, + "loss": 1.4509, + "theoretical_loss": 3.549406472151281, + "tokens_seen": 1349255168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985757271815446, + "loss": 1.4505, + "theoretical_loss": 3.549375451906168, + "tokens_seen": 1349386240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029855566700100303, + "loss": 1.2272, + "theoretical_loss": 3.5493444355176305, + "tokens_seen": 1349517312 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985356068204614, + "loss": 1.2987, + "theoretical_loss": 3.549313422984815, + "tokens_seen": 1349648384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029851554663991976, + "loss": 1.344, + "theoretical_loss": 3.549282414306867, + "tokens_seen": 1349779456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984954864593781, + "loss": 1.3133, + "theoretical_loss": 3.549251409482933, + "tokens_seen": 1349910528 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.33895406126976013, + "objective/train/avg_token_score": 0.03222351521253586, + "objective/train/docs_used": 764652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0899609327316284, + "objective/train/original_loss": 3.8389554023742676, + "objective/train/theoretical_loss": 3.549220408512161, + "objective/train/tokens_used": 1350041600, + "objective/train/unlikelihood_loss": 0.1967592090368271, + "theoretical_loss": 3.549220408512161, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984754262788365, + "loss": 1.3992, + "theoretical_loss": 3.549220408512161, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984553660982949, + "loss": 1.4038, + "theoretical_loss": 3.549189411393697, + "tokens_seen": 1350172672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029843530591775326, + "loss": 1.5086, + "theoretical_loss": 3.549158418126689, + "tokens_seen": 1350303744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984152457372116, + "loss": 1.5502, + "theoretical_loss": 3.549127428710285, + "tokens_seen": 1350434816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029839518555667, + "loss": 1.3481, + "theoretical_loss": 3.549096443143632, + "tokens_seen": 1350565888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983751253761284, + "loss": 1.2879, + "theoretical_loss": 3.5490654614258785, + "tokens_seen": 1350696960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029835506519558676, + "loss": 1.3624, + "theoretical_loss": 3.5490344835561736, + "tokens_seen": 1350828032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983350050150451, + "loss": 1.3609, + "theoretical_loss": 3.5490035095336654, + "tokens_seen": 1350959104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983149448345035, + "loss": 1.2784, + "theoretical_loss": 3.5489725393575036, + "tokens_seen": 1351090176 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029829488465396185, + "loss": 1.3689, + "theoretical_loss": 3.548941573026836, + "tokens_seen": 1351221248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029827482447342026, + "loss": 1.5363, + "theoretical_loss": 3.5489106105408137, + "tokens_seen": 1351352320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982547642928786, + "loss": 1.2733, + "theoretical_loss": 3.548879651898586, + "tokens_seen": 1351483392 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298234704112337, + "loss": 1.3972, + "theoretical_loss": 3.5488486970993027, + "tokens_seen": 1351614464 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.4323069453239441, + "objective/train/avg_token_score": 0.011420002207159996, + "objective/train/docs_used": 765372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3415284156799316, + "objective/train/original_loss": 3.6146607398986816, + "objective/train/theoretical_loss": 3.5488332211405, + "objective/train/tokens_used": 1351680000, + "objective/train/unlikelihood_loss": 0.20230014622211456, + "theoretical_loss": 3.5488332211405, + "tokens_seen": 1351680000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029821464393179535, + "loss": 1.3205, + "theoretical_loss": 3.548817746142115, + "tokens_seen": 1351745536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029819458375125377, + "loss": 1.432, + "theoretical_loss": 3.548786799026172, + "tokens_seen": 1351876608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029817452357071213, + "loss": 1.3938, + "theoretical_loss": 3.5487558557506267, + "tokens_seen": 1352007680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981544633901705, + "loss": 1.5368, + "theoretical_loss": 3.5487249163146286, + "tokens_seen": 1352138752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029813440320962885, + "loss": 1.4524, + "theoretical_loss": 3.5486939807173297, + "tokens_seen": 1352269824 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981143430290872, + "loss": 1.49, + "theoretical_loss": 3.548663048957882, + "tokens_seen": 1352400896 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029809428284854563, + "loss": 1.5409, + "theoretical_loss": 3.5486321210354372, + "tokens_seen": 1352531968 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298074222668004, + "loss": 1.3185, + "theoretical_loss": 3.5486011969491478, + "tokens_seen": 1352663040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029805416248746236, + "loss": 1.4291, + "theoretical_loss": 3.5485702766981655, + "tokens_seen": 1352794112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980341023069208, + "loss": 1.3834, + "theoretical_loss": 3.5485393602816444, + "tokens_seen": 1352925184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980140421263792, + "loss": 1.5338, + "theoretical_loss": 3.5485084476987367, + "tokens_seen": 1353056256 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029799398194583755, + "loss": 1.3972, + "theoretical_loss": 3.548477538948596, + "tokens_seen": 1353187328 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.4065249264240265, + "objective/train/avg_token_score": 0.010241014882922173, + "objective/train/docs_used": 766365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3413996696472168, + "objective/train/original_loss": 3.5842125415802, + "objective/train/theoretical_loss": 3.5484466340303755, + "objective/train/tokens_used": 1353318400, + "objective/train/unlikelihood_loss": 0.2105640470981598, + "theoretical_loss": 3.5484466340303755, + "tokens_seen": 1353318400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979739217652959, + "loss": 1.4877, + "theoretical_loss": 3.5484466340303755, + "tokens_seen": 1353318400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979538615847543, + "loss": 1.453, + "theoretical_loss": 3.54841573294323, + "tokens_seen": 1353449472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979338014042127, + "loss": 1.4765, + "theoretical_loss": 3.5483848356863126, + "tokens_seen": 1353580544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029791374122367106, + "loss": 1.3037, + "theoretical_loss": 3.548353942258778, + "tokens_seen": 1353711616 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978936810431294, + "loss": 1.3025, + "theoretical_loss": 3.548323052659782, + "tokens_seen": 1353842688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978736208625878, + "loss": 1.3941, + "theoretical_loss": 3.548292166888478, + "tokens_seen": 1353973760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029785356068204614, + "loss": 1.2565, + "theoretical_loss": 3.5482612849440223, + "tokens_seen": 1354104832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029783350050150456, + "loss": 1.5235, + "theoretical_loss": 3.548230406825569, + "tokens_seen": 1354235904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978134403209629, + "loss": 1.3694, + "theoretical_loss": 3.5481995325322755, + "tokens_seen": 1354366976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977933801404213, + "loss": 1.3509, + "theoretical_loss": 3.5481686620632966, + "tokens_seen": 1354498048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029777331995987965, + "loss": 1.2616, + "theoretical_loss": 3.5481377954177895, + "tokens_seen": 1354629120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029775325977933806, + "loss": 1.1786, + "theoretical_loss": 3.5481069325949104, + "tokens_seen": 1354760192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977331995987964, + "loss": 1.3739, + "theoretical_loss": 3.5480760735938155, + "tokens_seen": 1354891264 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.41055718064308167, + "objective/train/avg_token_score": 0.01602756790816784, + "objective/train/docs_used": 767078, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.402661919593811, + "objective/train/original_loss": 3.856752395629883, + "objective/train/theoretical_loss": 3.5480606455261743, + "objective/train/tokens_used": 1354956800, + "objective/train/unlikelihood_loss": 0.1429106593132019, + "theoretical_loss": 3.5480606455261743, + "tokens_seen": 1354956800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977131394182548, + "loss": 1.5704, + "theoretical_loss": 3.5480452184136633, + "tokens_seen": 1355022336 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029769307923771315, + "loss": 1.2845, + "theoretical_loss": 3.54801436705361, + "tokens_seen": 1355153408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976730190571715, + "loss": 1.4579, + "theoretical_loss": 3.547983519512813, + "tokens_seen": 1355284480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029765295887662993, + "loss": 1.4555, + "theoretical_loss": 3.547952675790431, + "tokens_seen": 1355415552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976328986960883, + "loss": 1.3434, + "theoretical_loss": 3.5479218358856226, + "tokens_seen": 1355546624 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029761283851554665, + "loss": 1.4855, + "theoretical_loss": 3.547890999797545, + "tokens_seen": 1355677696 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297592778335005, + "loss": 1.3368, + "theoretical_loss": 3.547860167525357, + "tokens_seen": 1355808768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029757271815446343, + "loss": 1.2822, + "theoretical_loss": 3.547829339068219, + "tokens_seen": 1355939840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975526579739218, + "loss": 1.3706, + "theoretical_loss": 3.5477985144252884, + "tokens_seen": 1356070912 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029753259779338016, + "loss": 1.4702, + "theoretical_loss": 3.547767693595726, + "tokens_seen": 1356201984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975125376128385, + "loss": 1.3886, + "theoretical_loss": 3.547736876578691, + "tokens_seen": 1356333056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974924774322969, + "loss": 1.2718, + "theoretical_loss": 3.547706063373343, + "tokens_seen": 1356464128 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.392778605222702, + "objective/train/avg_token_score": 0.008677978068590164, + "objective/train/docs_used": 768221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3046455383300781, + "objective/train/original_loss": 3.6662774085998535, + "objective/train/theoretical_loss": 3.547675253978843, + "objective/train/tokens_used": 1356595200, + "objective/train/unlikelihood_loss": 0.20256058871746063, + "theoretical_loss": 3.547675253978843, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974724172517553, + "loss": 1.355, + "theoretical_loss": 3.547675253978843, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029745235707121366, + "loss": 1.468, + "theoretical_loss": 3.547644448394352, + "tokens_seen": 1356726272 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297432296890672, + "loss": 1.2848, + "theoretical_loss": 3.5476136466190296, + "tokens_seen": 1356857344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974122367101304, + "loss": 1.1987, + "theoretical_loss": 3.5475828486520378, + "tokens_seen": 1356988416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973921765295888, + "loss": 1.3588, + "theoretical_loss": 3.547552054492537, + "tokens_seen": 1357119488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029737211634904716, + "loss": 1.2936, + "theoretical_loss": 3.5475212641396903, + "tokens_seen": 1357250560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973520561685055, + "loss": 1.3815, + "theoretical_loss": 3.5474904775926586, + "tokens_seen": 1357381632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973319959879639, + "loss": 1.2693, + "theoretical_loss": 3.547459694850604, + "tokens_seen": 1357512704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029731193580742225, + "loss": 1.417, + "theoretical_loss": 3.5474289159126893, + "tokens_seen": 1357643776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029729187562688067, + "loss": 1.6076, + "theoretical_loss": 3.5473981407780766, + "tokens_seen": 1357774848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029727181544633903, + "loss": 1.5102, + "theoretical_loss": 3.54736736944593, + "tokens_seen": 1357905920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972517552657974, + "loss": 1.3313, + "theoretical_loss": 3.5473366019154113, + "tokens_seen": 1358036992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029723169508525575, + "loss": 1.4392, + "theoretical_loss": 3.547305838185685, + "tokens_seen": 1358168064 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.5190615653991699, + "objective/train/avg_token_score": 0.015998471528291702, + "objective/train/docs_used": 768734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7258234024047852, + "objective/train/original_loss": 3.7690258026123047, + "objective/train/theoretical_loss": 3.5472904577458575, + "objective/train/tokens_used": 1358233600, + "objective/train/unlikelihood_loss": 0.18979965150356293, + "theoretical_loss": 3.5472904577458575, + "tokens_seen": 1358233600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029721163490471417, + "loss": 1.1748, + "theoretical_loss": 3.5472750782559146, + "tokens_seen": 1358299136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029719157472417253, + "loss": 1.3268, + "theoretical_loss": 3.5472443221252634, + "tokens_seen": 1358430208 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971715145436309, + "loss": 1.2791, + "theoretical_loss": 3.5472135697928966, + "tokens_seen": 1358561280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029715145436308926, + "loss": 1.216, + "theoretical_loss": 3.5471828212579792, + "tokens_seen": 1358692352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971313941825476, + "loss": 1.2804, + "theoretical_loss": 3.5471520765196747, + "tokens_seen": 1358823424 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029711133400200603, + "loss": 1.3608, + "theoretical_loss": 3.547121335577148, + "tokens_seen": 1358954496 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970912738214644, + "loss": 1.3725, + "theoretical_loss": 3.547090598429566, + "tokens_seen": 1359085568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029707121364092276, + "loss": 1.3397, + "theoretical_loss": 3.5470598650760934, + "tokens_seen": 1359216640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970511534603811, + "loss": 1.3449, + "theoretical_loss": 3.5470291355158956, + "tokens_seen": 1359347712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029703109327983954, + "loss": 1.1704, + "theoretical_loss": 3.5469984097481397, + "tokens_seen": 1359478784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970110330992979, + "loss": 1.4282, + "theoretical_loss": 3.5469676877719913, + "tokens_seen": 1359609856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029699097291875626, + "loss": 1.3197, + "theoretical_loss": 3.5469369695866177, + "tokens_seen": 1359740928 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.31579911708831787, + "objective/train/avg_token_score": 0.014897771179676056, + "objective/train/docs_used": 769315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9749266505241394, + "objective/train/original_loss": 4.02381706237793, + "objective/train/theoretical_loss": 3.5469062551911854, + "objective/train/tokens_used": 1359872000, + "objective/train/unlikelihood_loss": 0.17468640208244324, + "theoretical_loss": 3.5469062551911854, + "tokens_seen": 1359872000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969709127382146, + "loss": 1.1409, + "theoretical_loss": 3.5469062551911854, + "tokens_seen": 1359872000 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296950852557673, + "loss": 1.4472, + "theoretical_loss": 3.546875544584861, + "tokens_seen": 1360003072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969307923771314, + "loss": 1.3453, + "theoretical_loss": 3.5468448377668134, + "tokens_seen": 1360134144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029691073219658976, + "loss": 1.3096, + "theoretical_loss": 3.546814134736209, + "tokens_seen": 1360265216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968906720160481, + "loss": 1.3512, + "theoretical_loss": 3.5467834354922165, + "tokens_seen": 1360396288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968706118355065, + "loss": 1.2645, + "theoretical_loss": 3.546752740034004, + "tokens_seen": 1360527360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968505516549649, + "loss": 1.4728, + "theoretical_loss": 3.5467220483607393, + "tokens_seen": 1360658432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029683049147442327, + "loss": 1.3474, + "theoretical_loss": 3.546691360471592, + "tokens_seen": 1360789504 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029681043129388163, + "loss": 1.3469, + "theoretical_loss": 3.5466606763657316, + "tokens_seen": 1360920576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029679037111334, + "loss": 1.3925, + "theoretical_loss": 3.5466299960423258, + "tokens_seen": 1361051648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967703109327984, + "loss": 1.3474, + "theoretical_loss": 3.5465993195005456, + "tokens_seen": 1361182720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029675025075225677, + "loss": 1.2882, + "theoretical_loss": 3.5465686467395594, + "tokens_seen": 1361313792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029673019057171513, + "loss": 1.4982, + "theoretical_loss": 3.5465379777585384, + "tokens_seen": 1361444864 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.37609970569610596, + "objective/train/avg_token_score": 0.011834639124572277, + "objective/train/docs_used": 770761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2689532041549683, + "objective/train/original_loss": 4.130427360534668, + "objective/train/theoretical_loss": 3.546522644685256, + "objective/train/tokens_used": 1361510400, + "objective/train/unlikelihood_loss": 0.12086531519889832, + "theoretical_loss": 3.546522644685256, + "tokens_seen": 1361510400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967101303911735, + "loss": 1.3505, + "theoretical_loss": 3.5465073125566526, + "tokens_seen": 1361575936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029669007021063186, + "loss": 1.3037, + "theoretical_loss": 3.546476651133073, + "tokens_seen": 1361707008 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966700100300903, + "loss": 1.3853, + "theoretical_loss": 3.5464459934869694, + "tokens_seen": 1361838080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029664994984954864, + "loss": 1.2744, + "theoretical_loss": 3.5464153396175138, + "tokens_seen": 1361969152 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296629889669007, + "loss": 1.4691, + "theoretical_loss": 3.5463846895238773, + "tokens_seen": 1362100224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029660982948846536, + "loss": 1.4725, + "theoretical_loss": 3.5463540432052314, + "tokens_seen": 1362231296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965897693079238, + "loss": 1.3405, + "theoretical_loss": 3.5463234006607482, + "tokens_seen": 1362362368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029656970912738214, + "loss": 1.3733, + "theoretical_loss": 3.5462927618895996, + "tokens_seen": 1362493440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965496489468405, + "loss": 1.3671, + "theoretical_loss": 3.5462621268909587, + "tokens_seen": 1362624512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965295887662989, + "loss": 1.2408, + "theoretical_loss": 3.5462314956639975, + "tokens_seen": 1362755584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965095285857573, + "loss": 1.3075, + "theoretical_loss": 3.5462008682078894, + "tokens_seen": 1362886656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964894684052157, + "loss": 1.4789, + "theoretical_loss": 3.5461702445218073, + "tokens_seen": 1363017728 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.44703078269958496, + "objective/train/avg_token_score": 0.006488642655313015, + "objective/train/docs_used": 771430, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4047877788543701, + "objective/train/original_loss": 3.8531415462493896, + "objective/train/theoretical_loss": 3.5461396246049244, + "objective/train/tokens_used": 1363148800, + "objective/train/unlikelihood_loss": 0.14450199902057648, + "theoretical_loss": 3.5461396246049244, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029646940822467406, + "loss": 1.3975, + "theoretical_loss": 3.5461396246049244, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964493480441324, + "loss": 1.3673, + "theoretical_loss": 3.546109008456415, + "tokens_seen": 1363279872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964292878635908, + "loss": 1.219, + "theoretical_loss": 3.546078396075453, + "tokens_seen": 1363410944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964092276830492, + "loss": 1.4314, + "theoretical_loss": 3.5460477874612124, + "tokens_seen": 1363542016 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029638916750250756, + "loss": 1.349, + "theoretical_loss": 3.5460171826128675, + "tokens_seen": 1363673088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963691073219659, + "loss": 1.4665, + "theoretical_loss": 3.5459865815295935, + "tokens_seen": 1363804160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963490471414243, + "loss": 1.3137, + "theoretical_loss": 3.5459559842105657, + "tokens_seen": 1363935232 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029632898696088265, + "loss": 1.4276, + "theoretical_loss": 3.5459253906549586, + "tokens_seen": 1364066304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029630892678034107, + "loss": 1.4882, + "theoretical_loss": 3.545894800861948, + "tokens_seen": 1364197376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029628886659979943, + "loss": 1.2611, + "theoretical_loss": 3.54586421483071, + "tokens_seen": 1364328448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962688064192578, + "loss": 1.2766, + "theoretical_loss": 3.5458336325604205, + "tokens_seen": 1364459520 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029624874623871615, + "loss": 1.3535, + "theoretical_loss": 3.545803054050256, + "tokens_seen": 1364590592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029622868605817457, + "loss": 1.3949, + "theoretical_loss": 3.545772479299393, + "tokens_seen": 1364721664 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.2225073277950287, + "objective/train/avg_token_score": 0.013313478790223598, + "objective/train/docs_used": 772731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7895559072494507, + "objective/train/original_loss": 4.2685747146606445, + "objective/train/theoretical_loss": 3.5457571933334417, + "objective/train/tokens_used": 1364787200, + "objective/train/unlikelihood_loss": 0.14188525080680847, + "theoretical_loss": 3.5457571933334417, + "tokens_seen": 1364787200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029620862587763293, + "loss": 1.3706, + "theoretical_loss": 3.545741908307008, + "tokens_seen": 1364852736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961885656970913, + "loss": 1.2318, + "theoretical_loss": 3.5457113410722787, + "tokens_seen": 1364983808 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029616850551654966, + "loss": 1.2361, + "theoretical_loss": 3.5456807775943817, + "tokens_seen": 1365114880 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296148445336008, + "loss": 1.3977, + "theoretical_loss": 3.5456502178724953, + "tokens_seen": 1365245952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029612838515546643, + "loss": 1.3466, + "theoretical_loss": 3.545619661905797, + "tokens_seen": 1365377024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961083249749248, + "loss": 1.3434, + "theoretical_loss": 3.545589109693465, + "tokens_seen": 1365508096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029608826479438316, + "loss": 1.2816, + "theoretical_loss": 3.545558561234678, + "tokens_seen": 1365639168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960682046138415, + "loss": 1.23, + "theoretical_loss": 3.5455280165286145, + "tokens_seen": 1365770240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029604814443329994, + "loss": 1.2442, + "theoretical_loss": 3.545497475574453, + "tokens_seen": 1365901312 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002960280842527583, + "loss": 1.4104, + "theoretical_loss": 3.5454669383713737, + "tokens_seen": 1366032384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029600802407221666, + "loss": 1.3868, + "theoretical_loss": 3.5454364049185547, + "tokens_seen": 1366163456 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295987963891675, + "loss": 1.3741, + "theoretical_loss": 3.5454058752151765, + "tokens_seen": 1366294528 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.3832477927207947, + "objective/train/avg_token_score": 0.02153664268553257, + "objective/train/docs_used": 773461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1119582653045654, + "objective/train/original_loss": 4.065893650054932, + "objective/train/theoretical_loss": 3.545375349260419, + "objective/train/tokens_used": 1366425600, + "objective/train/unlikelihood_loss": 0.1231837272644043, + "theoretical_loss": 3.545375349260419, + "tokens_seen": 1366425600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959679037111334, + "loss": 1.2014, + "theoretical_loss": 3.545375349260419, + "tokens_seen": 1366425600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959478435305918, + "loss": 1.4217, + "theoretical_loss": 3.545344827053462, + "tokens_seen": 1366556672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029592778335005016, + "loss": 1.5003, + "theoretical_loss": 3.545314308593487, + "tokens_seen": 1366687744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029590772316950853, + "loss": 1.4066, + "theoretical_loss": 3.545283793879674, + "tokens_seen": 1366818816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958876629889669, + "loss": 1.1902, + "theoretical_loss": 3.5452532829112036, + "tokens_seen": 1366949888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958676028084253, + "loss": 1.3358, + "theoretical_loss": 3.5452227756872574, + "tokens_seen": 1367080960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029584754262788367, + "loss": 1.1917, + "theoretical_loss": 3.545192272207017, + "tokens_seen": 1367212032 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029582748244734203, + "loss": 1.4682, + "theoretical_loss": 3.5451617724696645, + "tokens_seen": 1367343104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958074222668004, + "loss": 1.2365, + "theoretical_loss": 3.5451312764743808, + "tokens_seen": 1367474176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957873620862588, + "loss": 1.3593, + "theoretical_loss": 3.545100784220349, + "tokens_seen": 1367605248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029576730190571717, + "loss": 1.3049, + "theoretical_loss": 3.5450702957067515, + "tokens_seen": 1367736320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029574724172517553, + "loss": 1.2968, + "theoretical_loss": 3.545039810932771, + "tokens_seen": 1367867392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957271815446339, + "loss": 1.2559, + "theoretical_loss": 3.5450093298975913, + "tokens_seen": 1367998464 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.5274926424026489, + "objective/train/avg_token_score": 0.003457870101556182, + "objective/train/docs_used": 774789, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6926124095916748, + "objective/train/original_loss": 3.2327685356140137, + "objective/train/theoretical_loss": 3.544994090781796, + "objective/train/tokens_used": 1368064000, + "objective/train/unlikelihood_loss": 0.19152002036571503, + "theoretical_loss": 3.544994090781796, + "tokens_seen": 1368064000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029570712136409226, + "loss": 1.5897, + "theoretical_loss": 3.5449788526003942, + "tokens_seen": 1368129536 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956870611835507, + "loss": 1.1808, + "theoretical_loss": 3.544948379040364, + "tokens_seen": 1368260608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029566700100300904, + "loss": 1.2728, + "theoretical_loss": 3.5449179092166854, + "tokens_seen": 1368391680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956469408224674, + "loss": 1.403, + "theoretical_loss": 3.544887443128541, + "tokens_seen": 1368522752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029562688064192576, + "loss": 1.4198, + "theoretical_loss": 3.544856980775116, + "tokens_seen": 1368653824 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956068204613842, + "loss": 1.4989, + "theoretical_loss": 3.544826522155595, + "tokens_seen": 1368784896 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029558676028084254, + "loss": 1.2888, + "theoretical_loss": 3.5447960672691616, + "tokens_seen": 1368915968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955667001003009, + "loss": 1.2687, + "theoretical_loss": 3.5447656161150025, + "tokens_seen": 1369047040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029554663991975926, + "loss": 1.2809, + "theoretical_loss": 3.5447351686923025, + "tokens_seen": 1369178112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955265797392176, + "loss": 1.4634, + "theoretical_loss": 3.544704725000247, + "tokens_seen": 1369309184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029550651955867604, + "loss": 1.4056, + "theoretical_loss": 3.5446742850380217, + "tokens_seen": 1369440256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954864593781344, + "loss": 1.3217, + "theoretical_loss": 3.5446438488048133, + "tokens_seen": 1369571328 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.37616080045700073, + "objective/train/avg_token_score": 0.0067526428028941154, + "objective/train/docs_used": 775516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.199377179145813, + "objective/train/original_loss": 3.344862937927246, + "objective/train/theoretical_loss": 3.544613416299808, + "objective/train/tokens_used": 1369702400, + "objective/train/unlikelihood_loss": 0.3153873682022095, + "theoretical_loss": 3.544613416299808, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029546639919759277, + "loss": 1.1882, + "theoretical_loss": 3.544613416299808, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029544633901705113, + "loss": 1.2943, + "theoretical_loss": 3.5445829875221917, + "tokens_seen": 1369833472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029542627883650955, + "loss": 1.2053, + "theoretical_loss": 3.544552562471152, + "tokens_seen": 1369964544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954062186559679, + "loss": 1.3088, + "theoretical_loss": 3.544522141145876, + "tokens_seen": 1370095616 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029538615847542627, + "loss": 1.3237, + "theoretical_loss": 3.544491723545551, + "tokens_seen": 1370226688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029536609829488463, + "loss": 1.3945, + "theoretical_loss": 3.5444613096693645, + "tokens_seen": 1370357760 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295346038114343, + "loss": 1.3082, + "theoretical_loss": 3.5444308995165046, + "tokens_seen": 1370488832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953259779338014, + "loss": 1.4763, + "theoretical_loss": 3.544400493086159, + "tokens_seen": 1370619904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953059177532598, + "loss": 1.4436, + "theoretical_loss": 3.544370090377517, + "tokens_seen": 1370750976 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029528585757271814, + "loss": 1.3531, + "theoretical_loss": 3.5443396913897667, + "tokens_seen": 1370882048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952657973921765, + "loss": 1.3074, + "theoretical_loss": 3.5443092961220968, + "tokens_seen": 1371013120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952457372116349, + "loss": 1.2854, + "theoretical_loss": 3.544278904573697, + "tokens_seen": 1371144192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952256770310933, + "loss": 1.4086, + "theoretical_loss": 3.544248516743756, + "tokens_seen": 1371275264 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.5130131840705872, + "objective/train/avg_token_score": 0.012053275480866432, + "objective/train/docs_used": 776338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7119686603546143, + "objective/train/original_loss": 3.9579577445983887, + "objective/train/theoretical_loss": 3.5442333242229545, + "objective/train/tokens_used": 1371340800, + "objective/train/unlikelihood_loss": 0.0924103632569313, + "theoretical_loss": 3.5442333242229545, + "tokens_seen": 1371340800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029520561685055164, + "loss": 1.3081, + "theoretical_loss": 3.544218132631464, + "tokens_seen": 1371406336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029518555667001, + "loss": 1.1758, + "theoretical_loss": 3.5441877522360112, + "tokens_seen": 1371537408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029516549648946836, + "loss": 1.3176, + "theoretical_loss": 3.544157375556587, + "tokens_seen": 1371668480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951454363089268, + "loss": 1.3926, + "theoretical_loss": 3.544127002592383, + "tokens_seen": 1371799552 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029512537612838514, + "loss": 1.3497, + "theoretical_loss": 3.5440966333425887, + "tokens_seen": 1371930624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951053159478435, + "loss": 1.332, + "theoretical_loss": 3.5440662678063957, + "tokens_seen": 1372061696 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029508525576730187, + "loss": 1.3399, + "theoretical_loss": 3.544035905982995, + "tokens_seen": 1372192768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950651955867603, + "loss": 1.4174, + "theoretical_loss": 3.5440055478715777, + "tokens_seen": 1372323840 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029504513540621865, + "loss": 1.4687, + "theoretical_loss": 3.5439751934713364, + "tokens_seen": 1372454912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029502507522567706, + "loss": 1.3839, + "theoretical_loss": 3.543944842781462, + "tokens_seen": 1372585984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950050150451354, + "loss": 1.3239, + "theoretical_loss": 3.5439144958011477, + "tokens_seen": 1372717056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949849548645938, + "loss": 1.215, + "theoretical_loss": 3.5438841525295857, + "tokens_seen": 1372848128 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.3090786933898926, + "objective/train/avg_token_score": 0.0593378059566021, + "objective/train/docs_used": 777606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9733283519744873, + "objective/train/original_loss": 4.13478422164917, + "objective/train/theoretical_loss": 3.5438538129659687, + "objective/train/tokens_used": 1372979200, + "objective/train/unlikelihood_loss": 0.15537269413471222, + "theoretical_loss": 3.5438538129659687, + "tokens_seen": 1372979200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949648946840522, + "loss": 1.4878, + "theoretical_loss": 3.5438538129659687, + "tokens_seen": 1372979200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029494483450351057, + "loss": 1.2223, + "theoretical_loss": 3.543823477109489, + "tokens_seen": 1373110272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029492477432296893, + "loss": 1.3993, + "theoretical_loss": 3.5437931449593405, + "tokens_seen": 1373241344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949047141424273, + "loss": 1.3469, + "theoretical_loss": 3.543762816514717, + "tokens_seen": 1373372416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948846539618857, + "loss": 1.1564, + "theoretical_loss": 3.5437324917748114, + "tokens_seen": 1373503488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029486459378134407, + "loss": 1.4159, + "theoretical_loss": 3.543702170738818, + "tokens_seen": 1373634560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029484453360080243, + "loss": 1.4433, + "theoretical_loss": 3.543671853405931, + "tokens_seen": 1373765632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948244734202608, + "loss": 1.2112, + "theoretical_loss": 3.5436415397753454, + "tokens_seen": 1373896704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948044132397192, + "loss": 1.484, + "theoretical_loss": 3.543611229846255, + "tokens_seen": 1374027776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029478435305917757, + "loss": 1.298, + "theoretical_loss": 3.543580923617856, + "tokens_seen": 1374158848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029476429287863593, + "loss": 1.5193, + "theoretical_loss": 3.5435506210893424, + "tokens_seen": 1374289920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947442326980943, + "loss": 1.2785, + "theoretical_loss": 3.5435203222599103, + "tokens_seen": 1374420992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029472417251755266, + "loss": 1.4969, + "theoretical_loss": 3.5434900271287555, + "tokens_seen": 1374552064 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.6265273690223694, + "objective/train/avg_token_score": 0.004024187568575144, + "objective/train/docs_used": 778164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.05932354927063, + "objective/train/original_loss": 3.4351367950439453, + "objective/train/theoretical_loss": 3.5434748809497814, + "objective/train/tokens_used": 1374617600, + "objective/train/unlikelihood_loss": 0.17800307273864746, + "theoretical_loss": 3.5434748809497814, + "tokens_seen": 1374617600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947041123370111, + "loss": 1.4847, + "theoretical_loss": 3.543459735695074, + "tokens_seen": 1374683136 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029468405215646944, + "loss": 1.2993, + "theoretical_loss": 3.5434294479580624, + "tokens_seen": 1374814208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946639919759278, + "loss": 1.3746, + "theoretical_loss": 3.5433991639169165, + "tokens_seen": 1374945280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029464393179538616, + "loss": 1.2956, + "theoretical_loss": 3.5433688835708326, + "tokens_seen": 1375076352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946238716148446, + "loss": 1.3928, + "theoretical_loss": 3.543338606919009, + "tokens_seen": 1375207424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029460381143430294, + "loss": 1.4016, + "theoretical_loss": 3.5433083339606424, + "tokens_seen": 1375338496 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945837512537613, + "loss": 1.4488, + "theoretical_loss": 3.5432780646949302, + "tokens_seen": 1375469568 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029456369107321966, + "loss": 1.2902, + "theoretical_loss": 3.5432477991210702, + "tokens_seen": 1375600640 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294543630892678, + "loss": 1.3363, + "theoretical_loss": 3.5432175372382604, + "tokens_seen": 1375731712 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029452357071213644, + "loss": 1.3246, + "theoretical_loss": 3.543187279045699, + "tokens_seen": 1375862784 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945035105315948, + "loss": 1.4936, + "theoretical_loss": 3.543157024542585, + "tokens_seen": 1375993856 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029448345035105317, + "loss": 1.3669, + "theoretical_loss": 3.5431267737281167, + "tokens_seen": 1376124928 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.43071848154067993, + "objective/train/avg_token_score": 0.01525172684341669, + "objective/train/docs_used": 779255, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3922213315963745, + "objective/train/original_loss": 3.840311050415039, + "objective/train/theoretical_loss": 3.5430965266014933, + "objective/train/tokens_used": 1376256000, + "objective/train/unlikelihood_loss": 0.15720538794994354, + "theoretical_loss": 3.5430965266014933, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029446339017051153, + "loss": 1.4027, + "theoretical_loss": 3.5430965266014933, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029444332998996995, + "loss": 1.424, + "theoretical_loss": 3.5430662831619135, + "tokens_seen": 1376387072 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944232698094283, + "loss": 1.4116, + "theoretical_loss": 3.5430360434085775, + "tokens_seen": 1376518144 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029440320962888667, + "loss": 1.3922, + "theoretical_loss": 3.543005807340685, + "tokens_seen": 1376649216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029438314944834503, + "loss": 1.3893, + "theoretical_loss": 3.542975574957435, + "tokens_seen": 1376780288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943630892678034, + "loss": 1.3915, + "theoretical_loss": 3.5429453462580294, + "tokens_seen": 1376911360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943430290872618, + "loss": 1.3656, + "theoretical_loss": 3.5429151212416676, + "tokens_seen": 1377042432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943229689067202, + "loss": 1.3159, + "theoretical_loss": 3.5428848999075506, + "tokens_seen": 1377173504 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029430290872617854, + "loss": 1.2237, + "theoretical_loss": 3.5428546822548794, + "tokens_seen": 1377304576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942828485456369, + "loss": 1.2281, + "theoretical_loss": 3.5428244682828556, + "tokens_seen": 1377435648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942627883650953, + "loss": 1.292, + "theoretical_loss": 3.54279425799068, + "tokens_seen": 1377566720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942427281845537, + "loss": 1.2123, + "theoretical_loss": 3.5427640513775556, + "tokens_seen": 1377697792 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029422266800401204, + "loss": 1.4149, + "theoretical_loss": 3.5427338484426825, + "tokens_seen": 1377828864 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.4788611829280853, + "objective/train/avg_token_score": 0.014572910964488983, + "objective/train/docs_used": 780014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4796091318130493, + "objective/train/original_loss": 3.473543882369995, + "objective/train/theoretical_loss": 3.542718748354342, + "objective/train/tokens_used": 1377894400, + "objective/train/unlikelihood_loss": 0.14301840960979462, + "theoretical_loss": 3.542718748354342, + "tokens_seen": 1377894400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942026078234704, + "loss": 1.2032, + "theoretical_loss": 3.542703649185265, + "tokens_seen": 1377959936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029418254764292876, + "loss": 1.2918, + "theoretical_loss": 3.542673453604504, + "tokens_seen": 1378091008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941624874623872, + "loss": 1.4196, + "theoretical_loss": 3.5426432616996033, + "tokens_seen": 1378222080 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029414242728184554, + "loss": 1.2509, + "theoretical_loss": 3.542613073469765, + "tokens_seen": 1378353152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941223671013039, + "loss": 1.5262, + "theoretical_loss": 3.5425828889141933, + "tokens_seen": 1378484224 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029410230692076227, + "loss": 1.2724, + "theoretical_loss": 3.542552708032091, + "tokens_seen": 1378615296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940822467402207, + "loss": 1.4093, + "theoretical_loss": 3.542522530822662, + "tokens_seen": 1378746368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029406218655967905, + "loss": 1.2874, + "theoretical_loss": 3.54249235728511, + "tokens_seen": 1378877440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940421263791374, + "loss": 1.4032, + "theoretical_loss": 3.5424621874186406, + "tokens_seen": 1379008512 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029402206619859577, + "loss": 1.4246, + "theoretical_loss": 3.5424320212224565, + "tokens_seen": 1379139584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029400200601805413, + "loss": 1.0657, + "theoretical_loss": 3.542401858695764, + "tokens_seen": 1379270656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029398194583751255, + "loss": 1.4502, + "theoretical_loss": 3.5423716998377666, + "tokens_seen": 1379401728 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.6254887580871582, + "objective/train/avg_token_score": 0.010154049843549728, + "objective/train/docs_used": 781246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8852906227111816, + "objective/train/original_loss": 3.136091470718384, + "objective/train/theoretical_loss": 3.5423415446476705, + "objective/train/tokens_used": 1379532800, + "objective/train/unlikelihood_loss": 0.21887630224227905, + "theoretical_loss": 3.5423415446476705, + "tokens_seen": 1379532800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939618856569709, + "loss": 1.2541, + "theoretical_loss": 3.5423415446476705, + "tokens_seen": 1379532800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939418254764293, + "loss": 1.3171, + "theoretical_loss": 3.542311393124681, + "tokens_seen": 1379663872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029392176529588764, + "loss": 1.177, + "theoretical_loss": 3.542281245268004, + "tokens_seen": 1379794944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029390170511534605, + "loss": 1.2663, + "theoretical_loss": 3.542251101076845, + "tokens_seen": 1379926016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938816449348044, + "loss": 1.3106, + "theoretical_loss": 3.5422209605504107, + "tokens_seen": 1380057088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938615847542628, + "loss": 1.4857, + "theoretical_loss": 3.542190823687907, + "tokens_seen": 1380188160 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029384152457372114, + "loss": 1.4585, + "theoretical_loss": 3.542160690488541, + "tokens_seen": 1380319232 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938214643931795, + "loss": 1.2744, + "theoretical_loss": 3.54213056095152, + "tokens_seen": 1380450304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938014042126379, + "loss": 1.3791, + "theoretical_loss": 3.5421004350760508, + "tokens_seen": 1380581376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937813440320963, + "loss": 1.2815, + "theoretical_loss": 3.54207031286134, + "tokens_seen": 1380712448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029376128385155464, + "loss": 1.2479, + "theoretical_loss": 3.5420401943065976, + "tokens_seen": 1380843520 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293741223671013, + "loss": 1.2584, + "theoretical_loss": 3.542010079411029, + "tokens_seen": 1380974592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937211634904714, + "loss": 1.3358, + "theoretical_loss": 3.5419799681738446, + "tokens_seen": 1381105664 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.3420698940753937, + "objective/train/avg_token_score": 0.01950184442102909, + "objective/train/docs_used": 781914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1941044330596924, + "objective/train/original_loss": 4.579495429992676, + "objective/train/theoretical_loss": 3.5419649139268987, + "objective/train/tokens_used": 1381171200, + "objective/train/unlikelihood_loss": 0.13264885544776917, + "theoretical_loss": 3.5419649139268987, + "tokens_seen": 1381171200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937011033099298, + "loss": 1.3652, + "theoretical_loss": 3.541949860594251, + "tokens_seen": 1381236736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029368104312938815, + "loss": 1.3669, + "theoretical_loss": 3.541919756671458, + "tokens_seen": 1381367808 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936609829488465, + "loss": 1.3824, + "theoretical_loss": 3.541889656404674, + "tokens_seen": 1381498880 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029364092276830487, + "loss": 1.4127, + "theoretical_loss": 3.541859559793109, + "tokens_seen": 1381629952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936208625877633, + "loss": 1.4117, + "theoretical_loss": 3.5418294668359716, + "tokens_seen": 1381761024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029360080240722165, + "loss": 1.4214, + "theoretical_loss": 3.5417993775324716, + "tokens_seen": 1381892096 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029358074222668, + "loss": 1.3621, + "theoretical_loss": 3.5417692918818187, + "tokens_seen": 1382023168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935606820461384, + "loss": 1.299, + "theoretical_loss": 3.5417392098832243, + "tokens_seen": 1382154240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029354062186559684, + "loss": 1.2671, + "theoretical_loss": 3.5417091315358973, + "tokens_seen": 1382285312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935205616850552, + "loss": 1.4055, + "theoretical_loss": 3.5416790568390493, + "tokens_seen": 1382416384 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029350050150451357, + "loss": 1.3663, + "theoretical_loss": 3.5416489857918902, + "tokens_seen": 1382547456 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029348044132397193, + "loss": 1.3255, + "theoretical_loss": 3.541618918393633, + "tokens_seen": 1382678528 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.43542277812957764, + "objective/train/avg_token_score": 0.014271651394665241, + "objective/train/docs_used": 783265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5443719625473022, + "objective/train/original_loss": 3.835261821746826, + "objective/train/theoretical_loss": 3.541588854643487, + "objective/train/tokens_used": 1382809600, + "objective/train/unlikelihood_loss": 0.18145227432250977, + "theoretical_loss": 3.541588854643487, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029346038114343035, + "loss": 1.3492, + "theoretical_loss": 3.541588854643487, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934403209628887, + "loss": 1.3669, + "theoretical_loss": 3.541558794540665, + "tokens_seen": 1382940672 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029342026078234707, + "loss": 1.4101, + "theoretical_loss": 3.541528738084379, + "tokens_seen": 1383071744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029340020060180543, + "loss": 1.2679, + "theoretical_loss": 3.54149868527384, + "tokens_seen": 1383202816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933801404212638, + "loss": 1.3141, + "theoretical_loss": 3.541468636108262, + "tokens_seen": 1383333888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933600802407222, + "loss": 1.3162, + "theoretical_loss": 3.541438590586857, + "tokens_seen": 1383464960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933400200601806, + "loss": 1.294, + "theoretical_loss": 3.541408548708837, + "tokens_seen": 1383596032 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029331995987963894, + "loss": 1.3565, + "theoretical_loss": 3.5413785104734163, + "tokens_seen": 1383727104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932998996990973, + "loss": 1.5906, + "theoretical_loss": 3.5413484758798073, + "tokens_seen": 1383858176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932798395185557, + "loss": 1.4618, + "theoretical_loss": 3.5413184449272244, + "tokens_seen": 1383989248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932597793380141, + "loss": 1.297, + "theoretical_loss": 3.5412884176148807, + "tokens_seen": 1384120320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029323971915747244, + "loss": 1.6127, + "theoretical_loss": 3.5412583939419906, + "tokens_seen": 1384251392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932196589769308, + "loss": 1.3545, + "theoretical_loss": 3.541228373907769, + "tokens_seen": 1384382464 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.43517839908599854, + "objective/train/avg_token_score": 0.014909998513758183, + "objective/train/docs_used": 783958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5483424663543701, + "objective/train/original_loss": 3.9835472106933594, + "objective/train/theoretical_loss": 3.541213365254913, + "objective/train/tokens_used": 1384448000, + "objective/train/unlikelihood_loss": 0.17587800323963165, + "theoretical_loss": 3.541213365254913, + "tokens_seen": 1384448000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029319959879638916, + "loss": 1.2458, + "theoretical_loss": 3.5411983575114294, + "tokens_seen": 1384513536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931795386158476, + "loss": 1.3542, + "theoretical_loss": 3.5411683447521876, + "tokens_seen": 1384644608 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029315947843530594, + "loss": 1.3299, + "theoretical_loss": 3.541138335629258, + "tokens_seen": 1384775680 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931394182547643, + "loss": 1.4009, + "theoretical_loss": 3.541108330141856, + "tokens_seen": 1384906752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029311935807422267, + "loss": 1.3684, + "theoretical_loss": 3.5410783282891973, + "tokens_seen": 1385037824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930992978936811, + "loss": 1.3394, + "theoretical_loss": 3.541048330070498, + "tokens_seen": 1385168896 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029307923771313945, + "loss": 1.3484, + "theoretical_loss": 3.541018335484974, + "tokens_seen": 1385299968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930591775325978, + "loss": 1.413, + "theoretical_loss": 3.540988344531841, + "tokens_seen": 1385431040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029303911735205617, + "loss": 1.4154, + "theoretical_loss": 3.540958357210316, + "tokens_seen": 1385562112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029301905717151453, + "loss": 1.3893, + "theoretical_loss": 3.5409283735196153, + "tokens_seen": 1385693184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029299899699097295, + "loss": 1.3307, + "theoretical_loss": 3.5408983934589573, + "tokens_seen": 1385824256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929789368104313, + "loss": 1.4159, + "theoretical_loss": 3.5408684170275575, + "tokens_seen": 1385955328 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.49114125967025757, + "objective/train/avg_token_score": 0.004459821153432131, + "objective/train/docs_used": 785103, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5754302740097046, + "objective/train/original_loss": 3.456366777420044, + "objective/train/theoretical_loss": 3.5408384442246343, + "objective/train/tokens_used": 1386086400, + "objective/train/unlikelihood_loss": 0.19322066009044647, + "theoretical_loss": 3.5408384442246343, + "tokens_seen": 1386086400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929588766298897, + "loss": 1.4232, + "theoretical_loss": 3.5408384442246343, + "tokens_seen": 1386086400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029293881644934804, + "loss": 1.1936, + "theoretical_loss": 3.5408084750494053, + "tokens_seen": 1386217472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029291875626880645, + "loss": 1.3424, + "theoretical_loss": 3.540778509501089, + "tokens_seen": 1386348544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928986960882648, + "loss": 1.3503, + "theoretical_loss": 3.540748547578902, + "tokens_seen": 1386479616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928786359077232, + "loss": 1.3919, + "theoretical_loss": 3.540718589282065, + "tokens_seen": 1386610688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029285857572718154, + "loss": 1.6609, + "theoretical_loss": 3.540688634609795, + "tokens_seen": 1386741760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928385155466399, + "loss": 1.3142, + "theoretical_loss": 3.540658683561311, + "tokens_seen": 1386872832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928184553660983, + "loss": 1.4651, + "theoretical_loss": 3.540628736135834, + "tokens_seen": 1387003904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927983951855567, + "loss": 1.2942, + "theoretical_loss": 3.540598792332581, + "tokens_seen": 1387134976 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029277833500501504, + "loss": 1.4312, + "theoretical_loss": 3.5405688521507734, + "tokens_seen": 1387266048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927582748244734, + "loss": 1.5232, + "theoretical_loss": 3.5405389155896305, + "tokens_seen": 1387397120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927382146439318, + "loss": 1.3119, + "theoretical_loss": 3.540508982648372, + "tokens_seen": 1387528192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927181544633902, + "loss": 1.2742, + "theoretical_loss": 3.5404790533262194, + "tokens_seen": 1387659264 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.4055474102497101, + "objective/train/avg_token_score": 0.008743871934711933, + "objective/train/docs_used": 785794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3564914464950562, + "objective/train/original_loss": 3.8764495849609375, + "objective/train/theoretical_loss": 3.5404640900220636, + "objective/train/tokens_used": 1387724800, + "objective/train/unlikelihood_loss": 0.16515983641147614, + "theoretical_loss": 3.5404640900220636, + "tokens_seen": 1387724800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029269809428284855, + "loss": 1.3914, + "theoretical_loss": 3.5404491276223924, + "tokens_seen": 1387790336 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926780341023069, + "loss": 1.2596, + "theoretical_loss": 3.5404192055361126, + "tokens_seen": 1387921408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029265797392176527, + "loss": 1.2465, + "theoretical_loss": 3.5403892870666, + "tokens_seen": 1388052480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926379137412237, + "loss": 1.4273, + "theoretical_loss": 3.5403593722130773, + "tokens_seen": 1388183552 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029261785356068205, + "loss": 1.1738, + "theoretical_loss": 3.5403294609747653, + "tokens_seen": 1388314624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925977933801404, + "loss": 1.2801, + "theoretical_loss": 3.540299553350886, + "tokens_seen": 1388445696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925777331995988, + "loss": 1.3457, + "theoretical_loss": 3.5402696493406616, + "tokens_seen": 1388576768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925576730190572, + "loss": 1.2544, + "theoretical_loss": 3.5402397489433146, + "tokens_seen": 1388707840 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029253761283851555, + "loss": 1.3713, + "theoretical_loss": 3.5402098521580676, + "tokens_seen": 1388838912 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925175526579739, + "loss": 1.3332, + "theoretical_loss": 3.5401799589841425, + "tokens_seen": 1388969984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924974924774323, + "loss": 1.4971, + "theoretical_loss": 3.5401500694207635, + "tokens_seen": 1389101056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924774322968907, + "loss": 1.2568, + "theoretical_loss": 3.540120183467153, + "tokens_seen": 1389232128 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.4174608886241913, + "objective/train/avg_token_score": 0.02466796152293682, + "objective/train/docs_used": 786491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.130980372428894, + "objective/train/original_loss": 3.3902463912963867, + "objective/train/theoretical_loss": 3.540090301122535, + "objective/train/tokens_used": 1389363200, + "objective/train/unlikelihood_loss": 0.2165214717388153, + "theoretical_loss": 3.540090301122535, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029245737211634906, + "loss": 1.5047, + "theoretical_loss": 3.540090301122535, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924373119358074, + "loss": 1.2047, + "theoretical_loss": 3.5400604223861336, + "tokens_seen": 1389494272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924172517552658, + "loss": 1.4467, + "theoretical_loss": 3.5400305472571723, + "tokens_seen": 1389625344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029239719157472414, + "loss": 1.3233, + "theoretical_loss": 3.540000675734875, + "tokens_seen": 1389756416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029237713139418256, + "loss": 1.3624, + "theoretical_loss": 3.5399708078184675, + "tokens_seen": 1389887488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923570712136409, + "loss": 1.3623, + "theoretical_loss": 3.5399409435071734, + "tokens_seen": 1390018560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923370110330993, + "loss": 1.1911, + "theoretical_loss": 3.5399110828002174, + "tokens_seen": 1390149632 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029231695085255765, + "loss": 1.3324, + "theoretical_loss": 3.539881225696826, + "tokens_seen": 1390280704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029229689067201606, + "loss": 1.4191, + "theoretical_loss": 3.5398513721962237, + "tokens_seen": 1390411776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922768304914744, + "loss": 1.2396, + "theoretical_loss": 3.539821522297636, + "tokens_seen": 1390542848 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922567703109328, + "loss": 1.4174, + "theoretical_loss": 3.53979167600029, + "tokens_seen": 1390673920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029223671013039115, + "loss": 1.3129, + "theoretical_loss": 3.5397618333034107, + "tokens_seen": 1390804992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922166499498495, + "loss": 1.2793, + "theoretical_loss": 3.5397319942062255, + "tokens_seen": 1390936064 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.4578445851802826, + "objective/train/avg_token_score": 0.009505200199782848, + "objective/train/docs_used": 787676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4734506607055664, + "objective/train/original_loss": 3.9642934799194336, + "objective/train/theoretical_loss": 3.539717076007276, + "objective/train/tokens_used": 1391001600, + "objective/train/unlikelihood_loss": 0.1418934166431427, + "theoretical_loss": 3.539717076007276, + "tokens_seen": 1391001600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921965897693079, + "loss": 1.3301, + "theoretical_loss": 3.53970215870796, + "tokens_seen": 1391067136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921765295887663, + "loss": 1.2613, + "theoretical_loss": 3.5396723268078416, + "tokens_seen": 1391198208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029215646940822465, + "loss": 1.3871, + "theoretical_loss": 3.5396424985050974, + "tokens_seen": 1391329280 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292136409227683, + "loss": 1.2738, + "theoretical_loss": 3.5396126737989553, + "tokens_seen": 1391460352 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029211634904714143, + "loss": 1.3163, + "theoretical_loss": 3.5395828526886417, + "tokens_seen": 1391591424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920962888665998, + "loss": 1.3357, + "theoretical_loss": 3.5395530351733853, + "tokens_seen": 1391722496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029207622868605815, + "loss": 1.4221, + "theoretical_loss": 3.539523221252414, + "tokens_seen": 1391853568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920561685055165, + "loss": 1.2708, + "theoretical_loss": 3.5394934109249565, + "tokens_seen": 1391984640 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029203610832497493, + "loss": 1.3976, + "theoretical_loss": 3.5394636041902405, + "tokens_seen": 1392115712 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029201604814443335, + "loss": 1.2995, + "theoretical_loss": 3.5394338010474957, + "tokens_seen": 1392246784 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919959879638917, + "loss": 1.1795, + "theoretical_loss": 3.5394040014959502, + "tokens_seen": 1392377856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919759277833501, + "loss": 1.517, + "theoretical_loss": 3.539374205534834, + "tokens_seen": 1392508928 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.47214075922966003, + "objective/train/avg_token_score": 0.01717517338693142, + "objective/train/docs_used": 788318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4890409708023071, + "objective/train/original_loss": 3.6603891849517822, + "objective/train/theoretical_loss": 3.5393444131633762, + "objective/train/tokens_used": 1392640000, + "objective/train/unlikelihood_loss": 0.17020604014396667, + "theoretical_loss": 3.5393444131633762, + "tokens_seen": 1392640000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029195586760280844, + "loss": 1.3025, + "theoretical_loss": 3.5393444131633762, + "tokens_seen": 1392640000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029193580742226685, + "loss": 1.2785, + "theoretical_loss": 3.5393146243808076, + "tokens_seen": 1392771072 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919157472417252, + "loss": 1.2514, + "theoretical_loss": 3.5392848391863563, + "tokens_seen": 1392902144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918956870611836, + "loss": 1.4418, + "theoretical_loss": 3.539255057579254, + "tokens_seen": 1393033216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029187562688064194, + "loss": 1.3331, + "theoretical_loss": 3.539225279558731, + "tokens_seen": 1393164288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918555667001003, + "loss": 1.3457, + "theoretical_loss": 3.539195505124017, + "tokens_seen": 1393295360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918355065195587, + "loss": 1.3345, + "theoretical_loss": 3.539165734274344, + "tokens_seen": 1393426432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918154463390171, + "loss": 1.3923, + "theoretical_loss": 3.539135967008943, + "tokens_seen": 1393557504 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029179538615847544, + "loss": 1.3376, + "theoretical_loss": 3.5391062033270453, + "tokens_seen": 1393688576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917753259779338, + "loss": 1.3833, + "theoretical_loss": 3.539076443227882, + "tokens_seen": 1393819648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917552657973922, + "loss": 1.3564, + "theoretical_loss": 3.5390466867106856, + "tokens_seen": 1393950720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917352056168506, + "loss": 1.4431, + "theoretical_loss": 3.5390169337746884, + "tokens_seen": 1394081792 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029171514543630895, + "loss": 1.3258, + "theoretical_loss": 3.5389871844191223, + "tokens_seen": 1394212864 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.3971163332462311, + "objective/train/avg_token_score": 0.007746445946395397, + "objective/train/docs_used": 789787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2720555067062378, + "objective/train/original_loss": 3.423855781555176, + "objective/train/theoretical_loss": 3.5389723110837608, + "objective/train/tokens_used": 1394278400, + "objective/train/unlikelihood_loss": 0.22334612905979156, + "theoretical_loss": 3.5389723110837608, + "tokens_seen": 1394278400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916950852557673, + "loss": 1.2249, + "theoretical_loss": 3.53895743864322, + "tokens_seen": 1394343936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029167502507522567, + "loss": 1.4762, + "theoretical_loss": 3.538927696446214, + "tokens_seen": 1394475008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916549648946841, + "loss": 1.2728, + "theoretical_loss": 3.5388979578273383, + "tokens_seen": 1394606080 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029163490471414245, + "loss": 1.2294, + "theoretical_loss": 3.5388682227858252, + "tokens_seen": 1394737152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916148445336008, + "loss": 1.2782, + "theoretical_loss": 3.538838491320909, + "tokens_seen": 1394868224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915947843530592, + "loss": 1.4809, + "theoretical_loss": 3.538808763431823, + "tokens_seen": 1394999296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915747241725176, + "loss": 1.2968, + "theoretical_loss": 3.5387790391178013, + "tokens_seen": 1395130368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029155466399197595, + "loss": 1.2696, + "theoretical_loss": 3.5387493183780783, + "tokens_seen": 1395261440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915346038114343, + "loss": 1.2707, + "theoretical_loss": 3.5387196012118887, + "tokens_seen": 1395392512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915145436308927, + "loss": 1.4156, + "theoretical_loss": 3.5386898876184665, + "tokens_seen": 1395523584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914944834503511, + "loss": 1.419, + "theoretical_loss": 3.538660177597047, + "tokens_seen": 1395654656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029147442326980946, + "loss": 1.3346, + "theoretical_loss": 3.538630471146866, + "tokens_seen": 1395785728 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.2877565920352936, + "objective/train/avg_token_score": 0.016363663598895073, + "objective/train/docs_used": 790377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0167137384414673, + "objective/train/original_loss": 3.836961030960083, + "objective/train/theoretical_loss": 3.5386007682671576, + "objective/train/tokens_used": 1395916800, + "objective/train/unlikelihood_loss": 0.20208749175071716, + "theoretical_loss": 3.5386007682671576, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914543630892678, + "loss": 1.2812, + "theoretical_loss": 3.5386007682671576, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914343029087262, + "loss": 1.3604, + "theoretical_loss": 3.538571068957159, + "tokens_seen": 1396047872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029141424272818454, + "loss": 1.2867, + "theoretical_loss": 3.5385413732161046, + "tokens_seen": 1396178944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029139418254764296, + "loss": 1.4707, + "theoretical_loss": 3.538511681043232, + "tokens_seen": 1396310016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913741223671013, + "loss": 1.2554, + "theoretical_loss": 3.5384819924377764, + "tokens_seen": 1396441088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913540621865597, + "loss": 1.2913, + "theoretical_loss": 3.538452307398975, + "tokens_seen": 1396572160 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029133400200601805, + "loss": 1.4779, + "theoretical_loss": 3.5384226259260645, + "tokens_seen": 1396703232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029131394182547646, + "loss": 1.4981, + "theoretical_loss": 3.5383929480182825, + "tokens_seen": 1396834304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912938816449348, + "loss": 1.2832, + "theoretical_loss": 3.5383632736748654, + "tokens_seen": 1396965376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912738214643932, + "loss": 1.3564, + "theoretical_loss": 3.538333602895051, + "tokens_seen": 1397096448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029125376128385155, + "loss": 1.3337, + "theoretical_loss": 3.538303935678077, + "tokens_seen": 1397227520 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912337011033099, + "loss": 1.3636, + "theoretical_loss": 3.538274272023182, + "tokens_seen": 1397358592 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029121364092276833, + "loss": 1.2, + "theoretical_loss": 3.5382446119296036, + "tokens_seen": 1397489664 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.41385629773139954, + "objective/train/avg_token_score": 0.012832061387598515, + "objective/train/docs_used": 791674, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.282406210899353, + "objective/train/original_loss": 3.8764142990112305, + "objective/train/theoretical_loss": 3.5382297832180702, + "objective/train/tokens_used": 1397555200, + "objective/train/unlikelihood_loss": 0.1881641447544098, + "theoretical_loss": 3.5382297832180702, + "tokens_seen": 1397555200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911935807422267, + "loss": 1.4252, + "theoretical_loss": 3.5382149553965805, + "tokens_seen": 1397620736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029117352056168505, + "loss": 1.436, + "theoretical_loss": 3.538185302423352, + "tokens_seen": 1397751808 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911534603811434, + "loss": 1.4061, + "theoretical_loss": 3.5381556530091562, + "tokens_seen": 1397882880 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029113340020060183, + "loss": 1.3854, + "theoretical_loss": 3.5381260071532328, + "tokens_seen": 1398013952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911133400200602, + "loss": 1.3859, + "theoretical_loss": 3.5380963648548205, + "tokens_seen": 1398145024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029109327983951855, + "loss": 1.4101, + "theoretical_loss": 3.5380667261131595, + "tokens_seen": 1398276096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910732196589769, + "loss": 1.4806, + "theoretical_loss": 3.5380370909274896, + "tokens_seen": 1398407168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910531594784353, + "loss": 1.251, + "theoretical_loss": 3.5380074592970514, + "tokens_seen": 1398538240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910330992978937, + "loss": 1.3192, + "theoretical_loss": 3.5379778312210846, + "tokens_seen": 1398669312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029101303911735206, + "loss": 1.395, + "theoretical_loss": 3.53794820669883, + "tokens_seen": 1398800384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909929789368104, + "loss": 1.3916, + "theoretical_loss": 3.5379185857295283, + "tokens_seen": 1398931456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909729187562688, + "loss": 1.3661, + "theoretical_loss": 3.537888968312421, + "tokens_seen": 1399062528 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.49144673347473145, + "objective/train/avg_token_score": 0.01360029075294733, + "objective/train/docs_used": 792315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5705522298812866, + "objective/train/original_loss": 4.270318508148193, + "objective/train/theoretical_loss": 3.5378593544467494, + "objective/train/tokens_used": 1399193600, + "objective/train/unlikelihood_loss": 0.1311744600534439, + "theoretical_loss": 3.5378593544467494, + "tokens_seen": 1399193600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909528585757272, + "loss": 1.3913, + "theoretical_loss": 3.5378593544467494, + "tokens_seen": 1399193600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029093279839518556, + "loss": 1.2836, + "theoretical_loss": 3.537829744131754, + "tokens_seen": 1399324672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909127382146439, + "loss": 1.4457, + "theoretical_loss": 3.537800137366677, + "tokens_seen": 1399455744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908926780341023, + "loss": 1.3359, + "theoretical_loss": 3.537770534150761, + "tokens_seen": 1399586816 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029087261785356065, + "loss": 1.3276, + "theoretical_loss": 3.537740934483248, + "tokens_seen": 1399717888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029085255767301906, + "loss": 1.4649, + "theoretical_loss": 3.5377113383633803, + "tokens_seen": 1399848960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908324974924774, + "loss": 1.5028, + "theoretical_loss": 3.5376817457904, + "tokens_seen": 1399980032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908124373119358, + "loss": 1.4372, + "theoretical_loss": 3.537652156763551, + "tokens_seen": 1400111104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029079237713139415, + "loss": 1.4138, + "theoretical_loss": 3.537622571282076, + "tokens_seen": 1400242176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029077231695085257, + "loss": 1.3341, + "theoretical_loss": 3.5375929893452187, + "tokens_seen": 1400373248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029075225677031093, + "loss": 1.2907, + "theoretical_loss": 3.537563410952222, + "tokens_seen": 1400504320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907321965897693, + "loss": 1.3734, + "theoretical_loss": 3.5375338361023303, + "tokens_seen": 1400635392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029071213640922765, + "loss": 1.1979, + "theoretical_loss": 3.5375042647947876, + "tokens_seen": 1400766464 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.4954789876937866, + "objective/train/avg_token_score": 0.005254681687802076, + "objective/train/docs_used": 793814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5649027824401855, + "objective/train/original_loss": 3.546599864959717, + "objective/train/theoretical_loss": 3.537489480469161, + "objective/train/tokens_used": 1400832000, + "objective/train/unlikelihood_loss": 0.1988324373960495, + "theoretical_loss": 3.537489480469161, + "tokens_seen": 1400832000 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290692076228686, + "loss": 1.3703, + "theoretical_loss": 3.5374746970288378, + "tokens_seen": 1400897536 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029067201604814443, + "loss": 1.3966, + "theoretical_loss": 3.537445132803726, + "tokens_seen": 1401028608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906519558676028, + "loss": 1.4002, + "theoretical_loss": 3.537415572118697, + "tokens_seen": 1401159680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029063189568706116, + "loss": 1.4681, + "theoretical_loss": 3.5373860149729954, + "tokens_seen": 1401290752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906118355065195, + "loss": 1.2934, + "theoretical_loss": 3.537356461365867, + "tokens_seen": 1401421824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029059177532597794, + "loss": 1.3296, + "theoretical_loss": 3.5373269112965566, + "tokens_seen": 1401552896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905717151454363, + "loss": 1.4501, + "theoretical_loss": 3.53729736476431, + "tokens_seen": 1401683968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029055165496489466, + "loss": 1.3191, + "theoretical_loss": 3.5372678217683733, + "tokens_seen": 1401815040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905315947843531, + "loss": 1.305, + "theoretical_loss": 3.5372382823079924, + "tokens_seen": 1401946112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905115346038115, + "loss": 1.4602, + "theoretical_loss": 3.5372087463824142, + "tokens_seen": 1402077184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029049147442326986, + "loss": 1.5651, + "theoretical_loss": 3.5371792139908855, + "tokens_seen": 1402208256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904714142427282, + "loss": 1.325, + "theoretical_loss": 3.5371496851326523, + "tokens_seen": 1402339328 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.5549853444099426, + "objective/train/avg_token_score": 0.009370463900268078, + "objective/train/docs_used": 794381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7006663084030151, + "objective/train/original_loss": 3.387856960296631, + "objective/train/theoretical_loss": 3.537120159806962, + "objective/train/tokens_used": 1402470400, + "objective/train/unlikelihood_loss": 0.15973490476608276, + "theoretical_loss": 3.537120159806962, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904513540621866, + "loss": 1.2819, + "theoretical_loss": 3.537120159806962, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029043129388164494, + "loss": 1.4739, + "theoretical_loss": 3.5370906380130624, + "tokens_seen": 1402601472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029041123370110336, + "loss": 1.2801, + "theoretical_loss": 3.5370611197502004, + "tokens_seen": 1402732544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903911735205617, + "loss": 1.4258, + "theoretical_loss": 3.5370316050176243, + "tokens_seen": 1402863616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903711133400201, + "loss": 1.5217, + "theoretical_loss": 3.5370020938145816, + "tokens_seen": 1402994688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029035105315947845, + "loss": 1.2902, + "theoretical_loss": 3.5369725861403207, + "tokens_seen": 1403125760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029033099297893686, + "loss": 1.3199, + "theoretical_loss": 3.5369430819940906, + "tokens_seen": 1403256832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903109327983952, + "loss": 1.5388, + "theoretical_loss": 3.5369135813751393, + "tokens_seen": 1403387904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902908726178536, + "loss": 1.3812, + "theoretical_loss": 3.5368840842827165, + "tokens_seen": 1403518976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029027081243731195, + "loss": 1.3698, + "theoretical_loss": 3.5368545907160707, + "tokens_seen": 1403650048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902507522567703, + "loss": 1.3926, + "theoretical_loss": 3.536825100674451, + "tokens_seen": 1403781120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029023069207622873, + "loss": 1.3807, + "theoretical_loss": 3.5367956141571075, + "tokens_seen": 1403912192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902106318956871, + "loss": 1.4185, + "theoretical_loss": 3.53676613116329, + "tokens_seen": 1404043264 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.404753178358078, + "objective/train/avg_token_score": 0.006307427771389484, + "objective/train/docs_used": 795506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3470999002456665, + "objective/train/original_loss": 3.617884635925293, + "objective/train/theoretical_loss": 3.5367513909874697, + "objective/train/tokens_used": 1404108800, + "objective/train/unlikelihood_loss": 0.20591498911380768, + "theoretical_loss": 3.5367513909874697, + "tokens_seen": 1404108800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029019057171514545, + "loss": 1.2726, + "theoretical_loss": 3.536736651692249, + "tokens_seen": 1404174336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901705115346038, + "loss": 1.4796, + "theoretical_loss": 3.5367071757432345, + "tokens_seen": 1404305408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029015045135406223, + "loss": 1.4319, + "theoretical_loss": 3.5366777033154966, + "tokens_seen": 1404436480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901303911735206, + "loss": 1.2268, + "theoretical_loss": 3.536648234408286, + "tokens_seen": 1404567552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029011033099297896, + "loss": 1.3206, + "theoretical_loss": 3.536618769020855, + "tokens_seen": 1404698624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900902708124373, + "loss": 1.2889, + "theoretical_loss": 3.5365893071524535, + "tokens_seen": 1404829696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900702106318957, + "loss": 1.4922, + "theoretical_loss": 3.5365598488023333, + "tokens_seen": 1404960768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900501504513541, + "loss": 1.5541, + "theoretical_loss": 3.5365303939697466, + "tokens_seen": 1405091840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029003009027081246, + "loss": 1.2781, + "theoretical_loss": 3.5365009426539444, + "tokens_seen": 1405222912 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900100300902708, + "loss": 1.4375, + "theoretical_loss": 3.5364714948541796, + "tokens_seen": 1405353984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899899699097292, + "loss": 1.4714, + "theoretical_loss": 3.536442050569704, + "tokens_seen": 1405485056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899699097291876, + "loss": 1.4288, + "theoretical_loss": 3.5364126097997706, + "tokens_seen": 1405616128 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.4387829899787903, + "objective/train/avg_token_score": 0.03665170818567276, + "objective/train/docs_used": 796159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5223362445831299, + "objective/train/original_loss": 4.2308268547058105, + "objective/train/theoretical_loss": 3.5363831725436317, + "objective/train/tokens_used": 1405747200, + "objective/train/unlikelihood_loss": 0.13712263107299805, + "theoretical_loss": 3.5363831725436317, + "tokens_seen": 1405747200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028994984954864596, + "loss": 1.3738, + "theoretical_loss": 3.5363831725436317, + "tokens_seen": 1405747200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899297893681043, + "loss": 1.3725, + "theoretical_loss": 3.5363537388005417, + "tokens_seen": 1405878272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899097291875627, + "loss": 1.3818, + "theoretical_loss": 3.5363243085697524, + "tokens_seen": 1406009344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028988966900702105, + "loss": 1.2686, + "theoretical_loss": 3.5362948818505178, + "tokens_seen": 1406140416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028986960882647946, + "loss": 1.2035, + "theoretical_loss": 3.536265458642092, + "tokens_seen": 1406271488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898495486459378, + "loss": 1.2962, + "theoretical_loss": 3.536236038943729, + "tokens_seen": 1406402560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898294884653962, + "loss": 1.3941, + "theoretical_loss": 3.5362066227546816, + "tokens_seen": 1406533632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028980942828485455, + "loss": 1.5574, + "theoretical_loss": 3.5361772100742064, + "tokens_seen": 1406664704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028978936810431297, + "loss": 1.3981, + "theoretical_loss": 3.5361478009015563, + "tokens_seen": 1406795776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028976930792377133, + "loss": 1.4371, + "theoretical_loss": 3.536118395235987, + "tokens_seen": 1406926848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897492477432297, + "loss": 1.248, + "theoretical_loss": 3.5360889930767536, + "tokens_seen": 1407057920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028972918756268805, + "loss": 1.3426, + "theoretical_loss": 3.5360595944231115, + "tokens_seen": 1407188992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897091273821464, + "loss": 1.3325, + "theoretical_loss": 3.5360301992743155, + "tokens_seen": 1407320064 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.38104838132858276, + "objective/train/avg_token_score": 0.03701358288526535, + "objective/train/docs_used": 797613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1247280836105347, + "objective/train/original_loss": 4.167716979980469, + "objective/train/theoretical_loss": 3.536015503014003, + "objective/train/tokens_used": 1407385600, + "objective/train/unlikelihood_loss": 0.15080274641513824, + "theoretical_loss": 3.536015503014003, + "tokens_seen": 1407385600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028968906720160483, + "loss": 1.4909, + "theoretical_loss": 3.5360008076296223, + "tokens_seen": 1407451136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896690070210632, + "loss": 1.2793, + "theoretical_loss": 3.5359714194882876, + "tokens_seen": 1407582208 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028964894684052156, + "loss": 1.3688, + "theoretical_loss": 3.535942034849567, + "tokens_seen": 1407713280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896288866599799, + "loss": 1.3515, + "theoretical_loss": 3.5359126537127183, + "tokens_seen": 1407844352 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028960882647943834, + "loss": 1.3727, + "theoretical_loss": 3.535883276076997, + "tokens_seen": 1407975424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895887662988967, + "loss": 1.2595, + "theoretical_loss": 3.535853901941661, + "tokens_seen": 1408106496 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028956870611835506, + "loss": 1.4035, + "theoretical_loss": 3.535824531305966, + "tokens_seen": 1408237568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895486459378134, + "loss": 1.3513, + "theoretical_loss": 3.5357951641691714, + "tokens_seen": 1408368640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895285857572718, + "loss": 1.5232, + "theoretical_loss": 3.5357658005305335, + "tokens_seen": 1408499712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895085255767302, + "loss": 1.525, + "theoretical_loss": 3.53573644038931, + "tokens_seen": 1408630784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028948846539618856, + "loss": 1.4385, + "theoretical_loss": 3.53570708374476, + "tokens_seen": 1408761856 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894684052156469, + "loss": 1.5124, + "theoretical_loss": 3.53567773059614, + "tokens_seen": 1408892928 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.5293254852294922, + "objective/train/avg_token_score": 0.010164051316678524, + "objective/train/docs_used": 798401, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5757200717926025, + "objective/train/original_loss": 3.614125967025757, + "objective/train/theoretical_loss": 3.5356483809427104, + "objective/train/tokens_used": 1409024000, + "objective/train/unlikelihood_loss": 0.15533043444156647, + "theoretical_loss": 3.5356483809427104, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894483450351053, + "loss": 1.4286, + "theoretical_loss": 3.5356483809427104, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894282848545637, + "loss": 1.4715, + "theoretical_loss": 3.5356190347837293, + "tokens_seen": 1409155072 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028940822467402207, + "loss": 1.443, + "theoretical_loss": 3.535589692118455, + "tokens_seen": 1409286144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028938816449348043, + "loss": 1.3707, + "theoretical_loss": 3.5355603529461477, + "tokens_seen": 1409417216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893681043129388, + "loss": 1.1646, + "theoretical_loss": 3.535531017266066, + "tokens_seen": 1409548288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893480441323972, + "loss": 1.461, + "theoretical_loss": 3.5355016850774703, + "tokens_seen": 1409679360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028932798395185557, + "loss": 1.3263, + "theoretical_loss": 3.5354723563796195, + "tokens_seen": 1409810432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028930792377131393, + "loss": 1.3228, + "theoretical_loss": 3.5354430311717744, + "tokens_seen": 1409941504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892878635907723, + "loss": 1.5291, + "theoretical_loss": 3.535413709453196, + "tokens_seen": 1410072576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028926780341023066, + "loss": 1.4873, + "theoretical_loss": 3.5353843912231433, + "tokens_seen": 1410203648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892477432296891, + "loss": 1.3965, + "theoretical_loss": 3.5353550764808777, + "tokens_seen": 1410334720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028922768304914744, + "loss": 1.342, + "theoretical_loss": 3.53532576522566, + "tokens_seen": 1410465792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892076228686058, + "loss": 1.3629, + "theoretical_loss": 3.5352964574567523, + "tokens_seen": 1410596864 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.33620479702949524, + "objective/train/avg_token_score": 0.011640033684670925, + "objective/train/docs_used": 799700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.963915228843689, + "objective/train/original_loss": 3.9411022663116455, + "objective/train/theoretical_loss": 3.5352818048794337, + "objective/train/tokens_used": 1410662400, + "objective/train/unlikelihood_loss": 0.1458776593208313, + "theoretical_loss": 3.5352818048794337, + "tokens_seen": 1410662400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028918756268806416, + "loss": 1.1825, + "theoretical_loss": 3.5352671531734154, + "tokens_seen": 1410727936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891675025075226, + "loss": 1.2553, + "theoretical_loss": 3.5352378523749106, + "tokens_seen": 1410859008 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028914744232698094, + "loss": 1.375, + "theoretical_loss": 3.5352085550605006, + "tokens_seen": 1410990080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891273821464393, + "loss": 1.4067, + "theoretical_loss": 3.5351792612294473, + "tokens_seen": 1411121152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028910732196589766, + "loss": 1.4236, + "theoretical_loss": 3.5351499708810126, + "tokens_seen": 1411252224 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289087261785356, + "loss": 1.3517, + "theoretical_loss": 3.5351206840144593, + "tokens_seen": 1411383296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028906720160481444, + "loss": 1.3429, + "theoretical_loss": 3.5350914006290504, + "tokens_seen": 1411514368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890471414242728, + "loss": 1.2472, + "theoretical_loss": 3.5350621207240485, + "tokens_seen": 1411645440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890270812437312, + "loss": 1.3471, + "theoretical_loss": 3.5350328442987173, + "tokens_seen": 1411776512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890070210631896, + "loss": 1.3828, + "theoretical_loss": 3.53500357135232, + "tokens_seen": 1411907584 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288986960882648, + "loss": 1.4747, + "theoretical_loss": 3.5349743018841204, + "tokens_seen": 1412038656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028896690070210636, + "loss": 1.3627, + "theoretical_loss": 3.5349450358933825, + "tokens_seen": 1412169728 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.46114370226860046, + "objective/train/avg_token_score": 0.01645650342106819, + "objective/train/docs_used": 800754, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.503489375114441, + "objective/train/original_loss": 3.5707452297210693, + "objective/train/theoretical_loss": 3.53491577337937, + "objective/train/tokens_used": 1412300800, + "objective/train/unlikelihood_loss": 0.16603858768939972, + "theoretical_loss": 3.53491577337937, + "tokens_seen": 1412300800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889468405215647, + "loss": 1.6221, + "theoretical_loss": 3.53491577337937, + "tokens_seen": 1412300800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889267803410231, + "loss": 1.4434, + "theoretical_loss": 3.5348865143413484, + "tokens_seen": 1412431872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028890672016048145, + "loss": 1.433, + "theoretical_loss": 3.534857258778581, + "tokens_seen": 1412562944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028888665997993986, + "loss": 1.419, + "theoretical_loss": 3.534828006690333, + "tokens_seen": 1412694016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028886659979939823, + "loss": 1.5272, + "theoretical_loss": 3.53479875807587, + "tokens_seen": 1412825088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888465396188566, + "loss": 1.3565, + "theoretical_loss": 3.5347695129344556, + "tokens_seen": 1412956160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028882647943831495, + "loss": 1.4338, + "theoretical_loss": 3.534740271265358, + "tokens_seen": 1413087232 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028880641925777337, + "loss": 1.385, + "theoretical_loss": 3.5347110330678406, + "tokens_seen": 1413218304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028878635907723173, + "loss": 1.3759, + "theoretical_loss": 3.53468179834117, + "tokens_seen": 1413349376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887662988966901, + "loss": 1.356, + "theoretical_loss": 3.5346525670846125, + "tokens_seen": 1413480448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028874623871614845, + "loss": 1.482, + "theoretical_loss": 3.534623339297435, + "tokens_seen": 1413611520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887261785356068, + "loss": 1.5001, + "theoretical_loss": 3.534594114978903, + "tokens_seen": 1413742592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028870611835506523, + "loss": 1.4235, + "theoretical_loss": 3.5345648941282843, + "tokens_seen": 1413873664 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.5464931726455688, + "objective/train/avg_token_score": 0.007018740754574537, + "objective/train/docs_used": 801653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6574599742889404, + "objective/train/original_loss": 3.4222655296325684, + "objective/train/theoretical_loss": 3.5345502850032133, + "objective/train/tokens_used": 1413939200, + "objective/train/unlikelihood_loss": 0.16873477399349213, + "theoretical_loss": 3.5345502850032133, + "tokens_seen": 1413939200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886860581745236, + "loss": 1.3367, + "theoretical_loss": 3.534535676744845, + "tokens_seen": 1414004736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028866599799398196, + "loss": 1.3376, + "theoretical_loss": 3.5345064628278537, + "tokens_seen": 1414135808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886459378134403, + "loss": 1.317, + "theoretical_loss": 3.5344772523765764, + "tokens_seen": 1414266880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028862587763289874, + "loss": 1.387, + "theoretical_loss": 3.534448045390282, + "tokens_seen": 1414397952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886058174523571, + "loss": 1.3634, + "theoretical_loss": 3.5344188418682374, + "tokens_seen": 1414529024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028858575727181546, + "loss": 1.4455, + "theoretical_loss": 3.5343896418097116, + "tokens_seen": 1414660096 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885656970912738, + "loss": 1.5606, + "theoretical_loss": 3.5343604452139727, + "tokens_seen": 1414791168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885456369107322, + "loss": 1.2514, + "theoretical_loss": 3.534331252080289, + "tokens_seen": 1414922240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885255767301906, + "loss": 1.4024, + "theoretical_loss": 3.5343020624079298, + "tokens_seen": 1415053312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028850551654964896, + "loss": 1.4942, + "theoretical_loss": 3.534272876196164, + "tokens_seen": 1415184384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884854563691073, + "loss": 1.4274, + "theoretical_loss": 3.5342436934442607, + "tokens_seen": 1415315456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884653961885657, + "loss": 1.2925, + "theoretical_loss": 3.53421451415149, + "tokens_seen": 1415446528 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.4655425250530243, + "objective/train/avg_token_score": 0.009494329802691936, + "objective/train/docs_used": 802505, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5244779586791992, + "objective/train/original_loss": 3.4723522663116455, + "objective/train/theoretical_loss": 3.5341853383171205, + "objective/train/tokens_used": 1415577600, + "objective/train/unlikelihood_loss": 0.1807432323694229, + "theoretical_loss": 3.5341853383171205, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884453360080241, + "loss": 1.3923, + "theoretical_loss": 3.5341853383171205, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028842527582748247, + "loss": 1.3341, + "theoretical_loss": 3.534156165940423, + "tokens_seen": 1415708672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028840521564694083, + "loss": 1.3184, + "theoretical_loss": 3.5341269970206675, + "tokens_seen": 1415839744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883851554663992, + "loss": 1.3455, + "theoretical_loss": 3.5340978315571245, + "tokens_seen": 1415970816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883650952858576, + "loss": 1.4345, + "theoretical_loss": 3.5340686695490637, + "tokens_seen": 1416101888 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028834503510531597, + "loss": 1.4972, + "theoretical_loss": 3.534039510995757, + "tokens_seen": 1416232960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028832497492477433, + "loss": 1.421, + "theoretical_loss": 3.5340103558964753, + "tokens_seen": 1416364032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883049147442327, + "loss": 1.4014, + "theoretical_loss": 3.5339812042504897, + "tokens_seen": 1416495104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028828485456369106, + "loss": 1.2573, + "theoretical_loss": 3.533952056057071, + "tokens_seen": 1416626176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882647943831495, + "loss": 1.3558, + "theoretical_loss": 3.5339229113154924, + "tokens_seen": 1416757248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028824473420260784, + "loss": 1.3506, + "theoretical_loss": 3.5338937700250246, + "tokens_seen": 1416888320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882246740220662, + "loss": 1.306, + "theoretical_loss": 3.53386463218494, + "tokens_seen": 1417019392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028820461384152456, + "loss": 1.3628, + "theoretical_loss": 3.5338354977945112, + "tokens_seen": 1417150464 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.3574657738208771, + "objective/train/avg_token_score": 0.01830158568918705, + "objective/train/docs_used": 803171, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2221176624298096, + "objective/train/original_loss": 3.9736123085021973, + "objective/train/theoretical_loss": 3.5338209318926905, + "objective/train/tokens_used": 1417216000, + "objective/train/unlikelihood_loss": 0.19019676744937897, + "theoretical_loss": 3.5338209318926905, + "tokens_seen": 1417216000 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288184553660983, + "loss": 1.2599, + "theoretical_loss": 3.5338063668530104, + "tokens_seen": 1417281536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028816449348044134, + "loss": 1.3584, + "theoretical_loss": 3.533777239359711, + "tokens_seen": 1417412608 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881444332998997, + "loss": 1.3821, + "theoretical_loss": 3.5337481153138857, + "tokens_seen": 1417543680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028812437311935806, + "loss": 1.3206, + "theoretical_loss": 3.5337189947148078, + "tokens_seen": 1417674752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881043129388164, + "loss": 1.2268, + "theoretical_loss": 3.5336898775617502, + "tokens_seen": 1417805824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028808425275827484, + "loss": 1.3802, + "theoretical_loss": 3.5336607638539874, + "tokens_seen": 1417936896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880641925777332, + "loss": 1.3553, + "theoretical_loss": 3.5336316535907937, + "tokens_seen": 1418067968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028804413239719157, + "loss": 1.4086, + "theoretical_loss": 3.5336025467714416, + "tokens_seen": 1418199040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028802407221664993, + "loss": 1.2197, + "theoretical_loss": 3.5335734433952073, + "tokens_seen": 1418330112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028800401203610835, + "loss": 1.4319, + "theoretical_loss": 3.5335443434613643, + "tokens_seen": 1418461184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879839518555667, + "loss": 1.2802, + "theoretical_loss": 3.533515246969187, + "tokens_seen": 1418592256 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028796389167502507, + "loss": 1.3663, + "theoretical_loss": 3.533486153917951, + "tokens_seen": 1418723328 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.3496456444263458, + "objective/train/avg_token_score": 0.01899789273738861, + "objective/train/docs_used": 804514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.183814525604248, + "objective/train/original_loss": 3.9938182830810547, + "objective/train/theoretical_loss": 3.5334570643069325, + "objective/train/tokens_used": 1418854400, + "objective/train/unlikelihood_loss": 0.1637943983078003, + "theoretical_loss": 3.5334570643069325, + "tokens_seen": 1418854400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028794383149448343, + "loss": 1.3675, + "theoretical_loss": 3.5334570643069325, + "tokens_seen": 1418854400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879237713139418, + "loss": 1.3933, + "theoretical_loss": 3.533427978135405, + "tokens_seen": 1418985472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879037111334002, + "loss": 1.4477, + "theoretical_loss": 3.533398895402646, + "tokens_seen": 1419116544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878836509528586, + "loss": 1.3597, + "theoretical_loss": 3.53336981610793, + "tokens_seen": 1419247616 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028786359077231694, + "loss": 1.2283, + "theoretical_loss": 3.5333407402505337, + "tokens_seen": 1419378688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878435305917753, + "loss": 1.4313, + "theoretical_loss": 3.533311667829734, + "tokens_seen": 1419509760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878234704112337, + "loss": 1.3521, + "theoretical_loss": 3.5332825988448064, + "tokens_seen": 1419640832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878034102306921, + "loss": 1.4299, + "theoretical_loss": 3.533253533295028, + "tokens_seen": 1419771904 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028778335005015044, + "loss": 1.4074, + "theoretical_loss": 3.5332244711796763, + "tokens_seen": 1419902976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877632898696088, + "loss": 1.2956, + "theoretical_loss": 3.533195412498028, + "tokens_seen": 1420034048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028774322968906716, + "loss": 1.3765, + "theoretical_loss": 3.5331663572493603, + "tokens_seen": 1420165120 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877231695085256, + "loss": 1.3156, + "theoretical_loss": 3.533137305432951, + "tokens_seen": 1420296192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028770310932798394, + "loss": 1.3838, + "theoretical_loss": 3.5331082570480787, + "tokens_seen": 1420427264 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.5452712774276733, + "objective/train/avg_token_score": 0.00961341243237257, + "objective/train/docs_used": 804990, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.639990210533142, + "objective/train/original_loss": 3.5425267219543457, + "objective/train/theoretical_loss": 3.533093734142243, + "objective/train/tokens_used": 1420492800, + "objective/train/unlikelihood_loss": 0.13118715584278107, + "theoretical_loss": 3.533093734142243, + "tokens_seen": 1420492800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876830491474423, + "loss": 1.391, + "theoretical_loss": 3.5330792120940204, + "tokens_seen": 1420558336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028766298896690067, + "loss": 1.4506, + "theoretical_loss": 3.533050170570055, + "tokens_seen": 1420689408 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876429287863591, + "loss": 1.2073, + "theoretical_loss": 3.5330211324754615, + "tokens_seen": 1420820480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028762286860581745, + "loss": 1.1692, + "theoretical_loss": 3.532992097809518, + "tokens_seen": 1420951552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876028084252758, + "loss": 1.3408, + "theoretical_loss": 3.532963066571503, + "tokens_seen": 1421082624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028758274824473417, + "loss": 1.5254, + "theoretical_loss": 3.532934038760697, + "tokens_seen": 1421213696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028756268806419253, + "loss": 1.3661, + "theoretical_loss": 3.532905014376378, + "tokens_seen": 1421344768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028754262788365095, + "loss": 1.4603, + "theoretical_loss": 3.532875993417826, + "tokens_seen": 1421475840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028752256770310936, + "loss": 1.3468, + "theoretical_loss": 3.532846975884321, + "tokens_seen": 1421606912 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875025075225677, + "loss": 1.3549, + "theoretical_loss": 3.5328179617751436, + "tokens_seen": 1421737984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874824473420261, + "loss": 1.2323, + "theoretical_loss": 3.5327889510895734, + "tokens_seen": 1421869056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874623871614845, + "loss": 1.3656, + "theoretical_loss": 3.532759943826891, + "tokens_seen": 1422000128 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.39125123620033264, + "objective/train/avg_token_score": 0.01862506940960884, + "objective/train/docs_used": 806295, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2153468132019043, + "objective/train/original_loss": 4.026674747467041, + "objective/train/theoretical_loss": 3.5327309399863767, + "objective/train/tokens_used": 1422131200, + "objective/train/unlikelihood_loss": 0.1433567851781845, + "theoretical_loss": 3.5327309399863767, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028744232698094287, + "loss": 1.3025, + "theoretical_loss": 3.5327309399863767, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028742226680040123, + "loss": 1.3392, + "theoretical_loss": 3.5327019395673123, + "tokens_seen": 1422262272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874022066198596, + "loss": 1.5323, + "theoretical_loss": 3.5326729425689782, + "tokens_seen": 1422393344 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287382146439318, + "loss": 1.381, + "theoretical_loss": 3.532643948990656, + "tokens_seen": 1422524416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028736208625877637, + "loss": 1.2717, + "theoretical_loss": 3.532614958831627, + "tokens_seen": 1422655488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028734202607823473, + "loss": 1.4553, + "theoretical_loss": 3.532585972091174, + "tokens_seen": 1422786560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873219658976931, + "loss": 1.3463, + "theoretical_loss": 3.5325569887685777, + "tokens_seen": 1422917632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028730190571715146, + "loss": 1.3678, + "theoretical_loss": 3.532528008863121, + "tokens_seen": 1423048704 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872818455366099, + "loss": 1.3966, + "theoretical_loss": 3.532499032374086, + "tokens_seen": 1423179776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028726178535606824, + "loss": 1.2948, + "theoretical_loss": 3.5324700593007554, + "tokens_seen": 1423310848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872417251755266, + "loss": 1.3971, + "theoretical_loss": 3.532441089642413, + "tokens_seen": 1423441920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028722166499498496, + "loss": 1.3622, + "theoretical_loss": 3.5324121233983403, + "tokens_seen": 1423572992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872016048144434, + "loss": 1.4284, + "theoretical_loss": 3.5323831605678215, + "tokens_seen": 1423704064 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.38734114170074463, + "objective/train/avg_token_score": 0.012464947998523712, + "objective/train/docs_used": 806909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1381407976150513, + "objective/train/original_loss": 3.6984543800354004, + "objective/train/theoretical_loss": 3.5323686804324215, + "objective/train/tokens_used": 1423769600, + "objective/train/unlikelihood_loss": 0.15951105952262878, + "theoretical_loss": 3.5323686804324215, + "tokens_seen": 1423769600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028718154463390174, + "loss": 1.4305, + "theoretical_loss": 3.5323542011501408, + "tokens_seen": 1423835136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871614844533601, + "loss": 1.2755, + "theoretical_loss": 3.5323252451445804, + "tokens_seen": 1423966208 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028714142427281846, + "loss": 1.3248, + "theoretical_loss": 3.5322962925504253, + "tokens_seen": 1424097280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871213640922768, + "loss": 1.2272, + "theoretical_loss": 3.532267343366959, + "tokens_seen": 1424228352 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028710130391173524, + "loss": 1.5902, + "theoretical_loss": 3.532238397593466, + "tokens_seen": 1424359424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870812437311936, + "loss": 1.4412, + "theoretical_loss": 3.532209455229232, + "tokens_seen": 1424490496 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028706118355065197, + "loss": 1.3174, + "theoretical_loss": 3.5321805162735407, + "tokens_seen": 1424621568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028704112337011033, + "loss": 1.3833, + "theoretical_loss": 3.532151580725677, + "tokens_seen": 1424752640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028702106318956875, + "loss": 1.323, + "theoretical_loss": 3.532122648584927, + "tokens_seen": 1424883712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870010030090271, + "loss": 1.356, + "theoretical_loss": 3.5320937198505753, + "tokens_seen": 1425014784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028698094282848547, + "loss": 1.4187, + "theoretical_loss": 3.532064794521908, + "tokens_seen": 1425145856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028696088264794383, + "loss": 1.259, + "theoretical_loss": 3.532035872598211, + "tokens_seen": 1425276928 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.41782745718955994, + "objective/train/avg_token_score": 0.005708466283977032, + "objective/train/docs_used": 808391, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3188856840133667, + "objective/train/original_loss": 3.695831298828125, + "objective/train/theoretical_loss": 3.5320069540787706, + "objective/train/tokens_used": 1425408000, + "objective/train/unlikelihood_loss": 0.18815873563289642, + "theoretical_loss": 3.5320069540787706, + "tokens_seen": 1425408000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869408224674022, + "loss": 1.3151, + "theoretical_loss": 3.5320069540787706, + "tokens_seen": 1425408000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869207622868606, + "loss": 1.2272, + "theoretical_loss": 3.5319780389628725, + "tokens_seen": 1425539072 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286900702106319, + "loss": 1.366, + "theoretical_loss": 3.531949127249804, + "tokens_seen": 1425670144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028688064192577734, + "loss": 1.2669, + "theoretical_loss": 3.5319202189388514, + "tokens_seen": 1425801216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868605817452357, + "loss": 1.5287, + "theoretical_loss": 3.531891314029301, + "tokens_seen": 1425932288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868405215646941, + "loss": 1.2885, + "theoretical_loss": 3.531862412520442, + "tokens_seen": 1426063360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868204613841525, + "loss": 1.48, + "theoretical_loss": 3.53183351441156, + "tokens_seen": 1426194432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028680040120361084, + "loss": 1.378, + "theoretical_loss": 3.5318046197019424, + "tokens_seen": 1426325504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867803410230692, + "loss": 1.3138, + "theoretical_loss": 3.531775728390879, + "tokens_seen": 1426456576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028676028084252756, + "loss": 1.4188, + "theoretical_loss": 3.5317468404776555, + "tokens_seen": 1426587648 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286740220661986, + "loss": 1.2363, + "theoretical_loss": 3.531717955961562, + "tokens_seen": 1426718720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028672016048144434, + "loss": 1.3378, + "theoretical_loss": 3.5316890748418857, + "tokens_seen": 1426849792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867001003009027, + "loss": 1.4354, + "theoretical_loss": 3.5316601971179162, + "tokens_seen": 1426980864 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.523949146270752, + "objective/train/avg_token_score": 0.007051435299217701, + "objective/train/docs_used": 809012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6669493913650513, + "objective/train/original_loss": 3.690777540206909, + "objective/train/theoretical_loss": 3.531645759529099, + "objective/train/tokens_used": 1427046400, + "objective/train/unlikelihood_loss": 0.13461323082447052, + "theoretical_loss": 3.531645759529099, + "tokens_seen": 1427046400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028668004012036107, + "loss": 1.3541, + "theoretical_loss": 3.5316313227889413, + "tokens_seen": 1427111936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866599799398195, + "loss": 1.2193, + "theoretical_loss": 3.5316024518542517, + "tokens_seen": 1427243008 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028663991975927785, + "loss": 1.2838, + "theoretical_loss": 3.5315735843131355, + "tokens_seen": 1427374080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866198595787362, + "loss": 1.3932, + "theoretical_loss": 3.5315447201648826, + "tokens_seen": 1427505152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028659979939819457, + "loss": 1.3549, + "theoretical_loss": 3.531515859408783, + "tokens_seen": 1427636224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028657973921765293, + "loss": 1.2938, + "theoretical_loss": 3.531487002044126, + "tokens_seen": 1427767296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028655967903711135, + "loss": 1.4496, + "theoretical_loss": 3.531458148070202, + "tokens_seen": 1427898368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865396188565697, + "loss": 1.3105, + "theoretical_loss": 3.5314292974863024, + "tokens_seen": 1428029440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865195586760281, + "loss": 1.3628, + "theoretical_loss": 3.5314004502917165, + "tokens_seen": 1428160512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028649949849548644, + "loss": 1.4531, + "theoretical_loss": 3.5313716064857354, + "tokens_seen": 1428291584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028647943831494485, + "loss": 1.2927, + "theoretical_loss": 3.5313427660676506, + "tokens_seen": 1428422656 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864593781344032, + "loss": 1.352, + "theoretical_loss": 3.5313139290367532, + "tokens_seen": 1428553728 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.48552054166793823, + "objective/train/avg_token_score": 0.012241683900356293, + "objective/train/docs_used": 809683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5028153657913208, + "objective/train/original_loss": 3.633737087249756, + "objective/train/theoretical_loss": 3.531285095392335, + "objective/train/tokens_used": 1428684800, + "objective/train/unlikelihood_loss": 0.15567556023597717, + "theoretical_loss": 3.531285095392335, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864393179538616, + "loss": 1.3704, + "theoretical_loss": 3.531285095392335, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028641925777331994, + "loss": 1.1071, + "theoretical_loss": 3.5312562651336865, + "tokens_seen": 1428815872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863991975927783, + "loss": 1.205, + "theoretical_loss": 3.5312274382601005, + "tokens_seen": 1428946944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863791374122367, + "loss": 1.3612, + "theoretical_loss": 3.5311986147708696, + "tokens_seen": 1429078016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863590772316951, + "loss": 1.42, + "theoretical_loss": 3.531169794665285, + "tokens_seen": 1429209088 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028633901705115344, + "loss": 1.5127, + "theoretical_loss": 3.5311409779426395, + "tokens_seen": 1429340160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863189568706118, + "loss": 1.3542, + "theoretical_loss": 3.5311121646022263, + "tokens_seen": 1429471232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862988966900702, + "loss": 1.3189, + "theoretical_loss": 3.531083354643338, + "tokens_seen": 1429602304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862788365095286, + "loss": 1.4901, + "theoretical_loss": 3.531054548065268, + "tokens_seen": 1429733376 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028625877632898694, + "loss": 1.3168, + "theoretical_loss": 3.5310257448673092, + "tokens_seen": 1429864448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862387161484453, + "loss": 1.4313, + "theoretical_loss": 3.5309969450487557, + "tokens_seen": 1429995520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028621865596790367, + "loss": 1.3244, + "theoretical_loss": 3.5309681486089013, + "tokens_seen": 1430126592 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861985957873621, + "loss": 1.4586, + "theoretical_loss": 3.53093935554704, + "tokens_seen": 1430257664 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.4526515007019043, + "objective/train/avg_token_score": 0.007326464634388685, + "objective/train/docs_used": 810937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.39878249168396, + "objective/train/original_loss": 3.59621524810791, + "objective/train/theoretical_loss": 3.530924960282636, + "objective/train/tokens_used": 1430323200, + "objective/train/unlikelihood_loss": 0.1673355996608734, + "theoretical_loss": 3.530924960282636, + "tokens_seen": 1430323200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028617853560682045, + "loss": 1.5085, + "theoretical_loss": 3.5309105658624658, + "tokens_seen": 1430388736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861584754262788, + "loss": 1.4241, + "theoretical_loss": 3.530881779554473, + "tokens_seen": 1430519808 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028613841524573717, + "loss": 1.3777, + "theoretical_loss": 3.5308529966223565, + "tokens_seen": 1430650880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861183550651956, + "loss": 1.2788, + "theoretical_loss": 3.530824217065411, + "tokens_seen": 1430781952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028609829488465395, + "loss": 1.2676, + "theoretical_loss": 3.530795440882932, + "tokens_seen": 1430913024 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860782347041123, + "loss": 1.4182, + "theoretical_loss": 3.5307666680742145, + "tokens_seen": 1431044096 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860581745235707, + "loss": 1.4331, + "theoretical_loss": 3.530737898638554, + "tokens_seen": 1431175168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860381143430291, + "loss": 1.5633, + "theoretical_loss": 3.5307091325752458, + "tokens_seen": 1431306240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860180541624875, + "loss": 1.4041, + "theoretical_loss": 3.530680369883587, + "tokens_seen": 1431437312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028599799398194587, + "loss": 1.3833, + "theoretical_loss": 3.530651610562872, + "tokens_seen": 1431568384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028597793380140423, + "loss": 1.2622, + "theoretical_loss": 3.530622854612399, + "tokens_seen": 1431699456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859578736208626, + "loss": 1.3359, + "theoretical_loss": 3.5305941020314635, + "tokens_seen": 1431830528 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.4924853444099426, + "objective/train/avg_token_score": 0.008443595841526985, + "objective/train/docs_used": 811574, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.816790223121643, + "objective/train/original_loss": 3.8578665256500244, + "objective/train/theoretical_loss": 3.5305653528193623, + "objective/train/tokens_used": 1431961600, + "objective/train/unlikelihood_loss": 0.12368126213550568, + "theoretical_loss": 3.5305653528193623, + "tokens_seen": 1431961600 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285937813440321, + "loss": 1.4331, + "theoretical_loss": 3.5305653528193623, + "tokens_seen": 1431961600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859177532597794, + "loss": 1.4045, + "theoretical_loss": 3.5305366069753927, + "tokens_seen": 1432092672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028589769307923774, + "loss": 1.3101, + "theoretical_loss": 3.5305078644988512, + "tokens_seen": 1432223744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858776328986961, + "loss": 1.5108, + "theoretical_loss": 3.5304791253890366, + "tokens_seen": 1432354816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858575727181545, + "loss": 1.4803, + "theoretical_loss": 3.530450389645245, + "tokens_seen": 1432485888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858375125376129, + "loss": 1.5983, + "theoretical_loss": 3.530421657266775, + "tokens_seen": 1432616960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028581745235707124, + "loss": 1.4385, + "theoretical_loss": 3.530392928252925, + "tokens_seen": 1432748032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857973921765296, + "loss": 1.3676, + "theoretical_loss": 3.5303642026029918, + "tokens_seen": 1432879104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028577733199598796, + "loss": 1.329, + "theoretical_loss": 3.5303354803162756, + "tokens_seen": 1433010176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857572718154464, + "loss": 1.3336, + "theoretical_loss": 3.530306761392074, + "tokens_seen": 1433141248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028573721163490474, + "loss": 1.1841, + "theoretical_loss": 3.530278045829686, + "tokens_seen": 1433272320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857171514543631, + "loss": 1.3891, + "theoretical_loss": 3.5302493336284106, + "tokens_seen": 1433403392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028569709127382147, + "loss": 1.4596, + "theoretical_loss": 3.530220624787548, + "tokens_seen": 1433534464 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.4647482931613922, + "objective/train/avg_token_score": 0.008312548510730267, + "objective/train/docs_used": 812859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.386775016784668, + "objective/train/original_loss": 3.3709511756896973, + "objective/train/theoretical_loss": 3.530206271627052, + "objective/train/tokens_used": 1433600000, + "objective/train/unlikelihood_loss": 0.23003259301185608, + "theoretical_loss": 3.530206271627052, + "tokens_seen": 1433600000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856770310932799, + "loss": 1.2293, + "theoretical_loss": 3.5301919193063966, + "tokens_seen": 1433665536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028565697091273825, + "loss": 1.2359, + "theoretical_loss": 3.5301632171842567, + "tokens_seen": 1433796608 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856369107321966, + "loss": 1.3677, + "theoretical_loss": 3.5301345184204282, + "tokens_seen": 1433927680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028561685055165497, + "loss": 1.354, + "theoretical_loss": 3.530105823014211, + "tokens_seen": 1434058752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028559679037111333, + "loss": 1.2601, + "theoretical_loss": 3.5300771309649055, + "tokens_seen": 1434189824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028557673019057175, + "loss": 1.3196, + "theoretical_loss": 3.530048442271812, + "tokens_seen": 1434320896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855566700100301, + "loss": 1.3704, + "theoretical_loss": 3.530019756934232, + "tokens_seen": 1434451968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855366098294885, + "loss": 1.2611, + "theoretical_loss": 3.5299910749514662, + "tokens_seen": 1434583040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028551654964894684, + "loss": 1.5659, + "theoretical_loss": 3.529962396322815, + "tokens_seen": 1434714112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028549648946840525, + "loss": 1.3719, + "theoretical_loss": 3.529933721047581, + "tokens_seen": 1434845184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854764292878636, + "loss": 1.4243, + "theoretical_loss": 3.529905049125065, + "tokens_seen": 1434976256 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285456369107322, + "loss": 1.3736, + "theoretical_loss": 3.529876380554569, + "tokens_seen": 1435107328 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.6410068273544312, + "objective/train/avg_token_score": 0.016829440370202065, + "objective/train/docs_used": 813625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8263188600540161, + "objective/train/original_loss": 3.3213415145874023, + "objective/train/theoretical_loss": 3.5298477153353955, + "objective/train/tokens_used": 1435238400, + "objective/train/unlikelihood_loss": 0.1127818152308464, + "theoretical_loss": 3.5298477153353955, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028543630892678034, + "loss": 1.503, + "theoretical_loss": 3.5298477153353955, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854162487462387, + "loss": 1.3259, + "theoretical_loss": 3.529819053466846, + "tokens_seen": 1435369472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853961885656971, + "loss": 1.3147, + "theoretical_loss": 3.529790394948223, + "tokens_seen": 1435500544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853761283851555, + "loss": 1.2745, + "theoretical_loss": 3.52976173977883, + "tokens_seen": 1435631616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028535606820461384, + "loss": 1.3501, + "theoretical_loss": 3.529733087957969, + "tokens_seen": 1435762688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853360080240722, + "loss": 1.4836, + "theoretical_loss": 3.5297044394849433, + "tokens_seen": 1435893760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853159478435306, + "loss": 1.5631, + "theoretical_loss": 3.529675794359056, + "tokens_seen": 1436024832 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285295887662989, + "loss": 1.2653, + "theoretical_loss": 3.529647152579611, + "tokens_seen": 1436155904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028527582748244735, + "loss": 1.3753, + "theoretical_loss": 3.5296185141459118, + "tokens_seen": 1436286976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852557673019057, + "loss": 1.3615, + "theoretical_loss": 3.529589879057262, + "tokens_seen": 1436418048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028523570712136407, + "loss": 1.4204, + "theoretical_loss": 3.5295612473129663, + "tokens_seen": 1436549120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852156469408225, + "loss": 1.4405, + "theoretical_loss": 3.529532618912328, + "tokens_seen": 1436680192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028519558676028085, + "loss": 1.2911, + "theoretical_loss": 3.5295039938546537, + "tokens_seen": 1436811264 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.5183895230293274, + "objective/train/avg_token_score": 0.010612035170197487, + "objective/train/docs_used": 814744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.723449468612671, + "objective/train/original_loss": 3.848266839981079, + "objective/train/theoretical_loss": 3.52948968257921, + "objective/train/tokens_used": 1436876800, + "objective/train/unlikelihood_loss": 0.11689840257167816, + "theoretical_loss": 3.52948968257921, + "tokens_seen": 1436876800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851755265797392, + "loss": 1.3288, + "theoretical_loss": 3.5294753721392453, + "tokens_seen": 1436942336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028515546639919757, + "loss": 1.3272, + "theoretical_loss": 3.52944675376541, + "tokens_seen": 1437073408 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285135406218656, + "loss": 1.2409, + "theoretical_loss": 3.529418138732452, + "tokens_seen": 1437204480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028511534603811435, + "loss": 1.4599, + "theoretical_loss": 3.529389527039677, + "tokens_seen": 1437335552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850952858575727, + "loss": 1.2565, + "theoretical_loss": 3.5293609186863906, + "tokens_seen": 1437466624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850752256770311, + "loss": 1.3581, + "theoretical_loss": 3.5293323136718975, + "tokens_seen": 1437597696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850551654964895, + "loss": 1.4715, + "theoretical_loss": 3.529303711995505, + "tokens_seen": 1437728768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028503510531594785, + "loss": 1.315, + "theoretical_loss": 3.529275113656519, + "tokens_seen": 1437859840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850150451354062, + "loss": 1.3808, + "theoretical_loss": 3.5292465186542454, + "tokens_seen": 1437990912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849949849548646, + "loss": 1.3502, + "theoretical_loss": 3.5292179269879913, + "tokens_seen": 1438121984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028497492477432294, + "loss": 1.1859, + "theoretical_loss": 3.529189338657063, + "tokens_seen": 1438253056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028495486459378136, + "loss": 1.3832, + "theoretical_loss": 3.5291607536607685, + "tokens_seen": 1438384128 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.43933284282684326, + "objective/train/avg_token_score": 0.011951694265007973, + "objective/train/docs_used": 815309, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5427814722061157, + "objective/train/original_loss": 4.193335056304932, + "objective/train/theoretical_loss": 3.5291321719984134, + "objective/train/tokens_used": 1438515200, + "objective/train/unlikelihood_loss": 0.11546733230352402, + "theoretical_loss": 3.5291321719984134, + "tokens_seen": 1438515200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849348044132397, + "loss": 1.1922, + "theoretical_loss": 3.5291321719984134, + "tokens_seen": 1438515200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849147442326981, + "loss": 1.2142, + "theoretical_loss": 3.5291035936693067, + "tokens_seen": 1438646272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028489468405215644, + "loss": 1.3574, + "theoretical_loss": 3.5290750186727555, + "tokens_seen": 1438777344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028487462387161486, + "loss": 1.2776, + "theoretical_loss": 3.5290464470080676, + "tokens_seen": 1438908416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848545636910732, + "loss": 1.3935, + "theoretical_loss": 3.529017878674551, + "tokens_seen": 1439039488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848345035105316, + "loss": 1.4603, + "theoretical_loss": 3.528989313671514, + "tokens_seen": 1439170560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028481444332998995, + "loss": 1.3198, + "theoretical_loss": 3.5289607519982646, + "tokens_seen": 1439301632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847943831494483, + "loss": 1.3968, + "theoretical_loss": 3.5289321936541125, + "tokens_seen": 1439432704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847743229689067, + "loss": 1.3597, + "theoretical_loss": 3.528903638638366, + "tokens_seen": 1439563776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847542627883651, + "loss": 1.3449, + "theoretical_loss": 3.5288750869503334, + "tokens_seen": 1439694848 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028473420260782345, + "loss": 1.277, + "theoretical_loss": 3.5288465385893257, + "tokens_seen": 1439825920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847141424272818, + "loss": 1.3171, + "theoretical_loss": 3.528817993554651, + "tokens_seen": 1439956992 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028469408224674023, + "loss": 1.1172, + "theoretical_loss": 3.5287894518456193, + "tokens_seen": 1440088064 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.44336509704589844, + "objective/train/avg_token_score": 0.01147431880235672, + "objective/train/docs_used": 816515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5335489511489868, + "objective/train/original_loss": 4.310601711273193, + "objective/train/theoretical_loss": 3.5287751822380047, + "objective/train/tokens_used": 1440153600, + "objective/train/unlikelihood_loss": 0.09573944658041, + "theoretical_loss": 3.5287751822380047, + "tokens_seen": 1440153600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846740220661986, + "loss": 1.5061, + "theoretical_loss": 3.528760913461541, + "tokens_seen": 1440219136 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028465396188565695, + "loss": 1.4237, + "theoretical_loss": 3.5287323784017266, + "tokens_seen": 1440350208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846339017051153, + "loss": 1.1899, + "theoretical_loss": 3.5287038466654845, + "tokens_seen": 1440481280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846138415245737, + "loss": 1.3096, + "theoretical_loss": 3.5286753182521275, + "tokens_seen": 1440612352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845937813440321, + "loss": 1.3652, + "theoretical_loss": 3.5286467931609646, + "tokens_seen": 1440743424 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028457372116349046, + "loss": 1.3769, + "theoretical_loss": 3.5286182713913075, + "tokens_seen": 1440874496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845536609829488, + "loss": 1.2921, + "theoretical_loss": 3.5285897529424677, + "tokens_seen": 1441005568 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028453360080240724, + "loss": 1.2958, + "theoretical_loss": 3.5285612378137556, + "tokens_seen": 1441136640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028451354062186565, + "loss": 1.3721, + "theoretical_loss": 3.5285327260044834, + "tokens_seen": 1441267712 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284493480441324, + "loss": 1.5621, + "theoretical_loss": 3.528504217513963, + "tokens_seen": 1441398784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844734202607824, + "loss": 1.3682, + "theoretical_loss": 3.528475712341506, + "tokens_seen": 1441529856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028445336008024074, + "loss": 1.2859, + "theoretical_loss": 3.5284472104864246, + "tokens_seen": 1441660928 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.34408602118492126, + "objective/train/avg_token_score": 0.02406410314142704, + "objective/train/docs_used": 817059, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1324615478515625, + "objective/train/original_loss": 3.4882781505584717, + "objective/train/theoretical_loss": 3.5284187119480315, + "objective/train/tokens_used": 1441792000, + "objective/train/unlikelihood_loss": 0.23261778056621552, + "theoretical_loss": 3.5284187119480315, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844332998996991, + "loss": 1.2701, + "theoretical_loss": 3.5284187119480315, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844132397191575, + "loss": 1.4702, + "theoretical_loss": 3.5283902167256382, + "tokens_seen": 1441923072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843931795386159, + "loss": 1.3158, + "theoretical_loss": 3.5283617248185593, + "tokens_seen": 1442054144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028437311935807424, + "loss": 1.473, + "theoretical_loss": 3.528333236226106, + "tokens_seen": 1442185216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843530591775326, + "loss": 1.3478, + "theoretical_loss": 3.5283047509475933, + "tokens_seen": 1442316288 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284332998996991, + "loss": 1.4351, + "theoretical_loss": 3.528276268982333, + "tokens_seen": 1442447360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843129388164494, + "loss": 1.3962, + "theoretical_loss": 3.5282477903296394, + "tokens_seen": 1442578432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028429287863590775, + "loss": 1.3675, + "theoretical_loss": 3.528219314988826, + "tokens_seen": 1442709504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842728184553661, + "loss": 1.3489, + "theoretical_loss": 3.528190842959207, + "tokens_seen": 1442840576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028425275827482447, + "loss": 1.395, + "theoretical_loss": 3.5281623742400967, + "tokens_seen": 1442971648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842326980942829, + "loss": 1.2651, + "theoretical_loss": 3.52813390883081, + "tokens_seen": 1443102720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028421263791374125, + "loss": 1.3836, + "theoretical_loss": 3.5281054467306605, + "tokens_seen": 1443233792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841925777331996, + "loss": 1.3167, + "theoretical_loss": 3.528076987938964, + "tokens_seen": 1443364864 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.4516128897666931, + "objective/train/avg_token_score": 0.02528996579349041, + "objective/train/docs_used": 818508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3822442293167114, + "objective/train/original_loss": 3.4121012687683105, + "objective/train/theoretical_loss": 3.5280627597835705, + "objective/train/tokens_used": 1443430400, + "objective/train/unlikelihood_loss": 0.18556709587574005, + "theoretical_loss": 3.5280627597835705, + "tokens_seen": 1443430400 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284172517552658, + "loss": 1.2981, + "theoretical_loss": 3.528048532455034, + "tokens_seen": 1443495936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841524573721164, + "loss": 1.2769, + "theoretical_loss": 3.528020080278188, + "tokens_seen": 1443627008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028413239719157475, + "loss": 1.2478, + "theoretical_loss": 3.5279916314077395, + "tokens_seen": 1443758080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841123370110331, + "loss": 1.5183, + "theoretical_loss": 3.527963185843005, + "tokens_seen": 1443889152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840922768304915, + "loss": 1.4273, + "theoretical_loss": 3.527934743583301, + "tokens_seen": 1444020224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840722166499499, + "loss": 1.3527, + "theoretical_loss": 3.5279063046279417, + "tokens_seen": 1444151296 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028405215646940825, + "loss": 1.4804, + "theoretical_loss": 3.527877868976245, + "tokens_seen": 1444282368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840320962888666, + "loss": 1.3327, + "theoretical_loss": 3.527849436627527, + "tokens_seen": 1444413440 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284012036108325, + "loss": 1.4314, + "theoretical_loss": 3.527821007581104, + "tokens_seen": 1444544512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028399197592778334, + "loss": 1.2909, + "theoretical_loss": 3.5277925818362927, + "tokens_seen": 1444675584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028397191574724176, + "loss": 1.411, + "theoretical_loss": 3.527764159392411, + "tokens_seen": 1444806656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839518555667001, + "loss": 1.3433, + "theoretical_loss": 3.5277357402487755, + "tokens_seen": 1444937728 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.6353861093521118, + "objective/train/avg_token_score": 0.010442422702908516, + "objective/train/docs_used": 819330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.161268472671509, + "objective/train/original_loss": 3.6179044246673584, + "objective/train/theoretical_loss": 3.527707324404704, + "objective/train/tokens_used": 1445068800, + "objective/train/unlikelihood_loss": 0.1325693130493164, + "theoretical_loss": 3.527707324404704, + "tokens_seen": 1445068800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839317953861585, + "loss": 1.6584, + "theoretical_loss": 3.527707324404704, + "tokens_seen": 1445068800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028391173520561684, + "loss": 1.3266, + "theoretical_loss": 3.527678911859514, + "tokens_seen": 1445199872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028389167502507526, + "loss": 1.5056, + "theoretical_loss": 3.5276505026125236, + "tokens_seen": 1445330944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838716148445336, + "loss": 1.3674, + "theoretical_loss": 3.5276220966630505, + "tokens_seen": 1445462016 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283851554663992, + "loss": 1.3789, + "theoretical_loss": 3.527593694010413, + "tokens_seen": 1445593088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028383149448345035, + "loss": 1.255, + "theoretical_loss": 3.52756529465393, + "tokens_seen": 1445724160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838114343029087, + "loss": 1.2164, + "theoretical_loss": 3.5275368985929205, + "tokens_seen": 1445855232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837913741223671, + "loss": 1.3851, + "theoretical_loss": 3.5275085058267024, + "tokens_seen": 1445986304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837713139418255, + "loss": 1.5662, + "theoretical_loss": 3.527480116354595, + "tokens_seen": 1446117376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028375125376128385, + "loss": 1.4458, + "theoretical_loss": 3.5274517301759185, + "tokens_seen": 1446248448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837311935807422, + "loss": 1.5504, + "theoretical_loss": 3.5274233472899916, + "tokens_seen": 1446379520 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028371113340020063, + "loss": 1.4437, + "theoretical_loss": 3.527394967696134, + "tokens_seen": 1446510592 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283691073219659, + "loss": 1.2386, + "theoretical_loss": 3.5273665913936663, + "tokens_seen": 1446641664 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.44678640365600586, + "objective/train/avg_token_score": 0.025752224028110504, + "objective/train/docs_used": 820636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.422412395477295, + "objective/train/original_loss": 4.176690578460693, + "objective/train/theoretical_loss": 3.527352404476491, + "objective/train/tokens_used": 1446707200, + "objective/train/unlikelihood_loss": 0.14174479246139526, + "theoretical_loss": 3.527352404476491, + "tokens_seen": 1446707200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028367101303911735, + "loss": 1.3917, + "theoretical_loss": 3.527338218381908, + "tokens_seen": 1446772736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836509528585757, + "loss": 1.4303, + "theoretical_loss": 3.5273098486601793, + "tokens_seen": 1446903808 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836308926780341, + "loss": 1.4013, + "theoretical_loss": 3.527281482227801, + "tokens_seen": 1447034880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836108324974925, + "loss": 1.4349, + "theoretical_loss": 3.5272531190840937, + "tokens_seen": 1447165952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028359077231695086, + "loss": 1.4089, + "theoretical_loss": 3.5272247592283787, + "tokens_seen": 1447297024 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835707121364092, + "loss": 1.4494, + "theoretical_loss": 3.527196402659977, + "tokens_seen": 1447428096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835506519558676, + "loss": 1.3711, + "theoretical_loss": 3.527168049378209, + "tokens_seen": 1447559168 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283530591775326, + "loss": 1.3937, + "theoretical_loss": 3.5271396993823974, + "tokens_seen": 1447690240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028351053159478436, + "loss": 1.6632, + "theoretical_loss": 3.527111352671864, + "tokens_seen": 1447821312 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834904714142427, + "loss": 1.3077, + "theoretical_loss": 3.5270830092459295, + "tokens_seen": 1447952384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834704112337011, + "loss": 1.4026, + "theoretical_loss": 3.527054669103917, + "tokens_seen": 1448083456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028345035105315945, + "loss": 1.3907, + "theoretical_loss": 3.527026332245148, + "tokens_seen": 1448214528 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.359665185213089, + "objective/train/avg_token_score": 0.019421303644776344, + "objective/train/docs_used": 821297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2012317180633545, + "objective/train/original_loss": 3.7231249809265137, + "objective/train/theoretical_loss": 3.5269979986689464, + "objective/train/tokens_used": 1448345600, + "objective/train/unlikelihood_loss": 0.13033714890480042, + "theoretical_loss": 3.5269979986689464, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028343029087261786, + "loss": 1.5801, + "theoretical_loss": 3.5269979986689464, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834102306920762, + "loss": 1.5478, + "theoretical_loss": 3.526969668374634, + "tokens_seen": 1448476672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833901705115346, + "loss": 1.3339, + "theoretical_loss": 3.5269413413615336, + "tokens_seen": 1448607744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028337011033099295, + "loss": 1.6195, + "theoretical_loss": 3.526913017628969, + "tokens_seen": 1448738816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028335005015045137, + "loss": 1.4683, + "theoretical_loss": 3.5268846971762624, + "tokens_seen": 1448869888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028332998996990973, + "loss": 1.3656, + "theoretical_loss": 3.526856380002739, + "tokens_seen": 1449000960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833099297893681, + "loss": 1.4545, + "theoretical_loss": 3.526828066107721, + "tokens_seen": 1449132032 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028328986960882645, + "loss": 1.3825, + "theoretical_loss": 3.526799755490533, + "tokens_seen": 1449263104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832698094282848, + "loss": 1.6363, + "theoretical_loss": 3.526771448150499, + "tokens_seen": 1449394176 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028324974924774323, + "loss": 1.5353, + "theoretical_loss": 3.526743144086944, + "tokens_seen": 1449525248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832296890672016, + "loss": 1.3692, + "theoretical_loss": 3.5267148432991915, + "tokens_seen": 1449656320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028320962888665996, + "loss": 1.3266, + "theoretical_loss": 3.526686545786567, + "tokens_seen": 1449787392 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831895687061183, + "loss": 1.5101, + "theoretical_loss": 3.5266582515483944, + "tokens_seen": 1449918464 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.43682795763015747, + "objective/train/avg_token_score": 0.028751306235790253, + "objective/train/docs_used": 822517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4858486652374268, + "objective/train/original_loss": 4.057677745819092, + "objective/train/theoretical_loss": 3.5266441056570175, + "objective/train/tokens_used": 1449984000, + "objective/train/unlikelihood_loss": 0.1128901019692421, + "theoretical_loss": 3.5266441056570175, + "tokens_seen": 1449984000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028316950852557674, + "loss": 1.342, + "theoretical_loss": 3.526629960584, + "tokens_seen": 1450049536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831494483450351, + "loss": 1.4038, + "theoretical_loss": 3.526601672892709, + "tokens_seen": 1450180608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028312938816449346, + "loss": 1.3536, + "theoretical_loss": 3.5265733884738464, + "tokens_seen": 1450311680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831093279839518, + "loss": 1.4916, + "theoretical_loss": 3.526545107326738, + "tokens_seen": 1450442752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830892678034102, + "loss": 1.4246, + "theoretical_loss": 3.52651682945071, + "tokens_seen": 1450573824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830692076228686, + "loss": 1.4759, + "theoretical_loss": 3.5264885548450886, + "tokens_seen": 1450704896 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028304914744232696, + "loss": 1.4693, + "theoretical_loss": 3.5264602835092003, + "tokens_seen": 1450835968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830290872617854, + "loss": 1.3919, + "theoretical_loss": 3.5264320154423707, + "tokens_seen": 1450967040 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028300902708124374, + "loss": 1.4624, + "theoretical_loss": 3.5264037506439276, + "tokens_seen": 1451098112 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028298896690070216, + "loss": 1.3228, + "theoretical_loss": 3.5263754891131973, + "tokens_seen": 1451229184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829689067201605, + "loss": 1.3911, + "theoretical_loss": 3.526347230849507, + "tokens_seen": 1451360256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829488465396189, + "loss": 1.4087, + "theoretical_loss": 3.5263189758521847, + "tokens_seen": 1451491328 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.46542033553123474, + "objective/train/avg_token_score": 0.012638341635465622, + "objective/train/docs_used": 823297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4112204313278198, + "objective/train/original_loss": 3.462179660797119, + "objective/train/theoretical_loss": 3.5262907241205568, + "objective/train/tokens_used": 1451622400, + "objective/train/unlikelihood_loss": 0.16298909485340118, + "theoretical_loss": 3.5262907241205568, + "tokens_seen": 1451622400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028292878635907725, + "loss": 1.4891, + "theoretical_loss": 3.5262907241205568, + "tokens_seen": 1451622400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028290872617853566, + "loss": 1.4766, + "theoretical_loss": 3.5262624756539522, + "tokens_seen": 1451753472 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282888665997994, + "loss": 1.3627, + "theoretical_loss": 3.526234230451698, + "tokens_seen": 1451884544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828686058174524, + "loss": 1.444, + "theoretical_loss": 3.526205988513123, + "tokens_seen": 1452015616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028284854563691075, + "loss": 1.5178, + "theoretical_loss": 3.5261777498375544, + "tokens_seen": 1452146688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828284854563691, + "loss": 1.4628, + "theoretical_loss": 3.526149514424322, + "tokens_seen": 1452277760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828084252758275, + "loss": 1.4911, + "theoretical_loss": 3.5261212822727535, + "tokens_seen": 1452408832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827883650952859, + "loss": 1.5003, + "theoretical_loss": 3.526093053382179, + "tokens_seen": 1452539904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028276830491474425, + "loss": 1.4985, + "theoretical_loss": 3.5260648277519264, + "tokens_seen": 1452670976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827482447342026, + "loss": 1.454, + "theoretical_loss": 3.526036605381326, + "tokens_seen": 1452802048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028272818455366103, + "loss": 1.4703, + "theoretical_loss": 3.5260083862697065, + "tokens_seen": 1452933120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827081243731194, + "loss": 1.4196, + "theoretical_loss": 3.525980170416398, + "tokens_seen": 1453064192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028268806419257775, + "loss": 1.2303, + "theoretical_loss": 3.525951957820731, + "tokens_seen": 1453195264 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.43609482049942017, + "objective/train/avg_token_score": 0.031401127576828, + "objective/train/docs_used": 824661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3900045156478882, + "objective/train/original_loss": 3.9992895126342773, + "objective/train/theoretical_loss": 3.5259378527443035, + "objective/train/tokens_used": 1453260800, + "objective/train/unlikelihood_loss": 0.1871781200170517, + "theoretical_loss": 3.5259378527443035, + "tokens_seen": 1453260800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826680040120361, + "loss": 1.5461, + "theoretical_loss": 3.5259237484820347, + "tokens_seen": 1453326336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826479438314945, + "loss": 1.5341, + "theoretical_loss": 3.5258955423996396, + "tokens_seen": 1453457408 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826278836509529, + "loss": 1.413, + "theoretical_loss": 3.5258673395728763, + "tokens_seen": 1453588480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028260782347041126, + "loss": 1.2985, + "theoretical_loss": 3.525839140001076, + "tokens_seen": 1453719552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825877632898696, + "loss": 1.39, + "theoretical_loss": 3.525810943683569, + "tokens_seen": 1453850624 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282567703109328, + "loss": 1.3726, + "theoretical_loss": 3.525782750619687, + "tokens_seen": 1453981696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825476429287864, + "loss": 1.3746, + "theoretical_loss": 3.5257545608087604, + "tokens_seen": 1454112768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028252758274824476, + "loss": 1.3915, + "theoretical_loss": 3.5257263742501213, + "tokens_seen": 1454243840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825075225677031, + "loss": 1.388, + "theoretical_loss": 3.525698190943101, + "tokens_seen": 1454374912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824874623871615, + "loss": 1.4031, + "theoretical_loss": 3.5256700108870325, + "tokens_seen": 1454505984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028246740220661985, + "loss": 1.4668, + "theoretical_loss": 3.525641834081246, + "tokens_seen": 1454637056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028244734202607826, + "loss": 1.262, + "theoretical_loss": 3.5256136605250763, + "tokens_seen": 1454768128 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.29001709818840027, + "objective/train/avg_token_score": 0.045937780290842056, + "objective/train/docs_used": 824875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0762373208999634, + "objective/train/original_loss": 4.401334762573242, + "objective/train/theoretical_loss": 3.5255854902178534, + "objective/train/tokens_used": 1454899200, + "objective/train/unlikelihood_loss": 0.15628454089164734, + "theoretical_loss": 3.5255854902178534, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824272818455366, + "loss": 1.4019, + "theoretical_loss": 3.5255854902178534, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282407221664995, + "loss": 1.4088, + "theoretical_loss": 3.5255573231589117, + "tokens_seen": 1455030272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028238716148445335, + "loss": 1.2714, + "theoretical_loss": 3.525529159347583, + "tokens_seen": 1455161344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028236710130391177, + "loss": 1.5009, + "theoretical_loss": 3.525500998783201, + "tokens_seen": 1455292416 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028234704112337013, + "loss": 1.2936, + "theoretical_loss": 3.525472841465099, + "tokens_seen": 1455423488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823269809428285, + "loss": 1.4522, + "theoretical_loss": 3.52544468739261, + "tokens_seen": 1455554560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028230692076228685, + "loss": 1.3741, + "theoretical_loss": 3.525416536565068, + "tokens_seen": 1455685632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822868605817452, + "loss": 1.3767, + "theoretical_loss": 3.525388388981807, + "tokens_seen": 1455816704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028226680040120363, + "loss": 1.3466, + "theoretical_loss": 3.525360244642161, + "tokens_seen": 1455947776 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282246740220662, + "loss": 1.2709, + "theoretical_loss": 3.525332103545464, + "tokens_seen": 1456078848 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028222668004012036, + "loss": 1.2804, + "theoretical_loss": 3.5253039656910508, + "tokens_seen": 1456209920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822066198595787, + "loss": 1.5978, + "theoretical_loss": 3.5252758310782557, + "tokens_seen": 1456340992 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028218655967903714, + "loss": 1.3843, + "theoretical_loss": 3.5252476997064135, + "tokens_seen": 1456472064 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.4890640377998352, + "objective/train/avg_token_score": 0.014791848137974739, + "objective/train/docs_used": 826174, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.62875497341156, + "objective/train/original_loss": 3.888981342315674, + "objective/train/theoretical_loss": 3.525233635235642, + "objective/train/tokens_used": 1456537600, + "objective/train/unlikelihood_loss": 0.15570108592510223, + "theoretical_loss": 3.525233635235642, + "tokens_seen": 1456537600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821664994984955, + "loss": 1.3804, + "theoretical_loss": 3.5252195715748598, + "tokens_seen": 1456603136 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028214643931795386, + "loss": 1.3458, + "theoretical_loss": 3.5251914466829293, + "tokens_seen": 1456734208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821263791374122, + "loss": 1.2623, + "theoretical_loss": 3.5251633250299577, + "tokens_seen": 1456865280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821063189568706, + "loss": 1.4871, + "theoretical_loss": 3.525135206615281, + "tokens_seen": 1456996352 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282086258776329, + "loss": 1.4599, + "theoretical_loss": 3.525107091438234, + "tokens_seen": 1457127424 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028206619859578736, + "loss": 1.5948, + "theoretical_loss": 3.525078979498154, + "tokens_seen": 1457258496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820461384152457, + "loss": 1.4815, + "theoretical_loss": 3.5250508707943764, + "tokens_seen": 1457389568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820260782347041, + "loss": 1.4147, + "theoretical_loss": 3.5250227653262374, + "tokens_seen": 1457520640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820060180541625, + "loss": 1.3812, + "theoretical_loss": 3.524994663093074, + "tokens_seen": 1457651712 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028198595787362087, + "loss": 1.3863, + "theoretical_loss": 3.524966564094224, + "tokens_seen": 1457782784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028196589769307923, + "loss": 1.4027, + "theoretical_loss": 3.5249384683290224, + "tokens_seen": 1457913856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819458375125376, + "loss": 1.475, + "theoretical_loss": 3.5249103757968077, + "tokens_seen": 1458044928 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.4125122129917145, + "objective/train/avg_token_score": 0.004481058102101088, + "objective/train/docs_used": 826810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2941863536834717, + "objective/train/original_loss": 3.0054266452789307, + "objective/train/theoretical_loss": 3.524882286496917, + "objective/train/tokens_used": 1458176000, + "objective/train/unlikelihood_loss": 0.26842546463012695, + "theoretical_loss": 3.524882286496917, + "tokens_seen": 1458176000 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281925777331996, + "loss": 1.3192, + "theoretical_loss": 3.524882286496917, + "tokens_seen": 1458176000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028190571715145437, + "loss": 1.6219, + "theoretical_loss": 3.5248542004286882, + "tokens_seen": 1458307072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028188565697091273, + "loss": 1.4629, + "theoretical_loss": 3.5248261175914584, + "tokens_seen": 1458438144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818655967903711, + "loss": 1.3308, + "theoretical_loss": 3.5247980379845663, + "tokens_seen": 1458569216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028184553660982946, + "loss": 1.2789, + "theoretical_loss": 3.5247699616073502, + "tokens_seen": 1458700288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818254764292879, + "loss": 1.3021, + "theoretical_loss": 3.5247418884591473, + "tokens_seen": 1458831360 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028180541624874624, + "loss": 1.4347, + "theoretical_loss": 3.524713818539297, + "tokens_seen": 1458962432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817853560682046, + "loss": 1.4017, + "theoretical_loss": 3.5246857518471386, + "tokens_seen": 1459093504 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028176529588766296, + "loss": 1.4378, + "theoretical_loss": 3.52465768838201, + "tokens_seen": 1459224576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817452357071214, + "loss": 1.2494, + "theoretical_loss": 3.5246296281432508, + "tokens_seen": 1459355648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028172517552657974, + "loss": 1.3497, + "theoretical_loss": 3.5246015711302006, + "tokens_seen": 1459486720 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817051153460381, + "loss": 1.3218, + "theoretical_loss": 3.5245735173421986, + "tokens_seen": 1459617792 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028168505516549646, + "loss": 1.5194, + "theoretical_loss": 3.5245454667785845, + "tokens_seen": 1459748864 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.42864125967025757, + "objective/train/avg_token_score": 0.004115052986890078, + "objective/train/docs_used": 827996, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3418611288070679, + "objective/train/original_loss": 3.5818307399749756, + "objective/train/theoretical_loss": 3.524531442705716, + "objective/train/tokens_used": 1459814400, + "objective/train/unlikelihood_loss": 0.19998368620872498, + "theoretical_loss": 3.524531442705716, + "tokens_seen": 1459814400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816649949849548, + "loss": 1.3922, + "theoretical_loss": 3.5245174194386983, + "tokens_seen": 1459879936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028164493480441324, + "loss": 1.3751, + "theoretical_loss": 3.5244893753218802, + "tokens_seen": 1460011008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816248746238716, + "loss": 1.4487, + "theoretical_loss": 3.5244613344274707, + "tokens_seen": 1460142080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028160481444332997, + "loss": 1.571, + "theoretical_loss": 3.52443329675481, + "tokens_seen": 1460273152 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028158475426278833, + "loss": 1.3712, + "theoretical_loss": 3.524405262303239, + "tokens_seen": 1460404224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028156469408224674, + "loss": 1.2673, + "theoretical_loss": 3.524377231072098, + "tokens_seen": 1460535296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815446339017051, + "loss": 1.3325, + "theoretical_loss": 3.5243492030607295, + "tokens_seen": 1460666368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815245737211635, + "loss": 1.4036, + "theoretical_loss": 3.5243211782684734, + "tokens_seen": 1460797440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815045135406219, + "loss": 1.2577, + "theoretical_loss": 3.5242931566946716, + "tokens_seen": 1460928512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028148445336008025, + "loss": 1.2627, + "theoretical_loss": 3.5242651383386665, + "tokens_seen": 1461059584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028146439317953866, + "loss": 1.331, + "theoretical_loss": 3.5242371231997986, + "tokens_seen": 1461190656 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281444332998997, + "loss": 1.2822, + "theoretical_loss": 3.524209111277411, + "tokens_seen": 1461321728 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.5160679221153259, + "objective/train/avg_token_score": 0.011202037334442139, + "objective/train/docs_used": 828576, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7984460592269897, + "objective/train/original_loss": 3.6972641944885254, + "objective/train/theoretical_loss": 3.524181102570846, + "objective/train/tokens_used": 1461452800, + "objective/train/unlikelihood_loss": 0.14342762529850006, + "theoretical_loss": 3.524181102570846, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814242728184554, + "loss": 1.4366, + "theoretical_loss": 3.524181102570846, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028140421263791375, + "loss": 1.4819, + "theoretical_loss": 3.5241530970794455, + "tokens_seen": 1461583872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028138415245737217, + "loss": 1.5484, + "theoretical_loss": 3.524125094802552, + "tokens_seen": 1461714944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028136409227683053, + "loss": 1.5316, + "theoretical_loss": 3.524097095739509, + "tokens_seen": 1461846016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813440320962889, + "loss": 1.4332, + "theoretical_loss": 3.5240690998896596, + "tokens_seen": 1461977088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028132397191574725, + "loss": 1.363, + "theoretical_loss": 3.524041107252346, + "tokens_seen": 1462108160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813039117352056, + "loss": 1.2676, + "theoretical_loss": 3.524013117826913, + "tokens_seen": 1462239232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028128385155466403, + "loss": 1.2743, + "theoretical_loss": 3.5239851316127035, + "tokens_seen": 1462370304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812637913741224, + "loss": 1.4369, + "theoretical_loss": 3.523957148609061, + "tokens_seen": 1462501376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028124373119358076, + "loss": 1.452, + "theoretical_loss": 3.5239291688153296, + "tokens_seen": 1462632448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812236710130391, + "loss": 1.3489, + "theoretical_loss": 3.523901192230854, + "tokens_seen": 1462763520 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028120361083249754, + "loss": 1.4226, + "theoretical_loss": 3.5238732188549777, + "tokens_seen": 1462894592 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811835506519559, + "loss": 1.3481, + "theoretical_loss": 3.5238452486870466, + "tokens_seen": 1463025664 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.4528958797454834, + "objective/train/avg_token_score": 0.01902177929878235, + "objective/train/docs_used": 830024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4812493324279785, + "objective/train/original_loss": 3.7723329067230225, + "objective/train/theoretical_loss": 3.5238312648058554, + "objective/train/tokens_used": 1463091200, + "objective/train/unlikelihood_loss": 0.15412835776805878, + "theoretical_loss": 3.5238312648058554, + "tokens_seen": 1463091200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028116349047141426, + "loss": 1.6208, + "theoretical_loss": 3.5238172817264046, + "tokens_seen": 1463156736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811434302908726, + "loss": 1.3365, + "theoretical_loss": 3.5237893179723967, + "tokens_seen": 1463287808 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281123370110331, + "loss": 1.4797, + "theoretical_loss": 3.523761357424368, + "tokens_seen": 1463418880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811033099297894, + "loss": 1.2835, + "theoretical_loss": 3.523733400081664, + "tokens_seen": 1463549952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028108324974924776, + "loss": 1.5386, + "theoretical_loss": 3.52370544594363, + "tokens_seen": 1463681024 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810631895687061, + "loss": 1.359, + "theoretical_loss": 3.5236774950096126, + "tokens_seen": 1463812096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810431293881645, + "loss": 1.4305, + "theoretical_loss": 3.5236495472789566, + "tokens_seen": 1463943168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810230692076229, + "loss": 1.2434, + "theoretical_loss": 3.5236216027510086, + "tokens_seen": 1464074240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028100300902708127, + "loss": 1.322, + "theoretical_loss": 3.523593661425115, + "tokens_seen": 1464205312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028098294884653963, + "loss": 1.2366, + "theoretical_loss": 3.5235657233006226, + "tokens_seen": 1464336384 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280962888665998, + "loss": 1.3324, + "theoretical_loss": 3.523537788376877, + "tokens_seen": 1464467456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028094282848545635, + "loss": 1.5018, + "theoretical_loss": 3.5235098566532264, + "tokens_seen": 1464598528 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.42027124762535095, + "objective/train/avg_token_score": 0.00850069522857666, + "objective/train/docs_used": 830651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2469569444656372, + "objective/train/original_loss": 3.1987521648406982, + "objective/train/theoretical_loss": 3.5234819281290166, + "objective/train/tokens_used": 1464729600, + "objective/train/unlikelihood_loss": 0.27428585290908813, + "theoretical_loss": 3.5234819281290166, + "tokens_seen": 1464729600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028092276830491477, + "loss": 1.3028, + "theoretical_loss": 3.5234819281290166, + "tokens_seen": 1464729600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028090270812437313, + "loss": 1.3697, + "theoretical_loss": 3.5234540028035957, + "tokens_seen": 1464860672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808826479438315, + "loss": 1.5056, + "theoretical_loss": 3.523426080676311, + "tokens_seen": 1464991744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028086258776328986, + "loss": 1.3124, + "theoretical_loss": 3.5233981617465107, + "tokens_seen": 1465122816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808425275827483, + "loss": 1.3912, + "theoretical_loss": 3.523370246013541, + "tokens_seen": 1465253888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028082246740220664, + "loss": 1.1593, + "theoretical_loss": 3.5233423334767515, + "tokens_seen": 1465384960 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280802407221665, + "loss": 1.4002, + "theoretical_loss": 3.5233144241354903, + "tokens_seen": 1465516032 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028078234704112336, + "loss": 1.5725, + "theoretical_loss": 3.5232865179891046, + "tokens_seen": 1465647104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807622868605818, + "loss": 1.4619, + "theoretical_loss": 3.5232586150369443, + "tokens_seen": 1465778176 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028074222668004014, + "loss": 1.3725, + "theoretical_loss": 3.5232307152783577, + "tokens_seen": 1465909248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807221664994985, + "loss": 1.3128, + "theoretical_loss": 3.523202818712694, + "tokens_seen": 1466040320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028070210631895686, + "loss": 1.4077, + "theoretical_loss": 3.523174925339302, + "tokens_seen": 1466171392 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806820461384152, + "loss": 1.3864, + "theoretical_loss": 3.523147035157531, + "tokens_seen": 1466302464 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.40163734555244446, + "objective/train/avg_token_score": 0.007136420346796513, + "objective/train/docs_used": 831840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.284009337425232, + "objective/train/original_loss": 3.844454050064087, + "objective/train/theoretical_loss": 3.5231330912633005, + "objective/train/tokens_used": 1466368000, + "objective/train/unlikelihood_loss": 0.17542870342731476, + "theoretical_loss": 3.5231330912633005, + "tokens_seen": 1466368000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028066198595787364, + "loss": 1.38, + "theoretical_loss": 3.523119148166731, + "tokens_seen": 1466433536 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280641925777332, + "loss": 1.3601, + "theoretical_loss": 3.523091264366252, + "tokens_seen": 1466564608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028062186559679037, + "loss": 1.3753, + "theoretical_loss": 3.523063383755443, + "tokens_seen": 1466695680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028060180541624873, + "loss": 1.3066, + "theoretical_loss": 3.5230355063336547, + "tokens_seen": 1466826752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028058174523570715, + "loss": 1.3639, + "theoretical_loss": 3.5230076321002373, + "tokens_seen": 1466957824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805616850551655, + "loss": 1.3292, + "theoretical_loss": 3.5229797610545415, + "tokens_seen": 1467088896 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028054162487462387, + "loss": 1.4448, + "theoretical_loss": 3.522951893195918, + "tokens_seen": 1467219968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028052156469408223, + "loss": 1.1931, + "theoretical_loss": 3.5229240285237173, + "tokens_seen": 1467351040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805015045135406, + "loss": 1.4402, + "theoretical_loss": 3.522896167037291, + "tokens_seen": 1467482112 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280481444332999, + "loss": 1.3446, + "theoretical_loss": 3.5228683087359896, + "tokens_seen": 1467613184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028046138415245737, + "loss": 1.2645, + "theoretical_loss": 3.5228404536191658, + "tokens_seen": 1467744256 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028044132397191574, + "loss": 1.3323, + "theoretical_loss": 3.5228126016861703, + "tokens_seen": 1467875328 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.47831133008003235, + "objective/train/avg_token_score": 0.02133219689130783, + "objective/train/docs_used": 832610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6741708517074585, + "objective/train/original_loss": 4.040345191955566, + "objective/train/theoretical_loss": 3.5227847529363547, + "objective/train/tokens_used": 1468006400, + "objective/train/unlikelihood_loss": 0.1244034618139267, + "theoretical_loss": 3.5227847529363547, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804212637913741, + "loss": 1.3575, + "theoretical_loss": 3.5227847529363547, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804012036108325, + "loss": 1.3082, + "theoretical_loss": 3.5227569073690717, + "tokens_seen": 1468137472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803811434302909, + "loss": 1.4665, + "theoretical_loss": 3.5227290649836736, + "tokens_seen": 1468268544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028036108324974924, + "loss": 1.3745, + "theoretical_loss": 3.5227012257795125, + "tokens_seen": 1468399616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803410230692076, + "loss": 1.3741, + "theoretical_loss": 3.522673389755941, + "tokens_seen": 1468530688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028032096288866596, + "loss": 1.5293, + "theoretical_loss": 3.522645556912312, + "tokens_seen": 1468661760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803009027081244, + "loss": 1.5662, + "theoretical_loss": 3.5226177272479786, + "tokens_seen": 1468792832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028028084252758274, + "loss": 1.2388, + "theoretical_loss": 3.522589900762293, + "tokens_seen": 1468923904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802607823470411, + "loss": 1.2487, + "theoretical_loss": 3.5225620774546105, + "tokens_seen": 1469054976 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028024072216649947, + "loss": 1.4283, + "theoretical_loss": 3.522534257324282, + "tokens_seen": 1469186048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802206619859579, + "loss": 1.3863, + "theoretical_loss": 3.5225064403706643, + "tokens_seen": 1469317120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028020060180541624, + "loss": 1.3449, + "theoretical_loss": 3.5224786265931085, + "tokens_seen": 1469448192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801805416248746, + "loss": 1.5329, + "theoretical_loss": 3.522450815990971, + "tokens_seen": 1469579264 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.37335044145584106, + "objective/train/avg_token_score": 0.01705220714211464, + "objective/train/docs_used": 834187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1379942893981934, + "objective/train/original_loss": 4.053225040435791, + "objective/train/theoretical_loss": 3.5224369118804817, + "objective/train/tokens_used": 1469644800, + "objective/train/unlikelihood_loss": 0.12494862079620361, + "theoretical_loss": 3.5224369118804817, + "tokens_seen": 1469644800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028016048144433297, + "loss": 1.4656, + "theoretical_loss": 3.5224230085636044, + "tokens_seen": 1469710336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028014042126379133, + "loss": 1.4919, + "theoretical_loss": 3.522395204310364, + "tokens_seen": 1469841408 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028012036108324975, + "loss": 1.3937, + "theoretical_loss": 3.5223674032306045, + "tokens_seen": 1469972480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801003009027081, + "loss": 1.2738, + "theoretical_loss": 3.5223396053236806, + "tokens_seen": 1470103552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028008024072216647, + "loss": 1.3066, + "theoretical_loss": 3.5223118105889473, + "tokens_seen": 1470234624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028006018054162483, + "loss": 1.3706, + "theoretical_loss": 3.52228401902576, + "tokens_seen": 1470365696 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028004012036108325, + "loss": 1.1803, + "theoretical_loss": 3.5222562306334737, + "tokens_seen": 1470496768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028002006018054167, + "loss": 1.2835, + "theoretical_loss": 3.5222284454114448, + "tokens_seen": 1470627840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028000000000000003, + "loss": 1.4694, + "theoretical_loss": 3.522200663359029, + "tokens_seen": 1470758912 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799799398194584, + "loss": 1.2843, + "theoretical_loss": 3.5221728844755815, + "tokens_seen": 1470889984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027995987963891675, + "loss": 1.386, + "theoretical_loss": 3.5221451087604594, + "tokens_seen": 1471021056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027993981945837517, + "loss": 1.4232, + "theoretical_loss": 3.522117336213018, + "tokens_seen": 1471152128 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.48405423760414124, + "objective/train/avg_token_score": 0.01983676478266716, + "objective/train/docs_used": 834912, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6344624757766724, + "objective/train/original_loss": 3.6980395317077637, + "objective/train/theoretical_loss": 3.522089566832615, + "objective/train/tokens_used": 1471283200, + "objective/train/unlikelihood_loss": 0.17185752093791962, + "theoretical_loss": 3.522089566832615, + "tokens_seen": 1471283200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027991975927783353, + "loss": 1.3355, + "theoretical_loss": 3.522089566832615, + "tokens_seen": 1471283200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798996990972919, + "loss": 1.423, + "theoretical_loss": 3.522061800618607, + "tokens_seen": 1471414272 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027987963891675026, + "loss": 1.4102, + "theoretical_loss": 3.52203403757035, + "tokens_seen": 1471545344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798595787362087, + "loss": 1.3824, + "theoretical_loss": 3.522006277687202, + "tokens_seen": 1471676416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027983951855566704, + "loss": 1.3632, + "theoretical_loss": 3.52197852096852, + "tokens_seen": 1471807488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798194583751254, + "loss": 1.3541, + "theoretical_loss": 3.521950767413662, + "tokens_seen": 1471938560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027979939819458376, + "loss": 1.2253, + "theoretical_loss": 3.521923017021985, + "tokens_seen": 1472069632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797793380140422, + "loss": 1.3943, + "theoretical_loss": 3.5218952697928474, + "tokens_seen": 1472200704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027975927783350054, + "loss": 1.382, + "theoretical_loss": 3.5218675257256065, + "tokens_seen": 1472331776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797392176529589, + "loss": 1.3806, + "theoretical_loss": 3.5218397848196217, + "tokens_seen": 1472462848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027971915747241726, + "loss": 1.375, + "theoretical_loss": 3.52181204707425, + "tokens_seen": 1472593920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796990972918756, + "loss": 1.3514, + "theoretical_loss": 3.5217843124888515, + "tokens_seen": 1472724992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027967903711133404, + "loss": 1.4805, + "theoretical_loss": 3.5217565810627844, + "tokens_seen": 1472856064 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.38508063554763794, + "objective/train/avg_token_score": 0.01531413197517395, + "objective/train/docs_used": 835528, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2243326902389526, + "objective/train/original_loss": 3.7222633361816406, + "objective/train/theoretical_loss": 3.5217427165343, + "objective/train/tokens_used": 1472921600, + "objective/train/unlikelihood_loss": 0.15625110268592834, + "theoretical_loss": 3.5217427165343, + "tokens_seen": 1472921600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796589769307924, + "loss": 1.2262, + "theoretical_loss": 3.521728852795407, + "tokens_seen": 1472987136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027963891675025077, + "loss": 1.2586, + "theoretical_loss": 3.52170112768608, + "tokens_seen": 1473118208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027961885656970913, + "loss": 1.2666, + "theoretical_loss": 3.5216734057341617, + "tokens_seen": 1473249280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027959879638916755, + "loss": 1.4829, + "theoretical_loss": 3.5216456869390123, + "tokens_seen": 1473380352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795787362086259, + "loss": 1.3834, + "theoretical_loss": 3.5216179712999907, + "tokens_seen": 1473511424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027955867602808427, + "loss": 1.5556, + "theoretical_loss": 3.521590258816457, + "tokens_seen": 1473642496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027953861584754263, + "loss": 1.3205, + "theoretical_loss": 3.521562549487772, + "tokens_seen": 1473773568 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279518555667001, + "loss": 1.3709, + "theoretical_loss": 3.521534843313296, + "tokens_seen": 1473904640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794984954864594, + "loss": 1.5196, + "theoretical_loss": 3.521507140292389, + "tokens_seen": 1474035712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794784353059178, + "loss": 1.3103, + "theoretical_loss": 3.521479440424412, + "tokens_seen": 1474166784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027945837512537614, + "loss": 1.3497, + "theoretical_loss": 3.5214517437087247, + "tokens_seen": 1474297856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794383149448345, + "loss": 1.4439, + "theoretical_loss": 3.52142405014469, + "tokens_seen": 1474428928 + }, + { + "debugging/length": 443.4, + "debugging/num_segments": 5, + "epoch": 0.45, + "objective/train/aligned_token_count": 0.44312071800231934, + "objective/train/avg_token_score": 0.013671123422682285, + "objective/train/docs_used": 836610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4569171667099, + "objective/train/original_loss": 3.928725242614746, + "objective/train/theoretical_loss": 3.521396359731668, + "objective/train/tokens_used": 1474560000, + "objective/train/unlikelihood_loss": 0.20717255771160126, + "theoretical_loss": 3.521396359731668, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794182547642929, + "loss": 1.3583, + "theoretical_loss": 3.521396359731668, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793981945837513, + "loss": 1.4315, + "theoretical_loss": 3.5213686724690207, + "tokens_seen": 1474691072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027937813440320964, + "loss": 1.3126, + "theoretical_loss": 3.5213409883561093, + "tokens_seen": 1474822144 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279358074222668, + "loss": 1.4177, + "theoretical_loss": 3.5213133073922958, + "tokens_seen": 1474953216 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027933801404212636, + "loss": 1.4679, + "theoretical_loss": 3.5212856295769424, + "tokens_seen": 1475084288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793179538615848, + "loss": 1.4214, + "theoretical_loss": 3.5212579549094105, + "tokens_seen": 1475215360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027929789368104314, + "loss": 1.4779, + "theoretical_loss": 3.5212302833890634, + "tokens_seen": 1475346432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792778335005015, + "loss": 1.4166, + "theoretical_loss": 3.5212026150152633, + "tokens_seen": 1475477504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027925777331995987, + "loss": 1.274, + "theoretical_loss": 3.5211749497873726, + "tokens_seen": 1475608576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792377131394183, + "loss": 1.3496, + "theoretical_loss": 3.5211472877047543, + "tokens_seen": 1475739648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027921765295887664, + "loss": 1.1587, + "theoretical_loss": 3.5211196287667716, + "tokens_seen": 1475870720 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279197592778335, + "loss": 1.3051, + "theoretical_loss": 3.5210919729727888, + "tokens_seen": 1476001792 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027917753259779337, + "loss": 1.3003, + "theoretical_loss": 3.521064320322168, + "tokens_seen": 1476132864 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.36614125967025757, + "objective/train/avg_token_score": 0.006083973217755556, + "objective/train/docs_used": 837052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2062969207763672, + "objective/train/original_loss": 3.5040762424468994, + "objective/train/theoretical_loss": 3.5210504951754196, + "objective/train/tokens_used": 1476198400, + "objective/train/unlikelihood_loss": 0.2551019489765167, + "theoretical_loss": 3.5210504951754196, + "tokens_seen": 1476198400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027915747241725173, + "loss": 1.3161, + "theoretical_loss": 3.521036670814273, + "tokens_seen": 1476263936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027913741223671015, + "loss": 1.4173, + "theoretical_loss": 3.521009024448468, + "tokens_seen": 1476395008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791173520561685, + "loss": 1.236, + "theoretical_loss": 3.520981381224117, + "tokens_seen": 1476526080 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027909729187562687, + "loss": 1.2818, + "theoretical_loss": 3.5209537411405845, + "tokens_seen": 1476657152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027907723169508523, + "loss": 1.3541, + "theoretical_loss": 3.520926104197234, + "tokens_seen": 1476788224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027905717151454365, + "loss": 1.3824, + "theoretical_loss": 3.5208984703934316, + "tokens_seen": 1476919296 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279037111334002, + "loss": 1.4982, + "theoretical_loss": 3.5208708397285404, + "tokens_seen": 1477050368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790170511534604, + "loss": 1.2696, + "theoretical_loss": 3.5208432122019264, + "tokens_seen": 1477181440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027899699097291874, + "loss": 1.3555, + "theoretical_loss": 3.5208155878129546, + "tokens_seen": 1477312512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789769307923771, + "loss": 1.3679, + "theoretical_loss": 3.5207879665609902, + "tokens_seen": 1477443584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789568706118355, + "loss": 1.3793, + "theoretical_loss": 3.520760348445399, + "tokens_seen": 1477574656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789368104312939, + "loss": 1.2822, + "theoretical_loss": 3.520732733465546, + "tokens_seen": 1477705728 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.419721394777298, + "objective/train/avg_token_score": 0.010134999640285969, + "objective/train/docs_used": 837925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3294548988342285, + "objective/train/original_loss": 3.74300479888916, + "objective/train/theoretical_loss": 3.520705121620797, + "objective/train/tokens_used": 1477836800, + "objective/train/unlikelihood_loss": 0.1622351109981537, + "theoretical_loss": 3.520705121620797, + "tokens_seen": 1477836800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027891675025075224, + "loss": 1.3724, + "theoretical_loss": 3.520705121620797, + "tokens_seen": 1477836800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788966900702106, + "loss": 1.3866, + "theoretical_loss": 3.520677512910519, + "tokens_seen": 1477967872 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278876629889669, + "loss": 1.3134, + "theoretical_loss": 3.520649907334078, + "tokens_seen": 1478098944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788565697091274, + "loss": 1.3535, + "theoretical_loss": 3.5206223048908396, + "tokens_seen": 1478230016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027883650952858574, + "loss": 1.4761, + "theoretical_loss": 3.520594705580171, + "tokens_seen": 1478361088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788164493480441, + "loss": 1.3729, + "theoretical_loss": 3.5205671094014397, + "tokens_seen": 1478492160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027879638916750247, + "loss": 1.463, + "theoretical_loss": 3.520539516354012, + "tokens_seen": 1478623232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787763289869609, + "loss": 1.2661, + "theoretical_loss": 3.520511926437255, + "tokens_seen": 1478754304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027875626880641925, + "loss": 1.4817, + "theoretical_loss": 3.520484339650536, + "tokens_seen": 1478885376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787362086258776, + "loss": 1.3149, + "theoretical_loss": 3.520456755993222, + "tokens_seen": 1479016448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027871614844533597, + "loss": 1.369, + "theoretical_loss": 3.520429175464682, + "tokens_seen": 1479147520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786960882647944, + "loss": 1.294, + "theoretical_loss": 3.5204015980642835, + "tokens_seen": 1479278592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027867602808425275, + "loss": 1.5173, + "theoretical_loss": 3.520374023791394, + "tokens_seen": 1479409664 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.35697701573371887, + "objective/train/avg_token_score": 0.018237190321087837, + "objective/train/docs_used": 838738, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1747971773147583, + "objective/train/original_loss": 4.5082902908325195, + "objective/train/theoretical_loss": 3.520360237827568, + "objective/train/tokens_used": 1479475200, + "objective/train/unlikelihood_loss": 0.1696925163269043, + "theoretical_loss": 3.520360237827568, + "tokens_seen": 1479475200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786559679037111, + "loss": 1.34, + "theoretical_loss": 3.5203464526453825, + "tokens_seen": 1479540736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786359077231695, + "loss": 1.2816, + "theoretical_loss": 3.520318884625617, + "tokens_seen": 1479671808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786158475426279, + "loss": 1.2122, + "theoretical_loss": 3.5202913197314665, + "tokens_seen": 1479802880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027859578736208625, + "loss": 1.3519, + "theoretical_loss": 3.520263757962299, + "tokens_seen": 1479933952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785757271815446, + "loss": 1.3942, + "theoretical_loss": 3.5202361993174844, + "tokens_seen": 1480065024 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278555667001003, + "loss": 1.2405, + "theoretical_loss": 3.5202086437963915, + "tokens_seen": 1480196096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027853560682046134, + "loss": 1.2413, + "theoretical_loss": 3.52018109139839, + "tokens_seen": 1480327168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785155466399198, + "loss": 1.3749, + "theoretical_loss": 3.520153542122849, + "tokens_seen": 1480458240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784954864593782, + "loss": 1.2645, + "theoretical_loss": 3.5201259959691384, + "tokens_seen": 1480589312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027847542627883654, + "loss": 1.1577, + "theoretical_loss": 3.520098452936628, + "tokens_seen": 1480720384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784553660982949, + "loss": 1.2947, + "theoretical_loss": 3.5200709130246883, + "tokens_seen": 1480851456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784353059177533, + "loss": 1.2736, + "theoretical_loss": 3.5200433762326897, + "tokens_seen": 1480982528 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.40866324305534363, + "objective/train/avg_token_score": 0.013213703408837318, + "objective/train/docs_used": 839976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2744966745376587, + "objective/train/original_loss": 3.802645683288574, + "objective/train/theoretical_loss": 3.520015842560001, + "objective/train/tokens_used": 1481113600, + "objective/train/unlikelihood_loss": 0.14176234602928162, + "theoretical_loss": 3.520015842560001, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784152457372117, + "loss": 1.3146, + "theoretical_loss": 3.520015842560001, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027839518555667004, + "loss": 1.3579, + "theoretical_loss": 3.519988312005996, + "tokens_seen": 1481244672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783751253761284, + "loss": 1.3749, + "theoretical_loss": 3.519960784570042, + "tokens_seen": 1481375744 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027835506519558676, + "loss": 1.2891, + "theoretical_loss": 3.5199332602515128, + "tokens_seen": 1481506816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783350050150452, + "loss": 1.2675, + "theoretical_loss": 3.5199057390497783, + "tokens_seen": 1481637888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027831494483450354, + "loss": 1.3169, + "theoretical_loss": 3.51987822096421, + "tokens_seen": 1481768960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782948846539619, + "loss": 1.104, + "theoretical_loss": 3.5198507059941795, + "tokens_seen": 1481900032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027827482447342027, + "loss": 1.3986, + "theoretical_loss": 3.5198231941390588, + "tokens_seen": 1482031104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782547642928787, + "loss": 1.2766, + "theoretical_loss": 3.5197956853982193, + "tokens_seen": 1482162176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027823470411233705, + "loss": 1.4605, + "theoretical_loss": 3.5197681797710336, + "tokens_seen": 1482293248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782146439317954, + "loss": 1.3204, + "theoretical_loss": 3.5197406772568742, + "tokens_seen": 1482424320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027819458375125377, + "loss": 1.5316, + "theoretical_loss": 3.519713177855113, + "tokens_seen": 1482555392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027817452357071213, + "loss": 1.2059, + "theoretical_loss": 3.519685681565123, + "tokens_seen": 1482686464 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.3903347849845886, + "objective/train/avg_token_score": 0.004387664142996073, + "objective/train/docs_used": 840589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1931071281433105, + "objective/train/original_loss": 3.631075859069824, + "objective/train/theoretical_loss": 3.519671934586846, + "objective/train/tokens_used": 1482752000, + "objective/train/unlikelihood_loss": 0.16786478459835052, + "theoretical_loss": 3.519671934586846, + "tokens_seen": 1482752000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027815446339017055, + "loss": 1.2407, + "theoretical_loss": 3.519658188386277, + "tokens_seen": 1482817536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781344032096289, + "loss": 1.2925, + "theoretical_loss": 3.5196306983179477, + "tokens_seen": 1482948608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781143430290873, + "loss": 1.4313, + "theoretical_loss": 3.5196032113595086, + "tokens_seen": 1483079680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027809428284854564, + "loss": 1.3857, + "theoretical_loss": 3.519575727510333, + "tokens_seen": 1483210752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027807422266800405, + "loss": 1.2144, + "theoretical_loss": 3.5195482467697943, + "tokens_seen": 1483341824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780541624874624, + "loss": 1.3354, + "theoretical_loss": 3.5195207691372667, + "tokens_seen": 1483472896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780341023069208, + "loss": 1.4301, + "theoretical_loss": 3.519493294612124, + "tokens_seen": 1483603968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027801404212637914, + "loss": 1.2793, + "theoretical_loss": 3.51946582319374, + "tokens_seen": 1483735040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779939819458375, + "loss": 1.4129, + "theoretical_loss": 3.5194383548814896, + "tokens_seen": 1483866112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779739217652959, + "loss": 1.3837, + "theoretical_loss": 3.519410889674746, + "tokens_seen": 1483997184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779538615847543, + "loss": 1.2374, + "theoretical_loss": 3.5193834275728855, + "tokens_seen": 1484128256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027793380140421264, + "loss": 1.4095, + "theoretical_loss": 3.519355968575282, + "tokens_seen": 1484259328 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.5956745147705078, + "objective/train/avg_token_score": 0.001881771837361157, + "objective/train/docs_used": 841910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8555012941360474, + "objective/train/original_loss": 3.1325271129608154, + "objective/train/theoretical_loss": 3.5193285126813105, + "objective/train/tokens_used": 1484390400, + "objective/train/unlikelihood_loss": 0.2226410061120987, + "theoretical_loss": 3.5193285126813105, + "tokens_seen": 1484390400 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277913741223671, + "loss": 1.4061, + "theoretical_loss": 3.5193285126813105, + "tokens_seen": 1484390400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778936810431294, + "loss": 1.4751, + "theoretical_loss": 3.5193010598903465, + "tokens_seen": 1484521472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778736208625878, + "loss": 1.4351, + "theoretical_loss": 3.5192736102017648, + "tokens_seen": 1484652544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027785356068204614, + "loss": 1.3366, + "theoretical_loss": 3.519246163614942, + "tokens_seen": 1484783616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778335005015045, + "loss": 1.349, + "theoretical_loss": 3.519218720129253, + "tokens_seen": 1484914688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027781344032096287, + "loss": 1.4872, + "theoretical_loss": 3.519191279744074, + "tokens_seen": 1485045760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777933801404213, + "loss": 1.4853, + "theoretical_loss": 3.5191638424587808, + "tokens_seen": 1485176832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027777331995987965, + "loss": 1.3446, + "theoretical_loss": 3.5191364082727503, + "tokens_seen": 1485307904 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277753259779338, + "loss": 1.348, + "theoretical_loss": 3.5191089771853585, + "tokens_seen": 1485438976 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027773319959879637, + "loss": 1.43, + "theoretical_loss": 3.519081549195982, + "tokens_seen": 1485570048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777131394182548, + "loss": 1.3873, + "theoretical_loss": 3.519054124303998, + "tokens_seen": 1485701120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027769307923771315, + "loss": 1.4688, + "theoretical_loss": 3.519026702508783, + "tokens_seen": 1485832192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776730190571715, + "loss": 1.2772, + "theoretical_loss": 3.518999283809715, + "tokens_seen": 1485963264 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.3427419364452362, + "objective/train/avg_token_score": 0.010530060157179832, + "objective/train/docs_used": 842517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0767756700515747, + "objective/train/original_loss": 3.592205047607422, + "objective/train/theoretical_loss": 3.5189855756210413, + "objective/train/tokens_used": 1486028800, + "objective/train/unlikelihood_loss": 0.2110508233308792, + "theoretical_loss": 3.5189855756210413, + "tokens_seen": 1486028800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776529588766299, + "loss": 1.2695, + "theoretical_loss": 3.5189718682061706, + "tokens_seen": 1486094336 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776328986960883, + "loss": 1.3524, + "theoretical_loss": 3.518944455697527, + "tokens_seen": 1486225408 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027761283851554665, + "loss": 1.4885, + "theoretical_loss": 3.518917046283163, + "tokens_seen": 1486356480 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277592778335005, + "loss": 1.2734, + "theoretical_loss": 3.518889639962456, + "tokens_seen": 1486487552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775727181544634, + "loss": 1.2606, + "theoretical_loss": 3.5188622367347846, + "tokens_seen": 1486618624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027755265797392174, + "loss": 1.4414, + "theoretical_loss": 3.5188348365995257, + "tokens_seen": 1486749696 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027753259779338016, + "loss": 1.4572, + "theoretical_loss": 3.518807439556059, + "tokens_seen": 1486880768 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775125376128385, + "loss": 1.3779, + "theoretical_loss": 3.518780045603763, + "tokens_seen": 1487011840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774924774322969, + "loss": 1.1656, + "theoretical_loss": 3.518752654742016, + "tokens_seen": 1487142912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027747241725175524, + "loss": 1.2722, + "theoretical_loss": 3.518725266970197, + "tokens_seen": 1487273984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027745235707121366, + "loss": 1.3233, + "theoretical_loss": 3.5186978822876855, + "tokens_seen": 1487405056 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277432296890672, + "loss": 1.4378, + "theoretical_loss": 3.518670500693861, + "tokens_seen": 1487536128 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.39546677470207214, + "objective/train/avg_token_score": 0.014961336739361286, + "objective/train/docs_used": 843832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1203639507293701, + "objective/train/original_loss": 3.822622060775757, + "objective/train/theoretical_loss": 3.5186431221881023, + "objective/train/tokens_used": 1487667200, + "objective/train/unlikelihood_loss": 0.12135648727416992, + "theoretical_loss": 3.5186431221881023, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774122367101304, + "loss": 1.4752, + "theoretical_loss": 3.5186431221881023, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027739217652958875, + "loss": 1.4262, + "theoretical_loss": 3.5186157467697896, + "tokens_seen": 1487798272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773721163490471, + "loss": 1.1961, + "theoretical_loss": 3.518588374438303, + "tokens_seen": 1487929344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773520561685055, + "loss": 1.3, + "theoretical_loss": 3.518561005193022, + "tokens_seen": 1488060416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773319959879639, + "loss": 1.3649, + "theoretical_loss": 3.5185336390333273, + "tokens_seen": 1488191488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027731193580742225, + "loss": 1.4783, + "theoretical_loss": 3.518506275958599, + "tokens_seen": 1488322560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772918756268806, + "loss": 1.3836, + "theoretical_loss": 3.5184789159682177, + "tokens_seen": 1488453632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027727181544633903, + "loss": 1.4124, + "theoretical_loss": 3.518451559061565, + "tokens_seen": 1488584704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772517552657974, + "loss": 1.3253, + "theoretical_loss": 3.5184242052380204, + "tokens_seen": 1488715776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027723169508525575, + "loss": 1.481, + "theoretical_loss": 3.5183968544969657, + "tokens_seen": 1488846848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772116349047141, + "loss": 1.3206, + "theoretical_loss": 3.518369506837783, + "tokens_seen": 1488977920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771915747241725, + "loss": 1.3373, + "theoretical_loss": 3.5183421622598527, + "tokens_seen": 1489108992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771715145436309, + "loss": 1.2936, + "theoretical_loss": 3.5183148207625567, + "tokens_seen": 1489240064 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.342253178358078, + "objective/train/avg_token_score": 0.023411698639392853, + "objective/train/docs_used": 844519, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.035590410232544, + "objective/train/original_loss": 3.7189109325408936, + "objective/train/theoretical_loss": 3.518301151168954, + "objective/train/tokens_used": 1489305600, + "objective/train/unlikelihood_loss": 0.1891334503889084, + "theoretical_loss": 3.518301151168954, + "tokens_seen": 1489305600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027715145436308926, + "loss": 1.3918, + "theoretical_loss": 3.5182874823452774, + "tokens_seen": 1489371136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771313941825476, + "loss": 1.3315, + "theoretical_loss": 3.518260147007396, + "tokens_seen": 1489502208 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277111334002006, + "loss": 1.4535, + "theoretical_loss": 3.5182328147482957, + "tokens_seen": 1489633280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770912738214644, + "loss": 1.2789, + "theoretical_loss": 3.5182054855673583, + "tokens_seen": 1489764352 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027707121364092276, + "loss": 1.4475, + "theoretical_loss": 3.5181781594639663, + "tokens_seen": 1489895424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770511534603811, + "loss": 1.4361, + "theoretical_loss": 3.5181508364375023, + "tokens_seen": 1490026496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770310932798395, + "loss": 1.2875, + "theoretical_loss": 3.51812351648735, + "tokens_seen": 1490157568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770110330992979, + "loss": 1.2707, + "theoretical_loss": 3.518096199612891, + "tokens_seen": 1490288640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769909729187563, + "loss": 1.2539, + "theoretical_loss": 3.51806888581351, + "tokens_seen": 1490419712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769709127382147, + "loss": 1.3055, + "theoretical_loss": 3.51804157508859, + "tokens_seen": 1490550784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027695085255767304, + "loss": 1.3852, + "theoretical_loss": 3.5180142674375148, + "tokens_seen": 1490681856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769307923771314, + "loss": 1.4149, + "theoretical_loss": 3.517986962859667, + "tokens_seen": 1490812928 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.4123900234699249, + "objective/train/avg_token_score": 0.0059684934094548225, + "objective/train/docs_used": 845874, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.450537085533142, + "objective/train/original_loss": 3.9143619537353516, + "objective/train/theoretical_loss": 3.517959661354433, + "objective/train/tokens_used": 1490944000, + "objective/train/unlikelihood_loss": 0.15612788498401642, + "theoretical_loss": 3.517959661354433, + "tokens_seen": 1490944000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769107321965898, + "loss": 1.2365, + "theoretical_loss": 3.517959661354433, + "tokens_seen": 1490944000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768906720160482, + "loss": 1.4224, + "theoretical_loss": 3.5179323629211945, + "tokens_seen": 1491075072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027687061183550655, + "loss": 1.3199, + "theoretical_loss": 3.5179050675593375, + "tokens_seen": 1491206144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768505516549649, + "loss": 1.344, + "theoretical_loss": 3.5178777752682455, + "tokens_seen": 1491337216 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027683049147442327, + "loss": 1.4263, + "theoretical_loss": 3.517850486047304, + "tokens_seen": 1491468288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768104312938817, + "loss": 1.3897, + "theoretical_loss": 3.517823199895897, + "tokens_seen": 1491599360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027679037111334005, + "loss": 1.5021, + "theoretical_loss": 3.51779591681341, + "tokens_seen": 1491730432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767703109327984, + "loss": 1.3525, + "theoretical_loss": 3.5177686367992287, + "tokens_seen": 1491861504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027675025075225677, + "loss": 1.4015, + "theoretical_loss": 3.517741359852738, + "tokens_seen": 1491992576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767301905717152, + "loss": 1.3112, + "theoretical_loss": 3.517714085973323, + "tokens_seen": 1492123648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027671013039117355, + "loss": 1.5094, + "theoretical_loss": 3.5176868151603697, + "tokens_seen": 1492254720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766900702106319, + "loss": 1.5194, + "theoretical_loss": 3.517659547413265, + "tokens_seen": 1492385792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766700100300903, + "loss": 1.4423, + "theoretical_loss": 3.5176322827313946, + "tokens_seen": 1492516864 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.4701857268810272, + "objective/train/avg_token_score": 0.008491098880767822, + "objective/train/docs_used": 846561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5772355794906616, + "objective/train/original_loss": 4.022024154663086, + "objective/train/theoretical_loss": 3.5176186515397303, + "objective/train/tokens_used": 1492582400, + "objective/train/unlikelihood_loss": 0.14138156175613403, + "theoretical_loss": 3.5176186515397303, + "tokens_seen": 1492582400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766499498495487, + "loss": 1.3931, + "theoretical_loss": 3.517605021114144, + "tokens_seen": 1492647936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027662988966900705, + "loss": 1.2783, + "theoretical_loss": 3.5175777625609004, + "tokens_seen": 1492779008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766098294884654, + "loss": 1.3334, + "theoretical_loss": 3.51755050707105, + "tokens_seen": 1492910080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765897693079238, + "loss": 1.4751, + "theoretical_loss": 3.51752325464398, + "tokens_seen": 1493041152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027656970912738214, + "loss": 1.3877, + "theoretical_loss": 3.517496005279077, + "tokens_seen": 1493172224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027654964894684056, + "loss": 1.4307, + "theoretical_loss": 3.5174687589757285, + "tokens_seen": 1493303296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765295887662989, + "loss": 1.4733, + "theoretical_loss": 3.5174415157333216, + "tokens_seen": 1493434368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765095285857573, + "loss": 1.3654, + "theoretical_loss": 3.5174142755512445, + "tokens_seen": 1493565440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027648946840521564, + "loss": 1.3315, + "theoretical_loss": 3.5173870384288843, + "tokens_seen": 1493696512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027646940822467406, + "loss": 1.2241, + "theoretical_loss": 3.5173598043656282, + "tokens_seen": 1493827584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764493480441324, + "loss": 1.4468, + "theoretical_loss": 3.517332573360866, + "tokens_seen": 1493958656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764292878635908, + "loss": 1.4068, + "theoretical_loss": 3.517305345413984, + "tokens_seen": 1494089728 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.35899314284324646, + "objective/train/avg_token_score": 0.04272521287202835, + "objective/train/docs_used": 848095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0888538360595703, + "objective/train/original_loss": 4.4244465827941895, + "objective/train/theoretical_loss": 3.517278120524372, + "objective/train/tokens_used": 1494220800, + "objective/train/unlikelihood_loss": 0.11898501217365265, + "theoretical_loss": 3.517278120524372, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027640922768304915, + "loss": 1.1668, + "theoretical_loss": 3.517278120524372, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763891675025075, + "loss": 1.4613, + "theoretical_loss": 3.517250898691418, + "tokens_seen": 1494351872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763691073219659, + "loss": 1.3338, + "theoretical_loss": 3.517223679914511, + "tokens_seen": 1494482944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763490471414243, + "loss": 1.3733, + "theoretical_loss": 3.5171964641930398, + "tokens_seen": 1494614016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027632898696088265, + "loss": 1.4306, + "theoretical_loss": 3.5171692515263935, + "tokens_seen": 1494745088 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276308926780341, + "loss": 1.2605, + "theoretical_loss": 3.5171420419139614, + "tokens_seen": 1494876160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027628886659979943, + "loss": 1.2881, + "theoretical_loss": 3.517114835355133, + "tokens_seen": 1495007232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762688064192578, + "loss": 1.442, + "theoretical_loss": 3.517087631849298, + "tokens_seen": 1495138304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027624874623871615, + "loss": 1.3812, + "theoretical_loss": 3.5170604313958456, + "tokens_seen": 1495269376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762286860581745, + "loss": 1.3719, + "theoretical_loss": 3.517033233994167, + "tokens_seen": 1495400448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762086258776329, + "loss": 1.2878, + "theoretical_loss": 3.5170060396436504, + "tokens_seen": 1495531520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761885656970913, + "loss": 1.3827, + "theoretical_loss": 3.5169788483436886, + "tokens_seen": 1495662592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027616850551654966, + "loss": 1.32, + "theoretical_loss": 3.51695166009367, + "tokens_seen": 1495793664 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.3382209241390228, + "objective/train/avg_token_score": 0.015063409693539143, + "objective/train/docs_used": 848766, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.024458885192871, + "objective/train/original_loss": 4.176711082458496, + "objective/train/theoretical_loss": 3.5169380671122, + "objective/train/tokens_used": 1495859200, + "objective/train/unlikelihood_loss": 0.20014770328998566, + "theoretical_loss": 3.5169380671122, + "tokens_seen": 1495859200 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276148445336008, + "loss": 1.3396, + "theoretical_loss": 3.516924474892987, + "tokens_seen": 1495924736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761283851554664, + "loss": 1.4357, + "theoretical_loss": 3.5168972927410294, + "tokens_seen": 1496055808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761083249749248, + "loss": 1.351, + "theoretical_loss": 3.5168701136371885, + "tokens_seen": 1496186880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027608826479438316, + "loss": 1.4833, + "theoretical_loss": 3.516842937580855, + "tokens_seen": 1496317952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760682046138415, + "loss": 1.3326, + "theoretical_loss": 3.5168157645714215, + "tokens_seen": 1496449024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760481444332999, + "loss": 1.463, + "theoretical_loss": 3.516788594608278, + "tokens_seen": 1496580096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027602808425275825, + "loss": 1.4285, + "theoretical_loss": 3.5167614276908177, + "tokens_seen": 1496711168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027600802407221666, + "loss": 1.431, + "theoretical_loss": 3.5167342638184316, + "tokens_seen": 1496842240 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275987963891675, + "loss": 1.4486, + "theoretical_loss": 3.5167071029905124, + "tokens_seen": 1496973312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759679037111334, + "loss": 1.3238, + "theoretical_loss": 3.5166799452064517, + "tokens_seen": 1497104384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027594784353059175, + "loss": 1.4206, + "theoretical_loss": 3.5166527904656424, + "tokens_seen": 1497235456 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027592778335005017, + "loss": 1.464, + "theoretical_loss": 3.516625638767477, + "tokens_seen": 1497366528 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.37512218952178955, + "objective/train/avg_token_score": 0.016274405643343925, + "objective/train/docs_used": 849888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1306674480438232, + "objective/train/original_loss": 3.8340864181518555, + "objective/train/theoretical_loss": 3.516598490111348, + "objective/train/tokens_used": 1497497600, + "objective/train/unlikelihood_loss": 0.17891336977481842, + "theoretical_loss": 3.516598490111348, + "tokens_seen": 1497497600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027590772316950853, + "loss": 1.3877, + "theoretical_loss": 3.516598490111348, + "tokens_seen": 1497497600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758876629889669, + "loss": 1.3648, + "theoretical_loss": 3.516571344496649, + "tokens_seen": 1497628672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027586760280842525, + "loss": 1.4015, + "theoretical_loss": 3.516544201922773, + "tokens_seen": 1497759744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758475426278836, + "loss": 1.3122, + "theoretical_loss": 3.516517062389112, + "tokens_seen": 1497890816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027582748244734203, + "loss": 1.3097, + "theoretical_loss": 3.5164899258950615, + "tokens_seen": 1498021888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758074222668004, + "loss": 1.2561, + "theoretical_loss": 3.516462792440014, + "tokens_seen": 1498152960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027578736208625876, + "loss": 1.4413, + "theoretical_loss": 3.516435662023363, + "tokens_seen": 1498284032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757673019057171, + "loss": 1.3777, + "theoretical_loss": 3.5164085346445035, + "tokens_seen": 1498415104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027574724172517554, + "loss": 1.2461, + "theoretical_loss": 3.516381410302829, + "tokens_seen": 1498546176 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757271815446339, + "loss": 1.4735, + "theoretical_loss": 3.5163542889977344, + "tokens_seen": 1498677248 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027570712136409226, + "loss": 1.3364, + "theoretical_loss": 3.516327170728614, + "tokens_seen": 1498808320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756870611835506, + "loss": 1.2608, + "theoretical_loss": 3.516300055494862, + "tokens_seen": 1498939392 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275667001003009, + "loss": 1.3422, + "theoretical_loss": 3.5162729432958733, + "tokens_seen": 1499070464 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.36522483825683594, + "objective/train/avg_token_score": 0.02051565609872341, + "objective/train/docs_used": 850610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1197175979614258, + "objective/train/original_loss": 3.872785806655884, + "objective/train/theoretical_loss": 3.516259388334227, + "objective/train/tokens_used": 1499136000, + "objective/train/unlikelihood_loss": 0.1799127608537674, + "theoretical_loss": 3.516259388334227, + "tokens_seen": 1499136000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756469408224674, + "loss": 1.4242, + "theoretical_loss": 3.516245834131044, + "tokens_seen": 1499201536 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027562688064192576, + "loss": 1.1896, + "theoretical_loss": 3.516218727999768, + "tokens_seen": 1499332608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756068204613841, + "loss": 1.3343, + "theoretical_loss": 3.5161916249014413, + "tokens_seen": 1499463680 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755867602808425, + "loss": 1.2686, + "theoretical_loss": 3.51616452483546, + "tokens_seen": 1499594752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755667001003009, + "loss": 1.3068, + "theoretical_loss": 3.516137427801219, + "tokens_seen": 1499725824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027554663991975927, + "loss": 1.4108, + "theoretical_loss": 3.5161103337981143, + "tokens_seen": 1499856896 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027552657973921763, + "loss": 1.4249, + "theoretical_loss": 3.516083242825542, + "tokens_seen": 1499987968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027550651955867604, + "loss": 1.4044, + "theoretical_loss": 3.5160561548828992, + "tokens_seen": 1500119040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027548645937813446, + "loss": 1.5543, + "theoretical_loss": 3.5160290699695813, + "tokens_seen": 1500250112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754663991975928, + "loss": 1.4464, + "theoretical_loss": 3.5160019880849855, + "tokens_seen": 1500381184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754463390170512, + "loss": 1.4785, + "theoretical_loss": 3.5159749092285084, + "tokens_seen": 1500512256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027542627883650955, + "loss": 1.4438, + "theoretical_loss": 3.5159478333995464, + "tokens_seen": 1500643328 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.34909579157829285, + "objective/train/avg_token_score": 0.03833184763789177, + "objective/train/docs_used": 851359, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1282553672790527, + "objective/train/original_loss": 3.85274338722229, + "objective/train/theoretical_loss": 3.515920760597498, + "objective/train/tokens_used": 1500774400, + "objective/train/unlikelihood_loss": 0.20481900870800018, + "theoretical_loss": 3.515920760597498, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754062186559679, + "loss": 1.4088, + "theoretical_loss": 3.515920760597498, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753861584754263, + "loss": 1.4724, + "theoretical_loss": 3.515893690821759, + "tokens_seen": 1500905472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753660982948847, + "loss": 1.302, + "theoretical_loss": 3.5158666240717276, + "tokens_seen": 1501036544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027534603811434305, + "loss": 1.1978, + "theoretical_loss": 3.5158395603468016, + "tokens_seen": 1501167616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753259779338014, + "loss": 1.434, + "theoretical_loss": 3.515812499646378, + "tokens_seen": 1501298688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027530591775325983, + "loss": 1.4641, + "theoretical_loss": 3.5157854419698555, + "tokens_seen": 1501429760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752858575727182, + "loss": 1.3257, + "theoretical_loss": 3.515758387316632, + "tokens_seen": 1501560832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027526579739217655, + "loss": 1.4956, + "theoretical_loss": 3.5157313356861066, + "tokens_seen": 1501691904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752457372116349, + "loss": 1.4076, + "theoretical_loss": 3.5157042870776767, + "tokens_seen": 1501822976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752256770310933, + "loss": 1.2939, + "theoretical_loss": 3.5156772414907413, + "tokens_seen": 1501954048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752056168505517, + "loss": 1.3799, + "theoretical_loss": 3.5156501989246993, + "tokens_seen": 1502085120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027518555667001006, + "loss": 1.3513, + "theoretical_loss": 3.5156231593789498, + "tokens_seen": 1502216192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751654964894684, + "loss": 1.2399, + "theoretical_loss": 3.515596122852892, + "tokens_seen": 1502347264 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.3881353735923767, + "objective/train/avg_token_score": 0.013223916292190552, + "objective/train/docs_used": 852603, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1726398468017578, + "objective/train/original_loss": 3.6908977031707764, + "objective/train/theoretical_loss": 3.51558260572206, + "objective/train/tokens_used": 1502412800, + "objective/train/unlikelihood_loss": 0.158029243350029, + "theoretical_loss": 3.51558260572206, + "tokens_seen": 1502412800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751454363089268, + "loss": 1.5589, + "theoretical_loss": 3.5155690893459255, + "tokens_seen": 1502478336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751253761283852, + "loss": 1.3646, + "theoretical_loss": 3.515542058857449, + "tokens_seen": 1502609408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027510531594784356, + "loss": 1.2178, + "theoretical_loss": 3.515515031386863, + "tokens_seen": 1502740480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750852557673019, + "loss": 1.357, + "theoretical_loss": 3.515488006933567, + "tokens_seen": 1502871552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750651955867603, + "loss": 1.4428, + "theoretical_loss": 3.5154609854969614, + "tokens_seen": 1503002624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027504513540621865, + "loss": 1.4738, + "theoretical_loss": 3.5154339670764463, + "tokens_seen": 1503133696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027502507522567706, + "loss": 1.5132, + "theoretical_loss": 3.515406951671422, + "tokens_seen": 1503264768 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750050150451354, + "loss": 1.4869, + "theoretical_loss": 3.5153799392812886, + "tokens_seen": 1503395840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749849548645938, + "loss": 1.315, + "theoretical_loss": 3.515352929905448, + "tokens_seen": 1503526912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027496489468405215, + "loss": 1.2761, + "theoretical_loss": 3.5153259235432994, + "tokens_seen": 1503657984 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027494483450351057, + "loss": 1.4229, + "theoretical_loss": 3.5152989201942457, + "tokens_seen": 1503789056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027492477432296893, + "loss": 1.2781, + "theoretical_loss": 3.515271919857687, + "tokens_seen": 1503920128 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.4008431136608124, + "objective/train/avg_token_score": 0.06508415937423706, + "objective/train/docs_used": 853123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2080678939819336, + "objective/train/original_loss": 3.6112747192382812, + "objective/train/theoretical_loss": 3.515244922533025, + "objective/train/tokens_used": 1504051200, + "objective/train/unlikelihood_loss": 0.16351859271526337, + "theoretical_loss": 3.515244922533025, + "tokens_seen": 1504051200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749047141424273, + "loss": 1.392, + "theoretical_loss": 3.515244922533025, + "tokens_seen": 1504051200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027488465396188565, + "loss": 1.284, + "theoretical_loss": 3.5152179282196614, + "tokens_seen": 1504182272 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274864593781344, + "loss": 1.2175, + "theoretical_loss": 3.5151909369169982, + "tokens_seen": 1504313344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027484453360080243, + "loss": 1.1671, + "theoretical_loss": 3.5151639486244366, + "tokens_seen": 1504444416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002748244734202608, + "loss": 1.363, + "theoretical_loss": 3.515136963341379, + "tokens_seen": 1504575488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027480441323971916, + "loss": 1.3599, + "theoretical_loss": 3.515109981067228, + "tokens_seen": 1504706560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747843530591775, + "loss": 1.4193, + "theoretical_loss": 3.5150830018013863, + "tokens_seen": 1504837632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027476429287863594, + "loss": 1.396, + "theoretical_loss": 3.515056025543256, + "tokens_seen": 1504968704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747442326980943, + "loss": 1.3849, + "theoretical_loss": 3.515029052292239, + "tokens_seen": 1505099776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027472417251755266, + "loss": 1.3775, + "theoretical_loss": 3.51500208204774, + "tokens_seen": 1505230848 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274704112337011, + "loss": 1.3505, + "theoretical_loss": 3.5149751148091615, + "tokens_seen": 1505361920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746840521564694, + "loss": 1.3196, + "theoretical_loss": 3.5149481505759064, + "tokens_seen": 1505492992 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746639919759278, + "loss": 1.3632, + "theoretical_loss": 3.5149211893473784, + "tokens_seen": 1505624064 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.45998290181159973, + "objective/train/avg_token_score": 0.00821203738451004, + "objective/train/docs_used": 854391, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.563668966293335, + "objective/train/original_loss": 3.854599714279175, + "objective/train/theoretical_loss": 3.514907709859701, + "objective/train/tokens_used": 1505689600, + "objective/train/unlikelihood_loss": 0.1520872563123703, + "theoretical_loss": 3.514907709859701, + "tokens_seen": 1505689600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027464393179538616, + "loss": 1.3985, + "theoretical_loss": 3.514894231122981, + "tokens_seen": 1505755136 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746238716148445, + "loss": 1.2599, + "theoretical_loss": 3.5148672759021182, + "tokens_seen": 1505886208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746038114343029, + "loss": 1.5408, + "theoretical_loss": 3.514840323684194, + "tokens_seen": 1506017280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745837512537613, + "loss": 1.2173, + "theoretical_loss": 3.5148133744686127, + "tokens_seen": 1506148352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027456369107321967, + "loss": 1.3394, + "theoretical_loss": 3.5147864282547783, + "tokens_seen": 1506279424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027454363089267803, + "loss": 1.288, + "theoretical_loss": 3.514759485042095, + "tokens_seen": 1506410496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745235707121364, + "loss": 1.2254, + "theoretical_loss": 3.514732544829968, + "tokens_seen": 1506541568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027450351053159475, + "loss": 1.174, + "theoretical_loss": 3.5147056076178025, + "tokens_seen": 1506672640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027448345035105317, + "loss": 1.2718, + "theoretical_loss": 3.514678673405003, + "tokens_seen": 1506803712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027446339017051153, + "loss": 1.314, + "theoretical_loss": 3.514651742190974, + "tokens_seen": 1506934784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744433299899699, + "loss": 1.5384, + "theoretical_loss": 3.5146248139751215, + "tokens_seen": 1507065856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027442326980942826, + "loss": 1.3054, + "theoretical_loss": 3.5145978887568514, + "tokens_seen": 1507196928 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.36870723962783813, + "objective/train/avg_token_score": 0.006511629559099674, + "objective/train/docs_used": 855086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1450027227401733, + "objective/train/original_loss": 3.8253138065338135, + "objective/train/theoretical_loss": 3.514570966535569, + "objective/train/tokens_used": 1507328000, + "objective/train/unlikelihood_loss": 0.15097831189632416, + "theoretical_loss": 3.514570966535569, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027440320962888667, + "loss": 1.3869, + "theoretical_loss": 3.514570966535569, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027438314944834503, + "loss": 1.3238, + "theoretical_loss": 3.51454404731068, + "tokens_seen": 1507459072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743630892678034, + "loss": 1.3439, + "theoretical_loss": 3.5145171310815906, + "tokens_seen": 1507590144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027434302908726176, + "loss": 1.3529, + "theoretical_loss": 3.514490217847707, + "tokens_seen": 1507721216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743229689067202, + "loss": 1.432, + "theoretical_loss": 3.514463307608435, + "tokens_seen": 1507852288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027430290872617854, + "loss": 1.3189, + "theoretical_loss": 3.5144364003631825, + "tokens_seen": 1507983360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742828485456369, + "loss": 1.3611, + "theoretical_loss": 3.5144094961113543, + "tokens_seen": 1508114432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027426278836509526, + "loss": 1.3276, + "theoretical_loss": 3.514382594852359, + "tokens_seen": 1508245504 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742427281845536, + "loss": 1.2738, + "theoretical_loss": 3.5143556965856027, + "tokens_seen": 1508376576 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027422266800401204, + "loss": 1.4431, + "theoretical_loss": 3.5143288013104925, + "tokens_seen": 1508507648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742026078234704, + "loss": 1.3832, + "theoretical_loss": 3.514301909026437, + "tokens_seen": 1508638720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027418254764292877, + "loss": 1.4085, + "theoretical_loss": 3.5142750197328416, + "tokens_seen": 1508769792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027416248746238713, + "loss": 1.4268, + "theoretical_loss": 3.514248133429116, + "tokens_seen": 1508900864 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.3487292230129242, + "objective/train/avg_token_score": 0.03324980288743973, + "objective/train/docs_used": 856541, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1456804275512695, + "objective/train/original_loss": 3.7558488845825195, + "objective/train/theoretical_loss": 3.514234691398269, + "objective/train/tokens_used": 1508966400, + "objective/train/unlikelihood_loss": 0.21186338365077972, + "theoretical_loss": 3.514234691398269, + "tokens_seen": 1508966400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027414242728184554, + "loss": 1.3246, + "theoretical_loss": 3.514221250114667, + "tokens_seen": 1509031936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741223671013039, + "loss": 1.4131, + "theoretical_loss": 3.514194369788903, + "tokens_seen": 1509163008 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027410230692076227, + "loss": 1.3363, + "theoretical_loss": 3.5141674924512323, + "tokens_seen": 1509294080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027408224674022063, + "loss": 1.2921, + "theoretical_loss": 3.514140618101063, + "tokens_seen": 1509425152 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274062186559679, + "loss": 1.2504, + "theoretical_loss": 3.514113746737804, + "tokens_seen": 1509556224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740421263791374, + "loss": 1.5474, + "theoretical_loss": 3.514086878360863, + "tokens_seen": 1509687296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740220661985958, + "loss": 1.2299, + "theoretical_loss": 3.514060012969651, + "tokens_seen": 1509818368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740020060180542, + "loss": 1.1685, + "theoretical_loss": 3.5140331505635745, + "tokens_seen": 1509949440 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027398194583751255, + "loss": 1.5197, + "theoretical_loss": 3.514006291142045, + "tokens_seen": 1510080512 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027396188565697097, + "loss": 1.4118, + "theoretical_loss": 3.5139794347044706, + "tokens_seen": 1510211584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027394182547642933, + "loss": 1.3581, + "theoretical_loss": 3.5139525812502606, + "tokens_seen": 1510342656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739217652958877, + "loss": 1.3248, + "theoretical_loss": 3.5139257307788254, + "tokens_seen": 1510473728 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.3256353735923767, + "objective/train/avg_token_score": 0.015720082446932793, + "objective/train/docs_used": 857299, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0216853618621826, + "objective/train/original_loss": 3.7378976345062256, + "objective/train/theoretical_loss": 3.513898883289575, + "objective/train/tokens_used": 1510604800, + "objective/train/unlikelihood_loss": 0.18417879939079285, + "theoretical_loss": 3.513898883289575, + "tokens_seen": 1510604800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027390170511534605, + "loss": 1.2078, + "theoretical_loss": 3.513898883289575, + "tokens_seen": 1510604800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738816449348044, + "loss": 1.3113, + "theoretical_loss": 3.5138720387819196, + "tokens_seen": 1510735872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027386158475426283, + "loss": 1.3889, + "theoretical_loss": 3.513845197255269, + "tokens_seen": 1510866944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738415245737212, + "loss": 1.3773, + "theoretical_loss": 3.513818358709033, + "tokens_seen": 1510998016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027382146439317956, + "loss": 1.4285, + "theoretical_loss": 3.5137915231426233, + "tokens_seen": 1511129088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738014042126379, + "loss": 1.4043, + "theoretical_loss": 3.51376469055545, + "tokens_seen": 1511260160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027378134403209634, + "loss": 1.3195, + "theoretical_loss": 3.5137378609469243, + "tokens_seen": 1511391232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737612838515547, + "loss": 1.4105, + "theoretical_loss": 3.513711034316457, + "tokens_seen": 1511522304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027374122367101306, + "loss": 1.3022, + "theoretical_loss": 3.5136842106634596, + "tokens_seen": 1511653376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737211634904714, + "loss": 1.3186, + "theoretical_loss": 3.513657389987343, + "tokens_seen": 1511784448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737011033099298, + "loss": 1.2234, + "theoretical_loss": 3.5136305722875196, + "tokens_seen": 1511915520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736810431293882, + "loss": 1.32, + "theoretical_loss": 3.5136037575634003, + "tokens_seen": 1512046592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027366098294884656, + "loss": 1.2704, + "theoretical_loss": 3.513576945814398, + "tokens_seen": 1512177664 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.42515885829925537, + "objective/train/avg_token_score": 0.01439465582370758, + "objective/train/docs_used": 858636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2745542526245117, + "objective/train/original_loss": 3.7984540462493896, + "objective/train/theoretical_loss": 3.5135635410553814, + "objective/train/tokens_used": 1512243200, + "objective/train/unlikelihood_loss": 0.19316226243972778, + "theoretical_loss": 3.5135635410553814, + "tokens_seen": 1512243200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736409227683049, + "loss": 1.4058, + "theoretical_loss": 3.513550137039924, + "tokens_seen": 1512308736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736208625877633, + "loss": 1.353, + "theoretical_loss": 3.5135233312393903, + "tokens_seen": 1512439808 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736008024072217, + "loss": 1.4296, + "theoretical_loss": 3.51349652841221, + "tokens_seen": 1512570880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027358074222668007, + "loss": 1.3181, + "theoretical_loss": 3.513469728557795, + "tokens_seen": 1512701952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027356068204613843, + "loss": 1.2863, + "theoretical_loss": 3.5134429316755593, + "tokens_seen": 1512833024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735406218655968, + "loss": 1.3911, + "theoretical_loss": 3.5134161377649145, + "tokens_seen": 1512964096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027352056168505515, + "loss": 1.3969, + "theoretical_loss": 3.513389346825274, + "tokens_seen": 1513095168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027350050150451357, + "loss": 1.4125, + "theoretical_loss": 3.513362558856051, + "tokens_seen": 1513226240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027348044132397193, + "loss": 1.5263, + "theoretical_loss": 3.5133357738566597, + "tokens_seen": 1513357312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734603811434303, + "loss": 1.4482, + "theoretical_loss": 3.513308991826513, + "tokens_seen": 1513488384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027344032096288866, + "loss": 1.2603, + "theoretical_loss": 3.513282212765024, + "tokens_seen": 1513619456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734202607823471, + "loss": 1.3302, + "theoretical_loss": 3.5132554366716082, + "tokens_seen": 1513750528 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.5330523252487183, + "objective/train/avg_token_score": 0.0025684942957013845, + "objective/train/docs_used": 859135, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7641527652740479, + "objective/train/original_loss": 3.303588628768921, + "objective/train/theoretical_loss": 3.513228663545678, + "objective/train/tokens_used": 1513881600, + "objective/train/unlikelihood_loss": 0.23598985373973846, + "theoretical_loss": 3.513228663545678, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027340020060180544, + "loss": 1.3463, + "theoretical_loss": 3.513228663545678, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733801404212638, + "loss": 1.3981, + "theoretical_loss": 3.513201893386649, + "tokens_seen": 1514012672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027336008024072216, + "loss": 1.3792, + "theoretical_loss": 3.513175126193935, + "tokens_seen": 1514143744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733400200601806, + "loss": 1.3729, + "theoretical_loss": 3.5131483619669504, + "tokens_seen": 1514274816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027331995987963894, + "loss": 1.2561, + "theoretical_loss": 3.5131216007051105, + "tokens_seen": 1514405888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732998996990973, + "loss": 1.4056, + "theoretical_loss": 3.5130948424078294, + "tokens_seen": 1514536960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027327983951855566, + "loss": 1.4119, + "theoretical_loss": 3.5130680870745232, + "tokens_seen": 1514668032 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273259779338014, + "loss": 1.4413, + "theoretical_loss": 3.5130413347046066, + "tokens_seen": 1514799104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027323971915747244, + "loss": 1.3179, + "theoretical_loss": 3.513014585297494, + "tokens_seen": 1514930176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732196589769308, + "loss": 1.4825, + "theoretical_loss": 3.5129878388526032, + "tokens_seen": 1515061248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027319959879638917, + "loss": 1.3117, + "theoretical_loss": 3.5129610953693478, + "tokens_seen": 1515192320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027317953861584753, + "loss": 1.2817, + "theoretical_loss": 3.512934354847145, + "tokens_seen": 1515323392 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027315947843530594, + "loss": 1.4097, + "theoretical_loss": 3.5129076172854106, + "tokens_seen": 1515454464 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.3603372573852539, + "objective/train/avg_token_score": 0.016839275136590004, + "objective/train/docs_used": 860397, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.067925214767456, + "objective/train/original_loss": 3.3837544918060303, + "objective/train/theoretical_loss": 3.5128942496145363, + "objective/train/tokens_used": 1515520000, + "objective/train/unlikelihood_loss": 0.2863500714302063, + "theoretical_loss": 3.5128942496145363, + "tokens_seen": 1515520000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731394182547643, + "loss": 1.3113, + "theoretical_loss": 3.5128808826835605, + "tokens_seen": 1515585536 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027311935807422267, + "loss": 1.2207, + "theoretical_loss": 3.5128541510410116, + "tokens_seen": 1515716608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027309929789368103, + "loss": 1.3515, + "theoretical_loss": 3.51282742235718, + "tokens_seen": 1515847680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730792377131394, + "loss": 1.4558, + "theoretical_loss": 3.5128006966314826, + "tokens_seen": 1515978752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730591775325978, + "loss": 1.4634, + "theoretical_loss": 3.512773973863336, + "tokens_seen": 1516109824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027303911735205617, + "loss": 1.4459, + "theoretical_loss": 3.512747254052158, + "tokens_seen": 1516240896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027301905717151453, + "loss": 1.3708, + "theoretical_loss": 3.512720537197365, + "tokens_seen": 1516371968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729989969909729, + "loss": 1.2834, + "theoretical_loss": 3.512693823298375, + "tokens_seen": 1516503040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729789368104313, + "loss": 1.3476, + "theoretical_loss": 3.5126671123546056, + "tokens_seen": 1516634112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729588766298897, + "loss": 1.4761, + "theoretical_loss": 3.5126404043654738, + "tokens_seen": 1516765184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027293881644934804, + "loss": 1.5425, + "theoretical_loss": 3.5126136993303985, + "tokens_seen": 1516896256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729187562688064, + "loss": 1.3211, + "theoretical_loss": 3.512586997248797, + "tokens_seen": 1517027328 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.3534335196018219, + "objective/train/avg_token_score": 0.009286854416131973, + "objective/train/docs_used": 860980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1767507791519165, + "objective/train/original_loss": 4.210026741027832, + "objective/train/theoretical_loss": 3.5125602981200874, + "objective/train/tokens_used": 1517158400, + "objective/train/unlikelihood_loss": 0.1728946715593338, + "theoretical_loss": 3.5125602981200874, + "tokens_seen": 1517158400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027289869608826476, + "loss": 1.389, + "theoretical_loss": 3.5125602981200874, + "tokens_seen": 1517158400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728786359077232, + "loss": 1.3839, + "theoretical_loss": 3.512533601943689, + "tokens_seen": 1517289472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027285857572718154, + "loss": 1.457, + "theoretical_loss": 3.5125069087190193, + "tokens_seen": 1517420544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728385155466399, + "loss": 1.2994, + "theoretical_loss": 3.5124802184454973, + "tokens_seen": 1517551616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027281845536609827, + "loss": 1.4067, + "theoretical_loss": 3.512453531122542, + "tokens_seen": 1517682688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727983951855567, + "loss": 1.2275, + "theoretical_loss": 3.512426846749573, + "tokens_seen": 1517813760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027277833500501504, + "loss": 1.4609, + "theoretical_loss": 3.5124001653260093, + "tokens_seen": 1517944832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727582748244734, + "loss": 1.3836, + "theoretical_loss": 3.5123734868512693, + "tokens_seen": 1518075904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027273821464393177, + "loss": 1.4131, + "theoretical_loss": 3.5123468113247736, + "tokens_seen": 1518206976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027271815446339013, + "loss": 1.3699, + "theoretical_loss": 3.512320138745941, + "tokens_seen": 1518338048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027269809428284855, + "loss": 1.3567, + "theoretical_loss": 3.512293469114192, + "tokens_seen": 1518469120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726780341023069, + "loss": 1.4744, + "theoretical_loss": 3.5122668024289467, + "tokens_seen": 1518600192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027265797392176527, + "loss": 1.3353, + "theoretical_loss": 3.512240138689625, + "tokens_seen": 1518731264 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.4055474102497101, + "objective/train/avg_token_score": 0.015353968366980553, + "objective/train/docs_used": 862373, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.21696937084198, + "objective/train/original_loss": 4.127537250518799, + "objective/train/theoretical_loss": 3.5122268079245043, + "objective/train/tokens_used": 1518796800, + "objective/train/unlikelihood_loss": 0.16004537045955658, + "theoretical_loss": 3.5122268079245043, + "tokens_seen": 1518796800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027263791374122363, + "loss": 1.2804, + "theoretical_loss": 3.5122134778956475, + "tokens_seen": 1518862336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027261785356068205, + "loss": 1.5079, + "theoretical_loss": 3.512186820046434, + "tokens_seen": 1518993408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725977933801404, + "loss": 1.426, + "theoretical_loss": 3.512160165141406, + "tokens_seen": 1519124480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725777331995988, + "loss": 1.3461, + "theoretical_loss": 3.5121335131799847, + "tokens_seen": 1519255552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027255767301905714, + "loss": 1.3251, + "theoretical_loss": 3.5121068641615896, + "tokens_seen": 1519386624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725376128385155, + "loss": 1.4085, + "theoretical_loss": 3.5120802180856434, + "tokens_seen": 1519517696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027251755265797397, + "loss": 1.3433, + "theoretical_loss": 3.5120535749515667, + "tokens_seen": 1519648768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027249749247743233, + "loss": 1.3303, + "theoretical_loss": 3.512026934758781, + "tokens_seen": 1519779840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724774322968907, + "loss": 1.338, + "theoretical_loss": 3.5120002975067077, + "tokens_seen": 1519910912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027245737211634906, + "loss": 1.4381, + "theoretical_loss": 3.5119736631947696, + "tokens_seen": 1520041984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724373119358075, + "loss": 1.4309, + "theoretical_loss": 3.511947031822388, + "tokens_seen": 1520173056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027241725175526584, + "loss": 1.4457, + "theoretical_loss": 3.5119204033889853, + "tokens_seen": 1520304128 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.39442816376686096, + "objective/train/avg_token_score": 0.01630260981619358, + "objective/train/docs_used": 862945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2706034183502197, + "objective/train/original_loss": 4.043051242828369, + "objective/train/theoretical_loss": 3.5118937778939836, + "objective/train/tokens_used": 1520435200, + "objective/train/unlikelihood_loss": 0.1544790416955948, + "theoretical_loss": 3.5118937778939836, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723971915747242, + "loss": 1.3058, + "theoretical_loss": 3.5118937778939836, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027237713139418256, + "loss": 1.4438, + "theoretical_loss": 3.5118671553368053, + "tokens_seen": 1520566272 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272357071213641, + "loss": 1.2969, + "theoretical_loss": 3.511840535716873, + "tokens_seen": 1520697344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027233701103309934, + "loss": 1.2501, + "theoretical_loss": 3.51181391903361, + "tokens_seen": 1520828416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723169508525577, + "loss": 1.3521, + "theoretical_loss": 3.5117873052864392, + "tokens_seen": 1520959488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027229689067201606, + "loss": 1.3902, + "theoretical_loss": 3.5117606944747832, + "tokens_seen": 1521090560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722768304914744, + "loss": 1.4633, + "theoretical_loss": 3.511734086598066, + "tokens_seen": 1521221632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027225677031093284, + "loss": 1.3573, + "theoretical_loss": 3.5117074816557103, + "tokens_seen": 1521352704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722367101303912, + "loss": 1.2798, + "theoretical_loss": 3.51168087964714, + "tokens_seen": 1521483776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027221664994984957, + "loss": 1.5099, + "theoretical_loss": 3.5116542805717796, + "tokens_seen": 1521614848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027219658976930793, + "loss": 1.4172, + "theoretical_loss": 3.5116276844290524, + "tokens_seen": 1521745920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027217652958876635, + "loss": 1.2893, + "theoretical_loss": 3.511601091218382, + "tokens_seen": 1521876992 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721564694082247, + "loss": 1.4479, + "theoretical_loss": 3.5115745009391937, + "tokens_seen": 1522008064 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.5047653913497925, + "objective/train/avg_token_score": 0.004550029523670673, + "objective/train/docs_used": 863367, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5946991443634033, + "objective/train/original_loss": 3.8085339069366455, + "objective/train/theoretical_loss": 3.5115612068987256, + "objective/train/tokens_used": 1522073600, + "objective/train/unlikelihood_loss": 0.1295635849237442, + "theoretical_loss": 3.5115612068987256, + "tokens_seen": 1522073600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027213640922768307, + "loss": 1.4921, + "theoretical_loss": 3.5115479135909116, + "tokens_seen": 1522139136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027211634904714143, + "loss": 1.3162, + "theoretical_loss": 3.51152132917296, + "tokens_seen": 1522270208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720962888665998, + "loss": 1.4059, + "theoretical_loss": 3.5114947476847638, + "tokens_seen": 1522401280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720762286860582, + "loss": 1.4162, + "theoretical_loss": 3.5114681691257483, + "tokens_seen": 1522532352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027205616850551657, + "loss": 1.3672, + "theoretical_loss": 3.511441593495338, + "tokens_seen": 1522663424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027203610832497493, + "loss": 1.3277, + "theoretical_loss": 3.511415020792959, + "tokens_seen": 1522794496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720160481444333, + "loss": 1.3386, + "theoretical_loss": 3.5113884510180355, + "tokens_seen": 1522925568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719959879638917, + "loss": 1.2575, + "theoretical_loss": 3.5113618841699936, + "tokens_seen": 1523056640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719759277833501, + "loss": 1.4819, + "theoretical_loss": 3.51133532024826, + "tokens_seen": 1523187712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027195586760280844, + "loss": 1.3481, + "theoretical_loss": 3.511308759252259, + "tokens_seen": 1523318784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719358074222668, + "loss": 1.2959, + "theoretical_loss": 3.5112822011814173, + "tokens_seen": 1523449856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027191574724172516, + "loss": 1.2104, + "theoretical_loss": 3.5112556460351616, + "tokens_seen": 1523580928 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.44183772802352905, + "objective/train/avg_token_score": 0.0069322604686021805, + "objective/train/docs_used": 864510, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3821964263916016, + "objective/train/original_loss": 3.9686009883880615, + "objective/train/theoretical_loss": 3.5112290938129176, + "objective/train/tokens_used": 1523712000, + "objective/train/unlikelihood_loss": 0.1749648004770279, + "theoretical_loss": 3.5112290938129176, + "tokens_seen": 1523712000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718956870611836, + "loss": 1.322, + "theoretical_loss": 3.5112290938129176, + "tokens_seen": 1523712000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027187562688064194, + "loss": 1.4597, + "theoretical_loss": 3.511202544514113, + "tokens_seen": 1523843072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718555667001003, + "loss": 1.4408, + "theoretical_loss": 3.5111759981381727, + "tokens_seen": 1523974144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027183550651955867, + "loss": 1.3916, + "theoretical_loss": 3.511149454684525, + "tokens_seen": 1524105216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718154463390171, + "loss": 1.3667, + "theoretical_loss": 3.5111229141525966, + "tokens_seen": 1524236288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027179538615847544, + "loss": 1.4578, + "theoretical_loss": 3.511096376541815, + "tokens_seen": 1524367360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717753259779338, + "loss": 1.2238, + "theoretical_loss": 3.511069841851606, + "tokens_seen": 1524498432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027175526579739217, + "loss": 1.4557, + "theoretical_loss": 3.5110433100813996, + "tokens_seen": 1524629504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027173520561685053, + "loss": 1.2064, + "theoretical_loss": 3.511016781230621, + "tokens_seen": 1524760576 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027171514543630895, + "loss": 1.3311, + "theoretical_loss": 3.5109902552987, + "tokens_seen": 1524891648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716950852557673, + "loss": 1.4133, + "theoretical_loss": 3.510963732285063, + "tokens_seen": 1525022720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027167502507522567, + "loss": 1.3656, + "theoretical_loss": 3.51093721218914, + "tokens_seen": 1525153792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027165496489468403, + "loss": 1.3699, + "theoretical_loss": 3.5109106950103577, + "tokens_seen": 1525284864 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.4868646264076233, + "objective/train/avg_token_score": 0.01318820845335722, + "objective/train/docs_used": 865194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.519594430923462, + "objective/train/original_loss": 3.7058987617492676, + "objective/train/theoretical_loss": 3.5108974375147155, + "objective/train/tokens_used": 1525350400, + "objective/train/unlikelihood_loss": 0.13879793882369995, + "theoretical_loss": 3.5108974375147155, + "tokens_seen": 1525350400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027163490471414245, + "loss": 1.3507, + "theoretical_loss": 3.510884180748145, + "tokens_seen": 1525415936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716148445336008, + "loss": 1.3083, + "theoretical_loss": 3.510857669401931, + "tokens_seen": 1525547008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715947843530592, + "loss": 1.4316, + "theoretical_loss": 3.5108311609711436, + "tokens_seen": 1525678080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027157472417251754, + "loss": 1.4426, + "theoretical_loss": 3.510804655455213, + "tokens_seen": 1525809152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715546639919759, + "loss": 1.1994, + "theoretical_loss": 3.5107781528535673, + "tokens_seen": 1525940224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715346038114343, + "loss": 1.5472, + "theoretical_loss": 3.510751653165636, + "tokens_seen": 1526071296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715145436308927, + "loss": 1.3105, + "theoretical_loss": 3.5107251563908495, + "tokens_seen": 1526202368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027149448345035104, + "loss": 1.3898, + "theoretical_loss": 3.5106986625286365, + "tokens_seen": 1526333440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714744232698094, + "loss": 1.3775, + "theoretical_loss": 3.5106721715784266, + "tokens_seen": 1526464512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714543630892678, + "loss": 1.2385, + "theoretical_loss": 3.5106456835396505, + "tokens_seen": 1526595584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714343029087262, + "loss": 1.3579, + "theoretical_loss": 3.510619198411738, + "tokens_seen": 1526726656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027141424272818454, + "loss": 1.3366, + "theoretical_loss": 3.510592716194118, + "tokens_seen": 1526857728 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.5571236610412598, + "objective/train/avg_token_score": 0.008076293393969536, + "objective/train/docs_used": 866480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7066460847854614, + "objective/train/original_loss": 3.3231730461120605, + "objective/train/theoretical_loss": 3.5105662368862234, + "objective/train/tokens_used": 1526988800, + "objective/train/unlikelihood_loss": 0.14562788605690002, + "theoretical_loss": 3.5105662368862234, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713941825476429, + "loss": 1.4795, + "theoretical_loss": 3.5105662368862234, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027137412236710127, + "loss": 1.2676, + "theoretical_loss": 3.5105397604874833, + "tokens_seen": 1527119872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713540621865597, + "loss": 1.3975, + "theoretical_loss": 3.510513286997328, + "tokens_seen": 1527250944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027133400200601805, + "loss": 1.3016, + "theoretical_loss": 3.5104868164151894, + "tokens_seen": 1527382016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713139418254764, + "loss": 1.4072, + "theoretical_loss": 3.510460348740498, + "tokens_seen": 1527513088 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027129388164493477, + "loss": 1.2563, + "theoretical_loss": 3.510433883972686, + "tokens_seen": 1527644160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712738214643932, + "loss": 1.4034, + "theoretical_loss": 3.510407422111183, + "tokens_seen": 1527775232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027125376128385155, + "loss": 1.5467, + "theoretical_loss": 3.5103809631554217, + "tokens_seen": 1527906304 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712337011033099, + "loss": 1.2076, + "theoretical_loss": 3.510354507104834, + "tokens_seen": 1528037376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712136409227683, + "loss": 1.3611, + "theoretical_loss": 3.510328053958851, + "tokens_seen": 1528168448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711935807422267, + "loss": 1.2749, + "theoretical_loss": 3.5103016037169055, + "tokens_seen": 1528299520 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027117352056168505, + "loss": 1.232, + "theoretical_loss": 3.510275156378429, + "tokens_seen": 1528430592 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711534603811434, + "loss": 1.3416, + "theoretical_loss": 3.510248711942854, + "tokens_seen": 1528561664 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.3853861093521118, + "objective/train/avg_token_score": 0.011287224479019642, + "objective/train/docs_used": 867112, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1327481269836426, + "objective/train/original_loss": 3.5473461151123047, + "objective/train/theoretical_loss": 3.5102354908134776, + "objective/train/tokens_used": 1528627200, + "objective/train/unlikelihood_loss": 0.17457419633865356, + "theoretical_loss": 3.5102354908134776, + "tokens_seen": 1528627200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711334002006018, + "loss": 1.1815, + "theoretical_loss": 3.510222270409613, + "tokens_seen": 1528692736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027111334002006014, + "loss": 1.515, + "theoretical_loss": 3.510195831778139, + "tokens_seen": 1528823808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027109327983951856, + "loss": 1.5553, + "theoretical_loss": 3.5101693960478646, + "tokens_seen": 1528954880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710732196589769, + "loss": 1.4471, + "theoretical_loss": 3.5101429632182226, + "tokens_seen": 1529085952 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710531594784353, + "loss": 1.35, + "theoretical_loss": 3.510116533288646, + "tokens_seen": 1529217024 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027103309929789364, + "loss": 1.5207, + "theoretical_loss": 3.510090106258569, + "tokens_seen": 1529348096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710130391173521, + "loss": 1.0707, + "theoretical_loss": 3.5100636821274236, + "tokens_seen": 1529479168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709929789368105, + "loss": 1.2925, + "theoretical_loss": 3.510037260894645, + "tokens_seen": 1529610240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027097291875626884, + "loss": 1.1557, + "theoretical_loss": 3.510010842559666, + "tokens_seen": 1529741312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709528585757272, + "loss": 1.4574, + "theoretical_loss": 3.5099844271219203, + "tokens_seen": 1529872384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027093279839518556, + "loss": 1.5013, + "theoretical_loss": 3.5099580145808424, + "tokens_seen": 1530003456 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270912738214644, + "loss": 1.2586, + "theoretical_loss": 3.5099316049358666, + "tokens_seen": 1530134528 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.39839932322502136, + "objective/train/avg_token_score": 0.009560000151395798, + "objective/train/docs_used": 868323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1761577129364014, + "objective/train/original_loss": 3.7391517162323, + "objective/train/theoretical_loss": 3.5099051981864275, + "objective/train/tokens_used": 1530265600, + "objective/train/unlikelihood_loss": 0.16838152706623077, + "theoretical_loss": 3.5099051981864275, + "tokens_seen": 1530265600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027089267803410234, + "loss": 1.314, + "theoretical_loss": 3.5099051981864275, + "tokens_seen": 1530265600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708726178535607, + "loss": 1.4209, + "theoretical_loss": 3.509878794331959, + "tokens_seen": 1530396672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027085255767301907, + "loss": 1.4389, + "theoretical_loss": 3.5098523933718964, + "tokens_seen": 1530527744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708324974924775, + "loss": 1.4397, + "theoretical_loss": 3.5098259953056745, + "tokens_seen": 1530658816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027081243731193584, + "loss": 1.3753, + "theoretical_loss": 3.5097996001327285, + "tokens_seen": 1530789888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707923771313942, + "loss": 1.4788, + "theoretical_loss": 3.509773207852493, + "tokens_seen": 1530920960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027077231695085257, + "loss": 1.5326, + "theoretical_loss": 3.509746818464404, + "tokens_seen": 1531052032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027075225677031093, + "loss": 1.4, + "theoretical_loss": 3.5097204319678967, + "tokens_seen": 1531183104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027073219658976935, + "loss": 1.2902, + "theoretical_loss": 3.509694048362406, + "tokens_seen": 1531314176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707121364092277, + "loss": 1.2057, + "theoretical_loss": 3.5096676676473697, + "tokens_seen": 1531445248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027069207622868607, + "loss": 1.2817, + "theoretical_loss": 3.509641289822222, + "tokens_seen": 1531576320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027067201604814443, + "loss": 1.2558, + "theoretical_loss": 3.5096149148864, + "tokens_seen": 1531707392 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027065195586760285, + "loss": 1.0744, + "theoretical_loss": 3.5095885428393396, + "tokens_seen": 1531838464 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.333272248506546, + "objective/train/avg_token_score": 0.025640349835157394, + "objective/train/docs_used": 869121, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9849498271942139, + "objective/train/original_loss": 4.778568267822266, + "objective/train/theoretical_loss": 3.5095753578989184, + "objective/train/tokens_used": 1531904000, + "objective/train/unlikelihood_loss": 0.0939873456954956, + "theoretical_loss": 3.5095753578989184, + "tokens_seen": 1531904000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706318956870612, + "loss": 1.2439, + "theoretical_loss": 3.509562173680477, + "tokens_seen": 1531969536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706118355065196, + "loss": 1.4376, + "theoretical_loss": 3.5095358074092493, + "tokens_seen": 1532100608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027059177532597794, + "loss": 1.4137, + "theoretical_loss": 3.5095094440250936, + "tokens_seen": 1532231680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705717151454363, + "loss": 1.4156, + "theoretical_loss": 3.5094830835274458, + "tokens_seen": 1532362752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705516549648947, + "loss": 1.4525, + "theoretical_loss": 3.509456725915744, + "tokens_seen": 1532493824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705315947843531, + "loss": 1.4157, + "theoretical_loss": 3.509430371189425, + "tokens_seen": 1532624896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027051153460381144, + "loss": 1.2636, + "theoretical_loss": 3.509404019347926, + "tokens_seen": 1532755968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704914744232698, + "loss": 1.2004, + "theoretical_loss": 3.509377670390685, + "tokens_seen": 1532887040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704714142427282, + "loss": 1.4057, + "theoretical_loss": 3.50935132431714, + "tokens_seen": 1533018112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704513540621866, + "loss": 1.3756, + "theoretical_loss": 3.5093249811267277, + "tokens_seen": 1533149184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027043129388164494, + "loss": 1.3411, + "theoretical_loss": 3.509298640818888, + "tokens_seen": 1533280256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704112337011033, + "loss": 1.2797, + "theoretical_loss": 3.5092723033930566, + "tokens_seen": 1533411328 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.49816715717315674, + "objective/train/avg_token_score": 0.0054593454115092754, + "objective/train/docs_used": 869825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5124175548553467, + "objective/train/original_loss": 3.4399688243865967, + "objective/train/theoretical_loss": 3.509245968848674, + "objective/train/tokens_used": 1533542400, + "objective/train/unlikelihood_loss": 0.16485247015953064, + "theoretical_loss": 3.509245968848674, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027039117352056167, + "loss": 1.4298, + "theoretical_loss": 3.509245968848674, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703711133400201, + "loss": 1.406, + "theoretical_loss": 3.5092196371851783, + "tokens_seen": 1533673472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027035105315947845, + "loss": 1.403, + "theoretical_loss": 3.5091933084020077, + "tokens_seen": 1533804544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703309929789368, + "loss": 1.361, + "theoretical_loss": 3.509166982498601, + "tokens_seen": 1533935616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027031093279839517, + "loss": 1.3693, + "theoretical_loss": 3.5091406594743972, + "tokens_seen": 1534066688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702908726178536, + "loss": 1.2458, + "theoretical_loss": 3.5091143393288355, + "tokens_seen": 1534197760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027027081243731195, + "loss": 1.4918, + "theoretical_loss": 3.509088022061356, + "tokens_seen": 1534328832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702507522567703, + "loss": 1.2944, + "theoretical_loss": 3.509061707671397, + "tokens_seen": 1534459904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702306920762287, + "loss": 1.2921, + "theoretical_loss": 3.5090353961583993, + "tokens_seen": 1534590976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702106318956871, + "loss": 1.2014, + "theoretical_loss": 3.509009087521801, + "tokens_seen": 1534722048 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027019057171514545, + "loss": 1.3675, + "theoretical_loss": 3.508982781761044, + "tokens_seen": 1534853120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701705115346038, + "loss": 1.3778, + "theoretical_loss": 3.5089564788755667, + "tokens_seen": 1534984192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701504513540622, + "loss": 1.5469, + "theoretical_loss": 3.5089301788648104, + "tokens_seen": 1535115264 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.47806695103645325, + "objective/train/avg_token_score": 0.00543417502194643, + "objective/train/docs_used": 871111, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.589242696762085, + "objective/train/original_loss": 3.714128255844116, + "objective/train/theoretical_loss": 3.5089170299372774, + "objective/train/tokens_used": 1535180800, + "objective/train/unlikelihood_loss": 0.14494438469409943, + "theoretical_loss": 3.5089170299372774, + "tokens_seen": 1535180800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027013039117352054, + "loss": 1.3403, + "theoretical_loss": 3.508903881728215, + "tokens_seen": 1535246336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027011033099297896, + "loss": 1.156, + "theoretical_loss": 3.5088775874652214, + "tokens_seen": 1535377408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700902708124373, + "loss": 1.2193, + "theoretical_loss": 3.50885129607527, + "tokens_seen": 1535508480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700702106318957, + "loss": 1.3645, + "theoretical_loss": 3.5088250075578014, + "tokens_seen": 1535639552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027005015045135404, + "loss": 1.4049, + "theoretical_loss": 3.5087987219122576, + "tokens_seen": 1535770624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027003009027081246, + "loss": 1.4022, + "theoretical_loss": 3.5087724391380792, + "tokens_seen": 1535901696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700100300902708, + "loss": 1.4354, + "theoretical_loss": 3.5087461592347076, + "tokens_seen": 1536032768 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699899699097292, + "loss": 1.389, + "theoretical_loss": 3.508719882201584, + "tokens_seen": 1536163840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026996990972918755, + "loss": 1.2523, + "theoretical_loss": 3.5086936080381506, + "tokens_seen": 1536294912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699498495486459, + "loss": 1.315, + "theoretical_loss": 3.5086673367438483, + "tokens_seen": 1536425984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699297893681043, + "loss": 1.4195, + "theoretical_loss": 3.50864106831812, + "tokens_seen": 1536557056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699097291875627, + "loss": 1.2062, + "theoretical_loss": 3.508614802760408, + "tokens_seen": 1536688128 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.32355815172195435, + "objective/train/avg_token_score": 0.019622644409537315, + "objective/train/docs_used": 871860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0351307392120361, + "objective/train/original_loss": 4.007094383239746, + "objective/train/theoretical_loss": 3.5085885400701535, + "objective/train/tokens_used": 1536819200, + "objective/train/unlikelihood_loss": 0.21199025213718414, + "theoretical_loss": 3.5085885400701535, + "tokens_seen": 1536819200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026988966900702105, + "loss": 1.1623, + "theoretical_loss": 3.5085885400701535, + "tokens_seen": 1536819200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698696088264794, + "loss": 1.4284, + "theoretical_loss": 3.5085622802467995, + "tokens_seen": 1536950272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026984954864593783, + "loss": 1.4439, + "theoretical_loss": 3.508536023289789, + "tokens_seen": 1537081344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698294884653962, + "loss": 1.4149, + "theoretical_loss": 3.5085097691985645, + "tokens_seen": 1537212416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026980942828485455, + "loss": 1.3379, + "theoretical_loss": 3.508483517972569, + "tokens_seen": 1537343488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697893681043129, + "loss": 1.3841, + "theoretical_loss": 3.5084572696112444, + "tokens_seen": 1537474560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697693079237713, + "loss": 1.3384, + "theoretical_loss": 3.508431024114035, + "tokens_seen": 1537605632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697492477432297, + "loss": 1.4058, + "theoretical_loss": 3.508404781480385, + "tokens_seen": 1537736704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026972918756268806, + "loss": 1.2109, + "theoretical_loss": 3.508378541709736, + "tokens_seen": 1537867776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697091273821464, + "loss": 1.3775, + "theoretical_loss": 3.508352304801533, + "tokens_seen": 1537998848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696890672016048, + "loss": 1.3404, + "theoretical_loss": 3.5083260707552193, + "tokens_seen": 1538129920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696690070210632, + "loss": 1.3437, + "theoretical_loss": 3.508299839570239, + "tokens_seen": 1538260992 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026964894684052156, + "loss": 1.3144, + "theoretical_loss": 3.5082736112460364, + "tokens_seen": 1538392064 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.4379887580871582, + "objective/train/avg_token_score": 0.03042302094399929, + "objective/train/docs_used": 873345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5273056030273438, + "objective/train/original_loss": 3.870345115661621, + "objective/train/theoretical_loss": 3.5082604981565533, + "objective/train/tokens_used": 1538457600, + "objective/train/unlikelihood_loss": 0.1623808741569519, + "theoretical_loss": 3.5082604981565533, + "tokens_seen": 1538457600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696288866599799, + "loss": 1.3817, + "theoretical_loss": 3.508247385782056, + "tokens_seen": 1538523136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696088264794383, + "loss": 1.2814, + "theoretical_loss": 3.5082211631777414, + "tokens_seen": 1538654208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026958876629889665, + "loss": 1.3688, + "theoretical_loss": 3.5081949434325383, + "tokens_seen": 1538785280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026956870611835506, + "loss": 1.4809, + "theoretical_loss": 3.5081687265458905, + "tokens_seen": 1538916352 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695486459378134, + "loss": 1.3542, + "theoretical_loss": 3.5081425125172436, + "tokens_seen": 1539047424 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695285857572718, + "loss": 1.4208, + "theoretical_loss": 3.5081163013460426, + "tokens_seen": 1539178496 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695085255767302, + "loss": 1.1573, + "theoretical_loss": 3.5080900930317327, + "tokens_seen": 1539309568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694884653961886, + "loss": 1.3969, + "theoretical_loss": 3.5080638875737584, + "tokens_seen": 1539440640 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269468405215647, + "loss": 1.4712, + "theoretical_loss": 3.5080376849715664, + "tokens_seen": 1539571712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026944834503510534, + "loss": 1.4367, + "theoretical_loss": 3.5080114852246025, + "tokens_seen": 1539702784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694282848545637, + "loss": 1.4541, + "theoretical_loss": 3.5079852883323115, + "tokens_seen": 1539833856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026940822467402207, + "loss": 1.3497, + "theoretical_loss": 3.50795909429414, + "tokens_seen": 1539964928 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.33345553278923035, + "objective/train/avg_token_score": 0.02210211008787155, + "objective/train/docs_used": 874032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.017038106918335, + "objective/train/original_loss": 3.801081418991089, + "objective/train/theoretical_loss": 3.507932903109534, + "objective/train/tokens_used": 1540096000, + "objective/train/unlikelihood_loss": 0.19799983501434326, + "theoretical_loss": 3.507932903109534, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693881644934805, + "loss": 1.2421, + "theoretical_loss": 3.507932903109534, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026936810431293885, + "loss": 1.4719, + "theoretical_loss": 3.50790671477794, + "tokens_seen": 1540227072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693480441323972, + "loss": 1.2436, + "theoretical_loss": 3.507880529298805, + "tokens_seen": 1540358144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026932798395185557, + "loss": 1.2897, + "theoretical_loss": 3.507854346671575, + "tokens_seen": 1540489216 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269307923771314, + "loss": 1.3453, + "theoretical_loss": 3.5078281668956963, + "tokens_seen": 1540620288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026928786359077235, + "loss": 1.158, + "theoretical_loss": 3.5078019899706163, + "tokens_seen": 1540751360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692678034102307, + "loss": 1.257, + "theoretical_loss": 3.5077758158957826, + "tokens_seen": 1540882432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692477432296891, + "loss": 1.2629, + "theoretical_loss": 3.5077496446706418, + "tokens_seen": 1541013504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692276830491475, + "loss": 1.4098, + "theoretical_loss": 3.5077234762946414, + "tokens_seen": 1541144576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026920762286860585, + "loss": 1.2186, + "theoretical_loss": 3.507697310767229, + "tokens_seen": 1541275648 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691875626880642, + "loss": 1.3731, + "theoretical_loss": 3.5076711480878524, + "tokens_seen": 1541406720 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691675025075226, + "loss": 1.3251, + "theoretical_loss": 3.5076449882559597, + "tokens_seen": 1541537792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026914744232698094, + "loss": 1.4394, + "theoretical_loss": 3.5076188312709986, + "tokens_seen": 1541668864 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.5486925840377808, + "objective/train/avg_token_score": 0.0031643458642065525, + "objective/train/docs_used": 874753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4820706844329834, + "objective/train/original_loss": 2.970688819885254, + "objective/train/theoretical_loss": 3.5076057538459446, + "objective/train/tokens_used": 1541734400, + "objective/train/unlikelihood_loss": 0.2138146162033081, + "theoretical_loss": 3.5076057538459446, + "tokens_seen": 1541734400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026912738214643936, + "loss": 1.3106, + "theoretical_loss": 3.5075926771324166, + "tokens_seen": 1541799936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691073219658977, + "loss": 1.2917, + "theoretical_loss": 3.5075665258396636, + "tokens_seen": 1541931008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690872617853561, + "loss": 1.4494, + "theoretical_loss": 3.5075403773921865, + "tokens_seen": 1542062080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026906720160481444, + "loss": 1.3634, + "theoretical_loss": 3.507514231789435, + "tokens_seen": 1542193152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026904714142427286, + "loss": 1.4575, + "theoretical_loss": 3.5074880890308577, + "tokens_seen": 1542324224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690270812437312, + "loss": 1.4517, + "theoretical_loss": 3.507461949115903, + "tokens_seen": 1542455296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690070210631896, + "loss": 1.5377, + "theoretical_loss": 3.5074358120440206, + "tokens_seen": 1542586368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026898696088264795, + "loss": 1.3252, + "theoretical_loss": 3.5074096778146586, + "tokens_seen": 1542717440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689669007021063, + "loss": 1.4767, + "theoretical_loss": 3.5073835464272682, + "tokens_seen": 1542848512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689468405215647, + "loss": 1.3503, + "theoretical_loss": 3.5073574178812974, + "tokens_seen": 1542979584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689267803410231, + "loss": 1.3393, + "theoretical_loss": 3.507331292176197, + "tokens_seen": 1543110656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026890672016048145, + "loss": 1.3786, + "theoretical_loss": 3.507305169311416, + "tokens_seen": 1543241728 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.563049852848053, + "objective/train/avg_token_score": 0.0034787754993885756, + "objective/train/docs_used": 875990, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8365345001220703, + "objective/train/original_loss": 3.5295379161834717, + "objective/train/theoretical_loss": 3.5072790492864057, + "objective/train/tokens_used": 1543372800, + "objective/train/unlikelihood_loss": 0.15045246481895447, + "theoretical_loss": 3.5072790492864057, + "tokens_seen": 1543372800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688866599799398, + "loss": 1.2566, + "theoretical_loss": 3.5072790492864057, + "tokens_seen": 1543372800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026886659979939823, + "loss": 1.3738, + "theoretical_loss": 3.5072529321006147, + "tokens_seen": 1543503872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688465396188566, + "loss": 1.3633, + "theoretical_loss": 3.5072268177534935, + "tokens_seen": 1543634944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026882647943831495, + "loss": 1.5255, + "theoretical_loss": 3.5072007062444936, + "tokens_seen": 1543766016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688064192577733, + "loss": 1.3692, + "theoretical_loss": 3.507174597573065, + "tokens_seen": 1543897088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687863590772317, + "loss": 1.3555, + "theoretical_loss": 3.5071484917386586, + "tokens_seen": 1544028160 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687662988966901, + "loss": 1.3213, + "theoretical_loss": 3.507122388740725, + "tokens_seen": 1544159232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026874623871614846, + "loss": 1.2667, + "theoretical_loss": 3.5070962885787154, + "tokens_seen": 1544290304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687261785356068, + "loss": 1.2779, + "theoretical_loss": 3.507070191252082, + "tokens_seen": 1544421376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687061183550652, + "loss": 1.4024, + "theoretical_loss": 3.507044096760275, + "tokens_seen": 1544552448 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686860581745236, + "loss": 1.2224, + "theoretical_loss": 3.5070180051027458, + "tokens_seen": 1544683520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026866599799398196, + "loss": 1.3126, + "theoretical_loss": 3.506991916278947, + "tokens_seen": 1544814592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686459378134403, + "loss": 1.2558, + "theoretical_loss": 3.5069658302883298, + "tokens_seen": 1544945664 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.2919110357761383, + "objective/train/avg_token_score": 0.016907580196857452, + "objective/train/docs_used": 876526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9070130586624146, + "objective/train/original_loss": 3.9409239292144775, + "objective/train/theoretical_loss": 3.5069527883552936, + "objective/train/tokens_used": 1545011200, + "objective/train/unlikelihood_loss": 0.1779026836156845, + "theoretical_loss": 3.5069527883552936, + "tokens_seen": 1545011200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686258776328987, + "loss": 1.2365, + "theoretical_loss": 3.506939747130347, + "tokens_seen": 1545076736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026860581745235705, + "loss": 1.3129, + "theoretical_loss": 3.50691366680445, + "tokens_seen": 1545207808 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026858575727181546, + "loss": 1.2182, + "theoretical_loss": 3.5068875893100913, + "tokens_seen": 1545338880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685656970912738, + "loss": 1.2687, + "theoretical_loss": 3.506861514646723, + "tokens_seen": 1545469952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685456369107322, + "loss": 1.4138, + "theoretical_loss": 3.5068354428137978, + "tokens_seen": 1545601024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026852557673019055, + "loss": 1.211, + "theoretical_loss": 3.5068093738107695, + "tokens_seen": 1545732096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026850551654964897, + "loss": 1.376, + "theoretical_loss": 3.5067833076370896, + "tokens_seen": 1545863168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026848545636910733, + "loss": 1.412, + "theoretical_loss": 3.5067572442922126, + "tokens_seen": 1545994240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684653961885657, + "loss": 1.2771, + "theoretical_loss": 3.50673118377559, + "tokens_seen": 1546125312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026844533600802405, + "loss": 1.3522, + "theoretical_loss": 3.5067051260866764, + "tokens_seen": 1546256384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684252758274824, + "loss": 1.2294, + "theoretical_loss": 3.5066790712249256, + "tokens_seen": 1546387456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026840521564694083, + "loss": 1.2766, + "theoretical_loss": 3.50665301918979, + "tokens_seen": 1546518528 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.5924364328384399, + "objective/train/avg_token_score": 0.01651097647845745, + "objective/train/docs_used": 877732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.767458200454712, + "objective/train/original_loss": 3.7146949768066406, + "objective/train/theoretical_loss": 3.506626969980724, + "objective/train/tokens_used": 1546649600, + "objective/train/unlikelihood_loss": 0.12983863055706024, + "theoretical_loss": 3.506626969980724, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683851554663992, + "loss": 1.3572, + "theoretical_loss": 3.506626969980724, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026836509528585756, + "loss": 1.4143, + "theoretical_loss": 3.5066009235971825, + "tokens_seen": 1546780672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683450351053159, + "loss": 1.4068, + "theoretical_loss": 3.5065748800386176, + "tokens_seen": 1546911744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026832497492477433, + "loss": 1.3326, + "theoretical_loss": 3.506548839304486, + "tokens_seen": 1547042816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683049147442327, + "loss": 1.3586, + "theoretical_loss": 3.50652280139424, + "tokens_seen": 1547173888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026828485456369106, + "loss": 1.423, + "theoretical_loss": 3.506496766307335, + "tokens_seen": 1547304960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682647943831494, + "loss": 1.3448, + "theoretical_loss": 3.5064707340432264, + "tokens_seen": 1547436032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682447342026078, + "loss": 1.292, + "theoretical_loss": 3.5064447046013685, + "tokens_seen": 1547567104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682246740220662, + "loss": 1.446, + "theoretical_loss": 3.506418677981216, + "tokens_seen": 1547698176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026820461384152456, + "loss": 1.3478, + "theoretical_loss": 3.506392654182225, + "tokens_seen": 1547829248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681845536609829, + "loss": 1.3565, + "theoretical_loss": 3.50636663320385, + "tokens_seen": 1547960320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681644934804413, + "loss": 1.2885, + "theoretical_loss": 3.5063406150455467, + "tokens_seen": 1548091392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681444332998997, + "loss": 1.2443, + "theoretical_loss": 3.50631459970677, + "tokens_seen": 1548222464 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.41184017062187195, + "objective/train/avg_token_score": 0.00911112129688263, + "objective/train/docs_used": 878530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.220145344734192, + "objective/train/original_loss": 4.1268839836120605, + "objective/train/theoretical_loss": 3.5063015930945354, + "objective/train/tokens_used": 1548288000, + "objective/train/unlikelihood_loss": 0.10617279261350632, + "theoretical_loss": 3.5063015930945354, + "tokens_seen": 1548288000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026812437311935807, + "loss": 1.4326, + "theoretical_loss": 3.506288587186978, + "tokens_seen": 1548353536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026810431293881643, + "loss": 1.3164, + "theoretical_loss": 3.506262577485624, + "tokens_seen": 1548484608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680842527582748, + "loss": 1.2821, + "theoretical_loss": 3.506236570602166, + "tokens_seen": 1548615680 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026806419257773315, + "loss": 1.1563, + "theoretical_loss": 3.5062105665360592, + "tokens_seen": 1548746752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026804413239719157, + "loss": 1.4078, + "theoretical_loss": 3.50618456528676, + "tokens_seen": 1548877824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026802407221664993, + "loss": 1.2611, + "theoretical_loss": 3.506158566853725, + "tokens_seen": 1549008896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026800401203610835, + "loss": 1.2885, + "theoretical_loss": 3.506132571236412, + "tokens_seen": 1549139968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679839518555667, + "loss": 1.2683, + "theoretical_loss": 3.5061065784342764, + "tokens_seen": 1549271040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679638916750251, + "loss": 1.3487, + "theoretical_loss": 3.5060805884467756, + "tokens_seen": 1549402112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679438314944835, + "loss": 1.3582, + "theoretical_loss": 3.5060546012733673, + "tokens_seen": 1549533184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026792377131394185, + "loss": 1.3293, + "theoretical_loss": 3.5060286169135084, + "tokens_seen": 1549664256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679037111334002, + "loss": 1.3018, + "theoretical_loss": 3.5060026353666562, + "tokens_seen": 1549795328 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.39772728085517883, + "objective/train/avg_token_score": 0.006444786675274372, + "objective/train/docs_used": 879841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3124042749404907, + "objective/train/original_loss": 4.184053421020508, + "objective/train/theoretical_loss": 3.505976656632269, + "objective/train/tokens_used": 1549926400, + "objective/train/unlikelihood_loss": 0.1287296712398529, + "theoretical_loss": 3.505976656632269, + "tokens_seen": 1549926400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026788365095285863, + "loss": 1.2028, + "theoretical_loss": 3.505976656632269, + "tokens_seen": 1549926400 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267863590772317, + "loss": 1.4904, + "theoretical_loss": 3.5059506807098035, + "tokens_seen": 1550057472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026784353059177535, + "loss": 1.4832, + "theoretical_loss": 3.505924707598718, + "tokens_seen": 1550188544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678234704112337, + "loss": 1.5127, + "theoretical_loss": 3.5058987372984713, + "tokens_seen": 1550319616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678034102306921, + "loss": 1.376, + "theoretical_loss": 3.50587276980852, + "tokens_seen": 1550450688 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677833500501505, + "loss": 1.3786, + "theoretical_loss": 3.5058468051283245, + "tokens_seen": 1550581760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026776328986960886, + "loss": 1.3547, + "theoretical_loss": 3.505820843257342, + "tokens_seen": 1550712832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677432296890672, + "loss": 1.4168, + "theoretical_loss": 3.505794884195031, + "tokens_seen": 1550843904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677231695085256, + "loss": 1.3541, + "theoretical_loss": 3.505768927940851, + "tokens_seen": 1550974976 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267703109327984, + "loss": 1.2963, + "theoretical_loss": 3.505742974494261, + "tokens_seen": 1551106048 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026768304914744236, + "loss": 1.4119, + "theoretical_loss": 3.505717023854719, + "tokens_seen": 1551237120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676629889669007, + "loss": 1.2816, + "theoretical_loss": 3.5056910760216855, + "tokens_seen": 1551368192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676429287863591, + "loss": 1.3766, + "theoretical_loss": 3.505665130994619, + "tokens_seen": 1551499264 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.5406280755996704, + "objective/train/avg_token_score": 0.029312819242477417, + "objective/train/docs_used": 880523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8097230195999146, + "objective/train/original_loss": 3.644073486328125, + "objective/train/theoretical_loss": 3.505652159533155, + "objective/train/tokens_used": 1551564800, + "objective/train/unlikelihood_loss": 0.2193147987127304, + "theoretical_loss": 3.505652159533155, + "tokens_seen": 1551564800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026762286860581745, + "loss": 1.4135, + "theoretical_loss": 3.5056391887729803, + "tokens_seen": 1551630336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026760280842527586, + "loss": 1.2781, + "theoretical_loss": 3.5056132493562275, + "tokens_seen": 1551761408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675827482447342, + "loss": 1.3758, + "theoretical_loss": 3.5055873127438217, + "tokens_seen": 1551892480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675626880641926, + "loss": 1.4436, + "theoretical_loss": 3.505561378935223, + "tokens_seen": 1552023552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026754262788365095, + "loss": 1.4345, + "theoretical_loss": 3.5055354479298897, + "tokens_seen": 1552154624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026752256770310937, + "loss": 1.4779, + "theoretical_loss": 3.5055095197272843, + "tokens_seen": 1552285696 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026750250752256773, + "loss": 1.3983, + "theoretical_loss": 3.505483594326866, + "tokens_seen": 1552416768 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674824473420261, + "loss": 1.3791, + "theoretical_loss": 3.5054576717280956, + "tokens_seen": 1552547840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026746238716148445, + "loss": 1.3783, + "theoretical_loss": 3.505431751930434, + "tokens_seen": 1552678912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674423269809428, + "loss": 1.4773, + "theoretical_loss": 3.5054058349333426, + "tokens_seen": 1552809984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026742226680040123, + "loss": 1.1208, + "theoretical_loss": 3.5053799207362815, + "tokens_seen": 1552941056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674022066198596, + "loss": 1.4127, + "theoretical_loss": 3.5053540093387126, + "tokens_seen": 1553072128 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.5148460268974304, + "objective/train/avg_token_score": 0.02032672055065632, + "objective/train/docs_used": 881131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4681516885757446, + "objective/train/original_loss": 3.4278547763824463, + "objective/train/theoretical_loss": 3.505328100740097, + "objective/train/tokens_used": 1553203200, + "objective/train/unlikelihood_loss": 0.16291972994804382, + "theoretical_loss": 3.505328100740097, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026738214643931796, + "loss": 1.3427, + "theoretical_loss": 3.505328100740097, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673620862587763, + "loss": 1.4182, + "theoretical_loss": 3.505302194939896, + "tokens_seen": 1553334272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026734202607823474, + "loss": 1.4397, + "theoretical_loss": 3.5052762919375717, + "tokens_seen": 1553465344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673219658976931, + "loss": 1.3013, + "theoretical_loss": 3.5052503917325852, + "tokens_seen": 1553596416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026730190571715146, + "loss": 1.3038, + "theoretical_loss": 3.5052244943243993, + "tokens_seen": 1553727488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672818455366098, + "loss": 1.4403, + "theoretical_loss": 3.5051985997124757, + "tokens_seen": 1553858560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672617853560682, + "loss": 1.3762, + "theoretical_loss": 3.505172707896276, + "tokens_seen": 1553989632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672417251755266, + "loss": 1.2077, + "theoretical_loss": 3.505146818875264, + "tokens_seen": 1554120704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026722166499498496, + "loss": 1.2367, + "theoretical_loss": 3.505120932648901, + "tokens_seen": 1554251776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672016048144433, + "loss": 1.3474, + "theoretical_loss": 3.50509504921665, + "tokens_seen": 1554382848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671815446339017, + "loss": 1.3904, + "theoretical_loss": 3.5050691685779745, + "tokens_seen": 1554513920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671614844533601, + "loss": 1.3057, + "theoretical_loss": 3.5050432907323366, + "tokens_seen": 1554644992 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026714142427281847, + "loss": 1.3425, + "theoretical_loss": 3.5050174156791996, + "tokens_seen": 1554776064 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.3675464391708374, + "objective/train/avg_token_score": 0.021707436069846153, + "objective/train/docs_used": 882511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2087987661361694, + "objective/train/original_loss": 3.9865882396698, + "objective/train/theoretical_loss": 3.505004479199651, + "objective/train/tokens_used": 1554841600, + "objective/train/unlikelihood_loss": 0.21248787641525269, + "theoretical_loss": 3.505004479199651, + "tokens_seen": 1554841600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026712136409227683, + "loss": 1.2291, + "theoretical_loss": 3.504991543418027, + "tokens_seen": 1554907136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671013039117352, + "loss": 1.4537, + "theoretical_loss": 3.5049656739482824, + "tokens_seen": 1555038208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026708124373119355, + "loss": 1.4278, + "theoretical_loss": 3.504939807269429, + "tokens_seen": 1555169280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026706118355065197, + "loss": 1.3203, + "theoretical_loss": 3.504913943380931, + "tokens_seen": 1555300352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026704112337011033, + "loss": 1.4427, + "theoretical_loss": 3.5048880822822515, + "tokens_seen": 1555431424 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670210631895687, + "loss": 1.4157, + "theoretical_loss": 3.504862223972855, + "tokens_seen": 1555562496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026700100300902706, + "loss": 1.2934, + "theoretical_loss": 3.5048363684522057, + "tokens_seen": 1555693568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026698094282848547, + "loss": 1.3079, + "theoretical_loss": 3.504810515719768, + "tokens_seen": 1555824640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026696088264794383, + "loss": 1.5012, + "theoretical_loss": 3.5047846657750066, + "tokens_seen": 1555955712 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669408224674022, + "loss": 1.3085, + "theoretical_loss": 3.5047588186173853, + "tokens_seen": 1556086784 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026692076228686056, + "loss": 1.4821, + "theoretical_loss": 3.5047329742463695, + "tokens_seen": 1556217856 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266900702106319, + "loss": 1.1825, + "theoretical_loss": 3.504707132661424, + "tokens_seen": 1556348928 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.4828934371471405, + "objective/train/avg_token_score": 0.03187386319041252, + "objective/train/docs_used": 882978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6838879585266113, + "objective/train/original_loss": 4.05211877822876, + "objective/train/theoretical_loss": 3.5046812938620135, + "objective/train/tokens_used": 1556480000, + "objective/train/unlikelihood_loss": 0.13135670125484467, + "theoretical_loss": 3.5046812938620135, + "tokens_seen": 1556480000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026688064192577734, + "loss": 1.5662, + "theoretical_loss": 3.5046812938620135, + "tokens_seen": 1556480000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668605817452357, + "loss": 1.2566, + "theoretical_loss": 3.504655457847604, + "tokens_seen": 1556611072 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026684052156469406, + "loss": 1.2799, + "theoretical_loss": 3.5046296246176594, + "tokens_seen": 1556742144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668204613841524, + "loss": 1.1945, + "theoretical_loss": 3.504603794171647, + "tokens_seen": 1556873216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026680040120361084, + "loss": 1.2682, + "theoretical_loss": 3.5045779665090313, + "tokens_seen": 1557004288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667803410230692, + "loss": 1.2623, + "theoretical_loss": 3.5045521416292784, + "tokens_seen": 1557135360 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026676028084252757, + "loss": 1.4457, + "theoretical_loss": 3.5045263195318546, + "tokens_seen": 1557266432 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026674022066198593, + "loss": 1.2147, + "theoretical_loss": 3.504500500216225, + "tokens_seen": 1557397504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026672016048144434, + "loss": 1.2178, + "theoretical_loss": 3.504474683681857, + "tokens_seen": 1557528576 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667001003009027, + "loss": 1.4862, + "theoretical_loss": 3.504448869928216, + "tokens_seen": 1557659648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026668004012036107, + "loss": 1.485, + "theoretical_loss": 3.504423058954769, + "tokens_seen": 1557790720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026665997993981943, + "loss": 1.3502, + "theoretical_loss": 3.5043972507609835, + "tokens_seen": 1557921792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666399197592778, + "loss": 1.2792, + "theoretical_loss": 3.504371445346325, + "tokens_seen": 1558052864 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.5040322542190552, + "objective/train/avg_token_score": 0.02961777150630951, + "objective/train/docs_used": 884109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5696574449539185, + "objective/train/original_loss": 3.4268500804901123, + "objective/train/theoretical_loss": 3.5043585436810014, + "objective/train/tokens_used": 1558118400, + "objective/train/unlikelihood_loss": 0.20903965830802917, + "theoretical_loss": 3.5043585436810014, + "tokens_seen": 1558118400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666198595787362, + "loss": 1.4298, + "theoretical_loss": 3.504345642710261, + "tokens_seen": 1558183936 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026659979939819457, + "loss": 1.2473, + "theoretical_loss": 3.504319842852258, + "tokens_seen": 1558315008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026657973921765293, + "loss": 1.4629, + "theoretical_loss": 3.5042940457717844, + "tokens_seen": 1558446080 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665596790371113, + "loss": 1.2113, + "theoretical_loss": 3.5042682514683072, + "tokens_seen": 1558577152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665396188565697, + "loss": 1.4761, + "theoretical_loss": 3.5042424599412936, + "tokens_seen": 1558708224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665195586760281, + "loss": 1.1034, + "theoretical_loss": 3.5042166711902114, + "tokens_seen": 1558839296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664994984954865, + "loss": 1.3439, + "theoretical_loss": 3.5041908852145287, + "tokens_seen": 1558970368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026647943831494485, + "loss": 1.3931, + "theoretical_loss": 3.5041651020137135, + "tokens_seen": 1559101440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664593781344032, + "loss": 1.5897, + "theoretical_loss": 3.504139321587234, + "tokens_seen": 1559232512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026643931795386163, + "loss": 1.1903, + "theoretical_loss": 3.504113543934558, + "tokens_seen": 1559363584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026641925777332, + "loss": 1.3641, + "theoretical_loss": 3.504087769055155, + "tokens_seen": 1559494656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026639919759277836, + "loss": 1.2849, + "theoretical_loss": 3.5040619969484923, + "tokens_seen": 1559625728 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.4476417303085327, + "objective/train/avg_token_score": 0.004074747208505869, + "objective/train/docs_used": 884823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3358041048049927, + "objective/train/original_loss": 3.313908815383911, + "objective/train/theoretical_loss": 3.5040362276140393, + "objective/train/tokens_used": 1559756800, + "objective/train/unlikelihood_loss": 0.21480469405651093, + "theoretical_loss": 3.5040362276140393, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663791374122367, + "loss": 1.3661, + "theoretical_loss": 3.5040362276140393, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026635907723169514, + "loss": 1.4007, + "theoretical_loss": 3.504010461051265, + "tokens_seen": 1559887872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663390170511535, + "loss": 1.3216, + "theoretical_loss": 3.503984697259638, + "tokens_seen": 1560018944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026631895687061186, + "loss": 1.2525, + "theoretical_loss": 3.503958936238628, + "tokens_seen": 1560150016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662988966900702, + "loss": 1.2782, + "theoretical_loss": 3.5039331779877037, + "tokens_seen": 1560281088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662788365095286, + "loss": 1.1676, + "theoretical_loss": 3.5039074225063347, + "tokens_seen": 1560412160 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266258776328987, + "loss": 1.3002, + "theoretical_loss": 3.5038816697939916, + "tokens_seen": 1560543232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026623871614844536, + "loss": 1.3767, + "theoretical_loss": 3.503855919850143, + "tokens_seen": 1560674304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662186559679037, + "loss": 1.3196, + "theoretical_loss": 3.503830172674259, + "tokens_seen": 1560805376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661985957873621, + "loss": 1.4551, + "theoretical_loss": 3.50380442826581, + "tokens_seen": 1560936448 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661785356068205, + "loss": 1.4577, + "theoretical_loss": 3.503778686624266, + "tokens_seen": 1561067520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026615847542627887, + "loss": 1.3769, + "theoretical_loss": 3.5037529477490974, + "tokens_seen": 1561198592 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026613841524573723, + "loss": 1.3637, + "theoretical_loss": 3.5037272116397746, + "tokens_seen": 1561329664 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.42570871114730835, + "objective/train/avg_token_score": 0.04138072207570076, + "objective/train/docs_used": 885915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.206316590309143, + "objective/train/original_loss": 3.891554355621338, + "objective/train/theoretical_loss": 3.50371434462214, + "objective/train/tokens_used": 1561395200, + "objective/train/unlikelihood_loss": 0.1374169886112213, + "theoretical_loss": 3.50371434462214, + "tokens_seen": 1561395200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661183550651956, + "loss": 1.2405, + "theoretical_loss": 3.5037014782957683, + "tokens_seen": 1561460736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026609829488465395, + "loss": 1.3651, + "theoretical_loss": 3.5036757477165494, + "tokens_seen": 1561591808 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026607823470411237, + "loss": 1.4689, + "theoretical_loss": 3.5036500199015883, + "tokens_seen": 1561722880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026605817452357073, + "loss": 1.5103, + "theoretical_loss": 3.503624294850357, + "tokens_seen": 1561853952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660381143430291, + "loss": 1.2531, + "theoretical_loss": 3.5035985725623258, + "tokens_seen": 1561985024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026601805416248746, + "loss": 1.4956, + "theoretical_loss": 3.503572853036967, + "tokens_seen": 1562116096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026599799398194587, + "loss": 1.3076, + "theoretical_loss": 3.503547136273751, + "tokens_seen": 1562247168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026597793380140423, + "loss": 1.1474, + "theoretical_loss": 3.5035214222721507, + "tokens_seen": 1562378240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659578736208626, + "loss": 1.4409, + "theoretical_loss": 3.5034957110316367, + "tokens_seen": 1562509312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026593781344032096, + "loss": 1.4164, + "theoretical_loss": 3.5034700025516816, + "tokens_seen": 1562640384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659177532597794, + "loss": 1.328, + "theoretical_loss": 3.5034442968317574, + "tokens_seen": 1562771456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026589769307923774, + "loss": 1.3942, + "theoretical_loss": 3.503418593871336, + "tokens_seen": 1562902528 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.47605082392692566, + "objective/train/avg_token_score": 0.010182133875787258, + "objective/train/docs_used": 886454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.416703224182129, + "objective/train/original_loss": 3.380857229232788, + "objective/train/theoretical_loss": 3.5033928936698904, + "objective/train/tokens_used": 1563033600, + "objective/train/unlikelihood_loss": 0.2105829268693924, + "theoretical_loss": 3.5033928936698904, + "tokens_seen": 1563033600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658776328986961, + "loss": 1.2955, + "theoretical_loss": 3.5033928936698904, + "tokens_seen": 1563033600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026585757271815446, + "loss": 1.2525, + "theoretical_loss": 3.5033671962268924, + "tokens_seen": 1563164672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658375125376128, + "loss": 1.2929, + "theoretical_loss": 3.503341501541816, + "tokens_seen": 1563295744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026581745235707124, + "loss": 1.4067, + "theoretical_loss": 3.503315809614132, + "tokens_seen": 1563426816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657973921765296, + "loss": 1.2823, + "theoretical_loss": 3.5032901204433156, + "tokens_seen": 1563557888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026577733199598797, + "loss": 1.3149, + "theoretical_loss": 3.5032644340288375, + "tokens_seen": 1563688960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026575727181544633, + "loss": 1.4579, + "theoretical_loss": 3.503238750370173, + "tokens_seen": 1563820032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026573721163490474, + "loss": 1.37, + "theoretical_loss": 3.5032130694667947, + "tokens_seen": 1563951104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657171514543631, + "loss": 1.4665, + "theoretical_loss": 3.5031873913181757, + "tokens_seen": 1564082176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026569709127382147, + "loss": 1.3251, + "theoretical_loss": 3.5031617159237904, + "tokens_seen": 1564213248 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026567703109327983, + "loss": 1.1342, + "theoretical_loss": 3.5031360432831127, + "tokens_seen": 1564344320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656569709127382, + "loss": 1.432, + "theoretical_loss": 3.5031103733956157, + "tokens_seen": 1564475392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656369107321966, + "loss": 1.3319, + "theoretical_loss": 3.5030847062607746, + "tokens_seen": 1564606464 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.303519070148468, + "objective/train/avg_token_score": 0.01796434074640274, + "objective/train/docs_used": 887471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.007061243057251, + "objective/train/original_loss": 4.125535011291504, + "objective/train/theoretical_loss": 3.5030718737254354, + "objective/train/tokens_used": 1564672000, + "objective/train/unlikelihood_loss": 0.1569800078868866, + "theoretical_loss": 3.5030718737254354, + "tokens_seen": 1564672000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026561685055165497, + "loss": 1.2412, + "theoretical_loss": 3.5030590418780623, + "tokens_seen": 1564737536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026559679037111333, + "loss": 1.347, + "theoretical_loss": 3.503033380246955, + "tokens_seen": 1564868608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655767301905717, + "loss": 1.3654, + "theoretical_loss": 3.5030077213669255, + "tokens_seen": 1564999680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655566700100301, + "loss": 1.3558, + "theoretical_loss": 3.5029820652374495, + "tokens_seen": 1565130752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655366098294885, + "loss": 1.3626, + "theoretical_loss": 3.502956411858002, + "tokens_seen": 1565261824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026551654964894684, + "loss": 1.2706, + "theoretical_loss": 3.502930761228057, + "tokens_seen": 1565392896 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654964894684052, + "loss": 1.2484, + "theoretical_loss": 3.5029051133470905, + "tokens_seen": 1565523968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026547642928786356, + "loss": 1.3835, + "theoretical_loss": 3.5028794682145774, + "tokens_seen": 1565655040 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265456369107322, + "loss": 1.3533, + "theoretical_loss": 3.5028538258299937, + "tokens_seen": 1565786112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026543630892678034, + "loss": 1.3064, + "theoretical_loss": 3.502828186192814, + "tokens_seen": 1565917184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654162487462387, + "loss": 1.2216, + "theoretical_loss": 3.5028025493025146, + "tokens_seen": 1566048256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026539618856569707, + "loss": 1.2639, + "theoretical_loss": 3.502776915158571, + "tokens_seen": 1566179328 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.4313294291496277, + "objective/train/avg_token_score": 0.0070722769014537334, + "objective/train/docs_used": 888126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3369563817977905, + "objective/train/original_loss": 3.812636613845825, + "objective/train/theoretical_loss": 3.50275128376046, + "objective/train/tokens_used": 1566310400, + "objective/train/unlikelihood_loss": 0.14208641648292542, + "theoretical_loss": 3.50275128376046, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653761283851555, + "loss": 1.2566, + "theoretical_loss": 3.50275128376046, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026535606820461384, + "loss": 1.3881, + "theoretical_loss": 3.502725655107657, + "tokens_seen": 1566441472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653360080240722, + "loss": 1.3764, + "theoretical_loss": 3.502700029199638, + "tokens_seen": 1566572544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026531594784353057, + "loss": 1.2947, + "theoretical_loss": 3.5026744060358803, + "tokens_seen": 1566703616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026529588766298893, + "loss": 1.4619, + "theoretical_loss": 3.50264878561586, + "tokens_seen": 1566834688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026527582748244735, + "loss": 1.2975, + "theoretical_loss": 3.5026231679390536, + "tokens_seen": 1566965760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652557673019057, + "loss": 1.333, + "theoretical_loss": 3.5025975530049385, + "tokens_seen": 1567096832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026523570712136407, + "loss": 1.3936, + "theoretical_loss": 3.5025719408129916, + "tokens_seen": 1567227904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026521564694082243, + "loss": 1.2078, + "theoretical_loss": 3.5025463313626894, + "tokens_seen": 1567358976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026519558676028085, + "loss": 1.1777, + "theoretical_loss": 3.5025207246535097, + "tokens_seen": 1567490048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651755265797392, + "loss": 1.226, + "theoretical_loss": 3.50249512068493, + "tokens_seen": 1567621120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651554663991976, + "loss": 1.3664, + "theoretical_loss": 3.5024695194564277, + "tokens_seen": 1567752192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026513540621865594, + "loss": 1.4358, + "theoretical_loss": 3.5024439209674805, + "tokens_seen": 1567883264 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.37267839908599854, + "objective/train/avg_token_score": 0.01723858341574669, + "objective/train/docs_used": 889384, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.33036208152771, + "objective/train/original_loss": 4.418459415435791, + "objective/train/theoretical_loss": 3.5024311227501768, + "objective/train/tokens_used": 1567948800, + "objective/train/unlikelihood_loss": 0.11782298237085342, + "theoretical_loss": 3.5024311227501768, + "tokens_seen": 1567948800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651153460381143, + "loss": 1.3421, + "theoretical_loss": 3.5024183252175662, + "tokens_seen": 1568014336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650952858575727, + "loss": 1.3117, + "theoretical_loss": 3.5023927322061637, + "tokens_seen": 1568145408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650752256770311, + "loss": 1.4693, + "theoretical_loss": 3.5023671419327496, + "tokens_seen": 1568276480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026505516549648944, + "loss": 1.3029, + "theoretical_loss": 3.5023415543968026, + "tokens_seen": 1568407552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650351053159478, + "loss": 1.3367, + "theoretical_loss": 3.5023159695978023, + "tokens_seen": 1568538624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650150451354062, + "loss": 1.4041, + "theoretical_loss": 3.502290387535226, + "tokens_seen": 1568669696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026499498495486464, + "loss": 1.182, + "theoretical_loss": 3.502264808208553, + "tokens_seen": 1568800768 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264974924774323, + "loss": 1.3296, + "theoretical_loss": 3.5022392316172617, + "tokens_seen": 1568931840 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026495486459378136, + "loss": 1.2014, + "theoretical_loss": 3.502213657760832, + "tokens_seen": 1569062912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649348044132398, + "loss": 1.3054, + "theoretical_loss": 3.5021880866387423, + "tokens_seen": 1569193984 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026491474423269814, + "loss": 1.2169, + "theoretical_loss": 3.502162518250472, + "tokens_seen": 1569325056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648946840521565, + "loss": 1.301, + "theoretical_loss": 3.5021369525955004, + "tokens_seen": 1569456128 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.5157013535499573, + "objective/train/avg_token_score": 0.015478887595236301, + "objective/train/docs_used": 890057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.673840880393982, + "objective/train/original_loss": 3.824557065963745, + "objective/train/theoretical_loss": 3.5021113896733076, + "objective/train/tokens_used": 1569587200, + "objective/train/unlikelihood_loss": 0.13192664086818695, + "theoretical_loss": 3.5021113896733076, + "tokens_seen": 1569587200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026487462387161486, + "loss": 1.3837, + "theoretical_loss": 3.5021113896733076, + "tokens_seen": 1569587200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648545636910732, + "loss": 1.1979, + "theoretical_loss": 3.5020858294833728, + "tokens_seen": 1569718272 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026483450351053164, + "loss": 1.5718, + "theoretical_loss": 3.5020602720251763, + "tokens_seen": 1569849344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026481444332999, + "loss": 1.3524, + "theoretical_loss": 3.502034717298198, + "tokens_seen": 1569980416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026479438314944837, + "loss": 1.394, + "theoretical_loss": 3.502009165301917, + "tokens_seen": 1570111488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026477432296890673, + "loss": 1.4249, + "theoretical_loss": 3.5019836160358153, + "tokens_seen": 1570242560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026475426278836514, + "loss": 1.4505, + "theoretical_loss": 3.501958069499372, + "tokens_seen": 1570373632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647342026078235, + "loss": 1.3765, + "theoretical_loss": 3.5019325256920686, + "tokens_seen": 1570504704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026471414242728187, + "loss": 1.3306, + "theoretical_loss": 3.501906984613385, + "tokens_seen": 1570635776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026469408224674023, + "loss": 1.3438, + "theoretical_loss": 3.5018814462628027, + "tokens_seen": 1570766848 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646740220661986, + "loss": 1.4862, + "theoretical_loss": 3.5018559106398026, + "tokens_seen": 1570897920 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264653961885657, + "loss": 1.3942, + "theoretical_loss": 3.5018303777438646, + "tokens_seen": 1571028992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026463390170511537, + "loss": 1.3715, + "theoretical_loss": 3.5018048475744723, + "tokens_seen": 1571160064 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.5315859913825989, + "objective/train/avg_token_score": 0.012114740908145905, + "objective/train/docs_used": 891281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.725339412689209, + "objective/train/original_loss": 3.555938720703125, + "objective/train/theoretical_loss": 3.501792083512068, + "objective/train/tokens_used": 1571225600, + "objective/train/unlikelihood_loss": 0.18413987755775452, + "theoretical_loss": 3.501792083512068, + "tokens_seen": 1571225600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026461384152457373, + "loss": 1.2848, + "theoretical_loss": 3.5017793201311056, + "tokens_seen": 1571291136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645937813440321, + "loss": 1.2903, + "theoretical_loss": 3.5017537954132463, + "tokens_seen": 1571422208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645737211634905, + "loss": 1.4472, + "theoretical_loss": 3.501728273420376, + "tokens_seen": 1571553280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645536609829489, + "loss": 1.4143, + "theoretical_loss": 3.5017027541519763, + "tokens_seen": 1571684352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026453360080240724, + "loss": 1.331, + "theoretical_loss": 3.50167723760753, + "tokens_seen": 1571815424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645135406218656, + "loss": 1.4009, + "theoretical_loss": 3.501651723786519, + "tokens_seen": 1571946496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026449348044132396, + "loss": 1.4436, + "theoretical_loss": 3.5016262126884254, + "tokens_seen": 1572077568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644734202607824, + "loss": 1.3179, + "theoretical_loss": 3.5016007043127315, + "tokens_seen": 1572208640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026445336008024074, + "loss": 1.3627, + "theoretical_loss": 3.50157519865892, + "tokens_seen": 1572339712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644332998996991, + "loss": 1.3634, + "theoretical_loss": 3.5015496957264736, + "tokens_seen": 1572470784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026441323971915747, + "loss": 1.3625, + "theoretical_loss": 3.5015241955148744, + "tokens_seen": 1572601856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643931795386159, + "loss": 1.5191, + "theoretical_loss": 3.5014986980236067, + "tokens_seen": 1572732928 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.4540567100048065, + "objective/train/avg_token_score": 0.02894679456949234, + "objective/train/docs_used": 891900, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3996063470840454, + "objective/train/original_loss": 3.4957447052001953, + "objective/train/theoretical_loss": 3.501473203252153, + "objective/train/tokens_used": 1572864000, + "objective/train/unlikelihood_loss": 0.14973536133766174, + "theoretical_loss": 3.501473203252153, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026437311935807424, + "loss": 1.3949, + "theoretical_loss": 3.501473203252153, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643530591775326, + "loss": 1.4512, + "theoretical_loss": 3.5014477111999964, + "tokens_seen": 1572995072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026433299899699097, + "loss": 1.3143, + "theoretical_loss": 3.501422221866621, + "tokens_seen": 1573126144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026431293881644933, + "loss": 1.3183, + "theoretical_loss": 3.501396735251509, + "tokens_seen": 1573257216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026429287863590775, + "loss": 1.4303, + "theoretical_loss": 3.5013712513541453, + "tokens_seen": 1573388288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642728184553661, + "loss": 1.2935, + "theoretical_loss": 3.501345770174013, + "tokens_seen": 1573519360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026425275827482447, + "loss": 1.4666, + "theoretical_loss": 3.501320291710597, + "tokens_seen": 1573650432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026423269809428283, + "loss": 1.5679, + "theoretical_loss": 3.501294815963381, + "tokens_seen": 1573781504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026421263791374125, + "loss": 1.3932, + "theoretical_loss": 3.501269342931848, + "tokens_seen": 1573912576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641925777331996, + "loss": 1.4471, + "theoretical_loss": 3.5012438726154844, + "tokens_seen": 1574043648 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264172517552658, + "loss": 1.3644, + "theoretical_loss": 3.5012184050137733, + "tokens_seen": 1574174720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026415245737211634, + "loss": 1.218, + "theoretical_loss": 3.5011929401261996, + "tokens_seen": 1574305792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641323971915747, + "loss": 1.3594, + "theoretical_loss": 3.501167477952249, + "tokens_seen": 1574436864 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.5511974692344666, + "objective/train/avg_token_score": 0.008297212421894073, + "objective/train/docs_used": 893256, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6454378366470337, + "objective/train/original_loss": 3.542894124984741, + "objective/train/theoretical_loss": 3.5011547478827207, + "objective/train/tokens_used": 1574502400, + "objective/train/unlikelihood_loss": 0.1942552775144577, + "theoretical_loss": 3.5011547478827207, + "tokens_seen": 1574502400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641123370110331, + "loss": 1.4515, + "theoretical_loss": 3.5011420184914055, + "tokens_seen": 1574567936 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640922768304915, + "loss": 1.4948, + "theoretical_loss": 3.5011165617431543, + "tokens_seen": 1574699008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026407221664994984, + "loss": 1.3159, + "theoretical_loss": 3.501091107706981, + "tokens_seen": 1574830080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640521564694082, + "loss": 1.3656, + "theoretical_loss": 3.501065656382371, + "tokens_seen": 1574961152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640320962888666, + "loss": 1.4141, + "theoretical_loss": 3.5010402077688094, + "tokens_seen": 1575092224 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264012036108325, + "loss": 1.4627, + "theoretical_loss": 3.5010147618657825, + "tokens_seen": 1575223296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026399197592778334, + "loss": 1.2896, + "theoretical_loss": 3.500989318672775, + "tokens_seen": 1575354368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639719157472417, + "loss": 1.4324, + "theoretical_loss": 3.5009638781892742, + "tokens_seen": 1575485440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026395185556670007, + "loss": 1.4862, + "theoretical_loss": 3.500938440414765, + "tokens_seen": 1575616512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639317953861585, + "loss": 1.4194, + "theoretical_loss": 3.5009130053487345, + "tokens_seen": 1575747584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026391173520561685, + "loss": 1.4097, + "theoretical_loss": 3.5008875729906688, + "tokens_seen": 1575878656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638916750250752, + "loss": 1.4764, + "theoretical_loss": 3.500862143340054, + "tokens_seen": 1576009728 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.3928396999835968, + "objective/train/avg_token_score": 0.018575115129351616, + "objective/train/docs_used": 893728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1646082401275635, + "objective/train/original_loss": 3.788179874420166, + "objective/train/theoretical_loss": 3.500836716396377, + "objective/train/tokens_used": 1576140800, + "objective/train/unlikelihood_loss": 0.15866708755493164, + "theoretical_loss": 3.500836716396377, + "tokens_seen": 1576140800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026387161484453357, + "loss": 1.3758, + "theoretical_loss": 3.500836716396377, + "tokens_seen": 1576140800 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263851554663992, + "loss": 1.3506, + "theoretical_loss": 3.5008112921591246, + "tokens_seen": 1576271872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026383149448345035, + "loss": 1.2939, + "theoretical_loss": 3.5007858706277837, + "tokens_seen": 1576402944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638114343029087, + "loss": 1.2951, + "theoretical_loss": 3.500760451801842, + "tokens_seen": 1576534016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637913741223671, + "loss": 1.2123, + "theoretical_loss": 3.500735035680785, + "tokens_seen": 1576665088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637713139418255, + "loss": 1.3953, + "theoretical_loss": 3.500709622264102, + "tokens_seen": 1576796160 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026375125376128385, + "loss": 1.4892, + "theoretical_loss": 3.5006842115512793, + "tokens_seen": 1576927232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637311935807422, + "loss": 1.4112, + "theoretical_loss": 3.5006588035418043, + "tokens_seen": 1577058304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637111334002006, + "loss": 1.4329, + "theoretical_loss": 3.5006333982351663, + "tokens_seen": 1577189376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026369107321965894, + "loss": 1.3797, + "theoretical_loss": 3.500607995630851, + "tokens_seen": 1577320448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026367101303911736, + "loss": 1.3291, + "theoretical_loss": 3.5005825957283485, + "tokens_seen": 1577451520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636509528585757, + "loss": 1.4574, + "theoretical_loss": 3.5005571985271455, + "tokens_seen": 1577582592 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636308926780341, + "loss": 1.481, + "theoretical_loss": 3.5005318040267315, + "tokens_seen": 1577713664 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.41287878155708313, + "objective/train/avg_token_score": 0.010119074955582619, + "objective/train/docs_used": 894313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.244351863861084, + "objective/train/original_loss": 3.6030240058898926, + "objective/train/theoretical_loss": 3.50051910778916, + "objective/train/tokens_used": 1577779200, + "objective/train/unlikelihood_loss": 0.20202146470546722, + "theoretical_loss": 3.50051910778916, + "tokens_seen": 1577779200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026361083249749244, + "loss": 1.1855, + "theoretical_loss": 3.500506412226594, + "tokens_seen": 1577844736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026359077231695086, + "loss": 1.3847, + "theoretical_loss": 3.5004810231262216, + "tokens_seen": 1577975808 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635707121364092, + "loss": 1.293, + "theoretical_loss": 3.5004556367251043, + "tokens_seen": 1578106880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635506519558676, + "loss": 1.4801, + "theoretical_loss": 3.500430253022729, + "tokens_seen": 1578237952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026353059177532595, + "loss": 1.3707, + "theoretical_loss": 3.5004048720185863, + "tokens_seen": 1578369024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026351053159478436, + "loss": 1.4884, + "theoretical_loss": 3.500379493712165, + "tokens_seen": 1578500096 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634904714142428, + "loss": 1.3134, + "theoretical_loss": 3.500354118102954, + "tokens_seen": 1578631168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026347041123370114, + "loss": 1.3416, + "theoretical_loss": 3.5003287451904423, + "tokens_seen": 1578762240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634503510531595, + "loss": 1.3526, + "theoretical_loss": 3.500303374974121, + "tokens_seen": 1578893312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026343029087261787, + "loss": 1.3828, + "theoretical_loss": 3.5002780074534785, + "tokens_seen": 1579024384 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634102306920763, + "loss": 1.3271, + "theoretical_loss": 3.5002526426280047, + "tokens_seen": 1579155456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026339017051153464, + "loss": 1.4863, + "theoretical_loss": 3.5002272804971906, + "tokens_seen": 1579286528 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.4157502353191376, + "objective/train/avg_token_score": 0.009588765911757946, + "objective/train/docs_used": 895615, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.209296464920044, + "objective/train/original_loss": 3.6651153564453125, + "objective/train/theoretical_loss": 3.5002019210605253, + "objective/train/tokens_used": 1579417600, + "objective/train/unlikelihood_loss": 0.13927039504051208, + "theoretical_loss": 3.5002019210605253, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263370110330993, + "loss": 1.4489, + "theoretical_loss": 3.5002019210605253, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026335005015045137, + "loss": 1.4985, + "theoretical_loss": 3.5001765643174996, + "tokens_seen": 1579548672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026332998996990973, + "loss": 1.4671, + "theoretical_loss": 3.500151210267604, + "tokens_seen": 1579679744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026330992978936815, + "loss": 1.5221, + "theoretical_loss": 3.5001258589103283, + "tokens_seen": 1579810816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632898696088265, + "loss": 1.2462, + "theoretical_loss": 3.5001005102451637, + "tokens_seen": 1579941888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026326980942828487, + "loss": 1.3776, + "theoretical_loss": 3.500075164271601, + "tokens_seen": 1580072960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026324974924774323, + "loss": 1.2836, + "theoretical_loss": 3.5000498209891306, + "tokens_seen": 1580204032 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026322968906720165, + "loss": 1.3422, + "theoretical_loss": 3.5000244803972453, + "tokens_seen": 1580335104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026320962888666, + "loss": 1.3103, + "theoretical_loss": 3.499999142495434, + "tokens_seen": 1580466176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631895687061184, + "loss": 1.2484, + "theoretical_loss": 3.49997380728319, + "tokens_seen": 1580597248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026316950852557674, + "loss": 1.3519, + "theoretical_loss": 3.4999484747600036, + "tokens_seen": 1580728320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631494483450351, + "loss": 1.4657, + "theoretical_loss": 3.499923144925367, + "tokens_seen": 1580859392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631293881644935, + "loss": 1.4429, + "theoretical_loss": 3.4998978177787716, + "tokens_seen": 1580990464 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.4247922897338867, + "objective/train/avg_token_score": 0.012541002593934536, + "objective/train/docs_used": 896135, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2515134811401367, + "objective/train/original_loss": 3.334340810775757, + "objective/train/theoretical_loss": 3.499885155213331, + "objective/train/tokens_used": 1581056000, + "objective/train/unlikelihood_loss": 0.21992668509483337, + "theoretical_loss": 3.499885155213331, + "tokens_seen": 1581056000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631093279839519, + "loss": 1.363, + "theoretical_loss": 3.4998724933197103, + "tokens_seen": 1581121536 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026308926780341024, + "loss": 1.3572, + "theoretical_loss": 3.4998471715476738, + "tokens_seen": 1581252608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630692076228686, + "loss": 1.2751, + "theoretical_loss": 3.499821852462155, + "tokens_seen": 1581383680 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263049147442327, + "loss": 1.4157, + "theoretical_loss": 3.4997965360626466, + "tokens_seen": 1581514752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630290872617854, + "loss": 1.3094, + "theoretical_loss": 3.49977122234864, + "tokens_seen": 1581645824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026300902708124374, + "loss": 1.371, + "theoretical_loss": 3.499745911319629, + "tokens_seen": 1581776896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629889669007021, + "loss": 1.3272, + "theoretical_loss": 3.4997206029751053, + "tokens_seen": 1581907968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026296890672016047, + "loss": 1.3233, + "theoretical_loss": 3.499695297314563, + "tokens_seen": 1582039040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629488465396189, + "loss": 1.3389, + "theoretical_loss": 3.499669994337494, + "tokens_seen": 1582170112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026292878635907725, + "loss": 1.3621, + "theoretical_loss": 3.499644694043392, + "tokens_seen": 1582301184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629087261785356, + "loss": 1.3678, + "theoretical_loss": 3.4996193964317506, + "tokens_seen": 1582432256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026288866599799397, + "loss": 1.4547, + "theoretical_loss": 3.499594101502062, + "tokens_seen": 1582563328 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.32869014143943787, + "objective/train/avg_token_score": 0.010668085888028145, + "objective/train/docs_used": 897089, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8933658003807068, + "objective/train/original_loss": 3.4249885082244873, + "objective/train/theoretical_loss": 3.4995688092538213, + "objective/train/tokens_used": 1582694400, + "objective/train/unlikelihood_loss": 0.20771679282188416, + "theoretical_loss": 3.4995688092538213, + "tokens_seen": 1582694400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628686058174524, + "loss": 1.4228, + "theoretical_loss": 3.4995688092538213, + "tokens_seen": 1582694400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026284854563691075, + "loss": 1.4337, + "theoretical_loss": 3.4995435196865214, + "tokens_seen": 1582825472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628284854563691, + "loss": 1.3344, + "theoretical_loss": 3.499518232799656, + "tokens_seen": 1582956544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628084252758275, + "loss": 1.5207, + "theoretical_loss": 3.49949294859272, + "tokens_seen": 1583087616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627883650952859, + "loss": 1.4446, + "theoretical_loss": 3.4994676670652067, + "tokens_seen": 1583218688 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026276830491474425, + "loss": 1.3512, + "theoretical_loss": 3.49944238821661, + "tokens_seen": 1583349760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627482447342026, + "loss": 1.2224, + "theoretical_loss": 3.4994171120464257, + "tokens_seen": 1583480832 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262728184553661, + "loss": 1.3968, + "theoretical_loss": 3.499391838554147, + "tokens_seen": 1583611904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026270812437311934, + "loss": 1.3745, + "theoretical_loss": 3.49936656773927, + "tokens_seen": 1583742976 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026268806419257776, + "loss": 1.2299, + "theoretical_loss": 3.4993412996012876, + "tokens_seen": 1583874048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626680040120361, + "loss": 1.2604, + "theoretical_loss": 3.4993160341396963, + "tokens_seen": 1584005120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626479438314945, + "loss": 1.5365, + "theoretical_loss": 3.4992907713539907, + "tokens_seen": 1584136192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026262788365095284, + "loss": 1.495, + "theoretical_loss": 3.4992655112436655, + "tokens_seen": 1584267264 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.43065738677978516, + "objective/train/avg_token_score": 0.02605409175157547, + "objective/train/docs_used": 897735, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1509511470794678, + "objective/train/original_loss": 3.3127286434173584, + "objective/train/theoretical_loss": 3.4992528821916133, + "objective/train/tokens_used": 1584332800, + "objective/train/unlikelihood_loss": 0.19494220614433289, + "theoretical_loss": 3.4992528821916133, + "tokens_seen": 1584332800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026260782347041126, + "loss": 1.1913, + "theoretical_loss": 3.4992402538082166, + "tokens_seen": 1584398336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625877632898696, + "loss": 1.3664, + "theoretical_loss": 3.49921499904714, + "tokens_seen": 1584529408 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262567703109328, + "loss": 1.4382, + "theoretical_loss": 3.4991897469599307, + "tokens_seen": 1584660480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026254764292878635, + "loss": 1.4967, + "theoretical_loss": 3.4991644975460843, + "tokens_seen": 1584791552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625275827482447, + "loss": 1.4243, + "theoretical_loss": 3.499139250805097, + "tokens_seen": 1584922624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625075225677031, + "loss": 1.2769, + "theoretical_loss": 3.4991140067364648, + "tokens_seen": 1585053696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624874623871615, + "loss": 1.456, + "theoretical_loss": 3.499088765339684, + "tokens_seen": 1585184768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026246740220661985, + "loss": 1.3911, + "theoretical_loss": 3.4990635266142505, + "tokens_seen": 1585315840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624473420260782, + "loss": 1.4687, + "theoretical_loss": 3.4990382905596618, + "tokens_seen": 1585446912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026242728184553663, + "loss": 1.4008, + "theoretical_loss": 3.4990130571754134, + "tokens_seen": 1585577984 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262407221664995, + "loss": 1.4583, + "theoretical_loss": 3.4989878264610024, + "tokens_seen": 1585709056 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026238716148445335, + "loss": 1.3133, + "theoretical_loss": 3.4989625984159254, + "tokens_seen": 1585840128 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.4529569745063782, + "objective/train/avg_token_score": 0.07449613511562347, + "objective/train/docs_used": 899130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.631143569946289, + "objective/train/original_loss": 4.274294853210449, + "objective/train/theoretical_loss": 3.49893737303968, + "objective/train/tokens_used": 1585971200, + "objective/train/unlikelihood_loss": 0.09464842081069946, + "theoretical_loss": 3.49893737303968, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623671013039117, + "loss": 1.2977, + "theoretical_loss": 3.49893737303968, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623470411233701, + "loss": 1.4023, + "theoretical_loss": 3.498912150331763, + "tokens_seen": 1586102272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623269809428285, + "loss": 1.3949, + "theoretical_loss": 3.498886930291672, + "tokens_seen": 1586233344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026230692076228686, + "loss": 1.4313, + "theoretical_loss": 3.4988617129189037, + "tokens_seen": 1586364416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622868605817452, + "loss": 1.4692, + "theoretical_loss": 3.498836498212956, + "tokens_seen": 1586495488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622668004012036, + "loss": 1.2504, + "theoretical_loss": 3.4988112861733267, + "tokens_seen": 1586626560 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262246740220662, + "loss": 1.2927, + "theoretical_loss": 3.4987860767995143, + "tokens_seen": 1586757632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026222668004012036, + "loss": 1.6315, + "theoretical_loss": 3.498760870091015, + "tokens_seen": 1586888704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622066198595787, + "loss": 1.4812, + "theoretical_loss": 3.4987356660473283, + "tokens_seen": 1587019776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621865596790371, + "loss": 1.3155, + "theoretical_loss": 3.498710464667952, + "tokens_seen": 1587150848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026216649949849545, + "loss": 1.4315, + "theoretical_loss": 3.4986852659523846, + "tokens_seen": 1587281920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026214643931795386, + "loss": 1.393, + "theoretical_loss": 3.498660069900124, + "tokens_seen": 1587412992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621263791374122, + "loss": 1.3213, + "theoretical_loss": 3.4986348765106703, + "tokens_seen": 1587544064 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.4136730134487152, + "objective/train/avg_token_score": 0.018380826339125633, + "objective/train/docs_used": 899656, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3421645164489746, + "objective/train/original_loss": 3.9078314304351807, + "objective/train/theoretical_loss": 3.4986222808143386, + "objective/train/tokens_used": 1587609600, + "objective/train/unlikelihood_loss": 0.16395781934261322, + "theoretical_loss": 3.4986222808143386, + "tokens_seen": 1587609600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621063189568706, + "loss": 1.4221, + "theoretical_loss": 3.4986096857835207, + "tokens_seen": 1587675136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026208625877632895, + "loss": 1.45, + "theoretical_loss": 3.4985844977181744, + "tokens_seen": 1587806208 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026206619859578737, + "loss": 1.4105, + "theoretical_loss": 3.498559312314131, + "tokens_seen": 1587937280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026204613841524573, + "loss": 1.3559, + "theoretical_loss": 3.4985341295708903, + "tokens_seen": 1588068352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620260782347041, + "loss": 1.322, + "theoretical_loss": 3.4985089494879498, + "tokens_seen": 1588199424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620060180541625, + "loss": 1.3565, + "theoretical_loss": 3.49848377206481, + "tokens_seen": 1588330496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026198595787362087, + "loss": 1.3585, + "theoretical_loss": 3.4984585973009708, + "tokens_seen": 1588461568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619658976930793, + "loss": 1.4065, + "theoretical_loss": 3.4984334251959313, + "tokens_seen": 1588592640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026194583751253765, + "loss": 1.3549, + "theoretical_loss": 3.498408255749191, + "tokens_seen": 1588723712 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261925777331996, + "loss": 1.4533, + "theoretical_loss": 3.498383088960251, + "tokens_seen": 1588854784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026190571715145437, + "loss": 1.4087, + "theoretical_loss": 3.4983579248286114, + "tokens_seen": 1588985856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618856569709128, + "loss": 1.3889, + "theoretical_loss": 3.498332763353771, + "tokens_seen": 1589116928 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.528286874294281, + "objective/train/avg_token_score": 0.019421083852648735, + "objective/train/docs_used": 900745, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7070245742797852, + "objective/train/original_loss": 3.474553346633911, + "objective/train/theoretical_loss": 3.4983076045352317, + "objective/train/tokens_used": 1589248000, + "objective/train/unlikelihood_loss": 0.16647744178771973, + "theoretical_loss": 3.4983076045352317, + "tokens_seen": 1589248000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026186559679037115, + "loss": 1.5826, + "theoretical_loss": 3.4983076045352317, + "tokens_seen": 1589248000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618455366098295, + "loss": 1.2389, + "theoretical_loss": 3.4982824483724935, + "tokens_seen": 1589379072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618254764292879, + "loss": 1.3241, + "theoretical_loss": 3.4982572948650565, + "tokens_seen": 1589510144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618054162487463, + "loss": 1.4604, + "theoretical_loss": 3.4982321440124227, + "tokens_seen": 1589641216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026178535606820465, + "loss": 1.3983, + "theoretical_loss": 3.4982069958140922, + "tokens_seen": 1589772288 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261765295887663, + "loss": 1.3543, + "theoretical_loss": 3.498181850269566, + "tokens_seen": 1589903360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617452357071214, + "loss": 1.4946, + "theoretical_loss": 3.498156707378346, + "tokens_seen": 1590034432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026172517552657974, + "loss": 1.333, + "theoretical_loss": 3.4981315671399327, + "tokens_seen": 1590165504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026170511534603816, + "loss": 1.5323, + "theoretical_loss": 3.498106429553828, + "tokens_seen": 1590296576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616850551654965, + "loss": 1.448, + "theoretical_loss": 3.4980812946195337, + "tokens_seen": 1590427648 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616649949849549, + "loss": 1.3053, + "theoretical_loss": 3.4980561623365514, + "tokens_seen": 1590558720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026164493480441324, + "loss": 1.303, + "theoretical_loss": 3.4980310327043824, + "tokens_seen": 1590689792 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026162487462387166, + "loss": 1.2749, + "theoretical_loss": 3.49800590572253, + "tokens_seen": 1590820864 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.47116324305534363, + "objective/train/avg_token_score": 0.01193602941930294, + "objective/train/docs_used": 902130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.640014410018921, + "objective/train/original_loss": 3.6377482414245605, + "objective/train/theoretical_loss": 3.4979933432253163, + "objective/train/tokens_used": 1590886400, + "objective/train/unlikelihood_loss": 0.18238569796085358, + "theoretical_loss": 3.4979933432253163, + "tokens_seen": 1590886400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026160481444333, + "loss": 1.2539, + "theoretical_loss": 3.497980781390495, + "tokens_seen": 1590951936 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615847542627884, + "loss": 1.3981, + "theoretical_loss": 3.49795565970778, + "tokens_seen": 1591083008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026156469408224675, + "loss": 1.377, + "theoretical_loss": 3.4979305406738885, + "tokens_seen": 1591214080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615446339017051, + "loss": 1.4831, + "theoretical_loss": 3.497905424288322, + "tokens_seen": 1591345152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615245737211635, + "loss": 1.215, + "theoretical_loss": 3.4978803105505833, + "tokens_seen": 1591476224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615045135406219, + "loss": 1.4292, + "theoretical_loss": 3.4978551994601754, + "tokens_seen": 1591607296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026148445336008025, + "loss": 1.4288, + "theoretical_loss": 3.4978300910166014, + "tokens_seen": 1591738368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614643931795386, + "loss": 1.3613, + "theoretical_loss": 3.497804985219364, + "tokens_seen": 1591869440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026144433299899703, + "loss": 1.5503, + "theoretical_loss": 3.4977798820679666, + "tokens_seen": 1592000512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614242728184554, + "loss": 1.3798, + "theoretical_loss": 3.4977547815619126, + "tokens_seen": 1592131584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026140421263791375, + "loss": 1.3692, + "theoretical_loss": 3.4977296837007055, + "tokens_seen": 1592262656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613841524573721, + "loss": 1.2044, + "theoretical_loss": 3.497704588483849, + "tokens_seen": 1592393728 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.4322458505630493, + "objective/train/avg_token_score": 0.019476717337965965, + "objective/train/docs_used": 902721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3302463293075562, + "objective/train/original_loss": 3.5585553646087646, + "objective/train/theoretical_loss": 3.4976794959108464, + "objective/train/tokens_used": 1592524800, + "objective/train/unlikelihood_loss": 0.15843220055103302, + "theoretical_loss": 3.4976794959108464, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613640922768305, + "loss": 1.5139, + "theoretical_loss": 3.4976794959108464, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613440320962889, + "loss": 1.3783, + "theoretical_loss": 3.4976544059812023, + "tokens_seen": 1592655872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026132397191574726, + "loss": 1.307, + "theoretical_loss": 3.49762931869442, + "tokens_seen": 1592786944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613039117352056, + "loss": 1.3892, + "theoretical_loss": 3.497604234050005, + "tokens_seen": 1592918016 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261283851554664, + "loss": 1.2132, + "theoretical_loss": 3.49757915204746, + "tokens_seen": 1593049088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612637913741224, + "loss": 1.2788, + "theoretical_loss": 3.49755407268629, + "tokens_seen": 1593180160 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026124373119358076, + "loss": 1.4875, + "theoretical_loss": 3.497528995966, + "tokens_seen": 1593311232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612236710130391, + "loss": 1.252, + "theoretical_loss": 3.497503921886094, + "tokens_seen": 1593442304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612036108324975, + "loss": 1.55, + "theoretical_loss": 3.497478850446077, + "tokens_seen": 1593573376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026118355065195585, + "loss": 1.3877, + "theoretical_loss": 3.497453781645455, + "tokens_seen": 1593704448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026116349047141426, + "loss": 1.3406, + "theoretical_loss": 3.4974287154837316, + "tokens_seen": 1593835520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611434302908726, + "loss": 1.3444, + "theoretical_loss": 3.497403651960413, + "tokens_seen": 1593966592 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261123370110331, + "loss": 1.2012, + "theoretical_loss": 3.4973785910750035, + "tokens_seen": 1594097664 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.527737021446228, + "objective/train/avg_token_score": 0.030332716181874275, + "objective/train/docs_used": 903500, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.546478271484375, + "objective/train/original_loss": 3.0401177406311035, + "objective/train/theoretical_loss": 3.497366061621361, + "objective/train/tokens_used": 1594163200, + "objective/train/unlikelihood_loss": 0.21184727549552917, + "theoretical_loss": 3.497366061621361, + "tokens_seen": 1594163200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026110330992978935, + "loss": 1.4618, + "theoretical_loss": 3.4973535328270104, + "tokens_seen": 1594228736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026108324974924777, + "loss": 1.2977, + "theoretical_loss": 3.4973284772159374, + "tokens_seen": 1594359808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026106318956870613, + "loss": 1.4902, + "theoretical_loss": 3.4973034242412915, + "tokens_seen": 1594490880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610431293881645, + "loss": 1.3168, + "theoretical_loss": 3.4972783739025783, + "tokens_seen": 1594621952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026102306920762285, + "loss": 1.4565, + "theoretical_loss": 3.4972533261993037, + "tokens_seen": 1594753024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610030090270812, + "loss": 1.2773, + "theoretical_loss": 3.4972282811309743, + "tokens_seen": 1594884096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026098294884653963, + "loss": 1.2836, + "theoretical_loss": 3.4972032386970957, + "tokens_seen": 1595015168 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260962888665998, + "loss": 1.3509, + "theoretical_loss": 3.4971781988971746, + "tokens_seen": 1595146240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026094282848545636, + "loss": 1.4105, + "theoretical_loss": 3.4971531617307177, + "tokens_seen": 1595277312 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609227683049147, + "loss": 1.3419, + "theoretical_loss": 3.4971281271972314, + "tokens_seen": 1595408384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026090270812437313, + "loss": 1.3207, + "theoretical_loss": 3.4971030952962234, + "tokens_seen": 1595539456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608826479438315, + "loss": 1.3184, + "theoretical_loss": 3.4970780660271994, + "tokens_seen": 1595670528 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.410190612077713, + "objective/train/avg_token_score": 0.030426690354943275, + "objective/train/docs_used": 904004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1541249752044678, + "objective/train/original_loss": 3.616422414779663, + "objective/train/theoretical_loss": 3.4970530393896673, + "objective/train/tokens_used": 1595801600, + "objective/train/unlikelihood_loss": 0.15032219886779785, + "theoretical_loss": 3.4970530393896673, + "tokens_seen": 1595801600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026086258776328986, + "loss": 1.3134, + "theoretical_loss": 3.4970530393896673, + "tokens_seen": 1595801600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608425275827482, + "loss": 1.2887, + "theoretical_loss": 3.497028015383134, + "tokens_seen": 1595932672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608224674022066, + "loss": 1.3837, + "theoretical_loss": 3.4970029940071075, + "tokens_seen": 1596063744 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260802407221665, + "loss": 1.3068, + "theoretical_loss": 3.4969779752610943, + "tokens_seen": 1596194816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026078234704112336, + "loss": 1.5407, + "theoretical_loss": 3.496952959144603, + "tokens_seen": 1596325888 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607622868605817, + "loss": 1.2811, + "theoretical_loss": 3.4969279456571405, + "tokens_seen": 1596456960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607422266800401, + "loss": 1.2884, + "theoretical_loss": 3.4969029347982152, + "tokens_seen": 1596588032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607221664994985, + "loss": 1.4782, + "theoretical_loss": 3.4968779265673353, + "tokens_seen": 1596719104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026070210631895687, + "loss": 1.2865, + "theoretical_loss": 3.4968529209640087, + "tokens_seen": 1596850176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026068204613841523, + "loss": 1.3048, + "theoretical_loss": 3.4968279179877433, + "tokens_seen": 1596981248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606619859578736, + "loss": 1.495, + "theoretical_loss": 3.496802917638048, + "tokens_seen": 1597112320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026064192577733195, + "loss": 1.3685, + "theoretical_loss": 3.4967779199144307, + "tokens_seen": 1597243392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026062186559679037, + "loss": 1.3591, + "theoretical_loss": 3.496752924816401, + "tokens_seen": 1597374464 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.5180840492248535, + "objective/train/avg_token_score": 0.019945958629250526, + "objective/train/docs_used": 904558, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6410958766937256, + "objective/train/original_loss": 3.6588375568389893, + "objective/train/theoretical_loss": 3.496740428251828, + "objective/train/tokens_used": 1597440000, + "objective/train/unlikelihood_loss": 0.16211923956871033, + "theoretical_loss": 3.496740428251828, + "tokens_seen": 1597440000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026060180541624873, + "loss": 1.5135, + "theoretical_loss": 3.4967279323434677, + "tokens_seen": 1597505536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605817452357071, + "loss": 1.4291, + "theoretical_loss": 3.4967029424951392, + "tokens_seen": 1597636608 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026056168505516546, + "loss": 1.3805, + "theoretical_loss": 3.4966779552709246, + "tokens_seen": 1597767680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026054162487462387, + "loss": 1.4425, + "theoretical_loss": 3.4966529706703335, + "tokens_seen": 1597898752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026052156469408223, + "loss": 1.4135, + "theoretical_loss": 3.4966279886928753, + "tokens_seen": 1598029824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026050150451354065, + "loss": 1.5005, + "theoretical_loss": 3.4966030093380587, + "tokens_seen": 1598160896 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260481444332999, + "loss": 1.4711, + "theoretical_loss": 3.496578032605394, + "tokens_seen": 1598291968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026046138415245743, + "loss": 1.2794, + "theoretical_loss": 3.4965530584943907, + "tokens_seen": 1598423040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604413239719158, + "loss": 1.3617, + "theoretical_loss": 3.4965280870045587, + "tokens_seen": 1598554112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026042126379137415, + "loss": 1.3145, + "theoretical_loss": 3.4965031181354083, + "tokens_seen": 1598685184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604012036108325, + "loss": 1.3487, + "theoretical_loss": 3.496478151886449, + "tokens_seen": 1598816256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603811434302909, + "loss": 1.4301, + "theoretical_loss": 3.496453188257192, + "tokens_seen": 1598947328 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.28861191868782043, + "objective/train/avg_token_score": 0.013060581870377064, + "objective/train/docs_used": 905731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9616495370864868, + "objective/train/original_loss": 3.736227035522461, + "objective/train/theoretical_loss": 3.4964282272471467, + "objective/train/tokens_used": 1599078400, + "objective/train/unlikelihood_loss": 0.1723061054944992, + "theoretical_loss": 3.4964282272471467, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603610832497493, + "loss": 1.2212, + "theoretical_loss": 3.4964282272471467, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026034102306920766, + "loss": 1.1963, + "theoretical_loss": 3.4964032688558246, + "tokens_seen": 1599209472 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260320962888666, + "loss": 1.3233, + "theoretical_loss": 3.4963783130827353, + "tokens_seen": 1599340544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603009027081244, + "loss": 1.4559, + "theoretical_loss": 3.49635335992739, + "tokens_seen": 1599471616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602808425275828, + "loss": 1.3361, + "theoretical_loss": 3.4963284093893003, + "tokens_seen": 1599602688 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026026078234704116, + "loss": 1.4457, + "theoretical_loss": 3.496303461467977, + "tokens_seen": 1599733760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602407221664995, + "loss": 1.4368, + "theoretical_loss": 3.4962785161629304, + "tokens_seen": 1599864832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602206619859579, + "loss": 1.4776, + "theoretical_loss": 3.4962535734736733, + "tokens_seen": 1599995904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026020060180541625, + "loss": 1.3092, + "theoretical_loss": 3.4962286333997152, + "tokens_seen": 1600126976 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026018054162487466, + "loss": 1.3168, + "theoretical_loss": 3.4962036959405696, + "tokens_seen": 1600258048 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260160481444333, + "loss": 1.3243, + "theoretical_loss": 3.496178761095747, + "tokens_seen": 1600389120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002601404212637914, + "loss": 1.3047, + "theoretical_loss": 3.49615382886476, + "tokens_seen": 1600520192 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026012036108324975, + "loss": 1.2649, + "theoretical_loss": 3.49612889924712, + "tokens_seen": 1600651264 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.4623045027256012, + "objective/train/avg_token_score": 0.03187735751271248, + "objective/train/docs_used": 906398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2697887420654297, + "objective/train/original_loss": 3.8459553718566895, + "objective/train/theoretical_loss": 3.496116435418153, + "objective/train/tokens_used": 1600716800, + "objective/train/unlikelihood_loss": 0.13044442236423492, + "theoretical_loss": 3.496116435418153, + "tokens_seen": 1600716800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026010030090270817, + "loss": 1.3145, + "theoretical_loss": 3.4961039722423397, + "tokens_seen": 1600782336 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026008024072216653, + "loss": 1.3987, + "theoretical_loss": 3.49607904784993, + "tokens_seen": 1600913408 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600601805416249, + "loss": 1.4151, + "theoretical_loss": 3.4960541260694056, + "tokens_seen": 1601044480 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026004012036108325, + "loss": 1.3823, + "theoretical_loss": 3.4960292069002765, + "tokens_seen": 1601175552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600200601805416, + "loss": 1.2822, + "theoretical_loss": 3.496004290342057, + "tokens_seen": 1601306624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026000000000000003, + "loss": 1.4314, + "theoretical_loss": 3.49597937639426, + "tokens_seen": 1601437696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599799398194584, + "loss": 1.4051, + "theoretical_loss": 3.495954465056397, + "tokens_seen": 1601568768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025995987963891676, + "loss": 1.2299, + "theoretical_loss": 3.4959295563279817, + "tokens_seen": 1601699840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599398194583751, + "loss": 1.2387, + "theoretical_loss": 3.4959046502085274, + "tokens_seen": 1601830912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025991975927783353, + "loss": 1.349, + "theoretical_loss": 3.4958797466975478, + "tokens_seen": 1601961984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598996990972919, + "loss": 1.3816, + "theoretical_loss": 3.4958548457945553, + "tokens_seen": 1602093056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025987963891675026, + "loss": 1.3599, + "theoretical_loss": 3.4958299474990646, + "tokens_seen": 1602224128 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.31708210706710815, + "objective/train/avg_token_score": 0.07058144360780716, + "objective/train/docs_used": 907690, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0090043544769287, + "objective/train/original_loss": 4.397782802581787, + "objective/train/theoretical_loss": 3.495805051810588, + "objective/train/tokens_used": 1602355200, + "objective/train/unlikelihood_loss": 0.21039800345897675, + "theoretical_loss": 3.495805051810588, + "tokens_seen": 1602355200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598595787362086, + "loss": 1.2619, + "theoretical_loss": 3.495805051810588, + "tokens_seen": 1602355200 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259839518555667, + "loss": 1.3931, + "theoretical_loss": 3.495780158728641, + "tokens_seen": 1602486272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598194583751254, + "loss": 1.1895, + "theoretical_loss": 3.4957552682527355, + "tokens_seen": 1602617344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025979939819458376, + "loss": 1.2337, + "theoretical_loss": 3.4957303803823874, + "tokens_seen": 1602748416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597793380140421, + "loss": 1.4357, + "theoretical_loss": 3.49570549511711, + "tokens_seen": 1602879488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597592778335005, + "loss": 1.2772, + "theoretical_loss": 3.495680612456418, + "tokens_seen": 1603010560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597392176529589, + "loss": 1.3204, + "theoretical_loss": 3.4956557323998254, + "tokens_seen": 1603141632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025971915747241727, + "loss": 1.3867, + "theoretical_loss": 3.4956308549468478, + "tokens_seen": 1603272704 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025969909729187563, + "loss": 1.361, + "theoretical_loss": 3.4956059800969985, + "tokens_seen": 1603403776 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259679037111334, + "loss": 1.3372, + "theoretical_loss": 3.4955811078497927, + "tokens_seen": 1603534848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025965897693079235, + "loss": 1.4468, + "theoretical_loss": 3.4955562382047463, + "tokens_seen": 1603665920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025963891675025077, + "loss": 1.4281, + "theoretical_loss": 3.495531371161374, + "tokens_seen": 1603796992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025961885656970913, + "loss": 1.4534, + "theoretical_loss": 3.4955065067191904, + "tokens_seen": 1603928064 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.4491080045700073, + "objective/train/avg_token_score": 0.009439226239919662, + "objective/train/docs_used": 908320, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2551159858703613, + "objective/train/original_loss": 3.779374837875366, + "objective/train/theoretical_loss": 3.4954940754733936, + "objective/train/tokens_used": 1603993600, + "objective/train/unlikelihood_loss": 0.1358586847782135, + "theoretical_loss": 3.4954940754733936, + "tokens_seen": 1603993600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595987963891675, + "loss": 1.406, + "theoretical_loss": 3.4954816448777115, + "tokens_seen": 1604059136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025957873620862586, + "loss": 1.4101, + "theoretical_loss": 3.4954567856364527, + "tokens_seen": 1604190208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025955867602808427, + "loss": 1.2274, + "theoretical_loss": 3.49543192899493, + "tokens_seen": 1604321280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025953861584754263, + "loss": 1.2594, + "theoretical_loss": 3.495407074952658, + "tokens_seen": 1604452352 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259518555667001, + "loss": 1.35, + "theoretical_loss": 3.4953822235091536, + "tokens_seen": 1604583424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025949849548645936, + "loss": 1.3379, + "theoretical_loss": 3.4953573746639326, + "tokens_seen": 1604714496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594784353059178, + "loss": 1.2631, + "theoretical_loss": 3.4953325284165113, + "tokens_seen": 1604845568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025945837512537614, + "loss": 1.3527, + "theoretical_loss": 3.4953076847664057, + "tokens_seen": 1604976640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594383149448345, + "loss": 1.3466, + "theoretical_loss": 3.4952828437131322, + "tokens_seen": 1605107712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025941825476429286, + "loss": 1.4462, + "theoretical_loss": 3.4952580052562077, + "tokens_seen": 1605238784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593981945837512, + "loss": 1.1982, + "theoretical_loss": 3.495233169395148, + "tokens_seen": 1605369856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025937813440320964, + "loss": 1.3246, + "theoretical_loss": 3.495208336129471, + "tokens_seen": 1605500928 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.32526880502700806, + "objective/train/avg_token_score": 0.025342857465147972, + "objective/train/docs_used": 909776, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0136632919311523, + "objective/train/original_loss": 3.974574089050293, + "objective/train/theoretical_loss": 3.4951835054586926, + "objective/train/tokens_used": 1605632000, + "objective/train/unlikelihood_loss": 0.16367752850055695, + "theoretical_loss": 3.4951835054586926, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259358074222668, + "loss": 1.3307, + "theoretical_loss": 3.4951835054586926, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025933801404212636, + "loss": 1.3718, + "theoretical_loss": 3.4951586773823307, + "tokens_seen": 1605763072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593179538615847, + "loss": 1.3407, + "theoretical_loss": 3.4951338518999027, + "tokens_seen": 1605894144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025929789368104314, + "loss": 1.4772, + "theoretical_loss": 3.4951090290109246, + "tokens_seen": 1606025216 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592778335005015, + "loss": 1.3639, + "theoretical_loss": 3.495084208714915, + "tokens_seen": 1606156288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025925777331995987, + "loss": 1.3769, + "theoretical_loss": 3.4950593910113907, + "tokens_seen": 1606287360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025923771313941823, + "loss": 1.4315, + "theoretical_loss": 3.4950345758998704, + "tokens_seen": 1606418432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592176529588766, + "loss": 1.4597, + "theoretical_loss": 3.495009763379871, + "tokens_seen": 1606549504 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259197592778335, + "loss": 1.2484, + "theoretical_loss": 3.4949849534509108, + "tokens_seen": 1606680576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025917753259779337, + "loss": 1.4069, + "theoretical_loss": 3.4949601461125077, + "tokens_seen": 1606811648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025915747241725173, + "loss": 1.2689, + "theoretical_loss": 3.4949353413641804, + "tokens_seen": 1606942720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591374122367101, + "loss": 1.3027, + "theoretical_loss": 3.494910539205447, + "tokens_seen": 1607073792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591173520561685, + "loss": 1.3385, + "theoretical_loss": 3.4948857396358255, + "tokens_seen": 1607204864 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.48405423760414124, + "objective/train/avg_token_score": 0.009268603287637234, + "objective/train/docs_used": 910493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.571372389793396, + "objective/train/original_loss": 3.7163569927215576, + "objective/train/theoretical_loss": 3.4948733408217816, + "objective/train/tokens_used": 1607270400, + "objective/train/unlikelihood_loss": 0.15302662551403046, + "theoretical_loss": 3.4948733408217816, + "tokens_seen": 1607270400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590972918756269, + "loss": 1.4495, + "theoretical_loss": 3.494860942654835, + "tokens_seen": 1607335936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025907723169508524, + "loss": 1.325, + "theoretical_loss": 3.494836148261995, + "tokens_seen": 1607467008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590571715145436, + "loss": 1.4439, + "theoretical_loss": 3.4948113564568226, + "tokens_seen": 1607598080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025903711133400196, + "loss": 1.3209, + "theoretical_loss": 3.4947865672388376, + "tokens_seen": 1607729152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590170511534604, + "loss": 1.2383, + "theoretical_loss": 3.49476178060756, + "tokens_seen": 1607860224 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589969909729188, + "loss": 1.3271, + "theoretical_loss": 3.4947369965625077, + "tokens_seen": 1607991296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025897693079237716, + "loss": 1.2799, + "theoretical_loss": 3.494712215103201, + "tokens_seen": 1608122368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589568706118355, + "loss": 1.3418, + "theoretical_loss": 3.494687436229159, + "tokens_seen": 1608253440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025893681043129393, + "loss": 1.3319, + "theoretical_loss": 3.494662659939901, + "tokens_seen": 1608384512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589167502507523, + "loss": 1.4145, + "theoretical_loss": 3.4946378862349476, + "tokens_seen": 1608515584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025889669007021066, + "loss": 1.2237, + "theoretical_loss": 3.4946131151138182, + "tokens_seen": 1608646656 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258876629889669, + "loss": 1.4102, + "theoretical_loss": 3.494588346576033, + "tokens_seen": 1608777728 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.5144794583320618, + "objective/train/avg_token_score": 0.0074701374396681786, + "objective/train/docs_used": 911872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.62159264087677, + "objective/train/original_loss": 3.3811094760894775, + "objective/train/theoretical_loss": 3.4945635806211115, + "objective/train/tokens_used": 1608908800, + "objective/train/unlikelihood_loss": 0.17801500856876373, + "theoretical_loss": 3.4945635806211115, + "tokens_seen": 1608908800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588565697091274, + "loss": 1.3427, + "theoretical_loss": 3.4945635806211115, + "tokens_seen": 1608908800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588365095285858, + "loss": 1.528, + "theoretical_loss": 3.494538817248575, + "tokens_seen": 1609039872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025881644934804416, + "loss": 1.2915, + "theoretical_loss": 3.494514056457943, + "tokens_seen": 1609170944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587963891675025, + "loss": 1.2772, + "theoretical_loss": 3.4944892982487366, + "tokens_seen": 1609302016 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587763289869609, + "loss": 1.1015, + "theoretical_loss": 3.4944645426204763, + "tokens_seen": 1609433088 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587562688064193, + "loss": 1.3141, + "theoretical_loss": 3.494439789572683, + "tokens_seen": 1609564160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025873620862587767, + "loss": 1.3451, + "theoretical_loss": 3.494415039104877, + "tokens_seen": 1609695232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025871614844533603, + "loss": 1.4911, + "theoretical_loss": 3.4943902912165807, + "tokens_seen": 1609826304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586960882647944, + "loss": 1.4902, + "theoretical_loss": 3.4943655459073133, + "tokens_seen": 1609957376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025867602808425275, + "loss": 1.333, + "theoretical_loss": 3.4943408031765983, + "tokens_seen": 1610088448 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025865596790371117, + "loss": 1.4259, + "theoretical_loss": 3.4943160630239554, + "tokens_seen": 1610219520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025863590772316953, + "loss": 1.4446, + "theoretical_loss": 3.4942913254489065, + "tokens_seen": 1610350592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586158475426279, + "loss": 1.324, + "theoretical_loss": 3.494266590450974, + "tokens_seen": 1610481664 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.44635874032974243, + "objective/train/avg_token_score": 0.025222182273864746, + "objective/train/docs_used": 912654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.551822304725647, + "objective/train/original_loss": 4.041503429412842, + "objective/train/theoretical_loss": 3.494254223918277, + "objective/train/tokens_used": 1610547200, + "objective/train/unlikelihood_loss": 0.11875547468662262, + "theoretical_loss": 3.494254223918277, + "tokens_seen": 1610547200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025859578736208626, + "loss": 1.3925, + "theoretical_loss": 3.4942418580296795, + "tokens_seen": 1610612736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025857572718154467, + "loss": 1.4197, + "theoretical_loss": 3.4942171281845447, + "tokens_seen": 1610743808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025855566700100303, + "loss": 1.2589, + "theoretical_loss": 3.4941924009150913, + "tokens_seen": 1610874880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585356068204614, + "loss": 1.2721, + "theoretical_loss": 3.494167676220842, + "tokens_seen": 1611005952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025851554663991976, + "loss": 1.2769, + "theoretical_loss": 3.494142954101319, + "tokens_seen": 1611137024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584954864593782, + "loss": 1.4557, + "theoretical_loss": 3.494118234556044, + "tokens_seen": 1611268096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025847542627883654, + "loss": 1.3077, + "theoretical_loss": 3.494093517584542, + "tokens_seen": 1611399168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584553660982949, + "loss": 1.4071, + "theoretical_loss": 3.4940688031863325, + "tokens_seen": 1611530240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025843530591775326, + "loss": 1.2598, + "theoretical_loss": 3.4940440913609407, + "tokens_seen": 1611661312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584152457372116, + "loss": 1.3226, + "theoretical_loss": 3.494019382107888, + "tokens_seen": 1611792384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025839518555667004, + "loss": 1.3381, + "theoretical_loss": 3.4939946754266984, + "tokens_seen": 1611923456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583751253761284, + "loss": 1.4018, + "theoretical_loss": 3.4939699713168952, + "tokens_seen": 1612054528 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.4496578574180603, + "objective/train/avg_token_score": 0.05626415088772774, + "objective/train/docs_used": 913987, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.42634916305542, + "objective/train/original_loss": 3.70835018157959, + "objective/train/theoretical_loss": 3.493945269778002, + "objective/train/tokens_used": 1612185600, + "objective/train/unlikelihood_loss": 0.12739773094654083, + "theoretical_loss": 3.493945269778002, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025835506519558677, + "loss": 1.3051, + "theoretical_loss": 3.493945269778002, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025833500501504513, + "loss": 1.3729, + "theoretical_loss": 3.4939205708095407, + "tokens_seen": 1612316672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025831494483450354, + "loss": 1.3569, + "theoretical_loss": 3.493895874411036, + "tokens_seen": 1612447744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582948846539619, + "loss": 1.2064, + "theoretical_loss": 3.493871180582012, + "tokens_seen": 1612578816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025827482447342027, + "loss": 1.4305, + "theoretical_loss": 3.4938464893219914, + "tokens_seen": 1612709888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025825476429287863, + "loss": 1.477, + "theoretical_loss": 3.4938218006304993, + "tokens_seen": 1612840960 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258234704112337, + "loss": 1.4296, + "theoretical_loss": 3.4937971145070597, + "tokens_seen": 1612972032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582146439317954, + "loss": 1.3688, + "theoretical_loss": 3.493772430951196, + "tokens_seen": 1613103104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025819458375125377, + "loss": 1.2671, + "theoretical_loss": 3.493747749962433, + "tokens_seen": 1613234176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025817452357071213, + "loss": 1.2052, + "theoretical_loss": 3.4937230715402956, + "tokens_seen": 1613365248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581544633901705, + "loss": 1.288, + "theoretical_loss": 3.4936983956843073, + "tokens_seen": 1613496320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581344032096289, + "loss": 1.2388, + "theoretical_loss": 3.4936737223939938, + "tokens_seen": 1613627392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581143430290873, + "loss": 1.4442, + "theoretical_loss": 3.4936490516688803, + "tokens_seen": 1613758464 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.4779447615146637, + "objective/train/avg_token_score": 0.013463466428220272, + "objective/train/docs_used": 914763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5665117502212524, + "objective/train/original_loss": 3.845942497253418, + "objective/train/theoretical_loss": 3.4936367172681244, + "objective/train/tokens_used": 1613824000, + "objective/train/unlikelihood_loss": 0.12711367011070251, + "theoretical_loss": 3.4936367172681244, + "tokens_seen": 1613824000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025809428284854564, + "loss": 1.2266, + "theoretical_loss": 3.4936243835084904, + "tokens_seen": 1613889536 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258074222668004, + "loss": 1.4489, + "theoretical_loss": 3.4935997179123506, + "tokens_seen": 1614020608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025805416248746236, + "loss": 1.2758, + "theoretical_loss": 3.493575054879985, + "tokens_seen": 1614151680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580341023069208, + "loss": 1.5538, + "theoretical_loss": 3.49355039441092, + "tokens_seen": 1614282752 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025801404212637914, + "loss": 1.2834, + "theoretical_loss": 3.4935257365046803, + "tokens_seen": 1614413824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579939819458375, + "loss": 1.3429, + "theoretical_loss": 3.493501081160792, + "tokens_seen": 1614544896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025797392176529586, + "loss": 1.4056, + "theoretical_loss": 3.4934764283787807, + "tokens_seen": 1614675968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579538615847543, + "loss": 1.3367, + "theoretical_loss": 3.493451778158172, + "tokens_seen": 1614807040 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025793380140421264, + "loss": 1.3282, + "theoretical_loss": 3.4934271304984925, + "tokens_seen": 1614938112 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257913741223671, + "loss": 1.279, + "theoretical_loss": 3.493402485399268, + "tokens_seen": 1615069184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025789368104312937, + "loss": 1.3813, + "theoretical_loss": 3.493377842860024, + "tokens_seen": 1615200256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025787362086258773, + "loss": 1.2688, + "theoretical_loss": 3.4933532028802885, + "tokens_seen": 1615331328 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.369562566280365, + "objective/train/avg_token_score": 0.009705348871648312, + "objective/train/docs_used": 916021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2213729619979858, + "objective/train/original_loss": 4.07781982421875, + "objective/train/theoretical_loss": 3.4933285654595867, + "objective/train/tokens_used": 1615462400, + "objective/train/unlikelihood_loss": 0.15859416127204895, + "theoretical_loss": 3.4933285654595867, + "tokens_seen": 1615462400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025785356068204615, + "loss": 1.324, + "theoretical_loss": 3.4933285654595867, + "tokens_seen": 1615462400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578335005015045, + "loss": 1.2582, + "theoretical_loss": 3.493303930597446, + "tokens_seen": 1615593472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025781344032096287, + "loss": 1.352, + "theoretical_loss": 3.493279298293392, + "tokens_seen": 1615724544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025779338014042123, + "loss": 1.2754, + "theoretical_loss": 3.493254668546953, + "tokens_seen": 1615855616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025777331995987965, + "loss": 1.3335, + "theoretical_loss": 3.4932300413576556, + "tokens_seen": 1615986688 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257753259779338, + "loss": 1.2045, + "theoretical_loss": 3.4932054167250257, + "tokens_seen": 1616117760 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577331995987964, + "loss": 1.3628, + "theoretical_loss": 3.4931807946485924, + "tokens_seen": 1616248832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025771313941825474, + "loss": 1.3642, + "theoretical_loss": 3.4931561751278823, + "tokens_seen": 1616379904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576930792377131, + "loss": 1.1523, + "theoretical_loss": 3.4931315581624225, + "tokens_seen": 1616510976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576730190571715, + "loss": 1.3385, + "theoretical_loss": 3.493106943751741, + "tokens_seen": 1616642048 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576529588766299, + "loss": 1.2433, + "theoretical_loss": 3.4930823318953657, + "tokens_seen": 1616773120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025763289869608824, + "loss": 1.3454, + "theoretical_loss": 3.493057722592824, + "tokens_seen": 1616904192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576128385155466, + "loss": 1.3599, + "theoretical_loss": 3.493033115843645, + "tokens_seen": 1617035264 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.37689393758773804, + "objective/train/avg_token_score": 0.011576359160244465, + "objective/train/docs_used": 916782, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1573508977890015, + "objective/train/original_loss": 3.7162678241729736, + "objective/train/theoretical_loss": 3.4930208134264182, + "objective/train/tokens_used": 1617100800, + "objective/train/unlikelihood_loss": 0.1839067041873932, + "theoretical_loss": 3.4930208134264182, + "tokens_seen": 1617100800 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257592778335005, + "loss": 1.3426, + "theoretical_loss": 3.4930085116473553, + "tokens_seen": 1617166336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575727181544634, + "loss": 1.4147, + "theoretical_loss": 3.4929839100034847, + "tokens_seen": 1617297408 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025755265797392174, + "loss": 1.3602, + "theoretical_loss": 3.4929593109115604, + "tokens_seen": 1617428480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575325977933801, + "loss": 1.2789, + "theoretical_loss": 3.4929347143711116, + "tokens_seen": 1617559552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025751253761283847, + "loss": 1.306, + "theoretical_loss": 3.4929101203816666, + "tokens_seen": 1617690624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025749247743229694, + "loss": 1.4786, + "theoretical_loss": 3.492885528942754, + "tokens_seen": 1617821696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574724172517553, + "loss": 1.4795, + "theoretical_loss": 3.4928609400539035, + "tokens_seen": 1617952768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025745235707121366, + "loss": 1.5339, + "theoretical_loss": 3.492836353714643, + "tokens_seen": 1618083840 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257432296890672, + "loss": 1.4289, + "theoretical_loss": 3.4928117699245025, + "tokens_seen": 1618214912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025741223671013044, + "loss": 1.2686, + "theoretical_loss": 3.492787188683011, + "tokens_seen": 1618345984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573921765295888, + "loss": 1.3385, + "theoretical_loss": 3.492762609989698, + "tokens_seen": 1618477056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025737211634904717, + "loss": 1.4187, + "theoretical_loss": 3.492738033844092, + "tokens_seen": 1618608128 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.38013195991516113, + "objective/train/avg_token_score": 0.010659412480890751, + "objective/train/docs_used": 918149, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2131035327911377, + "objective/train/original_loss": 3.8894460201263428, + "objective/train/theoretical_loss": 3.4927134602457244, + "objective/train/tokens_used": 1618739200, + "objective/train/unlikelihood_loss": 0.18793468177318573, + "theoretical_loss": 3.4927134602457244, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025735205616850553, + "loss": 1.3725, + "theoretical_loss": 3.4927134602457244, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025733199598796394, + "loss": 1.4583, + "theoretical_loss": 3.492688889194124, + "tokens_seen": 1618870272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573119358074223, + "loss": 1.3272, + "theoretical_loss": 3.492664320688821, + "tokens_seen": 1619001344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025729187562688067, + "loss": 1.4681, + "theoretical_loss": 3.4926397547293444, + "tokens_seen": 1619132416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025727181544633903, + "loss": 1.4865, + "theoretical_loss": 3.4926151913152257, + "tokens_seen": 1619263488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572517552657974, + "loss": 1.3684, + "theoretical_loss": 3.492590630445995, + "tokens_seen": 1619394560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572316950852558, + "loss": 1.3789, + "theoretical_loss": 3.492566072121181, + "tokens_seen": 1619525632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025721163490471417, + "loss": 1.244, + "theoretical_loss": 3.4925415163403164, + "tokens_seen": 1619656704 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025719157472417253, + "loss": 1.33, + "theoretical_loss": 3.492516963102931, + "tokens_seen": 1619787776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571715145436309, + "loss": 1.3849, + "theoretical_loss": 3.4924924124085557, + "tokens_seen": 1619918848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571514543630893, + "loss": 1.2888, + "theoretical_loss": 3.4924678642567204, + "tokens_seen": 1620049920 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571313941825477, + "loss": 1.3451, + "theoretical_loss": 3.4924433186469575, + "tokens_seen": 1620180992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025711133400200604, + "loss": 1.2213, + "theoretical_loss": 3.4924187755787974, + "tokens_seen": 1620312064 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.37634408473968506, + "objective/train/avg_token_score": 0.026361310854554176, + "objective/train/docs_used": 918561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2444372177124023, + "objective/train/original_loss": 3.814878225326538, + "objective/train/theoretical_loss": 3.492406504997672, + "objective/train/tokens_used": 1620377600, + "objective/train/unlikelihood_loss": 0.2048521339893341, + "theoretical_loss": 3.492406504997672, + "tokens_seen": 1620377600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570912738214644, + "loss": 1.1973, + "theoretical_loss": 3.4923942350517714, + "tokens_seen": 1620443136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025707121364092276, + "loss": 1.3218, + "theoretical_loss": 3.492369697065411, + "tokens_seen": 1620574208 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570511534603812, + "loss": 1.1687, + "theoretical_loss": 3.492345161619248, + "tokens_seen": 1620705280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025703109327983954, + "loss": 1.2769, + "theoretical_loss": 3.492320628712813, + "tokens_seen": 1620836352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570110330992979, + "loss": 1.4045, + "theoretical_loss": 3.492296098345639, + "tokens_seen": 1620967424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025699097291875626, + "loss": 1.1947, + "theoretical_loss": 3.492271570517257, + "tokens_seen": 1621098496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569709127382147, + "loss": 1.2837, + "theoretical_loss": 3.4922470452272, + "tokens_seen": 1621229568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025695085255767304, + "loss": 1.2423, + "theoretical_loss": 3.492222522474999, + "tokens_seen": 1621360640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569307923771314, + "loss": 1.305, + "theoretical_loss": 3.492198002260187, + "tokens_seen": 1621491712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025691073219658977, + "loss": 1.3417, + "theoretical_loss": 3.4921734845822963, + "tokens_seen": 1621622784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025689067201604813, + "loss": 1.3672, + "theoretical_loss": 3.4921489694408585, + "tokens_seen": 1621753856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025687061183550655, + "loss": 1.3289, + "theoretical_loss": 3.4921244568354077, + "tokens_seen": 1621884928 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.4144672453403473, + "objective/train/avg_token_score": 0.0033776697237044573, + "objective/train/docs_used": 919775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2946295738220215, + "objective/train/original_loss": 3.413480520248413, + "objective/train/theoretical_loss": 3.4920999467654754, + "objective/train/tokens_used": 1622016000, + "objective/train/unlikelihood_loss": 0.2035772055387497, + "theoretical_loss": 3.4920999467654754, + "tokens_seen": 1622016000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568505516549649, + "loss": 1.3193, + "theoretical_loss": 3.4920999467654754, + "tokens_seen": 1622016000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025683049147442327, + "loss": 1.2729, + "theoretical_loss": 3.4920754392305957, + "tokens_seen": 1622147072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025681043129388163, + "loss": 1.247, + "theoretical_loss": 3.4920509342303, + "tokens_seen": 1622278144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025679037111334005, + "loss": 1.2705, + "theoretical_loss": 3.492026431764123, + "tokens_seen": 1622409216 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567703109327984, + "loss": 1.3271, + "theoretical_loss": 3.492001931831597, + "tokens_seen": 1622540288 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567502507522568, + "loss": 1.3922, + "theoretical_loss": 3.491977434432256, + "tokens_seen": 1622671360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025673019057171514, + "loss": 1.3023, + "theoretical_loss": 3.4919529395656324, + "tokens_seen": 1622802432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567101303911735, + "loss": 1.4075, + "theoretical_loss": 3.4919284472312606, + "tokens_seen": 1622933504 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566900702106319, + "loss": 1.3726, + "theoretical_loss": 3.4919039574286748, + "tokens_seen": 1623064576 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566700100300903, + "loss": 1.4683, + "theoretical_loss": 3.491879470157408, + "tokens_seen": 1623195648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025664994984954864, + "loss": 1.4475, + "theoretical_loss": 3.4918549854169947, + "tokens_seen": 1623326720 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256629889669007, + "loss": 1.4146, + "theoretical_loss": 3.4918305032069687, + "tokens_seen": 1623457792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566098294884654, + "loss": 1.2161, + "theoretical_loss": 3.4918060235268644, + "tokens_seen": 1623588864 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.380437433719635, + "objective/train/avg_token_score": 0.01148695033043623, + "objective/train/docs_used": 920322, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2076101303100586, + "objective/train/original_loss": 3.6560652256011963, + "objective/train/theoretical_loss": 3.4917937846353873, + "objective/train/tokens_used": 1623654400, + "objective/train/unlikelihood_loss": 0.17890393733978271, + "theoretical_loss": 3.4917937846353873, + "tokens_seen": 1623654400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565897693079238, + "loss": 1.2346, + "theoretical_loss": 3.491781546376216, + "tokens_seen": 1623719936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025656970912738214, + "loss": 1.2696, + "theoretical_loss": 3.491757071754558, + "tokens_seen": 1623851008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565496489468405, + "loss": 1.3547, + "theoretical_loss": 3.4917325996614257, + "tokens_seen": 1623982080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025652958876629887, + "loss": 1.3314, + "theoretical_loss": 3.4917081300963524, + "tokens_seen": 1624113152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565095285857573, + "loss": 1.4378, + "theoretical_loss": 3.4916836630588746, + "tokens_seen": 1624244224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025648946840521565, + "loss": 1.3645, + "theoretical_loss": 3.491659198548526, + "tokens_seen": 1624375296 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256469408224674, + "loss": 1.2554, + "theoretical_loss": 3.4916347365648424, + "tokens_seen": 1624506368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025644934804413237, + "loss": 1.1638, + "theoretical_loss": 3.491610277107358, + "tokens_seen": 1624637440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564292878635908, + "loss": 1.3752, + "theoretical_loss": 3.49158582017561, + "tokens_seen": 1624768512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025640922768304915, + "loss": 1.2528, + "theoretical_loss": 3.4915613657691322, + "tokens_seen": 1624899584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563891675025075, + "loss": 1.3885, + "theoretical_loss": 3.491536913887461, + "tokens_seen": 1625030656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563691073219659, + "loss": 1.3321, + "theoretical_loss": 3.4915124645301314, + "tokens_seen": 1625161728 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.3388318717479706, + "objective/train/avg_token_score": 0.024084439501166344, + "objective/train/docs_used": 921030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1423226594924927, + "objective/train/original_loss": 4.457111835479736, + "objective/train/theoretical_loss": 3.49148801769668, + "objective/train/tokens_used": 1625292800, + "objective/train/unlikelihood_loss": 0.11843366175889969, + "theoretical_loss": 3.49148801769668, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025634904714142424, + "loss": 1.2393, + "theoretical_loss": 3.49148801769668, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025632898696088265, + "loss": 1.2432, + "theoretical_loss": 3.4914635733866426, + "tokens_seen": 1625423872 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256308926780341, + "loss": 1.2294, + "theoretical_loss": 3.491439131599555, + "tokens_seen": 1625554944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562888665997994, + "loss": 1.1645, + "theoretical_loss": 3.4914146923349536, + "tokens_seen": 1625686016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025626880641925774, + "loss": 1.3972, + "theoretical_loss": 3.491390255592374, + "tokens_seen": 1625817088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025624874623871616, + "loss": 1.3575, + "theoretical_loss": 3.491365821371354, + "tokens_seen": 1625948160 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562286860581745, + "loss": 1.2337, + "theoretical_loss": 3.4913413896714296, + "tokens_seen": 1626079232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562086258776329, + "loss": 1.2341, + "theoretical_loss": 3.491316960492137, + "tokens_seen": 1626210304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025618856569709124, + "loss": 1.3048, + "theoretical_loss": 3.491292533833014, + "tokens_seen": 1626341376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025616850551654966, + "loss": 1.1277, + "theoretical_loss": 3.4912681096935962, + "tokens_seen": 1626472448 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256148445336008, + "loss": 1.3795, + "theoretical_loss": 3.4912436880734212, + "tokens_seen": 1626603520 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561283851554664, + "loss": 1.192, + "theoretical_loss": 3.491219268972027, + "tokens_seen": 1626734592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025610832497492475, + "loss": 1.355, + "theoretical_loss": 3.49119485238895, + "tokens_seen": 1626865664 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.3839198350906372, + "objective/train/avg_token_score": 0.01756775937974453, + "objective/train/docs_used": 922470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2739691734313965, + "objective/train/original_loss": 4.220589637756348, + "objective/train/theoretical_loss": 3.4911826450416363, + "objective/train/tokens_used": 1626931200, + "objective/train/unlikelihood_loss": 0.12893523275852203, + "theoretical_loss": 3.4911826450416363, + "tokens_seen": 1626931200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560882647943831, + "loss": 1.4073, + "theoretical_loss": 3.491170438323728, + "tokens_seen": 1626996736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560682046138415, + "loss": 1.2198, + "theoretical_loss": 3.4911460267758985, + "tokens_seen": 1627127808 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560481444332999, + "loss": 1.2237, + "theoretical_loss": 3.491121617744999, + "tokens_seen": 1627258880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025602808425275825, + "loss": 1.2454, + "theoretical_loss": 3.4910972112305676, + "tokens_seen": 1627389952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560080240722166, + "loss": 1.2035, + "theoretical_loss": 3.4910728072321415, + "tokens_seen": 1627521024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559879638916751, + "loss": 1.268, + "theoretical_loss": 3.49104840574926, + "tokens_seen": 1627652096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025596790371113344, + "loss": 1.3223, + "theoretical_loss": 3.49102400678146, + "tokens_seen": 1627783168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559478435305918, + "loss": 1.3792, + "theoretical_loss": 3.4909996103282803, + "tokens_seen": 1627914240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025592778335005017, + "loss": 1.1595, + "theoretical_loss": 3.490975216389259, + "tokens_seen": 1628045312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025590772316950853, + "loss": 1.3768, + "theoretical_loss": 3.490950824963936, + "tokens_seen": 1628176384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025588766298896695, + "loss": 1.3244, + "theoretical_loss": 3.490926436051848, + "tokens_seen": 1628307456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558676028084253, + "loss": 1.3206, + "theoretical_loss": 3.4909020496525347, + "tokens_seen": 1628438528 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.39076244831085205, + "objective/train/avg_token_score": 0.009958990849554539, + "objective/train/docs_used": 922615, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2848808765411377, + "objective/train/original_loss": 3.845836639404297, + "objective/train/theoretical_loss": 3.4908776657655354, + "objective/train/tokens_used": 1628569600, + "objective/train/unlikelihood_loss": 0.17539362609386444, + "theoretical_loss": 3.4908776657655354, + "tokens_seen": 1628569600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025584754262788367, + "loss": 1.2728, + "theoretical_loss": 3.4908776657655354, + "tokens_seen": 1628569600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025582748244734203, + "loss": 1.4341, + "theoretical_loss": 3.490853284390388, + "tokens_seen": 1628700672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025580742226680045, + "loss": 1.158, + "theoretical_loss": 3.490828905526633, + "tokens_seen": 1628831744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557873620862588, + "loss": 1.3165, + "theoretical_loss": 3.490804529173808, + "tokens_seen": 1628962816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557673019057172, + "loss": 1.3332, + "theoretical_loss": 3.4907801553314544, + "tokens_seen": 1629093888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025574724172517554, + "loss": 1.333, + "theoretical_loss": 3.4907557839991097, + "tokens_seen": 1629224960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557271815446339, + "loss": 1.2999, + "theoretical_loss": 3.490731415176315, + "tokens_seen": 1629356032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557071213640923, + "loss": 1.1949, + "theoretical_loss": 3.4907070488626086, + "tokens_seen": 1629487104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556870611835507, + "loss": 1.2344, + "theoretical_loss": 3.490682685057532, + "tokens_seen": 1629618176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025566700100300904, + "loss": 1.1285, + "theoretical_loss": 3.4906583237606243, + "tokens_seen": 1629749248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556469408224674, + "loss": 1.194, + "theoretical_loss": 3.4906339649714258, + "tokens_seen": 1629880320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556268806419258, + "loss": 1.3913, + "theoretical_loss": 3.4906096086894767, + "tokens_seen": 1630011392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556068204613842, + "loss": 1.4201, + "theoretical_loss": 3.4905852549143166, + "tokens_seen": 1630142464 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.41605570912361145, + "objective/train/avg_token_score": 0.009028716944158077, + "objective/train/docs_used": 923942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3585327863693237, + "objective/train/original_loss": 3.6608142852783203, + "objective/train/theoretical_loss": 3.49057307896664, + "objective/train/tokens_used": 1630208000, + "objective/train/unlikelihood_loss": 0.16962948441505432, + "theoretical_loss": 3.49057307896664, + "tokens_seen": 1630208000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025558676028084254, + "loss": 1.262, + "theoretical_loss": 3.4905609036454877, + "tokens_seen": 1630273536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555667001003009, + "loss": 1.3124, + "theoretical_loss": 3.4905365548825285, + "tokens_seen": 1630404608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025554663991975927, + "loss": 1.2151, + "theoretical_loss": 3.490512208624981, + "tokens_seen": 1630535680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555265797392177, + "loss": 1.2989, + "theoretical_loss": 3.490487864872386, + "tokens_seen": 1630666752 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025550651955867605, + "loss": 1.2665, + "theoretical_loss": 3.4904635236242845, + "tokens_seen": 1630797824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554864593781344, + "loss": 1.3485, + "theoretical_loss": 3.490439184880217, + "tokens_seen": 1630928896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025546639919759277, + "loss": 1.2486, + "theoretical_loss": 3.490414848639725, + "tokens_seen": 1631059968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554463390170512, + "loss": 1.319, + "theoretical_loss": 3.4903905149023506, + "tokens_seen": 1631191040 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025542627883650955, + "loss": 1.2326, + "theoretical_loss": 3.490366183667634, + "tokens_seen": 1631322112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554062186559679, + "loss": 1.3351, + "theoretical_loss": 3.4903418549351164, + "tokens_seen": 1631453184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553861584754263, + "loss": 1.4382, + "theoretical_loss": 3.490317528704341, + "tokens_seen": 1631584256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025536609829488464, + "loss": 1.3167, + "theoretical_loss": 3.490293204974849, + "tokens_seen": 1631715328 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.33321115374565125, + "objective/train/avg_token_score": 0.025320002809166908, + "objective/train/docs_used": 924504, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.114802360534668, + "objective/train/original_loss": 3.5832254886627197, + "objective/train/theoretical_loss": 3.4902688837461824, + "objective/train/tokens_used": 1631846400, + "objective/train/unlikelihood_loss": 0.2736862897872925, + "theoretical_loss": 3.4902688837461824, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025534603811434305, + "loss": 1.2642, + "theoretical_loss": 3.4902688837461824, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553259779338014, + "loss": 1.3407, + "theoretical_loss": 3.490244565017883, + "tokens_seen": 1631977472 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553059177532598, + "loss": 1.4681, + "theoretical_loss": 3.4902202487894924, + "tokens_seen": 1632108544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025528585757271814, + "loss": 1.042, + "theoretical_loss": 3.490195935060554, + "tokens_seen": 1632239616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025526579739217656, + "loss": 1.2984, + "theoretical_loss": 3.4901716238306095, + "tokens_seen": 1632370688 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552457372116349, + "loss": 1.3404, + "theoretical_loss": 3.4901473150992017, + "tokens_seen": 1632501760 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552256770310933, + "loss": 1.3775, + "theoretical_loss": 3.490123008865873, + "tokens_seen": 1632632832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025520561685055164, + "loss": 1.3888, + "theoretical_loss": 3.4900987051301664, + "tokens_seen": 1632763904 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025518555667001006, + "loss": 1.2637, + "theoretical_loss": 3.490074403891625, + "tokens_seen": 1632894976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551654964894684, + "loss": 1.4326, + "theoretical_loss": 3.490050105149792, + "tokens_seen": 1633026048 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551454363089268, + "loss": 1.4248, + "theoretical_loss": 3.4900258089042095, + "tokens_seen": 1633157120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025512537612838515, + "loss": 1.3944, + "theoretical_loss": 3.490001515154421, + "tokens_seen": 1633288192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551053159478435, + "loss": 1.3272, + "theoretical_loss": 3.4899772238999707, + "tokens_seen": 1633419264 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.3908235430717468, + "objective/train/avg_token_score": 0.011087484657764435, + "objective/train/docs_used": 925849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3157711029052734, + "objective/train/original_loss": 3.6020219326019287, + "objective/train/theoretical_loss": 3.4899650792083543, + "objective/train/tokens_used": 1633484800, + "objective/train/unlikelihood_loss": 0.15067362785339355, + "theoretical_loss": 3.4899650792083543, + "tokens_seen": 1633484800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550852557673019, + "loss": 1.4163, + "theoretical_loss": 3.4899529351404013, + "tokens_seen": 1633550336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550651955867603, + "loss": 1.2651, + "theoretical_loss": 3.4899286488752566, + "tokens_seen": 1633681408 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025504513540621865, + "loss": 1.3105, + "theoretical_loss": 3.4899043651040804, + "tokens_seen": 1633812480 + }, + { + "epoch": 0.5, + "learning_rate": 0.000255025075225677, + "loss": 1.2872, + "theoretical_loss": 3.489880083826417, + "tokens_seen": 1633943552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025500501504513543, + "loss": 1.2119, + "theoretical_loss": 3.4898558050418096, + "tokens_seen": 1634074624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549849548645938, + "loss": 1.3127, + "theoretical_loss": 3.4898315287498023, + "tokens_seen": 1634205696 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025496489468405215, + "loss": 1.3847, + "theoretical_loss": 3.4898072549499397, + "tokens_seen": 1634336768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549448345035105, + "loss": 1.3887, + "theoretical_loss": 3.489782983641766, + "tokens_seen": 1634467840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549247743229689, + "loss": 1.2338, + "theoretical_loss": 3.489758714824826, + "tokens_seen": 1634598912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549047141424273, + "loss": 1.2506, + "theoretical_loss": 3.489734448498664, + "tokens_seen": 1634729984 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025488465396188566, + "loss": 1.346, + "theoretical_loss": 3.4897101846628242, + "tokens_seen": 1634861056 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254864593781344, + "loss": 1.3913, + "theoretical_loss": 3.489685923316852, + "tokens_seen": 1634992128 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.47061339020729065, + "objective/train/avg_token_score": 0.011187358759343624, + "objective/train/docs_used": 926453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3294051885604858, + "objective/train/original_loss": 3.2428388595581055, + "objective/train/theoretical_loss": 3.489661664460292, + "objective/train/tokens_used": 1635123200, + "objective/train/unlikelihood_loss": 0.20295561850070953, + "theoretical_loss": 3.489661664460292, + "tokens_seen": 1635123200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548445336008024, + "loss": 1.2081, + "theoretical_loss": 3.489661664460292, + "tokens_seen": 1635123200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548244734202608, + "loss": 1.2617, + "theoretical_loss": 3.4896374080926895, + "tokens_seen": 1635254272 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025480441323971916, + "loss": 1.3916, + "theoretical_loss": 3.4896131542135898, + "tokens_seen": 1635385344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547843530591775, + "loss": 1.2976, + "theoretical_loss": 3.489588902822538, + "tokens_seen": 1635516416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547642928786359, + "loss": 1.3783, + "theoretical_loss": 3.489564653919079, + "tokens_seen": 1635647488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025474423269809425, + "loss": 1.4086, + "theoretical_loss": 3.489540407502759, + "tokens_seen": 1635778560 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025472417251755266, + "loss": 1.2709, + "theoretical_loss": 3.489516163573123, + "tokens_seen": 1635909632 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254704112337011, + "loss": 1.4228, + "theoretical_loss": 3.4894919221297176, + "tokens_seen": 1636040704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546840521564694, + "loss": 1.5325, + "theoretical_loss": 3.4894676831720886, + "tokens_seen": 1636171776 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025466399197592775, + "loss": 1.251, + "theoretical_loss": 3.489443446699781, + "tokens_seen": 1636302848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025464393179538616, + "loss": 1.4049, + "theoretical_loss": 3.489419212712342, + "tokens_seen": 1636433920 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025462387161484453, + "loss": 1.2203, + "theoretical_loss": 3.4893949812093172, + "tokens_seen": 1636564992 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546038114343029, + "loss": 1.4133, + "theoretical_loss": 3.4893707521902533, + "tokens_seen": 1636696064 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.29478248953819275, + "objective/train/avg_token_score": 0.07206179201602936, + "objective/train/docs_used": 927487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9006825685501099, + "objective/train/original_loss": 3.6883935928344727, + "objective/train/theoretical_loss": 3.4893586386120647, + "objective/train/tokens_used": 1636761600, + "objective/train/unlikelihood_loss": 0.15889951586723328, + "theoretical_loss": 3.4893586386120647, + "tokens_seen": 1636761600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025458375125376125, + "loss": 1.3946, + "theoretical_loss": 3.489346525654696, + "tokens_seen": 1636827136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545636910732196, + "loss": 1.3378, + "theoretical_loss": 3.4893223016021935, + "tokens_seen": 1636958208 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025454363089267803, + "loss": 1.3084, + "theoretical_loss": 3.489298080032291, + "tokens_seen": 1637089280 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545235707121364, + "loss": 1.2938, + "theoretical_loss": 3.489273860944536, + "tokens_seen": 1637220352 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545035105315948, + "loss": 1.3318, + "theoretical_loss": 3.4892496443384755, + "tokens_seen": 1637351424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025448345035105317, + "loss": 1.2919, + "theoretical_loss": 3.489225430213656, + "tokens_seen": 1637482496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544633901705116, + "loss": 1.4762, + "theoretical_loss": 3.489201218569626, + "tokens_seen": 1637613568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025444332998996995, + "loss": 1.4373, + "theoretical_loss": 3.489177009405931, + "tokens_seen": 1637744640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544232698094283, + "loss": 1.2807, + "theoretical_loss": 3.4891528027221197, + "tokens_seen": 1637875712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544032096288867, + "loss": 1.4658, + "theoretical_loss": 3.4891285985177394, + "tokens_seen": 1638006784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025438314944834504, + "loss": 1.13, + "theoretical_loss": 3.4891043967923374, + "tokens_seen": 1638137856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025436308926780345, + "loss": 1.3644, + "theoretical_loss": 3.489080197545462, + "tokens_seen": 1638268928 + }, + { + "debugging/length": 458.6363636363636, + "debugging/num_segments": 11, + "epoch": 0.5, + "objective/train/aligned_token_count": 0.4280303120613098, + "objective/train/avg_token_score": 0.01861185021698475, + "objective/train/docs_used": 928998, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.470269799232483, + "objective/train/original_loss": 3.9114809036254883, + "objective/train/theoretical_loss": 3.4890560007766607, + "objective/train/tokens_used": 1638400000, + "objective/train/unlikelihood_loss": 0.1560528576374054, + "theoretical_loss": 3.4890560007766607, + "tokens_seen": 1638400000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543430290872618, + "loss": 1.4255, + "theoretical_loss": 3.4890560007766607, + "tokens_seen": 1638400000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543229689067202, + "loss": 1.2854, + "theoretical_loss": 3.4890318064854817, + "tokens_seen": 1638531072 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025430290872617854, + "loss": 1.1818, + "theoretical_loss": 3.489007614671473, + "tokens_seen": 1638662144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025428284854563696, + "loss": 1.4642, + "theoretical_loss": 3.488983425334183, + "tokens_seen": 1638793216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542627883650953, + "loss": 1.2822, + "theoretical_loss": 3.4889592384731603, + "tokens_seen": 1638924288 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542427281845537, + "loss": 1.2215, + "theoretical_loss": 3.4889350540879533, + "tokens_seen": 1639055360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025422266800401204, + "loss": 1.3202, + "theoretical_loss": 3.48891087217811, + "tokens_seen": 1639186432 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025420260782347046, + "loss": 1.3858, + "theoretical_loss": 3.4888866927431796, + "tokens_seen": 1639317504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541825476429288, + "loss": 1.1767, + "theoretical_loss": 3.488862515782711, + "tokens_seen": 1639448576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541624874623872, + "loss": 1.2528, + "theoretical_loss": 3.4888383412962534, + "tokens_seen": 1639579648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025414242728184555, + "loss": 1.2568, + "theoretical_loss": 3.4888141692833554, + "tokens_seen": 1639710720 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541223671013039, + "loss": 1.393, + "theoretical_loss": 3.4887899997435667, + "tokens_seen": 1639841792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541023069207623, + "loss": 1.1813, + "theoretical_loss": 3.488765832676436, + "tokens_seen": 1639972864 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.49413490295410156, + "objective/train/avg_token_score": 0.0052516870200634, + "objective/train/docs_used": 929588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6744496822357178, + "objective/train/original_loss": 3.675575017929077, + "objective/train/theoretical_loss": 3.488753750069977, + "objective/train/tokens_used": 1640038400, + "objective/train/unlikelihood_loss": 0.16906556487083435, + "theoretical_loss": 3.488753750069977, + "tokens_seen": 1640038400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540822467402207, + "loss": 1.3444, + "theoretical_loss": 3.488741668081513, + "tokens_seen": 1640103936 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025406218655967905, + "loss": 1.1941, + "theoretical_loss": 3.4887175059583475, + "tokens_seen": 1640235008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540421263791374, + "loss": 1.2403, + "theoretical_loss": 3.488693346306489, + "tokens_seen": 1640366080 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025402206619859583, + "loss": 1.3564, + "theoretical_loss": 3.4886691891254875, + "tokens_seen": 1640497152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540020060180542, + "loss": 1.2754, + "theoretical_loss": 3.4886450344148923, + "tokens_seen": 1640628224 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025398194583751255, + "loss": 1.3156, + "theoretical_loss": 3.488620882174254, + "tokens_seen": 1640759296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539618856569709, + "loss": 1.3817, + "theoretical_loss": 3.488596732403123, + "tokens_seen": 1640890368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539418254764293, + "loss": 1.3106, + "theoretical_loss": 3.488572585101049, + "tokens_seen": 1641021440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539217652958877, + "loss": 1.2598, + "theoretical_loss": 3.488548440267582, + "tokens_seen": 1641152512 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025390170511534606, + "loss": 1.3057, + "theoretical_loss": 3.488524297902274, + "tokens_seen": 1641283584 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538816449348044, + "loss": 1.2519, + "theoretical_loss": 3.4885001580046744, + "tokens_seen": 1641414656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538615847542628, + "loss": 1.4383, + "theoretical_loss": 3.4884760205743337, + "tokens_seen": 1641545728 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.3999877870082855, + "objective/train/avg_token_score": 0.02124297432601452, + "objective/train/docs_used": 930445, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.333105206489563, + "objective/train/original_loss": 4.2983903884887695, + "objective/train/theoretical_loss": 3.4884518856108038, + "objective/train/tokens_used": 1641676800, + "objective/train/unlikelihood_loss": 0.11932659149169922, + "theoretical_loss": 3.4884518856108038, + "tokens_seen": 1641676800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538415245737212, + "loss": 1.3511, + "theoretical_loss": 3.4884518856108038, + "tokens_seen": 1641676800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025382146439317956, + "loss": 1.1563, + "theoretical_loss": 3.488427753113635, + "tokens_seen": 1641807872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538014042126379, + "loss": 1.4604, + "theoretical_loss": 3.4884036230823785, + "tokens_seen": 1641938944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537813440320963, + "loss": 1.3005, + "theoretical_loss": 3.488379495516586, + "tokens_seen": 1642070016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025376128385155465, + "loss": 1.4066, + "theoretical_loss": 3.488355370415808, + "tokens_seen": 1642201088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025374122367101306, + "loss": 1.1876, + "theoretical_loss": 3.4883312477795965, + "tokens_seen": 1642332160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537211634904714, + "loss": 1.2545, + "theoretical_loss": 3.488307127607503, + "tokens_seen": 1642463232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537011033099298, + "loss": 1.2525, + "theoretical_loss": 3.488283009899079, + "tokens_seen": 1642594304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025368104312938815, + "loss": 1.0791, + "theoretical_loss": 3.488258894653876, + "tokens_seen": 1642725376 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025366098294884657, + "loss": 1.0845, + "theoretical_loss": 3.4882347818714465, + "tokens_seen": 1642856448 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025364092276830493, + "loss": 1.3564, + "theoretical_loss": 3.488210671551342, + "tokens_seen": 1642987520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536208625877633, + "loss": 1.1974, + "theoretical_loss": 3.488186563693116, + "tokens_seen": 1643118592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025360080240722165, + "loss": 1.3297, + "theoretical_loss": 3.488162458296319, + "tokens_seen": 1643249664 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.4965175986289978, + "objective/train/avg_token_score": 0.007709139492362738, + "objective/train/docs_used": 931444, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5427021980285645, + "objective/train/original_loss": 3.8396623134613037, + "objective/train/theoretical_loss": 3.488150406520817, + "objective/train/tokens_used": 1643315200, + "objective/train/unlikelihood_loss": 0.14254799485206604, + "theoretical_loss": 3.488150406520817, + "tokens_seen": 1643315200 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025358074222668, + "loss": 1.4092, + "theoretical_loss": 3.4881383553605043, + "tokens_seen": 1643380736 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025356068204613843, + "loss": 1.2501, + "theoretical_loss": 3.4881142548852244, + "tokens_seen": 1643511808 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535406218655968, + "loss": 1.4, + "theoretical_loss": 3.4880901568700313, + "tokens_seen": 1643642880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025352056168505516, + "loss": 1.3973, + "theoretical_loss": 3.4880660613144787, + "tokens_seen": 1643773952 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535005015045135, + "loss": 1.2279, + "theoretical_loss": 3.488041968218118, + "tokens_seen": 1643905024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025348044132397193, + "loss": 1.3723, + "theoretical_loss": 3.488017877580504, + "tokens_seen": 1644036096 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534603811434303, + "loss": 1.4475, + "theoretical_loss": 3.4879937894011888, + "tokens_seen": 1644167168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025344032096288866, + "loss": 1.4732, + "theoretical_loss": 3.4879697036797257, + "tokens_seen": 1644298240 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253420260782347, + "loss": 1.239, + "theoretical_loss": 3.4879456204156676, + "tokens_seen": 1644429312 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534002006018054, + "loss": 1.2265, + "theoretical_loss": 3.4879215396085685, + "tokens_seen": 1644560384 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533801404212638, + "loss": 1.3488, + "theoretical_loss": 3.487897461257982, + "tokens_seen": 1644691456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025336008024072216, + "loss": 1.3503, + "theoretical_loss": 3.487873385363461, + "tokens_seen": 1644822528 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.4337121248245239, + "objective/train/avg_token_score": 0.008525430224835873, + "objective/train/docs_used": 932028, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.279906153678894, + "objective/train/original_loss": 3.7635815143585205, + "objective/train/theoretical_loss": 3.4878493119245606, + "objective/train/tokens_used": 1644953600, + "objective/train/unlikelihood_loss": 0.24675671756267548, + "theoretical_loss": 3.4878493119245606, + "tokens_seen": 1644953600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533400200601805, + "loss": 1.3337, + "theoretical_loss": 3.4878493119245606, + "tokens_seen": 1644953600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533199598796389, + "loss": 1.4419, + "theoretical_loss": 3.4878252409408335, + "tokens_seen": 1645084672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532998996990973, + "loss": 1.2817, + "theoretical_loss": 3.4878011724118343, + "tokens_seen": 1645215744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025327983951855566, + "loss": 1.3271, + "theoretical_loss": 3.487777106337117, + "tokens_seen": 1645346816 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253259779338014, + "loss": 1.3321, + "theoretical_loss": 3.4877530427162355, + "tokens_seen": 1645477888 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532397191574724, + "loss": 1.5368, + "theoretical_loss": 3.4877289815487447, + "tokens_seen": 1645608960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025321965897693075, + "loss": 1.4216, + "theoretical_loss": 3.487704922834199, + "tokens_seen": 1645740032 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025319959879638917, + "loss": 1.3413, + "theoretical_loss": 3.487680866572153, + "tokens_seen": 1645871104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025317953861584753, + "loss": 1.2732, + "theoretical_loss": 3.487656812762161, + "tokens_seen": 1646002176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002531594784353059, + "loss": 1.162, + "theoretical_loss": 3.4876327614037788, + "tokens_seen": 1646133248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025313941825476425, + "loss": 1.2696, + "theoretical_loss": 3.4876087124965602, + "tokens_seen": 1646264320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025311935807422267, + "loss": 1.3772, + "theoretical_loss": 3.4875846660400605, + "tokens_seen": 1646395392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025309929789368103, + "loss": 1.213, + "theoretical_loss": 3.4875606220338358, + "tokens_seen": 1646526464 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.4188660681247711, + "objective/train/avg_token_score": 0.004527058452367783, + "objective/train/docs_used": 933210, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.461574673652649, + "objective/train/original_loss": 4.046353340148926, + "objective/train/theoretical_loss": 3.487548600949437, + "objective/train/tokens_used": 1646592000, + "objective/train/unlikelihood_loss": 0.161388099193573, + "theoretical_loss": 3.487548600949437, + "tokens_seen": 1646592000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530792377131394, + "loss": 1.3922, + "theoretical_loss": 3.4875365804774408, + "tokens_seen": 1646657536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025305917753259776, + "loss": 1.3333, + "theoretical_loss": 3.48751254137043, + "tokens_seen": 1646788608 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530391173520562, + "loss": 1.3846, + "theoretical_loss": 3.4874885047123607, + "tokens_seen": 1646919680 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025301905717151454, + "loss": 1.2528, + "theoretical_loss": 3.487464470502787, + "tokens_seen": 1647050752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025299899699097295, + "loss": 1.3113, + "theoretical_loss": 3.487440438741266, + "tokens_seen": 1647181824 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529789368104313, + "loss": 1.281, + "theoretical_loss": 3.487416409427352, + "tokens_seen": 1647312896 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529588766298897, + "loss": 1.229, + "theoretical_loss": 3.4873923825606026, + "tokens_seen": 1647443968 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529388164493481, + "loss": 1.3481, + "theoretical_loss": 3.487368358140573, + "tokens_seen": 1647575040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025291875626880646, + "loss": 1.2339, + "theoretical_loss": 3.4873443361668195, + "tokens_seen": 1647706112 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528986960882648, + "loss": 1.1952, + "theoretical_loss": 3.4873203166388986, + "tokens_seen": 1647837184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528786359077232, + "loss": 1.4166, + "theoretical_loss": 3.487296299556367, + "tokens_seen": 1647968256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528585757271816, + "loss": 1.3076, + "theoretical_loss": 3.4872722849187805, + "tokens_seen": 1648099328 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.44782501459121704, + "objective/train/avg_token_score": 0.015664178878068924, + "objective/train/docs_used": 933788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4355547428131104, + "objective/train/original_loss": 3.5010745525360107, + "objective/train/theoretical_loss": 3.4872482727256964, + "objective/train/tokens_used": 1648230400, + "objective/train/unlikelihood_loss": 0.15653972327709198, + "theoretical_loss": 3.4872482727256964, + "tokens_seen": 1648230400 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025283851554663996, + "loss": 1.4883, + "theoretical_loss": 3.4872482727256964, + "tokens_seen": 1648230400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528184553660983, + "loss": 1.253, + "theoretical_loss": 3.487224262976672, + "tokens_seen": 1648361472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527983951855567, + "loss": 1.1585, + "theoretical_loss": 3.487200255671263, + "tokens_seen": 1648492544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025277833500501505, + "loss": 1.2167, + "theoretical_loss": 3.4871762508090267, + "tokens_seen": 1648623616 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025275827482447346, + "loss": 1.2945, + "theoretical_loss": 3.4871522483895214, + "tokens_seen": 1648754688 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527382146439318, + "loss": 1.2069, + "theoretical_loss": 3.487128248412303, + "tokens_seen": 1648885760 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527181544633902, + "loss": 1.2262, + "theoretical_loss": 3.4871042508769294, + "tokens_seen": 1649016832 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025269809428284855, + "loss": 1.339, + "theoretical_loss": 3.4870802557829585, + "tokens_seen": 1649147904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025267803410230697, + "loss": 1.4088, + "theoretical_loss": 3.4870562631299475, + "tokens_seen": 1649278976 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025265797392176533, + "loss": 1.2937, + "theoretical_loss": 3.487032272917454, + "tokens_seen": 1649410048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002526379137412237, + "loss": 1.3656, + "theoretical_loss": 3.4870082851450364, + "tokens_seen": 1649541120 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025261785356068205, + "loss": 1.3124, + "theoretical_loss": 3.486984299812252, + "tokens_seen": 1649672192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525977933801404, + "loss": 1.2656, + "theoretical_loss": 3.486960316918659, + "tokens_seen": 1649803264 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.26472383737564087, + "objective/train/avg_token_score": 0.045460719615221024, + "objective/train/docs_used": 934285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7405925393104553, + "objective/train/original_loss": 3.8140108585357666, + "objective/train/theoretical_loss": 3.4869483263864214, + "objective/train/tokens_used": 1649868800, + "objective/train/unlikelihood_loss": 0.14122989773750305, + "theoretical_loss": 3.4869483263864214, + "tokens_seen": 1649868800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025257773319959883, + "loss": 1.3718, + "theoretical_loss": 3.486936336463816, + "tokens_seen": 1649934336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525576730190572, + "loss": 1.4126, + "theoretical_loss": 3.486912358447281, + "tokens_seen": 1650065408 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025253761283851556, + "loss": 1.3006, + "theoretical_loss": 3.486888382868612, + "tokens_seen": 1650196480 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525175526579739, + "loss": 1.2528, + "theoretical_loss": 3.4868644097273687, + "tokens_seen": 1650327552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025249749247743233, + "loss": 1.2973, + "theoretical_loss": 3.4868404390231085, + "tokens_seen": 1650458624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524774322968907, + "loss": 1.2484, + "theoretical_loss": 3.4868164707553913, + "tokens_seen": 1650589696 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025245737211634906, + "loss": 1.3294, + "theoretical_loss": 3.486792504923775, + "tokens_seen": 1650720768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524373119358074, + "loss": 1.2631, + "theoretical_loss": 3.486768541527819, + "tokens_seen": 1650851840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524172517552658, + "loss": 1.4252, + "theoretical_loss": 3.4867445805670827, + "tokens_seen": 1650982912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523971915747242, + "loss": 1.3484, + "theoretical_loss": 3.4867206220411244, + "tokens_seen": 1651113984 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025237713139418256, + "loss": 1.1583, + "theoretical_loss": 3.4866966659495047, + "tokens_seen": 1651245056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523570712136409, + "loss": 1.166, + "theoretical_loss": 3.486672712291782, + "tokens_seen": 1651376128 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.47537878155708313, + "objective/train/avg_token_score": 0.011349218897521496, + "objective/train/docs_used": 935410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5160956382751465, + "objective/train/original_loss": 3.6912083625793457, + "objective/train/theoretical_loss": 3.4866487610675163, + "objective/train/tokens_used": 1651507200, + "objective/train/unlikelihood_loss": 0.16990655660629272, + "theoretical_loss": 3.4866487610675163, + "tokens_seen": 1651507200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523370110330993, + "loss": 1.2499, + "theoretical_loss": 3.4866487610675163, + "tokens_seen": 1651507200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523169508525577, + "loss": 1.3882, + "theoretical_loss": 3.4866248122762675, + "tokens_seen": 1651638272 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025229689067201606, + "loss": 1.3731, + "theoretical_loss": 3.4866008659175955, + "tokens_seen": 1651769344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025227683049147443, + "loss": 1.4809, + "theoretical_loss": 3.486576921991059, + "tokens_seen": 1651900416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002522567703109328, + "loss": 1.2683, + "theoretical_loss": 3.4865529804962194, + "tokens_seen": 1652031488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025223671013039115, + "loss": 1.2211, + "theoretical_loss": 3.486529041432636, + "tokens_seen": 1652162560 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025221664994984957, + "loss": 1.4103, + "theoretical_loss": 3.4865051047998703, + "tokens_seen": 1652293632 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025219658976930793, + "loss": 1.293, + "theoretical_loss": 3.486481170597481, + "tokens_seen": 1652424704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002521765295887663, + "loss": 1.2536, + "theoretical_loss": 3.4864572388250292, + "tokens_seen": 1652555776 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025215646940822465, + "loss": 1.2247, + "theoretical_loss": 3.4864333094820763, + "tokens_seen": 1652686848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025213640922768307, + "loss": 1.3745, + "theoretical_loss": 3.4864093825681817, + "tokens_seen": 1652817920 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025211634904714143, + "loss": 1.2253, + "theoretical_loss": 3.486385458082907, + "tokens_seen": 1652948992 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002520962888665998, + "loss": 1.2734, + "theoretical_loss": 3.4863615360258136, + "tokens_seen": 1653080064 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.4454423189163208, + "objective/train/avg_token_score": 0.011107598431408405, + "objective/train/docs_used": 935942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2863366603851318, + "objective/train/original_loss": 3.3866539001464844, + "objective/train/theoretical_loss": 3.4863495759076972, + "objective/train/tokens_used": 1653145600, + "objective/train/unlikelihood_loss": 0.24813663959503174, + "theoretical_loss": 3.4863495759076972, + "tokens_seen": 1653145600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025207622868605816, + "loss": 1.1392, + "theoretical_loss": 3.4863376163964617, + "tokens_seen": 1653211136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002520561685055166, + "loss": 1.4119, + "theoretical_loss": 3.4863136991944126, + "tokens_seen": 1653342208 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025203610832497494, + "loss": 1.3748, + "theoretical_loss": 3.4862897844192275, + "tokens_seen": 1653473280 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002520160481444333, + "loss": 1.3952, + "theoretical_loss": 3.4862658720704682, + "tokens_seen": 1653604352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025199598796389166, + "loss": 1.345, + "theoretical_loss": 3.4862419621476963, + "tokens_seen": 1653735424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025197592778335, + "loss": 1.3164, + "theoretical_loss": 3.4862180546504726, + "tokens_seen": 1653866496 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025195586760280844, + "loss": 1.4405, + "theoretical_loss": 3.48619414957836, + "tokens_seen": 1653997568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002519358074222668, + "loss": 1.4076, + "theoretical_loss": 3.4861702469309197, + "tokens_seen": 1654128640 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025191574724172516, + "loss": 1.3878, + "theoretical_loss": 3.486146346707714, + "tokens_seen": 1654259712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002518956870611835, + "loss": 1.3177, + "theoretical_loss": 3.4861224489083042, + "tokens_seen": 1654390784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025187562688064194, + "loss": 1.3799, + "theoretical_loss": 3.4860985535322526, + "tokens_seen": 1654521856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002518555667001003, + "loss": 1.3129, + "theoretical_loss": 3.4860746605791224, + "tokens_seen": 1654652928 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.4734848439693451, + "objective/train/avg_token_score": 0.014535014517605305, + "objective/train/docs_used": 937360, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4967395067214966, + "objective/train/original_loss": 3.5561540126800537, + "objective/train/theoretical_loss": 3.4860507700484757, + "objective/train/tokens_used": 1654784000, + "objective/train/unlikelihood_loss": 0.15629826486110687, + "theoretical_loss": 3.4860507700484757, + "tokens_seen": 1654784000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025183550651955867, + "loss": 1.3555, + "theoretical_loss": 3.4860507700484757, + "tokens_seen": 1654784000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025181544633901703, + "loss": 1.3885, + "theoretical_loss": 3.4860268819398748, + "tokens_seen": 1654915072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517953861584754, + "loss": 1.3411, + "theoretical_loss": 3.4860029962528825, + "tokens_seen": 1655046144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517753259779338, + "loss": 1.4506, + "theoretical_loss": 3.485979112987061, + "tokens_seen": 1655177216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025175526579739217, + "loss": 1.3203, + "theoretical_loss": 3.4859552321419742, + "tokens_seen": 1655308288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025173520561685053, + "loss": 1.2207, + "theoretical_loss": 3.4859313537171843, + "tokens_seen": 1655439360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517151454363089, + "loss": 1.3362, + "theoretical_loss": 3.4859074777122547, + "tokens_seen": 1655570432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516950852557673, + "loss": 1.3631, + "theoretical_loss": 3.485883604126749, + "tokens_seen": 1655701504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516750250752257, + "loss": 1.177, + "theoretical_loss": 3.4858597329602294, + "tokens_seen": 1655832576 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025165496489468404, + "loss": 1.3986, + "theoretical_loss": 3.4858358642122607, + "tokens_seen": 1655963648 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516349047141424, + "loss": 1.2999, + "theoretical_loss": 3.485811997882405, + "tokens_seen": 1656094720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025161484453360076, + "loss": 1.3413, + "theoretical_loss": 3.4857881339702272, + "tokens_seen": 1656225792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515947843530592, + "loss": 1.3424, + "theoretical_loss": 3.485764272475291, + "tokens_seen": 1656356864 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.4725073277950287, + "objective/train/avg_token_score": 0.010051470249891281, + "objective/train/docs_used": 938134, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4361721277236938, + "objective/train/original_loss": 3.498169183731079, + "objective/train/theoretical_loss": 3.485752342634152, + "objective/train/tokens_used": 1656422400, + "objective/train/unlikelihood_loss": 0.16519752144813538, + "theoretical_loss": 3.485752342634152, + "tokens_seen": 1656422400 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025157472417251754, + "loss": 1.275, + "theoretical_loss": 3.48574041339716, + "tokens_seen": 1656487936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515546639919759, + "loss": 1.555, + "theoretical_loss": 3.4857165567353974, + "tokens_seen": 1656619008 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025153460381143426, + "loss": 1.3066, + "theoretical_loss": 3.4856927024895685, + "tokens_seen": 1656750080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515145436308927, + "loss": 1.324, + "theoretical_loss": 3.4856688506592373, + "tokens_seen": 1656881152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514944834503511, + "loss": 1.2228, + "theoretical_loss": 3.4856450012439675, + "tokens_seen": 1657012224 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025147442326980946, + "loss": 1.3398, + "theoretical_loss": 3.485621154243325, + "tokens_seen": 1657143296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514543630892678, + "loss": 1.2083, + "theoretical_loss": 3.4855973096568724, + "tokens_seen": 1657274368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514343029087262, + "loss": 1.2866, + "theoretical_loss": 3.485573467484176, + "tokens_seen": 1657405440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514142427281846, + "loss": 1.2888, + "theoretical_loss": 3.4855496277248, + "tokens_seen": 1657536512 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025139418254764296, + "loss": 1.1426, + "theoretical_loss": 3.4855257903783086, + "tokens_seen": 1657667584 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513741223671013, + "loss": 1.2615, + "theoretical_loss": 3.485501955444268, + "tokens_seen": 1657798656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513540621865597, + "loss": 1.2454, + "theoretical_loss": 3.485478122922243, + "tokens_seen": 1657929728 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.3350439965724945, + "objective/train/avg_token_score": 0.020752713084220886, + "objective/train/docs_used": 939068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9829416871070862, + "objective/train/original_loss": 3.8748042583465576, + "objective/train/theoretical_loss": 3.4854542928117986, + "objective/train/tokens_used": 1658060800, + "objective/train/unlikelihood_loss": 0.18852226436138153, + "theoretical_loss": 3.4854542928117986, + "tokens_seen": 1658060800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513340020060181, + "loss": 1.1946, + "theoretical_loss": 3.4854542928117986, + "tokens_seen": 1658060800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025131394182547647, + "loss": 1.195, + "theoretical_loss": 3.4854304651125005, + "tokens_seen": 1658191872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025129388164493483, + "loss": 1.3931, + "theoretical_loss": 3.485406639823914, + "tokens_seen": 1658322944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512738214643932, + "loss": 1.3744, + "theoretical_loss": 3.485382816945605, + "tokens_seen": 1658454016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025125376128385155, + "loss": 1.3444, + "theoretical_loss": 3.485358996477138, + "tokens_seen": 1658585088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025123370110330997, + "loss": 1.2719, + "theoretical_loss": 3.4853351784180804, + "tokens_seen": 1658716160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025121364092276833, + "loss": 1.2123, + "theoretical_loss": 3.4853113627679972, + "tokens_seen": 1658847232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511935807422267, + "loss": 1.2469, + "theoretical_loss": 3.485287549526454, + "tokens_seen": 1658978304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025117352056168506, + "loss": 1.3088, + "theoretical_loss": 3.4852637386930185, + "tokens_seen": 1659109376 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025115346038114347, + "loss": 1.42, + "theoretical_loss": 3.485239930267256, + "tokens_seen": 1659240448 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025113340020060183, + "loss": 1.2941, + "theoretical_loss": 3.4852161242487325, + "tokens_seen": 1659371520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511133400200602, + "loss": 1.2692, + "theoretical_loss": 3.4851923206370157, + "tokens_seen": 1659502592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025109327983951856, + "loss": 1.4555, + "theoretical_loss": 3.485168519431671, + "tokens_seen": 1659633664 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.5345185995101929, + "objective/train/avg_token_score": 0.0035236843395978212, + "objective/train/docs_used": 939826, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.637244701385498, + "objective/train/original_loss": 3.338872194290161, + "objective/train/theoretical_loss": 3.485156619731253, + "objective/train/tokens_used": 1659699200, + "objective/train/unlikelihood_loss": 0.15959185361862183, + "theoretical_loss": 3.485156619731253, + "tokens_seen": 1659699200 + }, + { + "epoch": 0.5, + "learning_rate": 0.000251073219658977, + "loss": 1.4451, + "theoretical_loss": 3.4851447206322654, + "tokens_seen": 1659764736 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025105315947843534, + "loss": 1.2529, + "theoretical_loss": 3.485120924238366, + "tokens_seen": 1659895808 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002510330992978937, + "loss": 1.2731, + "theoretical_loss": 3.4850971302495397, + "tokens_seen": 1660026880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025101303911735206, + "loss": 1.4237, + "theoretical_loss": 3.485073338665354, + "tokens_seen": 1660157952 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002509929789368104, + "loss": 1.2251, + "theoretical_loss": 3.4850495494853746, + "tokens_seen": 1660289024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025097291875626884, + "loss": 1.3841, + "theoretical_loss": 3.4850257627091707, + "tokens_seen": 1660420096 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002509528585757272, + "loss": 1.2151, + "theoretical_loss": 3.485001978336308, + "tokens_seen": 1660551168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025093279839518556, + "loss": 1.2195, + "theoretical_loss": 3.484978196366355, + "tokens_seen": 1660682240 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002509127382146439, + "loss": 1.275, + "theoretical_loss": 3.484954416798879, + "tokens_seen": 1660813312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025089267803410234, + "loss": 1.3225, + "theoretical_loss": 3.484930639633448, + "tokens_seen": 1660944384 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508726178535607, + "loss": 1.2064, + "theoretical_loss": 3.484906864869629, + "tokens_seen": 1661075456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025085255767301907, + "loss": 1.4629, + "theoretical_loss": 3.484883092506991, + "tokens_seen": 1661206528 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.5449047088623047, + "objective/train/avg_token_score": 0.012051751837134361, + "objective/train/docs_used": 941080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7467271089553833, + "objective/train/original_loss": 3.5269689559936523, + "objective/train/theoretical_loss": 3.484859322545101, + "objective/train/tokens_used": 1661337600, + "objective/train/unlikelihood_loss": 0.18456465005874634, + "theoretical_loss": 3.484859322545101, + "tokens_seen": 1661337600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025083249749247743, + "loss": 1.3709, + "theoretical_loss": 3.484859322545101, + "tokens_seen": 1661337600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508124373119358, + "loss": 1.3285, + "theoretical_loss": 3.484835554983529, + "tokens_seen": 1661468672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507923771313942, + "loss": 1.3476, + "theoretical_loss": 3.484811789821841, + "tokens_seen": 1661599744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025077231695085257, + "loss": 1.32, + "theoretical_loss": 3.4847880270596066, + "tokens_seen": 1661730816 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025075225677031093, + "loss": 1.184, + "theoretical_loss": 3.484764266696394, + "tokens_seen": 1661861888 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507321965897693, + "loss": 1.3799, + "theoretical_loss": 3.4847405087317727, + "tokens_seen": 1661992960 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507121364092277, + "loss": 1.5076, + "theoretical_loss": 3.48471675316531, + "tokens_seen": 1662124032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506920762286861, + "loss": 1.1505, + "theoretical_loss": 3.4846929999965757, + "tokens_seen": 1662255104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025067201604814444, + "loss": 1.3151, + "theoretical_loss": 3.4846692492251385, + "tokens_seen": 1662386176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506519558676028, + "loss": 1.273, + "theoretical_loss": 3.4846455008505677, + "tokens_seen": 1662517248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025063189568706116, + "loss": 1.3254, + "theoretical_loss": 3.484621754872432, + "tokens_seen": 1662648320 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506118355065196, + "loss": 1.3281, + "theoretical_loss": 3.4845980112903003, + "tokens_seen": 1662779392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025059177532597794, + "loss": 1.4176, + "theoretical_loss": 3.484574270103743, + "tokens_seen": 1662910464 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.47562316060066223, + "objective/train/avg_token_score": 0.038315482437610626, + "objective/train/docs_used": 941634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5213299989700317, + "objective/train/original_loss": 3.9211530685424805, + "objective/train/theoretical_loss": 3.484562400408671, + "objective/train/tokens_used": 1662976000, + "objective/train/unlikelihood_loss": 0.18545398116111755, + "theoretical_loss": 3.484562400408671, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505717151454363, + "loss": 1.4065, + "theoretical_loss": 3.4845505313123297, + "tokens_seen": 1663041536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025055165496489466, + "loss": 1.3938, + "theoretical_loss": 3.4845267949156296, + "tokens_seen": 1663172608 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505315947843531, + "loss": 1.2832, + "theoretical_loss": 3.484503060913212, + "tokens_seen": 1663303680 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025051153460381144, + "loss": 1.4793, + "theoretical_loss": 3.484479329304647, + "tokens_seen": 1663434752 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002504914744232698, + "loss": 1.2984, + "theoretical_loss": 3.484455600089505, + "tokens_seen": 1663565824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025047141424272817, + "loss": 1.3878, + "theoretical_loss": 3.4844318732673556, + "tokens_seen": 1663696896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025045135406218653, + "loss": 1.5352, + "theoretical_loss": 3.484408148837769, + "tokens_seen": 1663827968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025043129388164495, + "loss": 1.5178, + "theoretical_loss": 3.484384426800316, + "tokens_seen": 1663959040 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002504112337011033, + "loss": 1.3216, + "theoretical_loss": 3.484360707154566, + "tokens_seen": 1664090112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025039117352056167, + "loss": 1.3703, + "theoretical_loss": 3.4843369899000907, + "tokens_seen": 1664221184 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025037111334002003, + "loss": 1.442, + "theoretical_loss": 3.4843132750364596, + "tokens_seen": 1664352256 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025035105315947845, + "loss": 1.2274, + "theoretical_loss": 3.4842895625632444, + "tokens_seen": 1664483328 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.46413734555244446, + "objective/train/avg_token_score": 0.005206934176385403, + "objective/train/docs_used": 943199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3776192665100098, + "objective/train/original_loss": 3.6426937580108643, + "objective/train/theoretical_loss": 3.484265852480015, + "objective/train/tokens_used": 1664614400, + "objective/train/unlikelihood_loss": 0.1492820531129837, + "theoretical_loss": 3.484265852480015, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503309929789368, + "loss": 1.3907, + "theoretical_loss": 3.484265852480015, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503109327983952, + "loss": 1.359, + "theoretical_loss": 3.484242144786344, + "tokens_seen": 1664745472 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025029087261785354, + "loss": 1.3529, + "theoretical_loss": 3.4842184394818, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502708124373119, + "loss": 1.3259, + "theoretical_loss": 3.4841947365659562, + "tokens_seen": 1665007616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502507522567703, + "loss": 1.4936, + "theoretical_loss": 3.484171036038383, + "tokens_seen": 1665138688 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502306920762287, + "loss": 1.361, + "theoretical_loss": 3.4841473378986523, + "tokens_seen": 1665269760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025021063189568704, + "loss": 1.4968, + "theoretical_loss": 3.4841236421463346, + "tokens_seen": 1665400832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501905717151454, + "loss": 1.3234, + "theoretical_loss": 3.4840999487810027, + "tokens_seen": 1665531904 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501705115346038, + "loss": 1.2395, + "theoretical_loss": 3.484076257802228, + "tokens_seen": 1665662976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501504513540622, + "loss": 1.3845, + "theoretical_loss": 3.484052569209582, + "tokens_seen": 1665794048 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025013039117352054, + "loss": 1.226, + "theoretical_loss": 3.4840288830026367, + "tokens_seen": 1665925120 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501103309929789, + "loss": 1.4426, + "theoretical_loss": 3.4840051991809644, + "tokens_seen": 1666056192 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025009027081243727, + "loss": 1.3146, + "theoretical_loss": 3.4839815177441373, + "tokens_seen": 1666187264 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.45497313141822815, + "objective/train/avg_token_score": 0.01577458716928959, + "objective/train/docs_used": 943856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3804802894592285, + "objective/train/original_loss": 3.608287811279297, + "objective/train/theoretical_loss": 3.4839696779199065, + "objective/train/tokens_used": 1666252800, + "objective/train/unlikelihood_loss": 0.1412777602672577, + "theoretical_loss": 3.4839696779199065, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002500702106318957, + "loss": 1.2808, + "theoretical_loss": 3.483957838691727, + "tokens_seen": 1666318336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025005015045135405, + "loss": 1.0921, + "theoretical_loss": 3.483934162023307, + "tokens_seen": 1666449408 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002500300902708124, + "loss": 1.3715, + "theoretical_loss": 3.4839104877384486, + "tokens_seen": 1666580480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00025001003009027077, + "loss": 1.2589, + "theoretical_loss": 3.4838868158367253, + "tokens_seen": 1666711552 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002499899699097292, + "loss": 1.4713, + "theoretical_loss": 3.4838631463177103, + "tokens_seen": 1666842624 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024996990972918755, + "loss": 1.2953, + "theoretical_loss": 3.4838394791809746, + "tokens_seen": 1666973696 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024994984954864597, + "loss": 1.3555, + "theoretical_loss": 3.4838158144260927, + "tokens_seen": 1667104768 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024992978936810433, + "loss": 1.3528, + "theoretical_loss": 3.4837921520526374, + "tokens_seen": 1667235840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002499097291875627, + "loss": 1.3493, + "theoretical_loss": 3.4837684920601815, + "tokens_seen": 1667366912 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024988966900702105, + "loss": 1.1761, + "theoretical_loss": 3.483744834448298, + "tokens_seen": 1667497984 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498696088264794, + "loss": 1.3125, + "theoretical_loss": 3.483721179216561, + "tokens_seen": 1667629056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024984954864593783, + "loss": 1.2867, + "theoretical_loss": 3.4836975263645438, + "tokens_seen": 1667760128 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.46096041798591614, + "objective/train/avg_token_score": 0.02857697568833828, + "objective/train/docs_used": 945126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3720148801803589, + "objective/train/original_loss": 3.9922549724578857, + "objective/train/theoretical_loss": 3.48367387589182, + "objective/train/tokens_used": 1667891200, + "objective/train/unlikelihood_loss": 0.15566299855709076, + "theoretical_loss": 3.48367387589182, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498294884653962, + "loss": 1.321, + "theoretical_loss": 3.48367387589182, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024980942828485455, + "loss": 1.3973, + "theoretical_loss": 3.4836502277979626, + "tokens_seen": 1668022272 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497893681043129, + "loss": 1.3008, + "theoretical_loss": 3.4836265820825467, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024976930792377133, + "loss": 1.2745, + "theoretical_loss": 3.4836029387451455, + "tokens_seen": 1668284416 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497492477432297, + "loss": 1.3181, + "theoretical_loss": 3.483579297785333, + "tokens_seen": 1668415488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024972918756268806, + "loss": 1.2635, + "theoretical_loss": 3.483555659202683, + "tokens_seen": 1668546560 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497091273821464, + "loss": 1.2574, + "theoretical_loss": 3.483532022996771, + "tokens_seen": 1668677632 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496890672016048, + "loss": 1.4303, + "theoretical_loss": 3.48350838916717, + "tokens_seen": 1668808704 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496690070210632, + "loss": 1.2854, + "theoretical_loss": 3.4834847577134553, + "tokens_seen": 1668939776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024964894684052156, + "loss": 1.4735, + "theoretical_loss": 3.4834611286352013, + "tokens_seen": 1669070848 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496288866599799, + "loss": 1.3784, + "theoretical_loss": 3.483437501931982, + "tokens_seen": 1669201920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024960882647943834, + "loss": 1.2332, + "theoretical_loss": 3.4834138776033736, + "tokens_seen": 1669332992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495887662988967, + "loss": 1.4468, + "theoretical_loss": 3.48339025564895, + "tokens_seen": 1669464064 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.31854838132858276, + "objective/train/avg_token_score": 0.01755235716700554, + "objective/train/docs_used": 945699, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.036392092704773, + "objective/train/original_loss": 4.662958145141602, + "objective/train/theoretical_loss": 3.4833784455619248, + "objective/train/tokens_used": 1669529600, + "objective/train/unlikelihood_loss": 0.1304316371679306, + "theoretical_loss": 3.4833784455619248, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024956870611835506, + "loss": 1.3812, + "theoretical_loss": 3.4833666360682867, + "tokens_seen": 1669595136 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495486459378135, + "loss": 1.209, + "theoretical_loss": 3.4833430188609587, + "tokens_seen": 1669726208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024952858575727184, + "loss": 1.3502, + "theoretical_loss": 3.483319404026541, + "tokens_seen": 1669857280 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495085255767302, + "loss": 1.3923, + "theoretical_loss": 3.4832957915646086, + "tokens_seen": 1669988352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024948846539618857, + "loss": 1.2488, + "theoretical_loss": 3.4832721814747383, + "tokens_seen": 1670119424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024946840521564693, + "loss": 1.199, + "theoretical_loss": 3.4832485737565047, + "tokens_seen": 1670250496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024944834503510535, + "loss": 1.4986, + "theoretical_loss": 3.4832249684094836, + "tokens_seen": 1670381568 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002494282848545637, + "loss": 1.3587, + "theoretical_loss": 3.483201365433251, + "tokens_seen": 1670512640 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024940822467402207, + "loss": 1.3996, + "theoretical_loss": 3.4831777648273823, + "tokens_seen": 1670643712 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024938816449348043, + "loss": 1.5617, + "theoretical_loss": 3.4831541665914543, + "tokens_seen": 1670774784 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024936810431293885, + "loss": 1.2024, + "theoretical_loss": 3.4831305707250424, + "tokens_seen": 1670905856 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002493480441323972, + "loss": 1.3126, + "theoretical_loss": 3.4831069772277234, + "tokens_seen": 1671036928 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.4166666567325592, + "objective/train/avg_token_score": 0.010816219262778759, + "objective/train/docs_used": 946326, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.274077296257019, + "objective/train/original_loss": 3.661715269088745, + "objective/train/theoretical_loss": 3.4830833860990733, + "objective/train/tokens_used": 1671168000, + "objective/train/unlikelihood_loss": 0.22384794056415558, + "theoretical_loss": 3.4830833860990733, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002493279839518556, + "loss": 1.4077, + "theoretical_loss": 3.4830833860990733, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024930792377131394, + "loss": 1.2313, + "theoretical_loss": 3.4830597973386683, + "tokens_seen": 1671299072 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492878635907723, + "loss": 1.2292, + "theoretical_loss": 3.4830362109460857, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492678034102307, + "loss": 1.3053, + "theoretical_loss": 3.4830126269209014, + "tokens_seen": 1671561216 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492477432296891, + "loss": 1.501, + "theoretical_loss": 3.4829890452626926, + "tokens_seen": 1671692288 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024922768304914744, + "loss": 1.4541, + "theoretical_loss": 3.482965465971036, + "tokens_seen": 1671823360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492076228686058, + "loss": 1.4505, + "theoretical_loss": 3.482941889045509, + "tokens_seen": 1671954432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491875626880642, + "loss": 1.3059, + "theoretical_loss": 3.482918314485688, + "tokens_seen": 1672085504 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491675025075226, + "loss": 1.255, + "theoretical_loss": 3.482894742291151, + "tokens_seen": 1672216576 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024914744232698094, + "loss": 1.3683, + "theoretical_loss": 3.4828711724614747, + "tokens_seen": 1672347648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491273821464393, + "loss": 1.1753, + "theoretical_loss": 3.4828476049962367, + "tokens_seen": 1672478720 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024910732196589767, + "loss": 1.2373, + "theoretical_loss": 3.4828240398950143, + "tokens_seen": 1672609792 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490872617853561, + "loss": 1.2997, + "theoretical_loss": 3.482800477157386, + "tokens_seen": 1672740864 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.416727751493454, + "objective/train/avg_token_score": 0.01752919703722, + "objective/train/docs_used": 947668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2943665981292725, + "objective/train/original_loss": 3.701707124710083, + "objective/train/theoretical_loss": 3.4827886966747874, + "objective/train/tokens_used": 1672806400, + "objective/train/unlikelihood_loss": 0.19002114236354828, + "theoretical_loss": 3.4827886966747874, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024906720160481445, + "loss": 1.4289, + "theoretical_loss": 3.482776916782929, + "tokens_seen": 1672871936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490471414242728, + "loss": 1.3189, + "theoretical_loss": 3.4827533587712205, + "tokens_seen": 1673003008 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024902708124373117, + "loss": 1.2257, + "theoretical_loss": 3.4827298031218397, + "tokens_seen": 1673134080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490070210631896, + "loss": 1.3263, + "theoretical_loss": 3.4827062498343637, + "tokens_seen": 1673265152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024898696088264795, + "loss": 1.398, + "theoretical_loss": 3.4826826989083712, + "tokens_seen": 1673396224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002489669007021063, + "loss": 1.2559, + "theoretical_loss": 3.4826591503434408, + "tokens_seen": 1673527296 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002489468405215647, + "loss": 1.2634, + "theoretical_loss": 3.4826356041391495, + "tokens_seen": 1673658368 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024892678034102304, + "loss": 1.407, + "theoretical_loss": 3.4826120602950774, + "tokens_seen": 1673789440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024890672016048145, + "loss": 1.341, + "theoretical_loss": 3.482588518810802, + "tokens_seen": 1673920512 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488866599799398, + "loss": 1.4185, + "theoretical_loss": 3.482564979685903, + "tokens_seen": 1674051584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024886659979939823, + "loss": 1.2727, + "theoretical_loss": 3.482541442919959, + "tokens_seen": 1674182656 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488465396188566, + "loss": 1.3062, + "theoretical_loss": 3.4825179085125484, + "tokens_seen": 1674313728 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.40414223074913025, + "objective/train/avg_token_score": 0.03180708736181259, + "objective/train/docs_used": 948408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2921169996261597, + "objective/train/original_loss": 3.8018040657043457, + "objective/train/theoretical_loss": 3.4824943764632503, + "objective/train/tokens_used": 1674444800, + "objective/train/unlikelihood_loss": 0.21437034010887146, + "theoretical_loss": 3.4824943764632503, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024882647943831496, + "loss": 1.3751, + "theoretical_loss": 3.4824943764632503, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488064192577733, + "loss": 1.4352, + "theoretical_loss": 3.4824708467716445, + "tokens_seen": 1674575872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024878635907723173, + "loss": 1.3827, + "theoretical_loss": 3.48244731943731, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487662988966901, + "loss": 1.5418, + "theoretical_loss": 3.4824237944598257, + "tokens_seen": 1674838016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024874623871614846, + "loss": 1.1646, + "theoretical_loss": 3.482400271838771, + "tokens_seen": 1674969088 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487261785356068, + "loss": 1.3318, + "theoretical_loss": 3.4823767515737267, + "tokens_seen": 1675100160 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487061183550652, + "loss": 1.2438, + "theoretical_loss": 3.4823532336642717, + "tokens_seen": 1675231232 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486860581745236, + "loss": 1.494, + "theoretical_loss": 3.4823297181099857, + "tokens_seen": 1675362304 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024866599799398196, + "loss": 1.2139, + "theoretical_loss": 3.4823062049104485, + "tokens_seen": 1675493376 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486459378134403, + "loss": 1.4671, + "theoretical_loss": 3.4822826940652405, + "tokens_seen": 1675624448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486258776328987, + "loss": 1.4257, + "theoretical_loss": 3.482259185573942, + "tokens_seen": 1675755520 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486058174523571, + "loss": 1.2949, + "theoretical_loss": 3.4822356794361324, + "tokens_seen": 1675886592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024858575727181546, + "loss": 1.4281, + "theoretical_loss": 3.482212175651393, + "tokens_seen": 1676017664 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.39821603894233704, + "objective/train/avg_token_score": 0.01637439988553524, + "objective/train/docs_used": 949481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2431435585021973, + "objective/train/original_loss": 3.5968644618988037, + "objective/train/theoretical_loss": 3.4822004246412934, + "objective/train/tokens_used": 1676083200, + "objective/train/unlikelihood_loss": 0.21864719688892365, + "theoretical_loss": 3.4822004246412934, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485656970912738, + "loss": 1.3087, + "theoretical_loss": 3.4821886742193042, + "tokens_seen": 1676148736 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485456369107322, + "loss": 1.1968, + "theoretical_loss": 3.4821651751394453, + "tokens_seen": 1676279808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024852557673019055, + "loss": 1.3418, + "theoretical_loss": 3.4821416784113985, + "tokens_seen": 1676410880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024850551654964897, + "loss": 1.1862, + "theoretical_loss": 3.4821181840347437, + "tokens_seen": 1676541952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024848545636910733, + "loss": 1.3956, + "theoretical_loss": 3.4820946920090625, + "tokens_seen": 1676673024 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002484653961885657, + "loss": 1.4052, + "theoretical_loss": 3.482071202333935, + "tokens_seen": 1676804096 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024844533600802405, + "loss": 1.3808, + "theoretical_loss": 3.4820477150089433, + "tokens_seen": 1676935168 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024842527582748247, + "loss": 1.3435, + "theoretical_loss": 3.4820242300336677, + "tokens_seen": 1677066240 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024840521564694083, + "loss": 1.3134, + "theoretical_loss": 3.4820007474076893, + "tokens_seen": 1677197312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483851554663992, + "loss": 1.376, + "theoretical_loss": 3.4819772671305906, + "tokens_seen": 1677328384 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024836509528585756, + "loss": 1.4247, + "theoretical_loss": 3.4819537892019525, + "tokens_seen": 1677459456 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248345035105316, + "loss": 1.3873, + "theoretical_loss": 3.481930313621357, + "tokens_seen": 1677590528 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.3252077102661133, + "objective/train/avg_token_score": 0.03028898313641548, + "objective/train/docs_used": 950694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0555111169815063, + "objective/train/original_loss": 3.979170322418213, + "objective/train/theoretical_loss": 3.4819068403883855, + "objective/train/tokens_used": 1677721600, + "objective/train/unlikelihood_loss": 0.14277155697345734, + "theoretical_loss": 3.4819068403883855, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024832497492477434, + "loss": 1.3149, + "theoretical_loss": 3.4819068403883855, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483049147442327, + "loss": 1.3291, + "theoretical_loss": 3.4818833695026195, + "tokens_seen": 1677852672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024828485456369106, + "loss": 1.3454, + "theoretical_loss": 3.481859900963642, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482647943831494, + "loss": 1.4362, + "theoretical_loss": 3.481836434771034, + "tokens_seen": 1678114816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024824473420260784, + "loss": 1.3191, + "theoretical_loss": 3.481812970924379, + "tokens_seen": 1678245888 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482246740220662, + "loss": 1.3406, + "theoretical_loss": 3.4817895094232574, + "tokens_seen": 1678376960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024820461384152456, + "loss": 1.3919, + "theoretical_loss": 3.481766050267253, + "tokens_seen": 1678508032 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481845536609829, + "loss": 1.3979, + "theoretical_loss": 3.4817425934559476, + "tokens_seen": 1678639104 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024816449348044134, + "loss": 1.2537, + "theoretical_loss": 3.4817191389889244, + "tokens_seen": 1678770176 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481444332998997, + "loss": 1.324, + "theoretical_loss": 3.481695686865766, + "tokens_seen": 1678901248 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024812437311935807, + "loss": 1.2753, + "theoretical_loss": 3.4816722370860544, + "tokens_seen": 1679032320 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481043129388165, + "loss": 1.2948, + "theoretical_loss": 3.4816487896493733, + "tokens_seen": 1679163392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024808425275827485, + "loss": 1.3095, + "theoretical_loss": 3.4816253445553063, + "tokens_seen": 1679294464 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.5036656856536865, + "objective/train/avg_token_score": 0.030386311933398247, + "objective/train/docs_used": 951339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5700137615203857, + "objective/train/original_loss": 3.900503635406494, + "objective/train/theoretical_loss": 3.4816136228866217, + "objective/train/tokens_used": 1679360000, + "objective/train/unlikelihood_loss": 0.1619800329208374, + "theoretical_loss": 3.4816136228866217, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002480641925777332, + "loss": 1.4815, + "theoretical_loss": 3.481601901803435, + "tokens_seen": 1679425536 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024804413239719157, + "loss": 1.3973, + "theoretical_loss": 3.4815784613933434, + "tokens_seen": 1679556608 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024802407221665, + "loss": 1.355, + "theoretical_loss": 3.4815550233246153, + "tokens_seen": 1679687680 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024800401203610835, + "loss": 1.3329, + "theoretical_loss": 3.4815315875968333, + "tokens_seen": 1679818752 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002479839518555667, + "loss": 1.385, + "theoretical_loss": 3.481508154209582, + "tokens_seen": 1679949824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002479638916750251, + "loss": 1.3364, + "theoretical_loss": 3.4814847231624437, + "tokens_seen": 1680080896 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024794383149448344, + "loss": 1.4725, + "theoretical_loss": 3.4814612944550034, + "tokens_seen": 1680211968 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024792377131394185, + "loss": 1.3465, + "theoretical_loss": 3.4814378680868447, + "tokens_seen": 1680343040 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002479037111334002, + "loss": 1.3275, + "theoretical_loss": 3.4814144440575507, + "tokens_seen": 1680474112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002478836509528586, + "loss": 1.4379, + "theoretical_loss": 3.481391022366707, + "tokens_seen": 1680605184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024786359077231694, + "loss": 1.5313, + "theoretical_loss": 3.4813676030138963, + "tokens_seen": 1680736256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024784353059177536, + "loss": 1.5067, + "theoretical_loss": 3.481344185998704, + "tokens_seen": 1680867328 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.49841153621673584, + "objective/train/avg_token_score": 0.006667111534625292, + "objective/train/docs_used": 952727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5489822626113892, + "objective/train/original_loss": 3.4995641708374023, + "objective/train/theoretical_loss": 3.481320771320714, + "objective/train/tokens_used": 1680998400, + "objective/train/unlikelihood_loss": 0.17922058701515198, + "theoretical_loss": 3.481320771320714, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002478234704112337, + "loss": 1.395, + "theoretical_loss": 3.481320771320714, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002478034102306921, + "loss": 1.2694, + "theoretical_loss": 3.4812973589795106, + "tokens_seen": 1681129472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024778335005015044, + "loss": 1.2415, + "theoretical_loss": 3.4812739489746796, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024776328986960886, + "loss": 1.3662, + "theoretical_loss": 3.481250541305804, + "tokens_seen": 1681391616 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477432296890672, + "loss": 1.3605, + "theoretical_loss": 3.4812271359724702, + "tokens_seen": 1681522688 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477231695085256, + "loss": 1.3012, + "theoretical_loss": 3.481203732974262, + "tokens_seen": 1681653760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024770310932798395, + "loss": 1.3704, + "theoretical_loss": 3.481180332310765, + "tokens_seen": 1681784832 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476830491474423, + "loss": 1.2926, + "theoretical_loss": 3.4811569339815644, + "tokens_seen": 1681915904 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476629889669007, + "loss": 1.376, + "theoretical_loss": 3.4811335379862456, + "tokens_seen": 1682046976 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476429287863591, + "loss": 1.3669, + "theoretical_loss": 3.481110144324393, + "tokens_seen": 1682178048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024762286860581745, + "loss": 1.5293, + "theoretical_loss": 3.4810867529955933, + "tokens_seen": 1682309120 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476028084252758, + "loss": 1.2853, + "theoretical_loss": 3.4810633639994313, + "tokens_seen": 1682440192 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024758274824473423, + "loss": 1.3285, + "theoretical_loss": 3.4810399773354925, + "tokens_seen": 1682571264 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.38880741596221924, + "objective/train/avg_token_score": 0.01339726522564888, + "objective/train/docs_used": 953335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1646889448165894, + "objective/train/original_loss": 3.8061773777008057, + "objective/train/theoretical_loss": 3.4810282848779783, + "objective/train/tokens_used": 1682636800, + "objective/train/unlikelihood_loss": 0.17127735912799835, + "theoretical_loss": 3.4810282848779783, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475626880641926, + "loss": 1.47, + "theoretical_loss": 3.481016593003364, + "tokens_seen": 1682702336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024754262788365095, + "loss": 1.1842, + "theoretical_loss": 3.4809932110026303, + "tokens_seen": 1682833408 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475225677031093, + "loss": 1.4406, + "theoretical_loss": 3.480969831332878, + "tokens_seen": 1682964480 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475025075225677, + "loss": 1.126, + "theoretical_loss": 3.480946453993693, + "tokens_seen": 1683095552 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474824473420261, + "loss": 1.2874, + "theoretical_loss": 3.480923078984662, + "tokens_seen": 1683226624 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024746238716148445, + "loss": 1.3531, + "theoretical_loss": 3.4808997063053706, + "tokens_seen": 1683357696 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474423269809428, + "loss": 1.3597, + "theoretical_loss": 3.480876335955406, + "tokens_seen": 1683488768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474222668004012, + "loss": 1.1587, + "theoretical_loss": 3.4808529679343536, + "tokens_seen": 1683619840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474022066198596, + "loss": 1.2067, + "theoretical_loss": 3.4808296022418013, + "tokens_seen": 1683750912 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024738214643931796, + "loss": 1.3441, + "theoretical_loss": 3.480806238877335, + "tokens_seen": 1683881984 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002473620862587764, + "loss": 1.2422, + "theoretical_loss": 3.480782877840542, + "tokens_seen": 1684013056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024734202607823474, + "loss": 1.2791, + "theoretical_loss": 3.4807595191310092, + "tokens_seen": 1684144128 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.30687928199768066, + "objective/train/avg_token_score": 0.017612647265195847, + "objective/train/docs_used": 954676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9694479703903198, + "objective/train/original_loss": 3.810652256011963, + "objective/train/theoretical_loss": 3.4807361627483235, + "objective/train/tokens_used": 1684275200, + "objective/train/unlikelihood_loss": 0.24418771266937256, + "theoretical_loss": 3.4807361627483235, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002473219658976931, + "loss": 1.3515, + "theoretical_loss": 3.4807361627483235, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024730190571715146, + "loss": 1.358, + "theoretical_loss": 3.480712808692072, + "tokens_seen": 1684406272 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472818455366098, + "loss": 1.2475, + "theoretical_loss": 3.4806894569618425, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024726178535606824, + "loss": 1.2772, + "theoretical_loss": 3.480666107557221, + "tokens_seen": 1684668416 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472417251755266, + "loss": 1.2457, + "theoretical_loss": 3.4806427604777967, + "tokens_seen": 1684799488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024722166499498496, + "loss": 1.2359, + "theoretical_loss": 3.4806194157231567, + "tokens_seen": 1684930560 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472016048144433, + "loss": 1.2943, + "theoretical_loss": 3.4805960732928876, + "tokens_seen": 1685061632 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024718154463390174, + "loss": 1.2798, + "theoretical_loss": 3.4805727331865786, + "tokens_seen": 1685192704 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002471614844533601, + "loss": 1.3824, + "theoretical_loss": 3.480549395403817, + "tokens_seen": 1685323776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024714142427281847, + "loss": 1.2131, + "theoretical_loss": 3.4805260599441903, + "tokens_seen": 1685454848 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024712136409227683, + "loss": 1.3151, + "theoretical_loss": 3.4805027268072877, + "tokens_seen": 1685585920 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002471013039117352, + "loss": 1.4351, + "theoretical_loss": 3.4804793959926963, + "tokens_seen": 1685716992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470812437311936, + "loss": 1.3906, + "theoretical_loss": 3.4804560675000054, + "tokens_seen": 1685848064 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.44031035900115967, + "objective/train/avg_token_score": 0.021482110023498535, + "objective/train/docs_used": 955358, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1873342990875244, + "objective/train/original_loss": 4.094570159912109, + "objective/train/theoretical_loss": 3.4804444041242437, + "objective/train/tokens_used": 1685913600, + "objective/train/unlikelihood_loss": 0.1238083764910698, + "theoretical_loss": 3.4804444041242437, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024706118355065197, + "loss": 1.3823, + "theoretical_loss": 3.480432741328803, + "tokens_seen": 1685979136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024704112337011033, + "loss": 1.2945, + "theoretical_loss": 3.4804094174786773, + "tokens_seen": 1686110208 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470210631895687, + "loss": 1.338, + "theoretical_loss": 3.4803860959492177, + "tokens_seen": 1686241280 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470010030090271, + "loss": 1.4008, + "theoretical_loss": 3.4803627767400123, + "tokens_seen": 1686372352 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002469809428284855, + "loss": 1.3985, + "theoretical_loss": 3.48033945985065, + "tokens_seen": 1686503424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024696088264794384, + "loss": 1.5078, + "theoretical_loss": 3.4803161452807196, + "tokens_seen": 1686634496 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002469408224674022, + "loss": 1.5489, + "theoretical_loss": 3.480292833029811, + "tokens_seen": 1686765568 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024692076228686056, + "loss": 1.3458, + "theoretical_loss": 3.4802695230975127, + "tokens_seen": 1686896640 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246900702106319, + "loss": 1.2782, + "theoretical_loss": 3.4802462154834135, + "tokens_seen": 1687027712 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024688064192577734, + "loss": 1.2838, + "theoretical_loss": 3.480222910187104, + "tokens_seen": 1687158784 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468605817452357, + "loss": 1.3163, + "theoretical_loss": 3.4801996072081725, + "tokens_seen": 1687289856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024684052156469406, + "loss": 1.4061, + "theoretical_loss": 3.4801763065462095, + "tokens_seen": 1687420928 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.3630865216255188, + "objective/train/avg_token_score": 0.049899179488420486, + "objective/train/docs_used": 956671, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.24411141872406, + "objective/train/original_loss": 4.7759199142456055, + "objective/train/theoretical_loss": 3.480153008200804, + "objective/train/tokens_used": 1687552000, + "objective/train/unlikelihood_loss": 0.1116311177611351, + "theoretical_loss": 3.480153008200804, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468204613841525, + "loss": 1.4011, + "theoretical_loss": 3.480153008200804, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024680040120361084, + "loss": 1.2568, + "theoretical_loss": 3.4801297121715464, + "tokens_seen": 1687683072 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467803410230692, + "loss": 1.3227, + "theoretical_loss": 3.480106418458026, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024676028084252757, + "loss": 1.4424, + "theoretical_loss": 3.4800831270598334, + "tokens_seen": 1687945216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024674022066198593, + "loss": 1.3407, + "theoretical_loss": 3.4800598379765577, + "tokens_seen": 1688076288 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024672016048144435, + "loss": 1.3215, + "theoretical_loss": 3.48003655120779, + "tokens_seen": 1688207360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467001003009027, + "loss": 1.2787, + "theoretical_loss": 3.4800132667531205, + "tokens_seen": 1688338432 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024668004012036107, + "loss": 1.1583, + "theoretical_loss": 3.4799899846121396, + "tokens_seen": 1688469504 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024665997993981943, + "loss": 1.2846, + "theoretical_loss": 3.4799667047844376, + "tokens_seen": 1688600576 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024663991975927785, + "loss": 1.4574, + "theoretical_loss": 3.479943427269605, + "tokens_seen": 1688731648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002466198595787362, + "loss": 1.4387, + "theoretical_loss": 3.479920152067233, + "tokens_seen": 1688862720 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024659979939819463, + "loss": 1.3548, + "theoretical_loss": 3.479896879176912, + "tokens_seen": 1688993792 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246579739217653, + "loss": 1.2876, + "theoretical_loss": 3.4798736085982336, + "tokens_seen": 1689124864 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.5675097703933716, + "objective/train/avg_token_score": 0.024460725486278534, + "objective/train/docs_used": 957370, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.831213355064392, + "objective/train/original_loss": 3.898343563079834, + "objective/train/theoretical_loss": 3.479861974175632, + "objective/train/tokens_used": 1689190400, + "objective/train/unlikelihood_loss": 0.12086610496044159, + "theoretical_loss": 3.479861974175632, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024655967903711135, + "loss": 1.4328, + "theoretical_loss": 3.4798503403307874, + "tokens_seen": 1689255936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465396188565697, + "loss": 1.3566, + "theoretical_loss": 3.479827074374166, + "tokens_seen": 1689387008 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465195586760281, + "loss": 1.2137, + "theoretical_loss": 3.4798038107279603, + "tokens_seen": 1689518080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464994984954865, + "loss": 1.3378, + "theoretical_loss": 3.4797805493917613, + "tokens_seen": 1689649152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024647943831494486, + "loss": 1.3697, + "theoretical_loss": 3.4797572903651606, + "tokens_seen": 1689780224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464593781344032, + "loss": 1.3531, + "theoretical_loss": 3.47973403364775, + "tokens_seen": 1689911296 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464393179538616, + "loss": 1.3602, + "theoretical_loss": 3.4797107792391206, + "tokens_seen": 1690042368 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024641925777332, + "loss": 1.2529, + "theoretical_loss": 3.4796875271388648, + "tokens_seen": 1690173440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024639919759277836, + "loss": 1.2815, + "theoretical_loss": 3.479664277346574, + "tokens_seen": 1690304512 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463791374122367, + "loss": 1.233, + "theoretical_loss": 3.4796410298618405, + "tokens_seen": 1690435584 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463590772316951, + "loss": 1.3906, + "theoretical_loss": 3.4796177846842564, + "tokens_seen": 1690566656 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024633901705115345, + "loss": 1.3103, + "theoretical_loss": 3.4795945418134133, + "tokens_seen": 1690697728 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.4585166275501251, + "objective/train/avg_token_score": 0.02455119416117668, + "objective/train/docs_used": 957881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3647540807724, + "objective/train/original_loss": 3.2461812496185303, + "objective/train/theoretical_loss": 3.4795713012489045, + "objective/train/tokens_used": 1690828800, + "objective/train/unlikelihood_loss": 0.23378042876720428, + "theoretical_loss": 3.4795713012489045, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024631895687061186, + "loss": 1.448, + "theoretical_loss": 3.4795713012489045, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462988966900702, + "loss": 1.4105, + "theoretical_loss": 3.4795480629903217, + "tokens_seen": 1690959872 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462788365095286, + "loss": 1.464, + "theoretical_loss": 3.4795248270372574, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024625877632898695, + "loss": 1.3932, + "theoretical_loss": 3.479501593389304, + "tokens_seen": 1691222016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024623871614844536, + "loss": 1.2283, + "theoretical_loss": 3.4794783620460548, + "tokens_seen": 1691353088 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462186559679037, + "loss": 1.2027, + "theoretical_loss": 3.4794551330071024, + "tokens_seen": 1691484160 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461985957873621, + "loss": 1.3766, + "theoretical_loss": 3.4794319062720396, + "tokens_seen": 1691615232 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024617853560682045, + "loss": 1.3486, + "theoretical_loss": 3.479408681840459, + "tokens_seen": 1691746304 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461584754262788, + "loss": 1.2357, + "theoretical_loss": 3.4793854597119545, + "tokens_seen": 1691877376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024613841524573723, + "loss": 1.4283, + "theoretical_loss": 3.4793622398861186, + "tokens_seen": 1692008448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461183550651956, + "loss": 1.3135, + "theoretical_loss": 3.479339022362545, + "tokens_seen": 1692139520 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024609829488465395, + "loss": 1.3777, + "theoretical_loss": 3.4793158071408277, + "tokens_seen": 1692270592 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460782347041123, + "loss": 1.4248, + "theoretical_loss": 3.479292594220559, + "tokens_seen": 1692401664 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.39766618609428406, + "objective/train/avg_token_score": 0.02093455009162426, + "objective/train/docs_used": 959159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0620169639587402, + "objective/train/original_loss": 3.435926914215088, + "objective/train/theoretical_loss": 3.4792809886233416, + "objective/train/tokens_used": 1692467200, + "objective/train/unlikelihood_loss": 0.3267090916633606, + "theoretical_loss": 3.4792809886233416, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024605817452357073, + "loss": 1.2901, + "theoretical_loss": 3.4792693836013333, + "tokens_seen": 1692532736 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460381143430291, + "loss": 1.357, + "theoretical_loss": 3.4792461752827446, + "tokens_seen": 1692663808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024601805416248746, + "loss": 1.1861, + "theoretical_loss": 3.479222969264386, + "tokens_seen": 1692794880 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002459979939819458, + "loss": 1.3754, + "theoretical_loss": 3.479199765545852, + "tokens_seen": 1692925952 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002459779338014042, + "loss": 1.4714, + "theoretical_loss": 3.479176564126736, + "tokens_seen": 1693057024 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002459578736208626, + "loss": 1.3494, + "theoretical_loss": 3.479153365006632, + "tokens_seen": 1693188096 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024593781344032096, + "loss": 1.1529, + "theoretical_loss": 3.479130168185136, + "tokens_seen": 1693319168 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002459177532597793, + "loss": 1.2117, + "theoretical_loss": 3.4791069736618407, + "tokens_seen": 1693450240 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458976930792377, + "loss": 1.2212, + "theoretical_loss": 3.4790837814363407, + "tokens_seen": 1693581312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458776328986961, + "loss": 1.224, + "theoretical_loss": 3.4790605915082313, + "tokens_seen": 1693712384 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024585757271815446, + "loss": 1.2809, + "theoretical_loss": 3.479037403877107, + "tokens_seen": 1693843456 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458375125376129, + "loss": 1.2837, + "theoretical_loss": 3.479014218542561, + "tokens_seen": 1693974528 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.32942327857017517, + "objective/train/avg_token_score": 0.005918252281844616, + "objective/train/docs_used": 959847, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.034077525138855, + "objective/train/original_loss": 3.5762524604797363, + "objective/train/theoretical_loss": 3.478991035504191, + "objective/train/tokens_used": 1694105600, + "objective/train/unlikelihood_loss": 0.2530372142791748, + "theoretical_loss": 3.478991035504191, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024581745235707124, + "loss": 1.3065, + "theoretical_loss": 3.478991035504191, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457973921765296, + "loss": 1.5614, + "theoretical_loss": 3.4789678547615894, + "tokens_seen": 1694236672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024577733199598797, + "loss": 1.413, + "theoretical_loss": 3.478944676314353, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024575727181544633, + "loss": 1.5353, + "theoretical_loss": 3.478921500162076, + "tokens_seen": 1694498816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024573721163490475, + "loss": 1.3593, + "theoretical_loss": 3.4788983263043534, + "tokens_seen": 1694629888 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457171514543631, + "loss": 1.3544, + "theoretical_loss": 3.4788751547407815, + "tokens_seen": 1694760960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024569709127382147, + "loss": 1.1804, + "theoretical_loss": 3.478851985470956, + "tokens_seen": 1694892032 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024567703109327983, + "loss": 1.3521, + "theoretical_loss": 3.4788288184944713, + "tokens_seen": 1695023104 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024565697091273825, + "loss": 1.3395, + "theoretical_loss": 3.478805653810924, + "tokens_seen": 1695154176 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002456369107321966, + "loss": 1.4654, + "theoretical_loss": 3.478782491419909, + "tokens_seen": 1695285248 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245616850551655, + "loss": 1.3501, + "theoretical_loss": 3.478759331321023, + "tokens_seen": 1695416320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024559679037111334, + "loss": 1.3459, + "theoretical_loss": 3.4787361735138624, + "tokens_seen": 1695547392 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455767301905717, + "loss": 1.2998, + "theoretical_loss": 3.4787130179980217, + "tokens_seen": 1695678464 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.41330644488334656, + "objective/train/avg_token_score": 0.012362140230834484, + "objective/train/docs_used": 961177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2281874418258667, + "objective/train/original_loss": 3.44167423248291, + "objective/train/theoretical_loss": 3.478701441099221, + "objective/train/tokens_used": 1695744000, + "objective/train/unlikelihood_loss": 0.24482883512973785, + "theoretical_loss": 3.478701441099221, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455566700100301, + "loss": 1.3287, + "theoretical_loss": 3.4786898647730986, + "tokens_seen": 1695809536 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455366098294885, + "loss": 1.3054, + "theoretical_loss": 3.478666713838689, + "tokens_seen": 1695940608 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024551654964894684, + "loss": 1.3659, + "theoretical_loss": 3.478643565194389, + "tokens_seen": 1696071680 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454964894684052, + "loss": 1.3049, + "theoretical_loss": 3.478620418839795, + "tokens_seen": 1696202752 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454764292878636, + "loss": 1.4101, + "theoretical_loss": 3.478597274774504, + "tokens_seen": 1696333824 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245456369107322, + "loss": 1.3834, + "theoretical_loss": 3.478574132998113, + "tokens_seen": 1696464896 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024543630892678034, + "loss": 1.2995, + "theoretical_loss": 3.4785509935102183, + "tokens_seen": 1696595968 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454162487462387, + "loss": 1.4229, + "theoretical_loss": 3.478527856310417, + "tokens_seen": 1696727040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024539618856569707, + "loss": 1.3607, + "theoretical_loss": 3.478504721398306, + "tokens_seen": 1696858112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453761283851555, + "loss": 1.4164, + "theoretical_loss": 3.4784815887734823, + "tokens_seen": 1696989184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024535606820461385, + "loss": 1.554, + "theoretical_loss": 3.4784584584355436, + "tokens_seen": 1697120256 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453360080240722, + "loss": 1.4062, + "theoretical_loss": 3.4784353303840865, + "tokens_seen": 1697251328 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.3166544437408447, + "objective/train/avg_token_score": 0.004786605015397072, + "objective/train/docs_used": 961828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0478065013885498, + "objective/train/original_loss": 3.625047206878662, + "objective/train/theoretical_loss": 3.478412204618709, + "objective/train/tokens_used": 1697382400, + "objective/train/unlikelihood_loss": 0.2357628494501114, + "theoretical_loss": 3.478412204618709, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024531594784353057, + "loss": 1.4308, + "theoretical_loss": 3.478412204618709, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245295887662989, + "loss": 1.379, + "theoretical_loss": 3.478389081139009, + "tokens_seen": 1697513472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024527582748244735, + "loss": 1.4397, + "theoretical_loss": 3.4783659599445826, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452557673019057, + "loss": 1.2258, + "theoretical_loss": 3.4783428410350297, + "tokens_seen": 1697775616 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452357071213641, + "loss": 1.405, + "theoretical_loss": 3.478319724409946, + "tokens_seen": 1697906688 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024521564694082244, + "loss": 1.1624, + "theoretical_loss": 3.4782966100689308, + "tokens_seen": 1698037760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024519558676028085, + "loss": 1.4088, + "theoretical_loss": 3.478273498011582, + "tokens_seen": 1698168832 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451755265797392, + "loss": 1.4045, + "theoretical_loss": 3.478250388237497, + "tokens_seen": 1698299904 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451554663991976, + "loss": 1.3046, + "theoretical_loss": 3.4782272807462746, + "tokens_seen": 1698430976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024513540621865594, + "loss": 1.2642, + "theoretical_loss": 3.4782041755375133, + "tokens_seen": 1698562048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024511534603811435, + "loss": 1.3203, + "theoretical_loss": 3.478181072610811, + "tokens_seen": 1698693120 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002450952858575727, + "loss": 1.4002, + "theoretical_loss": 3.478157971965767, + "tokens_seen": 1698824192 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024507522567703113, + "loss": 1.3247, + "theoretical_loss": 3.478134873601979, + "tokens_seen": 1698955264 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.290628045797348, + "objective/train/avg_token_score": 0.04112381488084793, + "objective/train/docs_used": 963095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9128071665763855, + "objective/train/original_loss": 4.1264519691467285, + "objective/train/theoretical_loss": 3.4781233252754316, + "objective/train/tokens_used": 1699020800, + "objective/train/unlikelihood_loss": 0.2004564255475998, + "theoretical_loss": 3.4781233252754316, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002450551654964895, + "loss": 1.2321, + "theoretical_loss": 3.4781117775190467, + "tokens_seen": 1699086336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024503510531594786, + "loss": 1.382, + "theoretical_loss": 3.4780886837165688, + "tokens_seen": 1699217408 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002450150451354062, + "loss": 1.2852, + "theoretical_loss": 3.4780655921941435, + "tokens_seen": 1699348480 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002449949849548646, + "loss": 1.3492, + "theoretical_loss": 3.47804250295137, + "tokens_seen": 1699479552 + }, + { + "epoch": 0.52, + "learning_rate": 0.000244974924774323, + "loss": 1.3472, + "theoretical_loss": 3.4780194159878484, + "tokens_seen": 1699610624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024495486459378136, + "loss": 1.3182, + "theoretical_loss": 3.477996331303177, + "tokens_seen": 1699741696 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002449348044132397, + "loss": 1.242, + "theoretical_loss": 3.477973248896956, + "tokens_seen": 1699872768 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002449147442326981, + "loss": 1.4353, + "theoretical_loss": 3.4779501687687837, + "tokens_seen": 1700003840 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448946840521565, + "loss": 1.3457, + "theoretical_loss": 3.4779270909182607, + "tokens_seen": 1700134912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024487462387161486, + "loss": 1.3406, + "theoretical_loss": 3.477904015344986, + "tokens_seen": 1700265984 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448545636910732, + "loss": 1.3245, + "theoretical_loss": 3.47788094204856, + "tokens_seen": 1700397056 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448345035105316, + "loss": 1.2381, + "theoretical_loss": 3.4778578710285815, + "tokens_seen": 1700528128 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.5952468514442444, + "objective/train/avg_token_score": 0.009271438233554363, + "objective/train/docs_used": 963811, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.795974612236023, + "objective/train/original_loss": 3.301114082336426, + "objective/train/theoretical_loss": 3.4778348022846517, + "objective/train/tokens_used": 1700659200, + "objective/train/unlikelihood_loss": 0.16355566680431366, + "theoretical_loss": 3.4778348022846517, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024481444332998995, + "loss": 1.4087, + "theoretical_loss": 3.4778348022846517, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024479438314944837, + "loss": 1.3472, + "theoretical_loss": 3.4778117358163705, + "tokens_seen": 1700790272 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024477432296890673, + "loss": 1.5407, + "theoretical_loss": 3.477788671623337, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447542627883651, + "loss": 1.2724, + "theoretical_loss": 3.4777656097051524, + "tokens_seen": 1701052416 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024473420260782345, + "loss": 1.2998, + "theoretical_loss": 3.477742550061417, + "tokens_seen": 1701183488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024471414242728187, + "loss": 1.4025, + "theoretical_loss": 3.4777194926917305, + "tokens_seen": 1701314560 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024469408224674023, + "loss": 1.3619, + "theoretical_loss": 3.4776964375956947, + "tokens_seen": 1701445632 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446740220661986, + "loss": 1.3988, + "theoretical_loss": 3.4776733847729098, + "tokens_seen": 1701576704 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024465396188565696, + "loss": 1.2568, + "theoretical_loss": 3.477650334222976, + "tokens_seen": 1701707776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446339017051154, + "loss": 1.1975, + "theoretical_loss": 3.477627285945495, + "tokens_seen": 1701838848 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024461384152457374, + "loss": 1.3702, + "theoretical_loss": 3.4776042399400673, + "tokens_seen": 1701969920 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445937813440321, + "loss": 1.2844, + "theoretical_loss": 3.4775811962062937, + "tokens_seen": 1702100992 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024457372116349046, + "loss": 1.3449, + "theoretical_loss": 3.4775581547437757, + "tokens_seen": 1702232064 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.43438416719436646, + "objective/train/avg_token_score": 0.01639791950583458, + "objective/train/docs_used": 965136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3490318059921265, + "objective/train/original_loss": 3.4790992736816406, + "objective/train/theoretical_loss": 3.477546634864113, + "objective/train/tokens_used": 1702297600, + "objective/train/unlikelihood_loss": 0.2112819403409958, + "theoretical_loss": 3.477546634864113, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445536609829488, + "loss": 1.3382, + "theoretical_loss": 3.477535115552115, + "tokens_seen": 1702363136 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024453360080240724, + "loss": 1.3657, + "theoretical_loss": 3.4775120786309124, + "tokens_seen": 1702494208 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445135406218656, + "loss": 1.3113, + "theoretical_loss": 3.4774890439797694, + "tokens_seen": 1702625280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024449348044132396, + "loss": 1.308, + "theoretical_loss": 3.4774660115982883, + "tokens_seen": 1702756352 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002444734202607823, + "loss": 1.2999, + "theoretical_loss": 3.47744298148607, + "tokens_seen": 1702887424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024445336008024074, + "loss": 1.3362, + "theoretical_loss": 3.477419953642716, + "tokens_seen": 1703018496 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002444332998996991, + "loss": 1.4062, + "theoretical_loss": 3.4773969280678294, + "tokens_seen": 1703149568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024441323971915747, + "loss": 1.3162, + "theoretical_loss": 3.477373904761011, + "tokens_seen": 1703280640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024439317953861583, + "loss": 1.5138, + "theoretical_loss": 3.477350883721863, + "tokens_seen": 1703411712 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443731193580742, + "loss": 1.2547, + "theoretical_loss": 3.477327864949989, + "tokens_seen": 1703542784 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443530591775326, + "loss": 1.3488, + "theoretical_loss": 3.4773048484449895, + "tokens_seen": 1703673856 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024433299899699097, + "loss": 1.483, + "theoretical_loss": 3.477281834206468, + "tokens_seen": 1703804928 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.37188416719436646, + "objective/train/avg_token_score": 0.01578773371875286, + "objective/train/docs_used": 965825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2605202198028564, + "objective/train/original_loss": 4.280536651611328, + "objective/train/theoretical_loss": 3.477258822234026, + "objective/train/tokens_used": 1703936000, + "objective/train/unlikelihood_loss": 0.11020573228597641, + "theoretical_loss": 3.477258822234026, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443129388164494, + "loss": 1.2591, + "theoretical_loss": 3.477258822234026, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024429287863590775, + "loss": 1.3322, + "theoretical_loss": 3.477235812527267, + "tokens_seen": 1704067072 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442728184553661, + "loss": 1.3386, + "theoretical_loss": 3.4772128050857933, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442527582748245, + "loss": 1.3124, + "theoretical_loss": 3.4771897999092083, + "tokens_seen": 1704329216 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024423269809428284, + "loss": 1.3157, + "theoretical_loss": 3.4771667969971136, + "tokens_seen": 1704460288 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024421263791374125, + "loss": 1.2995, + "theoretical_loss": 3.477143796349113, + "tokens_seen": 1704591360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002441925777331996, + "loss": 1.2215, + "theoretical_loss": 3.47712079796481, + "tokens_seen": 1704722432 + }, + { + "epoch": 0.52, + "learning_rate": 0.000244172517552658, + "loss": 1.4683, + "theoretical_loss": 3.4770978018438066, + "tokens_seen": 1704853504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024415245737211634, + "loss": 1.3327, + "theoretical_loss": 3.4770748079857072, + "tokens_seen": 1704984576 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024413239719157473, + "loss": 1.2157, + "theoretical_loss": 3.4770518163901145, + "tokens_seen": 1705115648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024411233701103312, + "loss": 1.28, + "theoretical_loss": 3.4770288270566327, + "tokens_seen": 1705246720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024409227683049148, + "loss": 1.4156, + "theoretical_loss": 3.4770058399848645, + "tokens_seen": 1705377792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024407221664994987, + "loss": 1.2682, + "theoretical_loss": 3.476982855174414, + "tokens_seen": 1705508864 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.36241447925567627, + "objective/train/avg_token_score": 0.018051115795969963, + "objective/train/docs_used": 966739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1806671619415283, + "objective/train/original_loss": 4.018659591674805, + "objective/train/theoretical_loss": 3.476971363617059, + "objective/train/tokens_used": 1705574400, + "objective/train/unlikelihood_loss": 0.17842493951320648, + "theoretical_loss": 3.476971363617059, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024405215646940823, + "loss": 1.2569, + "theoretical_loss": 3.476959872624885, + "tokens_seen": 1705639936 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002440320962888666, + "loss": 1.3839, + "theoretical_loss": 3.4769368923358814, + "tokens_seen": 1705771008 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024401203610832498, + "loss": 1.2178, + "theoretical_loss": 3.4769139143070076, + "tokens_seen": 1705902080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024399197592778335, + "loss": 1.3991, + "theoretical_loss": 3.4768909385378666, + "tokens_seen": 1706033152 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024397191574724173, + "loss": 1.3054, + "theoretical_loss": 3.476867965028063, + "tokens_seen": 1706164224 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002439518555667001, + "loss": 1.461, + "theoretical_loss": 3.476844993777202, + "tokens_seen": 1706295296 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024393179538615849, + "loss": 1.2492, + "theoretical_loss": 3.4768220247848873, + "tokens_seen": 1706426368 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024391173520561685, + "loss": 1.2365, + "theoretical_loss": 3.476799058050723, + "tokens_seen": 1706557440 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024389167502507524, + "loss": 1.412, + "theoretical_loss": 3.476776093574314, + "tokens_seen": 1706688512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002438716148445336, + "loss": 1.306, + "theoretical_loss": 3.4767531313552658, + "tokens_seen": 1706819584 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024385155466399196, + "loss": 1.3338, + "theoretical_loss": 3.4767301713931813, + "tokens_seen": 1706950656 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024383149448345035, + "loss": 1.1718, + "theoretical_loss": 3.4767072136876673, + "tokens_seen": 1707081728 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.470246821641922, + "objective/train/avg_token_score": 0.019071055576205254, + "objective/train/docs_used": 967356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5092097520828247, + "objective/train/original_loss": 4.190674781799316, + "objective/train/theoretical_loss": 3.4766842582383277, + "objective/train/tokens_used": 1707212800, + "objective/train/unlikelihood_loss": 0.10261710733175278, + "theoretical_loss": 3.4766842582383277, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024381143430290871, + "loss": 1.2001, + "theoretical_loss": 3.4766842582383277, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002437913741223671, + "loss": 1.4883, + "theoretical_loss": 3.476661305044768, + "tokens_seen": 1707343872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024377131394182547, + "loss": 1.2586, + "theoretical_loss": 3.4766383541065933, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024375125376128385, + "loss": 1.3283, + "theoretical_loss": 3.4766154054234084, + "tokens_seen": 1707606016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024373119358074222, + "loss": 1.3017, + "theoretical_loss": 3.4765924589948196, + "tokens_seen": 1707737088 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002437111334002006, + "loss": 1.508, + "theoretical_loss": 3.476569514820432, + "tokens_seen": 1707868160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024369107321965897, + "loss": 1.231, + "theoretical_loss": 3.4765465728998506, + "tokens_seen": 1707999232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024367101303911736, + "loss": 1.2441, + "theoretical_loss": 3.4765236332326817, + "tokens_seen": 1708130304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024365095285857572, + "loss": 1.35, + "theoretical_loss": 3.476500695818531, + "tokens_seen": 1708261376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024363089267803408, + "loss": 1.2355, + "theoretical_loss": 3.476477760657004, + "tokens_seen": 1708392448 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002436108324974925, + "loss": 1.4986, + "theoretical_loss": 3.476454827747707, + "tokens_seen": 1708523520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024359077231695086, + "loss": 1.508, + "theoretical_loss": 3.4764318970902464, + "tokens_seen": 1708654592 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024357071213640925, + "loss": 1.4165, + "theoretical_loss": 3.476408968684228, + "tokens_seen": 1708785664 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.5128298997879028, + "objective/train/avg_token_score": 0.02624981850385666, + "objective/train/docs_used": 967795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6476236581802368, + "objective/train/original_loss": 3.7733631134033203, + "objective/train/theoretical_loss": 3.4763975053253864, + "objective/train/tokens_used": 1708851200, + "objective/train/unlikelihood_loss": 0.12670385837554932, + "theoretical_loss": 3.4763975053253864, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002435506519558676, + "loss": 1.2906, + "theoretical_loss": 3.4763860425292576, + "tokens_seen": 1708916736 + }, + { + "epoch": 0.52, + "learning_rate": 0.000243530591775326, + "loss": 1.2645, + "theoretical_loss": 3.476363118624943, + "tokens_seen": 1709047808 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024351053159478436, + "loss": 1.3446, + "theoretical_loss": 3.476340196970889, + "tokens_seen": 1709178880 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024349047141424275, + "loss": 1.3859, + "theoretical_loss": 3.476317277566703, + "tokens_seen": 1709309952 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024347041123370112, + "loss": 1.3635, + "theoretical_loss": 3.476294360411991, + "tokens_seen": 1709441024 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024345035105315948, + "loss": 1.2269, + "theoretical_loss": 3.4762714455063612, + "tokens_seen": 1709572096 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024343029087261787, + "loss": 1.2838, + "theoretical_loss": 3.476248532849419, + "tokens_seen": 1709703168 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024341023069207623, + "loss": 1.2729, + "theoretical_loss": 3.476225622440772, + "tokens_seen": 1709834240 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024339017051153462, + "loss": 1.2871, + "theoretical_loss": 3.476202714280028, + "tokens_seen": 1709965312 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024337011033099298, + "loss": 1.1882, + "theoretical_loss": 3.4761798083667923, + "tokens_seen": 1710096384 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024335005015045137, + "loss": 1.2097, + "theoretical_loss": 3.476156904700674, + "tokens_seen": 1710227456 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024332998996990973, + "loss": 1.2034, + "theoretical_loss": 3.4761340032812793, + "tokens_seen": 1710358528 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.3872189521789551, + "objective/train/avg_token_score": 0.007078302092850208, + "objective/train/docs_used": 968886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.173158049583435, + "objective/train/original_loss": 3.55717396736145, + "objective/train/theoretical_loss": 3.476111104108216, + "objective/train/tokens_used": 1710489600, + "objective/train/unlikelihood_loss": 0.26422014832496643, + "theoretical_loss": 3.476111104108216, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024330992978936812, + "loss": 1.29, + "theoretical_loss": 3.476111104108216, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024328986960882648, + "loss": 1.4197, + "theoretical_loss": 3.476088207181092, + "tokens_seen": 1710620672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024326980942828487, + "loss": 1.2824, + "theoretical_loss": 3.4760653124995144, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024324974924774324, + "loss": 1.3519, + "theoretical_loss": 3.4760424200630915, + "tokens_seen": 1710882816 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002432296890672016, + "loss": 1.1181, + "theoretical_loss": 3.476019529871431, + "tokens_seen": 1711013888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024320962888666, + "loss": 1.2415, + "theoretical_loss": 3.475996641924141, + "tokens_seen": 1711144960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024318956870611835, + "loss": 1.3519, + "theoretical_loss": 3.4759737562208284, + "tokens_seen": 1711276032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024316950852557674, + "loss": 1.4129, + "theoretical_loss": 3.4759508727611026, + "tokens_seen": 1711407104 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002431494483450351, + "loss": 1.3879, + "theoretical_loss": 3.4759279915445713, + "tokens_seen": 1711538176 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002431293881644935, + "loss": 1.3965, + "theoretical_loss": 3.4759051125708433, + "tokens_seen": 1711669248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024310932798395185, + "loss": 1.3521, + "theoretical_loss": 3.475882235839527, + "tokens_seen": 1711800320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024308926780341024, + "loss": 1.3452, + "theoretical_loss": 3.47585936135023, + "tokens_seen": 1711931392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002430692076228686, + "loss": 1.4338, + "theoretical_loss": 3.475836489102562, + "tokens_seen": 1712062464 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.40114858746528625, + "objective/train/avg_token_score": 0.012381168082356453, + "objective/train/docs_used": 969614, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4084160327911377, + "objective/train/original_loss": 4.037243366241455, + "objective/train/theoretical_loss": 3.4758250538192166, + "objective/train/tokens_used": 1712128000, + "objective/train/unlikelihood_loss": 0.1384529173374176, + "theoretical_loss": 3.4758250538192166, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024304914744232697, + "loss": 1.4211, + "theoretical_loss": 3.4758136190961313, + "tokens_seen": 1712193536 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024302908726178536, + "loss": 1.382, + "theoretical_loss": 3.475790751330547, + "tokens_seen": 1712324608 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024300902708124372, + "loss": 1.2941, + "theoretical_loss": 3.4757678858054177, + "tokens_seen": 1712455680 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002429889669007021, + "loss": 1.4144, + "theoretical_loss": 3.4757450225203526, + "tokens_seen": 1712586752 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024296890672016047, + "loss": 1.3148, + "theoretical_loss": 3.4757221614749607, + "tokens_seen": 1712717824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024294884653961886, + "loss": 1.3838, + "theoretical_loss": 3.4756993026688514, + "tokens_seen": 1712848896 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024292878635907722, + "loss": 1.388, + "theoretical_loss": 3.4756764461016347, + "tokens_seen": 1712979968 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002429087261785356, + "loss": 1.2933, + "theoretical_loss": 3.475653591772918, + "tokens_seen": 1713111040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024288866599799397, + "loss": 1.3681, + "theoretical_loss": 3.475630739682313, + "tokens_seen": 1713242112 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024286860581745234, + "loss": 1.2452, + "theoretical_loss": 3.475607889829428, + "tokens_seen": 1713373184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024284854563691075, + "loss": 1.2448, + "theoretical_loss": 3.475585042213874, + "tokens_seen": 1713504256 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024282848545636911, + "loss": 1.3711, + "theoretical_loss": 3.4755621968352592, + "tokens_seen": 1713635328 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.30015885829925537, + "objective/train/avg_token_score": 0.01989401876926422, + "objective/train/docs_used": 971016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9191915988922119, + "objective/train/original_loss": 3.859710931777954, + "objective/train/theoretical_loss": 3.4755393536931942, + "objective/train/tokens_used": 1713766400, + "objective/train/unlikelihood_loss": 0.14533783495426178, + "theoretical_loss": 3.4755393536931942, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002428084252758275, + "loss": 1.1233, + "theoretical_loss": 3.4755393536931942, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024278836509528587, + "loss": 1.3356, + "theoretical_loss": 3.4755165127872893, + "tokens_seen": 1713897472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024276830491474426, + "loss": 1.3065, + "theoretical_loss": 3.4754936741171543, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024274824473420262, + "loss": 1.0562, + "theoretical_loss": 3.4754708376823995, + "tokens_seen": 1714159616 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242728184553661, + "loss": 1.3773, + "theoretical_loss": 3.4754480034826356, + "tokens_seen": 1714290688 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024270812437311937, + "loss": 1.1453, + "theoretical_loss": 3.4754251715174727, + "tokens_seen": 1714421760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024268806419257776, + "loss": 1.2741, + "theoretical_loss": 3.4754023417865207, + "tokens_seen": 1714552832 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024266800401203612, + "loss": 1.2273, + "theoretical_loss": 3.475379514289391, + "tokens_seen": 1714683904 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024264794383149448, + "loss": 1.3026, + "theoretical_loss": 3.475356689025694, + "tokens_seen": 1714814976 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024262788365095287, + "loss": 1.1395, + "theoretical_loss": 3.475333865995041, + "tokens_seen": 1714946048 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024260782347041123, + "loss": 1.4376, + "theoretical_loss": 3.4753110451970417, + "tokens_seen": 1715077120 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024258776328986962, + "loss": 1.2902, + "theoretical_loss": 3.475288226631308, + "tokens_seen": 1715208192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024256770310932799, + "loss": 1.4468, + "theoretical_loss": 3.475265410297451, + "tokens_seen": 1715339264 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.5105693936347961, + "objective/train/avg_token_score": 0.009695515036582947, + "objective/train/docs_used": 971702, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6010916233062744, + "objective/train/original_loss": 3.8771517276763916, + "objective/train/theoretical_loss": 3.4752540029673544, + "objective/train/tokens_used": 1715404800, + "objective/train/unlikelihood_loss": 0.14445847272872925, + "theoretical_loss": 3.4752540029673544, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024254764292878638, + "loss": 1.2841, + "theoretical_loss": 3.4752425961950815, + "tokens_seen": 1715470336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024252758274824474, + "loss": 1.3472, + "theoretical_loss": 3.475219784323811, + "tokens_seen": 1715601408 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024250752256770313, + "loss": 1.2511, + "theoretical_loss": 3.475196974683251, + "tokens_seen": 1715732480 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002424874623871615, + "loss": 1.2335, + "theoretical_loss": 3.4751741672730123, + "tokens_seen": 1715863552 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024246740220661985, + "loss": 1.4083, + "theoretical_loss": 3.4751513620927073, + "tokens_seen": 1715994624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024244734202607824, + "loss": 1.2305, + "theoretical_loss": 3.4751285591419476, + "tokens_seen": 1716125696 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002424272818455366, + "loss": 1.3892, + "theoretical_loss": 3.4751057584203444, + "tokens_seen": 1716256768 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242407221664995, + "loss": 1.2029, + "theoretical_loss": 3.47508295992751, + "tokens_seen": 1716387840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024238716148445335, + "loss": 1.3173, + "theoretical_loss": 3.4750601636630565, + "tokens_seen": 1716518912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024236710130391174, + "loss": 1.3681, + "theoretical_loss": 3.4750373696265955, + "tokens_seen": 1716649984 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002423470411233701, + "loss": 1.2645, + "theoretical_loss": 3.4750145778177393, + "tokens_seen": 1716781056 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002423269809428285, + "loss": 1.2758, + "theoretical_loss": 3.4749917882360997, + "tokens_seen": 1716912128 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.5343353152275085, + "objective/train/avg_token_score": 0.009466072544455528, + "objective/train/docs_used": 972817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6441320180892944, + "objective/train/original_loss": 3.5202884674072266, + "objective/train/theoretical_loss": 3.4749690008812903, + "objective/train/tokens_used": 1717043200, + "objective/train/unlikelihood_loss": 0.17875421047210693, + "theoretical_loss": 3.4749690008812903, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024230692076228686, + "loss": 1.1969, + "theoretical_loss": 3.4749690008812903, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024228686058174522, + "loss": 1.2876, + "theoretical_loss": 3.4749462157529227, + "tokens_seen": 1717174272 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002422668004012036, + "loss": 1.2683, + "theoretical_loss": 3.47492343285061, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024224674022066197, + "loss": 1.3495, + "theoretical_loss": 3.4749006521739636, + "tokens_seen": 1717436416 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024222668004012036, + "loss": 1.365, + "theoretical_loss": 3.4748778737225976, + "tokens_seen": 1717567488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024220661985957872, + "loss": 1.4176, + "theoretical_loss": 3.474855097496124, + "tokens_seen": 1717698560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002421865596790371, + "loss": 1.5862, + "theoretical_loss": 3.474832323494156, + "tokens_seen": 1717829632 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024216649949849547, + "loss": 1.3584, + "theoretical_loss": 3.474809551716307, + "tokens_seen": 1717960704 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024214643931795386, + "loss": 1.2359, + "theoretical_loss": 3.4747867821621896, + "tokens_seen": 1718091776 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024212637913741223, + "loss": 1.2827, + "theoretical_loss": 3.474764014831417, + "tokens_seen": 1718222848 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024210631895687064, + "loss": 1.2294, + "theoretical_loss": 3.474741249723603, + "tokens_seen": 1718353920 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242086258776329, + "loss": 1.3731, + "theoretical_loss": 3.4747184868383605, + "tokens_seen": 1718484992 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024206619859578737, + "loss": 1.3344, + "theoretical_loss": 3.4746957261753035, + "tokens_seen": 1718616064 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.3709677457809448, + "objective/train/avg_token_score": 0.020207742229104042, + "objective/train/docs_used": 973424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.186741828918457, + "objective/train/original_loss": 4.011867046356201, + "objective/train/theoretical_loss": 3.4746843466769737, + "objective/train/tokens_used": 1718681600, + "objective/train/unlikelihood_loss": 0.13356520235538483, + "theoretical_loss": 3.4746843466769737, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024204613841524576, + "loss": 1.2119, + "theoretical_loss": 3.4746729677340453, + "tokens_seen": 1718747136 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024202607823470412, + "loss": 1.3827, + "theoretical_loss": 3.4746502115141995, + "tokens_seen": 1718878208 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002420060180541625, + "loss": 1.2444, + "theoretical_loss": 3.47462745751538, + "tokens_seen": 1719009280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024198595787362087, + "loss": 1.431, + "theoretical_loss": 3.4746047057372014, + "tokens_seen": 1719140352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024196589769307926, + "loss": 1.2978, + "theoretical_loss": 3.474581956179277, + "tokens_seen": 1719271424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024194583751253762, + "loss": 1.4245, + "theoretical_loss": 3.4745592088412205, + "tokens_seen": 1719402496 + }, + { + "epoch": 0.52, + "learning_rate": 0.000241925777331996, + "loss": 1.2987, + "theoretical_loss": 3.4745364637226466, + "tokens_seen": 1719533568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024190571715145437, + "loss": 1.3193, + "theoretical_loss": 3.47451372082317, + "tokens_seen": 1719664640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024188565697091274, + "loss": 1.4961, + "theoretical_loss": 3.474490980142404, + "tokens_seen": 1719795712 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024186559679037113, + "loss": 1.234, + "theoretical_loss": 3.474468241679964, + "tokens_seen": 1719926784 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002418455366098295, + "loss": 1.3275, + "theoretical_loss": 3.4744455054354644, + "tokens_seen": 1720057856 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024182547642928788, + "loss": 1.3409, + "theoretical_loss": 3.47442277140852, + "tokens_seen": 1720188928 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.2717497646808624, + "objective/train/avg_token_score": 0.043517209589481354, + "objective/train/docs_used": 974650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8329663276672363, + "objective/train/original_loss": 3.923548936843872, + "objective/train/theoretical_loss": 3.474400039598745, + "objective/train/tokens_used": 1720320000, + "objective/train/unlikelihood_loss": 0.1716012954711914, + "theoretical_loss": 3.474400039598745, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024180541624874624, + "loss": 1.1426, + "theoretical_loss": 3.474400039598745, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024178535606820463, + "loss": 1.1861, + "theoretical_loss": 3.4743773100057544, + "tokens_seen": 1720451072 + }, + { + "epoch": 0.52, + "learning_rate": 0.000241765295887663, + "loss": 1.2323, + "theoretical_loss": 3.4743545826291635, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024174523570712138, + "loss": 1.2826, + "theoretical_loss": 3.4743318574685875, + "tokens_seen": 1720713216 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024172517552657974, + "loss": 1.3914, + "theoretical_loss": 3.474309134523641, + "tokens_seen": 1720844288 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024170511534603813, + "loss": 1.1026, + "theoretical_loss": 3.4742864137939398, + "tokens_seen": 1720975360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416850551654965, + "loss": 1.1591, + "theoretical_loss": 3.4742636952790984, + "tokens_seen": 1721106432 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024166499498495486, + "loss": 1.1875, + "theoretical_loss": 3.474240978978733, + "tokens_seen": 1721237504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024164493480441325, + "loss": 1.1815, + "theoretical_loss": 3.474218264892459, + "tokens_seen": 1721368576 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416248746238716, + "loss": 1.2928, + "theoretical_loss": 3.474195553019892, + "tokens_seen": 1721499648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024160481444333, + "loss": 1.2072, + "theoretical_loss": 3.4741728433606482, + "tokens_seen": 1721630720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024158475426278836, + "loss": 1.266, + "theoretical_loss": 3.4741501359143423, + "tokens_seen": 1721761792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024156469408224675, + "loss": 1.3978, + "theoretical_loss": 3.474127430680591, + "tokens_seen": 1721892864 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.3122556209564209, + "objective/train/avg_token_score": 0.031719084829092026, + "objective/train/docs_used": 975130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.857271671295166, + "objective/train/original_loss": 3.937514543533325, + "objective/train/theoretical_loss": 3.474116078893303, + "objective/train/tokens_used": 1721958400, + "objective/train/unlikelihood_loss": 0.1427450031042099, + "theoretical_loss": 3.474116078893303, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002415446339017051, + "loss": 1.3104, + "theoretical_loss": 3.47410472765901, + "tokens_seen": 1722023936 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002415245737211635, + "loss": 1.2863, + "theoretical_loss": 3.474082026849216, + "tokens_seen": 1722155008 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024150451354062186, + "loss": 1.2444, + "theoretical_loss": 3.4740593282508243, + "tokens_seen": 1722286080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024148445336008022, + "loss": 1.3732, + "theoretical_loss": 3.474036631863452, + "tokens_seen": 1722417152 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024146439317953861, + "loss": 1.2852, + "theoretical_loss": 3.474013937686715, + "tokens_seen": 1722548224 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024144433299899698, + "loss": 1.3499, + "theoretical_loss": 3.4739912457202307, + "tokens_seen": 1722679296 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024142427281845537, + "loss": 1.287, + "theoretical_loss": 3.473968555963614, + "tokens_seen": 1722810368 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024140421263791373, + "loss": 1.2371, + "theoretical_loss": 3.4739458684164832, + "tokens_seen": 1722941440 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024138415245737212, + "loss": 1.3711, + "theoretical_loss": 3.4739231830784543, + "tokens_seen": 1723072512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002413640922768305, + "loss": 1.2526, + "theoretical_loss": 3.473900499949144, + "tokens_seen": 1723203584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002413440320962889, + "loss": 1.3392, + "theoretical_loss": 3.4738778190281705, + "tokens_seen": 1723334656 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024132397191574726, + "loss": 1.2406, + "theoretical_loss": 3.473855140315149, + "tokens_seen": 1723465728 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.41403958201408386, + "objective/train/avg_token_score": 0.017859043553471565, + "objective/train/docs_used": 976352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3346071243286133, + "objective/train/original_loss": 3.931220769882202, + "objective/train/theoretical_loss": 3.473832463809698, + "objective/train/tokens_used": 1723596800, + "objective/train/unlikelihood_loss": 0.12141155451536179, + "theoretical_loss": 3.473832463809698, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024130391173520562, + "loss": 1.1711, + "theoretical_loss": 3.473832463809698, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.52, + "learning_rate": 0.000241283851554664, + "loss": 1.192, + "theoretical_loss": 3.473809789511434, + "tokens_seen": 1723727872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024126379137412237, + "loss": 1.4046, + "theoretical_loss": 3.4737871174199757, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024124373119358076, + "loss": 1.2161, + "theoretical_loss": 3.473764447534939, + "tokens_seen": 1723990016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024122367101303912, + "loss": 1.2908, + "theoretical_loss": 3.473741779855942, + "tokens_seen": 1724121088 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002412036108324975, + "loss": 1.2832, + "theoretical_loss": 3.473719114382603, + "tokens_seen": 1724252160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024118355065195587, + "loss": 1.3437, + "theoretical_loss": 3.4736964511145385, + "tokens_seen": 1724383232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024116349047141426, + "loss": 1.3203, + "theoretical_loss": 3.4736737900513672, + "tokens_seen": 1724514304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024114343029087263, + "loss": 1.1439, + "theoretical_loss": 3.4736511311927067, + "tokens_seen": 1724645376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024112337011033102, + "loss": 1.3508, + "theoretical_loss": 3.4736284745381756, + "tokens_seen": 1724776448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024110330992978938, + "loss": 1.448, + "theoretical_loss": 3.473605820087391, + "tokens_seen": 1724907520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024108324974924774, + "loss": 1.2811, + "theoretical_loss": 3.473583167839972, + "tokens_seen": 1725038592 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024106318956870613, + "loss": 1.2074, + "theoretical_loss": 3.473560517795536, + "tokens_seen": 1725169664 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.4528958797454834, + "objective/train/avg_token_score": 0.014270788989961147, + "objective/train/docs_used": 977030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5083386898040771, + "objective/train/original_loss": 3.6323282718658447, + "objective/train/theoretical_loss": 3.473549193599318, + "objective/train/tokens_used": 1725235200, + "objective/train/unlikelihood_loss": 0.16081441938877106, + "theoretical_loss": 3.473549193599318, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002410431293881645, + "loss": 1.4729, + "theoretical_loss": 3.4735378699537027, + "tokens_seen": 1725300736 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024102306920762288, + "loss": 1.255, + "theoretical_loss": 3.4735152243140894, + "tokens_seen": 1725431808 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024100300902708124, + "loss": 1.2488, + "theoretical_loss": 3.4734925808763153, + "tokens_seen": 1725562880 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024098294884653963, + "loss": 1.1701, + "theoretical_loss": 3.4734699396399993, + "tokens_seen": 1725693952 + }, + { + "epoch": 0.52, + "learning_rate": 0.000240962888665998, + "loss": 1.3723, + "theoretical_loss": 3.4734473006047595, + "tokens_seen": 1725825024 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024094282848545638, + "loss": 1.2351, + "theoretical_loss": 3.4734246637702153, + "tokens_seen": 1725956096 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024092276830491475, + "loss": 1.3144, + "theoretical_loss": 3.4734020291359853, + "tokens_seen": 1726087168 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002409027081243731, + "loss": 1.2727, + "theoretical_loss": 3.473379396701689, + "tokens_seen": 1726218240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002408826479438315, + "loss": 1.3572, + "theoretical_loss": 3.473356766466945, + "tokens_seen": 1726349312 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024086258776328986, + "loss": 1.3108, + "theoretical_loss": 3.4733341384313734, + "tokens_seen": 1726480384 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024084252758274825, + "loss": 1.3968, + "theoretical_loss": 3.4733115125945924, + "tokens_seen": 1726611456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002408224674022066, + "loss": 1.2894, + "theoretical_loss": 3.4732888889562226, + "tokens_seen": 1726742528 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.2817082107067108, + "objective/train/avg_token_score": 0.019839085638523102, + "objective/train/docs_used": 977402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8908194899559021, + "objective/train/original_loss": 4.019481658935547, + "objective/train/theoretical_loss": 3.473266267515883, + "objective/train/tokens_used": 1726873600, + "objective/train/unlikelihood_loss": 0.16856689751148224, + "theoretical_loss": 3.473266267515883, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.52, + "learning_rate": 0.000240802407221665, + "loss": 1.162, + "theoretical_loss": 3.473266267515883, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024078234704112336, + "loss": 1.2517, + "theoretical_loss": 3.4732436482731925, + "tokens_seen": 1727004672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024076228686058175, + "loss": 1.3488, + "theoretical_loss": 3.473221031227773, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024074222668004012, + "loss": 1.3887, + "theoretical_loss": 3.473198416379242, + "tokens_seen": 1727266816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024072216649949848, + "loss": 1.3564, + "theoretical_loss": 3.4731758037272202, + "tokens_seen": 1727397888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024070210631895687, + "loss": 1.3954, + "theoretical_loss": 3.4731531932713287, + "tokens_seen": 1727528960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024068204613841523, + "loss": 1.4632, + "theoretical_loss": 3.4731305850111855, + "tokens_seen": 1727660032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024066198595787362, + "loss": 1.1822, + "theoretical_loss": 3.473107978946413, + "tokens_seen": 1727791104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024064192577733198, + "loss": 1.236, + "theoretical_loss": 3.4730853750766304, + "tokens_seen": 1727922176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024062186559679037, + "loss": 1.3805, + "theoretical_loss": 3.4730627734014576, + "tokens_seen": 1728053248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024060180541624876, + "loss": 1.1614, + "theoretical_loss": 3.4730401739205163, + "tokens_seen": 1728184320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024058174523570715, + "loss": 1.3994, + "theoretical_loss": 3.473017576633426, + "tokens_seen": 1728315392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002405616850551655, + "loss": 1.1937, + "theoretical_loss": 3.472994981539808, + "tokens_seen": 1728446464 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.29777616262435913, + "objective/train/avg_token_score": 0.012793656438589096, + "objective/train/docs_used": 978835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8825053572654724, + "objective/train/original_loss": 4.230527400970459, + "objective/train/theoretical_loss": 3.4729836848154325, + "objective/train/tokens_used": 1728512000, + "objective/train/unlikelihood_loss": 0.15972572565078735, + "theoretical_loss": 3.4729836848154325, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002405416248746239, + "loss": 1.2161, + "theoretical_loss": 3.472972388639283, + "tokens_seen": 1728577536 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024052156469408226, + "loss": 1.3158, + "theoretical_loss": 3.472949797931472, + "tokens_seen": 1728708608 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024050150451354062, + "loss": 1.2815, + "theoretical_loss": 3.472927209415995, + "tokens_seen": 1728839680 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024048144433299901, + "loss": 1.1436, + "theoretical_loss": 3.4729046230924743, + "tokens_seen": 1728970752 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024046138415245738, + "loss": 1.3741, + "theoretical_loss": 3.4728820389605306, + "tokens_seen": 1729101824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024044132397191577, + "loss": 1.238, + "theoretical_loss": 3.472859457019785, + "tokens_seen": 1729232896 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024042126379137413, + "loss": 1.4013, + "theoretical_loss": 3.4728368772698586, + "tokens_seen": 1729363968 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024040120361083252, + "loss": 1.2035, + "theoretical_loss": 3.4728142997103735, + "tokens_seen": 1729495040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024038114343029088, + "loss": 1.3165, + "theoretical_loss": 3.472791724340951, + "tokens_seen": 1729626112 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024036108324974927, + "loss": 1.2132, + "theoretical_loss": 3.472769151161213, + "tokens_seen": 1729757184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024034102306920763, + "loss": 1.177, + "theoretical_loss": 3.4727465801707798, + "tokens_seen": 1729888256 + }, + { + "epoch": 0.52, + "learning_rate": 0.000240320962888666, + "loss": 1.0924, + "theoretical_loss": 3.4727240113692748, + "tokens_seen": 1730019328 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.3437194526195526, + "objective/train/avg_token_score": 0.009118499234318733, + "objective/train/docs_used": 979527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1072030067443848, + "objective/train/original_loss": 4.144057750701904, + "objective/train/theoretical_loss": 3.4727014447563196, + "objective/train/tokens_used": 1730150400, + "objective/train/unlikelihood_loss": 0.15046684443950653, + "theoretical_loss": 3.4727014447563196, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024030090270812438, + "loss": 1.2414, + "theoretical_loss": 3.4727014447563196, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024028084252758274, + "loss": 1.1664, + "theoretical_loss": 3.472678880331535, + "tokens_seen": 1730281472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024026078234704113, + "loss": 1.1902, + "theoretical_loss": 3.472656318094545, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002402407221664995, + "loss": 1.1072, + "theoretical_loss": 3.47263375804497, + "tokens_seen": 1730543616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024022066198595789, + "loss": 1.2185, + "theoretical_loss": 3.472611200182434, + "tokens_seen": 1730674688 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024020060180541625, + "loss": 1.2861, + "theoretical_loss": 3.472588644506558, + "tokens_seen": 1730805760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024018054162487464, + "loss": 1.2871, + "theoretical_loss": 3.4725660910169647, + "tokens_seen": 1730936832 + }, + { + "epoch": 0.52, + "learning_rate": 0.000240160481444333, + "loss": 1.352, + "theoretical_loss": 3.472543539713277, + "tokens_seen": 1731067904 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024014042126379136, + "loss": 1.2125, + "theoretical_loss": 3.4725209905951173, + "tokens_seen": 1731198976 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024012036108324975, + "loss": 1.2678, + "theoretical_loss": 3.4724984436621087, + "tokens_seen": 1731330048 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024010030090270811, + "loss": 1.2837, + "theoretical_loss": 3.4724758989138738, + "tokens_seen": 1731461120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002400802407221665, + "loss": 1.2196, + "theoretical_loss": 3.4724533563500355, + "tokens_seen": 1731592192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024006018054162487, + "loss": 1.3421, + "theoretical_loss": 3.472430815970217, + "tokens_seen": 1731723264 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.4078079164028168, + "objective/train/avg_token_score": 0.04142635688185692, + "objective/train/docs_used": 980288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3886165618896484, + "objective/train/original_loss": 4.043662071228027, + "objective/train/theoretical_loss": 3.4724195465991974, + "objective/train/tokens_used": 1731788800, + "objective/train/unlikelihood_loss": 0.15409962832927704, + "theoretical_loss": 3.4724195465991974, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024004012036108325, + "loss": 1.3482, + "theoretical_loss": 3.4724082777740413, + "tokens_seen": 1731854336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024002006018054162, + "loss": 1.1561, + "theoretical_loss": 3.4723857417611317, + "tokens_seen": 1731985408 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024, + "loss": 1.2601, + "theoretical_loss": 3.472363207931111, + "tokens_seen": 1732116480 + }, + { + "epoch": 0.52, + "learning_rate": 0.00023997993981945837, + "loss": 1.2517, + "theoretical_loss": 3.4723406762836033, + "tokens_seen": 1732247552 + }, + { + "epoch": 0.52, + "learning_rate": 0.00023995987963891676, + "loss": 1.3775, + "theoretical_loss": 3.472318146818232, + "tokens_seen": 1732378624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023993981945837512, + "loss": 1.4381, + "theoretical_loss": 3.47229561953462, + "tokens_seen": 1732509696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023991975927783348, + "loss": 1.4084, + "theoretical_loss": 3.4722730944323916, + "tokens_seen": 1732640768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023989969909729187, + "loss": 1.1909, + "theoretical_loss": 3.4722505715111707, + "tokens_seen": 1732771840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023987963891675023, + "loss": 1.3794, + "theoretical_loss": 3.472228050770581, + "tokens_seen": 1732902912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023985957873620865, + "loss": 1.2803, + "theoretical_loss": 3.4722055322102463, + "tokens_seen": 1733033984 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239839518555667, + "loss": 1.311, + "theoretical_loss": 3.47218301582979, + "tokens_seen": 1733165056 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002398194583751254, + "loss": 1.2684, + "theoretical_loss": 3.472160501628838, + "tokens_seen": 1733296128 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.4060361683368683, + "objective/train/avg_token_score": 0.028027016669511795, + "objective/train/docs_used": 981145, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4067858457565308, + "objective/train/original_loss": 4.15606689453125, + "objective/train/theoretical_loss": 3.4721379896070133, + "objective/train/tokens_used": 1733427200, + "objective/train/unlikelihood_loss": 0.1413305699825287, + "theoretical_loss": 3.4721379896070133, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023979939819458376, + "loss": 1.2739, + "theoretical_loss": 3.4721379896070133, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023977933801404215, + "loss": 1.2644, + "theoretical_loss": 3.4721154797639406, + "tokens_seen": 1733558272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023975927783350052, + "loss": 1.3965, + "theoretical_loss": 3.472092972099244, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023973921765295888, + "loss": 1.1836, + "theoretical_loss": 3.472070466612548, + "tokens_seen": 1733820416 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023971915747241727, + "loss": 1.204, + "theoretical_loss": 3.4720479633034778, + "tokens_seen": 1733951488 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023969909729187563, + "loss": 1.3413, + "theoretical_loss": 3.4720254621716578, + "tokens_seen": 1734082560 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023967903711133402, + "loss": 1.3993, + "theoretical_loss": 3.4720029632167124, + "tokens_seen": 1734213632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023965897693079238, + "loss": 1.2617, + "theoretical_loss": 3.471980466438267, + "tokens_seen": 1734344704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023963891675025077, + "loss": 1.3196, + "theoretical_loss": 3.4719579718359466, + "tokens_seen": 1734475776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023961885656970913, + "loss": 1.3397, + "theoretical_loss": 3.471935479409376, + "tokens_seen": 1734606848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023959879638916752, + "loss": 1.4013, + "theoretical_loss": 3.471912989158181, + "tokens_seen": 1734737920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023957873620862588, + "loss": 1.324, + "theoretical_loss": 3.471890501081985, + "tokens_seen": 1734868992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023955867602808427, + "loss": 1.1944, + "theoretical_loss": 3.4718680151804158, + "tokens_seen": 1735000064 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.38361436128616333, + "objective/train/avg_token_score": 0.06325226277112961, + "objective/train/docs_used": 982466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2645334005355835, + "objective/train/original_loss": 4.190939903259277, + "objective/train/theoretical_loss": 3.4718567730449985, + "objective/train/tokens_used": 1735065600, + "objective/train/unlikelihood_loss": 0.1751450151205063, + "theoretical_loss": 3.4718567730449985, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023953861584754264, + "loss": 1.4032, + "theoretical_loss": 3.4718455314530976, + "tokens_seen": 1735131136 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239518555667001, + "loss": 1.2349, + "theoretical_loss": 3.4718230498996556, + "tokens_seen": 1735262208 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394984954864594, + "loss": 1.3283, + "theoretical_loss": 3.4718005705197164, + "tokens_seen": 1735393280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023947843530591775, + "loss": 1.4201, + "theoretical_loss": 3.471778093312905, + "tokens_seen": 1735524352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023945837512537614, + "loss": 1.3081, + "theoretical_loss": 3.4717556182788476, + "tokens_seen": 1735655424 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394383149448345, + "loss": 1.3496, + "theoretical_loss": 3.4717331454171703, + "tokens_seen": 1735786496 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394182547642929, + "loss": 1.29, + "theoretical_loss": 3.4717106747274986, + "tokens_seen": 1735917568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023939819458375125, + "loss": 1.3423, + "theoretical_loss": 3.4716882062094587, + "tokens_seen": 1736048640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023937813440320964, + "loss": 1.2963, + "theoretical_loss": 3.4716657398626767, + "tokens_seen": 1736179712 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239358074222668, + "loss": 1.3158, + "theoretical_loss": 3.4716432756867794, + "tokens_seen": 1736310784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023933801404212637, + "loss": 1.3706, + "theoretical_loss": 3.471620813681393, + "tokens_seen": 1736441856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023931795386158476, + "loss": 1.3696, + "theoretical_loss": 3.4715983538461432, + "tokens_seen": 1736572928 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.4292522072792053, + "objective/train/avg_token_score": 0.003412567777559161, + "objective/train/docs_used": 983129, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3720672130584717, + "objective/train/original_loss": 3.7713356018066406, + "objective/train/theoretical_loss": 3.4715758961806573, + "objective/train/tokens_used": 1736704000, + "objective/train/unlikelihood_loss": 0.1456233710050583, + "theoretical_loss": 3.4715758961806573, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023929789368104312, + "loss": 1.308, + "theoretical_loss": 3.4715758961806573, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002392778335005015, + "loss": 1.2911, + "theoretical_loss": 3.4715534406845627, + "tokens_seen": 1736835072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023925777331995987, + "loss": 1.3173, + "theoretical_loss": 3.4715309873574847, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023923771313941826, + "loss": 1.2384, + "theoretical_loss": 3.4715085361990505, + "tokens_seen": 1737097216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023921765295887662, + "loss": 1.2671, + "theoretical_loss": 3.4714860872088877, + "tokens_seen": 1737228288 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239197592778335, + "loss": 1.3592, + "theoretical_loss": 3.471463640386623, + "tokens_seen": 1737359360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023917753259779337, + "loss": 1.2362, + "theoretical_loss": 3.4714411957318827, + "tokens_seen": 1737490432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023915747241725174, + "loss": 1.2939, + "theoretical_loss": 3.4714187532442953, + "tokens_seen": 1737621504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023913741223671012, + "loss": 1.514, + "theoretical_loss": 3.4713963129234875, + "tokens_seen": 1737752576 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002391173520561685, + "loss": 1.4482, + "theoretical_loss": 3.4713738747690863, + "tokens_seen": 1737883648 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002390972918756269, + "loss": 1.2442, + "theoretical_loss": 3.4713514387807196, + "tokens_seen": 1738014720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023907723169508527, + "loss": 1.2882, + "theoretical_loss": 3.4713290049580157, + "tokens_seen": 1738145792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023905717151454365, + "loss": 1.2741, + "theoretical_loss": 3.471306573300601, + "tokens_seen": 1738276864 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.31983137130737305, + "objective/train/avg_token_score": 0.019290989264845848, + "objective/train/docs_used": 984348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.991836667060852, + "objective/train/original_loss": 3.4936046600341797, + "objective/train/theoretical_loss": 3.4712953582837613, + "objective/train/tokens_used": 1738342400, + "objective/train/unlikelihood_loss": 0.24434490501880646, + "theoretical_loss": 3.4712953582837613, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023903711133400202, + "loss": 1.1777, + "theoretical_loss": 3.471284143808104, + "tokens_seen": 1738407936 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002390170511534604, + "loss": 1.3926, + "theoretical_loss": 3.471261716480152, + "tokens_seen": 1738539008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023899699097291877, + "loss": 1.2849, + "theoretical_loss": 3.4712392913163734, + "tokens_seen": 1738670080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023897693079237716, + "loss": 1.3451, + "theoretical_loss": 3.4712168683163966, + "tokens_seen": 1738801152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023895687061183552, + "loss": 1.2554, + "theoretical_loss": 3.4711944474798493, + "tokens_seen": 1738932224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023893681043129388, + "loss": 1.1695, + "theoretical_loss": 3.4711720288063592, + "tokens_seen": 1739063296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023891675025075227, + "loss": 1.2715, + "theoretical_loss": 3.471149612295555, + "tokens_seen": 1739194368 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023889669007021063, + "loss": 1.4254, + "theoretical_loss": 3.471127197947066, + "tokens_seen": 1739325440 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023887662988966902, + "loss": 1.3844, + "theoretical_loss": 3.4711047857605193, + "tokens_seen": 1739456512 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023885656970912739, + "loss": 1.2562, + "theoretical_loss": 3.471082375735544, + "tokens_seen": 1739587584 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023883650952858577, + "loss": 1.3103, + "theoretical_loss": 3.471059967871769, + "tokens_seen": 1739718656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023881644934804414, + "loss": 1.3212, + "theoretical_loss": 3.4710375621688234, + "tokens_seen": 1739849728 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.4883919954299927, + "objective/train/avg_token_score": 0.006087710149586201, + "objective/train/docs_used": 985100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.537111759185791, + "objective/train/original_loss": 3.4930083751678467, + "objective/train/theoretical_loss": 3.471015158626335, + "objective/train/tokens_used": 1739980800, + "objective/train/unlikelihood_loss": 0.1743123084306717, + "theoretical_loss": 3.471015158626335, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023879638916750253, + "loss": 1.306, + "theoretical_loss": 3.471015158626335, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002387763289869609, + "loss": 1.1381, + "theoretical_loss": 3.470992757243934, + "tokens_seen": 1740111872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023875626880641925, + "loss": 1.4563, + "theoretical_loss": 3.4709703580212485, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023873620862587764, + "loss": 1.4361, + "theoretical_loss": 3.4709479609579077, + "tokens_seen": 1740374016 + }, + { + "epoch": 0.53, + "learning_rate": 0.000238716148445336, + "loss": 1.354, + "theoretical_loss": 3.470925566053541, + "tokens_seen": 1740505088 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386960882647944, + "loss": 1.2918, + "theoretical_loss": 3.4709031733077786, + "tokens_seen": 1740636160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023867602808425275, + "loss": 1.3036, + "theoretical_loss": 3.470880782720249, + "tokens_seen": 1740767232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023865596790371114, + "loss": 1.3367, + "theoretical_loss": 3.4708583942905813, + "tokens_seen": 1740898304 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386359077231695, + "loss": 1.3515, + "theoretical_loss": 3.4708360080184057, + "tokens_seen": 1741029376 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386158475426279, + "loss": 1.2508, + "theoretical_loss": 3.470813623903352, + "tokens_seen": 1741160448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023859578736208626, + "loss": 1.4055, + "theoretical_loss": 3.47079124194505, + "tokens_seen": 1741291520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023857572718154462, + "loss": 1.1826, + "theoretical_loss": 3.470768862143129, + "tokens_seen": 1741422592 + }, + { + "epoch": 0.53, + "learning_rate": 0.000238555667001003, + "loss": 1.2622, + "theoretical_loss": 3.4707464844972193, + "tokens_seen": 1741553664 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.378543496131897, + "objective/train/avg_token_score": 0.04502832144498825, + "objective/train/docs_used": 986266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2691749334335327, + "objective/train/original_loss": 3.986555576324463, + "objective/train/theoretical_loss": 3.470735296482653, + "objective/train/tokens_used": 1741619200, + "objective/train/unlikelihood_loss": 0.1581888645887375, + "theoretical_loss": 3.470735296482653, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023853560682046137, + "loss": 1.2865, + "theoretical_loss": 3.470724109006951, + "tokens_seen": 1741684736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023851554663991976, + "loss": 1.3085, + "theoretical_loss": 3.4707017356719545, + "tokens_seen": 1741815808 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023849548645937812, + "loss": 1.1557, + "theoretical_loss": 3.47067936449186, + "tokens_seen": 1741946880 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002384754262788365, + "loss": 1.2124, + "theoretical_loss": 3.470656995466297, + "tokens_seen": 1742077952 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023845536609829487, + "loss": 1.2496, + "theoretical_loss": 3.470634628594897, + "tokens_seen": 1742209024 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023843530591775326, + "loss": 1.3508, + "theoretical_loss": 3.47061226387729, + "tokens_seen": 1742340096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023841524573721163, + "loss": 1.2218, + "theoretical_loss": 3.4705899013131063, + "tokens_seen": 1742471168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023839518555667002, + "loss": 1.2687, + "theoretical_loss": 3.4705675409019774, + "tokens_seen": 1742602240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023837512537612838, + "loss": 1.3321, + "theoretical_loss": 3.4705451826435327, + "tokens_seen": 1742733312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023835506519558677, + "loss": 1.2692, + "theoretical_loss": 3.4705228265374046, + "tokens_seen": 1742864384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023833500501504516, + "loss": 1.2503, + "theoretical_loss": 3.4705004725832236, + "tokens_seen": 1742995456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023831494483450352, + "loss": 1.1851, + "theoretical_loss": 3.470478120780621, + "tokens_seen": 1743126528 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.6091764569282532, + "objective/train/avg_token_score": 0.014462779276072979, + "objective/train/docs_used": 986906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6863874197006226, + "objective/train/original_loss": 3.2984488010406494, + "objective/train/theoretical_loss": 3.4704557711292265, + "objective/train/tokens_used": 1743257600, + "objective/train/unlikelihood_loss": 0.14984466135501862, + "theoretical_loss": 3.4704557711292265, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002382948846539619, + "loss": 1.3043, + "theoretical_loss": 3.4704557711292265, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023827482447342027, + "loss": 1.2744, + "theoretical_loss": 3.4704334236286734, + "tokens_seen": 1743388672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023825476429287866, + "loss": 1.3733, + "theoretical_loss": 3.470411078278591, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023823470411233702, + "loss": 1.2194, + "theoretical_loss": 3.470388735078613, + "tokens_seen": 1743650816 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002382146439317954, + "loss": 1.3735, + "theoretical_loss": 3.4703663940283684, + "tokens_seen": 1743781888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023819458375125377, + "loss": 1.3722, + "theoretical_loss": 3.470344055127491, + "tokens_seen": 1743912960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023817452357071214, + "loss": 1.3325, + "theoretical_loss": 3.4703217183756108, + "tokens_seen": 1744044032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023815446339017052, + "loss": 1.2122, + "theoretical_loss": 3.470299383772361, + "tokens_seen": 1744175104 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002381344032096289, + "loss": 1.4574, + "theoretical_loss": 3.4702770513173724, + "tokens_seen": 1744306176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023811434302908728, + "loss": 1.3591, + "theoretical_loss": 3.4702547210102774, + "tokens_seen": 1744437248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023809428284854564, + "loss": 1.2043, + "theoretical_loss": 3.470232392850708, + "tokens_seen": 1744568320 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023807422266800403, + "loss": 1.3593, + "theoretical_loss": 3.4702100668382965, + "tokens_seen": 1744699392 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002380541624874624, + "loss": 1.2662, + "theoretical_loss": 3.470187742972675, + "tokens_seen": 1744830464 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.49786168336868286, + "objective/train/avg_token_score": 0.08227621018886566, + "objective/train/docs_used": 987618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4971760511398315, + "objective/train/original_loss": 3.486266851425171, + "objective/train/theoretical_loss": 3.470176581844796, + "objective/train/tokens_used": 1744896000, + "objective/train/unlikelihood_loss": 0.17308412492275238, + "theoretical_loss": 3.470176581844796, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023803410230692078, + "loss": 1.305, + "theoretical_loss": 3.4701654212534763, + "tokens_seen": 1744961536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023801404212637914, + "loss": 1.4288, + "theoretical_loss": 3.470143101680332, + "tokens_seen": 1745092608 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023799398194583753, + "loss": 1.2843, + "theoretical_loss": 3.4701207842528747, + "tokens_seen": 1745223680 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002379739217652959, + "loss": 1.4993, + "theoretical_loss": 3.470098468970738, + "tokens_seen": 1745354752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023795386158475426, + "loss": 1.3344, + "theoretical_loss": 3.470076155833553, + "tokens_seen": 1745485824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023793380140421265, + "loss": 1.3575, + "theoretical_loss": 3.4700538448409537, + "tokens_seen": 1745616896 + }, + { + "epoch": 0.53, + "learning_rate": 0.000237913741223671, + "loss": 1.3378, + "theoretical_loss": 3.4700315359925726, + "tokens_seen": 1745747968 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378936810431294, + "loss": 1.2747, + "theoretical_loss": 3.470009229288043, + "tokens_seen": 1745879040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023787362086258776, + "loss": 1.4152, + "theoretical_loss": 3.469986924726997, + "tokens_seen": 1746010112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023785356068204615, + "loss": 1.2348, + "theoretical_loss": 3.4699646223090688, + "tokens_seen": 1746141184 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378335005015045, + "loss": 1.3599, + "theoretical_loss": 3.469942322033891, + "tokens_seen": 1746272256 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378134403209629, + "loss": 1.2279, + "theoretical_loss": 3.469920023901097, + "tokens_seen": 1746403328 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.41758307814598083, + "objective/train/avg_token_score": 0.0069565302692353725, + "objective/train/docs_used": 989102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1292983293533325, + "objective/train/original_loss": 3.132930040359497, + "objective/train/theoretical_loss": 3.469897727910321, + "objective/train/tokens_used": 1746534400, + "objective/train/unlikelihood_loss": 0.290651798248291, + "theoretical_loss": 3.469897727910321, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023779338014042126, + "loss": 1.3283, + "theoretical_loss": 3.469897727910321, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023777331995987962, + "loss": 1.2308, + "theoretical_loss": 3.4698754340611955, + "tokens_seen": 1746665472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023775325977933801, + "loss": 1.1773, + "theoretical_loss": 3.4698531423533545, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023773319959879638, + "loss": 1.3192, + "theoretical_loss": 3.4698308527864317, + "tokens_seen": 1746927616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023771313941825477, + "loss": 1.3252, + "theoretical_loss": 3.4698085653600605, + "tokens_seen": 1747058688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023769307923771313, + "loss": 1.2804, + "theoretical_loss": 3.4697862800738752, + "tokens_seen": 1747189760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023767301905717152, + "loss": 1.4528, + "theoretical_loss": 3.46976399692751, + "tokens_seen": 1747320832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023765295887662988, + "loss": 1.2943, + "theoretical_loss": 3.469741715920598, + "tokens_seen": 1747451904 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023763289869608827, + "loss": 1.3217, + "theoretical_loss": 3.469719437052774, + "tokens_seen": 1747582976 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023761283851554663, + "loss": 1.2094, + "theoretical_loss": 3.469697160323672, + "tokens_seen": 1747714048 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023759277833500502, + "loss": 1.2192, + "theoretical_loss": 3.469674885732927, + "tokens_seen": 1747845120 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002375727181544634, + "loss": 1.1815, + "theoretical_loss": 3.4696526132801724, + "tokens_seen": 1747976192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023755265797392177, + "loss": 1.3122, + "theoretical_loss": 3.4696303429650435, + "tokens_seen": 1748107264 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.342803031206131, + "objective/train/avg_token_score": 0.021170923486351967, + "objective/train/docs_used": 989762, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0390747785568237, + "objective/train/original_loss": 4.1245436668396, + "objective/train/theoretical_loss": 3.469619208608974, + "objective/train/tokens_used": 1748172800, + "objective/train/unlikelihood_loss": 0.12949968874454498, + "theoretical_loss": 3.469619208608974, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023753259779338016, + "loss": 1.3528, + "theoretical_loss": 3.469608074787174, + "tokens_seen": 1748238336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023751253761283852, + "loss": 1.4238, + "theoretical_loss": 3.4695858087461993, + "tokens_seen": 1748369408 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002374924774322969, + "loss": 1.2142, + "theoretical_loss": 3.469563544841754, + "tokens_seen": 1748500480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023747241725175527, + "loss": 1.3899, + "theoretical_loss": 3.4695412830734726, + "tokens_seen": 1748631552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023745235707121366, + "loss": 1.1809, + "theoretical_loss": 3.469519023440991, + "tokens_seen": 1748762624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023743229689067203, + "loss": 1.2361, + "theoretical_loss": 3.4694967659439433, + "tokens_seen": 1748893696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023741223671013042, + "loss": 1.4021, + "theoretical_loss": 3.4694745105819647, + "tokens_seen": 1749024768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023739217652958878, + "loss": 1.2982, + "theoretical_loss": 3.469452257354691, + "tokens_seen": 1749155840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023737211634904714, + "loss": 1.1346, + "theoretical_loss": 3.4694300062617565, + "tokens_seen": 1749286912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023735205616850553, + "loss": 1.3313, + "theoretical_loss": 3.4694077573027977, + "tokens_seen": 1749417984 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002373319959879639, + "loss": 1.3425, + "theoretical_loss": 3.4693855104774496, + "tokens_seen": 1749549056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023731193580742228, + "loss": 1.2359, + "theoretical_loss": 3.469363265785348, + "tokens_seen": 1749680128 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.5081866979598999, + "objective/train/avg_token_score": 0.01055136974900961, + "objective/train/docs_used": 990903, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.554496169090271, + "objective/train/original_loss": 3.4128406047821045, + "objective/train/theoretical_loss": 3.4693410232261277, + "objective/train/tokens_used": 1749811200, + "objective/train/unlikelihood_loss": 0.2113381028175354, + "theoretical_loss": 3.4693410232261277, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023729187562688064, + "loss": 1.1955, + "theoretical_loss": 3.4693410232261277, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023727181544633903, + "loss": 1.3434, + "theoretical_loss": 3.469318782799425, + "tokens_seen": 1749942272 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002372517552657974, + "loss": 1.3624, + "theoretical_loss": 3.4692965445048767, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023723169508525578, + "loss": 1.195, + "theoretical_loss": 3.469274308342117, + "tokens_seen": 1750204416 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023721163490471415, + "loss": 1.3262, + "theoretical_loss": 3.4692520743107833, + "tokens_seen": 1750335488 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002371915747241725, + "loss": 1.2564, + "theoretical_loss": 3.469229842410511, + "tokens_seen": 1750466560 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002371715145436309, + "loss": 1.1772, + "theoretical_loss": 3.469207612640936, + "tokens_seen": 1750597632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023715145436308926, + "loss": 1.319, + "theoretical_loss": 3.469185385001696, + "tokens_seen": 1750728704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023713139418254765, + "loss": 1.2546, + "theoretical_loss": 3.469163159492426, + "tokens_seen": 1750859776 + }, + { + "epoch": 0.53, + "learning_rate": 0.000237111334002006, + "loss": 1.1584, + "theoretical_loss": 3.4691409361127628, + "tokens_seen": 1750990848 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002370912738214644, + "loss": 1.1511, + "theoretical_loss": 3.469118714862343, + "tokens_seen": 1751121920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023707121364092276, + "loss": 1.4431, + "theoretical_loss": 3.4690964957408035, + "tokens_seen": 1751252992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023705115346038115, + "loss": 1.4096, + "theoretical_loss": 3.4690742787477804, + "tokens_seen": 1751384064 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.4223484992980957, + "objective/train/avg_token_score": 0.0037719833198934793, + "objective/train/docs_used": 991641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.364516258239746, + "objective/train/original_loss": 3.4095308780670166, + "objective/train/theoretical_loss": 3.4690631710493496, + "objective/train/tokens_used": 1751449600, + "objective/train/unlikelihood_loss": 0.2689692974090576, + "theoretical_loss": 3.4690631710493496, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023703109327983952, + "loss": 1.2675, + "theoretical_loss": 3.469052063882912, + "tokens_seen": 1751515136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023701103309929788, + "loss": 1.374, + "theoretical_loss": 3.469029851145833, + "tokens_seen": 1751646208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023699097291875627, + "loss": 1.2902, + "theoretical_loss": 3.469007640536182, + "tokens_seen": 1751777280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023697091273821463, + "loss": 1.2642, + "theoretical_loss": 3.4689854320535956, + "tokens_seen": 1751908352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023695085255767302, + "loss": 1.2734, + "theoretical_loss": 3.468963225697711, + "tokens_seen": 1752039424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023693079237713138, + "loss": 1.2689, + "theoretical_loss": 3.468941021468166, + "tokens_seen": 1752170496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023691073219658977, + "loss": 1.2914, + "theoretical_loss": 3.468918819364597, + "tokens_seen": 1752301568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023689067201604813, + "loss": 1.3179, + "theoretical_loss": 3.4688966193866415, + "tokens_seen": 1752432640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023687061183550652, + "loss": 1.265, + "theoretical_loss": 3.468874421533938, + "tokens_seen": 1752563712 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002368505516549649, + "loss": 1.3514, + "theoretical_loss": 3.468852225806123, + "tokens_seen": 1752694784 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002368304914744233, + "loss": 1.2705, + "theoretical_loss": 3.4688300322028356, + "tokens_seen": 1752825856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023681043129388166, + "loss": 1.2237, + "theoretical_loss": 3.4688078407237124, + "tokens_seen": 1752956928 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.3206256031990051, + "objective/train/avg_token_score": 0.02291475050151348, + "objective/train/docs_used": 992944, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9407128691673279, + "objective/train/original_loss": 4.312899589538574, + "objective/train/theoretical_loss": 3.4687856513683917, + "objective/train/tokens_used": 1753088000, + "objective/train/unlikelihood_loss": 0.09350314736366272, + "theoretical_loss": 3.4687856513683917, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023679037111334002, + "loss": 1.2252, + "theoretical_loss": 3.4687856513683917, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023677031093279841, + "loss": 1.409, + "theoretical_loss": 3.4687634641365115, + "tokens_seen": 1753219072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023675025075225678, + "loss": 1.4127, + "theoretical_loss": 3.468741279027709, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023673019057171517, + "loss": 1.3337, + "theoretical_loss": 3.4687190960416245, + "tokens_seen": 1753481216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023671013039117353, + "loss": 1.1885, + "theoretical_loss": 3.468696915177894, + "tokens_seen": 1753612288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023669007021063192, + "loss": 1.3316, + "theoretical_loss": 3.4686747364361565, + "tokens_seen": 1753743360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023667001003009028, + "loss": 1.3228, + "theoretical_loss": 3.4686525598160514, + "tokens_seen": 1753874432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023664994984954867, + "loss": 1.3044, + "theoretical_loss": 3.4686303853172165, + "tokens_seen": 1754005504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023662988966900703, + "loss": 1.3112, + "theoretical_loss": 3.46860821293929, + "tokens_seen": 1754136576 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002366098294884654, + "loss": 1.3658, + "theoretical_loss": 3.4685860426819106, + "tokens_seen": 1754267648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023658976930792378, + "loss": 1.2567, + "theoretical_loss": 3.468563874544718, + "tokens_seen": 1754398720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023656970912738214, + "loss": 1.3122, + "theoretical_loss": 3.4685417085273507, + "tokens_seen": 1754529792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023654964894684053, + "loss": 1.4384, + "theoretical_loss": 3.468519544629447, + "tokens_seen": 1754660864 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.4200879633426666, + "objective/train/avg_token_score": 0.010140310041606426, + "objective/train/docs_used": 993452, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3142601251602173, + "objective/train/original_loss": 3.8705384731292725, + "objective/train/theoretical_loss": 3.468508463475181, + "objective/train/tokens_used": 1754726400, + "objective/train/unlikelihood_loss": 0.1671108901500702, + "theoretical_loss": 3.468508463475181, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002365295887662989, + "loss": 1.3054, + "theoretical_loss": 3.4684973828506465, + "tokens_seen": 1754791936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023650952858575729, + "loss": 1.4116, + "theoretical_loss": 3.4684752231905875, + "tokens_seen": 1754923008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023648946840521565, + "loss": 1.3856, + "theoretical_loss": 3.468453065648911, + "tokens_seen": 1755054080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023646940822467404, + "loss": 1.3741, + "theoretical_loss": 3.468430910225255, + "tokens_seen": 1755185152 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002364493480441324, + "loss": 1.4398, + "theoretical_loss": 3.468408756919259, + "tokens_seen": 1755316224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023642928786359076, + "loss": 1.2584, + "theoretical_loss": 3.4683866057305623, + "tokens_seen": 1755447296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023640922768304915, + "loss": 1.3984, + "theoretical_loss": 3.468364456658805, + "tokens_seen": 1755578368 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002363891675025075, + "loss": 1.2734, + "theoretical_loss": 3.468342309703627, + "tokens_seen": 1755709440 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002363691073219659, + "loss": 1.1805, + "theoretical_loss": 3.4683201648646675, + "tokens_seen": 1755840512 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023634904714142426, + "loss": 1.3525, + "theoretical_loss": 3.4682980221415662, + "tokens_seen": 1755971584 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023632898696088265, + "loss": 1.3746, + "theoretical_loss": 3.4682758815339634, + "tokens_seen": 1756102656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023630892678034102, + "loss": 1.446, + "theoretical_loss": 3.4682537430414992, + "tokens_seen": 1756233728 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.4275415539741516, + "objective/train/avg_token_score": 0.015417186543345451, + "objective/train/docs_used": 994709, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2658015489578247, + "objective/train/original_loss": 3.7915198802948, + "objective/train/theoretical_loss": 3.468231606663813, + "objective/train/tokens_used": 1756364800, + "objective/train/unlikelihood_loss": 0.15812501311302185, + "theoretical_loss": 3.468231606663813, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002362888665997994, + "loss": 1.309, + "theoretical_loss": 3.468231606663813, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023626880641925777, + "loss": 1.2323, + "theoretical_loss": 3.4682094724005457, + "tokens_seen": 1756495872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023624874623871616, + "loss": 1.2892, + "theoretical_loss": 3.4681873402513372, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023622868605817452, + "loss": 1.2161, + "theoretical_loss": 3.4681652102158282, + "tokens_seen": 1756758016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023620862587763288, + "loss": 1.3516, + "theoretical_loss": 3.468143082293659, + "tokens_seen": 1756889088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023618856569709127, + "loss": 1.1423, + "theoretical_loss": 3.46812095648447, + "tokens_seen": 1757020160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023616850551654963, + "loss": 1.3442, + "theoretical_loss": 3.468098832787902, + "tokens_seen": 1757151232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023614844533600802, + "loss": 1.3592, + "theoretical_loss": 3.4680767112035955, + "tokens_seen": 1757282304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023612838515546639, + "loss": 1.2801, + "theoretical_loss": 3.4680545917311916, + "tokens_seen": 1757413376 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361083249749248, + "loss": 1.3006, + "theoretical_loss": 3.4680324743703315, + "tokens_seen": 1757544448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023608826479438316, + "loss": 1.2017, + "theoretical_loss": 3.468010359120655, + "tokens_seen": 1757675520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023606820461384155, + "loss": 1.3743, + "theoretical_loss": 3.4679882459818043, + "tokens_seen": 1757806592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023604814443329992, + "loss": 1.2871, + "theoretical_loss": 3.4679661349534197, + "tokens_seen": 1757937664 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.5237658619880676, + "objective/train/avg_token_score": 0.016879012808203697, + "objective/train/docs_used": 995465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.612779140472412, + "objective/train/original_loss": 3.162489175796509, + "objective/train/theoretical_loss": 3.4679550802305408, + "objective/train/tokens_used": 1758003200, + "objective/train/unlikelihood_loss": 0.23880966007709503, + "theoretical_loss": 3.4679550802305408, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023602808425275828, + "loss": 1.2533, + "theoretical_loss": 3.467944026035144, + "tokens_seen": 1758068736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023600802407221667, + "loss": 1.329, + "theoretical_loss": 3.467921919226616, + "tokens_seen": 1758199808 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023598796389167503, + "loss": 1.2491, + "theoretical_loss": 3.4678998145274793, + "tokens_seen": 1758330880 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023596790371113342, + "loss": 1.2269, + "theoretical_loss": 3.467877711937375, + "tokens_seen": 1758461952 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023594784353059178, + "loss": 1.2404, + "theoretical_loss": 3.467855611455944, + "tokens_seen": 1758593024 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023592778335005017, + "loss": 1.391, + "theoretical_loss": 3.4678335130828284, + "tokens_seen": 1758724096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023590772316950853, + "loss": 1.37, + "theoretical_loss": 3.4678114168176695, + "tokens_seen": 1758855168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023588766298896692, + "loss": 1.3348, + "theoretical_loss": 3.46778932266011, + "tokens_seen": 1758986240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023586760280842528, + "loss": 1.1679, + "theoretical_loss": 3.467767230609791, + "tokens_seen": 1759117312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023584754262788367, + "loss": 1.0902, + "theoretical_loss": 3.4677451406663558, + "tokens_seen": 1759248384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023582748244734204, + "loss": 1.4159, + "theoretical_loss": 3.467723052829445, + "tokens_seen": 1759379456 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002358074222668004, + "loss": 1.29, + "theoretical_loss": 3.4677009670987013, + "tokens_seen": 1759510528 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.47733381390571594, + "objective/train/avg_token_score": 0.029847566038370132, + "objective/train/docs_used": 995841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4250075817108154, + "objective/train/original_loss": 4.258460998535156, + "objective/train/theoretical_loss": 3.4676788834737677, + "objective/train/tokens_used": 1759641600, + "objective/train/unlikelihood_loss": 0.1157526969909668, + "theoretical_loss": 3.4676788834737677, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357873620862588, + "loss": 1.2386, + "theoretical_loss": 3.4676788834737677, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023576730190571715, + "loss": 1.3402, + "theoretical_loss": 3.467656801954286, + "tokens_seen": 1759772672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023574724172517554, + "loss": 1.3356, + "theoretical_loss": 3.4676347225398985, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357271815446339, + "loss": 1.2767, + "theoretical_loss": 3.467612645230248, + "tokens_seen": 1760034816 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357071213640923, + "loss": 1.2306, + "theoretical_loss": 3.467590570024977, + "tokens_seen": 1760165888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023568706118355065, + "loss": 1.1659, + "theoretical_loss": 3.467568496923729, + "tokens_seen": 1760296960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023566700100300904, + "loss": 1.2476, + "theoretical_loss": 3.4675464259261455, + "tokens_seen": 1760428032 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002356469408224674, + "loss": 1.3675, + "theoretical_loss": 3.46752435703187, + "tokens_seen": 1760559104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023562688064192577, + "loss": 1.258, + "theoretical_loss": 3.467502290240546, + "tokens_seen": 1760690176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023560682046138416, + "loss": 1.3702, + "theoretical_loss": 3.4674802255518156, + "tokens_seen": 1760821248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023558676028084252, + "loss": 1.3878, + "theoretical_loss": 3.4674581629653227, + "tokens_seen": 1760952320 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002355667001003009, + "loss": 1.2746, + "theoretical_loss": 3.467436102480711, + "tokens_seen": 1761083392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023554663991975927, + "loss": 1.3213, + "theoretical_loss": 3.4674140440976227, + "tokens_seen": 1761214464 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.37060117721557617, + "objective/train/avg_token_score": 0.013845223933458328, + "objective/train/docs_used": 997243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0882796049118042, + "objective/train/original_loss": 4.006743431091309, + "objective/train/theoretical_loss": 3.467403015694039, + "objective/train/tokens_used": 1761280000, + "objective/train/unlikelihood_loss": 0.15563353896141052, + "theoretical_loss": 3.467403015694039, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023552657973921766, + "loss": 1.4235, + "theoretical_loss": 3.467391987815702, + "tokens_seen": 1761345536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023550651955867602, + "loss": 1.3195, + "theoretical_loss": 3.4673699336345916, + "tokens_seen": 1761476608 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002354864593781344, + "loss": 1.2361, + "theoretical_loss": 3.4673478815539363, + "tokens_seen": 1761607680 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023546639919759277, + "loss": 1.4249, + "theoretical_loss": 3.467325831573379, + "tokens_seen": 1761738752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023544633901705113, + "loss": 1.3973, + "theoretical_loss": 3.4673037836925635, + "tokens_seen": 1761869824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023542627883650952, + "loss": 1.3261, + "theoretical_loss": 3.467281737911134, + "tokens_seen": 1762000896 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002354062186559679, + "loss": 1.2893, + "theoretical_loss": 3.467259694228734, + "tokens_seen": 1762131968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023538615847542628, + "loss": 1.2999, + "theoretical_loss": 3.467237652645008, + "tokens_seen": 1762263040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023536609829488464, + "loss": 1.3262, + "theoretical_loss": 3.4672156131595995, + "tokens_seen": 1762394112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023534603811434305, + "loss": 1.2045, + "theoretical_loss": 3.4671935757721535, + "tokens_seen": 1762525184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023532597793380142, + "loss": 1.1978, + "theoretical_loss": 3.4671715404823136, + "tokens_seen": 1762656256 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002353059177532598, + "loss": 1.2727, + "theoretical_loss": 3.467149507289725, + "tokens_seen": 1762787328 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.2626466155052185, + "objective/train/avg_token_score": 0.015402340330183506, + "objective/train/docs_used": 997805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8023011088371277, + "objective/train/original_loss": 4.462587356567383, + "objective/train/theoretical_loss": 3.4671274761940314, + "objective/train/tokens_used": 1762918400, + "objective/train/unlikelihood_loss": 0.13776002824306488, + "theoretical_loss": 3.4671274761940314, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023528585757271817, + "loss": 1.2924, + "theoretical_loss": 3.4671274761940314, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023526579739217656, + "loss": 1.3912, + "theoretical_loss": 3.4671054471948777, + "tokens_seen": 1763049472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023524573721163492, + "loss": 1.2349, + "theoretical_loss": 3.4670834202919085, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023522567703109328, + "loss": 1.2183, + "theoretical_loss": 3.4670613954847678, + "tokens_seen": 1763311616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023520561685055167, + "loss": 1.2869, + "theoretical_loss": 3.467039372773102, + "tokens_seen": 1763442688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023518555667001003, + "loss": 1.3954, + "theoretical_loss": 3.467017352156555, + "tokens_seen": 1763573760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023516549648946842, + "loss": 1.429, + "theoretical_loss": 3.466995333634772, + "tokens_seen": 1763704832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023514543630892679, + "loss": 1.4897, + "theoretical_loss": 3.4669733172073975, + "tokens_seen": 1763835904 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023512537612838517, + "loss": 1.4124, + "theoretical_loss": 3.4669513028740777, + "tokens_seen": 1763966976 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023510531594784354, + "loss": 1.3813, + "theoretical_loss": 3.466929290634457, + "tokens_seen": 1764098048 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023508525576730193, + "loss": 1.406, + "theoretical_loss": 3.466907280488181, + "tokens_seen": 1764229120 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002350651955867603, + "loss": 1.2922, + "theoretical_loss": 3.4668852724348955, + "tokens_seen": 1764360192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023504513540621865, + "loss": 1.2266, + "theoretical_loss": 3.4668632664742454, + "tokens_seen": 1764491264 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.39149561524391174, + "objective/train/avg_token_score": 0.01643899641931057, + "objective/train/docs_used": 998902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4230890274047852, + "objective/train/original_loss": 4.301625728607178, + "objective/train/theoretical_loss": 3.4668522642785478, + "objective/train/tokens_used": 1764556800, + "objective/train/unlikelihood_loss": 0.11482984572649002, + "theoretical_loss": 3.4668522642785478, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023502507522567704, + "loss": 1.317, + "theoretical_loss": 3.466841262605876, + "tokens_seen": 1764622336 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002350050150451354, + "loss": 1.2714, + "theoretical_loss": 3.4668192608294346, + "tokens_seen": 1764753408 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002349849548645938, + "loss": 1.4313, + "theoretical_loss": 3.4667972611445657, + "tokens_seen": 1764884480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023496489468405215, + "loss": 1.3353, + "theoretical_loss": 3.466775263550915, + "tokens_seen": 1765015552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023494483450351054, + "loss": 1.208, + "theoretical_loss": 3.466753268048129, + "tokens_seen": 1765146624 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002349247743229689, + "loss": 1.5344, + "theoretical_loss": 3.466731274635854, + "tokens_seen": 1765277696 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002349047141424273, + "loss": 1.142, + "theoretical_loss": 3.466709283313735, + "tokens_seen": 1765408768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023488465396188566, + "loss": 1.2714, + "theoretical_loss": 3.466687294081419, + "tokens_seen": 1765539840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023486459378134402, + "loss": 1.222, + "theoretical_loss": 3.4666653069385527, + "tokens_seen": 1765670912 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002348445336008024, + "loss": 1.2378, + "theoretical_loss": 3.4666433218847814, + "tokens_seen": 1765801984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023482447342026077, + "loss": 1.3238, + "theoretical_loss": 3.4666213389197518, + "tokens_seen": 1765933056 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023480441323971916, + "loss": 1.3184, + "theoretical_loss": 3.466599358043111, + "tokens_seen": 1766064128 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.3728005886077881, + "objective/train/avg_token_score": 0.013118423521518707, + "objective/train/docs_used": 999561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.247544288635254, + "objective/train/original_loss": 3.898468017578125, + "objective/train/theoretical_loss": 3.466577379254506, + "objective/train/tokens_used": 1766195200, + "objective/train/unlikelihood_loss": 0.1700659841299057, + "theoretical_loss": 3.466577379254506, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023478435305917752, + "loss": 1.1294, + "theoretical_loss": 3.466577379254506, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002347642928786359, + "loss": 1.3321, + "theoretical_loss": 3.466555402553582, + "tokens_seen": 1766326272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023474423269809427, + "loss": 1.3011, + "theoretical_loss": 3.4665334279399875, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023472417251755266, + "loss": 1.2531, + "theoretical_loss": 3.466511455413368, + "tokens_seen": 1766588416 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023470411233701103, + "loss": 1.3973, + "theoretical_loss": 3.466489484973371, + "tokens_seen": 1766719488 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023468405215646942, + "loss": 1.2722, + "theoretical_loss": 3.4664675166196437, + "tokens_seen": 1766850560 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023466399197592778, + "loss": 1.0889, + "theoretical_loss": 3.4664455503518337, + "tokens_seen": 1766981632 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023464393179538614, + "loss": 1.3008, + "theoretical_loss": 3.4664235861695873, + "tokens_seen": 1767112704 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023462387161484453, + "loss": 1.3481, + "theoretical_loss": 3.466401624072552, + "tokens_seen": 1767243776 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023460381143430292, + "loss": 1.0834, + "theoretical_loss": 3.4663796640603755, + "tokens_seen": 1767374848 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002345837512537613, + "loss": 1.1675, + "theoretical_loss": 3.466357706132705, + "tokens_seen": 1767505920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023456369107321967, + "loss": 1.2998, + "theoretical_loss": 3.466335750289189, + "tokens_seen": 1767636992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023454363089267806, + "loss": 1.167, + "theoretical_loss": 3.466313796529474, + "tokens_seen": 1767768064 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.3030914068222046, + "objective/train/avg_token_score": 0.032471418380737305, + "objective/train/docs_used": 1000589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8860474824905396, + "objective/train/original_loss": 4.219391345977783, + "objective/train/theoretical_loss": 3.466302820430932, + "objective/train/tokens_used": 1767833600, + "objective/train/unlikelihood_loss": 0.1501479297876358, + "theoretical_loss": 3.466302820430932, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023452357071213642, + "loss": 1.1853, + "theoretical_loss": 3.466291844853208, + "tokens_seen": 1767899136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002345035105315948, + "loss": 1.3396, + "theoretical_loss": 3.466269895260039, + "tokens_seen": 1768030208 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023448345035105317, + "loss": 1.0462, + "theoretical_loss": 3.466247947749615, + "tokens_seen": 1768161280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023446339017051154, + "loss": 1.2224, + "theoretical_loss": 3.4662260023215836, + "tokens_seen": 1768292352 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023444332998996992, + "loss": 1.2705, + "theoretical_loss": 3.4662040589755936, + "tokens_seen": 1768423424 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002344232698094283, + "loss": 1.2029, + "theoretical_loss": 3.466182117711293, + "tokens_seen": 1768554496 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023440320962888668, + "loss": 1.16, + "theoretical_loss": 3.46616017852833, + "tokens_seen": 1768685568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023438314944834504, + "loss": 1.1306, + "theoretical_loss": 3.466138241426352, + "tokens_seen": 1768816640 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023436308926780343, + "loss": 1.2032, + "theoretical_loss": 3.466116306405009, + "tokens_seen": 1768947712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002343430290872618, + "loss": 1.3062, + "theoretical_loss": 3.466094373463948, + "tokens_seen": 1769078784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023432296890672018, + "loss": 1.3181, + "theoretical_loss": 3.466072442602819, + "tokens_seen": 1769209856 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023430290872617854, + "loss": 1.325, + "theoretical_loss": 3.4660505138212696, + "tokens_seen": 1769340928 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.39125123620033264, + "objective/train/avg_token_score": 0.0170467346906662, + "objective/train/docs_used": 1001288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2494152784347534, + "objective/train/original_loss": 3.6377789974212646, + "objective/train/theoretical_loss": 3.4660285871189496, + "objective/train/tokens_used": 1769472000, + "objective/train/unlikelihood_loss": 0.17810146510601044, + "theoretical_loss": 3.4660285871189496, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002342828485456369, + "loss": 1.3371, + "theoretical_loss": 3.4660285871189496, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002342627883650953, + "loss": 1.3107, + "theoretical_loss": 3.4660066624955066, + "tokens_seen": 1769603072 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023424272818455366, + "loss": 1.3172, + "theoretical_loss": 3.4659847399505903, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023422266800401204, + "loss": 1.2571, + "theoretical_loss": 3.46596281948385, + "tokens_seen": 1769865216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002342026078234704, + "loss": 1.2547, + "theoretical_loss": 3.4659409010949345, + "tokens_seen": 1769996288 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341825476429288, + "loss": 1.1479, + "theoretical_loss": 3.4659189847834924, + "tokens_seen": 1770127360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023416248746238716, + "loss": 1.2199, + "theoretical_loss": 3.465897070549174, + "tokens_seen": 1770258432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023414242728184555, + "loss": 1.2883, + "theoretical_loss": 3.4658751583916283, + "tokens_seen": 1770389504 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341223671013039, + "loss": 1.2436, + "theoretical_loss": 3.4658532483105042, + "tokens_seen": 1770520576 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341023069207623, + "loss": 1.3009, + "theoretical_loss": 3.465831340305452, + "tokens_seen": 1770651648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023408224674022066, + "loss": 1.306, + "theoretical_loss": 3.465809434376121, + "tokens_seen": 1770782720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023406218655967902, + "loss": 1.3489, + "theoretical_loss": 3.465787530522161, + "tokens_seen": 1770913792 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002340421263791374, + "loss": 1.185, + "theoretical_loss": 3.4657656287432212, + "tokens_seen": 1771044864 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.5327468514442444, + "objective/train/avg_token_score": 0.004544450435787439, + "objective/train/docs_used": 1002447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7120349407196045, + "objective/train/original_loss": 3.526111364364624, + "objective/train/theoretical_loss": 3.465754678631775, + "objective/train/tokens_used": 1771110400, + "objective/train/unlikelihood_loss": 0.14082397520542145, + "theoretical_loss": 3.465754678631775, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023402206619859578, + "loss": 1.4685, + "theoretical_loss": 3.4657437290389526, + "tokens_seen": 1771175936 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023400200601805416, + "loss": 1.308, + "theoretical_loss": 3.4657218314090046, + "tokens_seen": 1771307008 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023398194583751253, + "loss": 1.1825, + "theoretical_loss": 3.465699935853027, + "tokens_seen": 1771438080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023396188565697092, + "loss": 1.1201, + "theoretical_loss": 3.46567804237067, + "tokens_seen": 1771569152 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023394182547642928, + "loss": 1.3398, + "theoretical_loss": 3.4656561509615846, + "tokens_seen": 1771700224 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023392176529588767, + "loss": 1.3476, + "theoretical_loss": 3.4656342616254197, + "tokens_seen": 1771831296 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023390170511534603, + "loss": 1.2261, + "theoretical_loss": 3.465612374361827, + "tokens_seen": 1771962368 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002338816449348044, + "loss": 1.3884, + "theoretical_loss": 3.4655904891704563, + "tokens_seen": 1772093440 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023386158475426278, + "loss": 1.1832, + "theoretical_loss": 3.465568606050958, + "tokens_seen": 1772224512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023384152457372117, + "loss": 1.2617, + "theoretical_loss": 3.4655467250029837, + "tokens_seen": 1772355584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023382146439317956, + "loss": 1.4348, + "theoretical_loss": 3.4655248460261827, + "tokens_seen": 1772486656 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023380140421263792, + "loss": 1.1761, + "theoretical_loss": 3.465502969120207, + "tokens_seen": 1772617728 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.3105449676513672, + "objective/train/avg_token_score": 0.01538858376443386, + "objective/train/docs_used": 1002969, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.04959237575531, + "objective/train/original_loss": 4.507214546203613, + "objective/train/theoretical_loss": 3.465481094284707, + "objective/train/tokens_used": 1772748800, + "objective/train/unlikelihood_loss": 0.1443428099155426, + "theoretical_loss": 3.465481094284707, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002337813440320963, + "loss": 1.2524, + "theoretical_loss": 3.465481094284707, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023376128385155467, + "loss": 1.3961, + "theoretical_loss": 3.4654592215193345, + "tokens_seen": 1772879872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023374122367101306, + "loss": 1.2758, + "theoretical_loss": 3.465437350823739, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023372116349047143, + "loss": 1.3468, + "theoretical_loss": 3.4654154821975727, + "tokens_seen": 1773142016 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023370110330992982, + "loss": 1.3096, + "theoretical_loss": 3.4653936156404868, + "tokens_seen": 1773273088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023368104312938818, + "loss": 1.4006, + "theoretical_loss": 3.465371751152132, + "tokens_seen": 1773404160 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023366098294884654, + "loss": 1.1954, + "theoretical_loss": 3.4653498887321605, + "tokens_seen": 1773535232 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023364092276830493, + "loss": 1.3168, + "theoretical_loss": 3.465328028380223, + "tokens_seen": 1773666304 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002336208625877633, + "loss": 1.3874, + "theoretical_loss": 3.465306170095972, + "tokens_seen": 1773797376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023360080240722168, + "loss": 1.4067, + "theoretical_loss": 3.4652843138790583, + "tokens_seen": 1773928448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023358074222668004, + "loss": 1.5209, + "theoretical_loss": 3.465262459729135, + "tokens_seen": 1774059520 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023356068204613843, + "loss": 1.3159, + "theoretical_loss": 3.4652406076458515, + "tokens_seen": 1774190592 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002335406218655968, + "loss": 1.208, + "theoretical_loss": 3.465218757628862, + "tokens_seen": 1774321664 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.37341153621673584, + "objective/train/avg_token_score": 0.0039716134779155254, + "objective/train/docs_used": 1004180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3328355550765991, + "objective/train/original_loss": 3.7340519428253174, + "objective/train/theoretical_loss": 3.4652078333951186, + "objective/train/tokens_used": 1774387200, + "objective/train/unlikelihood_loss": 0.18588504195213318, + "theoretical_loss": 3.4652078333951186, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023352056168505518, + "loss": 1.3147, + "theoretical_loss": 3.4651969096778177, + "tokens_seen": 1774452736 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023350050150451355, + "loss": 1.476, + "theoretical_loss": 3.465175063792371, + "tokens_seen": 1774583808 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002334804413239719, + "loss": 1.3198, + "theoretical_loss": 3.465153219972173, + "tokens_seen": 1774714880 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002334603811434303, + "loss": 1.2279, + "theoretical_loss": 3.4651313782168764, + "tokens_seen": 1774845952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023344032096288866, + "loss": 1.4865, + "theoretical_loss": 3.4651095385261343, + "tokens_seen": 1774977024 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023342026078234705, + "loss": 1.2635, + "theoretical_loss": 3.4650877008995984, + "tokens_seen": 1775108096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002334002006018054, + "loss": 1.3018, + "theoretical_loss": 3.4650658653369213, + "tokens_seen": 1775239168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002333801404212638, + "loss": 1.3133, + "theoretical_loss": 3.4650440318377562, + "tokens_seen": 1775370240 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023336008024072216, + "loss": 1.4103, + "theoretical_loss": 3.465022200401755, + "tokens_seen": 1775501312 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023334002006018055, + "loss": 1.4257, + "theoretical_loss": 3.4650003710285704, + "tokens_seen": 1775632384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023331995987963891, + "loss": 1.2268, + "theoretical_loss": 3.464978543717856, + "tokens_seen": 1775763456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023329989969909728, + "loss": 1.2341, + "theoretical_loss": 3.464956718469264, + "tokens_seen": 1775894528 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.484115332365036, + "objective/train/avg_token_score": 0.008319268934428692, + "objective/train/docs_used": 1004852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5533770322799683, + "objective/train/original_loss": 3.5351057052612305, + "objective/train/theoretical_loss": 3.4649348952824477, + "objective/train/tokens_used": 1776025600, + "objective/train/unlikelihood_loss": 0.18108296394348145, + "theoretical_loss": 3.4649348952824477, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023327983951855567, + "loss": 1.2748, + "theoretical_loss": 3.4649348952824477, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023325977933801403, + "loss": 1.3324, + "theoretical_loss": 3.46491307415706, + "tokens_seen": 1776156672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023323971915747242, + "loss": 1.3405, + "theoretical_loss": 3.4648912550927546, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023321965897693078, + "loss": 1.2383, + "theoretical_loss": 3.4648694380891847, + "tokens_seen": 1776418816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023319959879638917, + "loss": 1.3093, + "theoretical_loss": 3.464847623146003, + "tokens_seen": 1776549888 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023317953861584753, + "loss": 1.3006, + "theoretical_loss": 3.464825810262863, + "tokens_seen": 1776680960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023315947843530592, + "loss": 1.2178, + "theoretical_loss": 3.464803999439418, + "tokens_seen": 1776812032 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023313941825476428, + "loss": 1.2143, + "theoretical_loss": 3.464782190675323, + "tokens_seen": 1776943104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023311935807422267, + "loss": 1.4491, + "theoretical_loss": 3.464760383970231, + "tokens_seen": 1777074176 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023309929789368106, + "loss": 1.2132, + "theoretical_loss": 3.464738579323795, + "tokens_seen": 1777205248 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023307923771313942, + "loss": 1.2624, + "theoretical_loss": 3.4647167767356692, + "tokens_seen": 1777336320 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023305917753259781, + "loss": 1.4125, + "theoretical_loss": 3.4646949762055077, + "tokens_seen": 1777467392 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023303911735205618, + "loss": 1.2815, + "theoretical_loss": 3.4646731777329647, + "tokens_seen": 1777598464 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.3216642141342163, + "objective/train/avg_token_score": 0.017556222155690193, + "objective/train/docs_used": 1006253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9627801179885864, + "objective/train/original_loss": 3.994245767593384, + "objective/train/theoretical_loss": 3.4646622792681923, + "objective/train/tokens_used": 1777664000, + "objective/train/unlikelihood_loss": 0.1347261518239975, + "theoretical_loss": 3.4646622792681923, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023301905717151457, + "loss": 1.1533, + "theoretical_loss": 3.4646513813176942, + "tokens_seen": 1777729536 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023299899699097293, + "loss": 1.4574, + "theoretical_loss": 3.46462958695935, + "tokens_seen": 1777860608 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023297893681043132, + "loss": 1.2251, + "theoretical_loss": 3.464607794657587, + "tokens_seen": 1777991680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023295887662988968, + "loss": 1.3062, + "theoretical_loss": 3.464586004412059, + "tokens_seen": 1778122752 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023293881644934807, + "loss": 1.3533, + "theoretical_loss": 3.4645642162224206, + "tokens_seen": 1778253824 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023291875626880643, + "loss": 1.1789, + "theoretical_loss": 3.464542430088326, + "tokens_seen": 1778384896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002328986960882648, + "loss": 1.2628, + "theoretical_loss": 3.46452064600943, + "tokens_seen": 1778515968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023287863590772318, + "loss": 1.2363, + "theoretical_loss": 3.4644988639853884, + "tokens_seen": 1778647040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023285857572718154, + "loss": 1.3823, + "theoretical_loss": 3.4644770840158543, + "tokens_seen": 1778778112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023283851554663993, + "loss": 1.419, + "theoretical_loss": 3.4644553061004832, + "tokens_seen": 1778909184 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002328184553660983, + "loss": 1.1216, + "theoretical_loss": 3.46443353023893, + "tokens_seen": 1779040256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023279839518555669, + "loss": 1.3962, + "theoretical_loss": 3.4644117564308505, + "tokens_seen": 1779171328 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.5458822250366211, + "objective/train/avg_token_score": 0.01742255687713623, + "objective/train/docs_used": 1006973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7530816793441772, + "objective/train/original_loss": 3.512991189956665, + "objective/train/theoretical_loss": 3.4643899846758983, + "objective/train/tokens_used": 1779302400, + "objective/train/unlikelihood_loss": 0.13341659307479858, + "theoretical_loss": 3.4643899846758983, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023277833500501505, + "loss": 1.5775, + "theoretical_loss": 3.4643899846758983, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023275827482447344, + "loss": 1.3185, + "theoretical_loss": 3.46436821497373, + "tokens_seen": 1779433472 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002327382146439318, + "loss": 1.2452, + "theoretical_loss": 3.464346447323999, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023271815446339016, + "loss": 1.2961, + "theoretical_loss": 3.464324681726363, + "tokens_seen": 1779695616 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023269809428284855, + "loss": 1.3866, + "theoretical_loss": 3.4643029181804756, + "tokens_seen": 1779826688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002326780341023069, + "loss": 1.3435, + "theoretical_loss": 3.4642811566859937, + "tokens_seen": 1779957760 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002326579739217653, + "loss": 1.2423, + "theoretical_loss": 3.4642593972425715, + "tokens_seen": 1780088832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023263791374122366, + "loss": 1.236, + "theoretical_loss": 3.464237639849866, + "tokens_seen": 1780219904 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023261785356068205, + "loss": 1.3694, + "theoretical_loss": 3.464215884507532, + "tokens_seen": 1780350976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023259779338014042, + "loss": 1.1897, + "theoretical_loss": 3.464194131215226, + "tokens_seen": 1780482048 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002325777331995988, + "loss": 1.3769, + "theoretical_loss": 3.464172379972603, + "tokens_seen": 1780613120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023255767301905717, + "loss": 1.326, + "theoretical_loss": 3.4641506307793204, + "tokens_seen": 1780744192 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023253761283851556, + "loss": 1.2587, + "theoretical_loss": 3.464128883635033, + "tokens_seen": 1780875264 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.4748900234699249, + "objective/train/avg_token_score": 0.008687560446560383, + "objective/train/docs_used": 1008497, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.547424554824829, + "objective/train/original_loss": 3.383101224899292, + "objective/train/theoretical_loss": 3.4641180108311556, + "objective/train/tokens_used": 1780940800, + "objective/train/unlikelihood_loss": 0.1927439272403717, + "theoretical_loss": 3.4641180108311556, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023251755265797392, + "loss": 1.3307, + "theoretical_loss": 3.4641071385393976, + "tokens_seen": 1781006336 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023249749247743228, + "loss": 1.3078, + "theoretical_loss": 3.4640853954920705, + "tokens_seen": 1781137408 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023247743229689067, + "loss": 1.2806, + "theoretical_loss": 3.4640636544927075, + "tokens_seen": 1781268480 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023245737211634903, + "loss": 1.3513, + "theoretical_loss": 3.4640419155409665, + "tokens_seen": 1781399552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023243731193580742, + "loss": 1.252, + "theoretical_loss": 3.4640201786365017, + "tokens_seen": 1781530624 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023241725175526578, + "loss": 1.0955, + "theoretical_loss": 3.463998443778972, + "tokens_seen": 1781661696 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023239719157472417, + "loss": 1.3872, + "theoretical_loss": 3.4639767109680326, + "tokens_seen": 1781792768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023237713139418254, + "loss": 1.4077, + "theoretical_loss": 3.46395498020334, + "tokens_seen": 1781923840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023235707121364095, + "loss": 1.312, + "theoretical_loss": 3.463933251484553, + "tokens_seen": 1782054912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023233701103309932, + "loss": 1.4029, + "theoretical_loss": 3.4639115248113264, + "tokens_seen": 1782185984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023231695085255768, + "loss": 1.2912, + "theoretical_loss": 3.4638898001833183, + "tokens_seen": 1782317056 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023229689067201607, + "loss": 1.3739, + "theoretical_loss": 3.4638680776001856, + "tokens_seen": 1782448128 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.3390151560306549, + "objective/train/avg_token_score": 0.021074416115880013, + "objective/train/docs_used": 1009251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.133033275604248, + "objective/train/original_loss": 4.666356563568115, + "objective/train/theoretical_loss": 3.4638463570615854, + "objective/train/tokens_used": 1782579200, + "objective/train/unlikelihood_loss": 0.08731876313686371, + "theoretical_loss": 3.4638463570615854, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023227683049147443, + "loss": 1.223, + "theoretical_loss": 3.4638463570615854, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023225677031093282, + "loss": 1.4541, + "theoretical_loss": 3.463824638567175, + "tokens_seen": 1782710272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023223671013039118, + "loss": 1.1568, + "theoretical_loss": 3.463802922116612, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023221664994984957, + "loss": 1.3186, + "theoretical_loss": 3.463781207709553, + "tokens_seen": 1782972416 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023219658976930793, + "loss": 1.4428, + "theoretical_loss": 3.4637594953456565, + "tokens_seen": 1783103488 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023217652958876632, + "loss": 1.2444, + "theoretical_loss": 3.4637377850245796, + "tokens_seen": 1783234560 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023215646940822468, + "loss": 1.3878, + "theoretical_loss": 3.46371607674598, + "tokens_seen": 1783365632 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023213640922768307, + "loss": 1.2832, + "theoretical_loss": 3.463694370509515, + "tokens_seen": 1783496704 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023211634904714144, + "loss": 1.358, + "theoretical_loss": 3.4636726663148436, + "tokens_seen": 1783627776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002320962888665998, + "loss": 1.2709, + "theoretical_loss": 3.463650964161623, + "tokens_seen": 1783758848 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002320762286860582, + "loss": 1.1676, + "theoretical_loss": 3.4636292640495103, + "tokens_seen": 1783889920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023205616850551655, + "loss": 1.3128, + "theoretical_loss": 3.4636075659781653, + "tokens_seen": 1784020992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023203610832497494, + "loss": 1.323, + "theoretical_loss": 3.463585869947245, + "tokens_seen": 1784152064 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.2529325485229492, + "objective/train/avg_token_score": 0.04381867125630379, + "objective/train/docs_used": 1009780, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8110944032669067, + "objective/train/original_loss": 4.074629306793213, + "objective/train/theoretical_loss": 3.463575022696838, + "objective/train/tokens_used": 1784217600, + "objective/train/unlikelihood_loss": 0.2258375585079193, + "theoretical_loss": 3.463575022696838, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002320160481444333, + "loss": 1.167, + "theoretical_loss": 3.4635641759564084, + "tokens_seen": 1784283136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002319959879638917, + "loss": 1.2658, + "theoretical_loss": 3.463542484005313, + "tokens_seen": 1784414208 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023197592778335005, + "loss": 1.3566, + "theoretical_loss": 3.463520794093618, + "tokens_seen": 1784545280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023195586760280844, + "loss": 1.5095, + "theoretical_loss": 3.463499106220981, + "tokens_seen": 1784676352 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002319358074222668, + "loss": 1.4901, + "theoretical_loss": 3.4634774203870617, + "tokens_seen": 1784807424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023191574724172517, + "loss": 1.3483, + "theoretical_loss": 3.463455736591518, + "tokens_seen": 1784938496 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023189568706118356, + "loss": 1.311, + "theoretical_loss": 3.463434054834009, + "tokens_seen": 1785069568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023187562688064192, + "loss": 1.4141, + "theoretical_loss": 3.463412375114193, + "tokens_seen": 1785200640 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002318555667001003, + "loss": 1.2491, + "theoretical_loss": 3.4633906974317292, + "tokens_seen": 1785331712 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023183550651955867, + "loss": 1.1847, + "theoretical_loss": 3.463369021786277, + "tokens_seen": 1785462784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023181544633901706, + "loss": 1.2703, + "theoretical_loss": 3.4633473481774946, + "tokens_seen": 1785593856 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023179538615847542, + "loss": 1.2253, + "theoretical_loss": 3.463325676605042, + "tokens_seen": 1785724928 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.32478004693984985, + "objective/train/avg_token_score": 0.010185401886701584, + "objective/train/docs_used": 1010805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9511911273002625, + "objective/train/original_loss": 3.979628324508667, + "objective/train/theoretical_loss": 3.4633040070685777, + "objective/train/tokens_used": 1785856000, + "objective/train/unlikelihood_loss": 0.2001226395368576, + "theoretical_loss": 3.4633040070685777, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002317753259779338, + "loss": 1.2067, + "theoretical_loss": 3.4633040070685777, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023175526579739217, + "loss": 1.2934, + "theoretical_loss": 3.463282339567762, + "tokens_seen": 1785987072 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023173520561685053, + "loss": 1.3207, + "theoretical_loss": 3.463260674102253, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023171514543630892, + "loss": 1.2826, + "theoretical_loss": 3.4632390106717112, + "tokens_seen": 1786249216 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023169508525576729, + "loss": 1.0968, + "theoretical_loss": 3.463217349275796, + "tokens_seen": 1786380288 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023167502507522568, + "loss": 1.2124, + "theoretical_loss": 3.4631956899141665, + "tokens_seen": 1786511360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023165496489468404, + "loss": 1.1902, + "theoretical_loss": 3.4631740325864833, + "tokens_seen": 1786642432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023163490471414243, + "loss": 1.4263, + "theoretical_loss": 3.4631523772924053, + "tokens_seen": 1786773504 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002316148445336008, + "loss": 1.1987, + "theoretical_loss": 3.4631307240315934, + "tokens_seen": 1786904576 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002315947843530592, + "loss": 1.481, + "theoretical_loss": 3.463109072803706, + "tokens_seen": 1787035648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023157472417251757, + "loss": 1.1817, + "theoretical_loss": 3.4630874236084046, + "tokens_seen": 1787166720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023155466399197596, + "loss": 1.2639, + "theoretical_loss": 3.463065776445349, + "tokens_seen": 1787297792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023153460381143432, + "loss": 1.2597, + "theoretical_loss": 3.4630441313141986, + "tokens_seen": 1787428864 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.33742669224739075, + "objective/train/avg_token_score": 0.023008624091744423, + "objective/train/docs_used": 1011138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9465935826301575, + "objective/train/original_loss": 3.727376937866211, + "objective/train/theoretical_loss": 3.4630333095104824, + "objective/train/tokens_used": 1787494400, + "objective/train/unlikelihood_loss": 0.1610458493232727, + "theoretical_loss": 3.4630333095104824, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023151454363089268, + "loss": 1.2025, + "theoretical_loss": 3.463022488214615, + "tokens_seen": 1787559936 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023149448345035107, + "loss": 1.2983, + "theoretical_loss": 3.463000847146258, + "tokens_seen": 1787691008 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023147442326980943, + "loss": 1.3216, + "theoretical_loss": 3.462979208108788, + "tokens_seen": 1787822080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023145436308926782, + "loss": 1.2714, + "theoretical_loss": 3.4629575711018648, + "tokens_seen": 1787953152 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023143430290872619, + "loss": 1.3259, + "theoretical_loss": 3.4629359361251506, + "tokens_seen": 1788084224 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023141424272818457, + "loss": 1.2774, + "theoretical_loss": 3.462914303178305, + "tokens_seen": 1788215296 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023139418254764294, + "loss": 1.2742, + "theoretical_loss": 3.4628926722609887, + "tokens_seen": 1788346368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023137412236710133, + "loss": 1.0862, + "theoretical_loss": 3.4628710433728633, + "tokens_seen": 1788477440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002313540621865597, + "loss": 1.3324, + "theoretical_loss": 3.4628494165135892, + "tokens_seen": 1788608512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023133400200601805, + "loss": 1.3916, + "theoretical_loss": 3.4628277916828276, + "tokens_seen": 1788739584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023131394182547644, + "loss": 1.2476, + "theoretical_loss": 3.46280616888024, + "tokens_seen": 1788870656 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312938816449348, + "loss": 1.2972, + "theoretical_loss": 3.462784548105487, + "tokens_seen": 1789001728 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.3797653913497925, + "objective/train/avg_token_score": 0.011035677045583725, + "objective/train/docs_used": 1012559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2061752080917358, + "objective/train/original_loss": 3.944977045059204, + "objective/train/theoretical_loss": 3.46276292935823, + "objective/train/tokens_used": 1789132800, + "objective/train/unlikelihood_loss": 0.17685022950172424, + "theoretical_loss": 3.46276292935823, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312738214643932, + "loss": 1.3726, + "theoretical_loss": 3.46276292935823, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023125376128385155, + "loss": 1.3041, + "theoretical_loss": 3.4627413126381303, + "tokens_seen": 1789263872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023123370110330994, + "loss": 1.2739, + "theoretical_loss": 3.46271969794485, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312136409227683, + "loss": 1.1648, + "theoretical_loss": 3.4626980852780496, + "tokens_seen": 1789526016 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002311935807422267, + "loss": 1.247, + "theoretical_loss": 3.4626764746373913, + "tokens_seen": 1789657088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023117352056168506, + "loss": 1.2033, + "theoretical_loss": 3.462654866022537, + "tokens_seen": 1789788160 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023115346038114342, + "loss": 1.1793, + "theoretical_loss": 3.4626332594331473, + "tokens_seen": 1789919232 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002311334002006018, + "loss": 1.1849, + "theoretical_loss": 3.4626116548688857, + "tokens_seen": 1790050304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023111334002006017, + "loss": 1.2326, + "theoretical_loss": 3.4625900523294133, + "tokens_seen": 1790181376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023109327983951856, + "loss": 1.3255, + "theoretical_loss": 3.4625684518143918, + "tokens_seen": 1790312448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023107321965897692, + "loss": 1.2534, + "theoretical_loss": 3.4625468533234836, + "tokens_seen": 1790443520 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002310531594784353, + "loss": 1.2788, + "theoretical_loss": 3.462525256856351, + "tokens_seen": 1790574592 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023103309929789367, + "loss": 1.2669, + "theoretical_loss": 3.462503662412656, + "tokens_seen": 1790705664 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.3564271628856659, + "objective/train/avg_token_score": 0.0334165096282959, + "objective/train/docs_used": 1013169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1401686668395996, + "objective/train/original_loss": 4.225780010223389, + "objective/train/theoretical_loss": 3.4624928659494927, + "objective/train/tokens_used": 1790771200, + "objective/train/unlikelihood_loss": 0.15489980578422546, + "theoretical_loss": 3.4624928659494927, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023101303911735206, + "loss": 1.367, + "theoretical_loss": 3.4624820699920615, + "tokens_seen": 1790836736 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023099297893681043, + "loss": 1.3486, + "theoretical_loss": 3.4624604795942293, + "tokens_seen": 1790967808 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023097291875626881, + "loss": 1.156, + "theoretical_loss": 3.462438891218822, + "tokens_seen": 1791098880 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023095285857572718, + "loss": 1.291, + "theoretical_loss": 3.462417304865502, + "tokens_seen": 1791229952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023093279839518554, + "loss": 1.2874, + "theoretical_loss": 3.462395720533933, + "tokens_seen": 1791361024 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023091273821464393, + "loss": 1.2458, + "theoretical_loss": 3.462374138223776, + "tokens_seen": 1791492096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002308926780341023, + "loss": 1.3368, + "theoretical_loss": 3.4623525579346954, + "tokens_seen": 1791623168 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023087261785356068, + "loss": 1.4035, + "theoretical_loss": 3.462330979666353, + "tokens_seen": 1791754240 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023085255767301907, + "loss": 1.3682, + "theoretical_loss": 3.4623094034184128, + "tokens_seen": 1791885312 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023083249749247746, + "loss": 1.2251, + "theoretical_loss": 3.4622878291905366, + "tokens_seen": 1792016384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023081243731193582, + "loss": 1.2758, + "theoretical_loss": 3.4622662569823888, + "tokens_seen": 1792147456 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002307923771313942, + "loss": 1.3647, + "theoretical_loss": 3.462244686793632, + "tokens_seen": 1792278528 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.4223484992980957, + "objective/train/avg_token_score": 0.01925511471927166, + "objective/train/docs_used": 1014342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3984503746032715, + "objective/train/original_loss": 4.036617279052734, + "objective/train/theoretical_loss": 3.4622231186239296, + "objective/train/tokens_used": 1792409600, + "objective/train/unlikelihood_loss": 0.16442471742630005, + "theoretical_loss": 3.4622231186239296, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023077231695085257, + "loss": 1.0604, + "theoretical_loss": 3.4622231186239296, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023075225677031094, + "loss": 1.2015, + "theoretical_loss": 3.462201552472945, + "tokens_seen": 1792540672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023073219658976932, + "loss": 1.2388, + "theoretical_loss": 3.462179988340341, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002307121364092277, + "loss": 1.3161, + "theoretical_loss": 3.4621584262257823, + "tokens_seen": 1792802816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023069207622868608, + "loss": 1.3432, + "theoretical_loss": 3.4621368661289322, + "tokens_seen": 1792933888 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023067201604814444, + "loss": 1.3343, + "theoretical_loss": 3.4621153080494538, + "tokens_seen": 1793064960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023065195586760283, + "loss": 1.2585, + "theoretical_loss": 3.462093751987011, + "tokens_seen": 1793196032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002306318956870612, + "loss": 1.25, + "theoretical_loss": 3.4620721979412687, + "tokens_seen": 1793327104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023061183550651958, + "loss": 1.2695, + "theoretical_loss": 3.46205064591189, + "tokens_seen": 1793458176 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023059177532597794, + "loss": 1.2339, + "theoretical_loss": 3.4620290958985382, + "tokens_seen": 1793589248 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002305717151454363, + "loss": 1.286, + "theoretical_loss": 3.462007547900879, + "tokens_seen": 1793720320 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002305516549648947, + "loss": 1.2638, + "theoretical_loss": 3.4619860019185755, + "tokens_seen": 1793851392 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023053159478435306, + "loss": 1.2234, + "theoretical_loss": 3.461964457951292, + "tokens_seen": 1793982464 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.3893572688102722, + "objective/train/avg_token_score": 0.02078082039952278, + "objective/train/docs_used": 1015045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.188867211341858, + "objective/train/original_loss": 3.8381540775299072, + "objective/train/theoretical_loss": 3.461953686723178, + "objective/train/tokens_used": 1794048000, + "objective/train/unlikelihood_loss": 0.12606054544448853, + "theoretical_loss": 3.461953686723178, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023051153460381144, + "loss": 1.5211, + "theoretical_loss": 3.461942915998694, + "tokens_seen": 1794113536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304914744232698, + "loss": 1.3222, + "theoretical_loss": 3.4619213760604444, + "tokens_seen": 1794244608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304714142427282, + "loss": 1.2857, + "theoretical_loss": 3.4618998381362087, + "tokens_seen": 1794375680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023045135406218656, + "loss": 1.2989, + "theoretical_loss": 3.461878302225651, + "tokens_seen": 1794506752 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023043129388164495, + "loss": 1.4167, + "theoretical_loss": 3.4618567683284365, + "tokens_seen": 1794637824 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304112337011033, + "loss": 1.2559, + "theoretical_loss": 3.4618352364442293, + "tokens_seen": 1794768896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002303911735205617, + "loss": 1.3909, + "theoretical_loss": 3.4618137065726944, + "tokens_seen": 1794899968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023037111334002006, + "loss": 1.3015, + "theoretical_loss": 3.461792178713497, + "tokens_seen": 1795031040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023035105315947842, + "loss": 1.3664, + "theoretical_loss": 3.461770652866302, + "tokens_seen": 1795162112 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002303309929789368, + "loss": 1.344, + "theoretical_loss": 3.4617491290307743, + "tokens_seen": 1795293184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023031093279839518, + "loss": 1.4772, + "theoretical_loss": 3.461727607206579, + "tokens_seen": 1795424256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023029087261785356, + "loss": 1.4638, + "theoretical_loss": 3.461706087393382, + "tokens_seen": 1795555328 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.5193670392036438, + "objective/train/avg_token_score": 0.007617725990712643, + "objective/train/docs_used": 1016473, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6132582426071167, + "objective/train/original_loss": 3.4622602462768555, + "objective/train/theoretical_loss": 3.461684569590848, + "objective/train/tokens_used": 1795686400, + "objective/train/unlikelihood_loss": 0.16788122057914734, + "theoretical_loss": 3.461684569590848, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023027081243731193, + "loss": 1.3128, + "theoretical_loss": 3.461684569590848, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023025075225677032, + "loss": 1.2328, + "theoretical_loss": 3.4616630537986426, + "tokens_seen": 1795817472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023023069207622868, + "loss": 1.371, + "theoretical_loss": 3.461641540016431, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023021063189568707, + "loss": 1.335, + "theoretical_loss": 3.4616200282438783, + "tokens_seen": 1796079616 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023019057171514543, + "loss": 1.3032, + "theoretical_loss": 3.4615985184806517, + "tokens_seen": 1796210688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002301705115346038, + "loss": 1.1293, + "theoretical_loss": 3.4615770107264154, + "tokens_seen": 1796341760 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023015045135406218, + "loss": 1.2015, + "theoretical_loss": 3.4615555049808364, + "tokens_seen": 1796472832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023013039117352054, + "loss": 1.2437, + "theoretical_loss": 3.46153400124358, + "tokens_seen": 1796603904 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023011033099297893, + "loss": 1.3315, + "theoretical_loss": 3.4615124995143116, + "tokens_seen": 1796734976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023009027081243732, + "loss": 1.1804, + "theoretical_loss": 3.461490999792698, + "tokens_seen": 1796866048 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002300702106318957, + "loss": 1.2157, + "theoretical_loss": 3.4614695020784048, + "tokens_seen": 1796997120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023005015045135407, + "loss": 1.1839, + "theoretical_loss": 3.4614480063710986, + "tokens_seen": 1797128192 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023003009027081246, + "loss": 1.3656, + "theoretical_loss": 3.4614265126704455, + "tokens_seen": 1797259264 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.4662145674228668, + "objective/train/avg_token_score": 0.032763104885816574, + "objective/train/docs_used": 1017203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4318561553955078, + "objective/train/original_loss": 3.9768178462982178, + "objective/train/theoretical_loss": 3.46141576657251, + "objective/train/tokens_used": 1797324800, + "objective/train/unlikelihood_loss": 0.13404768705368042, + "theoretical_loss": 3.46141576657251, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023001003009027083, + "loss": 1.3442, + "theoretical_loss": 3.461405020976112, + "tokens_seen": 1797390336 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022998996990972922, + "loss": 1.2263, + "theoretical_loss": 3.4613835312877645, + "tokens_seen": 1797521408 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022996990972918758, + "loss": 1.1794, + "theoretical_loss": 3.461362043605069, + "tokens_seen": 1797652480 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022994984954864594, + "loss": 1.3425, + "theoretical_loss": 3.4613405579276932, + "tokens_seen": 1797783552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022992978936810433, + "loss": 1.3173, + "theoretical_loss": 3.461319074255303, + "tokens_seen": 1797914624 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002299097291875627, + "loss": 1.3875, + "theoretical_loss": 3.461297592587565, + "tokens_seen": 1798045696 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022988966900702108, + "loss": 1.3197, + "theoretical_loss": 3.4612761129241463, + "tokens_seen": 1798176768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022986960882647944, + "loss": 1.2312, + "theoretical_loss": 3.4612546352647136, + "tokens_seen": 1798307840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022984954864593783, + "loss": 1.1631, + "theoretical_loss": 3.461233159608934, + "tokens_seen": 1798438912 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002298294884653962, + "loss": 1.3315, + "theoretical_loss": 3.461211685956475, + "tokens_seen": 1798569984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022980942828485458, + "loss": 1.4705, + "theoretical_loss": 3.4611902143070035, + "tokens_seen": 1798701056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022978936810431295, + "loss": 1.3256, + "theoretical_loss": 3.4611687446601866, + "tokens_seen": 1798832128 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.32948437333106995, + "objective/train/avg_token_score": 0.0097274761646986, + "objective/train/docs_used": 1018428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0569645166397095, + "objective/train/original_loss": 4.004675388336182, + "objective/train/theoretical_loss": 3.4611472770156917, + "objective/train/tokens_used": 1798963200, + "objective/train/unlikelihood_loss": 0.18949832022190094, + "theoretical_loss": 3.4611472770156917, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002297693079237713, + "loss": 1.0449, + "theoretical_loss": 3.4611472770156917, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002297492477432297, + "loss": 1.1707, + "theoretical_loss": 3.4611258113731855, + "tokens_seen": 1799094272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022972918756268806, + "loss": 1.2616, + "theoretical_loss": 3.461104347732337, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022970912738214645, + "loss": 1.283, + "theoretical_loss": 3.4610828860928122, + "tokens_seen": 1799356416 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002296890672016048, + "loss": 1.3335, + "theoretical_loss": 3.46106142645428, + "tokens_seen": 1799487488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002296690070210632, + "loss": 1.339, + "theoretical_loss": 3.4610399688164066, + "tokens_seen": 1799618560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022964894684052156, + "loss": 1.288, + "theoretical_loss": 3.4610185131788613, + "tokens_seen": 1799749632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022962888665997995, + "loss": 1.3246, + "theoretical_loss": 3.4609970595413113, + "tokens_seen": 1799880704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022960882647943831, + "loss": 1.4125, + "theoretical_loss": 3.460975607903425, + "tokens_seen": 1800011776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022958876629889668, + "loss": 1.2623, + "theoretical_loss": 3.460954158264869, + "tokens_seen": 1800142848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022956870611835507, + "loss": 1.3185, + "theoretical_loss": 3.460932710625313, + "tokens_seen": 1800273920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022954864593781343, + "loss": 1.4288, + "theoretical_loss": 3.460911264984425, + "tokens_seen": 1800404992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022952858575727182, + "loss": 1.3195, + "theoretical_loss": 3.460889821341872, + "tokens_seen": 1800536064 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.470246821641922, + "objective/train/avg_token_score": 0.004619506187736988, + "objective/train/docs_used": 1018922, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4636540412902832, + "objective/train/original_loss": 3.622443914413452, + "objective/train/theoretical_loss": 3.4608791002698682, + "objective/train/tokens_used": 1800601600, + "objective/train/unlikelihood_loss": 0.2025619000196457, + "theoretical_loss": 3.4608791002698682, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022950852557673018, + "loss": 1.4141, + "theoretical_loss": 3.460868379697324, + "tokens_seen": 1800667136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022948846539618857, + "loss": 1.1814, + "theoretical_loss": 3.4608469400504482, + "tokens_seen": 1800798208 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022946840521564693, + "loss": 1.4121, + "theoretical_loss": 3.4608255024009136, + "tokens_seen": 1800929280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022944834503510532, + "loss": 1.1342, + "theoretical_loss": 3.4608040667483886, + "tokens_seen": 1801060352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022942828485456368, + "loss": 1.2765, + "theoretical_loss": 3.460782633092542, + "tokens_seen": 1801191424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022940822467402207, + "loss": 1.3801, + "theoretical_loss": 3.4607612014330424, + "tokens_seen": 1801322496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022938816449348043, + "loss": 1.4706, + "theoretical_loss": 3.460739771769559, + "tokens_seen": 1801453568 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002293681043129388, + "loss": 1.3683, + "theoretical_loss": 3.4607183441017604, + "tokens_seen": 1801584640 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002293480441323972, + "loss": 1.1901, + "theoretical_loss": 3.4606969184293153, + "tokens_seen": 1801715712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022932798395185558, + "loss": 1.3315, + "theoretical_loss": 3.4606754947518934, + "tokens_seen": 1801846784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022930792377131397, + "loss": 1.2587, + "theoretical_loss": 3.460654073069163, + "tokens_seen": 1801977856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022928786359077233, + "loss": 1.4254, + "theoretical_loss": 3.4606326533807943, + "tokens_seen": 1802108928 + }, + { + "debugging/length": 461.2, + "debugging/num_segments": 5, + "epoch": 0.55, + "objective/train/aligned_token_count": 0.43914955854415894, + "objective/train/avg_token_score": 0.00775171210989356, + "objective/train/docs_used": 1019481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3137940168380737, + "objective/train/original_loss": 3.678178310394287, + "objective/train/theoretical_loss": 3.460611235686456, + "objective/train/tokens_used": 1802240000, + "objective/train/unlikelihood_loss": 0.21046657860279083, + "theoretical_loss": 3.460611235686456, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022926780341023072, + "loss": 1.3076, + "theoretical_loss": 3.460611235686456, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022924774322968908, + "loss": 1.3218, + "theoretical_loss": 3.460589819985817, + "tokens_seen": 1802371072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022922768304914747, + "loss": 1.2999, + "theoretical_loss": 3.460568406278548, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022920762286860583, + "loss": 1.3088, + "theoretical_loss": 3.460546994564317, + "tokens_seen": 1802633216 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002291875626880642, + "loss": 1.1827, + "theoretical_loss": 3.460525584842795, + "tokens_seen": 1802764288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022916750250752258, + "loss": 1.4264, + "theoretical_loss": 3.460504177113651, + "tokens_seen": 1802895360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022914744232698094, + "loss": 1.2362, + "theoretical_loss": 3.460482771376555, + "tokens_seen": 1803026432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022912738214643933, + "loss": 1.2535, + "theoretical_loss": 3.4604613676311766, + "tokens_seen": 1803157504 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002291073219658977, + "loss": 1.3027, + "theoretical_loss": 3.4604399658771863, + "tokens_seen": 1803288576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022908726178535609, + "loss": 1.173, + "theoretical_loss": 3.460418566114253, + "tokens_seen": 1803419648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022906720160481445, + "loss": 1.2092, + "theoretical_loss": 3.4603971683420474, + "tokens_seen": 1803550720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022904714142427284, + "loss": 1.2199, + "theoretical_loss": 3.4603757725602398, + "tokens_seen": 1803681792 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002290270812437312, + "loss": 1.1533, + "theoretical_loss": 3.4603543787685, + "tokens_seen": 1803812864 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.36265885829925537, + "objective/train/avg_token_score": 0.0411720871925354, + "objective/train/docs_used": 1020148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1860411167144775, + "objective/train/original_loss": 3.715235471725464, + "objective/train/theoretical_loss": 3.460343682618803, + "objective/train/tokens_used": 1803878400, + "objective/train/unlikelihood_loss": 0.22270280122756958, + "theoretical_loss": 3.460343682618803, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022900702106318956, + "loss": 1.3586, + "theoretical_loss": 3.460332986966499, + "tokens_seen": 1803943936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022898696088264795, + "loss": 1.3222, + "theoretical_loss": 3.4603115971539067, + "tokens_seen": 1804075008 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002289669007021063, + "loss": 1.2406, + "theoretical_loss": 3.4602902093303936, + "tokens_seen": 1804206080 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002289468405215647, + "loss": 1.186, + "theoretical_loss": 3.460268823495631, + "tokens_seen": 1804337152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022892678034102306, + "loss": 1.1165, + "theoretical_loss": 3.460247439649288, + "tokens_seen": 1804468224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022890672016048145, + "loss": 1.3377, + "theoretical_loss": 3.4602260577910364, + "tokens_seen": 1804599296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022888665997993982, + "loss": 1.1204, + "theoretical_loss": 3.4602046779205464, + "tokens_seen": 1804730368 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002288665997993982, + "loss": 1.2894, + "theoretical_loss": 3.4601833000374898, + "tokens_seen": 1804861440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022884653961885657, + "loss": 1.3139, + "theoretical_loss": 3.460161924141537, + "tokens_seen": 1804992512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022882647943831496, + "loss": 1.1867, + "theoretical_loss": 3.4601405502323583, + "tokens_seen": 1805123584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022880641925777332, + "loss": 1.0885, + "theoretical_loss": 3.460119178309626, + "tokens_seen": 1805254656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022878635907723168, + "loss": 1.4201, + "theoretical_loss": 3.4600978083730105, + "tokens_seen": 1805385728 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.5075757503509521, + "objective/train/avg_token_score": 0.004167946521192789, + "objective/train/docs_used": 1020822, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5035169124603271, + "objective/train/original_loss": 3.252401828765869, + "objective/train/theoretical_loss": 3.4600764404221835, + "objective/train/tokens_used": 1805516800, + "objective/train/unlikelihood_loss": 0.20417453348636627, + "theoretical_loss": 3.4600764404221835, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022876629889669007, + "loss": 1.4248, + "theoretical_loss": 3.4600764404221835, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022874623871614843, + "loss": 1.263, + "theoretical_loss": 3.460055074456816, + "tokens_seen": 1805647872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022872617853560682, + "loss": 1.3433, + "theoretical_loss": 3.4600337104765795, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022870611835506518, + "loss": 1.1567, + "theoretical_loss": 3.4600123484811456, + "tokens_seen": 1805910016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022868605817452357, + "loss": 1.3388, + "theoretical_loss": 3.459990988470186, + "tokens_seen": 1806041088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022866599799398194, + "loss": 1.3511, + "theoretical_loss": 3.459969630443372, + "tokens_seen": 1806172160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022864593781344033, + "loss": 1.2393, + "theoretical_loss": 3.459948274400375, + "tokens_seen": 1806303232 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002286258776328987, + "loss": 1.2848, + "theoretical_loss": 3.459926920340868, + "tokens_seen": 1806434304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022860581745235705, + "loss": 1.2695, + "theoretical_loss": 3.4599055682645217, + "tokens_seen": 1806565376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022858575727181547, + "loss": 1.0796, + "theoretical_loss": 3.459884218171008, + "tokens_seen": 1806696448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022856569709127383, + "loss": 1.1523, + "theoretical_loss": 3.4598628700600003, + "tokens_seen": 1806827520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022854563691073222, + "loss": 1.2071, + "theoretical_loss": 3.4598415239311695, + "tokens_seen": 1806958592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022852557673019058, + "loss": 1.2922, + "theoretical_loss": 3.4598201797841885, + "tokens_seen": 1807089664 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.4556451737880707, + "objective/train/avg_token_score": 0.003934022970497608, + "objective/train/docs_used": 1022105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3802043199539185, + "objective/train/original_loss": 3.431511640548706, + "objective/train/theoretical_loss": 3.459809508453789, + "objective/train/tokens_used": 1807155200, + "objective/train/unlikelihood_loss": 0.1847713589668274, + "theoretical_loss": 3.459809508453789, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022850551654964897, + "loss": 1.3263, + "theoretical_loss": 3.459798837618729, + "tokens_seen": 1807220736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022848545636910733, + "loss": 1.2549, + "theoretical_loss": 3.4597774974344637, + "tokens_seen": 1807351808 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022846539618856572, + "loss": 1.2777, + "theoretical_loss": 3.4597561592310644, + "tokens_seen": 1807482880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022844533600802408, + "loss": 1.3251, + "theoretical_loss": 3.459734823008205, + "tokens_seen": 1807613952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022842527582748247, + "loss": 1.3524, + "theoretical_loss": 3.4597134887655567, + "tokens_seen": 1807745024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022840521564694084, + "loss": 1.2538, + "theoretical_loss": 3.4596921565027925, + "tokens_seen": 1807876096 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283851554663992, + "loss": 1.067, + "theoretical_loss": 3.459670826219585, + "tokens_seen": 1808007168 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283650952858576, + "loss": 1.1408, + "theoretical_loss": 3.459649497915608, + "tokens_seen": 1808138240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022834503510531595, + "loss": 1.1533, + "theoretical_loss": 3.4596281715905333, + "tokens_seen": 1808269312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022832497492477434, + "loss": 1.1468, + "theoretical_loss": 3.459606847244034, + "tokens_seen": 1808400384 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283049147442327, + "loss": 1.2179, + "theoretical_loss": 3.459585524875784, + "tokens_seen": 1808531456 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002282848545636911, + "loss": 1.1304, + "theoretical_loss": 3.4595642044854555, + "tokens_seen": 1808662528 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.3107282519340515, + "objective/train/avg_token_score": 0.03576691448688507, + "objective/train/docs_used": 1022607, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8535788059234619, + "objective/train/original_loss": 3.8179516792297363, + "objective/train/theoretical_loss": 3.4595428860727218, + "objective/train/tokens_used": 1808793600, + "objective/train/unlikelihood_loss": 0.19588857889175415, + "theoretical_loss": 3.4595428860727218, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022826479438314945, + "loss": 1.2683, + "theoretical_loss": 3.4595428860727218, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022824473420260784, + "loss": 1.2232, + "theoretical_loss": 3.459521569637257, + "tokens_seen": 1808924672 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002282246740220662, + "loss": 1.1408, + "theoretical_loss": 3.4595002551787335, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022820461384152457, + "loss": 1.3368, + "theoretical_loss": 3.4594789426968253, + "tokens_seen": 1809186816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022818455366098296, + "loss": 1.2408, + "theoretical_loss": 3.4594576321912056, + "tokens_seen": 1809317888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022816449348044132, + "loss": 1.1574, + "theoretical_loss": 3.4594363236615484, + "tokens_seen": 1809448960 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002281444332998997, + "loss": 1.1694, + "theoretical_loss": 3.459415017107527, + "tokens_seen": 1809580032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022812437311935807, + "loss": 1.437, + "theoretical_loss": 3.459393712528815, + "tokens_seen": 1809711104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022810431293881646, + "loss": 1.2481, + "theoretical_loss": 3.459372409925087, + "tokens_seen": 1809842176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022808425275827482, + "loss": 1.1762, + "theoretical_loss": 3.4593511092960156, + "tokens_seen": 1809973248 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002280641925777332, + "loss": 1.3189, + "theoretical_loss": 3.459329810641276, + "tokens_seen": 1810104320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022804413239719157, + "loss": 1.2046, + "theoretical_loss": 3.4593085139605417, + "tokens_seen": 1810235392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022802407221664993, + "loss": 1.1422, + "theoretical_loss": 3.459287219253487, + "tokens_seen": 1810366464 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.4571114480495453, + "objective/train/avg_token_score": 0.01232603657990694, + "objective/train/docs_used": 1024104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.418099045753479, + "objective/train/original_loss": 3.775533676147461, + "objective/train/theoretical_loss": 3.459276572639988, + "objective/train/tokens_used": 1810432000, + "objective/train/unlikelihood_loss": 0.15346667170524597, + "theoretical_loss": 3.459276572639988, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022800401203610832, + "loss": 1.2278, + "theoretical_loss": 3.459265926519786, + "tokens_seen": 1810497536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022798395185556669, + "loss": 1.2887, + "theoretical_loss": 3.459244635759113, + "tokens_seen": 1810628608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022796389167502508, + "loss": 1.2254, + "theoretical_loss": 3.4592233469711426, + "tokens_seen": 1810759680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022794383149448344, + "loss": 1.2777, + "theoretical_loss": 3.4592020601555493, + "tokens_seen": 1810890752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022792377131394183, + "loss": 1.3153, + "theoretical_loss": 3.4591807753120065, + "tokens_seen": 1811021824 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002279037111334002, + "loss": 1.4053, + "theoretical_loss": 3.4591594924401905, + "tokens_seen": 1811152896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022788365095285858, + "loss": 1.1817, + "theoretical_loss": 3.459138211539775, + "tokens_seen": 1811283968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022786359077231694, + "loss": 1.2502, + "theoretical_loss": 3.459116932610435, + "tokens_seen": 1811415040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022784353059177536, + "loss": 1.2748, + "theoretical_loss": 3.4590956556518444, + "tokens_seen": 1811546112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022782347041123372, + "loss": 1.2597, + "theoretical_loss": 3.45907438066368, + "tokens_seen": 1811677184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022780341023069208, + "loss": 1.3113, + "theoretical_loss": 3.459053107645615, + "tokens_seen": 1811808256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022778335005015047, + "loss": 1.2572, + "theoretical_loss": 3.4590318365973256, + "tokens_seen": 1811939328 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.46566471457481384, + "objective/train/avg_token_score": 0.025856217369437218, + "objective/train/docs_used": 1024786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5257141590118408, + "objective/train/original_loss": 3.992875576019287, + "objective/train/theoretical_loss": 3.4590105675184866, + "objective/train/tokens_used": 1812070400, + "objective/train/unlikelihood_loss": 0.19636161625385284, + "theoretical_loss": 3.4590105675184866, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022776328986960883, + "loss": 1.2157, + "theoretical_loss": 3.4590105675184866, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022774322968906722, + "loss": 1.2537, + "theoretical_loss": 3.4589893004087733, + "tokens_seen": 1812201472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022772316950852558, + "loss": 1.1774, + "theoretical_loss": 3.4589680352678602, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022770310932798397, + "loss": 1.4091, + "theoretical_loss": 3.4589467720954232, + "tokens_seen": 1812463616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022768304914744234, + "loss": 1.312, + "theoretical_loss": 3.4589255108911385, + "tokens_seen": 1812594688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022766298896690073, + "loss": 1.1483, + "theoretical_loss": 3.458904251654681, + "tokens_seen": 1812725760 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002276429287863591, + "loss": 1.2732, + "theoretical_loss": 3.4588829943857258, + "tokens_seen": 1812856832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022762286860581745, + "loss": 1.2897, + "theoretical_loss": 3.4588617390839493, + "tokens_seen": 1812987904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022760280842527584, + "loss": 1.162, + "theoretical_loss": 3.458840485749027, + "tokens_seen": 1813118976 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002275827482447342, + "loss": 1.2885, + "theoretical_loss": 3.458819234380635, + "tokens_seen": 1813250048 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002275626880641926, + "loss": 1.2315, + "theoretical_loss": 3.4587979849784487, + "tokens_seen": 1813381120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022754262788365095, + "loss": 1.1743, + "theoretical_loss": 3.4587767375421445, + "tokens_seen": 1813512192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022752256770310934, + "loss": 1.3601, + "theoretical_loss": 3.458755492071398, + "tokens_seen": 1813643264 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.3797653913497925, + "objective/train/avg_token_score": 0.031106898561120033, + "objective/train/docs_used": 1026014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2469812631607056, + "objective/train/original_loss": 4.441781997680664, + "objective/train/theoretical_loss": 3.458744870073008, + "objective/train/tokens_used": 1813708800, + "objective/train/unlikelihood_loss": 0.11959458887577057, + "theoretical_loss": 3.458744870073008, + "tokens_seen": 1813708800 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002275025075225677, + "loss": 1.1637, + "theoretical_loss": 3.458734248565886, + "tokens_seen": 1813774336 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002274824473420261, + "loss": 1.2195, + "theoretical_loss": 3.458713007025284, + "tokens_seen": 1813905408 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022746238716148446, + "loss": 1.324, + "theoretical_loss": 3.458691767449269, + "tokens_seen": 1814036480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022744232698094282, + "loss": 1.2098, + "theoretical_loss": 3.4586705298375167, + "tokens_seen": 1814167552 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002274222668004012, + "loss": 1.3227, + "theoretical_loss": 3.458649294189704, + "tokens_seen": 1814298624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022740220661985957, + "loss": 1.3536, + "theoretical_loss": 3.4586280605055073, + "tokens_seen": 1814429696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022738214643931796, + "loss": 1.2858, + "theoretical_loss": 3.4586068287846032, + "tokens_seen": 1814560768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022736208625877632, + "loss": 1.3113, + "theoretical_loss": 3.4585855990266685, + "tokens_seen": 1814691840 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002273420260782347, + "loss": 1.2932, + "theoretical_loss": 3.45856437123138, + "tokens_seen": 1814822912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022732196589769307, + "loss": 1.3718, + "theoretical_loss": 3.458543145398414, + "tokens_seen": 1814953984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022730190571715146, + "loss": 1.4884, + "theoretical_loss": 3.4585219215274474, + "tokens_seen": 1815085056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022728184553660983, + "loss": 1.3705, + "theoretical_loss": 3.458500699618158, + "tokens_seen": 1815216128 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.4875977635383606, + "objective/train/avg_token_score": 0.019424427300691605, + "objective/train/docs_used": 1026828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5851179361343384, + "objective/train/original_loss": 3.638456344604492, + "objective/train/theoretical_loss": 3.4584794796702223, + "objective/train/tokens_used": 1815347200, + "objective/train/unlikelihood_loss": 0.14131706953048706, + "theoretical_loss": 3.4584794796702223, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022726178535606821, + "loss": 1.2936, + "theoretical_loss": 3.4584794796702223, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022724172517552658, + "loss": 1.2743, + "theoretical_loss": 3.4584582616833175, + "tokens_seen": 1815478272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022722166499498494, + "loss": 1.1813, + "theoretical_loss": 3.4584370456571207, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022720160481444333, + "loss": 1.383, + "theoretical_loss": 3.4584158315913096, + "tokens_seen": 1815740416 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002271815446339017, + "loss": 1.455, + "theoretical_loss": 3.458394619485561, + "tokens_seen": 1815871488 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022716148445336008, + "loss": 1.2087, + "theoretical_loss": 3.458373409339553, + "tokens_seen": 1816002560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022714142427281844, + "loss": 1.2344, + "theoretical_loss": 3.4583522011529624, + "tokens_seen": 1816133632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022712136409227683, + "loss": 1.337, + "theoretical_loss": 3.4583309949254675, + "tokens_seen": 1816264704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022710130391173522, + "loss": 1.334, + "theoretical_loss": 3.4583097906567453, + "tokens_seen": 1816395776 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002270812437311936, + "loss": 1.3598, + "theoretical_loss": 3.458288588346474, + "tokens_seen": 1816526848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022706118355065197, + "loss": 1.2805, + "theoretical_loss": 3.458267387994331, + "tokens_seen": 1816657920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022704112337011033, + "loss": 1.2367, + "theoretical_loss": 3.4582461895999947, + "tokens_seen": 1816788992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022702106318956872, + "loss": 1.3511, + "theoretical_loss": 3.458224993163143, + "tokens_seen": 1816920064 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.4635263979434967, + "objective/train/avg_token_score": 0.00874384492635727, + "objective/train/docs_used": 1028029, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4905285835266113, + "objective/train/original_loss": 3.64540696144104, + "objective/train/theoretical_loss": 3.458214395678673, + "objective/train/tokens_used": 1816985600, + "objective/train/unlikelihood_loss": 0.1476590782403946, + "theoretical_loss": 3.458214395678673, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022700100300902709, + "loss": 1.3618, + "theoretical_loss": 3.4582037986834537, + "tokens_seen": 1817051136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022698094282848548, + "loss": 1.405, + "theoretical_loss": 3.4581826061606047, + "tokens_seen": 1817182208 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022696088264794384, + "loss": 1.247, + "theoretical_loss": 3.458161415594275, + "tokens_seen": 1817313280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022694082246740223, + "loss": 1.2096, + "theoretical_loss": 3.458140226984142, + "tokens_seen": 1817444352 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002269207622868606, + "loss": 1.3903, + "theoretical_loss": 3.4581190403298847, + "tokens_seen": 1817575424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022690070210631898, + "loss": 1.221, + "theoretical_loss": 3.4580978556311814, + "tokens_seen": 1817706496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022688064192577734, + "loss": 1.2097, + "theoretical_loss": 3.45807667288771, + "tokens_seen": 1817837568 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002268605817452357, + "loss": 1.144, + "theoretical_loss": 3.45805549209915, + "tokens_seen": 1817968640 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002268405215646941, + "loss": 1.2303, + "theoretical_loss": 3.4580343132651796, + "tokens_seen": 1818099712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022682046138415245, + "loss": 1.1739, + "theoretical_loss": 3.4580131363854782, + "tokens_seen": 1818230784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022680040120361084, + "loss": 1.1705, + "theoretical_loss": 3.457991961459723, + "tokens_seen": 1818361856 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002267803410230692, + "loss": 1.4904, + "theoretical_loss": 3.4579707884875948, + "tokens_seen": 1818492928 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.42741936445236206, + "objective/train/avg_token_score": 0.017135243862867355, + "objective/train/docs_used": 1028792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4546442031860352, + "objective/train/original_loss": 4.0928144454956055, + "objective/train/theoretical_loss": 3.457949617468771, + "objective/train/tokens_used": 1818624000, + "objective/train/unlikelihood_loss": 0.14676019549369812, + "theoretical_loss": 3.457949617468771, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002267602808425276, + "loss": 1.3765, + "theoretical_loss": 3.457949617468771, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022674022066198596, + "loss": 1.4647, + "theoretical_loss": 3.4579284484029316, + "tokens_seen": 1818755072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022672016048144435, + "loss": 1.2974, + "theoretical_loss": 3.4579072812897547, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002267001003009027, + "loss": 1.3476, + "theoretical_loss": 3.457886116128921, + "tokens_seen": 1819017216 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002266800401203611, + "loss": 1.34, + "theoretical_loss": 3.457864952920109, + "tokens_seen": 1819148288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022665997993981946, + "loss": 1.4318, + "theoretical_loss": 3.4578437916629974, + "tokens_seen": 1819279360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022663991975927782, + "loss": 1.2875, + "theoretical_loss": 3.4578226323572663, + "tokens_seen": 1819410432 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002266198595787362, + "loss": 1.3137, + "theoretical_loss": 3.4578014750025954, + "tokens_seen": 1819541504 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022659979939819458, + "loss": 1.3236, + "theoretical_loss": 3.4577803195986636, + "tokens_seen": 1819672576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022657973921765296, + "loss": 1.3695, + "theoretical_loss": 3.4577591661451517, + "tokens_seen": 1819803648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022655967903711133, + "loss": 1.512, + "theoretical_loss": 3.457738014641738, + "tokens_seen": 1819934720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022653961885656972, + "loss": 1.2508, + "theoretical_loss": 3.457716865088103, + "tokens_seen": 1820065792 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022651955867602808, + "loss": 1.2475, + "theoretical_loss": 3.4576957174839267, + "tokens_seen": 1820196864 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.36424732208251953, + "objective/train/avg_token_score": 0.009563930332660675, + "objective/train/docs_used": 1029550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.304864525794983, + "objective/train/original_loss": 4.12615442276001, + "objective/train/theoretical_loss": 3.4576851444127854, + "objective/train/tokens_used": 1820262400, + "objective/train/unlikelihood_loss": 0.11520551145076752, + "theoretical_loss": 3.4576851444127854, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022649949849548647, + "loss": 1.4137, + "theoretical_loss": 3.457674571828889, + "tokens_seen": 1820327936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022647943831494483, + "loss": 1.1881, + "theoretical_loss": 3.4576534281226694, + "tokens_seen": 1820459008 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002264593781344032, + "loss": 1.3106, + "theoretical_loss": 3.457632286364949, + "tokens_seen": 1820590080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022643931795386158, + "loss": 1.3795, + "theoretical_loss": 3.457611146555407, + "tokens_seen": 1820721152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022641925777331994, + "loss": 1.2652, + "theoretical_loss": 3.4575900086937237, + "tokens_seen": 1820852224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022639919759277833, + "loss": 1.1344, + "theoretical_loss": 3.4575688727795804, + "tokens_seen": 1820983296 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002263791374122367, + "loss": 1.2681, + "theoretical_loss": 3.4575477388126563, + "tokens_seen": 1821114368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022635907723169508, + "loss": 1.2641, + "theoretical_loss": 3.4575266067926327, + "tokens_seen": 1821245440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022633901705115347, + "loss": 1.3093, + "theoretical_loss": 3.4575054767191897, + "tokens_seen": 1821376512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022631895687061186, + "loss": 1.3616, + "theoretical_loss": 3.457484348592008, + "tokens_seen": 1821507584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022629889669007023, + "loss": 1.2993, + "theoretical_loss": 3.4574632224107686, + "tokens_seen": 1821638656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022627883650952861, + "loss": 1.3919, + "theoretical_loss": 3.457442098175152, + "tokens_seen": 1821769728 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.4154447615146637, + "objective/train/avg_token_score": 0.008914188481867313, + "objective/train/docs_used": 1030259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2882378101348877, + "objective/train/original_loss": 3.623776435852051, + "objective/train/theoretical_loss": 3.4574209758848395, + "objective/train/tokens_used": 1821900800, + "objective/train/unlikelihood_loss": 0.18308565020561218, + "theoretical_loss": 3.4574209758848395, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022625877632898698, + "loss": 1.2725, + "theoretical_loss": 3.4574209758848395, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022623871614844534, + "loss": 1.3879, + "theoretical_loss": 3.4573998555395113, + "tokens_seen": 1822031872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022621865596790373, + "loss": 1.3594, + "theoretical_loss": 3.457378737138849, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002261985957873621, + "loss": 1.3114, + "theoretical_loss": 3.457357620682533, + "tokens_seen": 1822294016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022617853560682048, + "loss": 1.2216, + "theoretical_loss": 3.457336506170245, + "tokens_seen": 1822425088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022615847542627884, + "loss": 1.2172, + "theoretical_loss": 3.4573153936016663, + "tokens_seen": 1822556160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022613841524573723, + "loss": 1.2705, + "theoretical_loss": 3.4572942829764783, + "tokens_seen": 1822687232 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002261183550651956, + "loss": 1.4588, + "theoretical_loss": 3.457273174294362, + "tokens_seen": 1822818304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022609829488465398, + "loss": 1.3737, + "theoretical_loss": 3.4572520675549985, + "tokens_seen": 1822949376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022607823470411235, + "loss": 1.2013, + "theoretical_loss": 3.4572309627580706, + "tokens_seen": 1823080448 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002260581745235707, + "loss": 1.2455, + "theoretical_loss": 3.4572098599032586, + "tokens_seen": 1823211520 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002260381143430291, + "loss": 1.2146, + "theoretical_loss": 3.4571887589902444, + "tokens_seen": 1823342592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022601805416248746, + "loss": 1.2853, + "theoretical_loss": 3.457167660018711, + "tokens_seen": 1823473664 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.4176441729068756, + "objective/train/avg_token_score": 0.012667980045080185, + "objective/train/docs_used": 1030966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3059900999069214, + "objective/train/original_loss": 4.118127346038818, + "objective/train/theoretical_loss": 3.4571571112608988, + "objective/train/tokens_used": 1823539200, + "objective/train/unlikelihood_loss": 0.17281955480575562, + "theoretical_loss": 3.4571571112608988, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022599799398194585, + "loss": 1.214, + "theoretical_loss": 3.4571465629883384, + "tokens_seen": 1823604736 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002259779338014042, + "loss": 1.2453, + "theoretical_loss": 3.4571254678988095, + "tokens_seen": 1823735808 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002259578736208626, + "loss": 1.3521, + "theoretical_loss": 3.4571043747498065, + "tokens_seen": 1823866880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022593781344032096, + "loss": 1.2329, + "theoretical_loss": 3.457083283541011, + "tokens_seen": 1823997952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022591775325977935, + "loss": 1.2765, + "theoretical_loss": 3.4570621942721047, + "tokens_seen": 1824129024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022589769307923771, + "loss": 1.2233, + "theoretical_loss": 3.4570411069427713, + "tokens_seen": 1824260096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022587763289869608, + "loss": 1.4633, + "theoretical_loss": 3.457020021552691, + "tokens_seen": 1824391168 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022585757271815447, + "loss": 1.193, + "theoretical_loss": 3.456998938101548, + "tokens_seen": 1824522240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022583751253761283, + "loss": 1.385, + "theoretical_loss": 3.4569778565890243, + "tokens_seen": 1824653312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022581745235707122, + "loss": 1.3582, + "theoretical_loss": 3.4569567770148018, + "tokens_seen": 1824784384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022579739217652958, + "loss": 1.3831, + "theoretical_loss": 3.4569356993785636, + "tokens_seen": 1824915456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022577733199598797, + "loss": 1.3495, + "theoretical_loss": 3.456914623679992, + "tokens_seen": 1825046528 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.40738025307655334, + "objective/train/avg_token_score": 0.013075040653347969, + "objective/train/docs_used": 1032081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1345117092132568, + "objective/train/original_loss": 4.016002178192139, + "objective/train/theoretical_loss": 3.4568935499187696, + "objective/train/tokens_used": 1825177600, + "objective/train/unlikelihood_loss": 0.14521053433418274, + "theoretical_loss": 3.4568935499187696, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022575727181544633, + "loss": 1.2247, + "theoretical_loss": 3.4568935499187696, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022573721163490472, + "loss": 1.2799, + "theoretical_loss": 3.4568724780945796, + "tokens_seen": 1825308672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022571715145436308, + "loss": 1.2388, + "theoretical_loss": 3.456851408207105, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022569709127382147, + "loss": 1.1342, + "theoretical_loss": 3.4568303402560288, + "tokens_seen": 1825570816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022567703109327983, + "loss": 1.2493, + "theoretical_loss": 3.4568092742410332, + "tokens_seen": 1825701888 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002256569709127382, + "loss": 1.2489, + "theoretical_loss": 3.456788210161802, + "tokens_seen": 1825832960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022563691073219659, + "loss": 1.3676, + "theoretical_loss": 3.4567671480180184, + "tokens_seen": 1825964032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022561685055165495, + "loss": 1.425, + "theoretical_loss": 3.456746087809365, + "tokens_seen": 1826095104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022559679037111336, + "loss": 1.3377, + "theoretical_loss": 3.4567250295355265, + "tokens_seen": 1826226176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022557673019057173, + "loss": 1.3104, + "theoretical_loss": 3.4567039731961846, + "tokens_seen": 1826357248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022555667001003012, + "loss": 1.2933, + "theoretical_loss": 3.456682918791024, + "tokens_seen": 1826488320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022553660982948848, + "loss": 1.3629, + "theoretical_loss": 3.4566618663197275, + "tokens_seen": 1826619392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022551654964894687, + "loss": 1.2257, + "theoretical_loss": 3.4566408157819795, + "tokens_seen": 1826750464 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.2227517068386078, + "objective/train/avg_token_score": 0.011948438361287117, + "objective/train/docs_used": 1032630, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.6764900088310242, + "objective/train/original_loss": 3.7054250240325928, + "objective/train/theoretical_loss": 3.456630291238087, + "objective/train/tokens_used": 1826816000, + "objective/train/unlikelihood_loss": 0.17984451353549957, + "theoretical_loss": 3.456630291238087, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022549648946840523, + "loss": 1.0602, + "theoretical_loss": 3.4566197671774628, + "tokens_seen": 1826881536 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002254764292878636, + "loss": 1.3706, + "theoretical_loss": 3.456598720505862, + "tokens_seen": 1827012608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022545636910732198, + "loss": 1.3083, + "theoretical_loss": 3.4565776757668605, + "tokens_seen": 1827143680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022543630892678034, + "loss": 1.3343, + "theoretical_loss": 3.4565566329601425, + "tokens_seen": 1827274752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022541624874623873, + "loss": 1.3098, + "theoretical_loss": 3.4565355920853915, + "tokens_seen": 1827405824 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002253961885656971, + "loss": 1.434, + "theoretical_loss": 3.456514553142292, + "tokens_seen": 1827536896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022537612838515548, + "loss": 1.393, + "theoretical_loss": 3.4564935161305277, + "tokens_seen": 1827667968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022535606820461385, + "loss": 1.2349, + "theoretical_loss": 3.4564724810497838, + "tokens_seen": 1827799040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022533600802407224, + "loss": 1.2107, + "theoretical_loss": 3.4564514478997435, + "tokens_seen": 1827930112 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002253159478435306, + "loss": 1.1709, + "theoretical_loss": 3.4564304166800914, + "tokens_seen": 1828061184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022529588766298896, + "loss": 1.3625, + "theoretical_loss": 3.4564093873905124, + "tokens_seen": 1828192256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022527582748244735, + "loss": 1.2532, + "theoretical_loss": 3.456388360030691, + "tokens_seen": 1828323328 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.5082477927207947, + "objective/train/avg_token_score": 0.017813192680478096, + "objective/train/docs_used": 1034021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7169173955917358, + "objective/train/original_loss": 3.690929412841797, + "objective/train/theoretical_loss": 3.456367334600311, + "objective/train/tokens_used": 1828454400, + "objective/train/unlikelihood_loss": 0.19483494758605957, + "theoretical_loss": 3.456367334600311, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002252557673019057, + "loss": 1.3715, + "theoretical_loss": 3.456367334600311, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002252357071213641, + "loss": 1.3709, + "theoretical_loss": 3.456346311099058, + "tokens_seen": 1828585472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022521564694082246, + "loss": 1.3934, + "theoretical_loss": 3.456325289526616, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022519558676028085, + "loss": 1.2921, + "theoretical_loss": 3.4563042698826707, + "tokens_seen": 1828847616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022517552657973922, + "loss": 1.3196, + "theoretical_loss": 3.456283252166906, + "tokens_seen": 1828978688 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002251554663991976, + "loss": 1.3465, + "theoretical_loss": 3.4562622363790076, + "tokens_seen": 1829109760 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022513540621865597, + "loss": 1.25, + "theoretical_loss": 3.45624122251866, + "tokens_seen": 1829240832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022511534603811436, + "loss": 1.1868, + "theoretical_loss": 3.456220210585549, + "tokens_seen": 1829371904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022509528585757272, + "loss": 1.4284, + "theoretical_loss": 3.456199200579359, + "tokens_seen": 1829502976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022507522567703108, + "loss": 1.4911, + "theoretical_loss": 3.4561781924997756, + "tokens_seen": 1829634048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022505516549648947, + "loss": 1.3145, + "theoretical_loss": 3.456157186346484, + "tokens_seen": 1829765120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022503510531594783, + "loss": 1.378, + "theoretical_loss": 3.4561361821191703, + "tokens_seen": 1829896192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022501504513540622, + "loss": 1.408, + "theoretical_loss": 3.4561151798175187, + "tokens_seen": 1830027264 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.3526392877101898, + "objective/train/avg_token_score": 0.021257884800434113, + "objective/train/docs_used": 1034491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1240291595458984, + "objective/train/original_loss": 3.7955236434936523, + "objective/train/theoretical_loss": 3.4561046793887185, + "objective/train/tokens_used": 1830092800, + "objective/train/unlikelihood_loss": 0.15107472240924835, + "theoretical_loss": 3.4561046793887185, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022499498495486458, + "loss": 1.3583, + "theoretical_loss": 3.456094179441216, + "tokens_seen": 1830158336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022497492477432297, + "loss": 1.327, + "theoretical_loss": 3.456073180989947, + "tokens_seen": 1830289408 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022495486459378134, + "loss": 1.3891, + "theoretical_loss": 3.4560521844633976, + "tokens_seen": 1830420480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022493480441323973, + "loss": 1.1877, + "theoretical_loss": 3.456031189861254, + "tokens_seen": 1830551552 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002249147442326981, + "loss": 1.221, + "theoretical_loss": 3.4560101971832013, + "tokens_seen": 1830682624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022489468405215645, + "loss": 1.4332, + "theoretical_loss": 3.455989206428926, + "tokens_seen": 1830813696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022487462387161484, + "loss": 1.3888, + "theoretical_loss": 3.455968217598114, + "tokens_seen": 1830944768 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002248545636910732, + "loss": 1.299, + "theoretical_loss": 3.4559472306904517, + "tokens_seen": 1831075840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022483450351053162, + "loss": 1.344, + "theoretical_loss": 3.4559262457056246, + "tokens_seen": 1831206912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022481444332998998, + "loss": 1.3853, + "theoretical_loss": 3.455905262643319, + "tokens_seen": 1831337984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022479438314944837, + "loss": 1.3892, + "theoretical_loss": 3.4558842815032214, + "tokens_seen": 1831469056 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022477432296890673, + "loss": 1.3494, + "theoretical_loss": 3.4558633022850183, + "tokens_seen": 1831600128 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.4153836667537689, + "objective/train/avg_token_score": 0.009566809982061386, + "objective/train/docs_used": 1035908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4007099866867065, + "objective/train/original_loss": 4.031271934509277, + "objective/train/theoretical_loss": 3.455842324988396, + "objective/train/tokens_used": 1831731200, + "objective/train/unlikelihood_loss": 0.1561894714832306, + "theoretical_loss": 3.455842324988396, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022475426278836512, + "loss": 1.2868, + "theoretical_loss": 3.455842324988396, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022473420260782348, + "loss": 1.2887, + "theoretical_loss": 3.45582134961304, + "tokens_seen": 1831862272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022471414242728187, + "loss": 1.2869, + "theoretical_loss": 3.455800376158639, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022469408224674023, + "loss": 1.4943, + "theoretical_loss": 3.455779404624878, + "tokens_seen": 1832124416 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002246740220661986, + "loss": 1.4098, + "theoretical_loss": 3.4557584350114445, + "tokens_seen": 1832255488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022465396188565699, + "loss": 1.4412, + "theoretical_loss": 3.455737467318025, + "tokens_seen": 1832386560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022463390170511535, + "loss": 1.465, + "theoretical_loss": 3.4557165015443063, + "tokens_seen": 1832517632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022461384152457374, + "loss": 1.2833, + "theoretical_loss": 3.4556955376899756, + "tokens_seen": 1832648704 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245937813440321, + "loss": 1.1691, + "theoretical_loss": 3.4556745757547196, + "tokens_seen": 1832779776 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245737211634905, + "loss": 1.234, + "theoretical_loss": 3.4556536157382256, + "tokens_seen": 1832910848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022455366098294885, + "loss": 1.2011, + "theoretical_loss": 3.455632657640181, + "tokens_seen": 1833041920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022453360080240724, + "loss": 1.4453, + "theoretical_loss": 3.4556117014602727, + "tokens_seen": 1833172992 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245135406218656, + "loss": 1.2839, + "theoretical_loss": 3.4555907471981886, + "tokens_seen": 1833304064 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.431512713432312, + "objective/train/avg_token_score": 0.009787185117602348, + "objective/train/docs_used": 1036404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.303069829940796, + "objective/train/original_loss": 3.6601321697235107, + "objective/train/theoretical_loss": 3.455580270786232, + "objective/train/tokens_used": 1833369600, + "objective/train/unlikelihood_loss": 0.11018586158752441, + "theoretical_loss": 3.455580270786232, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022449348044132397, + "loss": 1.359, + "theoretical_loss": 3.455569794853615, + "tokens_seen": 1833435136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022447342026078236, + "loss": 1.2755, + "theoretical_loss": 3.4555488444262403, + "tokens_seen": 1833566208 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022445336008024072, + "loss": 1.1784, + "theoretical_loss": 3.4555278959157514, + "tokens_seen": 1833697280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002244332998996991, + "loss": 1.2825, + "theoretical_loss": 3.455506949321837, + "tokens_seen": 1833828352 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022441323971915747, + "loss": 1.357, + "theoretical_loss": 3.4554860046441833, + "tokens_seen": 1833959424 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022439317953861586, + "loss": 1.3285, + "theoretical_loss": 3.455465061882479, + "tokens_seen": 1834090496 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022437311935807422, + "loss": 1.3296, + "theoretical_loss": 3.455444121036412, + "tokens_seen": 1834221568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002243530591775326, + "loss": 1.3433, + "theoretical_loss": 3.45542318210567, + "tokens_seen": 1834352640 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022433299899699097, + "loss": 1.3363, + "theoretical_loss": 3.45540224508994, + "tokens_seen": 1834483712 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022431293881644933, + "loss": 1.2526, + "theoretical_loss": 3.455381309988912, + "tokens_seen": 1834614784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022429287863590772, + "loss": 1.1744, + "theoretical_loss": 3.4553603768022727, + "tokens_seen": 1834745856 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022427281845536609, + "loss": 1.3078, + "theoretical_loss": 3.4553394455297104, + "tokens_seen": 1834876928 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.4511852264404297, + "objective/train/avg_token_score": 0.014104153029620647, + "objective/train/docs_used": 1037817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4311113357543945, + "objective/train/original_loss": 3.5987093448638916, + "objective/train/theoretical_loss": 3.455318516170914, + "objective/train/tokens_used": 1835008000, + "objective/train/unlikelihood_loss": 0.16582712531089783, + "theoretical_loss": 3.455318516170914, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022425275827482448, + "loss": 1.4142, + "theoretical_loss": 3.455318516170914, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022423269809428284, + "loss": 1.2912, + "theoretical_loss": 3.4552975887255717, + "tokens_seen": 1835139072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022421263791374123, + "loss": 1.2265, + "theoretical_loss": 3.455276663193372, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002241925777331996, + "loss": 1.1518, + "theoretical_loss": 3.4552557395740022, + "tokens_seen": 1835401216 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022417251755265798, + "loss": 1.1803, + "theoretical_loss": 3.455234817867152, + "tokens_seen": 1835532288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022415245737211634, + "loss": 1.4413, + "theoretical_loss": 3.4552138980725102, + "tokens_seen": 1835663360 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002241323971915747, + "loss": 1.2829, + "theoretical_loss": 3.4551929801897647, + "tokens_seen": 1835794432 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002241123370110331, + "loss": 1.3375, + "theoretical_loss": 3.455172064218605, + "tokens_seen": 1835925504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022409227683049148, + "loss": 1.3867, + "theoretical_loss": 3.4551511501587195, + "tokens_seen": 1836056576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022407221664994987, + "loss": 1.3092, + "theoretical_loss": 3.4551302380097972, + "tokens_seen": 1836187648 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022405215646940823, + "loss": 1.2935, + "theoretical_loss": 3.4551093277715275, + "tokens_seen": 1836318720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022403209628886662, + "loss": 1.1554, + "theoretical_loss": 3.455088419443599, + "tokens_seen": 1836449792 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022401203610832498, + "loss": 1.2624, + "theoretical_loss": 3.455067513025701, + "tokens_seen": 1836580864 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.3460410535335541, + "objective/train/avg_token_score": 0.020406225696206093, + "objective/train/docs_used": 1038484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1461899280548096, + "objective/train/original_loss": 3.8839941024780273, + "objective/train/theoretical_loss": 3.4550570605329165, + "objective/train/tokens_used": 1836646400, + "objective/train/unlikelihood_loss": 0.15742819011211395, + "theoretical_loss": 3.4550570605329165, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022399197592778337, + "loss": 1.2964, + "theoretical_loss": 3.455046608517523, + "tokens_seen": 1836711936 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022397191574724174, + "loss": 1.3793, + "theoretical_loss": 3.4550257059187537, + "tokens_seen": 1836843008 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022395185556670013, + "loss": 1.0856, + "theoretical_loss": 3.4550048052290823, + "tokens_seen": 1836974080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002239317953861585, + "loss": 1.1857, + "theoretical_loss": 3.4549839064481995, + "tokens_seen": 1837105152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022391173520561685, + "loss": 1.3736, + "theoretical_loss": 3.4549630095757937, + "tokens_seen": 1837236224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022389167502507524, + "loss": 1.3332, + "theoretical_loss": 3.454942114611555, + "tokens_seen": 1837367296 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002238716148445336, + "loss": 1.268, + "theoretical_loss": 3.4549212215551726, + "tokens_seen": 1837498368 + }, + { + "epoch": 0.56, + "learning_rate": 0.000223851554663992, + "loss": 1.2771, + "theoretical_loss": 3.4549003304063364, + "tokens_seen": 1837629440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022383149448345035, + "loss": 1.4647, + "theoretical_loss": 3.4548794411647368, + "tokens_seen": 1837760512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022381143430290874, + "loss": 1.0881, + "theoretical_loss": 3.4548585538300625, + "tokens_seen": 1837891584 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002237913741223671, + "loss": 1.2963, + "theoretical_loss": 3.4548376684020043, + "tokens_seen": 1838022656 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002237713139418255, + "loss": 1.1672, + "theoretical_loss": 3.4548167848802525, + "tokens_seen": 1838153728 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.5342742204666138, + "objective/train/avg_token_score": 0.008332007564604282, + "objective/train/docs_used": 1039769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8049894571304321, + "objective/train/original_loss": 3.7792322635650635, + "objective/train/theoretical_loss": 3.454795903264496, + "objective/train/tokens_used": 1838284800, + "objective/train/unlikelihood_loss": 0.13258232176303864, + "theoretical_loss": 3.454795903264496, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022375125376128386, + "loss": 1.2464, + "theoretical_loss": 3.454795903264496, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022373119358074222, + "loss": 1.4055, + "theoretical_loss": 3.454775023554426, + "tokens_seen": 1838415872 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002237111334002006, + "loss": 1.3121, + "theoretical_loss": 3.4547541457497326, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022369107321965897, + "loss": 1.2143, + "theoretical_loss": 3.454733269850106, + "tokens_seen": 1838678016 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022367101303911736, + "loss": 1.2152, + "theoretical_loss": 3.454712395855236, + "tokens_seen": 1838809088 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022365095285857572, + "loss": 1.2282, + "theoretical_loss": 3.4546915237648137, + "tokens_seen": 1838940160 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002236308926780341, + "loss": 1.1974, + "theoretical_loss": 3.454670653578529, + "tokens_seen": 1839071232 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022361083249749247, + "loss": 1.1896, + "theoretical_loss": 3.4546497852960742, + "tokens_seen": 1839202304 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022359077231695086, + "loss": 1.3714, + "theoretical_loss": 3.4546289189171375, + "tokens_seen": 1839333376 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022357071213640923, + "loss": 1.3313, + "theoretical_loss": 3.4546080544414117, + "tokens_seen": 1839464448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022355065195586761, + "loss": 1.302, + "theoretical_loss": 3.454587191868587, + "tokens_seen": 1839595520 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022353059177532598, + "loss": 1.31, + "theoretical_loss": 3.454566331198353, + "tokens_seen": 1839726592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022351053159478434, + "loss": 1.4753, + "theoretical_loss": 3.4545454724304028, + "tokens_seen": 1839857664 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.24450147151947021, + "objective/train/avg_token_score": 0.019046537578105927, + "objective/train/docs_used": 1040587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8506798148155212, + "objective/train/original_loss": 5.014407157897949, + "objective/train/theoretical_loss": 3.4545350437596865, + "objective/train/tokens_used": 1839923200, + "objective/train/unlikelihood_loss": 0.10890599340200424, + "theoretical_loss": 3.4545350437596865, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022349047141424273, + "loss": 1.3361, + "theoretical_loss": 3.4545246155644254, + "tokens_seen": 1839988736 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002234704112337011, + "loss": 1.2376, + "theoretical_loss": 3.4545037606001134, + "tokens_seen": 1840119808 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022345035105315948, + "loss": 1.2592, + "theoretical_loss": 3.4544829075371575, + "tokens_seen": 1840250880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022343029087261784, + "loss": 1.3449, + "theoretical_loss": 3.454462056375249, + "tokens_seen": 1840381952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022341023069207623, + "loss": 1.311, + "theoretical_loss": 3.454441207114079, + "tokens_seen": 1840513024 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002233901705115346, + "loss": 1.3277, + "theoretical_loss": 3.4544203597533385, + "tokens_seen": 1840644096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022337011033099298, + "loss": 1.3354, + "theoretical_loss": 3.45439951429272, + "tokens_seen": 1840775168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022335005015045135, + "loss": 1.3881, + "theoretical_loss": 3.454378670731914, + "tokens_seen": 1840906240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022332998996990973, + "loss": 1.2693, + "theoretical_loss": 3.454357829070613, + "tokens_seen": 1841037312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022330992978936812, + "loss": 1.3276, + "theoretical_loss": 3.4543369893085085, + "tokens_seen": 1841168384 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022328986960882649, + "loss": 1.3259, + "theoretical_loss": 3.4543161514452914, + "tokens_seen": 1841299456 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022326980942828488, + "loss": 1.4062, + "theoretical_loss": 3.4542953154806546, + "tokens_seen": 1841430528 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.47385141253471375, + "objective/train/avg_token_score": 0.007492596283555031, + "objective/train/docs_used": 1041999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4385663270950317, + "objective/train/original_loss": 3.6961052417755127, + "objective/train/theoretical_loss": 3.4542744814142896, + "objective/train/tokens_used": 1841561600, + "objective/train/unlikelihood_loss": 0.1431102305650711, + "theoretical_loss": 3.4542744814142896, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022324974924774324, + "loss": 1.3661, + "theoretical_loss": 3.4542744814142896, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022322968906720163, + "loss": 1.3559, + "theoretical_loss": 3.454253649245888, + "tokens_seen": 1841692672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022320962888666, + "loss": 1.3326, + "theoretical_loss": 3.4542328189751426, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022318956870611838, + "loss": 1.3296, + "theoretical_loss": 3.4542119906017446, + "tokens_seen": 1841954816 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022316950852557674, + "loss": 1.3703, + "theoretical_loss": 3.454191164125387, + "tokens_seen": 1842085888 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002231494483450351, + "loss": 1.3297, + "theoretical_loss": 3.4541703395457617, + "tokens_seen": 1842216960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002231293881644935, + "loss": 1.2706, + "theoretical_loss": 3.454149516862561, + "tokens_seen": 1842348032 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022310932798395185, + "loss": 1.3528, + "theoretical_loss": 3.454128696075477, + "tokens_seen": 1842479104 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022308926780341024, + "loss": 1.2731, + "theoretical_loss": 3.454107877184203, + "tokens_seen": 1842610176 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002230692076228686, + "loss": 1.3434, + "theoretical_loss": 3.4540870601884306, + "tokens_seen": 1842741248 + }, + { + "epoch": 0.56, + "learning_rate": 0.000223049147442327, + "loss": 1.2112, + "theoretical_loss": 3.454066245087853, + "tokens_seen": 1842872320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022302908726178536, + "loss": 1.3644, + "theoretical_loss": 3.4540454318821627, + "tokens_seen": 1843003392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022300902708124375, + "loss": 1.3857, + "theoretical_loss": 3.4540246205710528, + "tokens_seen": 1843134464 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.42570871114730835, + "objective/train/avg_token_score": 0.02565133012831211, + "objective/train/docs_used": 1042423, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2848546504974365, + "objective/train/original_loss": 3.6387691497802734, + "objective/train/theoretical_loss": 3.454014215625869, + "objective/train/tokens_used": 1843200000, + "objective/train/unlikelihood_loss": 0.14040718972682953, + "theoretical_loss": 3.454014215625869, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002229889669007021, + "loss": 1.4106, + "theoretical_loss": 3.4540038111542155, + "tokens_seen": 1843265536 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002229689067201605, + "loss": 1.2846, + "theoretical_loss": 3.453983003631344, + "tokens_seen": 1843396608 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022294884653961886, + "loss": 1.3201, + "theoretical_loss": 3.453962198002132, + "tokens_seen": 1843527680 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022292878635907722, + "loss": 1.3871, + "theoretical_loss": 3.4539413942662707, + "tokens_seen": 1843658752 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002229087261785356, + "loss": 1.3098, + "theoretical_loss": 3.4539205924234553, + "tokens_seen": 1843789824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022288866599799397, + "loss": 1.3391, + "theoretical_loss": 3.4538997924733774, + "tokens_seen": 1843920896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022286860581745236, + "loss": 1.337, + "theoretical_loss": 3.4538789944157315, + "tokens_seen": 1844051968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022284854563691073, + "loss": 1.3178, + "theoretical_loss": 3.45385819825021, + "tokens_seen": 1844183040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022282848545636912, + "loss": 1.402, + "theoretical_loss": 3.4538374039765065, + "tokens_seen": 1844314112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022280842527582748, + "loss": 1.4143, + "theoretical_loss": 3.4538166115943145, + "tokens_seen": 1844445184 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022278836509528587, + "loss": 1.3245, + "theoretical_loss": 3.453795821103328, + "tokens_seen": 1844576256 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022276830491474423, + "loss": 1.2381, + "theoretical_loss": 3.4537750325032404, + "tokens_seen": 1844707328 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.4173997938632965, + "objective/train/avg_token_score": 0.008677545003592968, + "objective/train/docs_used": 1043833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3148137331008911, + "objective/train/original_loss": 3.747074842453003, + "objective/train/theoretical_loss": 3.4537542457937453, + "objective/train/tokens_used": 1844838400, + "objective/train/unlikelihood_loss": 0.13828729093074799, + "theoretical_loss": 3.4537542457937453, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002227482447342026, + "loss": 1.2544, + "theoretical_loss": 3.4537542457937453, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022272818455366098, + "loss": 1.3093, + "theoretical_loss": 3.4537334609745356, + "tokens_seen": 1844969472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022270812437311934, + "loss": 1.3019, + "theoretical_loss": 3.4537126780453065, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022268806419257773, + "loss": 1.2703, + "theoretical_loss": 3.4536918970057515, + "tokens_seen": 1845231616 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002226680040120361, + "loss": 1.2962, + "theoretical_loss": 3.4536711178555644, + "tokens_seen": 1845362688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022264794383149448, + "loss": 1.4246, + "theoretical_loss": 3.453650340594439, + "tokens_seen": 1845493760 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022262788365095285, + "loss": 1.3635, + "theoretical_loss": 3.4536295652220694, + "tokens_seen": 1845624832 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022260782347041124, + "loss": 1.3042, + "theoretical_loss": 3.4536087917381506, + "tokens_seen": 1845755904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022258776328986963, + "loss": 1.4393, + "theoretical_loss": 3.4535880201423765, + "tokens_seen": 1845886976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022256770310932801, + "loss": 1.2667, + "theoretical_loss": 3.453567250434441, + "tokens_seen": 1846018048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022254764292878638, + "loss": 1.3959, + "theoretical_loss": 3.453546482614038, + "tokens_seen": 1846149120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022252758274824474, + "loss": 1.3877, + "theoretical_loss": 3.453525716680863, + "tokens_seen": 1846280192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022250752256770313, + "loss": 1.3943, + "theoretical_loss": 3.453504952634611, + "tokens_seen": 1846411264 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.3945503532886505, + "objective/train/avg_token_score": 0.011826961301267147, + "objective/train/docs_used": 1044447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1861060857772827, + "objective/train/original_loss": 3.552360773086548, + "objective/train/theoretical_loss": 3.453494571318985, + "objective/train/tokens_used": 1846476800, + "objective/train/unlikelihood_loss": 0.2250012457370758, + "theoretical_loss": 3.453494571318985, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002224874623871615, + "loss": 1.4052, + "theoretical_loss": 3.453484190474976, + "tokens_seen": 1846542336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022246740220661988, + "loss": 1.1692, + "theoretical_loss": 3.4534634302016514, + "tokens_seen": 1846673408 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022244734202607824, + "loss": 1.3963, + "theoretical_loss": 3.453442671814334, + "tokens_seen": 1846804480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022242728184553663, + "loss": 1.4818, + "theoretical_loss": 3.453421915312717, + "tokens_seen": 1846935552 + }, + { + "epoch": 0.56, + "learning_rate": 0.000222407221664995, + "loss": 1.4301, + "theoretical_loss": 3.4534011606964965, + "tokens_seen": 1847066624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022238716148445338, + "loss": 1.2717, + "theoretical_loss": 3.453380407965367, + "tokens_seen": 1847197696 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022236710130391175, + "loss": 1.3595, + "theoretical_loss": 3.4533596571190235, + "tokens_seen": 1847328768 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002223470411233701, + "loss": 1.2926, + "theoretical_loss": 3.453338908157161, + "tokens_seen": 1847459840 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002223269809428285, + "loss": 1.357, + "theoretical_loss": 3.453318161079475, + "tokens_seen": 1847590912 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022230692076228686, + "loss": 1.3616, + "theoretical_loss": 3.45329741588566, + "tokens_seen": 1847721984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022228686058174525, + "loss": 1.4467, + "theoretical_loss": 3.453276672575413, + "tokens_seen": 1847853056 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002222668004012036, + "loss": 1.378, + "theoretical_loss": 3.4532559311484277, + "tokens_seen": 1847984128 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.4030425250530243, + "objective/train/avg_token_score": 0.009174083359539509, + "objective/train/docs_used": 1045763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2596713304519653, + "objective/train/original_loss": 3.5567409992218018, + "objective/train/theoretical_loss": 3.4532351916044, + "objective/train/tokens_used": 1848115200, + "objective/train/unlikelihood_loss": 0.17909519374370575, + "theoretical_loss": 3.4532351916044, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.56, + "learning_rate": 0.000222246740220662, + "loss": 1.292, + "theoretical_loss": 3.4532351916044, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022222668004012036, + "loss": 1.1276, + "theoretical_loss": 3.453214453943026, + "tokens_seen": 1848246272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022220661985957875, + "loss": 1.5019, + "theoretical_loss": 3.453193718164001, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022218655967903711, + "loss": 1.288, + "theoretical_loss": 3.45317298426702, + "tokens_seen": 1848508416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022216649949849548, + "loss": 1.2469, + "theoretical_loss": 3.4531522522517797, + "tokens_seen": 1848639488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022214643931795387, + "loss": 1.2749, + "theoretical_loss": 3.453131522117975, + "tokens_seen": 1848770560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022212637913741223, + "loss": 1.3774, + "theoretical_loss": 3.453110793865303, + "tokens_seen": 1848901632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022210631895687062, + "loss": 1.5726, + "theoretical_loss": 3.4530900674934584, + "tokens_seen": 1849032704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022208625877632898, + "loss": 1.2539, + "theoretical_loss": 3.4530693430021384, + "tokens_seen": 1849163776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022206619859578737, + "loss": 1.3172, + "theoretical_loss": 3.453048620391038, + "tokens_seen": 1849294848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022204613841524573, + "loss": 1.4477, + "theoretical_loss": 3.4530278996598547, + "tokens_seen": 1849425920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022202607823470412, + "loss": 1.1913, + "theoretical_loss": 3.4530071808082834, + "tokens_seen": 1849556992 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022200601805416248, + "loss": 1.1726, + "theoretical_loss": 3.4529864638360213, + "tokens_seen": 1849688064 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.48967498540878296, + "objective/train/avg_token_score": 0.014848665334284306, + "objective/train/docs_used": 1046397, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.517645239830017, + "objective/train/original_loss": 3.2858333587646484, + "objective/train/theoretical_loss": 3.452976106054536, + "objective/train/tokens_used": 1849753600, + "objective/train/unlikelihood_loss": 0.1940404772758484, + "theoretical_loss": 3.452976106054536, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022198595787362084, + "loss": 1.3227, + "theoretical_loss": 3.4529657487427636, + "tokens_seen": 1849819136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022196589769307923, + "loss": 1.3116, + "theoretical_loss": 3.4529450355282085, + "tokens_seen": 1849950208 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002219458375125376, + "loss": 1.2199, + "theoretical_loss": 3.4529243241920513, + "tokens_seen": 1850081280 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022192577733199599, + "loss": 1.3545, + "theoretical_loss": 3.4529036147339887, + "tokens_seen": 1850212352 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022190571715145435, + "loss": 1.3523, + "theoretical_loss": 3.4528829071537177, + "tokens_seen": 1850343424 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022188565697091274, + "loss": 1.3313, + "theoretical_loss": 3.452862201450935, + "tokens_seen": 1850474496 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002218655967903711, + "loss": 1.1954, + "theoretical_loss": 3.452841497625337, + "tokens_seen": 1850605568 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022184553660982952, + "loss": 1.354, + "theoretical_loss": 3.4528207956766215, + "tokens_seen": 1850736640 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022182547642928788, + "loss": 1.2507, + "theoretical_loss": 3.4528000956044846, + "tokens_seen": 1850867712 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022180541624874627, + "loss": 1.3284, + "theoretical_loss": 3.452779397408624, + "tokens_seen": 1850998784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022178535606820463, + "loss": 1.1802, + "theoretical_loss": 3.4527587010887357, + "tokens_seen": 1851129856 + }, + { + "epoch": 0.56, + "learning_rate": 0.000221765295887663, + "loss": 1.4049, + "theoretical_loss": 3.4527380066445175, + "tokens_seen": 1851260928 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.38856303691864014, + "objective/train/avg_token_score": 0.012979085557162762, + "objective/train/docs_used": 1047036, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2689204216003418, + "objective/train/original_loss": 4.084848403930664, + "objective/train/theoretical_loss": 3.4527173140756666, + "objective/train/tokens_used": 1851392000, + "objective/train/unlikelihood_loss": 0.1310286819934845, + "theoretical_loss": 3.4527173140756666, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022174523570712138, + "loss": 1.3915, + "theoretical_loss": 3.4527173140756666, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022172517552657974, + "loss": 1.3878, + "theoretical_loss": 3.4526966233818808, + "tokens_seen": 1851523072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022170511534603813, + "loss": 1.388, + "theoretical_loss": 3.4526759345628566, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002216850551654965, + "loss": 1.3221, + "theoretical_loss": 3.452655247618292, + "tokens_seen": 1851785216 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022166499498495488, + "loss": 1.4251, + "theoretical_loss": 3.4526345625478845, + "tokens_seen": 1851916288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022164493480441325, + "loss": 1.1943, + "theoretical_loss": 3.4526138793513317, + "tokens_seen": 1852047360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022162487462387164, + "loss": 1.2945, + "theoretical_loss": 3.452593198028331, + "tokens_seen": 1852178432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022160481444333, + "loss": 1.3539, + "theoretical_loss": 3.45257251857858, + "tokens_seen": 1852309504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022158475426278836, + "loss": 1.1944, + "theoretical_loss": 3.452551841001777, + "tokens_seen": 1852440576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022156469408224675, + "loss": 1.2232, + "theoretical_loss": 3.452531165297619, + "tokens_seen": 1852571648 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002215446339017051, + "loss": 1.4058, + "theoretical_loss": 3.452510491465805, + "tokens_seen": 1852702720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002215245737211635, + "loss": 1.2909, + "theoretical_loss": 3.4524898195060323, + "tokens_seen": 1852833792 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022150451354062186, + "loss": 1.3925, + "theoretical_loss": 3.4524691494179995, + "tokens_seen": 1852964864 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.4089687168598175, + "objective/train/avg_token_score": 0.023404888808727264, + "objective/train/docs_used": 1048172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2324931621551514, + "objective/train/original_loss": 3.7030065059661865, + "objective/train/theoretical_loss": 3.452458815075791, + "objective/train/tokens_used": 1853030400, + "objective/train/unlikelihood_loss": 0.15494054555892944, + "theoretical_loss": 3.452458815075791, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022148445336008025, + "loss": 1.4225, + "theoretical_loss": 3.452448481201404, + "tokens_seen": 1853095936 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022146439317953862, + "loss": 1.2606, + "theoretical_loss": 3.4524278148559446, + "tokens_seen": 1853227008 + }, + { + "epoch": 0.56, + "learning_rate": 0.000221444332998997, + "loss": 1.2639, + "theoretical_loss": 3.452407150381319, + "tokens_seen": 1853358080 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022142427281845537, + "loss": 1.2381, + "theoretical_loss": 3.452386487777227, + "tokens_seen": 1853489152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022140421263791376, + "loss": 1.2405, + "theoretical_loss": 3.452365827043365, + "tokens_seen": 1853620224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022138415245737212, + "loss": 1.3922, + "theoretical_loss": 3.452345168179433, + "tokens_seen": 1853751296 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022136409227683048, + "loss": 1.5331, + "theoretical_loss": 3.452324511185129, + "tokens_seen": 1853882368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022134403209628887, + "loss": 1.3153, + "theoretical_loss": 3.4523038560601513, + "tokens_seen": 1854013440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022132397191574723, + "loss": 1.197, + "theoretical_loss": 3.4522832028041988, + "tokens_seen": 1854144512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022130391173520562, + "loss": 1.2706, + "theoretical_loss": 3.452262551416971, + "tokens_seen": 1854275584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022128385155466398, + "loss": 1.455, + "theoretical_loss": 3.452241901898166, + "tokens_seen": 1854406656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022126379137412237, + "loss": 1.2737, + "theoretical_loss": 3.452221254247483, + "tokens_seen": 1854537728 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.36620235443115234, + "objective/train/avg_token_score": 0.02949846163392067, + "objective/train/docs_used": 1049323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9726887345314026, + "objective/train/original_loss": 3.692697286605835, + "objective/train/theoretical_loss": 3.4522006084646204, + "objective/train/tokens_used": 1854668800, + "objective/train/unlikelihood_loss": 0.2694639563560486, + "theoretical_loss": 3.4522006084646204, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022124373119358074, + "loss": 1.3069, + "theoretical_loss": 3.4522006084646204, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022122367101303913, + "loss": 1.3844, + "theoretical_loss": 3.4521799645492783, + "tokens_seen": 1854799872 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002212036108324975, + "loss": 1.2408, + "theoretical_loss": 3.452159322501155, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022118355065195585, + "loss": 1.3534, + "theoretical_loss": 3.45213868231995, + "tokens_seen": 1855062016 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022116349047141424, + "loss": 1.2499, + "theoretical_loss": 3.4521180440053625, + "tokens_seen": 1855193088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002211434302908726, + "loss": 1.4884, + "theoretical_loss": 3.4520974075570914, + "tokens_seen": 1855324160 + }, + { + "epoch": 0.56, + "learning_rate": 0.000221123370110331, + "loss": 1.39, + "theoretical_loss": 3.452076772974837, + "tokens_seen": 1855455232 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022110330992978935, + "loss": 1.1831, + "theoretical_loss": 3.452056140258298, + "tokens_seen": 1855586304 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022108324974924777, + "loss": 1.3285, + "theoretical_loss": 3.4520355094071746, + "tokens_seen": 1855717376 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022106318956870613, + "loss": 1.2331, + "theoretical_loss": 3.452014880421166, + "tokens_seen": 1855848448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022104312938816452, + "loss": 1.195, + "theoretical_loss": 3.4519942532999712, + "tokens_seen": 1855979520 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022102306920762288, + "loss": 1.392, + "theoretical_loss": 3.4519736280432918, + "tokens_seen": 1856110592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022100300902708125, + "loss": 1.3138, + "theoretical_loss": 3.4519530046508256, + "tokens_seen": 1856241664 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.40811339020729065, + "objective/train/avg_token_score": 0.012535694055259228, + "objective/train/docs_used": 1049929, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2220669984817505, + "objective/train/original_loss": 3.7320573329925537, + "objective/train/theoretical_loss": 3.451942693653579, + "objective/train/tokens_used": 1856307200, + "objective/train/unlikelihood_loss": 0.17686794698238373, + "theoretical_loss": 3.451942693653579, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022098294884653963, + "loss": 1.2798, + "theoretical_loss": 3.4519323831222737, + "tokens_seen": 1856372736 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220962888665998, + "loss": 1.4532, + "theoretical_loss": 3.4519117634573355, + "tokens_seen": 1856503808 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022094282848545639, + "loss": 1.3129, + "theoretical_loss": 3.4518911456557113, + "tokens_seen": 1856634880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022092276830491475, + "loss": 1.2756, + "theoretical_loss": 3.4518705297171013, + "tokens_seen": 1856765952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022090270812437314, + "loss": 1.2821, + "theoretical_loss": 3.4518499156412052, + "tokens_seen": 1856897024 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002208826479438315, + "loss": 1.4096, + "theoretical_loss": 3.451829303427724, + "tokens_seen": 1857028096 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002208625877632899, + "loss": 1.3998, + "theoretical_loss": 3.451808693076357, + "tokens_seen": 1857159168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022084252758274825, + "loss": 1.3745, + "theoretical_loss": 3.4517880845868056, + "tokens_seen": 1857290240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022082246740220664, + "loss": 1.4097, + "theoretical_loss": 3.4517674779587693, + "tokens_seen": 1857421312 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220802407221665, + "loss": 1.3555, + "theoretical_loss": 3.451746873191949, + "tokens_seen": 1857552384 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022078234704112337, + "loss": 1.2808, + "theoretical_loss": 3.451726270286046, + "tokens_seen": 1857683456 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022076228686058175, + "loss": 1.3643, + "theoretical_loss": 3.4517056692407597, + "tokens_seen": 1857814528 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.456378310918808, + "objective/train/avg_token_score": 0.01903766579926014, + "objective/train/docs_used": 1050597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5160365104675293, + "objective/train/original_loss": 3.7752833366394043, + "objective/train/theoretical_loss": 3.4516850700557917, + "objective/train/tokens_used": 1857945600, + "objective/train/unlikelihood_loss": 0.1638176143169403, + "theoretical_loss": 3.4516850700557917, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022074222668004012, + "loss": 1.3816, + "theoretical_loss": 3.4516850700557917, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002207221664994985, + "loss": 1.2401, + "theoretical_loss": 3.451664472730842, + "tokens_seen": 1858076672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022070210631895687, + "loss": 1.3052, + "theoretical_loss": 3.451643877265613, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022068204613841526, + "loss": 1.389, + "theoretical_loss": 3.4516232836598033, + "tokens_seen": 1858338816 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022066198595787362, + "loss": 1.3692, + "theoretical_loss": 3.451602691913116, + "tokens_seen": 1858469888 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220641925777332, + "loss": 1.3859, + "theoretical_loss": 3.4515821020252506, + "tokens_seen": 1858600960 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022062186559679037, + "loss": 1.2494, + "theoretical_loss": 3.4515615139959097, + "tokens_seen": 1858732032 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022060180541624873, + "loss": 1.496, + "theoretical_loss": 3.4515409278247935, + "tokens_seen": 1858863104 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022058174523570712, + "loss": 1.2592, + "theoretical_loss": 3.4515203435116035, + "tokens_seen": 1858994176 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022056168505516549, + "loss": 1.2032, + "theoretical_loss": 3.4514997610560414, + "tokens_seen": 1859125248 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022054162487462387, + "loss": 1.2862, + "theoretical_loss": 3.4514791804578078, + "tokens_seen": 1859256320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022052156469408224, + "loss": 1.2899, + "theoretical_loss": 3.451458601716605, + "tokens_seen": 1859387392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022050150451354063, + "loss": 1.2604, + "theoretical_loss": 3.451438024832134, + "tokens_seen": 1859518464 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.4801441729068756, + "objective/train/avg_token_score": 0.010358082130551338, + "objective/train/docs_used": 1051049, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5439846515655518, + "objective/train/original_loss": 3.246432304382324, + "objective/train/theoretical_loss": 3.45142773708608, + "objective/train/tokens_used": 1859584000, + "objective/train/unlikelihood_loss": 0.2205262929201126, + "theoretical_loss": 3.45142773708608, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220481444332999, + "loss": 1.4021, + "theoretical_loss": 3.451417449804097, + "tokens_seen": 1859649536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022046138415245738, + "loss": 1.3716, + "theoretical_loss": 3.451396876632195, + "tokens_seen": 1859780608 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022044132397191574, + "loss": 1.2671, + "theoretical_loss": 3.4513763053161304, + "tokens_seen": 1859911680 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002204212637913741, + "loss": 1.4254, + "theoretical_loss": 3.4513557358556044, + "tokens_seen": 1860042752 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002204012036108325, + "loss": 1.2868, + "theoretical_loss": 3.4513351682503193, + "tokens_seen": 1860173824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022038114343029085, + "loss": 1.2487, + "theoretical_loss": 3.451314602499977, + "tokens_seen": 1860304896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022038114343029085, + "loss": 1.2386, + "theoretical_loss": 3.451294038604279, + "tokens_seen": 1860435968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022036108324974924, + "loss": 1.3557, + "theoretical_loss": 3.4512734765629283, + "tokens_seen": 1860567040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022034102306920763, + "loss": 1.2904, + "theoretical_loss": 3.451252916375627, + "tokens_seen": 1860698112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022032096288866602, + "loss": 1.3636, + "theoretical_loss": 3.4512323580420765, + "tokens_seen": 1860829184 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022030090270812438, + "loss": 1.1686, + "theoretical_loss": 3.4512118015619797, + "tokens_seen": 1860960256 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022028084252758277, + "loss": 1.2929, + "theoretical_loss": 3.451191246935039, + "tokens_seen": 1861091328 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.41055718064308167, + "objective/train/avg_token_score": 0.033672790974378586, + "objective/train/docs_used": 1052031, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4056320190429688, + "objective/train/original_loss": 4.618156433105469, + "objective/train/theoretical_loss": 3.4511706941609566, + "objective/train/tokens_used": 1861222400, + "objective/train/unlikelihood_loss": 0.1225542351603508, + "theoretical_loss": 3.4511706941609566, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022026078234704114, + "loss": 1.3488, + "theoretical_loss": 3.4511706941609566, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022024072216649953, + "loss": 1.1645, + "theoretical_loss": 3.4511501432394356, + "tokens_seen": 1861353472 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002202206619859579, + "loss": 1.3867, + "theoretical_loss": 3.4511295941701774, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022020060180541625, + "loss": 1.3409, + "theoretical_loss": 3.451109046952886, + "tokens_seen": 1861615616 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022018054162487464, + "loss": 1.399, + "theoretical_loss": 3.4510885015872628, + "tokens_seen": 1861746688 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220160481444333, + "loss": 1.3195, + "theoretical_loss": 3.4510679580730117, + "tokens_seen": 1861877760 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002201404212637914, + "loss": 1.3612, + "theoretical_loss": 3.4510474164098346, + "tokens_seen": 1862008832 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022012036108324975, + "loss": 1.3285, + "theoretical_loss": 3.4510268765974357, + "tokens_seen": 1862139904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022010030090270814, + "loss": 1.1569, + "theoretical_loss": 3.451006338635517, + "tokens_seen": 1862270976 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002200802407221665, + "loss": 1.2992, + "theoretical_loss": 3.450985802523782, + "tokens_seen": 1862402048 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002200601805416249, + "loss": 1.3106, + "theoretical_loss": 3.4509652682619327, + "tokens_seen": 1862533120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022004012036108326, + "loss": 1.2705, + "theoretical_loss": 3.4509447358496743, + "tokens_seen": 1862664192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022002006018054162, + "loss": 1.257, + "theoretical_loss": 3.450924205286708, + "tokens_seen": 1862795264 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.4781891405582428, + "objective/train/avg_token_score": 0.01192160602658987, + "objective/train/docs_used": 1052757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6278313398361206, + "objective/train/original_loss": 4.146297454833984, + "objective/train/theoretical_loss": 3.4509139406986176, + "objective/train/tokens_used": 1862860800, + "objective/train/unlikelihood_loss": 0.14082418382167816, + "theoretical_loss": 3.4509139406986176, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022, + "loss": 1.344, + "theoretical_loss": 3.450903676572739, + "tokens_seen": 1862926336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021997993981945837, + "loss": 1.2792, + "theoretical_loss": 3.4508831497074697, + "tokens_seen": 1863057408 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021995987963891676, + "loss": 1.446, + "theoretical_loss": 3.450862624690603, + "tokens_seen": 1863188480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021993981945837512, + "loss": 1.345, + "theoretical_loss": 3.450842101521844, + "tokens_seen": 1863319552 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002199197592778335, + "loss": 1.1446, + "theoretical_loss": 3.4508215802008952, + "tokens_seen": 1863450624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021989969909729187, + "loss": 1.5129, + "theoretical_loss": 3.4508010607274606, + "tokens_seen": 1863581696 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021987963891675026, + "loss": 1.4177, + "theoretical_loss": 3.4507805431012435, + "tokens_seen": 1863712768 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021985957873620862, + "loss": 1.3995, + "theoretical_loss": 3.450760027321948, + "tokens_seen": 1863843840 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021983951855566701, + "loss": 1.2628, + "theoretical_loss": 3.4507395133892786, + "tokens_seen": 1863974912 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021981945837512538, + "loss": 1.2414, + "theoretical_loss": 3.4507190013029385, + "tokens_seen": 1864105984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021979939819458374, + "loss": 1.1969, + "theoretical_loss": 3.4506984910626324, + "tokens_seen": 1864237056 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021977933801404213, + "loss": 1.2066, + "theoretical_loss": 3.450677982668063, + "tokens_seen": 1864368128 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.4010874927043915, + "objective/train/avg_token_score": 0.01795024983584881, + "objective/train/docs_used": 1054143, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2174545526504517, + "objective/train/original_loss": 3.859652519226074, + "objective/train/theoretical_loss": 3.450657476118936, + "objective/train/tokens_used": 1864499200, + "objective/train/unlikelihood_loss": 0.14504680037498474, + "theoretical_loss": 3.450657476118936, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002197592778335005, + "loss": 1.2262, + "theoretical_loss": 3.450657476118936, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021973921765295888, + "loss": 1.2329, + "theoretical_loss": 3.450636971414955, + "tokens_seen": 1864630272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021971915747241724, + "loss": 1.2419, + "theoretical_loss": 3.4506164685558245, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021969909729187563, + "loss": 1.196, + "theoretical_loss": 3.450595967541248, + "tokens_seen": 1864892416 + }, + { + "epoch": 0.57, + "learning_rate": 0.000219679037111334, + "loss": 1.1605, + "theoretical_loss": 3.450575468370931, + "tokens_seen": 1865023488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021965897693079238, + "loss": 1.1331, + "theoretical_loss": 3.4505549710445775, + "tokens_seen": 1865154560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021963891675025075, + "loss": 1.4048, + "theoretical_loss": 3.4505344755618923, + "tokens_seen": 1865285632 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002196188565697091, + "loss": 1.0081, + "theoretical_loss": 3.4505139819225796, + "tokens_seen": 1865416704 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002195987963891675, + "loss": 1.2938, + "theoretical_loss": 3.4504934901263447, + "tokens_seen": 1865547776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021957873620862589, + "loss": 1.2749, + "theoretical_loss": 3.450473000172892, + "tokens_seen": 1865678848 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021955867602808428, + "loss": 1.22, + "theoretical_loss": 3.4504525120619265, + "tokens_seen": 1865809920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021953861584754264, + "loss": 1.1633, + "theoretical_loss": 3.4504320257931527, + "tokens_seen": 1865940992 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021951855566700103, + "loss": 1.4717, + "theoretical_loss": 3.450411541366276, + "tokens_seen": 1866072064 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.3226417303085327, + "objective/train/avg_token_score": 0.024696554988622665, + "objective/train/docs_used": 1054832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8233851194381714, + "objective/train/original_loss": 3.6810128688812256, + "objective/train/theoretical_loss": 3.450401299843457, + "objective/train/tokens_used": 1866137600, + "objective/train/unlikelihood_loss": 0.20938819646835327, + "theoretical_loss": 3.450401299843457, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194984954864594, + "loss": 1.2376, + "theoretical_loss": 3.450391058781001, + "tokens_seen": 1866203136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021947843530591778, + "loss": 1.3722, + "theoretical_loss": 3.4503705780370337, + "tokens_seen": 1866334208 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021945837512537614, + "loss": 1.2517, + "theoretical_loss": 3.450350099134078, + "tokens_seen": 1866465280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194383149448345, + "loss": 1.1353, + "theoretical_loss": 3.4503296220718402, + "tokens_seen": 1866596352 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194182547642929, + "loss": 1.4574, + "theoretical_loss": 3.4503091468500253, + "tokens_seen": 1866727424 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021939819458375125, + "loss": 1.5114, + "theoretical_loss": 3.450288673468339, + "tokens_seen": 1866858496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021937813440320964, + "loss": 1.3166, + "theoretical_loss": 3.4502682019264856, + "tokens_seen": 1866989568 + }, + { + "epoch": 0.57, + "learning_rate": 0.000219358074222668, + "loss": 1.3982, + "theoretical_loss": 3.450247732224172, + "tokens_seen": 1867120640 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002193380140421264, + "loss": 1.3728, + "theoretical_loss": 3.4502272643611027, + "tokens_seen": 1867251712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021931795386158476, + "loss": 1.3339, + "theoretical_loss": 3.4502067983369846, + "tokens_seen": 1867382784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021929789368104315, + "loss": 1.2609, + "theoretical_loss": 3.450186334151522, + "tokens_seen": 1867513856 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002192778335005015, + "loss": 1.2062, + "theoretical_loss": 3.4501658718044217, + "tokens_seen": 1867644928 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.41782745718955994, + "objective/train/avg_token_score": 0.01819363422691822, + "objective/train/docs_used": 1056292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2156109809875488, + "objective/train/original_loss": 3.730748414993286, + "objective/train/theoretical_loss": 3.450145411295389, + "objective/train/tokens_used": 1867776000, + "objective/train/unlikelihood_loss": 0.15366435050964355, + "theoretical_loss": 3.450145411295389, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002192577733199599, + "loss": 1.3552, + "theoretical_loss": 3.450145411295389, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021923771313941826, + "loss": 1.3428, + "theoretical_loss": 3.4501249526241295, + "tokens_seen": 1867907072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021921765295887662, + "loss": 1.287, + "theoretical_loss": 3.4501044957903506, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.57, + "learning_rate": 0.000219197592778335, + "loss": 1.3852, + "theoretical_loss": 3.4500840407937567, + "tokens_seen": 1868169216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021917753259779337, + "loss": 1.3738, + "theoretical_loss": 3.4500635876340553, + "tokens_seen": 1868300288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021915747241725176, + "loss": 1.27, + "theoretical_loss": 3.450043136310952, + "tokens_seen": 1868431360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021913741223671013, + "loss": 1.4818, + "theoretical_loss": 3.4500226868241524, + "tokens_seen": 1868562432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021911735205616852, + "loss": 1.3846, + "theoretical_loss": 3.450002239173364, + "tokens_seen": 1868693504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021909729187562688, + "loss": 1.2276, + "theoretical_loss": 3.449981793358292, + "tokens_seen": 1868824576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021907723169508527, + "loss": 1.1697, + "theoretical_loss": 3.4499613493786443, + "tokens_seen": 1868955648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021905717151454363, + "loss": 1.2879, + "theoretical_loss": 3.4499409072341263, + "tokens_seen": 1869086720 + }, + { + "epoch": 0.57, + "learning_rate": 0.000219037111334002, + "loss": 1.1692, + "theoretical_loss": 3.449920466924445, + "tokens_seen": 1869217792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021901705115346038, + "loss": 1.3154, + "theoretical_loss": 3.4499000284493073, + "tokens_seen": 1869348864 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.5884652733802795, + "objective/train/avg_token_score": 0.022282807156443596, + "objective/train/docs_used": 1056874, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5097851753234863, + "objective/train/original_loss": 3.332839250564575, + "objective/train/theoretical_loss": 3.4498898098996, + "objective/train/tokens_used": 1869414400, + "objective/train/unlikelihood_loss": 0.13246940076351166, + "theoretical_loss": 3.4498898098996, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021899699097291874, + "loss": 1.4082, + "theoretical_loss": 3.4498795918084193, + "tokens_seen": 1869479936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021897693079237713, + "loss": 1.3323, + "theoretical_loss": 3.4498591570014883, + "tokens_seen": 1869611008 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002189568706118355, + "loss": 1.2166, + "theoretical_loss": 3.4498387240282207, + "tokens_seen": 1869742080 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021893681043129388, + "loss": 1.3572, + "theoretical_loss": 3.449818292888324, + "tokens_seen": 1869873152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021891675025075225, + "loss": 1.3564, + "theoretical_loss": 3.449797863581505, + "tokens_seen": 1870004224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021889669007021064, + "loss": 1.4288, + "theoretical_loss": 3.4497774361074707, + "tokens_seen": 1870135296 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218876629889669, + "loss": 1.1252, + "theoretical_loss": 3.4497570104659285, + "tokens_seen": 1870266368 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021885656970912736, + "loss": 1.3045, + "theoretical_loss": 3.4497365866565852, + "tokens_seen": 1870397440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021883650952858578, + "loss": 1.457, + "theoretical_loss": 3.449716164679148, + "tokens_seen": 1870528512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021881644934804414, + "loss": 1.2829, + "theoretical_loss": 3.449695744533325, + "tokens_seen": 1870659584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021879638916750253, + "loss": 1.2722, + "theoretical_loss": 3.4496753262188227, + "tokens_seen": 1870790656 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002187763289869609, + "loss": 1.4594, + "theoretical_loss": 3.449654909735349, + "tokens_seen": 1870921728 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.4332844614982605, + "objective/train/avg_token_score": 0.02448432147502899, + "objective/train/docs_used": 1058498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3367446660995483, + "objective/train/original_loss": 4.202612400054932, + "objective/train/theoretical_loss": 3.4496344950826114, + "objective/train/tokens_used": 1871052800, + "objective/train/unlikelihood_loss": 0.08500269800424576, + "theoretical_loss": 3.4496344950826114, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021875626880641928, + "loss": 1.2731, + "theoretical_loss": 3.4496344950826114, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021873620862587764, + "loss": 1.206, + "theoretical_loss": 3.449614082260317, + "tokens_seen": 1871183872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021871614844533603, + "loss": 1.4062, + "theoretical_loss": 3.449593671268175, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186960882647944, + "loss": 1.1691, + "theoretical_loss": 3.4495732621058917, + "tokens_seen": 1871446016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021867602808425278, + "loss": 1.0849, + "theoretical_loss": 3.4495528547731755, + "tokens_seen": 1871577088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021865596790371115, + "loss": 1.3112, + "theoretical_loss": 3.449532449269734, + "tokens_seen": 1871708160 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186359077231695, + "loss": 1.463, + "theoretical_loss": 3.449512045595275, + "tokens_seen": 1871839232 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186158475426279, + "loss": 1.3397, + "theoretical_loss": 3.449491643749507, + "tokens_seen": 1871970304 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021859578736208626, + "loss": 1.1075, + "theoretical_loss": 3.449471243732138, + "tokens_seen": 1872101376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021857572718154465, + "loss": 1.2688, + "theoretical_loss": 3.449450845542876, + "tokens_seen": 1872232448 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218555667001003, + "loss": 1.2877, + "theoretical_loss": 3.4494304491814285, + "tokens_seen": 1872363520 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002185356068204614, + "loss": 1.2709, + "theoretical_loss": 3.4494100546475055, + "tokens_seen": 1872494592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021851554663991976, + "loss": 1.277, + "theoretical_loss": 3.4493896619408138, + "tokens_seen": 1872625664 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.5256597995758057, + "objective/train/avg_token_score": 0.00493689626455307, + "objective/train/docs_used": 1059136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3248929977416992, + "objective/train/original_loss": 2.904456853866577, + "objective/train/theoretical_loss": 3.449379466272589, + "objective/train/tokens_used": 1872691200, + "objective/train/unlikelihood_loss": 0.27488401532173157, + "theoretical_loss": 3.449379466272589, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021849548645937815, + "loss": 1.245, + "theoretical_loss": 3.4493692710610624, + "tokens_seen": 1872756736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021847542627883651, + "loss": 1.3134, + "theoretical_loss": 3.4493488820079596, + "tokens_seen": 1872887808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021845536609829488, + "loss": 1.3594, + "theoretical_loss": 3.4493284947812146, + "tokens_seen": 1873018880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021843530591775327, + "loss": 1.235, + "theoretical_loss": 3.449308109380535, + "tokens_seen": 1873149952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021841524573721163, + "loss": 1.3775, + "theoretical_loss": 3.44928772580563, + "tokens_seen": 1873281024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021839518555667002, + "loss": 1.3688, + "theoretical_loss": 3.4492673440562083, + "tokens_seen": 1873412096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021837512537612838, + "loss": 1.3092, + "theoretical_loss": 3.449246964131979, + "tokens_seen": 1873543168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021835506519558677, + "loss": 1.3452, + "theoretical_loss": 3.4492265860326508, + "tokens_seen": 1873674240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021833500501504513, + "loss": 1.4351, + "theoretical_loss": 3.449206209757932, + "tokens_seen": 1873805312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021831494483450352, + "loss": 1.435, + "theoretical_loss": 3.4491858353075324, + "tokens_seen": 1873936384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021829488465396188, + "loss": 1.435, + "theoretical_loss": 3.449165462681161, + "tokens_seen": 1874067456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021827482447342024, + "loss": 1.3492, + "theoretical_loss": 3.449145091878527, + "tokens_seen": 1874198528 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.5190004706382751, + "objective/train/avg_token_score": 0.022021424025297165, + "objective/train/docs_used": 1059915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4837833642959595, + "objective/train/original_loss": 3.4097607135772705, + "objective/train/theoretical_loss": 3.449124722899339, + "objective/train/tokens_used": 1874329600, + "objective/train/unlikelihood_loss": 0.15578673779964447, + "theoretical_loss": 3.449124722899339, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021825476429287863, + "loss": 1.5191, + "theoretical_loss": 3.449124722899339, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218234704112337, + "loss": 1.2893, + "theoretical_loss": 3.449104355743307, + "tokens_seen": 1874460672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021821464393179539, + "loss": 1.3622, + "theoretical_loss": 3.44908399041014, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021819458375125375, + "loss": 1.2947, + "theoretical_loss": 3.4490636268995476, + "tokens_seen": 1874722816 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021817452357071214, + "loss": 1.3562, + "theoretical_loss": 3.449043265211239, + "tokens_seen": 1874853888 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002181544633901705, + "loss": 1.2433, + "theoretical_loss": 3.4490229053449237, + "tokens_seen": 1874984960 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002181344032096289, + "loss": 1.3499, + "theoretical_loss": 3.449002547300312, + "tokens_seen": 1875116032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021811434302908725, + "loss": 1.1692, + "theoretical_loss": 3.4489821910771123, + "tokens_seen": 1875247104 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021809428284854564, + "loss": 1.2123, + "theoretical_loss": 3.4489618366750356, + "tokens_seen": 1875378176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021807422266800403, + "loss": 1.1669, + "theoretical_loss": 3.4489414840937913, + "tokens_seen": 1875509248 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002180541624874624, + "loss": 1.2431, + "theoretical_loss": 3.4489211333330894, + "tokens_seen": 1875640320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021803410230692078, + "loss": 1.2133, + "theoretical_loss": 3.44890078439264, + "tokens_seen": 1875771392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021801404212637914, + "loss": 1.3267, + "theoretical_loss": 3.4488804372721518, + "tokens_seen": 1875902464 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.31158357858657837, + "objective/train/avg_token_score": 0.013181167654693127, + "objective/train/docs_used": 1060624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0544226169586182, + "objective/train/original_loss": 3.7511813640594482, + "objective/train/theoretical_loss": 3.448870264394303, + "objective/train/tokens_used": 1875968000, + "objective/train/unlikelihood_loss": 0.19581592082977295, + "theoretical_loss": 3.448870264394303, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021799398194583753, + "loss": 1.4799, + "theoretical_loss": 3.448860091971336, + "tokens_seen": 1876033536 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002179739217652959, + "loss": 1.3565, + "theoretical_loss": 3.4488397484899034, + "tokens_seen": 1876164608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021795386158475428, + "loss": 1.3522, + "theoretical_loss": 3.448819406827563, + "tokens_seen": 1876295680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021793380140421265, + "loss": 1.2026, + "theoretical_loss": 3.4487990669840256, + "tokens_seen": 1876426752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021791374122367104, + "loss": 1.3062, + "theoretical_loss": 3.4487787289590015, + "tokens_seen": 1876557824 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002178936810431294, + "loss": 1.2929, + "theoretical_loss": 3.4487583927522008, + "tokens_seen": 1876688896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021787362086258776, + "loss": 1.3192, + "theoretical_loss": 3.4487380583633342, + "tokens_seen": 1876819968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021785356068204615, + "loss": 1.2781, + "theoretical_loss": 3.448717725792113, + "tokens_seen": 1876951040 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002178335005015045, + "loss": 1.2752, + "theoretical_loss": 3.4486973950382467, + "tokens_seen": 1877082112 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002178134403209629, + "loss": 1.2696, + "theoretical_loss": 3.448677066101446, + "tokens_seen": 1877213184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021779338014042126, + "loss": 1.4191, + "theoretical_loss": 3.4486567389814224, + "tokens_seen": 1877344256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021777331995987965, + "loss": 1.3513, + "theoretical_loss": 3.448636413677886, + "tokens_seen": 1877475328 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.5450268983840942, + "objective/train/avg_token_score": 0.010001312009990215, + "objective/train/docs_used": 1061320, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8128176927566528, + "objective/train/original_loss": 3.697716236114502, + "objective/train/theoretical_loss": 3.4486160901905487, + "objective/train/tokens_used": 1877606400, + "objective/train/unlikelihood_loss": 0.1870923489332199, + "theoretical_loss": 3.4486160901905487, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021775325977933802, + "loss": 1.4165, + "theoretical_loss": 3.4486160901905487, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002177331995987964, + "loss": 1.2786, + "theoretical_loss": 3.4485957685191204, + "tokens_seen": 1877737472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021771313941825477, + "loss": 1.2019, + "theoretical_loss": 3.4485754486633127, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021769307923771316, + "loss": 1.1245, + "theoretical_loss": 3.448555130622836, + "tokens_seen": 1877999616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021767301905717152, + "loss": 1.2647, + "theoretical_loss": 3.4485348143974024, + "tokens_seen": 1878130688 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021765295887662988, + "loss": 1.2365, + "theoretical_loss": 3.448514499986722, + "tokens_seen": 1878261760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021763289869608827, + "loss": 1.1855, + "theoretical_loss": 3.448494187390507, + "tokens_seen": 1878392832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021761283851554663, + "loss": 1.3676, + "theoretical_loss": 3.4484738766084684, + "tokens_seen": 1878523904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021759277833500502, + "loss": 1.3726, + "theoretical_loss": 3.4484535676403176, + "tokens_seen": 1878654976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021757271815446338, + "loss": 1.2717, + "theoretical_loss": 3.448433260485766, + "tokens_seen": 1878786048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021755265797392177, + "loss": 1.24, + "theoretical_loss": 3.4484129551445255, + "tokens_seen": 1878917120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021753259779338014, + "loss": 1.2138, + "theoretical_loss": 3.4483926516163073, + "tokens_seen": 1879048192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021751253761283852, + "loss": 1.2976, + "theoretical_loss": 3.448372349900823, + "tokens_seen": 1879179264 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.49120235443115234, + "objective/train/avg_token_score": 0.007590876426547766, + "objective/train/docs_used": 1062685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5080890655517578, + "objective/train/original_loss": 3.4243054389953613, + "objective/train/theoretical_loss": 3.448362199722766, + "objective/train/tokens_used": 1879244800, + "objective/train/unlikelihood_loss": 0.15223416686058044, + "theoretical_loss": 3.448362199722766, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174924774322969, + "loss": 1.2978, + "theoretical_loss": 3.4483520499977844, + "tokens_seen": 1879310336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021747241725175525, + "loss": 1.3775, + "theoretical_loss": 3.4483317519069043, + "tokens_seen": 1879441408 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021745235707121364, + "loss": 1.2553, + "theoretical_loss": 3.4483114556278927, + "tokens_seen": 1879572480 + }, + { + "epoch": 0.57, + "learning_rate": 0.000217432296890672, + "loss": 1.406, + "theoretical_loss": 3.448291161160463, + "tokens_seen": 1879703552 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174122367101304, + "loss": 1.2516, + "theoretical_loss": 3.4482708685043266, + "tokens_seen": 1879834624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021739217652958875, + "loss": 1.4377, + "theoretical_loss": 3.448250577659196, + "tokens_seen": 1879965696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021737211634904714, + "loss": 1.3074, + "theoretical_loss": 3.4482302886247824, + "tokens_seen": 1880096768 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002173520561685055, + "loss": 1.3702, + "theoretical_loss": 3.448210001400799, + "tokens_seen": 1880227840 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021733199598796392, + "loss": 1.308, + "theoretical_loss": 3.448189715986958, + "tokens_seen": 1880358912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021731193580742228, + "loss": 1.2764, + "theoretical_loss": 3.448169432382971, + "tokens_seen": 1880489984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021729187562688065, + "loss": 1.211, + "theoretical_loss": 3.448149150588551, + "tokens_seen": 1880621056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021727181544633903, + "loss": 1.2987, + "theoretical_loss": 3.44812887060341, + "tokens_seen": 1880752128 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.41831621527671814, + "objective/train/avg_token_score": 0.041238442063331604, + "objective/train/docs_used": 1063493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3138890266418457, + "objective/train/original_loss": 3.773663282394409, + "objective/train/theoretical_loss": 3.448108592427261, + "objective/train/tokens_used": 1880883200, + "objective/train/unlikelihood_loss": 0.1804686337709427, + "theoretical_loss": 3.448108592427261, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002172517552657974, + "loss": 1.2279, + "theoretical_loss": 3.448108592427261, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021723169508525579, + "loss": 1.4692, + "theoretical_loss": 3.4480883160598164, + "tokens_seen": 1881014272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021721163490471415, + "loss": 1.297, + "theoretical_loss": 3.4480680415007887, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021719157472417254, + "loss": 1.2134, + "theoretical_loss": 3.448047768749891, + "tokens_seen": 1881276416 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002171715145436309, + "loss": 1.2792, + "theoretical_loss": 3.4480274978068355, + "tokens_seen": 1881407488 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002171514543630893, + "loss": 1.3984, + "theoretical_loss": 3.4480072286713357, + "tokens_seen": 1881538560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021713139418254765, + "loss": 1.3648, + "theoretical_loss": 3.447986961343104, + "tokens_seen": 1881669632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021711133400200604, + "loss": 1.3803, + "theoretical_loss": 3.447966695821854, + "tokens_seen": 1881800704 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002170912738214644, + "loss": 1.2542, + "theoretical_loss": 3.4479464321072983, + "tokens_seen": 1881931776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021707121364092277, + "loss": 1.4933, + "theoretical_loss": 3.4479261701991497, + "tokens_seen": 1882062848 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021705115346038115, + "loss": 1.2019, + "theoretical_loss": 3.4479059100971217, + "tokens_seen": 1882193920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021703109327983952, + "loss": 1.2499, + "theoretical_loss": 3.447885651800928, + "tokens_seen": 1882324992 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002170110330992979, + "loss": 1.373, + "theoretical_loss": 3.4478653953102816, + "tokens_seen": 1882456064 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.3628421425819397, + "objective/train/avg_token_score": 0.0319361537694931, + "objective/train/docs_used": 1064917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0118926763534546, + "objective/train/original_loss": 3.7626707553863525, + "objective/train/theoretical_loss": 3.4478552677419487, + "objective/train/tokens_used": 1882521600, + "objective/train/unlikelihood_loss": 0.14935363829135895, + "theoretical_loss": 3.4478552677419487, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021699097291875627, + "loss": 1.388, + "theoretical_loss": 3.4478451406248953, + "tokens_seen": 1882587136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021697091273821466, + "loss": 1.3269, + "theoretical_loss": 3.447824887744483, + "tokens_seen": 1882718208 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021695085255767302, + "loss": 1.207, + "theoretical_loss": 3.447804636668759, + "tokens_seen": 1882849280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002169307923771314, + "loss": 1.3522, + "theoretical_loss": 3.4477843873974354, + "tokens_seen": 1882980352 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021691073219658977, + "loss": 1.1773, + "theoretical_loss": 3.447764139930227, + "tokens_seen": 1883111424 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021689067201604813, + "loss": 1.3015, + "theoretical_loss": 3.4477438942668472, + "tokens_seen": 1883242496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021687061183550652, + "loss": 1.3109, + "theoretical_loss": 3.447723650407009, + "tokens_seen": 1883373568 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021685055165496489, + "loss": 1.3923, + "theoretical_loss": 3.447703408350427, + "tokens_seen": 1883504640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021683049147442327, + "loss": 1.2149, + "theoretical_loss": 3.447683168096815, + "tokens_seen": 1883635712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021681043129388164, + "loss": 1.2557, + "theoretical_loss": 3.4476629296458876, + "tokens_seen": 1883766784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021679037111334003, + "loss": 1.1998, + "theoretical_loss": 3.447642692997358, + "tokens_seen": 1883897856 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002167703109327984, + "loss": 1.2713, + "theoretical_loss": 3.4476224581509403, + "tokens_seen": 1884028928 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.42448681592941284, + "objective/train/avg_token_score": 0.013448099605739117, + "objective/train/docs_used": 1065450, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3421076536178589, + "objective/train/original_loss": 3.710195302963257, + "objective/train/theoretical_loss": 3.4476022251063485, + "objective/train/tokens_used": 1884160000, + "objective/train/unlikelihood_loss": 0.17768067121505737, + "theoretical_loss": 3.4476022251063485, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021675025075225678, + "loss": 1.4943, + "theoretical_loss": 3.4476022251063485, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021673019057171514, + "loss": 1.3822, + "theoretical_loss": 3.447581993863297, + "tokens_seen": 1884291072 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002167101303911735, + "loss": 1.1646, + "theoretical_loss": 3.4475617644215006, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002166900702106319, + "loss": 1.374, + "theoretical_loss": 3.4475415367806734, + "tokens_seen": 1884553216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021667001003009025, + "loss": 1.2601, + "theoretical_loss": 3.44752131094053, + "tokens_seen": 1884684288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021664994984954864, + "loss": 1.2635, + "theoretical_loss": 3.4475010869007843, + "tokens_seen": 1884815360 + }, + { + "epoch": 0.57, + "learning_rate": 0.000216629889669007, + "loss": 1.3163, + "theoretical_loss": 3.4474808646611512, + "tokens_seen": 1884946432 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002166098294884654, + "loss": 1.2566, + "theoretical_loss": 3.447460644221345, + "tokens_seen": 1885077504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021658976930792378, + "loss": 1.3903, + "theoretical_loss": 3.447440425581081, + "tokens_seen": 1885208576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021656970912738217, + "loss": 1.3753, + "theoretical_loss": 3.4474202087400734, + "tokens_seen": 1885339648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021654964894684054, + "loss": 1.2551, + "theoretical_loss": 3.4473999936980375, + "tokens_seen": 1885470720 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021652958876629893, + "loss": 1.1922, + "theoretical_loss": 3.447379780454688, + "tokens_seen": 1885601792 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002165095285857573, + "loss": 1.2042, + "theoretical_loss": 3.4473595690097394, + "tokens_seen": 1885732864 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.35697701573371887, + "objective/train/avg_token_score": 0.011370758526027203, + "objective/train/docs_used": 1066425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0941872596740723, + "objective/train/original_loss": 3.62231183052063, + "objective/train/theoretical_loss": 3.4473494639615767, + "objective/train/tokens_used": 1885798400, + "objective/train/unlikelihood_loss": 0.196159228682518, + "theoretical_loss": 3.4473494639615767, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021648946840521565, + "loss": 1.4117, + "theoretical_loss": 3.447339359362907, + "tokens_seen": 1885863936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021646940822467404, + "loss": 1.3477, + "theoretical_loss": 3.4473191515139066, + "tokens_seen": 1885995008 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002164493480441324, + "loss": 1.2633, + "theoretical_loss": 3.4472989454624523, + "tokens_seen": 1886126080 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002164292878635908, + "loss": 1.2316, + "theoretical_loss": 3.4472787412082595, + "tokens_seen": 1886257152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021640922768304915, + "loss": 1.2762, + "theoretical_loss": 3.4472585387510435, + "tokens_seen": 1886388224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021638916750250754, + "loss": 1.1709, + "theoretical_loss": 3.4472383380905205, + "tokens_seen": 1886519296 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002163691073219659, + "loss": 1.4111, + "theoretical_loss": 3.4472181392264045, + "tokens_seen": 1886650368 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002163490471414243, + "loss": 1.1242, + "theoretical_loss": 3.447197942158412, + "tokens_seen": 1886781440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021632898696088266, + "loss": 1.1975, + "theoretical_loss": 3.447177746886258, + "tokens_seen": 1886912512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021630892678034102, + "loss": 1.3692, + "theoretical_loss": 3.4471575534096583, + "tokens_seen": 1887043584 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002162888665997994, + "loss": 1.3614, + "theoretical_loss": 3.4471373617283287, + "tokens_seen": 1887174656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021626880641925777, + "loss": 1.3816, + "theoretical_loss": 3.4471171718419846, + "tokens_seen": 1887305728 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.4172165095806122, + "objective/train/avg_token_score": 0.057213760912418365, + "objective/train/docs_used": 1067085, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3066139221191406, + "objective/train/original_loss": 4.040600299835205, + "objective/train/theoretical_loss": 3.447096983750342, + "objective/train/tokens_used": 1887436800, + "objective/train/unlikelihood_loss": 0.14897114038467407, + "theoretical_loss": 3.447096983750342, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021624874623871616, + "loss": 1.3563, + "theoretical_loss": 3.447096983750342, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021622868605817452, + "loss": 1.3182, + "theoretical_loss": 3.447076797453117, + "tokens_seen": 1887567872 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002162086258776329, + "loss": 1.3864, + "theoretical_loss": 3.4470566129500244, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021618856569709127, + "loss": 1.2843, + "theoretical_loss": 3.4470364302407814, + "tokens_seen": 1887830016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021616850551654966, + "loss": 1.2645, + "theoretical_loss": 3.4470162493251038, + "tokens_seen": 1887961088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021614844533600802, + "loss": 1.2884, + "theoretical_loss": 3.446996070202707, + "tokens_seen": 1888092160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021612838515546641, + "loss": 1.4332, + "theoretical_loss": 3.4469758928733083, + "tokens_seen": 1888223232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021610832497492478, + "loss": 1.2347, + "theoretical_loss": 3.446955717336623, + "tokens_seen": 1888354304 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021608826479438314, + "loss": 1.3263, + "theoretical_loss": 3.4469355435923683, + "tokens_seen": 1888485376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021606820461384153, + "loss": 1.2908, + "theoretical_loss": 3.4469153716402596, + "tokens_seen": 1888616448 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002160481444332999, + "loss": 1.206, + "theoretical_loss": 3.4468952014800136, + "tokens_seen": 1888747520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021602808425275828, + "loss": 1.286, + "theoretical_loss": 3.446875033111347, + "tokens_seen": 1888878592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021600802407221664, + "loss": 1.4759, + "theoretical_loss": 3.4468548665339767, + "tokens_seen": 1889009664 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.4257698059082031, + "objective/train/avg_token_score": 0.0512680783867836, + "objective/train/docs_used": 1068281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3766205310821533, + "objective/train/original_loss": 4.451517581939697, + "objective/train/theoretical_loss": 3.446844783916939, + "objective/train/tokens_used": 1889075200, + "objective/train/unlikelihood_loss": 0.129115492105484, + "theoretical_loss": 3.446844783916939, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021598796389167503, + "loss": 1.2999, + "theoretical_loss": 3.4468347017476186, + "tokens_seen": 1889140736 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002159679037111334, + "loss": 1.235, + "theoretical_loss": 3.44681453875199, + "tokens_seen": 1889271808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021594784353059178, + "loss": 1.3091, + "theoretical_loss": 3.446794377546807, + "tokens_seen": 1889402880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021592778335005014, + "loss": 1.4402, + "theoretical_loss": 3.4467742181317873, + "tokens_seen": 1889533952 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002159077231695085, + "loss": 1.1869, + "theoretical_loss": 3.446754060506647, + "tokens_seen": 1889665024 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002158876629889669, + "loss": 1.3204, + "theoretical_loss": 3.4467339046711034, + "tokens_seen": 1889796096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021586760280842526, + "loss": 1.3447, + "theoretical_loss": 3.4467137506248733, + "tokens_seen": 1889927168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021584754262788365, + "loss": 1.47, + "theoretical_loss": 3.4466935983676743, + "tokens_seen": 1890058240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021582748244734204, + "loss": 1.3383, + "theoretical_loss": 3.446673447899223, + "tokens_seen": 1890189312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021580742226680043, + "loss": 1.3599, + "theoretical_loss": 3.4466532992192365, + "tokens_seen": 1890320384 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157873620862588, + "loss": 1.2259, + "theoretical_loss": 3.4466331523274327, + "tokens_seen": 1890451456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021576730190571718, + "loss": 1.2881, + "theoretical_loss": 3.4466130072235286, + "tokens_seen": 1890582528 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.39302298426628113, + "objective/train/avg_token_score": 0.011853698641061783, + "objective/train/docs_used": 1069021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2090446949005127, + "objective/train/original_loss": 3.498725652694702, + "objective/train/theoretical_loss": 3.4465928639072416, + "objective/train/tokens_used": 1890713600, + "objective/train/unlikelihood_loss": 0.18100960552692413, + "theoretical_loss": 3.4465928639072416, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021574724172517554, + "loss": 1.2148, + "theoretical_loss": 3.4465928639072416, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157271815446339, + "loss": 1.4336, + "theoretical_loss": 3.446572722378289, + "tokens_seen": 1890844672 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157071213640923, + "loss": 1.1447, + "theoretical_loss": 3.4465525826363885, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021568706118355065, + "loss": 1.4749, + "theoretical_loss": 3.4465324446812575, + "tokens_seen": 1891106816 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021566700100300904, + "loss": 1.1925, + "theoretical_loss": 3.446512308512614, + "tokens_seen": 1891237888 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002156469408224674, + "loss": 1.2895, + "theoretical_loss": 3.4464921741301757, + "tokens_seen": 1891368960 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002156268806419258, + "loss": 1.2362, + "theoretical_loss": 3.44647204153366, + "tokens_seen": 1891500032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021560682046138416, + "loss": 1.1798, + "theoretical_loss": 3.4464519107227853, + "tokens_seen": 1891631104 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021558676028084255, + "loss": 1.2046, + "theoretical_loss": 3.446431781697269, + "tokens_seen": 1891762176 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155667001003009, + "loss": 1.2667, + "theoretical_loss": 3.4464116544568295, + "tokens_seen": 1891893248 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155466399197593, + "loss": 1.2009, + "theoretical_loss": 3.4463915290011844, + "tokens_seen": 1892024320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021552657973921766, + "loss": 1.362, + "theoretical_loss": 3.4463714053300523, + "tokens_seen": 1892155392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021550651955867602, + "loss": 1.2938, + "theoretical_loss": 3.4463512834431507, + "tokens_seen": 1892286464 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.39119014143943787, + "objective/train/avg_token_score": 0.025414373725652695, + "objective/train/docs_used": 1069644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1698155403137207, + "objective/train/original_loss": 3.824716091156006, + "objective/train/theoretical_loss": 3.446341223168698, + "objective/train/tokens_used": 1892352000, + "objective/train/unlikelihood_loss": 0.18700602650642395, + "theoretical_loss": 3.446341223168698, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002154864593781344, + "loss": 1.2523, + "theoretical_loss": 3.446331163340198, + "tokens_seen": 1892417536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021546639919759277, + "loss": 1.2992, + "theoretical_loss": 3.4463110450209133, + "tokens_seen": 1892548608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021544633901705116, + "loss": 1.1779, + "theoretical_loss": 3.4462909284850145, + "tokens_seen": 1892679680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021542627883650953, + "loss": 1.3269, + "theoretical_loss": 3.4462708137322196, + "tokens_seen": 1892810752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021540621865596792, + "loss": 1.4482, + "theoretical_loss": 3.4462507007622474, + "tokens_seen": 1892941824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021538615847542628, + "loss": 1.2963, + "theoretical_loss": 3.4462305895748164, + "tokens_seen": 1893072896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021536609829488467, + "loss": 1.3513, + "theoretical_loss": 3.446210480169645, + "tokens_seen": 1893203968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021534603811434303, + "loss": 1.3971, + "theoretical_loss": 3.4461903725464524, + "tokens_seen": 1893335040 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002153259779338014, + "loss": 1.298, + "theoretical_loss": 3.446170266704957, + "tokens_seen": 1893466112 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021530591775325978, + "loss": 1.3383, + "theoretical_loss": 3.4461501626448783, + "tokens_seen": 1893597184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021528585757271814, + "loss": 1.3508, + "theoretical_loss": 3.4461300603659337, + "tokens_seen": 1893728256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021526579739217653, + "loss": 1.4628, + "theoretical_loss": 3.4461099598678437, + "tokens_seen": 1893859328 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.35056206583976746, + "objective/train/avg_token_score": 0.03810567036271095, + "objective/train/docs_used": 1070823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.034933090209961, + "objective/train/original_loss": 4.157607078552246, + "objective/train/theoretical_loss": 3.446089861150326, + "objective/train/tokens_used": 1893990400, + "objective/train/unlikelihood_loss": 0.14728206396102905, + "theoretical_loss": 3.446089861150326, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002152457372116349, + "loss": 1.3327, + "theoretical_loss": 3.446089861150326, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021522567703109328, + "loss": 1.1947, + "theoretical_loss": 3.446069764213101, + "tokens_seen": 1894121472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021520561685055165, + "loss": 1.2576, + "theoretical_loss": 3.4460496690558866, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021518555667001004, + "loss": 1.3322, + "theoretical_loss": 3.4460295756784025, + "tokens_seen": 1894383616 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002151654964894684, + "loss": 1.315, + "theoretical_loss": 3.446009484080368, + "tokens_seen": 1894514688 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021514543630892676, + "loss": 1.3307, + "theoretical_loss": 3.4459893942615025, + "tokens_seen": 1894645760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021512537612838515, + "loss": 1.3878, + "theoretical_loss": 3.4459693062215253, + "tokens_seen": 1894776832 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002151053159478435, + "loss": 1.1794, + "theoretical_loss": 3.4459492199601556, + "tokens_seen": 1894907904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021508525576730193, + "loss": 1.2088, + "theoretical_loss": 3.445929135477113, + "tokens_seen": 1895038976 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002150651955867603, + "loss": 1.3582, + "theoretical_loss": 3.4459090527721177, + "tokens_seen": 1895170048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021504513540621868, + "loss": 1.2095, + "theoretical_loss": 3.445888971844888, + "tokens_seen": 1895301120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021502507522567704, + "loss": 1.3343, + "theoretical_loss": 3.4458688926951453, + "tokens_seen": 1895432192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021500501504513543, + "loss": 1.2698, + "theoretical_loss": 3.4458488153226083, + "tokens_seen": 1895563264 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.3342497646808624, + "objective/train/avg_token_score": 0.0575401708483696, + "objective/train/docs_used": 1071554, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0937868356704712, + "objective/train/original_loss": 4.585005760192871, + "objective/train/theoretical_loss": 3.445838777302704, + "objective/train/tokens_used": 1895628800, + "objective/train/unlikelihood_loss": 0.1173008382320404, + "theoretical_loss": 3.445838777302704, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002149849548645938, + "loss": 1.1969, + "theoretical_loss": 3.4458287397269967, + "tokens_seen": 1895694336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021496489468405218, + "loss": 1.2119, + "theoretical_loss": 3.4458086659080305, + "tokens_seen": 1895825408 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021494483450351055, + "loss": 1.2972, + "theoretical_loss": 3.4457885938654305, + "tokens_seen": 1895956480 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002149247743229689, + "loss": 1.3456, + "theoretical_loss": 3.4457685235989155, + "tokens_seen": 1896087552 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002149047141424273, + "loss": 1.1978, + "theoretical_loss": 3.4457484551082063, + "tokens_seen": 1896218624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021488465396188566, + "loss": 1.213, + "theoretical_loss": 3.4457283883930234, + "tokens_seen": 1896349696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021486459378134405, + "loss": 1.1845, + "theoretical_loss": 3.4457083234530863, + "tokens_seen": 1896480768 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002148445336008024, + "loss": 1.2955, + "theoretical_loss": 3.445688260288115, + "tokens_seen": 1896611840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002148244734202608, + "loss": 1.3845, + "theoretical_loss": 3.4456681988978306, + "tokens_seen": 1896742912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021480441323971916, + "loss": 1.2057, + "theoretical_loss": 3.4456481392819533, + "tokens_seen": 1896873984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021478435305917755, + "loss": 1.3261, + "theoretical_loss": 3.445628081440203, + "tokens_seen": 1897005056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021476429287863591, + "loss": 1.196, + "theoretical_loss": 3.4456080253723016, + "tokens_seen": 1897136128 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.43242913484573364, + "objective/train/avg_token_score": 0.024419045075774193, + "objective/train/docs_used": 1072893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3768832683563232, + "objective/train/original_loss": 3.509005308151245, + "objective/train/theoretical_loss": 3.445587971077968, + "objective/train/tokens_used": 1897267200, + "objective/train/unlikelihood_loss": 0.29701992869377136, + "theoretical_loss": 3.445587971077968, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021474423269809428, + "loss": 1.2108, + "theoretical_loss": 3.445587971077968, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021472417251755267, + "loss": 1.253, + "theoretical_loss": 3.4455679185569243, + "tokens_seen": 1897398272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021470411233701103, + "loss": 1.3857, + "theoretical_loss": 3.44554786780889, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021468405215646942, + "loss": 1.1423, + "theoretical_loss": 3.445527818833587, + "tokens_seen": 1897660416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021466399197592778, + "loss": 1.246, + "theoretical_loss": 3.445507771630735, + "tokens_seen": 1897791488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021464393179538617, + "loss": 1.2594, + "theoretical_loss": 3.4454877262000556, + "tokens_seen": 1897922560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021462387161484453, + "loss": 1.18, + "theoretical_loss": 3.44546768254127, + "tokens_seen": 1898053632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021460381143430292, + "loss": 1.2849, + "theoretical_loss": 3.4454476406540984, + "tokens_seen": 1898184704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021458375125376128, + "loss": 1.3834, + "theoretical_loss": 3.445427600538263, + "tokens_seen": 1898315776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021456369107321964, + "loss": 1.513, + "theoretical_loss": 3.445407562193484, + "tokens_seen": 1898446848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021454363089267803, + "loss": 1.271, + "theoretical_loss": 3.445387525619483, + "tokens_seen": 1898577920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002145235707121364, + "loss": 1.3697, + "theoretical_loss": 3.445367490815981, + "tokens_seen": 1898708992 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021450351053159479, + "loss": 1.2931, + "theoretical_loss": 3.4453474577826997, + "tokens_seen": 1898840064 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.2875122129917145, + "objective/train/avg_token_score": 0.06583469361066818, + "objective/train/docs_used": 1073632, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8564949631690979, + "objective/train/original_loss": 3.902287483215332, + "objective/train/theoretical_loss": 3.445337441929805, + "objective/train/tokens_used": 1898905600, + "objective/train/unlikelihood_loss": 0.17169082164764404, + "theoretical_loss": 3.445337441929805, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021448345035105315, + "loss": 1.0702, + "theoretical_loss": 3.445327426519361, + "tokens_seen": 1898971136 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021446339017051154, + "loss": 1.2605, + "theoretical_loss": 3.445307397025685, + "tokens_seen": 1899102208 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002144433299899699, + "loss": 1.1531, + "theoretical_loss": 3.4452873693013943, + "tokens_seen": 1899233280 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002144232698094283, + "loss": 1.2087, + "theoretical_loss": 3.4452673433462104, + "tokens_seen": 1899364352 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021440320962888665, + "loss": 1.3201, + "theoretical_loss": 3.4452473191598547, + "tokens_seen": 1899495424 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021438314944834504, + "loss": 1.2517, + "theoretical_loss": 3.4452272967420488, + "tokens_seen": 1899626496 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002143630892678034, + "loss": 1.2354, + "theoretical_loss": 3.445207276092515, + "tokens_seen": 1899757568 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021434302908726176, + "loss": 1.1358, + "theoretical_loss": 3.4451872572109745, + "tokens_seen": 1899888640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021432296890672018, + "loss": 1.3754, + "theoretical_loss": 3.44516724009715, + "tokens_seen": 1900019712 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021430290872617854, + "loss": 1.1777, + "theoretical_loss": 3.4451472247507624, + "tokens_seen": 1900150784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021428284854563693, + "loss": 1.1901, + "theoretical_loss": 3.4451272111715348, + "tokens_seen": 1900281856 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002142627883650953, + "loss": 1.0352, + "theoretical_loss": 3.445107199359189, + "tokens_seen": 1900412928 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.37988758087158203, + "objective/train/avg_token_score": 0.00978877767920494, + "objective/train/docs_used": 1074230, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1855452060699463, + "objective/train/original_loss": 3.511742115020752, + "objective/train/theoretical_loss": 3.4450871893134467, + "objective/train/tokens_used": 1900544000, + "objective/train/unlikelihood_loss": 0.2077323943376541, + "theoretical_loss": 3.4450871893134467, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021424272818455368, + "loss": 1.2456, + "theoretical_loss": 3.4450871893134467, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021422266800401205, + "loss": 1.3763, + "theoretical_loss": 3.4450671810340308, + "tokens_seen": 1900675072 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021420260782347044, + "loss": 1.1765, + "theoretical_loss": 3.445047174520663, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141825476429288, + "loss": 1.2169, + "theoretical_loss": 3.4450271697730663, + "tokens_seen": 1900937216 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021416248746238716, + "loss": 1.1967, + "theoretical_loss": 3.4450071667909623, + "tokens_seen": 1901068288 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021414242728184555, + "loss": 1.3367, + "theoretical_loss": 3.4449871655740743, + "tokens_seen": 1901199360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141223671013039, + "loss": 1.1884, + "theoretical_loss": 3.444967166122124, + "tokens_seen": 1901330432 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141023069207623, + "loss": 1.4466, + "theoretical_loss": 3.444947168434835, + "tokens_seen": 1901461504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021408224674022066, + "loss": 1.1993, + "theoretical_loss": 3.444927172511929, + "tokens_seen": 1901592576 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021406218655967905, + "loss": 1.1398, + "theoretical_loss": 3.4449071783531293, + "tokens_seen": 1901723648 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021404212637913742, + "loss": 1.4719, + "theoretical_loss": 3.4448871859581587, + "tokens_seen": 1901854720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002140220661985958, + "loss": 1.2844, + "theoretical_loss": 3.44486719532674, + "tokens_seen": 1901985792 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021400200601805417, + "loss": 1.1126, + "theoretical_loss": 3.4448472064585958, + "tokens_seen": 1902116864 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.44984114170074463, + "objective/train/avg_token_score": 0.006575016770511866, + "objective/train/docs_used": 1075669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3423633575439453, + "objective/train/original_loss": 3.341994047164917, + "objective/train/theoretical_loss": 3.4448372126856652, + "objective/train/tokens_used": 1902182400, + "objective/train/unlikelihood_loss": 0.20209302008152008, + "theoretical_loss": 3.4448372126856652, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021398194583751256, + "loss": 1.1684, + "theoretical_loss": 3.4448272193534493, + "tokens_seen": 1902247936 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021396188565697092, + "loss": 1.398, + "theoretical_loss": 3.444807234011024, + "tokens_seen": 1902379008 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021394182547642928, + "loss": 1.1932, + "theoretical_loss": 3.444787250431042, + "tokens_seen": 1902510080 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021392176529588767, + "loss": 1.3784, + "theoretical_loss": 3.4447672686132274, + "tokens_seen": 1902641152 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021390170511534603, + "loss": 1.2756, + "theoretical_loss": 3.444747288557303, + "tokens_seen": 1902772224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021388164493480442, + "loss": 1.2478, + "theoretical_loss": 3.444727310262992, + "tokens_seen": 1902903296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021386158475426278, + "loss": 1.3091, + "theoretical_loss": 3.444707333730018, + "tokens_seen": 1903034368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021384152457372117, + "loss": 1.1853, + "theoretical_loss": 3.444687358958104, + "tokens_seen": 1903165440 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021382146439317954, + "loss": 1.2117, + "theoretical_loss": 3.4446673859469747, + "tokens_seen": 1903296512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021380140421263792, + "loss": 1.334, + "theoretical_loss": 3.444647414696352, + "tokens_seen": 1903427584 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137813440320963, + "loss": 1.1421, + "theoretical_loss": 3.4446274452059606, + "tokens_seen": 1903558656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021376128385155465, + "loss": 1.324, + "theoretical_loss": 3.444607477475524, + "tokens_seen": 1903689728 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.363025426864624, + "objective/train/avg_token_score": 0.023712707683444023, + "objective/train/docs_used": 1076340, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0491011142730713, + "objective/train/original_loss": 3.9789304733276367, + "objective/train/theoretical_loss": 3.4445875115047655, + "objective/train/tokens_used": 1903820800, + "objective/train/unlikelihood_loss": 0.1412980854511261, + "theoretical_loss": 3.4445875115047655, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021374122367101304, + "loss": 1.38, + "theoretical_loss": 3.4445875115047655, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137211634904714, + "loss": 1.3217, + "theoretical_loss": 3.444567547293409, + "tokens_seen": 1903951872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137011033099298, + "loss": 1.2942, + "theoretical_loss": 3.444547584841179, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021368104312938815, + "loss": 1.2689, + "theoretical_loss": 3.444527624147799, + "tokens_seen": 1904214016 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021366098294884654, + "loss": 1.2306, + "theoretical_loss": 3.4445076652129925, + "tokens_seen": 1904345088 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002136409227683049, + "loss": 1.4317, + "theoretical_loss": 3.444487708036484, + "tokens_seen": 1904476160 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002136208625877633, + "loss": 1.3986, + "theoretical_loss": 3.4444677526179976, + "tokens_seen": 1904607232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021360080240722166, + "loss": 1.3389, + "theoretical_loss": 3.444447798957258, + "tokens_seen": 1904738304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021358074222668004, + "loss": 1.2214, + "theoretical_loss": 3.4444278470539884, + "tokens_seen": 1904869376 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021356068204613843, + "loss": 1.4103, + "theoretical_loss": 3.4444078969079133, + "tokens_seen": 1905000448 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002135406218655968, + "loss": 1.3968, + "theoretical_loss": 3.4443879485187576, + "tokens_seen": 1905131520 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021352056168505519, + "loss": 1.4196, + "theoretical_loss": 3.4443680018862457, + "tokens_seen": 1905262592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021350050150451355, + "loss": 1.373, + "theoretical_loss": 3.4443480570101017, + "tokens_seen": 1905393664 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.5097751617431641, + "objective/train/avg_token_score": 0.016621354967355728, + "objective/train/docs_used": 1077487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.517067551612854, + "objective/train/original_loss": 3.3470466136932373, + "objective/train/theoretical_loss": 3.444338085230582, + "objective/train/tokens_used": 1905459200, + "objective/train/unlikelihood_loss": 0.22249506413936615, + "theoretical_loss": 3.444338085230582, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021348044132397194, + "loss": 1.2817, + "theoretical_loss": 3.44432811389005, + "tokens_seen": 1905524736 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134603811434303, + "loss": 1.2884, + "theoretical_loss": 3.444308172525816, + "tokens_seen": 1905655808 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134403209628887, + "loss": 1.3042, + "theoretical_loss": 3.444288232917123, + "tokens_seen": 1905786880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021342026078234705, + "loss": 1.3382, + "theoretical_loss": 3.444268295063697, + "tokens_seen": 1905917952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021340020060180544, + "loss": 1.2848, + "theoretical_loss": 3.444248358965263, + "tokens_seen": 1906049024 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002133801404212638, + "loss": 1.1479, + "theoretical_loss": 3.4442284246215444, + "tokens_seen": 1906180096 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021336008024072216, + "loss": 1.2896, + "theoretical_loss": 3.444208492032267, + "tokens_seen": 1906311168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021334002006018055, + "loss": 1.2878, + "theoretical_loss": 3.444188561197156, + "tokens_seen": 1906442240 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021331995987963892, + "loss": 1.3602, + "theoretical_loss": 3.444168632115936, + "tokens_seen": 1906573312 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002132998996990973, + "loss": 1.4238, + "theoretical_loss": 3.444148704788332, + "tokens_seen": 1906704384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021327983951855567, + "loss": 1.1546, + "theoretical_loss": 3.4441287792140693, + "tokens_seen": 1906835456 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021325977933801406, + "loss": 1.3189, + "theoretical_loss": 3.444108855392874, + "tokens_seen": 1906966528 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.43609482049942017, + "objective/train/avg_token_score": 0.018311558291316032, + "objective/train/docs_used": 1078017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3344192504882812, + "objective/train/original_loss": 3.709261178970337, + "objective/train/theoretical_loss": 3.4440889333244695, + "objective/train/tokens_used": 1907097600, + "objective/train/unlikelihood_loss": 0.18454064428806305, + "theoretical_loss": 3.4440889333244695, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021323971915747242, + "loss": 1.25, + "theoretical_loss": 3.4440889333244695, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002132196589769308, + "loss": 1.2955, + "theoretical_loss": 3.4440690130085825, + "tokens_seen": 1907228672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021319959879638917, + "loss": 1.1765, + "theoretical_loss": 3.4440490944449387, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021317953861584753, + "loss": 1.2446, + "theoretical_loss": 3.444029177633263, + "tokens_seen": 1907490816 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021315947843530592, + "loss": 1.1539, + "theoretical_loss": 3.4440092625732808, + "tokens_seen": 1907621888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021313941825476429, + "loss": 1.1722, + "theoretical_loss": 3.443989349264718, + "tokens_seen": 1907752960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021311935807422267, + "loss": 1.3505, + "theoretical_loss": 3.4439694377073, + "tokens_seen": 1907884032 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021309929789368104, + "loss": 1.4516, + "theoretical_loss": 3.4439495279007524, + "tokens_seen": 1908015104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021307923771313943, + "loss": 1.1666, + "theoretical_loss": 3.443929619844801, + "tokens_seen": 1908146176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002130591775325978, + "loss": 1.2091, + "theoretical_loss": 3.4439097135391723, + "tokens_seen": 1908277248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021303911735205618, + "loss": 1.233, + "theoretical_loss": 3.4438898089835916, + "tokens_seen": 1908408320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021301905717151454, + "loss": 1.2667, + "theoretical_loss": 3.4438699061777855, + "tokens_seen": 1908539392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002129989969909729, + "loss": 1.5355, + "theoretical_loss": 3.4438500051214787, + "tokens_seen": 1908670464 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.4912634491920471, + "objective/train/avg_token_score": 0.012508946470916271, + "objective/train/docs_used": 1079511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3436886072158813, + "objective/train/original_loss": 3.410301685333252, + "objective/train/theoretical_loss": 3.4438400552493027, + "objective/train/tokens_used": 1908736000, + "objective/train/unlikelihood_loss": 0.13294106721878052, + "theoretical_loss": 3.4438400552493027, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002129789368104313, + "loss": 1.3108, + "theoretical_loss": 3.4438301058143987, + "tokens_seen": 1908801536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021295887662988965, + "loss": 1.3779, + "theoretical_loss": 3.443810208256271, + "tokens_seen": 1908932608 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021293881644934804, + "loss": 1.2022, + "theoretical_loss": 3.443790312446822, + "tokens_seen": 1909063680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002129187562688064, + "loss": 1.2472, + "theoretical_loss": 3.4437704183857782, + "tokens_seen": 1909194752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002128986960882648, + "loss": 1.4301, + "theoretical_loss": 3.443750526072865, + "tokens_seen": 1909325824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021287863590772316, + "loss": 1.3354, + "theoretical_loss": 3.4437306355078094, + "tokens_seen": 1909456896 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021285857572718155, + "loss": 1.2931, + "theoretical_loss": 3.4437107466903383, + "tokens_seen": 1909587968 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002128385155466399, + "loss": 1.3299, + "theoretical_loss": 3.443690859620178, + "tokens_seen": 1909719040 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021281845536609832, + "loss": 1.1298, + "theoretical_loss": 3.443670974297054, + "tokens_seen": 1909850112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127983951855567, + "loss": 1.1983, + "theoretical_loss": 3.4436510907206945, + "tokens_seen": 1909981184 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021277833500501505, + "loss": 1.3196, + "theoretical_loss": 3.4436312088908254, + "tokens_seen": 1910112256 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021275827482447344, + "loss": 1.4433, + "theoretical_loss": 3.4436113288071732, + "tokens_seen": 1910243328 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.33938172459602356, + "objective/train/avg_token_score": 0.026301808655261993, + "objective/train/docs_used": 1080108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9394913911819458, + "objective/train/original_loss": 4.001698017120361, + "objective/train/theoretical_loss": 3.443591450469466, + "objective/train/tokens_used": 1910374400, + "objective/train/unlikelihood_loss": 0.2126522809267044, + "theoretical_loss": 3.443591450469466, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127382146439318, + "loss": 1.3621, + "theoretical_loss": 3.443591450469466, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127181544633902, + "loss": 1.2521, + "theoretical_loss": 3.443571573877429, + "tokens_seen": 1910505472 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021269809428284855, + "loss": 1.3842, + "theoretical_loss": 3.4435516990307904, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021267803410230694, + "loss": 1.2834, + "theoretical_loss": 3.4435318259292766, + "tokens_seen": 1910767616 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002126579739217653, + "loss": 1.3017, + "theoretical_loss": 3.4435119545726147, + "tokens_seen": 1910898688 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002126379137412237, + "loss": 1.2033, + "theoretical_loss": 3.4434920849605324, + "tokens_seen": 1911029760 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021261785356068206, + "loss": 1.2156, + "theoretical_loss": 3.443472217092756, + "tokens_seen": 1911160832 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021259779338014042, + "loss": 1.2369, + "theoretical_loss": 3.4434523509690136, + "tokens_seen": 1911291904 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002125777331995988, + "loss": 1.2077, + "theoretical_loss": 3.4434324865890322, + "tokens_seen": 1911422976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021255767301905717, + "loss": 1.386, + "theoretical_loss": 3.443412623952539, + "tokens_seen": 1911554048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021253761283851556, + "loss": 1.2529, + "theoretical_loss": 3.4433927630592613, + "tokens_seen": 1911685120 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021251755265797392, + "loss": 1.2068, + "theoretical_loss": 3.4433729039089274, + "tokens_seen": 1911816192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002124974924774323, + "loss": 1.3313, + "theoretical_loss": 3.4433530465012643, + "tokens_seen": 1911947264 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.3253909945487976, + "objective/train/avg_token_score": 0.024844123050570488, + "objective/train/docs_used": 1081319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0753298997879028, + "objective/train/original_loss": 3.8704843521118164, + "objective/train/theoretical_loss": 3.443343118450849, + "objective/train/tokens_used": 1912012800, + "objective/train/unlikelihood_loss": 0.21136721968650818, + "theoretical_loss": 3.443343118450849, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021247743229689067, + "loss": 1.2762, + "theoretical_loss": 3.4433331908359994, + "tokens_seen": 1912078336 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021245737211634906, + "loss": 1.3189, + "theoretical_loss": 3.443313336912861, + "tokens_seen": 1912209408 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021243731193580742, + "loss": 1.1463, + "theoretical_loss": 3.4432934847315764, + "tokens_seen": 1912340480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002124172517552658, + "loss": 1.3744, + "theoretical_loss": 3.4432736342918737, + "tokens_seen": 1912471552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021239719157472418, + "loss": 1.2868, + "theoretical_loss": 3.4432537855934804, + "tokens_seen": 1912602624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021237713139418254, + "loss": 1.1088, + "theoretical_loss": 3.4432339386361246, + "tokens_seen": 1912733696 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021235707121364093, + "loss": 1.2715, + "theoretical_loss": 3.443214093419534, + "tokens_seen": 1912864768 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002123370110330993, + "loss": 1.3082, + "theoretical_loss": 3.4431942499434376, + "tokens_seen": 1912995840 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021231695085255768, + "loss": 1.467, + "theoretical_loss": 3.443174408207563, + "tokens_seen": 1913126912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021229689067201604, + "loss": 1.2894, + "theoretical_loss": 3.4431545682116376, + "tokens_seen": 1913257984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021227683049147443, + "loss": 1.2309, + "theoretical_loss": 3.44313472995539, + "tokens_seen": 1913389056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002122567703109328, + "loss": 1.2004, + "theoretical_loss": 3.44311489343855, + "tokens_seen": 1913520128 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.3669965863227844, + "objective/train/avg_token_score": 0.016051271930336952, + "objective/train/docs_used": 1081978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1902230978012085, + "objective/train/original_loss": 3.8286728858947754, + "objective/train/theoretical_loss": 3.4430950586608438, + "objective/train/tokens_used": 1913651200, + "objective/train/unlikelihood_loss": 0.17121806740760803, + "theoretical_loss": 3.4430950586608438, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021223671013039118, + "loss": 1.1998, + "theoretical_loss": 3.4430950586608438, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021221664994984954, + "loss": 1.4105, + "theoretical_loss": 3.4430752256220014, + "tokens_seen": 1913782272 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121965897693079, + "loss": 1.2056, + "theoretical_loss": 3.4430553943217506, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121765295887663, + "loss": 1.4615, + "theoretical_loss": 3.4430355647598194, + "tokens_seen": 1914044416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021215646940822466, + "loss": 1.3018, + "theoretical_loss": 3.443015736935938, + "tokens_seen": 1914175488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021213640922768305, + "loss": 1.2531, + "theoretical_loss": 3.4429959108498336, + "tokens_seen": 1914306560 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121163490471414, + "loss": 1.1447, + "theoretical_loss": 3.4429760865012353, + "tokens_seen": 1914437632 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002120962888665998, + "loss": 1.2022, + "theoretical_loss": 3.4429562638898723, + "tokens_seen": 1914568704 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002120762286860582, + "loss": 1.3367, + "theoretical_loss": 3.4429364430154727, + "tokens_seen": 1914699776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021205616850551658, + "loss": 1.2261, + "theoretical_loss": 3.4429166238777658, + "tokens_seen": 1914830848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021203610832497494, + "loss": 1.2185, + "theoretical_loss": 3.4428968064764813, + "tokens_seen": 1914961920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002120160481444333, + "loss": 1.2005, + "theoretical_loss": 3.442876990811347, + "tokens_seen": 1915092992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002119959879638917, + "loss": 1.2029, + "theoretical_loss": 3.442857176882093, + "tokens_seen": 1915224064 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.44654202461242676, + "objective/train/avg_token_score": 0.02138923481106758, + "objective/train/docs_used": 1082665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3626799583435059, + "objective/train/original_loss": 3.75476336479187, + "objective/train/theoretical_loss": 3.442847270568336, + "objective/train/tokens_used": 1915289600, + "objective/train/unlikelihood_loss": 0.15580429136753082, + "theoretical_loss": 3.442847270568336, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021197592778335005, + "loss": 1.1595, + "theoretical_loss": 3.4428373646884474, + "tokens_seen": 1915355136 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021195586760280844, + "loss": 1.1742, + "theoretical_loss": 3.4428175542301407, + "tokens_seen": 1915486208 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002119358074222668, + "loss": 1.2858, + "theoretical_loss": 3.4427977455069008, + "tokens_seen": 1915617280 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002119157472417252, + "loss": 1.2707, + "theoretical_loss": 3.442777938518458, + "tokens_seen": 1915748352 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021189568706118356, + "loss": 1.4358, + "theoretical_loss": 3.442758133264541, + "tokens_seen": 1915879424 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021187562688064195, + "loss": 1.2555, + "theoretical_loss": 3.4427383297448806, + "tokens_seen": 1916010496 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002118555667001003, + "loss": 1.0672, + "theoretical_loss": 3.442718527959205, + "tokens_seen": 1916141568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002118355065195587, + "loss": 1.194, + "theoretical_loss": 3.442698727907244, + "tokens_seen": 1916272640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021181544633901706, + "loss": 1.3069, + "theoretical_loss": 3.4426789295887277, + "tokens_seen": 1916403712 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021179538615847542, + "loss": 1.2234, + "theoretical_loss": 3.442659133003385, + "tokens_seen": 1916534784 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002117753259779338, + "loss": 1.2215, + "theoretical_loss": 3.442639338150946, + "tokens_seen": 1916665856 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021175526579739217, + "loss": 1.1635, + "theoretical_loss": 3.4426195450311416, + "tokens_seen": 1916796928 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.4890640377998352, + "objective/train/avg_token_score": 0.009474965743720531, + "objective/train/docs_used": 1083753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5438741445541382, + "objective/train/original_loss": 3.630955219268799, + "objective/train/theoretical_loss": 3.4425997536437, + "objective/train/tokens_used": 1916928000, + "objective/train/unlikelihood_loss": 0.15519310534000397, + "theoretical_loss": 3.4425997536437, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021173520561685056, + "loss": 1.205, + "theoretical_loss": 3.4425997536437, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021171514543630893, + "loss": 1.4479, + "theoretical_loss": 3.4425799639883525, + "tokens_seen": 1917059072 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021169508525576732, + "loss": 1.2751, + "theoretical_loss": 3.442560176064828, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021167502507522568, + "loss": 1.2569, + "theoretical_loss": 3.4425403898728573, + "tokens_seen": 1917321216 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021165496489468407, + "loss": 1.3004, + "theoretical_loss": 3.44252060541217, + "tokens_seen": 1917452288 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021163490471414243, + "loss": 1.299, + "theoretical_loss": 3.4425008226824967, + "tokens_seen": 1917583360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002116148445336008, + "loss": 1.172, + "theoretical_loss": 3.4424810416835676, + "tokens_seen": 1917714432 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021159478435305918, + "loss": 1.3219, + "theoretical_loss": 3.442461262415113, + "tokens_seen": 1917845504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021157472417251754, + "loss": 1.2761, + "theoretical_loss": 3.4424414848768627, + "tokens_seen": 1917976576 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021155466399197593, + "loss": 1.3308, + "theoretical_loss": 3.4424217090685483, + "tokens_seen": 1918107648 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002115346038114343, + "loss": 1.1741, + "theoretical_loss": 3.442401934989899, + "tokens_seen": 1918238720 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021151454363089268, + "loss": 1.2899, + "theoretical_loss": 3.442382162640646, + "tokens_seen": 1918369792 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021149448345035105, + "loss": 1.1669, + "theoretical_loss": 3.44236239202052, + "tokens_seen": 1918500864 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.3958333432674408, + "objective/train/avg_token_score": 0.021457023918628693, + "objective/train/docs_used": 1084225, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.300883173942566, + "objective/train/original_loss": 4.229017734527588, + "objective/train/theoretical_loss": 3.4423525073587955, + "objective/train/tokens_used": 1918566400, + "objective/train/unlikelihood_loss": 0.13041551411151886, + "theoretical_loss": 3.4423525073587955, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021147442326980944, + "loss": 1.0799, + "theoretical_loss": 3.4423426231292518, + "tokens_seen": 1918631936 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002114543630892678, + "loss": 1.2725, + "theoretical_loss": 3.4423228559665713, + "tokens_seen": 1918763008 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021143430290872616, + "loss": 1.3808, + "theoretical_loss": 3.4423030905322105, + "tokens_seen": 1918894080 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021141424272818455, + "loss": 1.2291, + "theoretical_loss": 3.442283326825899, + "tokens_seen": 1919025152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002113941825476429, + "loss": 1.3848, + "theoretical_loss": 3.4422635648473685, + "tokens_seen": 1919156224 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002113741223671013, + "loss": 1.2886, + "theoretical_loss": 3.4422438045963495, + "tokens_seen": 1919287296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021135406218655966, + "loss": 1.0356, + "theoretical_loss": 3.4422240460725737, + "tokens_seen": 1919418368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021133400200601808, + "loss": 1.2037, + "theoretical_loss": 3.4422042892757716, + "tokens_seen": 1919549440 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021131394182547644, + "loss": 1.1894, + "theoretical_loss": 3.442184534205674, + "tokens_seen": 1919680512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021129388164493483, + "loss": 1.2861, + "theoretical_loss": 3.4421647808620133, + "tokens_seen": 1919811584 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002112738214643932, + "loss": 1.2427, + "theoretical_loss": 3.4421450292445197, + "tokens_seen": 1919942656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021125376128385158, + "loss": 1.3227, + "theoretical_loss": 3.4421252793529247, + "tokens_seen": 1920073728 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.3578323423862457, + "objective/train/avg_token_score": 0.008121492341160774, + "objective/train/docs_used": 1085471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.00901198387146, + "objective/train/original_loss": 3.7845864295959473, + "objective/train/theoretical_loss": 3.4421055311869604, + "objective/train/tokens_used": 1920204800, + "objective/train/unlikelihood_loss": 0.15960928797721863, + "theoretical_loss": 3.4421055311869604, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021123370110330994, + "loss": 1.257, + "theoretical_loss": 3.4421055311869604, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002112136409227683, + "loss": 1.2304, + "theoretical_loss": 3.4420857847463573, + "tokens_seen": 1920335872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002111935807422267, + "loss": 1.3458, + "theoretical_loss": 3.4420660400308476, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021117352056168506, + "loss": 1.3831, + "theoretical_loss": 3.442046297040163, + "tokens_seen": 1920598016 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021115346038114345, + "loss": 1.3006, + "theoretical_loss": 3.442026555774034, + "tokens_seen": 1920729088 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002111334002006018, + "loss": 1.3634, + "theoretical_loss": 3.442006816232194, + "tokens_seen": 1920860160 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002111133400200602, + "loss": 1.283, + "theoretical_loss": 3.441987078414373, + "tokens_seen": 1920991232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021109327983951856, + "loss": 1.3629, + "theoretical_loss": 3.4419673423203037, + "tokens_seen": 1921122304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021107321965897695, + "loss": 1.3927, + "theoretical_loss": 3.441947607949718, + "tokens_seen": 1921253376 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002110531594784353, + "loss": 1.2542, + "theoretical_loss": 3.441927875302348, + "tokens_seen": 1921384448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021103309929789368, + "loss": 1.1667, + "theoretical_loss": 3.4419081443779254, + "tokens_seen": 1921515520 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021101303911735207, + "loss": 1.2335, + "theoretical_loss": 3.441888415176182, + "tokens_seen": 1921646592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021099297893681043, + "loss": 1.3424, + "theoretical_loss": 3.44186868769685, + "tokens_seen": 1921777664 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.4151392877101898, + "objective/train/avg_token_score": 0.017439980059862137, + "objective/train/docs_used": 1086172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.283548355102539, + "objective/train/original_loss": 3.907247304916382, + "objective/train/theoretical_loss": 3.441858824603005, + "objective/train/tokens_used": 1921843200, + "objective/train/unlikelihood_loss": 0.15037135779857635, + "theoretical_loss": 3.441858824603005, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021097291875626882, + "loss": 1.3174, + "theoretical_loss": 3.4418489619396624, + "tokens_seen": 1921908736 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021095285857572718, + "loss": 1.3912, + "theoretical_loss": 3.44182923790435, + "tokens_seen": 1922039808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021093279839518557, + "loss": 1.2544, + "theoretical_loss": 3.4418095155906463, + "tokens_seen": 1922170880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021091273821464393, + "loss": 1.2119, + "theoretical_loss": 3.441789794998283, + "tokens_seen": 1922301952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021089267803410232, + "loss": 1.2371, + "theoretical_loss": 3.4417700761269927, + "tokens_seen": 1922433024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021087261785356068, + "loss": 1.1837, + "theoretical_loss": 3.441750358976508, + "tokens_seen": 1922564096 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021085255767301904, + "loss": 1.4425, + "theoretical_loss": 3.441730643546561, + "tokens_seen": 1922695168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021083249749247743, + "loss": 1.4012, + "theoretical_loss": 3.441710929836885, + "tokens_seen": 1922826240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002108124373119358, + "loss": 1.1015, + "theoretical_loss": 3.441691217847212, + "tokens_seen": 1922957312 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021079237713139419, + "loss": 1.2616, + "theoretical_loss": 3.441671507577275, + "tokens_seen": 1923088384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021077231695085255, + "loss": 1.3809, + "theoretical_loss": 3.4416517990268067, + "tokens_seen": 1923219456 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021075225677031094, + "loss": 1.417, + "theoretical_loss": 3.4416320921955403, + "tokens_seen": 1923350528 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.3891128897666931, + "objective/train/avg_token_score": 0.036930207163095474, + "objective/train/docs_used": 1087440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2392277717590332, + "objective/train/original_loss": 3.844510555267334, + "objective/train/theoretical_loss": 3.441612387083208, + "objective/train/tokens_used": 1923481600, + "objective/train/unlikelihood_loss": 0.14572468400001526, + "theoretical_loss": 3.441612387083208, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002107321965897693, + "loss": 1.5259, + "theoretical_loss": 3.441612387083208, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002107121364092277, + "loss": 1.4639, + "theoretical_loss": 3.441592683689543, + "tokens_seen": 1923612672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021069207622868605, + "loss": 1.2544, + "theoretical_loss": 3.4415729820142786, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021067201604814444, + "loss": 1.3886, + "theoretical_loss": 3.4415532820571473, + "tokens_seen": 1923874816 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002106519558676028, + "loss": 1.0783, + "theoretical_loss": 3.441533583817883, + "tokens_seen": 1924005888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021063189568706116, + "loss": 1.2934, + "theoretical_loss": 3.4415138872962183, + "tokens_seen": 1924136960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021061183550651955, + "loss": 1.425, + "theoretical_loss": 3.441494192491887, + "tokens_seen": 1924268032 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021059177532597792, + "loss": 1.3039, + "theoretical_loss": 3.441474499404621, + "tokens_seen": 1924399104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021057171514543633, + "loss": 1.3098, + "theoretical_loss": 3.4414548080341554, + "tokens_seen": 1924530176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002105516549648947, + "loss": 1.3529, + "theoretical_loss": 3.441435118380223, + "tokens_seen": 1924661248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021053159478435308, + "loss": 1.2084, + "theoretical_loss": 3.4414154304425573, + "tokens_seen": 1924792320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021051153460381145, + "loss": 1.289, + "theoretical_loss": 3.441395744220891, + "tokens_seen": 1924923392 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021049147442326984, + "loss": 1.3349, + "theoretical_loss": 3.441376059714959, + "tokens_seen": 1925054464 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.3771994113922119, + "objective/train/avg_token_score": 0.019752413034439087, + "objective/train/docs_used": 1088918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1480333805084229, + "objective/train/original_loss": 3.7906956672668457, + "objective/train/theoretical_loss": 3.44136621810531, + "objective/train/tokens_used": 1925120000, + "objective/train/unlikelihood_loss": 0.23767469823360443, + "theoretical_loss": 3.44136621810531, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104714142427282, + "loss": 1.1407, + "theoretical_loss": 3.4413563769244946, + "tokens_seen": 1925185536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021045135406218656, + "loss": 1.3261, + "theoretical_loss": 3.441336695849231, + "tokens_seen": 1925316608 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021043129388164495, + "loss": 1.2842, + "theoretical_loss": 3.4413170164889024, + "tokens_seen": 1925447680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104112337011033, + "loss": 1.3592, + "theoretical_loss": 3.4412973388432424, + "tokens_seen": 1925578752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103911735205617, + "loss": 1.2826, + "theoretical_loss": 3.4412776629119852, + "tokens_seen": 1925709824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021037111334002006, + "loss": 1.2447, + "theoretical_loss": 3.4412579886948644, + "tokens_seen": 1925840896 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021035105315947845, + "loss": 1.3837, + "theoretical_loss": 3.4412383161916145, + "tokens_seen": 1925971968 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021033099297893681, + "loss": 1.3763, + "theoretical_loss": 3.441218645401969, + "tokens_seen": 1926103040 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103109327983952, + "loss": 1.1838, + "theoretical_loss": 3.4411989763256625, + "tokens_seen": 1926234112 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021029087261785357, + "loss": 1.0417, + "theoretical_loss": 3.441179308962429, + "tokens_seen": 1926365184 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021027081243731196, + "loss": 1.2851, + "theoretical_loss": 3.4411596433120026, + "tokens_seen": 1926496256 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021025075225677032, + "loss": 1.5503, + "theoretical_loss": 3.4411399793741184, + "tokens_seen": 1926627328 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.2983260154724121, + "objective/train/avg_token_score": 0.04583496227860451, + "objective/train/docs_used": 1089550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.08523690700531, + "objective/train/original_loss": 4.301586627960205, + "objective/train/theoretical_loss": 3.44112031714851, + "objective/train/tokens_used": 1926758400, + "objective/train/unlikelihood_loss": 0.1520206779241562, + "theoretical_loss": 3.44112031714851, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021023069207622868, + "loss": 1.3107, + "theoretical_loss": 3.44112031714851, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021021063189568707, + "loss": 1.3289, + "theoretical_loss": 3.4411006566349114, + "tokens_seen": 1926889472 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021019057171514543, + "loss": 1.0778, + "theoretical_loss": 3.4410809978330583, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021017051153460382, + "loss": 1.1774, + "theoretical_loss": 3.441061340742684, + "tokens_seen": 1927151616 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021015045135406218, + "loss": 1.392, + "theoretical_loss": 3.4410416853635244, + "tokens_seen": 1927282688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021013039117352057, + "loss": 1.2966, + "theoretical_loss": 3.4410220316953133, + "tokens_seen": 1927413760 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021011033099297894, + "loss": 1.3365, + "theoretical_loss": 3.441002379737786, + "tokens_seen": 1927544832 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021009027081243732, + "loss": 1.3627, + "theoretical_loss": 3.4409827294906767, + "tokens_seen": 1927675904 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002100702106318957, + "loss": 1.2905, + "theoretical_loss": 3.440963080953721, + "tokens_seen": 1927806976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021005015045135405, + "loss": 1.3038, + "theoretical_loss": 3.4409434341266527, + "tokens_seen": 1927938048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021003009027081244, + "loss": 1.5306, + "theoretical_loss": 3.4409237890092075, + "tokens_seen": 1928069120 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002100100300902708, + "loss": 1.1806, + "theoretical_loss": 3.4409041456011202, + "tokens_seen": 1928200192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002099899699097292, + "loss": 1.2619, + "theoretical_loss": 3.4408845039021263, + "tokens_seen": 1928331264 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.4065249264240265, + "objective/train/avg_token_score": 0.04939766600728035, + "objective/train/docs_used": 1091004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.340605616569519, + "objective/train/original_loss": 4.010280132293701, + "objective/train/theoretical_loss": 3.4408746836934565, + "objective/train/tokens_used": 1928396800, + "objective/train/unlikelihood_loss": 0.12202414125204086, + "theoretical_loss": 3.4408746836934565, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020996990972918755, + "loss": 1.3606, + "theoretical_loss": 3.4408648639119606, + "tokens_seen": 1928462336 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020994984954864594, + "loss": 1.2423, + "theoretical_loss": 3.4408452256303583, + "tokens_seen": 1928593408 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002099297893681043, + "loss": 1.3852, + "theoretical_loss": 3.4408255890570545, + "tokens_seen": 1928724480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002099097291875627, + "loss": 1.4151, + "theoretical_loss": 3.440805954191785, + "tokens_seen": 1928855552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020988966900702106, + "loss": 1.3926, + "theoretical_loss": 3.4407863210342846, + "tokens_seen": 1928986624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020986960882647942, + "loss": 1.2577, + "theoretical_loss": 3.440766689584289, + "tokens_seen": 1929117696 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002098495486459378, + "loss": 1.353, + "theoretical_loss": 3.4407470598415344, + "tokens_seen": 1929248768 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002098294884653962, + "loss": 1.302, + "theoretical_loss": 3.4407274318057555, + "tokens_seen": 1929379840 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020980942828485459, + "loss": 1.3126, + "theoretical_loss": 3.4407078054766878, + "tokens_seen": 1929510912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020978936810431295, + "loss": 1.2768, + "theoretical_loss": 3.4406881808540675, + "tokens_seen": 1929641984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020976930792377134, + "loss": 1.3658, + "theoretical_loss": 3.44066855793763, + "tokens_seen": 1929773056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002097492477432297, + "loss": 1.3184, + "theoretical_loss": 3.440648936727111, + "tokens_seen": 1929904128 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.46535924077033997, + "objective/train/avg_token_score": 0.00794178992509842, + "objective/train/docs_used": 1091522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4894250631332397, + "objective/train/original_loss": 3.370820999145508, + "objective/train/theoretical_loss": 3.4406293172222466, + "objective/train/tokens_used": 1930035200, + "objective/train/unlikelihood_loss": 0.20787984132766724, + "theoretical_loss": 3.4406293172222466, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002097291875626881, + "loss": 1.3363, + "theoretical_loss": 3.4406293172222466, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020970912738214645, + "loss": 1.1518, + "theoretical_loss": 3.4406096994227733, + "tokens_seen": 1930166272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020968906720160484, + "loss": 1.3696, + "theoretical_loss": 3.4405900833284258, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002096690070210632, + "loss": 1.2431, + "theoretical_loss": 3.440570468938941, + "tokens_seen": 1930428416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020964894684052156, + "loss": 1.2204, + "theoretical_loss": 3.4405508562540543, + "tokens_seen": 1930559488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020962888665997995, + "loss": 1.2704, + "theoretical_loss": 3.440531245273503, + "tokens_seen": 1930690560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020960882647943832, + "loss": 1.3943, + "theoretical_loss": 3.4405116359970225, + "tokens_seen": 1930821632 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002095887662988967, + "loss": 1.3018, + "theoretical_loss": 3.440492028424349, + "tokens_seen": 1930952704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020956870611835507, + "loss": 1.3761, + "theoretical_loss": 3.440472422555219, + "tokens_seen": 1931083776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020954864593781346, + "loss": 1.2569, + "theoretical_loss": 3.4404528183893692, + "tokens_seen": 1931214848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020952858575727182, + "loss": 1.4206, + "theoretical_loss": 3.4404332159265354, + "tokens_seen": 1931345920 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002095085255767302, + "loss": 1.2615, + "theoretical_loss": 3.440413615166454, + "tokens_seen": 1931476992 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020948846539618857, + "loss": 1.1978, + "theoretical_loss": 3.440394016108863, + "tokens_seen": 1931608064 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.380987286567688, + "objective/train/avg_token_score": 0.019535914063453674, + "objective/train/docs_used": 1091970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9542360305786133, + "objective/train/original_loss": 3.9765820503234863, + "objective/train/theoretical_loss": 3.4403842172184183, + "objective/train/tokens_used": 1931673600, + "objective/train/unlikelihood_loss": 0.1626909077167511, + "theoretical_loss": 3.4403842172184183, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020946840521564693, + "loss": 1.2972, + "theoretical_loss": 3.440374418753497, + "tokens_seen": 1931739136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020944834503510532, + "loss": 1.1996, + "theoretical_loss": 3.440354823100094, + "tokens_seen": 1931870208 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020942828485456368, + "loss": 1.3732, + "theoretical_loss": 3.4403352291483906, + "tokens_seen": 1932001280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020940822467402207, + "loss": 1.3665, + "theoretical_loss": 3.4403156368981236, + "tokens_seen": 1932132352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020938816449348044, + "loss": 1.3773, + "theoretical_loss": 3.4402960463490286, + "tokens_seen": 1932263424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020936810431293883, + "loss": 1.1677, + "theoretical_loss": 3.440276457500844, + "tokens_seen": 1932394496 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002093480441323972, + "loss": 1.3494, + "theoretical_loss": 3.440256870353307, + "tokens_seen": 1932525568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020932798395185558, + "loss": 1.2233, + "theoretical_loss": 3.4402372849061535, + "tokens_seen": 1932656640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020930792377131394, + "loss": 1.2604, + "theoretical_loss": 3.4402177011591206, + "tokens_seen": 1932787712 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002092878635907723, + "loss": 1.3325, + "theoretical_loss": 3.4401981191119466, + "tokens_seen": 1932918784 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002092678034102307, + "loss": 1.3038, + "theoretical_loss": 3.4401785387643677, + "tokens_seen": 1933049856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020924774322968905, + "loss": 1.2852, + "theoretical_loss": 3.440158960116121, + "tokens_seen": 1933180928 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.41477271914482117, + "objective/train/avg_token_score": 0.014390777796506882, + "objective/train/docs_used": 1092955, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3431459665298462, + "objective/train/original_loss": 3.9108927249908447, + "objective/train/theoretical_loss": 3.4401393831669447, + "objective/train/tokens_used": 1933312000, + "objective/train/unlikelihood_loss": 0.16139693558216095, + "theoretical_loss": 3.4401393831669447, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020922768304914744, + "loss": 1.3387, + "theoretical_loss": 3.4401393831669447, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002092076228686058, + "loss": 1.2599, + "theoretical_loss": 3.440119807916575, + "tokens_seen": 1933443072 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091875626880642, + "loss": 1.2415, + "theoretical_loss": 3.440100234364751, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020916750250752256, + "loss": 1.2755, + "theoretical_loss": 3.4400806625112086, + "tokens_seen": 1933705216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020914744232698095, + "loss": 1.2181, + "theoretical_loss": 3.440061092355686, + "tokens_seen": 1933836288 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091273821464393, + "loss": 1.3055, + "theoretical_loss": 3.440041523897921, + "tokens_seen": 1933967360 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091073219658977, + "loss": 1.1559, + "theoretical_loss": 3.4400219571376507, + "tokens_seen": 1934098432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020908726178535606, + "loss": 1.3719, + "theoretical_loss": 3.4400023920746134, + "tokens_seen": 1934229504 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020906720160481445, + "loss": 1.4331, + "theoretical_loss": 3.4399828287085468, + "tokens_seen": 1934360576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020904714142427284, + "loss": 1.2664, + "theoretical_loss": 3.4399632670391886, + "tokens_seen": 1934491648 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002090270812437312, + "loss": 1.2873, + "theoretical_loss": 3.4399437070662766, + "tokens_seen": 1934622720 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002090070210631896, + "loss": 1.3324, + "theoretical_loss": 3.439924148789549, + "tokens_seen": 1934753792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020898696088264795, + "loss": 1.1663, + "theoretical_loss": 3.4399045922087437, + "tokens_seen": 1934884864 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.3615591526031494, + "objective/train/avg_token_score": 0.016087079420685768, + "objective/train/docs_used": 1093336, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2029565572738647, + "objective/train/original_loss": 4.164218902587891, + "objective/train/theoretical_loss": 3.4398948145542296, + "objective/train/tokens_used": 1934950400, + "objective/train/unlikelihood_loss": 0.10311036556959152, + "theoretical_loss": 3.4398948145542296, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020896690070210634, + "loss": 1.3038, + "theoretical_loss": 3.4398850373235987, + "tokens_seen": 1935015936 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002089468405215647, + "loss": 1.2186, + "theoretical_loss": 3.439865484133852, + "tokens_seen": 1935147008 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002089267803410231, + "loss": 1.3898, + "theoretical_loss": 3.439845932639242, + "tokens_seen": 1935278080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020890672016048146, + "loss": 1.4022, + "theoretical_loss": 3.439826382839507, + "tokens_seen": 1935409152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020888665997993982, + "loss": 1.3566, + "theoretical_loss": 3.4398068347343855, + "tokens_seen": 1935540224 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002088665997993982, + "loss": 1.3335, + "theoretical_loss": 3.439787288323615, + "tokens_seen": 1935671296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020884653961885657, + "loss": 1.3577, + "theoretical_loss": 3.439767743606935, + "tokens_seen": 1935802368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020882647943831496, + "loss": 1.2588, + "theoretical_loss": 3.4397482005840834, + "tokens_seen": 1935933440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020880641925777332, + "loss": 1.3119, + "theoretical_loss": 3.439728659254799, + "tokens_seen": 1936064512 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002087863590772317, + "loss": 1.3951, + "theoretical_loss": 3.4397091196188203, + "tokens_seen": 1936195584 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020876629889669007, + "loss": 1.3863, + "theoretical_loss": 3.4396895816758857, + "tokens_seen": 1936326656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020874623871614846, + "loss": 1.3778, + "theoretical_loss": 3.439670045425734, + "tokens_seen": 1936457728 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.3465298116207123, + "objective/train/avg_token_score": 0.014081096276640892, + "objective/train/docs_used": 1094637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.172161340713501, + "objective/train/original_loss": 3.876873731613159, + "objective/train/theoretical_loss": 3.4396505108681046, + "objective/train/tokens_used": 1936588800, + "objective/train/unlikelihood_loss": 0.17920628190040588, + "theoretical_loss": 3.4396505108681046, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020872617853560682, + "loss": 1.3303, + "theoretical_loss": 3.4396505108681046, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020870611835506519, + "loss": 1.3567, + "theoretical_loss": 3.4396309780027354, + "tokens_seen": 1936719872 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020868605817452358, + "loss": 1.4224, + "theoretical_loss": 3.4396114468293657, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020866599799398194, + "loss": 1.3954, + "theoretical_loss": 3.4395919173477347, + "tokens_seen": 1936982016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020864593781344033, + "loss": 1.3925, + "theoretical_loss": 3.439572389557581, + "tokens_seen": 1937113088 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002086258776328987, + "loss": 1.2502, + "theoretical_loss": 3.439552863458644, + "tokens_seen": 1937244160 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020860581745235708, + "loss": 1.2652, + "theoretical_loss": 3.439533339050662, + "tokens_seen": 1937375232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020858575727181544, + "loss": 1.2329, + "theoretical_loss": 3.439513816333376, + "tokens_seen": 1937506304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020856569709127383, + "loss": 1.3397, + "theoretical_loss": 3.439494295306523, + "tokens_seen": 1937637376 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002085456369107322, + "loss": 1.4598, + "theoretical_loss": 3.4394747759698436, + "tokens_seen": 1937768448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020852557673019058, + "loss": 1.312, + "theoretical_loss": 3.4394552583230773, + "tokens_seen": 1937899520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020850551654964894, + "loss": 1.4701, + "theoretical_loss": 3.439435742365963, + "tokens_seen": 1938030592 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084854563691073, + "loss": 1.351, + "theoretical_loss": 3.43941622809824, + "tokens_seen": 1938161664 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.5395894646644592, + "objective/train/avg_token_score": 0.017774518579244614, + "objective/train/docs_used": 1095318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7296571731567383, + "objective/train/original_loss": 3.61208438873291, + "objective/train/theoretical_loss": 3.439406471597819, + "objective/train/tokens_used": 1938227200, + "objective/train/unlikelihood_loss": 0.12611813843250275, + "theoretical_loss": 3.439406471597819, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084653961885657, + "loss": 1.313, + "theoretical_loss": 3.4393967155196483, + "tokens_seen": 1938292736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020844533600802406, + "loss": 1.3931, + "theoretical_loss": 3.439377204629927, + "tokens_seen": 1938423808 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020842527582748245, + "loss": 1.2374, + "theoretical_loss": 3.439357695428816, + "tokens_seen": 1938554880 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084052156469408, + "loss": 1.2581, + "theoretical_loss": 3.4393381879160554, + "tokens_seen": 1938685952 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002083851554663992, + "loss": 1.0974, + "theoretical_loss": 3.439318682091385, + "tokens_seen": 1938817024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020836509528585756, + "loss": 1.324, + "theoretical_loss": 3.4392991779545437, + "tokens_seen": 1938948096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020834503510531595, + "loss": 1.362, + "theoretical_loss": 3.4392796755052712, + "tokens_seen": 1939079168 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020832497492477434, + "loss": 1.2461, + "theoretical_loss": 3.439260174743309, + "tokens_seen": 1939210240 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002083049147442327, + "loss": 1.3129, + "theoretical_loss": 3.4392406756683958, + "tokens_seen": 1939341312 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082848545636911, + "loss": 1.4181, + "theoretical_loss": 3.439221178280272, + "tokens_seen": 1939472384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020826479438314945, + "loss": 1.4124, + "theoretical_loss": 3.439201682578678, + "tokens_seen": 1939603456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020824473420260784, + "loss": 1.4532, + "theoretical_loss": 3.439182188563354, + "tokens_seen": 1939734528 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.4020650088787079, + "objective/train/avg_token_score": 0.014071573503315449, + "objective/train/docs_used": 1096386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4408215284347534, + "objective/train/original_loss": 3.792991876602173, + "objective/train/theoretical_loss": 3.4391626962340394, + "objective/train/tokens_used": 1939865600, + "objective/train/unlikelihood_loss": 0.17302802205085754, + "theoretical_loss": 3.4391626962340394, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082246740220662, + "loss": 1.3252, + "theoretical_loss": 3.4391626962340394, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082046138415246, + "loss": 1.3488, + "theoretical_loss": 3.439143205590475, + "tokens_seen": 1939996672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020818455366098296, + "loss": 1.2953, + "theoretical_loss": 3.439123716632401, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020816449348044135, + "loss": 1.4107, + "theoretical_loss": 3.4391042293595584, + "tokens_seen": 1940258816 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002081444332998997, + "loss": 1.2626, + "theoretical_loss": 3.4390847437716863, + "tokens_seen": 1940389888 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002081243731193581, + "loss": 1.399, + "theoretical_loss": 3.439065259868527, + "tokens_seen": 1940520960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020810431293881646, + "loss": 1.5866, + "theoretical_loss": 3.4390457776498193, + "tokens_seen": 1940652032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020808425275827482, + "loss": 1.4367, + "theoretical_loss": 3.439026297115305, + "tokens_seen": 1940783104 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002080641925777332, + "loss": 1.4514, + "theoretical_loss": 3.4390068182647244, + "tokens_seen": 1940914176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020804413239719157, + "loss": 1.2857, + "theoretical_loss": 3.4389873410978185, + "tokens_seen": 1941045248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020802407221664996, + "loss": 1.3898, + "theoretical_loss": 3.4389678656143277, + "tokens_seen": 1941176320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020800401203610833, + "loss": 1.3759, + "theoretical_loss": 3.4389483918139927, + "tokens_seen": 1941307392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020798395185556671, + "loss": 1.4776, + "theoretical_loss": 3.438928919696555, + "tokens_seen": 1941438464 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.3447580635547638, + "objective/train/avg_token_score": 0.02481227181851864, + "objective/train/docs_used": 1097058, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1756653785705566, + "objective/train/original_loss": 4.512879848480225, + "objective/train/theoretical_loss": 3.4389191842688414, + "objective/train/tokens_used": 1941504000, + "objective/train/unlikelihood_loss": 0.1251474767923355, + "theoretical_loss": 3.4389191842688414, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020796389167502508, + "loss": 1.2849, + "theoretical_loss": 3.438909449261755, + "tokens_seen": 1941569536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020794383149448347, + "loss": 1.362, + "theoretical_loss": 3.4388899805093343, + "tokens_seen": 1941700608 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020792377131394183, + "loss": 1.3454, + "theoretical_loss": 3.438870513439033, + "tokens_seen": 1941831680 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002079037111334002, + "loss": 1.3439, + "theoretical_loss": 3.438851048050594, + "tokens_seen": 1941962752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020788365095285858, + "loss": 1.483, + "theoretical_loss": 3.438831584343757, + "tokens_seen": 1942093824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020786359077231694, + "loss": 1.3353, + "theoretical_loss": 3.438812122318263, + "tokens_seen": 1942224896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020784353059177533, + "loss": 1.2789, + "theoretical_loss": 3.4387926619738547, + "tokens_seen": 1942355968 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002078234704112337, + "loss": 1.3751, + "theoretical_loss": 3.438773203310272, + "tokens_seen": 1942487040 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020780341023069208, + "loss": 1.1783, + "theoretical_loss": 3.4387537463272575, + "tokens_seen": 1942618112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020778335005015045, + "loss": 1.4315, + "theoretical_loss": 3.438734291024552, + "tokens_seen": 1942749184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020776328986960884, + "loss": 1.3074, + "theoretical_loss": 3.438714837401897, + "tokens_seen": 1942880256 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002077432296890672, + "loss": 1.5534, + "theoretical_loss": 3.4386953854590354, + "tokens_seen": 1943011328 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.4211876690387726, + "objective/train/avg_token_score": 0.020965728908777237, + "objective/train/docs_used": 1098461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2947056293487549, + "objective/train/original_loss": 3.9896326065063477, + "objective/train/theoretical_loss": 3.4386759351957066, + "objective/train/tokens_used": 1943142400, + "objective/train/unlikelihood_loss": 0.09931595623493195, + "theoretical_loss": 3.4386759351957066, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020772316950852556, + "loss": 1.4538, + "theoretical_loss": 3.4386759351957066, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020770310932798395, + "loss": 1.3546, + "theoretical_loss": 3.438656486611654, + "tokens_seen": 1943273472 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002076830491474423, + "loss": 1.415, + "theoretical_loss": 3.4386370397066193, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002076629889669007, + "loss": 1.288, + "theoretical_loss": 3.4386175944803434, + "tokens_seen": 1943535616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020764292878635906, + "loss": 1.303, + "theoretical_loss": 3.438598150932569, + "tokens_seen": 1943666688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020762286860581745, + "loss": 1.4167, + "theoretical_loss": 3.4385787090630373, + "tokens_seen": 1943797760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020760280842527581, + "loss": 1.4444, + "theoretical_loss": 3.438559268871491, + "tokens_seen": 1943928832 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002075827482447342, + "loss": 1.307, + "theoretical_loss": 3.4385398303576715, + "tokens_seen": 1944059904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002075626880641926, + "loss": 1.2835, + "theoretical_loss": 3.4385203935213218, + "tokens_seen": 1944190976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020754262788365098, + "loss": 1.194, + "theoretical_loss": 3.438500958362183, + "tokens_seen": 1944322048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020752256770310934, + "loss": 1.2123, + "theoretical_loss": 3.438481524879998, + "tokens_seen": 1944453120 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002075025075225677, + "loss": 1.3802, + "theoretical_loss": 3.4384620930745093, + "tokens_seen": 1944584192 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002074824473420261, + "loss": 1.2382, + "theoretical_loss": 3.4384426629454587, + "tokens_seen": 1944715264 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.5130131840705872, + "objective/train/avg_token_score": 0.00923915859311819, + "objective/train/docs_used": 1099095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.502394437789917, + "objective/train/original_loss": 3.6624698638916016, + "objective/train/theoretical_loss": 3.438432948509517, + "objective/train/tokens_used": 1944780800, + "objective/train/unlikelihood_loss": 0.13872729241847992, + "theoretical_loss": 3.438432948509517, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020746238716148446, + "loss": 1.2892, + "theoretical_loss": 3.4384232344925882, + "tokens_seen": 1944846336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020744232698094285, + "loss": 1.2061, + "theoretical_loss": 3.438403807715641, + "tokens_seen": 1944977408 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002074222668004012, + "loss": 1.3506, + "theoretical_loss": 3.43838438261436, + "tokens_seen": 1945108480 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002074022066198596, + "loss": 1.1833, + "theoretical_loss": 3.438364959188487, + "tokens_seen": 1945239552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020738214643931796, + "loss": 1.3835, + "theoretical_loss": 3.4383455374377645, + "tokens_seen": 1945370624 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020736208625877635, + "loss": 1.3808, + "theoretical_loss": 3.4383261173619353, + "tokens_seen": 1945501696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002073420260782347, + "loss": 1.2103, + "theoretical_loss": 3.438306698960743, + "tokens_seen": 1945632768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020732196589769308, + "loss": 1.4196, + "theoretical_loss": 3.4382872822339294, + "tokens_seen": 1945763840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020730190571715146, + "loss": 1.4609, + "theoretical_loss": 3.4382678671812377, + "tokens_seen": 1945894912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020728184553660983, + "loss": 1.3656, + "theoretical_loss": 3.4382484538024105, + "tokens_seen": 1946025984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020726178535606822, + "loss": 1.4207, + "theoretical_loss": 3.4382290420971913, + "tokens_seen": 1946157056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020724172517552658, + "loss": 1.4004, + "theoretical_loss": 3.4382096320653224, + "tokens_seen": 1946288128 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.3022971749305725, + "objective/train/avg_token_score": 0.05815090984106064, + "objective/train/docs_used": 1099984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0587533712387085, + "objective/train/original_loss": 4.369499206542969, + "objective/train/theoretical_loss": 3.438190223706548, + "objective/train/tokens_used": 1946419200, + "objective/train/unlikelihood_loss": 0.14050383865833282, + "theoretical_loss": 3.438190223706548, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020722166499498497, + "loss": 1.2783, + "theoretical_loss": 3.438190223706548, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020720160481444333, + "loss": 1.4258, + "theoretical_loss": 3.43817081702061, + "tokens_seen": 1946550272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020718154463390172, + "loss": 1.1965, + "theoretical_loss": 3.4381514120072523, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020716148445336008, + "loss": 1.3819, + "theoretical_loss": 3.438132008666218, + "tokens_seen": 1946812416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020714142427281844, + "loss": 1.3116, + "theoretical_loss": 3.43811260699725, + "tokens_seen": 1946943488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020712136409227683, + "loss": 1.4102, + "theoretical_loss": 3.438093207000093, + "tokens_seen": 1947074560 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002071013039117352, + "loss": 1.3237, + "theoretical_loss": 3.4380738086744884, + "tokens_seen": 1947205632 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020708124373119358, + "loss": 1.5059, + "theoretical_loss": 3.4380544120201812, + "tokens_seen": 1947336704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020706118355065195, + "loss": 1.2981, + "theoretical_loss": 3.4380350170369143, + "tokens_seen": 1947467776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020704112337011034, + "loss": 1.1253, + "theoretical_loss": 3.438015623724432, + "tokens_seen": 1947598848 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002070210631895687, + "loss": 1.3158, + "theoretical_loss": 3.437996232082477, + "tokens_seen": 1947729920 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002070010030090271, + "loss": 1.2765, + "theoretical_loss": 3.437976842110793, + "tokens_seen": 1947860992 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020698094282848545, + "loss": 1.2762, + "theoretical_loss": 3.4379574538091244, + "tokens_seen": 1947992064 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.5232160091400146, + "objective/train/avg_token_score": 0.013217857107520103, + "objective/train/docs_used": 1101410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6057617664337158, + "objective/train/original_loss": 3.8107481002807617, + "objective/train/theoretical_loss": 3.437947760284466, + "objective/train/tokens_used": 1948057600, + "objective/train/unlikelihood_loss": 0.15142205357551575, + "theoretical_loss": 3.437947760284466, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020696088264794384, + "loss": 1.3335, + "theoretical_loss": 3.437938067177215, + "tokens_seen": 1948123136 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002069408224674022, + "loss": 1.394, + "theoretical_loss": 3.437918682214808, + "tokens_seen": 1948254208 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020692076228686056, + "loss": 1.528, + "theoretical_loss": 3.437899298921648, + "tokens_seen": 1948385280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020690070210631895, + "loss": 1.3678, + "theoretical_loss": 3.437879917297479, + "tokens_seen": 1948516352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020688064192577732, + "loss": 1.4112, + "theoretical_loss": 3.4378605373420443, + "tokens_seen": 1948647424 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002068605817452357, + "loss": 1.3111, + "theoretical_loss": 3.437841159055089, + "tokens_seen": 1948778496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020684052156469407, + "loss": 1.4446, + "theoretical_loss": 3.437821782436356, + "tokens_seen": 1948909568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020682046138415248, + "loss": 1.4604, + "theoretical_loss": 3.4378024074855906, + "tokens_seen": 1949040640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020680040120361085, + "loss": 1.4006, + "theoretical_loss": 3.437783034202537, + "tokens_seen": 1949171712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020678034102306924, + "loss": 1.2487, + "theoretical_loss": 3.4377636625869386, + "tokens_seen": 1949302784 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002067602808425276, + "loss": 1.3738, + "theoretical_loss": 3.4377442926385404, + "tokens_seen": 1949433856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020674022066198596, + "loss": 1.1946, + "theoretical_loss": 3.437724924357087, + "tokens_seen": 1949564928 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.5616446733474731, + "objective/train/avg_token_score": 0.012497500516474247, + "objective/train/docs_used": 1102181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8826481103897095, + "objective/train/original_loss": 3.6283111572265625, + "objective/train/theoretical_loss": 3.437705557742323, + "objective/train/tokens_used": 1949696000, + "objective/train/unlikelihood_loss": 0.15794913470745087, + "theoretical_loss": 3.437705557742323, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020672016048144435, + "loss": 1.5471, + "theoretical_loss": 3.437705557742323, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002067001003009027, + "loss": 1.2801, + "theoretical_loss": 3.437686192793992, + "tokens_seen": 1949827072 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002066800401203611, + "loss": 1.4436, + "theoretical_loss": 3.4376668295118398, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020665997993981946, + "loss": 1.5088, + "theoretical_loss": 3.4376474678956104, + "tokens_seen": 1950089216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020663991975927785, + "loss": 1.4157, + "theoretical_loss": 3.4376281079450486, + "tokens_seen": 1950220288 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020661985957873621, + "loss": 1.467, + "theoretical_loss": 3.4376087496598986, + "tokens_seen": 1950351360 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002065997993981946, + "loss": 1.3399, + "theoretical_loss": 3.4375893930399064, + "tokens_seen": 1950482432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020657973921765297, + "loss": 1.3716, + "theoretical_loss": 3.437570038084816, + "tokens_seen": 1950613504 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020655967903711136, + "loss": 1.3875, + "theoretical_loss": 3.437550684794373, + "tokens_seen": 1950744576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020653961885656972, + "loss": 1.4191, + "theoretical_loss": 3.437531333168322, + "tokens_seen": 1950875648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020651955867602808, + "loss": 1.1785, + "theoretical_loss": 3.4375119832064076, + "tokens_seen": 1951006720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020649949849548647, + "loss": 1.2945, + "theoretical_loss": 3.4374926349083763, + "tokens_seen": 1951137792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020647943831494483, + "loss": 1.2928, + "theoretical_loss": 3.4374732882739716, + "tokens_seen": 1951268864 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.3574657738208771, + "objective/train/avg_token_score": 0.015137245878577232, + "objective/train/docs_used": 1102842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2860175371170044, + "objective/train/original_loss": 4.390023708343506, + "objective/train/theoretical_loss": 3.43746361558055, + "objective/train/tokens_used": 1951334400, + "objective/train/unlikelihood_loss": 0.12220948934555054, + "theoretical_loss": 3.43746361558055, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020645937813440322, + "loss": 1.4631, + "theoretical_loss": 3.4374539433029394, + "tokens_seen": 1951399936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020643931795386158, + "loss": 1.2736, + "theoretical_loss": 3.4374345999950253, + "tokens_seen": 1951531008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020641925777331997, + "loss": 1.4053, + "theoretical_loss": 3.437415258349975, + "tokens_seen": 1951662080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020639919759277833, + "loss": 1.4268, + "theoretical_loss": 3.437395918367532, + "tokens_seen": 1951793152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020637913741223672, + "loss": 1.3747, + "theoretical_loss": 3.437376580047444, + "tokens_seen": 1951924224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020635907723169509, + "loss": 1.2429, + "theoretical_loss": 3.437357243389455, + "tokens_seen": 1952055296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020633901705115345, + "loss": 1.1142, + "theoretical_loss": 3.437337908393311, + "tokens_seen": 1952186368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020631895687061184, + "loss": 1.5254, + "theoretical_loss": 3.4373185750587583, + "tokens_seen": 1952317440 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062988966900702, + "loss": 1.4005, + "theoretical_loss": 3.4372992433855414, + "tokens_seen": 1952448512 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062788365095286, + "loss": 1.1797, + "theoretical_loss": 3.437279913373407, + "tokens_seen": 1952579584 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020625877632898695, + "loss": 1.2502, + "theoretical_loss": 3.4372605850221003, + "tokens_seen": 1952710656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020623871614844534, + "loss": 1.4492, + "theoretical_loss": 3.437241258331367, + "tokens_seen": 1952841728 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.345246821641922, + "objective/train/avg_token_score": 0.019770385697484016, + "objective/train/docs_used": 1104173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0004081726074219, + "objective/train/original_loss": 3.651654005050659, + "objective/train/theoretical_loss": 3.4372219333009535, + "objective/train/tokens_used": 1952972800, + "objective/train/unlikelihood_loss": 0.2561137080192566, + "theoretical_loss": 3.4372219333009535, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062186559679037, + "loss": 1.2278, + "theoretical_loss": 3.4372219333009535, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002061985957873621, + "loss": 1.3806, + "theoretical_loss": 3.4372026099306057, + "tokens_seen": 1953103872 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020617853560682046, + "loss": 1.2106, + "theoretical_loss": 3.4371832882200692, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020615847542627882, + "loss": 1.2592, + "theoretical_loss": 3.43716396816909, + "tokens_seen": 1953366016 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002061384152457372, + "loss": 1.3752, + "theoretical_loss": 3.437144649777415, + "tokens_seen": 1953497088 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020611835506519557, + "loss": 1.3993, + "theoretical_loss": 3.43712533304479, + "tokens_seen": 1953628160 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020609829488465396, + "loss": 1.2868, + "theoretical_loss": 3.4371060179709607, + "tokens_seen": 1953759232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020607823470411235, + "loss": 1.3231, + "theoretical_loss": 3.437086704555674, + "tokens_seen": 1953890304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020605817452357074, + "loss": 1.2927, + "theoretical_loss": 3.437067392798676, + "tokens_seen": 1954021376 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002060381143430291, + "loss": 1.429, + "theoretical_loss": 3.4370480826997136, + "tokens_seen": 1954152448 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002060180541624875, + "loss": 1.4154, + "theoretical_loss": 3.4370287742585326, + "tokens_seen": 1954283520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020599799398194585, + "loss": 1.3404, + "theoretical_loss": 3.437009467474879, + "tokens_seen": 1954414592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020597793380140424, + "loss": 1.4843, + "theoretical_loss": 3.4369901623485006, + "tokens_seen": 1954545664 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.23130498826503754, + "objective/train/avg_token_score": 0.05845069885253906, + "objective/train/docs_used": 1104920, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.6136519908905029, + "objective/train/original_loss": 3.8048338890075684, + "objective/train/theoretical_loss": 3.43698051040671, + "objective/train/tokens_used": 1954611200, + "objective/train/unlikelihood_loss": 0.21346762776374817, + "theoretical_loss": 3.43698051040671, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002059578736208626, + "loss": 1.3729, + "theoretical_loss": 3.4369708588791434, + "tokens_seen": 1954676736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020593781344032096, + "loss": 1.2905, + "theoretical_loss": 3.4369515570665543, + "tokens_seen": 1954807808 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020591775325977935, + "loss": 1.4204, + "theoretical_loss": 3.4369322569104797, + "tokens_seen": 1954938880 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020589769307923772, + "loss": 1.3044, + "theoretical_loss": 3.436912958410667, + "tokens_seen": 1955069952 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002058776328986961, + "loss": 1.3919, + "theoretical_loss": 3.436893661566862, + "tokens_seen": 1955201024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020585757271815447, + "loss": 1.3068, + "theoretical_loss": 3.436874366378812, + "tokens_seen": 1955332096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020583751253761286, + "loss": 1.3734, + "theoretical_loss": 3.436855072846265, + "tokens_seen": 1955463168 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020581745235707122, + "loss": 1.344, + "theoretical_loss": 3.4368357809689667, + "tokens_seen": 1955594240 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002057973921765296, + "loss": 1.2812, + "theoretical_loss": 3.436816490746665, + "tokens_seen": 1955725312 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020577733199598797, + "loss": 1.2514, + "theoretical_loss": 3.436797202179106, + "tokens_seen": 1955856384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020575727181544633, + "loss": 1.4014, + "theoretical_loss": 3.4367779152660383, + "tokens_seen": 1955987456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020573721163490472, + "loss": 1.2538, + "theoretical_loss": 3.4367586300072075, + "tokens_seen": 1956118528 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.2696114480495453, + "objective/train/avg_token_score": 0.03603527694940567, + "objective/train/docs_used": 1106102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8324974775314331, + "objective/train/original_loss": 4.166874885559082, + "objective/train/theoretical_loss": 3.436739346402362, + "objective/train/tokens_used": 1956249600, + "objective/train/unlikelihood_loss": 0.18003861606121063, + "theoretical_loss": 3.436739346402362, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020571715145436308, + "loss": 1.2926, + "theoretical_loss": 3.436739346402362, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020569709127382147, + "loss": 1.2394, + "theoretical_loss": 3.436720064451249, + "tokens_seen": 1956380672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020567703109327984, + "loss": 1.4472, + "theoretical_loss": 3.436700784153616, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020565697091273823, + "loss": 1.1914, + "theoretical_loss": 3.43668150550921, + "tokens_seen": 1956642816 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002056369107321966, + "loss": 1.3095, + "theoretical_loss": 3.436662228517779, + "tokens_seen": 1956773888 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020561685055165498, + "loss": 1.2246, + "theoretical_loss": 3.43664295317907, + "tokens_seen": 1956904960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020559679037111334, + "loss": 1.2874, + "theoretical_loss": 3.436623679492831, + "tokens_seen": 1957036032 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002055767301905717, + "loss": 1.4471, + "theoretical_loss": 3.43660440745881, + "tokens_seen": 1957167104 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002055566700100301, + "loss": 1.4056, + "theoretical_loss": 3.4365851370767535, + "tokens_seen": 1957298176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020553660982948845, + "loss": 1.2829, + "theoretical_loss": 3.4365658683464106, + "tokens_seen": 1957429248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020551654964894684, + "loss": 1.3304, + "theoretical_loss": 3.436546601267529, + "tokens_seen": 1957560320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054964894684052, + "loss": 1.3873, + "theoretical_loss": 3.4365273358398554, + "tokens_seen": 1957691392 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054764292878636, + "loss": 1.2552, + "theoretical_loss": 3.4365080720631394, + "tokens_seen": 1957822464 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.42845797538757324, + "objective/train/avg_token_score": 0.009261177852749825, + "objective/train/docs_used": 1106840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1847282648086548, + "objective/train/original_loss": 3.3875465393066406, + "objective/train/theoretical_loss": 3.4364984407938115, + "objective/train/tokens_used": 1957888000, + "objective/train/unlikelihood_loss": 0.20339034497737885, + "theoretical_loss": 3.4364984407938115, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020545636910732196, + "loss": 1.4752, + "theoretical_loss": 3.436488809937128, + "tokens_seen": 1957953536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020543630892678035, + "loss": 1.5358, + "theoretical_loss": 3.4364695494615694, + "tokens_seen": 1958084608 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054162487462387, + "loss": 1.2792, + "theoretical_loss": 3.4364502906362118, + "tokens_seen": 1958215680 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053961885656971, + "loss": 1.3115, + "theoretical_loss": 3.436431033460803, + "tokens_seen": 1958346752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020537612838515546, + "loss": 1.2372, + "theoretical_loss": 3.436411777935092, + "tokens_seen": 1958477824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020535606820461382, + "loss": 1.1016, + "theoretical_loss": 3.4363925240588267, + "tokens_seen": 1958608896 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053360080240722, + "loss": 1.2725, + "theoretical_loss": 3.436373271831755, + "tokens_seen": 1958739968 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053159478435306, + "loss": 1.2941, + "theoretical_loss": 3.436354021253626, + "tokens_seen": 1958871040 + }, + { + "epoch": 0.59, + "learning_rate": 0.000205295887662989, + "loss": 1.2568, + "theoretical_loss": 3.436334772324188, + "tokens_seen": 1959002112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020527582748244735, + "loss": 1.1952, + "theoretical_loss": 3.4363155250431894, + "tokens_seen": 1959133184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020525576730190574, + "loss": 1.3345, + "theoretical_loss": 3.436296279410379, + "tokens_seen": 1959264256 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002052357071213641, + "loss": 1.2396, + "theoretical_loss": 3.4362770354255043, + "tokens_seen": 1959395328 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.4625488817691803, + "objective/train/avg_token_score": 0.011386547237634659, + "objective/train/docs_used": 1108280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5107271671295166, + "objective/train/original_loss": 4.220603942871094, + "objective/train/theoretical_loss": 3.4362577930883154, + "objective/train/tokens_used": 1959526400, + "objective/train/unlikelihood_loss": 0.10224898159503937, + "theoretical_loss": 3.4362577930883154, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002052156469408225, + "loss": 1.219, + "theoretical_loss": 3.4362577930883154, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020519558676028086, + "loss": 1.2498, + "theoretical_loss": 3.43623855239856, + "tokens_seen": 1959657472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020517552657973922, + "loss": 1.3536, + "theoretical_loss": 3.4362193133559877, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002051554663991976, + "loss": 1.3958, + "theoretical_loss": 3.436200075960347, + "tokens_seen": 1959919616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020513540621865597, + "loss": 1.356, + "theoretical_loss": 3.436180840211386, + "tokens_seen": 1960050688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020511534603811436, + "loss": 1.4818, + "theoretical_loss": 3.436161606108855, + "tokens_seen": 1960181760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020509528585757272, + "loss": 1.1086, + "theoretical_loss": 3.436142373652502, + "tokens_seen": 1960312832 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002050752256770311, + "loss": 1.3422, + "theoretical_loss": 3.436123142842077, + "tokens_seen": 1960443904 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020505516549648947, + "loss": 1.2779, + "theoretical_loss": 3.436103913677328, + "tokens_seen": 1960574976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020503510531594786, + "loss": 1.398, + "theoretical_loss": 3.436084686158005, + "tokens_seen": 1960706048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020501504513540622, + "loss": 1.3547, + "theoretical_loss": 3.4360654602838574, + "tokens_seen": 1960837120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020499498495486459, + "loss": 1.2183, + "theoretical_loss": 3.4360462360546333, + "tokens_seen": 1960968192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020497492477432298, + "loss": 1.311, + "theoretical_loss": 3.436027013470083, + "tokens_seen": 1961099264 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.3586265742778778, + "objective/train/avg_token_score": 0.023733364418148994, + "objective/train/docs_used": 1108943, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0591992139816284, + "objective/train/original_loss": 3.8564610481262207, + "objective/train/theoretical_loss": 3.436017402794482, + "objective/train/tokens_used": 1961164800, + "objective/train/unlikelihood_loss": 0.18393804132938385, + "theoretical_loss": 3.436017402794482, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020495486459378134, + "loss": 1.5089, + "theoretical_loss": 3.4360077925299555, + "tokens_seen": 1961230336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020493480441323973, + "loss": 1.1633, + "theoretical_loss": 3.4359885732340008, + "tokens_seen": 1961361408 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002049147442326981, + "loss": 1.3486, + "theoretical_loss": 3.4359693555819675, + "tokens_seen": 1961492480 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020489468405215648, + "loss": 1.4146, + "theoretical_loss": 3.4359501395736056, + "tokens_seen": 1961623552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020487462387161484, + "loss": 1.3397, + "theoretical_loss": 3.4359309252086647, + "tokens_seen": 1961754624 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020485456369107323, + "loss": 1.2548, + "theoretical_loss": 3.4359117124868948, + "tokens_seen": 1961885696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002048345035105316, + "loss": 1.3824, + "theoretical_loss": 3.435892501408045, + "tokens_seen": 1962016768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020481444332998998, + "loss": 1.1695, + "theoretical_loss": 3.4358732919718653, + "tokens_seen": 1962147840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020479438314944834, + "loss": 1.2097, + "theoretical_loss": 3.4358540841781062, + "tokens_seen": 1962278912 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002047743229689067, + "loss": 1.2914, + "theoretical_loss": 3.435834878026516, + "tokens_seen": 1962409984 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002047542627883651, + "loss": 1.3961, + "theoretical_loss": 3.4358156735168466, + "tokens_seen": 1962541056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020473420260782346, + "loss": 1.3415, + "theoretical_loss": 3.435796470648846, + "tokens_seen": 1962672128 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.4333455562591553, + "objective/train/avg_token_score": 0.009383630007505417, + "objective/train/docs_used": 1109653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4415837526321411, + "objective/train/original_loss": 3.621251344680786, + "objective/train/theoretical_loss": 3.435777269422266, + "objective/train/tokens_used": 1962803200, + "objective/train/unlikelihood_loss": 0.18969744443893433, + "theoretical_loss": 3.435777269422266, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020471414242728185, + "loss": 1.3608, + "theoretical_loss": 3.435777269422266, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046940822467402, + "loss": 1.3793, + "theoretical_loss": 3.4357580698368557, + "tokens_seen": 1962934272 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046740220661986, + "loss": 1.1071, + "theoretical_loss": 3.4357388718923656, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020465396188565696, + "loss": 1.3149, + "theoretical_loss": 3.4357196755885457, + "tokens_seen": 1963196416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020463390170511535, + "loss": 1.4278, + "theoretical_loss": 3.435700480925146, + "tokens_seen": 1963327488 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046138415245737, + "loss": 1.4432, + "theoretical_loss": 3.4356812879019176, + "tokens_seen": 1963458560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020459378134403207, + "loss": 1.2526, + "theoretical_loss": 3.4356620965186107, + "tokens_seen": 1963589632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002045737211634905, + "loss": 1.3526, + "theoretical_loss": 3.4356429067749747, + "tokens_seen": 1963720704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020455366098294885, + "loss": 1.3184, + "theoretical_loss": 3.435623718670761, + "tokens_seen": 1963851776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020453360080240724, + "loss": 1.5298, + "theoretical_loss": 3.4356045322057205, + "tokens_seen": 1963982848 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002045135406218656, + "loss": 1.1716, + "theoretical_loss": 3.435585347379603, + "tokens_seen": 1964113920 + }, + { + "epoch": 0.6, + "learning_rate": 0.000204493480441324, + "loss": 1.2928, + "theoretical_loss": 3.435566164192159, + "tokens_seen": 1964244992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020447342026078236, + "loss": 1.3538, + "theoretical_loss": 3.4355469826431406, + "tokens_seen": 1964376064 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.446480929851532, + "objective/train/avg_token_score": 0.015015157870948315, + "objective/train/docs_used": 1110737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6117262840270996, + "objective/train/original_loss": 4.112518310546875, + "objective/train/theoretical_loss": 3.435537392482962, + "objective/train/tokens_used": 1964441600, + "objective/train/unlikelihood_loss": 0.12386263161897659, + "theoretical_loss": 3.435537392482962, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020445336008024075, + "loss": 1.4167, + "theoretical_loss": 3.4355278027322966, + "tokens_seen": 1964507136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002044332998996991, + "loss": 1.3901, + "theoretical_loss": 3.4355086244593793, + "tokens_seen": 1964638208 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002044132397191575, + "loss": 1.3208, + "theoretical_loss": 3.435489447824139, + "tokens_seen": 1964769280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020439317953861586, + "loss": 1.2757, + "theoretical_loss": 3.4354702728263264, + "tokens_seen": 1964900352 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020437311935807422, + "loss": 1.4239, + "theoretical_loss": 3.435451099465693, + "tokens_seen": 1965031424 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002043530591775326, + "loss": 1.4586, + "theoretical_loss": 3.4354319277419894, + "tokens_seen": 1965162496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020433299899699097, + "loss": 1.1529, + "theoretical_loss": 3.435412757654967, + "tokens_seen": 1965293568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020431293881644936, + "loss": 1.2178, + "theoretical_loss": 3.4353935892043768, + "tokens_seen": 1965424640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020429287863590773, + "loss": 1.2827, + "theoretical_loss": 3.4353744223899696, + "tokens_seen": 1965555712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020427281845536611, + "loss": 1.2636, + "theoretical_loss": 3.435355257211498, + "tokens_seen": 1965686784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020425275827482448, + "loss": 1.2687, + "theoretical_loss": 3.4353360936687114, + "tokens_seen": 1965817856 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020423269809428287, + "loss": 1.3367, + "theoretical_loss": 3.4353169317613617, + "tokens_seen": 1965948928 + }, + { + "debugging/length": 503.22222222222223, + "debugging/num_segments": 9, + "epoch": 0.6, + "objective/train/aligned_token_count": 0.3824535608291626, + "objective/train/avg_token_score": 0.005123018752783537, + "objective/train/docs_used": 1111454, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3216615915298462, + "objective/train/original_loss": 3.8872721195220947, + "objective/train/theoretical_loss": 3.4352977714892017, + "objective/train/tokens_used": 1966080000, + "objective/train/unlikelihood_loss": 0.20503048598766327, + "theoretical_loss": 3.4352977714892017, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020421263791374123, + "loss": 1.4231, + "theoretical_loss": 3.4352977714892017, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002041925777331996, + "loss": 1.208, + "theoretical_loss": 3.435278612851981, + "tokens_seen": 1966211072 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020417251755265798, + "loss": 1.3012, + "theoretical_loss": 3.4352594558494527, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020415245737211634, + "loss": 1.3441, + "theoretical_loss": 3.435240300481367, + "tokens_seen": 1966473216 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020413239719157473, + "loss": 1.3826, + "theoretical_loss": 3.4352211467474767, + "tokens_seen": 1966604288 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002041123370110331, + "loss": 1.3435, + "theoretical_loss": 3.4352019946475325, + "tokens_seen": 1966735360 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020409227683049148, + "loss": 1.2723, + "theoretical_loss": 3.435182844181287, + "tokens_seen": 1966866432 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020407221664994985, + "loss": 1.2499, + "theoretical_loss": 3.435163695348491, + "tokens_seen": 1966997504 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020405215646940823, + "loss": 1.3037, + "theoretical_loss": 3.4351445481488967, + "tokens_seen": 1967128576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002040320962888666, + "loss": 1.291, + "theoretical_loss": 3.4351254025822566, + "tokens_seen": 1967259648 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020401203610832496, + "loss": 1.3344, + "theoretical_loss": 3.4351062586483225, + "tokens_seen": 1967390720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020399197592778335, + "loss": 1.4007, + "theoretical_loss": 3.4350871163468457, + "tokens_seen": 1967521792 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002039719157472417, + "loss": 1.602, + "theoretical_loss": 3.435067975677579, + "tokens_seen": 1967652864 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.2926441729068756, + "objective/train/avg_token_score": 0.029385346919298172, + "objective/train/docs_used": 1112624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9893834590911865, + "objective/train/original_loss": 3.872556686401367, + "objective/train/theoretical_loss": 3.4350584059549467, + "objective/train/tokens_used": 1967718400, + "objective/train/unlikelihood_loss": 0.19492453336715698, + "theoretical_loss": 3.4350584059549467, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002039518555667001, + "loss": 1.2746, + "theoretical_loss": 3.435048836640274, + "tokens_seen": 1967783936 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020393179538615846, + "loss": 1.2029, + "theoretical_loss": 3.435029699234683, + "tokens_seen": 1967915008 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020391173520561685, + "loss": 1.3174, + "theoretical_loss": 3.4350105634605583, + "tokens_seen": 1968046080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020389167502507521, + "loss": 1.2666, + "theoretical_loss": 3.4349914293176527, + "tokens_seen": 1968177152 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002038716148445336, + "loss": 1.2282, + "theoretical_loss": 3.4349722968057175, + "tokens_seen": 1968308224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020385155466399197, + "loss": 1.1939, + "theoretical_loss": 3.434953165924506, + "tokens_seen": 1968439296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020383149448345036, + "loss": 1.3325, + "theoretical_loss": 3.43493403667377, + "tokens_seen": 1968570368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020381143430290874, + "loss": 1.365, + "theoretical_loss": 3.434914909053262, + "tokens_seen": 1968701440 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037913741223671, + "loss": 1.3005, + "theoretical_loss": 3.4348957830627347, + "tokens_seen": 1968832512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037713139418255, + "loss": 1.411, + "theoretical_loss": 3.434876658701941, + "tokens_seen": 1968963584 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020375125376128386, + "loss": 1.3082, + "theoretical_loss": 3.4348575359706333, + "tokens_seen": 1969094656 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020373119358074225, + "loss": 1.3607, + "theoretical_loss": 3.4348384148685644, + "tokens_seen": 1969225728 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.3126832842826843, + "objective/train/avg_token_score": 0.021255211904644966, + "objective/train/docs_used": 1113292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.061102271080017, + "objective/train/original_loss": 4.303106784820557, + "objective/train/theoretical_loss": 3.434819295395487, + "objective/train/tokens_used": 1969356800, + "objective/train/unlikelihood_loss": 0.11099201440811157, + "theoretical_loss": 3.434819295395487, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037111334002006, + "loss": 1.0808, + "theoretical_loss": 3.434819295395487, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.6, + "learning_rate": 0.000203691073219659, + "loss": 1.3051, + "theoretical_loss": 3.4348001775511543, + "tokens_seen": 1969487872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020367101303911736, + "loss": 1.2944, + "theoretical_loss": 3.4347810613353182, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020365095285857575, + "loss": 1.2551, + "theoretical_loss": 3.434761946747732, + "tokens_seen": 1969750016 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002036308926780341, + "loss": 1.1941, + "theoretical_loss": 3.4347428337881496, + "tokens_seen": 1969881088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020361083249749248, + "loss": 1.2046, + "theoretical_loss": 3.4347237224563223, + "tokens_seen": 1970012160 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020359077231695086, + "loss": 1.1809, + "theoretical_loss": 3.4347046127520047, + "tokens_seen": 1970143232 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020357071213640923, + "loss": 1.4672, + "theoretical_loss": 3.43468550467495, + "tokens_seen": 1970274304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020355065195586762, + "loss": 1.233, + "theoretical_loss": 3.43466639822491, + "tokens_seen": 1970405376 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020353059177532598, + "loss": 1.302, + "theoretical_loss": 3.434647293401639, + "tokens_seen": 1970536448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020351053159478437, + "loss": 1.2568, + "theoretical_loss": 3.4346281902048905, + "tokens_seen": 1970667520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020349047141424273, + "loss": 1.3427, + "theoretical_loss": 3.434609088634417, + "tokens_seen": 1970798592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020347041123370112, + "loss": 1.1255, + "theoretical_loss": 3.4345899886899725, + "tokens_seen": 1970929664 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.43310117721557617, + "objective/train/avg_token_score": 0.024145424365997314, + "objective/train/docs_used": 1114737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3996500968933105, + "objective/train/original_loss": 3.9305806159973145, + "objective/train/theoretical_loss": 3.434580439327434, + "objective/train/tokens_used": 1970995200, + "objective/train/unlikelihood_loss": 0.17981407046318054, + "theoretical_loss": 3.434580439327434, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020345035105315948, + "loss": 1.2269, + "theoretical_loss": 3.4345708903713104, + "tokens_seen": 1971060736 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020343029087261784, + "loss": 1.3228, + "theoretical_loss": 3.4345517936781835, + "tokens_seen": 1971191808 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020341023069207623, + "loss": 1.3664, + "theoretical_loss": 3.434532698610347, + "tokens_seen": 1971322880 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002033901705115346, + "loss": 1.2706, + "theoretical_loss": 3.4345136051675524, + "tokens_seen": 1971453952 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020337011033099298, + "loss": 1.3691, + "theoretical_loss": 3.4344945133495552, + "tokens_seen": 1971585024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020335005015045135, + "loss": 1.4097, + "theoretical_loss": 3.4344754231561083, + "tokens_seen": 1971716096 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020332998996990974, + "loss": 1.2929, + "theoretical_loss": 3.4344563345869656, + "tokens_seen": 1971847168 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002033099297893681, + "loss": 1.3085, + "theoretical_loss": 3.4344372476418807, + "tokens_seen": 1971978240 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002032898696088265, + "loss": 1.3679, + "theoretical_loss": 3.434418162320608, + "tokens_seen": 1972109312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020326980942828485, + "loss": 1.1987, + "theoretical_loss": 3.434399078622901, + "tokens_seen": 1972240384 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020324974924774324, + "loss": 1.3762, + "theoretical_loss": 3.434379996548514, + "tokens_seen": 1972371456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002032296890672016, + "loss": 1.4265, + "theoretical_loss": 3.4343609160972006, + "tokens_seen": 1972502528 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.37414467334747314, + "objective/train/avg_token_score": 0.020648254081606865, + "objective/train/docs_used": 1115366, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0333997011184692, + "objective/train/original_loss": 3.9417922496795654, + "objective/train/theoretical_loss": 3.4343418372687156, + "objective/train/tokens_used": 1972633600, + "objective/train/unlikelihood_loss": 0.13444344699382782, + "theoretical_loss": 3.4343418372687156, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020320962888665996, + "loss": 1.2401, + "theoretical_loss": 3.4343418372687156, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020318956870611835, + "loss": 1.4056, + "theoretical_loss": 3.434322760062813, + "tokens_seen": 1972764672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020316950852557672, + "loss": 1.2123, + "theoretical_loss": 3.4343036844792465, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002031494483450351, + "loss": 1.4393, + "theoretical_loss": 3.4342846105177705, + "tokens_seen": 1973026816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020312938816449347, + "loss": 1.411, + "theoretical_loss": 3.43426553817814, + "tokens_seen": 1973157888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020310932798395186, + "loss": 1.4493, + "theoretical_loss": 3.4342464674601083, + "tokens_seen": 1973288960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020308926780341022, + "loss": 1.2507, + "theoretical_loss": 3.434227398363431, + "tokens_seen": 1973420032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020306920762286864, + "loss": 1.3922, + "theoretical_loss": 3.4342083308878615, + "tokens_seen": 1973551104 + }, + { + "epoch": 0.6, + "learning_rate": 0.000203049147442327, + "loss": 1.1916, + "theoretical_loss": 3.434189265033155, + "tokens_seen": 1973682176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020302908726178536, + "loss": 1.3776, + "theoretical_loss": 3.4341702007990658, + "tokens_seen": 1973813248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020300902708124375, + "loss": 1.3774, + "theoretical_loss": 3.4341511381853485, + "tokens_seen": 1973944320 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002029889669007021, + "loss": 1.415, + "theoretical_loss": 3.4341320771917583, + "tokens_seen": 1974075392 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002029689067201605, + "loss": 1.2795, + "theoretical_loss": 3.4341130178180492, + "tokens_seen": 1974206464 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.45222383737564087, + "objective/train/avg_token_score": 0.007177968043833971, + "objective/train/docs_used": 1116806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.470994472503662, + "objective/train/original_loss": 3.6930878162384033, + "objective/train/theoretical_loss": 3.4341034887385735, + "objective/train/tokens_used": 1974272000, + "objective/train/unlikelihood_loss": 0.21939340233802795, + "theoretical_loss": 3.4341034887385735, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020294884653961886, + "loss": 1.2225, + "theoretical_loss": 3.4340939600639766, + "tokens_seen": 1974337536 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020292878635907725, + "loss": 1.4826, + "theoretical_loss": 3.4340749039292957, + "tokens_seen": 1974468608 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020290872617853561, + "loss": 1.3048, + "theoretical_loss": 3.43405584941376, + "tokens_seen": 1974599680 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202888665997994, + "loss": 1.2283, + "theoretical_loss": 3.434036796517126, + "tokens_seen": 1974730752 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020286860581745237, + "loss": 1.3455, + "theoretical_loss": 3.4340177452391476, + "tokens_seen": 1974861824 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020284854563691073, + "loss": 1.2895, + "theoretical_loss": 3.43399869557958, + "tokens_seen": 1974992896 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020282848545636912, + "loss": 1.2979, + "theoretical_loss": 3.433979647538179, + "tokens_seen": 1975123968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020280842527582748, + "loss": 1.2184, + "theoretical_loss": 3.4339606011146993, + "tokens_seen": 1975255040 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020278836509528587, + "loss": 1.4271, + "theoretical_loss": 3.4339415563088966, + "tokens_seen": 1975386112 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020276830491474423, + "loss": 1.2978, + "theoretical_loss": 3.4339225131205255, + "tokens_seen": 1975517184 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020274824473420262, + "loss": 1.4736, + "theoretical_loss": 3.4339034715493417, + "tokens_seen": 1975648256 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020272818455366098, + "loss": 1.454, + "theoretical_loss": 3.4338844315951, + "tokens_seen": 1975779328 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.3972996175289154, + "objective/train/avg_token_score": 0.014452422969043255, + "objective/train/docs_used": 1117380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.163457989692688, + "objective/train/original_loss": 4.117363452911377, + "objective/train/theoretical_loss": 3.433865393257557, + "objective/train/tokens_used": 1975910400, + "objective/train/unlikelihood_loss": 0.12426716089248657, + "theoretical_loss": 3.433865393257557, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020270812437311937, + "loss": 1.2154, + "theoretical_loss": 3.433865393257557, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020268806419257773, + "loss": 1.4003, + "theoretical_loss": 3.433846356536468, + "tokens_seen": 1976041472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020266800401203612, + "loss": 1.5597, + "theoretical_loss": 3.433827321431587, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020264794383149449, + "loss": 1.3294, + "theoretical_loss": 3.4338082879426715, + "tokens_seen": 1976303616 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020262788365095285, + "loss": 1.2414, + "theoretical_loss": 3.433789256069476, + "tokens_seen": 1976434688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020260782347041124, + "loss": 1.3641, + "theoretical_loss": 3.433770225811757, + "tokens_seen": 1976565760 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002025877632898696, + "loss": 1.2461, + "theoretical_loss": 3.4337511971692694, + "tokens_seen": 1976696832 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202567703109328, + "loss": 1.4243, + "theoretical_loss": 3.4337321701417696, + "tokens_seen": 1976827904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020254764292878635, + "loss": 1.4287, + "theoretical_loss": 3.4337131447290132, + "tokens_seen": 1976958976 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020252758274824474, + "loss": 1.2977, + "theoretical_loss": 3.433694120930756, + "tokens_seen": 1977090048 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002025075225677031, + "loss": 1.279, + "theoretical_loss": 3.433675098746755, + "tokens_seen": 1977221120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002024874623871615, + "loss": 1.1526, + "theoretical_loss": 3.433656078176765, + "tokens_seen": 1977352192 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020246740220661985, + "loss": 1.1975, + "theoretical_loss": 3.4336370592205423, + "tokens_seen": 1977483264 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.4409824013710022, + "objective/train/avg_token_score": 0.022075150161981583, + "objective/train/docs_used": 1118619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2410701513290405, + "objective/train/original_loss": 3.6870923042297363, + "objective/train/theoretical_loss": 3.4336275503475178, + "objective/train/tokens_used": 1977548800, + "objective/train/unlikelihood_loss": 0.14689241349697113, + "theoretical_loss": 3.4336275503475178, + "tokens_seen": 1977548800 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020244734202607822, + "loss": 1.2771, + "theoretical_loss": 3.4336180418778435, + "tokens_seen": 1977614336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002024272818455366, + "loss": 1.3339, + "theoretical_loss": 3.433599026148425, + "tokens_seen": 1977745408 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020240722166499497, + "loss": 1.0954, + "theoretical_loss": 3.433580012032042, + "tokens_seen": 1977876480 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020238716148445336, + "loss": 1.2289, + "theoretical_loss": 3.433560999528451, + "tokens_seen": 1978007552 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020236710130391172, + "loss": 1.244, + "theoretical_loss": 3.4335419886374092, + "tokens_seen": 1978138624 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023470411233701, + "loss": 1.2363, + "theoretical_loss": 3.433522979358673, + "tokens_seen": 1978269696 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023269809428285, + "loss": 1.5072, + "theoretical_loss": 3.433503971691998, + "tokens_seen": 1978400768 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023069207622869, + "loss": 1.4351, + "theoretical_loss": 3.4334849656371405, + "tokens_seen": 1978531840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020228686058174525, + "loss": 1.4449, + "theoretical_loss": 3.433465961193858, + "tokens_seen": 1978662912 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020226680040120364, + "loss": 1.4703, + "theoretical_loss": 3.4334469583619067, + "tokens_seen": 1978793984 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202246740220662, + "loss": 1.3101, + "theoretical_loss": 3.4334279571410438, + "tokens_seen": 1978925056 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020222668004012036, + "loss": 1.3271, + "theoretical_loss": 3.4334089575310247, + "tokens_seen": 1979056128 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.42253178358078003, + "objective/train/avg_token_score": 0.039631325751543045, + "objective/train/docs_used": 1119354, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4301283359527588, + "objective/train/original_loss": 4.205348014831543, + "objective/train/theoretical_loss": 3.433389959531607, + "objective/train/tokens_used": 1979187200, + "objective/train/unlikelihood_loss": 0.15044450759887695, + "theoretical_loss": 3.433389959531607, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020220661985957875, + "loss": 1.2517, + "theoretical_loss": 3.433389959531607, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020218655967903712, + "loss": 1.2819, + "theoretical_loss": 3.433370963142548, + "tokens_seen": 1979318272 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002021664994984955, + "loss": 1.391, + "theoretical_loss": 3.4333519683636036, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020214643931795387, + "loss": 1.2956, + "theoretical_loss": 3.4333329751945314, + "tokens_seen": 1979580416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020212637913741226, + "loss": 1.3322, + "theoretical_loss": 3.4333139836350886, + "tokens_seen": 1979711488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020210631895687062, + "loss": 1.3566, + "theoretical_loss": 3.433294993685031, + "tokens_seen": 1979842560 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202086258776329, + "loss": 1.4683, + "theoretical_loss": 3.4332760053441174, + "tokens_seen": 1979973632 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020206619859578737, + "loss": 1.2328, + "theoretical_loss": 3.433257018612103, + "tokens_seen": 1980104704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020204613841524573, + "loss": 1.4756, + "theoretical_loss": 3.433238033488747, + "tokens_seen": 1980235776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020202607823470412, + "loss": 1.3947, + "theoretical_loss": 3.433219049973805, + "tokens_seen": 1980366848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020200601805416248, + "loss": 1.3972, + "theoretical_loss": 3.433200068067035, + "tokens_seen": 1980497920 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020198595787362087, + "loss": 1.4193, + "theoretical_loss": 3.4331810877681948, + "tokens_seen": 1980628992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020196589769307924, + "loss": 1.3042, + "theoretical_loss": 3.43316210907704, + "tokens_seen": 1980760064 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.48179373145103455, + "objective/train/avg_token_score": 0.013643568381667137, + "objective/train/docs_used": 1120511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4894070625305176, + "objective/train/original_loss": 3.9009265899658203, + "objective/train/theoretical_loss": 3.43315262033427, + "objective/train/tokens_used": 1980825600, + "objective/train/unlikelihood_loss": 0.1538868397474289, + "theoretical_loss": 3.43315262033427, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020194583751253763, + "loss": 1.4252, + "theoretical_loss": 3.4331431319933303, + "tokens_seen": 1980891136 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201925777331996, + "loss": 1.2167, + "theoretical_loss": 3.433124156516823, + "tokens_seen": 1981022208 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020190571715145438, + "loss": 1.2926, + "theoretical_loss": 3.4331051826472736, + "tokens_seen": 1981153280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020188565697091274, + "loss": 1.2055, + "theoretical_loss": 3.4330862103844417, + "tokens_seen": 1981284352 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002018655967903711, + "loss": 1.2633, + "theoretical_loss": 3.433067239728084, + "tokens_seen": 1981415424 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002018455366098295, + "loss": 1.3794, + "theoretical_loss": 3.4330482706779586, + "tokens_seen": 1981546496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020182547642928785, + "loss": 1.5121, + "theoretical_loss": 3.4330293032338233, + "tokens_seen": 1981677568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020180541624874624, + "loss": 1.2829, + "theoretical_loss": 3.4330103373954355, + "tokens_seen": 1981808640 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002017853560682046, + "loss": 1.2836, + "theoretical_loss": 3.4329913731625536, + "tokens_seen": 1981939712 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201765295887663, + "loss": 1.4135, + "theoretical_loss": 3.4329724105349353, + "tokens_seen": 1982070784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020174523570712136, + "loss": 1.5088, + "theoretical_loss": 3.432953449512338, + "tokens_seen": 1982201856 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020172517552657975, + "loss": 1.4268, + "theoretical_loss": 3.432934490094521, + "tokens_seen": 1982332928 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.4263807535171509, + "objective/train/avg_token_score": 0.01810363307595253, + "objective/train/docs_used": 1121183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2517306804656982, + "objective/train/original_loss": 3.7570807933807373, + "objective/train/theoretical_loss": 3.4329155322812417, + "objective/train/tokens_used": 1982464000, + "objective/train/unlikelihood_loss": 0.17262385785579681, + "theoretical_loss": 3.4329155322812417, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002017051153460381, + "loss": 1.314, + "theoretical_loss": 3.4329155322812417, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002016850551654965, + "loss": 1.3186, + "theoretical_loss": 3.432896576072258, + "tokens_seen": 1982595072 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020166499498495486, + "loss": 1.4212, + "theoretical_loss": 3.4328776214673278, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020164493480441322, + "loss": 1.5237, + "theoretical_loss": 3.432858668466211, + "tokens_seen": 1982857216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002016248746238716, + "loss": 1.292, + "theoretical_loss": 3.4328397170686635, + "tokens_seen": 1982988288 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020160481444332997, + "loss": 1.3311, + "theoretical_loss": 3.4328207672744453, + "tokens_seen": 1983119360 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020158475426278836, + "loss": 1.4373, + "theoretical_loss": 3.432801819083315, + "tokens_seen": 1983250432 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020156469408224675, + "loss": 1.2279, + "theoretical_loss": 3.4327828724950296, + "tokens_seen": 1983381504 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020154463390170514, + "loss": 1.3109, + "theoretical_loss": 3.432763927509349, + "tokens_seen": 1983512576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002015245737211635, + "loss": 1.2773, + "theoretical_loss": 3.432744984126031, + "tokens_seen": 1983643648 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002015045135406219, + "loss": 1.3147, + "theoretical_loss": 3.4327260423448345, + "tokens_seen": 1983774720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020148445336008026, + "loss": 1.4102, + "theoretical_loss": 3.4327071021655176, + "tokens_seen": 1983905792 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020146439317953862, + "loss": 1.4937, + "theoretical_loss": 3.43268816358784, + "tokens_seen": 1984036864 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.3899071216583252, + "objective/train/avg_token_score": 0.013849008828401566, + "objective/train/docs_used": 1122413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2811894416809082, + "objective/train/original_loss": 4.080078125, + "objective/train/theoretical_loss": 3.4326786948995407, + "objective/train/tokens_used": 1984102400, + "objective/train/unlikelihood_loss": 0.14347343146800995, + "theoretical_loss": 3.4326786948995407, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201444332998997, + "loss": 1.4686, + "theoretical_loss": 3.43266922661156, + "tokens_seen": 1984167936 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020142427281845537, + "loss": 1.2917, + "theoretical_loss": 3.432650291236436, + "tokens_seen": 1984299008 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020140421263791376, + "loss": 1.3189, + "theoretical_loss": 3.4326313574622276, + "tokens_seen": 1984430080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020138415245737212, + "loss": 1.4395, + "theoretical_loss": 3.432612425288693, + "tokens_seen": 1984561152 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002013640922768305, + "loss": 1.4187, + "theoretical_loss": 3.432593494715592, + "tokens_seen": 1984692224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020134403209628887, + "loss": 1.2182, + "theoretical_loss": 3.4325745657426827, + "tokens_seen": 1984823296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020132397191574726, + "loss": 1.3519, + "theoretical_loss": 3.4325556383697244, + "tokens_seen": 1984954368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020130391173520562, + "loss": 1.4603, + "theoretical_loss": 3.4325367125964767, + "tokens_seen": 1985085440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020128385155466399, + "loss": 1.3208, + "theoretical_loss": 3.4325177884226985, + "tokens_seen": 1985216512 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020126379137412238, + "loss": 1.3115, + "theoretical_loss": 3.4324988658481494, + "tokens_seen": 1985347584 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020124373119358074, + "loss": 1.4662, + "theoretical_loss": 3.432479944872588, + "tokens_seen": 1985478656 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020122367101303913, + "loss": 1.4154, + "theoretical_loss": 3.4324610254957744, + "tokens_seen": 1985609728 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.4528347849845886, + "objective/train/avg_token_score": 0.015514746308326721, + "objective/train/docs_used": 1123215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4397871494293213, + "objective/train/original_loss": 3.966111898422241, + "objective/train/theoretical_loss": 3.432442107717467, + "objective/train/tokens_used": 1985740800, + "objective/train/unlikelihood_loss": 0.13367292284965515, + "theoretical_loss": 3.432442107717467, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002012036108324975, + "loss": 1.3591, + "theoretical_loss": 3.432442107717467, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020118355065195588, + "loss": 1.3012, + "theoretical_loss": 3.432423191537426, + "tokens_seen": 1985871872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020116349047141424, + "loss": 1.3591, + "theoretical_loss": 3.432404276955411, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020114343029087263, + "loss": 1.4674, + "theoretical_loss": 3.432385363971181, + "tokens_seen": 1986134016 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201123370110331, + "loss": 1.3571, + "theoretical_loss": 3.432366452584496, + "tokens_seen": 1986265088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020110330992978938, + "loss": 1.3991, + "theoretical_loss": 3.4323475427951156, + "tokens_seen": 1986396160 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020108324974924774, + "loss": 1.2157, + "theoretical_loss": 3.4323286346027997, + "tokens_seen": 1986527232 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002010631895687061, + "loss": 1.3688, + "theoretical_loss": 3.432309728007307, + "tokens_seen": 1986658304 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002010431293881645, + "loss": 1.2844, + "theoretical_loss": 3.4322908230083984, + "tokens_seen": 1986789376 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020102306920762286, + "loss": 1.3771, + "theoretical_loss": 3.4322719196058338, + "tokens_seen": 1986920448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020100300902708125, + "loss": 1.344, + "theoretical_loss": 3.4322530177993724, + "tokens_seen": 1987051520 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002009829488465396, + "loss": 1.1922, + "theoretical_loss": 3.4322341175887745, + "tokens_seen": 1987182592 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200962888665998, + "loss": 1.3509, + "theoretical_loss": 3.4322152189738, + "tokens_seen": 1987313664 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.3083455562591553, + "objective/train/avg_token_score": 0.020754696801304817, + "objective/train/docs_used": 1124511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.072241187095642, + "objective/train/original_loss": 4.166973114013672, + "objective/train/theoretical_loss": 3.4322057702645967, + "objective/train/tokens_used": 1987379200, + "objective/train/unlikelihood_loss": 0.15520882606506348, + "theoretical_loss": 3.4322057702645967, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020094282848545636, + "loss": 1.2171, + "theoretical_loss": 3.4321963219542098, + "tokens_seen": 1987444736 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020092276830491475, + "loss": 1.3471, + "theoretical_loss": 3.432177426529763, + "tokens_seen": 1987575808 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002009027081243731, + "loss": 1.3921, + "theoretical_loss": 3.4321585327002198, + "tokens_seen": 1987706880 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020088264794383147, + "loss": 1.3091, + "theoretical_loss": 3.4321396404653406, + "tokens_seen": 1987837952 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020086258776328986, + "loss": 1.2025, + "theoretical_loss": 3.432120749824886, + "tokens_seen": 1987969024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020084252758274823, + "loss": 1.3112, + "theoretical_loss": 3.4321018607786167, + "tokens_seen": 1988100096 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020082246740220664, + "loss": 1.2762, + "theoretical_loss": 3.432082973326292, + "tokens_seen": 1988231168 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200802407221665, + "loss": 1.3152, + "theoretical_loss": 3.4320640874676727, + "tokens_seen": 1988362240 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007823470411234, + "loss": 1.2225, + "theoretical_loss": 3.4320452032025193, + "tokens_seen": 1988493312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020076228686058176, + "loss": 1.5124, + "theoretical_loss": 3.4320263205305928, + "tokens_seen": 1988624384 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020074222668004015, + "loss": 1.3371, + "theoretical_loss": 3.432007439451653, + "tokens_seen": 1988755456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007221664994985, + "loss": 1.4112, + "theoretical_loss": 3.431988559965461, + "tokens_seen": 1988886528 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.4149560034275055, + "objective/train/avg_token_score": 0.006040042731910944, + "objective/train/docs_used": 1125297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2501800060272217, + "objective/train/original_loss": 3.321730136871338, + "objective/train/theoretical_loss": 3.4319696820717778, + "objective/train/tokens_used": 1989017600, + "objective/train/unlikelihood_loss": 0.21500474214553833, + "theoretical_loss": 3.4319696820717778, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007021063189569, + "loss": 1.3156, + "theoretical_loss": 3.4319696820717778, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020068204613841526, + "loss": 1.3248, + "theoretical_loss": 3.4319508057703634, + "tokens_seen": 1989148672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020066198595787362, + "loss": 1.3799, + "theoretical_loss": 3.431931931060979, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200641925777332, + "loss": 1.3516, + "theoretical_loss": 3.431913057943386, + "tokens_seen": 1989410816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020062186559679037, + "loss": 1.3233, + "theoretical_loss": 3.4318941864173445, + "tokens_seen": 1989541888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020060180541624876, + "loss": 1.3468, + "theoretical_loss": 3.431875316482615, + "tokens_seen": 1989672960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020058174523570713, + "loss": 1.2207, + "theoretical_loss": 3.43185644813896, + "tokens_seen": 1989804032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020056168505516551, + "loss": 1.2532, + "theoretical_loss": 3.431837581386139, + "tokens_seen": 1989935104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020054162487462388, + "loss": 1.2117, + "theoretical_loss": 3.4318187162239147, + "tokens_seen": 1990066176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020052156469408227, + "loss": 1.3395, + "theoretical_loss": 3.4317998526520466, + "tokens_seen": 1990197248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020050150451354063, + "loss": 1.2707, + "theoretical_loss": 3.4317809906702967, + "tokens_seen": 1990328320 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200481444332999, + "loss": 1.2634, + "theoretical_loss": 3.4317621302784262, + "tokens_seen": 1990459392 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020046138415245738, + "loss": 1.409, + "theoretical_loss": 3.431743271476197, + "tokens_seen": 1990590464 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.2690615952014923, + "objective/train/avg_token_score": 0.0935102328658104, + "objective/train/docs_used": 1126404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.72669917345047, + "objective/train/original_loss": 3.6423254013061523, + "objective/train/theoretical_loss": 3.4317338426711226, + "objective/train/tokens_used": 1990656000, + "objective/train/unlikelihood_loss": 0.27089741826057434, + "theoretical_loss": 3.4317338426711226, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020044132397191574, + "loss": 1.3577, + "theoretical_loss": 3.4317244142633694, + "tokens_seen": 1990721536 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020042126379137413, + "loss": 1.2368, + "theoretical_loss": 3.4317055586397056, + "tokens_seen": 1990852608 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002004012036108325, + "loss": 1.4212, + "theoretical_loss": 3.431686704604967, + "tokens_seen": 1990983680 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020038114343029088, + "loss": 1.2479, + "theoretical_loss": 3.431667852158914, + "tokens_seen": 1991114752 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020036108324974925, + "loss": 1.2969, + "theoretical_loss": 3.43164900130131, + "tokens_seen": 1991245824 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020034102306920763, + "loss": 1.2732, + "theoretical_loss": 3.4316301520319152, + "tokens_seen": 1991376896 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200320962888666, + "loss": 1.2004, + "theoretical_loss": 3.4316113043504917, + "tokens_seen": 1991507968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020030090270812436, + "loss": 1.2668, + "theoretical_loss": 3.4315924582568016, + "tokens_seen": 1991639040 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020028084252758275, + "loss": 1.2993, + "theoretical_loss": 3.431573613750606, + "tokens_seen": 1991770112 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002002607823470411, + "loss": 1.3802, + "theoretical_loss": 3.431554770831667, + "tokens_seen": 1991901184 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002002407221664995, + "loss": 1.3781, + "theoretical_loss": 3.4315359294997467, + "tokens_seen": 1992032256 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020022066198595786, + "loss": 1.2655, + "theoretical_loss": 3.431517089754607, + "tokens_seen": 1992163328 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.33174487948417664, + "objective/train/avg_token_score": 0.011509726755321026, + "objective/train/docs_used": 1126980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.054763913154602, + "objective/train/original_loss": 3.4354071617126465, + "objective/train/theoretical_loss": 3.4314982515960093, + "objective/train/tokens_used": 1992294400, + "objective/train/unlikelihood_loss": 0.2248244732618332, + "theoretical_loss": 3.4314982515960093, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020020060180541625, + "loss": 1.3776, + "theoretical_loss": 3.4314982515960093, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020018054162487461, + "loss": 1.2795, + "theoretical_loss": 3.4314794150237162, + "tokens_seen": 1992425472 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200160481444333, + "loss": 1.2517, + "theoretical_loss": 3.4314605800374895, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020014042126379137, + "loss": 1.2793, + "theoretical_loss": 3.431441746637092, + "tokens_seen": 1992687616 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020012036108324973, + "loss": 1.3768, + "theoretical_loss": 3.431422914822285, + "tokens_seen": 1992818688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020010030090270812, + "loss": 1.317, + "theoretical_loss": 3.431404084592831, + "tokens_seen": 1992949760 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020008024072216648, + "loss": 1.2102, + "theoretical_loss": 3.4313852559484928, + "tokens_seen": 1993080832 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002000601805416249, + "loss": 1.2195, + "theoretical_loss": 3.4313664288890324, + "tokens_seen": 1993211904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020004012036108326, + "loss": 1.2714, + "theoretical_loss": 3.4313476034142116, + "tokens_seen": 1993342976 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020002006018054165, + "loss": 1.4785, + "theoretical_loss": 3.4313287795237932, + "tokens_seen": 1993474048 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002, + "loss": 1.1934, + "theoretical_loss": 3.4313099572175405, + "tokens_seen": 1993605120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001999799398194584, + "loss": 1.1247, + "theoretical_loss": 3.431291136495215, + "tokens_seen": 1993736192 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019995987963891676, + "loss": 1.3649, + "theoretical_loss": 3.431272317356579, + "tokens_seen": 1993867264 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.4694525897502899, + "objective/train/avg_token_score": 0.012774664908647537, + "objective/train/docs_used": 1127596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5192607641220093, + "objective/train/original_loss": 3.7576711177825928, + "objective/train/theoretical_loss": 3.431262908381071, + "objective/train/tokens_used": 1993932800, + "objective/train/unlikelihood_loss": 0.14828087389469147, + "theoretical_loss": 3.431262908381071, + "tokens_seen": 1993932800 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019993981945837515, + "loss": 1.3478, + "theoretical_loss": 3.4312534998013966, + "tokens_seen": 1993998336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001999197592778335, + "loss": 1.385, + "theoretical_loss": 3.4312346838294294, + "tokens_seen": 1994129408 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019989969909729187, + "loss": 1.3139, + "theoretical_loss": 3.431215869440441, + "tokens_seen": 1994260480 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019987963891675026, + "loss": 1.2789, + "theoretical_loss": 3.4311970566341934, + "tokens_seen": 1994391552 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019985957873620863, + "loss": 1.2504, + "theoretical_loss": 3.4311782454104494, + "tokens_seen": 1994522624 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019983951855566702, + "loss": 1.2466, + "theoretical_loss": 3.431159435768972, + "tokens_seen": 1994653696 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019981945837512538, + "loss": 1.2576, + "theoretical_loss": 3.431140627709525, + "tokens_seen": 1994784768 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019979939819458377, + "loss": 1.2205, + "theoretical_loss": 3.43112182123187, + "tokens_seen": 1994915840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019977933801404213, + "loss": 1.389, + "theoretical_loss": 3.431103016335771, + "tokens_seen": 1995046912 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019975927783350052, + "loss": 1.3472, + "theoretical_loss": 3.4310842130209913, + "tokens_seen": 1995177984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019973921765295888, + "loss": 1.3041, + "theoretical_loss": 3.4310654112872934, + "tokens_seen": 1995309056 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019971915747241724, + "loss": 1.2734, + "theoretical_loss": 3.431046611134441, + "tokens_seen": 1995440128 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.5531525015830994, + "objective/train/avg_token_score": 0.007041353732347488, + "objective/train/docs_used": 1128921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5888235569000244, + "objective/train/original_loss": 3.310046911239624, + "objective/train/theoretical_loss": 3.4310278125621965, + "objective/train/tokens_used": 1995571200, + "objective/train/unlikelihood_loss": 0.16202636063098907, + "theoretical_loss": 3.4310278125621965, + "tokens_seen": 1995571200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019969909729187563, + "loss": 1.2327, + "theoretical_loss": 3.4310278125621965, + "tokens_seen": 1995571200 + }, + { + "epoch": 0.6, + "learning_rate": 0.000199679037111334, + "loss": 1.3411, + "theoretical_loss": 3.4310090155703246, + "tokens_seen": 1995702272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019965897693079238, + "loss": 1.2905, + "theoretical_loss": 3.430990220158588, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019963891675025075, + "loss": 1.4936, + "theoretical_loss": 3.430971426326749, + "tokens_seen": 1995964416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019961885656970914, + "loss": 1.1577, + "theoretical_loss": 3.430952634074573, + "tokens_seen": 1996095488 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001995987963891675, + "loss": 1.1712, + "theoretical_loss": 3.4309338434018217, + "tokens_seen": 1996226560 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001995787362086259, + "loss": 1.2877, + "theoretical_loss": 3.43091505430826, + "tokens_seen": 1996357632 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019955867602808425, + "loss": 1.2713, + "theoretical_loss": 3.430896266793651, + "tokens_seen": 1996488704 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019953861584754264, + "loss": 1.4193, + "theoretical_loss": 3.4308774808577587, + "tokens_seen": 1996619776 + }, + { + "epoch": 0.61, + "learning_rate": 0.000199518555667001, + "loss": 1.309, + "theoretical_loss": 3.430858696500346, + "tokens_seen": 1996750848 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019949849548645936, + "loss": 1.527, + "theoretical_loss": 3.4308399137211776, + "tokens_seen": 1996881920 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019947843530591775, + "loss": 1.3102, + "theoretical_loss": 3.4308211325200166, + "tokens_seen": 1997012992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019945837512537612, + "loss": 1.3411, + "theoretical_loss": 3.4308023528966274, + "tokens_seen": 1997144064 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.3593597114086151, + "objective/train/avg_token_score": 0.028130266815423965, + "objective/train/docs_used": 1129541, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0629225969314575, + "objective/train/original_loss": 3.7758021354675293, + "objective/train/theoretical_loss": 3.4307929636765233, + "objective/train/tokens_used": 1997209600, + "objective/train/unlikelihood_loss": 0.2492760866880417, + "theoretical_loss": 3.4307929636765233, + "tokens_seen": 1997209600 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001994383149448345, + "loss": 1.4336, + "theoretical_loss": 3.4307835748507736, + "tokens_seen": 1997275136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019941825476429287, + "loss": 1.2058, + "theoretical_loss": 3.4307647983822194, + "tokens_seen": 1997406208 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019939819458375126, + "loss": 1.3177, + "theoretical_loss": 3.4307460234907285, + "tokens_seen": 1997537280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019937813440320962, + "loss": 1.315, + "theoretical_loss": 3.430727250176065, + "tokens_seen": 1997668352 + }, + { + "epoch": 0.61, + "learning_rate": 0.000199358074222668, + "loss": 1.3017, + "theoretical_loss": 3.4307084784379933, + "tokens_seen": 1997799424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019933801404212637, + "loss": 1.3935, + "theoretical_loss": 3.4306897082762777, + "tokens_seen": 1997930496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019931795386158476, + "loss": 1.2622, + "theoretical_loss": 3.4306709396906823, + "tokens_seen": 1998061568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019929789368104315, + "loss": 1.1944, + "theoretical_loss": 3.4306521726809707, + "tokens_seen": 1998192640 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001992778335005015, + "loss": 1.3157, + "theoretical_loss": 3.4306334072469085, + "tokens_seen": 1998323712 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001992577733199599, + "loss": 1.2421, + "theoretical_loss": 3.4306146433882585, + "tokens_seen": 1998454784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019923771313941826, + "loss": 1.2301, + "theoretical_loss": 3.4305958811047863, + "tokens_seen": 1998585856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019921765295887665, + "loss": 1.2557, + "theoretical_loss": 3.430577120396256, + "tokens_seen": 1998716928 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.3708455562591553, + "objective/train/avg_token_score": 0.06954240798950195, + "objective/train/docs_used": 1130465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1751244068145752, + "objective/train/original_loss": 4.401905059814453, + "objective/train/theoretical_loss": 3.430558361262433, + "objective/train/tokens_used": 1998848000, + "objective/train/unlikelihood_loss": 0.17418831586837769, + "theoretical_loss": 3.430558361262433, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019919759277833501, + "loss": 1.2904, + "theoretical_loss": 3.430558361262433, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001991775325977934, + "loss": 1.323, + "theoretical_loss": 3.43053960370308, + "tokens_seen": 1998979072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019915747241725177, + "loss": 1.3068, + "theoretical_loss": 3.430520847717963, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019913741223671013, + "loss": 1.2007, + "theoretical_loss": 3.4305020933068464, + "tokens_seen": 1999241216 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019911735205616852, + "loss": 1.3506, + "theoretical_loss": 3.430483340469495, + "tokens_seen": 1999372288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019909729187562688, + "loss": 1.3386, + "theoretical_loss": 3.4304645892056733, + "tokens_seen": 1999503360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019907723169508527, + "loss": 1.1748, + "theoretical_loss": 3.4304458395151465, + "tokens_seen": 1999634432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019905717151454363, + "loss": 1.1813, + "theoretical_loss": 3.4304270913976795, + "tokens_seen": 1999765504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019903711133400202, + "loss": 1.3236, + "theoretical_loss": 3.4304083448530367, + "tokens_seen": 1999896576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019901705115346038, + "loss": 1.1968, + "theoretical_loss": 3.430389599880983, + "tokens_seen": 2000027648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019899699097291877, + "loss": 1.1966, + "theoretical_loss": 3.4303708564812845, + "tokens_seen": 2000158720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019897693079237713, + "loss": 1.2142, + "theoretical_loss": 3.4303521146537053, + "tokens_seen": 2000289792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019895687061183552, + "loss": 1.3102, + "theoretical_loss": 3.430333374398011, + "tokens_seen": 2000420864 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.3238636255264282, + "objective/train/avg_token_score": 0.04398213326931, + "objective/train/docs_used": 1131148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0160489082336426, + "objective/train/original_loss": 4.453743934631348, + "objective/train/theoretical_loss": 3.430324004859547, + "objective/train/tokens_used": 2000486400, + "objective/train/unlikelihood_loss": 0.10650455206632614, + "theoretical_loss": 3.430324004859547, + "tokens_seen": 2000486400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019893681043129389, + "loss": 1.3244, + "theoretical_loss": 3.430314635713967, + "tokens_seen": 2000551936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019891675025075225, + "loss": 1.496, + "theoretical_loss": 3.4302958986013374, + "tokens_seen": 2000683008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019889669007021064, + "loss": 1.3747, + "theoretical_loss": 3.4302771630598885, + "tokens_seen": 2000814080 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198876629889669, + "loss": 1.2306, + "theoretical_loss": 3.4302584290893856, + "tokens_seen": 2000945152 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001988565697091274, + "loss": 1.1501, + "theoretical_loss": 3.430239696689594, + "tokens_seen": 2001076224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019883650952858575, + "loss": 1.1297, + "theoretical_loss": 3.4302209658602787, + "tokens_seen": 2001207296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019881644934804414, + "loss": 1.1805, + "theoretical_loss": 3.4302022366012057, + "tokens_seen": 2001338368 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001987963891675025, + "loss": 1.2778, + "theoretical_loss": 3.4301835089121404, + "tokens_seen": 2001469440 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001987763289869609, + "loss": 1.4565, + "theoretical_loss": 3.4301647827928483, + "tokens_seen": 2001600512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019875626880641925, + "loss": 1.4566, + "theoretical_loss": 3.430146058243095, + "tokens_seen": 2001731584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019873620862587762, + "loss": 1.32, + "theoretical_loss": 3.4301273352626462, + "tokens_seen": 2001862656 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198716148445336, + "loss": 1.2739, + "theoretical_loss": 3.4301086138512678, + "tokens_seen": 2001993728 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.3044965863227844, + "objective/train/avg_token_score": 0.023212619125843048, + "objective/train/docs_used": 1132271, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0299471616744995, + "objective/train/original_loss": 4.340414047241211, + "objective/train/theoretical_loss": 3.4300898940087254, + "objective/train/tokens_used": 2002124800, + "objective/train/unlikelihood_loss": 0.16442349553108215, + "theoretical_loss": 3.4300898940087254, + "tokens_seen": 2002124800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019869608826479437, + "loss": 1.1479, + "theoretical_loss": 3.4300898940087254, + "tokens_seen": 2002124800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019867602808425276, + "loss": 1.1606, + "theoretical_loss": 3.4300711757347853, + "tokens_seen": 2002255872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019865596790371112, + "loss": 1.2141, + "theoretical_loss": 3.4300524590292127, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001986359077231695, + "loss": 1.2947, + "theoretical_loss": 3.430033743891774, + "tokens_seen": 2002518016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019861584754262787, + "loss": 1.3189, + "theoretical_loss": 3.430015030322235, + "tokens_seen": 2002649088 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019859578736208626, + "loss": 1.1936, + "theoretical_loss": 3.4299963183203617, + "tokens_seen": 2002780160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019857572718154462, + "loss": 1.2925, + "theoretical_loss": 3.4299776078859203, + "tokens_seen": 2002911232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019855566700100304, + "loss": 1.4431, + "theoretical_loss": 3.4299588990186773, + "tokens_seen": 2003042304 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001985356068204614, + "loss": 1.2313, + "theoretical_loss": 3.429940191718398, + "tokens_seen": 2003173376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019851554663991976, + "loss": 1.2709, + "theoretical_loss": 3.4299214859848495, + "tokens_seen": 2003304448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019849548645937815, + "loss": 1.2997, + "theoretical_loss": 3.4299027818177974, + "tokens_seen": 2003435520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019847542627883652, + "loss": 1.2686, + "theoretical_loss": 3.4298840792170084, + "tokens_seen": 2003566592 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001984553660982949, + "loss": 1.3963, + "theoretical_loss": 3.4298653781822495, + "tokens_seen": 2003697664 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.32056450843811035, + "objective/train/avg_token_score": 0.027280980721116066, + "objective/train/docs_used": 1132794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1010472774505615, + "objective/train/original_loss": 4.3502936363220215, + "objective/train/theoretical_loss": 3.429856028252058, + "objective/train/tokens_used": 2003763200, + "objective/train/unlikelihood_loss": 0.16821575164794922, + "theoretical_loss": 3.429856028252058, + "tokens_seen": 2003763200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019843530591775327, + "loss": 1.2951, + "theoretical_loss": 3.429846678713286, + "tokens_seen": 2003828736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019841524573721166, + "loss": 1.3281, + "theoretical_loss": 3.4298279808098853, + "tokens_seen": 2003959808 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019839518555667002, + "loss": 1.2099, + "theoretical_loss": 3.4298092844718133, + "tokens_seen": 2004090880 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001983751253761284, + "loss": 1.1784, + "theoretical_loss": 3.429790589698837, + "tokens_seen": 2004221952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019835506519558677, + "loss": 1.3039, + "theoretical_loss": 3.429771896490723, + "tokens_seen": 2004353024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019833500501504513, + "loss": 1.2937, + "theoretical_loss": 3.4297532048472377, + "tokens_seen": 2004484096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019831494483450352, + "loss": 1.2374, + "theoretical_loss": 3.429734514768148, + "tokens_seen": 2004615168 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019829488465396188, + "loss": 1.3728, + "theoretical_loss": 3.4297158262532204, + "tokens_seen": 2004746240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019827482447342027, + "loss": 1.376, + "theoretical_loss": 3.4296971393022226, + "tokens_seen": 2004877312 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019825476429287864, + "loss": 1.1958, + "theoretical_loss": 3.4296784539149208, + "tokens_seen": 2005008384 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019823470411233703, + "loss": 1.2402, + "theoretical_loss": 3.4296597700910825, + "tokens_seen": 2005139456 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001982146439317954, + "loss": 1.3568, + "theoretical_loss": 3.4296410878304737, + "tokens_seen": 2005270528 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.3172042965888977, + "objective/train/avg_token_score": 0.020166559144854546, + "objective/train/docs_used": 1134045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9376850128173828, + "objective/train/original_loss": 3.613009452819824, + "objective/train/theoretical_loss": 3.4296224071328623, + "objective/train/tokens_used": 2005401600, + "objective/train/unlikelihood_loss": 0.18017402291297913, + "theoretical_loss": 3.4296224071328623, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019819458375125378, + "loss": 1.2609, + "theoretical_loss": 3.4296224071328623, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019817452357071214, + "loss": 1.263, + "theoretical_loss": 3.429603727998015, + "tokens_seen": 2005532672 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001981544633901705, + "loss": 1.244, + "theoretical_loss": 3.429585050425699, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001981344032096289, + "loss": 1.2007, + "theoretical_loss": 3.429566374415682, + "tokens_seen": 2005794816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019811434302908725, + "loss": 1.3192, + "theoretical_loss": 3.4295476999677303, + "tokens_seen": 2005925888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019809428284854564, + "loss": 1.3504, + "theoretical_loss": 3.429529027081612, + "tokens_seen": 2006056960 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198074222668004, + "loss": 1.1155, + "theoretical_loss": 3.429510355757094, + "tokens_seen": 2006188032 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001980541624874624, + "loss": 1.3263, + "theoretical_loss": 3.429491685993944, + "tokens_seen": 2006319104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019803410230692076, + "loss": 1.5319, + "theoretical_loss": 3.4294730177919295, + "tokens_seen": 2006450176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019801404212637915, + "loss": 1.2553, + "theoretical_loss": 3.429454351150817, + "tokens_seen": 2006581248 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001979939819458375, + "loss": 1.331, + "theoretical_loss": 3.4294356860703754, + "tokens_seen": 2006712320 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001979739217652959, + "loss": 1.3928, + "theoretical_loss": 3.4294170225503717, + "tokens_seen": 2006843392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019795386158475426, + "loss": 1.3692, + "theoretical_loss": 3.4293983605905733, + "tokens_seen": 2006974464 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.3006476163864136, + "objective/train/avg_token_score": 0.02843763493001461, + "objective/train/docs_used": 1134700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.041620135307312, + "objective/train/original_loss": 3.8274757862091064, + "objective/train/theoretical_loss": 3.429389030195679, + "objective/train/tokens_used": 2007040000, + "objective/train/unlikelihood_loss": 0.4981590509414673, + "theoretical_loss": 3.429389030195679, + "tokens_seen": 2007040000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019793380140421262, + "loss": 1.3742, + "theoretical_loss": 3.4293797001907484, + "tokens_seen": 2007105536 + }, + { + "epoch": 0.61, + "learning_rate": 0.000197913741223671, + "loss": 1.3195, + "theoretical_loss": 3.4293610413506643, + "tokens_seen": 2007236608 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019789368104312937, + "loss": 1.1635, + "theoretical_loss": 3.429342384070089, + "tokens_seen": 2007367680 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019787362086258776, + "loss": 1.259, + "theoretical_loss": 3.4293237283487903, + "tokens_seen": 2007498752 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019785356068204612, + "loss": 1.2914, + "theoretical_loss": 3.4293050741865363, + "tokens_seen": 2007629824 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019783350050150451, + "loss": 1.3101, + "theoretical_loss": 3.4292864215830945, + "tokens_seen": 2007760896 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001978134403209629, + "loss": 1.308, + "theoretical_loss": 3.4292677705382335, + "tokens_seen": 2007891968 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001977933801404213, + "loss": 1.3685, + "theoretical_loss": 3.42924912105172, + "tokens_seen": 2008023040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019777331995987965, + "loss": 1.4854, + "theoretical_loss": 3.429230473123324, + "tokens_seen": 2008154112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019775325977933802, + "loss": 1.3016, + "theoretical_loss": 3.4292118267528124, + "tokens_seen": 2008285184 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001977331995987964, + "loss": 1.358, + "theoretical_loss": 3.4291931819399535, + "tokens_seen": 2008416256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019771313941825477, + "loss": 1.3034, + "theoretical_loss": 3.4291745386845163, + "tokens_seen": 2008547328 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.5497311949729919, + "objective/train/avg_token_score": 0.010504922829568386, + "objective/train/docs_used": 1135755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7056809663772583, + "objective/train/original_loss": 3.5224740505218506, + "objective/train/theoretical_loss": 3.429155896986268, + "objective/train/tokens_used": 2008678400, + "objective/train/unlikelihood_loss": 0.12507140636444092, + "theoretical_loss": 3.429155896986268, + "tokens_seen": 2008678400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019769307923771316, + "loss": 1.3053, + "theoretical_loss": 3.429155896986268, + "tokens_seen": 2008678400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019767301905717152, + "loss": 1.3214, + "theoretical_loss": 3.429137256844977, + "tokens_seen": 2008809472 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001976529588766299, + "loss": 1.3978, + "theoretical_loss": 3.429118618260413, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019763289869608827, + "loss": 1.2452, + "theoretical_loss": 3.4290999812323433, + "tokens_seen": 2009071616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019761283851554666, + "loss": 1.3513, + "theoretical_loss": 3.4290813457605362, + "tokens_seen": 2009202688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019759277833500502, + "loss": 1.3219, + "theoretical_loss": 3.429062711844761, + "tokens_seen": 2009333760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019757271815446339, + "loss": 1.3133, + "theoretical_loss": 3.429044079484786, + "tokens_seen": 2009464832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019755265797392178, + "loss": 1.1835, + "theoretical_loss": 3.42902544868038, + "tokens_seen": 2009595904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019753259779338014, + "loss": 1.2744, + "theoretical_loss": 3.4290068194313115, + "tokens_seen": 2009726976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019751253761283853, + "loss": 1.3355, + "theoretical_loss": 3.4289881917373486, + "tokens_seen": 2009858048 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001974924774322969, + "loss": 1.5631, + "theoretical_loss": 3.428969565598261, + "tokens_seen": 2009989120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019747241725175528, + "loss": 1.3559, + "theoretical_loss": 3.4289509410138175, + "tokens_seen": 2010120192 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019745235707121364, + "loss": 1.4614, + "theoretical_loss": 3.4289323179837865, + "tokens_seen": 2010251264 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.39027369022369385, + "objective/train/avg_token_score": 0.011012593284249306, + "objective/train/docs_used": 1136321, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1733808517456055, + "objective/train/original_loss": 3.684664249420166, + "objective/train/theoretical_loss": 3.4289230070516035, + "objective/train/tokens_used": 2010316800, + "objective/train/unlikelihood_loss": 0.1680251955986023, + "theoretical_loss": 3.4289230070516035, + "tokens_seen": 2010316800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019743229689067203, + "loss": 1.2816, + "theoretical_loss": 3.428913696507937, + "tokens_seen": 2010382336 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001974122367101304, + "loss": 1.2202, + "theoretical_loss": 3.428895076586038, + "tokens_seen": 2010513408 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019739217652958878, + "loss": 1.4655, + "theoretical_loss": 3.428876458217859, + "tokens_seen": 2010644480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019737211634904714, + "loss": 1.2975, + "theoretical_loss": 3.4288578414031687, + "tokens_seen": 2010775552 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973520561685055, + "loss": 1.5026, + "theoretical_loss": 3.4288392261417364, + "tokens_seen": 2010906624 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973319959879639, + "loss": 1.3225, + "theoretical_loss": 3.4288206124333307, + "tokens_seen": 2011037696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019731193580742226, + "loss": 1.2694, + "theoretical_loss": 3.4288020002777215, + "tokens_seen": 2011168768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019729187562688065, + "loss": 1.4602, + "theoretical_loss": 3.4287833896746776, + "tokens_seen": 2011299840 + }, + { + "epoch": 0.61, + "learning_rate": 0.000197271815446339, + "loss": 1.1825, + "theoretical_loss": 3.428764780623969, + "tokens_seen": 2011430912 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001972517552657974, + "loss": 1.3361, + "theoretical_loss": 3.428746173125364, + "tokens_seen": 2011561984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019723169508525576, + "loss": 1.3144, + "theoretical_loss": 3.4287275671786333, + "tokens_seen": 2011693056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019721163490471415, + "loss": 1.2324, + "theoretical_loss": 3.4287089627835456, + "tokens_seen": 2011824128 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.3922898471355438, + "objective/train/avg_token_score": 0.017566844820976257, + "objective/train/docs_used": 1137326, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2167068719863892, + "objective/train/original_loss": 3.7684452533721924, + "objective/train/theoretical_loss": 3.4286903599398704, + "objective/train/tokens_used": 2011955200, + "objective/train/unlikelihood_loss": 0.17020252346992493, + "theoretical_loss": 3.4286903599398704, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001971915747241725, + "loss": 1.3725, + "theoretical_loss": 3.4286903599398704, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019717151454363087, + "loss": 1.2208, + "theoretical_loss": 3.4286717586473774, + "tokens_seen": 2012086272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019715145436308926, + "loss": 1.2285, + "theoretical_loss": 3.428653158905836, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019713139418254763, + "loss": 1.3611, + "theoretical_loss": 3.428634560715017, + "tokens_seen": 2012348416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019711133400200602, + "loss": 1.3963, + "theoretical_loss": 3.4286159640746883, + "tokens_seen": 2012479488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019709127382146438, + "loss": 1.4059, + "theoretical_loss": 3.428597368984621, + "tokens_seen": 2012610560 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001970712136409228, + "loss": 1.17, + "theoretical_loss": 3.428578775444585, + "tokens_seen": 2012741632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019705115346038116, + "loss": 1.2927, + "theoretical_loss": 3.4285601834543487, + "tokens_seen": 2012872704 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019703109327983955, + "loss": 1.2174, + "theoretical_loss": 3.4285415930136836, + "tokens_seen": 2013003776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001970110330992979, + "loss": 1.2514, + "theoretical_loss": 3.4285230041223587, + "tokens_seen": 2013134848 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001969909729187563, + "loss": 1.3146, + "theoretical_loss": 3.428504416780145, + "tokens_seen": 2013265920 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019697091273821466, + "loss": 1.4374, + "theoretical_loss": 3.428485830986811, + "tokens_seen": 2013396992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019695085255767302, + "loss": 1.2373, + "theoretical_loss": 3.4284672467421284, + "tokens_seen": 2013528064 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.2991202473640442, + "objective/train/avg_token_score": 0.023575175553560257, + "objective/train/docs_used": 1138031, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0826133489608765, + "objective/train/original_loss": 3.9465839862823486, + "objective/train/theoretical_loss": 3.4284579552004595, + "objective/train/tokens_used": 2013593600, + "objective/train/unlikelihood_loss": 0.16640309989452362, + "theoretical_loss": 3.4284579552004595, + "tokens_seen": 2013593600 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001969307923771314, + "loss": 1.2958, + "theoretical_loss": 3.4284486640458667, + "tokens_seen": 2013659136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019691073219658977, + "loss": 1.3342, + "theoretical_loss": 3.4284300828977967, + "tokens_seen": 2013790208 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019689067201604816, + "loss": 1.4753, + "theoretical_loss": 3.428411503297687, + "tokens_seen": 2013921280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019687061183550652, + "loss": 1.3139, + "theoretical_loss": 3.4283929252453094, + "tokens_seen": 2014052352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019685055165496491, + "loss": 1.3306, + "theoretical_loss": 3.428374348740434, + "tokens_seen": 2014183424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019683049147442328, + "loss": 1.2927, + "theoretical_loss": 3.4283557737828314, + "tokens_seen": 2014314496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019681043129388167, + "loss": 1.2345, + "theoretical_loss": 3.428337200372271, + "tokens_seen": 2014445568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019679037111334003, + "loss": 1.4063, + "theoretical_loss": 3.4283186285085243, + "tokens_seen": 2014576640 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001967703109327984, + "loss": 1.4259, + "theoretical_loss": 3.428300058191362, + "tokens_seen": 2014707712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019675025075225678, + "loss": 1.2563, + "theoretical_loss": 3.428281489420554, + "tokens_seen": 2014838784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019673019057171514, + "loss": 1.3936, + "theoretical_loss": 3.428262922195871, + "tokens_seen": 2014969856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019671013039117353, + "loss": 1.3681, + "theoretical_loss": 3.428244356517084, + "tokens_seen": 2015100928 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.6485214829444885, + "objective/train/avg_token_score": 0.00412459671497345, + "objective/train/docs_used": 1139429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0337560176849365, + "objective/train/original_loss": 3.2045295238494873, + "objective/train/theoretical_loss": 3.428225792383964, + "objective/train/tokens_used": 2015232000, + "objective/train/unlikelihood_loss": 0.13168199360370636, + "theoretical_loss": 3.428225792383964, + "tokens_seen": 2015232000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001966900702106319, + "loss": 1.3785, + "theoretical_loss": 3.428225792383964, + "tokens_seen": 2015232000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019667001003009028, + "loss": 1.3803, + "theoretical_loss": 3.428207229796281, + "tokens_seen": 2015363072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019664994984954865, + "loss": 1.2737, + "theoretical_loss": 3.4281886687538066, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019662988966900703, + "loss": 1.3399, + "theoretical_loss": 3.4281701092563117, + "tokens_seen": 2015625216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001966098294884654, + "loss": 1.3637, + "theoretical_loss": 3.428151551303567, + "tokens_seen": 2015756288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019658976930792376, + "loss": 1.448, + "theoretical_loss": 3.4281329948953427, + "tokens_seen": 2015887360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019656970912738215, + "loss": 1.2868, + "theoretical_loss": 3.428114440031411, + "tokens_seen": 2016018432 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001965496489468405, + "loss": 1.4044, + "theoretical_loss": 3.428095886711543, + "tokens_seen": 2016149504 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001965295887662989, + "loss": 1.2507, + "theoretical_loss": 3.4280773349355087, + "tokens_seen": 2016280576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019650952858575726, + "loss": 1.4203, + "theoretical_loss": 3.4280587847030812, + "tokens_seen": 2016411648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019648946840521565, + "loss": 1.2273, + "theoretical_loss": 3.4280402360140294, + "tokens_seen": 2016542720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019646940822467401, + "loss": 1.2711, + "theoretical_loss": 3.4280216888681263, + "tokens_seen": 2016673792 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001964493480441324, + "loss": 1.4915, + "theoretical_loss": 3.4280031432651423, + "tokens_seen": 2016804864 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.22550097107887268, + "objective/train/avg_token_score": 0.016780702397227287, + "objective/train/docs_used": 1140637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7870764136314392, + "objective/train/original_loss": 4.28874397277832, + "objective/train/theoretical_loss": 3.4279938710421742, + "objective/train/tokens_used": 2016870400, + "objective/train/unlikelihood_loss": 0.15464521944522858, + "theoretical_loss": 3.4279938710421742, + "tokens_seen": 2016870400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019642928786359077, + "loss": 1.2999, + "theoretical_loss": 3.4279845992048497, + "tokens_seen": 2016935936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019640922768304913, + "loss": 1.4627, + "theoretical_loss": 3.427966056687019, + "tokens_seen": 2017067008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019638916750250752, + "loss": 1.4122, + "theoretical_loss": 3.4279475157114225, + "tokens_seen": 2017198080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019636910732196588, + "loss": 1.2898, + "theoretical_loss": 3.427928976277831, + "tokens_seen": 2017329152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019634904714142427, + "loss": 1.2587, + "theoretical_loss": 3.4279104383860166, + "tokens_seen": 2017460224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019632898696088263, + "loss": 1.3335, + "theoretical_loss": 3.427891902035751, + "tokens_seen": 2017591296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019630892678034105, + "loss": 1.2948, + "theoretical_loss": 3.4278733672268045, + "tokens_seen": 2017722368 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001962888665997994, + "loss": 1.2394, + "theoretical_loss": 3.4278548339589507, + "tokens_seen": 2017853440 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001962688064192578, + "loss": 1.4808, + "theoretical_loss": 3.427836302231961, + "tokens_seen": 2017984512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019624874623871616, + "loss": 1.212, + "theoretical_loss": 3.4278177720456062, + "tokens_seen": 2018115584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019622868605817455, + "loss": 1.2433, + "theoretical_loss": 3.427799243399659, + "tokens_seen": 2018246656 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001962086258776329, + "loss": 1.5278, + "theoretical_loss": 3.427780716293891, + "tokens_seen": 2018377728 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.32869014143943787, + "objective/train/avg_token_score": 0.008385313674807549, + "objective/train/docs_used": 1141300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0460286140441895, + "objective/train/original_loss": 3.887025833129883, + "objective/train/theoretical_loss": 3.4277621907280746, + "objective/train/tokens_used": 2018508800, + "objective/train/unlikelihood_loss": 0.16319917142391205, + "theoretical_loss": 3.4277621907280746, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019618856569709127, + "loss": 1.4727, + "theoretical_loss": 3.4277621907280746, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019616850551654966, + "loss": 1.3009, + "theoretical_loss": 3.427743666701981, + "tokens_seen": 2018639872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019614844533600803, + "loss": 1.2084, + "theoretical_loss": 3.4277251442153833, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019612838515546642, + "loss": 1.3419, + "theoretical_loss": 3.427706623268053, + "tokens_seen": 2018902016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019610832497492478, + "loss": 1.2486, + "theoretical_loss": 3.4276881038597615, + "tokens_seen": 2019033088 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019608826479438317, + "loss": 1.1542, + "theoretical_loss": 3.427669585990283, + "tokens_seen": 2019164160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019606820461384153, + "loss": 1.2175, + "theoretical_loss": 3.4276510696593876, + "tokens_seen": 2019295232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019604814443329992, + "loss": 1.3244, + "theoretical_loss": 3.4276325548668494, + "tokens_seen": 2019426304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019602808425275828, + "loss": 1.2702, + "theoretical_loss": 3.42761404161244, + "tokens_seen": 2019557376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019600802407221664, + "loss": 1.3581, + "theoretical_loss": 3.4275955298959313, + "tokens_seen": 2019688448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019598796389167503, + "loss": 1.3194, + "theoretical_loss": 3.4275770197170963, + "tokens_seen": 2019819520 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001959679037111334, + "loss": 1.4004, + "theoretical_loss": 3.4275585110757074, + "tokens_seen": 2019950592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019594784353059178, + "loss": 1.188, + "theoretical_loss": 3.4275400039715374, + "tokens_seen": 2020081664 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.2844574749469757, + "objective/train/avg_token_score": 0.008559354580938816, + "objective/train/docs_used": 1142665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8628281950950623, + "objective/train/original_loss": 3.7099802494049072, + "objective/train/theoretical_loss": 3.427530750995839, + "objective/train/tokens_used": 2020147200, + "objective/train/unlikelihood_loss": 0.2253696322441101, + "theoretical_loss": 3.427530750995839, + "tokens_seen": 2020147200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019592778335005015, + "loss": 1.3119, + "theoretical_loss": 3.427521498404359, + "tokens_seen": 2020212736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019590772316950854, + "loss": 1.3459, + "theoretical_loss": 3.427502994373944, + "tokens_seen": 2020343808 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958876629889669, + "loss": 1.3622, + "theoretical_loss": 3.4274844918800658, + "tokens_seen": 2020474880 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958676028084253, + "loss": 1.3355, + "theoretical_loss": 3.427465990922497, + "tokens_seen": 2020605952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019584754262788365, + "loss": 1.2608, + "theoretical_loss": 3.4274474915010105, + "tokens_seen": 2020737024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019582748244734204, + "loss": 1.4432, + "theoretical_loss": 3.427428993615379, + "tokens_seen": 2020868096 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958074222668004, + "loss": 1.4421, + "theoretical_loss": 3.4274104972653747, + "tokens_seen": 2020999168 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019578736208625876, + "loss": 1.156, + "theoretical_loss": 3.4273920024507722, + "tokens_seen": 2021130240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019576730190571715, + "loss": 1.32, + "theoretical_loss": 3.427373509171343, + "tokens_seen": 2021261312 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019574724172517552, + "loss": 1.1643, + "theoretical_loss": 3.4273550174268603, + "tokens_seen": 2021392384 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001957271815446339, + "loss": 1.3721, + "theoretical_loss": 3.4273365272170984, + "tokens_seen": 2021523456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019570712136409227, + "loss": 1.3747, + "theoretical_loss": 3.427318038541829, + "tokens_seen": 2021654528 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.4005987346172333, + "objective/train/avg_token_score": 0.010463309474289417, + "objective/train/docs_used": 1143239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1437504291534424, + "objective/train/original_loss": 3.8517277240753174, + "objective/train/theoretical_loss": 3.427299551400826, + "objective/train/tokens_used": 2021785600, + "objective/train/unlikelihood_loss": 0.16504551470279694, + "theoretical_loss": 3.427299551400826, + "tokens_seen": 2021785600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019568706118355066, + "loss": 1.3842, + "theoretical_loss": 3.427299551400826, + "tokens_seen": 2021785600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019566700100300902, + "loss": 1.2103, + "theoretical_loss": 3.4272810657938626, + "tokens_seen": 2021916672 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001956469408224674, + "loss": 1.2919, + "theoretical_loss": 3.4272625817207114, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019562688064192577, + "loss": 1.268, + "theoretical_loss": 3.427244099181147, + "tokens_seen": 2022178816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019560682046138413, + "loss": 1.2507, + "theoretical_loss": 3.4272256181749414, + "tokens_seen": 2022309888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019558676028084252, + "loss": 1.4201, + "theoretical_loss": 3.4272071387018688, + "tokens_seen": 2022440960 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001955667001003009, + "loss": 1.3493, + "theoretical_loss": 3.4271886607617024, + "tokens_seen": 2022572032 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001955466399197593, + "loss": 1.4458, + "theoretical_loss": 3.427170184354216, + "tokens_seen": 2022703104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019552657973921766, + "loss": 1.2487, + "theoretical_loss": 3.4271517094791832, + "tokens_seen": 2022834176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019550651955867605, + "loss": 1.3038, + "theoretical_loss": 3.4271332361363775, + "tokens_seen": 2022965248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019548645937813441, + "loss": 1.3087, + "theoretical_loss": 3.427114764325572, + "tokens_seen": 2023096320 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001954663991975928, + "loss": 1.3688, + "theoretical_loss": 3.427096294046541, + "tokens_seen": 2023227392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019544633901705117, + "loss": 1.3488, + "theoretical_loss": 3.427077825299058, + "tokens_seen": 2023358464 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.42748045921325684, + "objective/train/avg_token_score": 0.01034439168870449, + "objective/train/docs_used": 1144661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3509773015975952, + "objective/train/original_loss": 3.7184510231018066, + "objective/train/theoretical_loss": 3.4270685914995767, + "objective/train/tokens_used": 2023424000, + "objective/train/unlikelihood_loss": 0.1552148461341858, + "theoretical_loss": 3.4270685914995767, + "tokens_seen": 2023424000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019542627883650953, + "loss": 1.0367, + "theoretical_loss": 3.4270593580828974, + "tokens_seen": 2023489536 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019540621865596792, + "loss": 1.4274, + "theoretical_loss": 3.4270408923978324, + "tokens_seen": 2023620608 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019538615847542628, + "loss": 1.3091, + "theoretical_loss": 3.427022428243637, + "tokens_seen": 2023751680 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019536609829488467, + "loss": 1.3857, + "theoretical_loss": 3.4270039656200852, + "tokens_seen": 2023882752 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019534603811434303, + "loss": 1.2092, + "theoretical_loss": 3.426985504526951, + "tokens_seen": 2024013824 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019532597793380142, + "loss": 1.3122, + "theoretical_loss": 3.426967044964009, + "tokens_seen": 2024144896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019530591775325978, + "loss": 1.3569, + "theoretical_loss": 3.426948586931032, + "tokens_seen": 2024275968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019528585757271817, + "loss": 1.3488, + "theoretical_loss": 3.4269301304277953, + "tokens_seen": 2024407040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019526579739217653, + "loss": 1.4512, + "theoretical_loss": 3.4269116754540727, + "tokens_seen": 2024538112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019524573721163492, + "loss": 1.503, + "theoretical_loss": 3.426893222009639, + "tokens_seen": 2024669184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019522567703109329, + "loss": 1.2176, + "theoretical_loss": 3.4268747700942668, + "tokens_seen": 2024800256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019520561685055165, + "loss": 1.3346, + "theoretical_loss": 3.426856319707732, + "tokens_seen": 2024931328 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.40316471457481384, + "objective/train/avg_token_score": 0.03769855946302414, + "objective/train/docs_used": 1145363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.140990138053894, + "objective/train/original_loss": 4.019776821136475, + "objective/train/theoretical_loss": 3.4268378708498086, + "objective/train/tokens_used": 2025062400, + "objective/train/unlikelihood_loss": 0.14280542731285095, + "theoretical_loss": 3.4268378708498086, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019518555667001004, + "loss": 1.4817, + "theoretical_loss": 3.4268378708498086, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001951654964894684, + "loss": 1.2762, + "theoretical_loss": 3.4268194235202714, + "tokens_seen": 2025193472 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001951454363089268, + "loss": 1.435, + "theoretical_loss": 3.4268009777188935, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019512537612838515, + "loss": 1.4926, + "theoretical_loss": 3.426782533445451, + "tokens_seen": 2025455616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019510531594784354, + "loss": 1.569, + "theoretical_loss": 3.426764090699718, + "tokens_seen": 2025586688 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001950852557673019, + "loss": 1.4233, + "theoretical_loss": 3.4267456494814685, + "tokens_seen": 2025717760 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001950651955867603, + "loss": 1.2581, + "theoretical_loss": 3.426727209790478, + "tokens_seen": 2025848832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019504513540621865, + "loss": 1.4462, + "theoretical_loss": 3.4267087716265205, + "tokens_seen": 2025979904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019502507522567702, + "loss": 1.466, + "theoretical_loss": 3.426690334989371, + "tokens_seen": 2026110976 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001950050150451354, + "loss": 1.427, + "theoretical_loss": 3.426671899878804, + "tokens_seen": 2026242048 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019498495486459377, + "loss": 1.2108, + "theoretical_loss": 3.4266534662945958, + "tokens_seen": 2026373120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019496489468405216, + "loss": 1.4219, + "theoretical_loss": 3.42663503423652, + "tokens_seen": 2026504192 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019494483450351052, + "loss": 1.3768, + "theoretical_loss": 3.4266166037043515, + "tokens_seen": 2026635264 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.4427541494369507, + "objective/train/avg_token_score": 0.006872382014989853, + "objective/train/docs_used": 1146045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2685174942016602, + "objective/train/original_loss": 3.6097118854522705, + "objective/train/theoretical_loss": 3.4266073890104125, + "objective/train/tokens_used": 2026700800, + "objective/train/unlikelihood_loss": 0.19050636887550354, + "theoretical_loss": 3.4266073890104125, + "tokens_seen": 2026700800 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001949247743229689, + "loss": 1.2648, + "theoretical_loss": 3.426598174697866, + "tokens_seen": 2026766336 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019490471414242727, + "loss": 1.0763, + "theoretical_loss": 3.426579747216838, + "tokens_seen": 2026897408 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019488465396188566, + "loss": 1.431, + "theoretical_loss": 3.426561321261042, + "tokens_seen": 2027028480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019486459378134402, + "loss": 1.2882, + "theoretical_loss": 3.426542896830255, + "tokens_seen": 2027159552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019484453360080239, + "loss": 1.4982, + "theoretical_loss": 3.4265244739242506, + "tokens_seen": 2027290624 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019482447342026077, + "loss": 1.3784, + "theoretical_loss": 3.426506052542805, + "tokens_seen": 2027421696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019480441323971916, + "loss": 1.2833, + "theoretical_loss": 3.4264876326856926, + "tokens_seen": 2027552768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019478435305917755, + "loss": 1.2973, + "theoretical_loss": 3.4264692143526894, + "tokens_seen": 2027683840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019476429287863592, + "loss": 1.257, + "theoretical_loss": 3.4264507975435707, + "tokens_seen": 2027814912 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001947442326980943, + "loss": 1.3429, + "theoretical_loss": 3.4264323822581115, + "tokens_seen": 2027945984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019472417251755267, + "loss": 1.5082, + "theoretical_loss": 3.4264139684960875, + "tokens_seen": 2028077056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019470411233701106, + "loss": 1.3958, + "theoretical_loss": 3.4263955562572743, + "tokens_seen": 2028208128 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.5306695699691772, + "objective/train/avg_token_score": 0.015986405313014984, + "objective/train/docs_used": 1147311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6920746564865112, + "objective/train/original_loss": 3.7163572311401367, + "objective/train/theoretical_loss": 3.426377145541448, + "objective/train/tokens_used": 2028339200, + "objective/train/unlikelihood_loss": 0.13381460309028625, + "theoretical_loss": 3.426377145541448, + "tokens_seen": 2028339200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019468405215646942, + "loss": 1.4789, + "theoretical_loss": 3.426377145541448, + "tokens_seen": 2028339200 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001946639919759278, + "loss": 1.2298, + "theoretical_loss": 3.4263587363483827, + "tokens_seen": 2028470272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019464393179538617, + "loss": 1.3683, + "theoretical_loss": 3.4263403286778558, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019462387161484453, + "loss": 1.3486, + "theoretical_loss": 3.4263219225296417, + "tokens_seen": 2028732416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019460381143430292, + "loss": 1.2542, + "theoretical_loss": 3.4263035179035173, + "tokens_seen": 2028863488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019458375125376128, + "loss": 1.2327, + "theoretical_loss": 3.4262851147992577, + "tokens_seen": 2028994560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019456369107321967, + "loss": 1.3688, + "theoretical_loss": 3.4262667132166387, + "tokens_seen": 2029125632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019454363089267804, + "loss": 1.1933, + "theoretical_loss": 3.426248313155437, + "tokens_seen": 2029256704 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019452357071213642, + "loss": 1.2357, + "theoretical_loss": 3.426229914615427, + "tokens_seen": 2029387776 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001945035105315948, + "loss": 1.2853, + "theoretical_loss": 3.4262115175963865, + "tokens_seen": 2029518848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019448345035105318, + "loss": 1.3338, + "theoretical_loss": 3.4261931220980903, + "tokens_seen": 2029649920 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019446339017051154, + "loss": 1.3058, + "theoretical_loss": 3.4261747281203148, + "tokens_seen": 2029780992 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001944433299899699, + "loss": 1.3971, + "theoretical_loss": 3.426156335662837, + "tokens_seen": 2029912064 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.3677908182144165, + "objective/train/avg_token_score": 0.01966213434934616, + "objective/train/docs_used": 1147924, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1883107423782349, + "objective/train/original_loss": 3.7858433723449707, + "objective/train/theoretical_loss": 3.426147140004139, + "objective/train/tokens_used": 2029977600, + "objective/train/unlikelihood_loss": 0.17653721570968628, + "theoretical_loss": 3.426147140004139, + "tokens_seen": 2029977600 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001944232698094283, + "loss": 1.3162, + "theoretical_loss": 3.4261379447254314, + "tokens_seen": 2030043136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019440320962888665, + "loss": 1.3185, + "theoretical_loss": 3.4261195553078756, + "tokens_seen": 2030174208 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019438314944834504, + "loss": 1.3167, + "theoretical_loss": 3.4261011674099455, + "tokens_seen": 2030305280 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001943630892678034, + "loss": 1.1265, + "theoretical_loss": 3.4260827810314174, + "tokens_seen": 2030436352 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001943430290872618, + "loss": 1.2607, + "theoretical_loss": 3.426064396172068, + "tokens_seen": 2030567424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019432296890672016, + "loss": 1.2422, + "theoretical_loss": 3.4260460128316725, + "tokens_seen": 2030698496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019430290872617855, + "loss": 1.1891, + "theoretical_loss": 3.426027631010009, + "tokens_seen": 2030829568 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001942828485456369, + "loss": 1.2843, + "theoretical_loss": 3.426009250706853, + "tokens_seen": 2030960640 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001942627883650953, + "loss": 1.2402, + "theoretical_loss": 3.4259908719219814, + "tokens_seen": 2031091712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019424272818455366, + "loss": 1.3856, + "theoretical_loss": 3.425972494655171, + "tokens_seen": 2031222784 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019422266800401202, + "loss": 1.3109, + "theoretical_loss": 3.4259541189061977, + "tokens_seen": 2031353856 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001942026078234704, + "loss": 1.2278, + "theoretical_loss": 3.4259357446748395, + "tokens_seen": 2031484928 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.388868510723114, + "objective/train/avg_token_score": 0.01857711374759674, + "objective/train/docs_used": 1149081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.13739812374115, + "objective/train/original_loss": 3.8995728492736816, + "objective/train/theoretical_loss": 3.425917371960872, + "objective/train/tokens_used": 2031616000, + "objective/train/unlikelihood_loss": 0.16916348040103912, + "theoretical_loss": 3.425917371960872, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019418254764292877, + "loss": 1.5235, + "theoretical_loss": 3.425917371960872, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019416248746238716, + "loss": 1.256, + "theoretical_loss": 3.4258990007640726, + "tokens_seen": 2031747072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019414242728184552, + "loss": 1.3441, + "theoretical_loss": 3.425880631084218, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019412236710130391, + "loss": 1.152, + "theoretical_loss": 3.425862262921085, + "tokens_seen": 2032009216 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019410230692076228, + "loss": 1.136, + "theoretical_loss": 3.4258438962744506, + "tokens_seen": 2032140288 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019408224674022067, + "loss": 1.3288, + "theoretical_loss": 3.425825531144092, + "tokens_seen": 2032271360 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019406218655967905, + "loss": 1.3472, + "theoretical_loss": 3.4258071675297854, + "tokens_seen": 2032402432 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019404212637913742, + "loss": 1.4627, + "theoretical_loss": 3.425788805431309, + "tokens_seen": 2032533504 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001940220661985958, + "loss": 1.4205, + "theoretical_loss": 3.42577044484844, + "tokens_seen": 2032664576 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019400200601805417, + "loss": 1.4317, + "theoretical_loss": 3.4257520857809545, + "tokens_seen": 2032795648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019398194583751256, + "loss": 1.2739, + "theoretical_loss": 3.4257337282286304, + "tokens_seen": 2032926720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019396188565697092, + "loss": 1.3848, + "theoretical_loss": 3.4257153721912452, + "tokens_seen": 2033057792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001939418254764293, + "loss": 1.3008, + "theoretical_loss": 3.4256970176685755, + "tokens_seen": 2033188864 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.3519061505794525, + "objective/train/avg_token_score": 0.025898415595293045, + "objective/train/docs_used": 1149676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1244767904281616, + "objective/train/original_loss": 4.000161170959473, + "objective/train/theoretical_loss": 3.42568784097519, + "objective/train/tokens_used": 2033254400, + "objective/train/unlikelihood_loss": 0.14115819334983826, + "theoretical_loss": 3.42568784097519, + "tokens_seen": 2033254400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019392176529588767, + "loss": 1.1916, + "theoretical_loss": 3.425678664660399, + "tokens_seen": 2033319936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019390170511534606, + "loss": 1.5538, + "theoretical_loss": 3.4256603131664933, + "tokens_seen": 2033451008 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019388164493480442, + "loss": 1.3516, + "theoretical_loss": 3.425641963186636, + "tokens_seen": 2033582080 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019386158475426279, + "loss": 1.353, + "theoretical_loss": 3.425623614720604, + "tokens_seen": 2033713152 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019384152457372117, + "loss": 1.2143, + "theoretical_loss": 3.4256052677681756, + "tokens_seen": 2033844224 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019382146439317954, + "loss": 1.173, + "theoretical_loss": 3.425586922329128, + "tokens_seen": 2033975296 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019380140421263793, + "loss": 1.0683, + "theoretical_loss": 3.4255685784032393, + "tokens_seen": 2034106368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001937813440320963, + "loss": 1.1977, + "theoretical_loss": 3.4255502359902863, + "tokens_seen": 2034237440 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019376128385155468, + "loss": 1.2632, + "theoretical_loss": 3.425531895090047, + "tokens_seen": 2034368512 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019374122367101304, + "loss": 1.4042, + "theoretical_loss": 3.4255135557022998, + "tokens_seen": 2034499584 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019372116349047143, + "loss": 1.2158, + "theoretical_loss": 3.425495217826822, + "tokens_seen": 2034630656 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001937011033099298, + "loss": 1.226, + "theoretical_loss": 3.425476881463392, + "tokens_seen": 2034761728 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.24712854623794556, + "objective/train/avg_token_score": 0.018686605617403984, + "objective/train/docs_used": 1150523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.711345374584198, + "objective/train/original_loss": 3.7477927207946777, + "objective/train/theoretical_loss": 3.425458546611787, + "objective/train/tokens_used": 2034892800, + "objective/train/unlikelihood_loss": 0.27138465642929077, + "theoretical_loss": 3.425458546611787, + "tokens_seen": 2034892800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019368104312938818, + "loss": 1.2619, + "theoretical_loss": 3.425458546611787, + "tokens_seen": 2034892800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019366098294884654, + "loss": 1.2352, + "theoretical_loss": 3.4254402132717856, + "tokens_seen": 2035023872 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001936409227683049, + "loss": 1.1437, + "theoretical_loss": 3.4254218814431656, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001936208625877633, + "loss": 1.2313, + "theoretical_loss": 3.4254035511257053, + "tokens_seen": 2035286016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019360080240722166, + "loss": 1.4129, + "theoretical_loss": 3.425385222319182, + "tokens_seen": 2035417088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019358074222668005, + "loss": 1.125, + "theoretical_loss": 3.425366895023375, + "tokens_seen": 2035548160 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001935606820461384, + "loss": 1.3427, + "theoretical_loss": 3.425348569238062, + "tokens_seen": 2035679232 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001935406218655968, + "loss": 1.4264, + "theoretical_loss": 3.425330244963021, + "tokens_seen": 2035810304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019352056168505516, + "loss": 1.2679, + "theoretical_loss": 3.425311922198031, + "tokens_seen": 2035941376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019350050150451355, + "loss": 1.2748, + "theoretical_loss": 3.4252936009428696, + "tokens_seen": 2036072448 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001934804413239719, + "loss": 1.2781, + "theoretical_loss": 3.4252752811973157, + "tokens_seen": 2036203520 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019346038114343027, + "loss": 1.3301, + "theoretical_loss": 3.4252569629611473, + "tokens_seen": 2036334592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019344032096288866, + "loss": 1.2218, + "theoretical_loss": 3.4252386462341438, + "tokens_seen": 2036465664 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.2792644202709198, + "objective/train/avg_token_score": 0.014644605107605457, + "objective/train/docs_used": 1151127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8597248792648315, + "objective/train/original_loss": 3.6483452320098877, + "objective/train/theoretical_loss": 3.4252294884365093, + "objective/train/tokens_used": 2036531200, + "objective/train/unlikelihood_loss": 0.21187205612659454, + "theoretical_loss": 3.4252294884365093, + "tokens_seen": 2036531200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019342026078234703, + "loss": 1.1974, + "theoretical_loss": 3.425220331016083, + "tokens_seen": 2036596736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019340020060180542, + "loss": 1.2341, + "theoretical_loss": 3.4252020173067432, + "tokens_seen": 2036727808 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019338014042126378, + "loss": 1.3838, + "theoretical_loss": 3.4251837051059035, + "tokens_seen": 2036858880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019336008024072217, + "loss": 1.3765, + "theoretical_loss": 3.4251653944133427, + "tokens_seen": 2036989952 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019334002006018053, + "loss": 1.3451, + "theoretical_loss": 3.4251470852288395, + "tokens_seen": 2037121024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019331995987963892, + "loss": 1.2376, + "theoretical_loss": 3.4251287775521724, + "tokens_seen": 2037252096 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001932998996990973, + "loss": 1.2049, + "theoretical_loss": 3.42511047138312, + "tokens_seen": 2037383168 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001932798395185557, + "loss": 1.2372, + "theoretical_loss": 3.4250921667214627, + "tokens_seen": 2037514240 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019325977933801406, + "loss": 1.5108, + "theoretical_loss": 3.4250738635669773, + "tokens_seen": 2037645312 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019323971915747242, + "loss": 1.277, + "theoretical_loss": 3.425055561919444, + "tokens_seen": 2037776384 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001932196589769308, + "loss": 1.179, + "theoretical_loss": 3.4250372617786415, + "tokens_seen": 2037907456 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019319959879638917, + "loss": 1.4128, + "theoretical_loss": 3.4250189631443493, + "tokens_seen": 2038038528 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.40786901116371155, + "objective/train/avg_token_score": 0.01708858273923397, + "objective/train/docs_used": 1152471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2183659076690674, + "objective/train/original_loss": 3.896066188812256, + "objective/train/theoretical_loss": 3.425000666016346, + "objective/train/tokens_used": 2038169600, + "objective/train/unlikelihood_loss": 0.1406579315662384, + "theoretical_loss": 3.425000666016346, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019317953861584756, + "loss": 1.2599, + "theoretical_loss": 3.425000666016346, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019315947843530592, + "loss": 1.2787, + "theoretical_loss": 3.4249823703944102, + "tokens_seen": 2038300672 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019313941825476431, + "loss": 1.3544, + "theoretical_loss": 3.4249640762783224, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019311935807422268, + "loss": 1.3682, + "theoretical_loss": 3.424945783667861, + "tokens_seen": 2038562816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019309929789368107, + "loss": 1.5372, + "theoretical_loss": 3.4249274925628055, + "tokens_seen": 2038693888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019307923771313943, + "loss": 1.4753, + "theoretical_loss": 3.4249092029629353, + "tokens_seen": 2038824960 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001930591775325978, + "loss": 1.4235, + "theoretical_loss": 3.4248909148680298, + "tokens_seen": 2038956032 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019303911735205618, + "loss": 1.3222, + "theoretical_loss": 3.424872628277868, + "tokens_seen": 2039087104 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019301905717151454, + "loss": 1.2822, + "theoretical_loss": 3.42485434319223, + "tokens_seen": 2039218176 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019299899699097293, + "loss": 1.2569, + "theoretical_loss": 3.424836059610895, + "tokens_seen": 2039349248 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001929789368104313, + "loss": 1.4989, + "theoretical_loss": 3.424817777533643, + "tokens_seen": 2039480320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019295887662988968, + "loss": 1.3222, + "theoretical_loss": 3.4247994969602527, + "tokens_seen": 2039611392 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019293881644934804, + "loss": 1.4253, + "theoretical_loss": 3.424781217890504, + "tokens_seen": 2039742464 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.4469085931777954, + "objective/train/avg_token_score": 0.008446778170764446, + "objective/train/docs_used": 1153219, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.539851427078247, + "objective/train/original_loss": 3.7520570755004883, + "objective/train/theoretical_loss": 3.4247720789194274, + "objective/train/tokens_used": 2039808000, + "objective/train/unlikelihood_loss": 0.13463537395000458, + "theoretical_loss": 3.4247720789194274, + "tokens_seen": 2039808000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019291875626880643, + "loss": 1.3778, + "theoretical_loss": 3.424762940324178, + "tokens_seen": 2039873536 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928986960882648, + "loss": 1.3096, + "theoretical_loss": 3.4247446642610524, + "tokens_seen": 2040004608 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019287863590772316, + "loss": 1.4484, + "theoretical_loss": 3.4247263897009086, + "tokens_seen": 2040135680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019285857572718155, + "loss": 1.3066, + "theoretical_loss": 3.4247081166435254, + "tokens_seen": 2040266752 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928385155466399, + "loss": 1.3173, + "theoretical_loss": 3.4246898450886833, + "tokens_seen": 2040397824 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928184553660983, + "loss": 1.3338, + "theoretical_loss": 3.424671575036162, + "tokens_seen": 2040528896 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019279839518555666, + "loss": 1.2928, + "theoretical_loss": 3.4246533064857414, + "tokens_seen": 2040659968 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019277833500501505, + "loss": 1.2588, + "theoretical_loss": 3.424635039437202, + "tokens_seen": 2040791040 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001927582748244734, + "loss": 1.3798, + "theoretical_loss": 3.4246167738903233, + "tokens_seen": 2040922112 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001927382146439318, + "loss": 1.3602, + "theoretical_loss": 3.424598509844886, + "tokens_seen": 2041053184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019271815446339017, + "loss": 1.4177, + "theoretical_loss": 3.4245802473006695, + "tokens_seen": 2041184256 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019269809428284853, + "loss": 1.349, + "theoretical_loss": 3.4245619862574546, + "tokens_seen": 2041315328 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.4329789876937866, + "objective/train/avg_token_score": 0.006498618051409721, + "objective/train/docs_used": 1154501, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3364782333374023, + "objective/train/original_loss": 3.4117558002471924, + "objective/train/theoretical_loss": 3.4245437267150214, + "objective/train/tokens_used": 2041446400, + "objective/train/unlikelihood_loss": 0.2190694361925125, + "theoretical_loss": 3.4245437267150214, + "tokens_seen": 2041446400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019267803410230692, + "loss": 1.3503, + "theoretical_loss": 3.4245437267150214, + "tokens_seen": 2041446400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019265797392176528, + "loss": 1.3091, + "theoretical_loss": 3.4245254686731506, + "tokens_seen": 2041577472 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019263791374122367, + "loss": 1.2828, + "theoretical_loss": 3.424507212131622, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019261785356068203, + "loss": 1.2733, + "theoretical_loss": 3.424488957090216, + "tokens_seen": 2041839616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019259779338014042, + "loss": 1.3694, + "theoretical_loss": 3.424470703548714, + "tokens_seen": 2041970688 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019257773319959878, + "loss": 1.2855, + "theoretical_loss": 3.4244524515068946, + "tokens_seen": 2042101760 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001925576730190572, + "loss": 1.1566, + "theoretical_loss": 3.4244342009645403, + "tokens_seen": 2042232832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019253761283851556, + "loss": 1.4194, + "theoretical_loss": 3.424415951921431, + "tokens_seen": 2042363904 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019251755265797395, + "loss": 1.2292, + "theoretical_loss": 3.4243977043773466, + "tokens_seen": 2042494976 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001924974924774323, + "loss": 1.2508, + "theoretical_loss": 3.424379458332069, + "tokens_seen": 2042626048 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019247743229689067, + "loss": 1.3329, + "theoretical_loss": 3.424361213785378, + "tokens_seen": 2042757120 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019245737211634906, + "loss": 1.2788, + "theoretical_loss": 3.424342970737055, + "tokens_seen": 2042888192 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019243731193580743, + "loss": 1.3596, + "theoretical_loss": 3.42432472918688, + "tokens_seen": 2043019264 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.4496578574180603, + "objective/train/avg_token_score": 0.015820374712347984, + "objective/train/docs_used": 1155127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4653973579406738, + "objective/train/original_loss": 3.735863208770752, + "objective/train/theoretical_loss": 3.4243156089735303, + "objective/train/tokens_used": 2043084800, + "objective/train/unlikelihood_loss": 0.19792599976062775, + "theoretical_loss": 3.4243156089735303, + "tokens_seen": 2043084800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019241725175526582, + "loss": 1.5318, + "theoretical_loss": 3.424306489134635, + "tokens_seen": 2043150336 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019239719157472418, + "loss": 1.4483, + "theoretical_loss": 3.4242882505801, + "tokens_seen": 2043281408 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019237713139418257, + "loss": 1.3635, + "theoretical_loss": 3.4242700135230564, + "tokens_seen": 2043412480 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019235707121364093, + "loss": 1.4775, + "theoretical_loss": 3.424251777963285, + "tokens_seen": 2043543552 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019233701103309932, + "loss": 1.1651, + "theoretical_loss": 3.424233543900567, + "tokens_seen": 2043674624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019231695085255768, + "loss": 1.2092, + "theoretical_loss": 3.4242153113346836, + "tokens_seen": 2043805696 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019229689067201604, + "loss": 1.2156, + "theoretical_loss": 3.424197080265415, + "tokens_seen": 2043936768 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019227683049147443, + "loss": 1.1812, + "theoretical_loss": 3.4241788506925444, + "tokens_seen": 2044067840 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001922567703109328, + "loss": 1.4056, + "theoretical_loss": 3.424160622615851, + "tokens_seen": 2044198912 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019223671013039118, + "loss": 1.4166, + "theoretical_loss": 3.4241423960351165, + "tokens_seen": 2044329984 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019221664994984955, + "loss": 1.3897, + "theoretical_loss": 3.4241241709501233, + "tokens_seen": 2044461056 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019219658976930794, + "loss": 1.2977, + "theoretical_loss": 3.424105947360652, + "tokens_seen": 2044592128 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.43536168336868286, + "objective/train/avg_token_score": 0.03392334654927254, + "objective/train/docs_used": 1156711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4475445747375488, + "objective/train/original_loss": 4.269580364227295, + "objective/train/theoretical_loss": 3.4240877252664834, + "objective/train/tokens_used": 2044723200, + "objective/train/unlikelihood_loss": 0.12212424725294113, + "theoretical_loss": 3.4240877252664834, + "tokens_seen": 2044723200 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921765295887663, + "loss": 1.3686, + "theoretical_loss": 3.4240877252664834, + "tokens_seen": 2044723200 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921564694082247, + "loss": 1.2838, + "theoretical_loss": 3.4240695046674006, + "tokens_seen": 2044854272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019213640922768305, + "loss": 1.1874, + "theoretical_loss": 3.4240512855631833, + "tokens_seen": 2044985344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019211634904714144, + "loss": 1.2373, + "theoretical_loss": 3.424033067953615, + "tokens_seen": 2045116416 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001920962888665998, + "loss": 1.286, + "theoretical_loss": 3.424014851838475, + "tokens_seen": 2045247488 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019207622868605816, + "loss": 1.2294, + "theoretical_loss": 3.423996637217547, + "tokens_seen": 2045378560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019205616850551655, + "loss": 1.1562, + "theoretical_loss": 3.4239784240906115, + "tokens_seen": 2045509632 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019203610832497491, + "loss": 1.3549, + "theoretical_loss": 3.423960212457451, + "tokens_seen": 2045640704 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001920160481444333, + "loss": 1.1887, + "theoretical_loss": 3.423942002317846, + "tokens_seen": 2045771776 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019199598796389167, + "loss": 1.2474, + "theoretical_loss": 3.4239237936715803, + "tokens_seen": 2045902848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019197592778335006, + "loss": 1.2338, + "theoretical_loss": 3.4239055865184342, + "tokens_seen": 2046033920 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019195586760280842, + "loss": 1.3494, + "theoretical_loss": 3.42388738085819, + "tokens_seen": 2046164992 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001919358074222668, + "loss": 1.2523, + "theoretical_loss": 3.42386917669063, + "tokens_seen": 2046296064 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.3826368451118469, + "objective/train/avg_token_score": 0.02491237409412861, + "objective/train/docs_used": 1157531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2968149185180664, + "objective/train/original_loss": 4.028264999389648, + "objective/train/theoretical_loss": 3.4238600751665382, + "objective/train/tokens_used": 2046361600, + "objective/train/unlikelihood_loss": 0.20052728056907654, + "theoretical_loss": 3.4238600751665382, + "tokens_seen": 2046361600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019191574724172517, + "loss": 1.1865, + "theoretical_loss": 3.4238509740155356, + "tokens_seen": 2046427136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019189568706118353, + "loss": 1.2486, + "theoretical_loss": 3.42383277283269, + "tokens_seen": 2046558208 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019187562688064192, + "loss": 1.3969, + "theoretical_loss": 3.4238145731418737, + "tokens_seen": 2046689280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019185556670010028, + "loss": 1.5216, + "theoretical_loss": 3.4237963749428704, + "tokens_seen": 2046820352 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019183550651955867, + "loss": 1.3715, + "theoretical_loss": 3.4237781782354615, + "tokens_seen": 2046951424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019181544633901706, + "loss": 1.2429, + "theoretical_loss": 3.423759983019429, + "tokens_seen": 2047082496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019179538615847545, + "loss": 1.2624, + "theoretical_loss": 3.423741789294556, + "tokens_seen": 2047213568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019177532597793381, + "loss": 1.1586, + "theoretical_loss": 3.4237235970606243, + "tokens_seen": 2047344640 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001917552657973922, + "loss": 1.2547, + "theoretical_loss": 3.4237054063174166, + "tokens_seen": 2047475712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019173520561685057, + "loss": 1.3392, + "theoretical_loss": 3.423687217064715, + "tokens_seen": 2047606784 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019171514543630893, + "loss": 1.3896, + "theoretical_loss": 3.4236690293023018, + "tokens_seen": 2047737856 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019169508525576732, + "loss": 1.2296, + "theoretical_loss": 3.42365084302996, + "tokens_seen": 2047868928 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.3274071216583252, + "objective/train/avg_token_score": 0.025432568043470383, + "objective/train/docs_used": 1159006, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0220048427581787, + "objective/train/original_loss": 4.2149858474731445, + "objective/train/theoretical_loss": 3.4236326582474716, + "objective/train/tokens_used": 2048000000, + "objective/train/unlikelihood_loss": 0.1484479457139969, + "theoretical_loss": 3.4236326582474716, + "tokens_seen": 2048000000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019167502507522568, + "loss": 1.3738, + "theoretical_loss": 3.4236326582474716, + "tokens_seen": 2048000000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019165496489468407, + "loss": 1.2834, + "theoretical_loss": 3.42361447495462, + "tokens_seen": 2048131072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019163490471414243, + "loss": 1.4235, + "theoretical_loss": 3.4235962931511867, + "tokens_seen": 2048262144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019161484453360082, + "loss": 1.2888, + "theoretical_loss": 3.423578112836956, + "tokens_seen": 2048393216 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019159478435305918, + "loss": 1.1617, + "theoretical_loss": 3.4235599340117098, + "tokens_seen": 2048524288 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019157472417251757, + "loss": 1.3092, + "theoretical_loss": 3.42354175667523, + "tokens_seen": 2048655360 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019155466399197593, + "loss": 1.2706, + "theoretical_loss": 3.4235235808273012, + "tokens_seen": 2048786432 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019153460381143432, + "loss": 1.392, + "theoretical_loss": 3.423505406467705, + "tokens_seen": 2048917504 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019151454363089269, + "loss": 1.2192, + "theoretical_loss": 3.4234872335962248, + "tokens_seen": 2049048576 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019149448345035105, + "loss": 1.3371, + "theoretical_loss": 3.423469062212643, + "tokens_seen": 2049179648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019147442326980944, + "loss": 1.3552, + "theoretical_loss": 3.4234508923167435, + "tokens_seen": 2049310720 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001914543630892678, + "loss": 1.3405, + "theoretical_loss": 3.423432723908309, + "tokens_seen": 2049441792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001914343029087262, + "loss": 1.5371, + "theoretical_loss": 3.4234145569871224, + "tokens_seen": 2049572864 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.5641495585441589, + "objective/train/avg_token_score": 0.003495381912216544, + "objective/train/docs_used": 1159734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9136897325515747, + "objective/train/original_loss": 3.6474218368530273, + "objective/train/theoretical_loss": 3.4234054740841797, + "objective/train/tokens_used": 2049638400, + "objective/train/unlikelihood_loss": 0.1220756471157074, + "theoretical_loss": 3.4234054740841797, + "tokens_seen": 2049638400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019141424272818455, + "loss": 1.3243, + "theoretical_loss": 3.4233963915529673, + "tokens_seen": 2049703936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019139418254764294, + "loss": 1.4569, + "theoretical_loss": 3.4233782276056264, + "tokens_seen": 2049835008 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001913741223671013, + "loss": 1.3257, + "theoretical_loss": 3.4233600651448834, + "tokens_seen": 2049966080 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001913540621865597, + "loss": 1.3735, + "theoretical_loss": 3.4233419041705213, + "tokens_seen": 2050097152 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019133400200601805, + "loss": 1.3743, + "theoretical_loss": 3.4233237446823237, + "tokens_seen": 2050228224 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019131394182547642, + "loss": 1.3561, + "theoretical_loss": 3.4233055866800735, + "tokens_seen": 2050359296 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001912938816449348, + "loss": 1.2436, + "theoretical_loss": 3.4232874301635543, + "tokens_seen": 2050490368 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019127382146439317, + "loss": 1.2332, + "theoretical_loss": 3.42326927513255, + "tokens_seen": 2050621440 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019125376128385156, + "loss": 1.3591, + "theoretical_loss": 3.4232511215868433, + "tokens_seen": 2050752512 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019123370110330992, + "loss": 1.1879, + "theoretical_loss": 3.423232969526219, + "tokens_seen": 2050883584 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001912136409227683, + "loss": 1.2917, + "theoretical_loss": 3.4232148189504596, + "tokens_seen": 2051014656 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019119358074222667, + "loss": 1.1857, + "theoretical_loss": 3.4231966698593492, + "tokens_seen": 2051145728 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.39198437333106995, + "objective/train/avg_token_score": 0.022142184898257256, + "objective/train/docs_used": 1160185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3584237098693848, + "objective/train/original_loss": 4.353691577911377, + "objective/train/theoretical_loss": 3.423178522252672, + "objective/train/tokens_used": 2051276800, + "objective/train/unlikelihood_loss": 0.16669486463069916, + "theoretical_loss": 3.423178522252672, + "tokens_seen": 2051276800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019117352056168506, + "loss": 1.3531, + "theoretical_loss": 3.423178522252672, + "tokens_seen": 2051276800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019115346038114342, + "loss": 1.1287, + "theoretical_loss": 3.4231603761302107, + "tokens_seen": 2051407872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019113340020060178, + "loss": 1.1829, + "theoretical_loss": 3.4231422314917497, + "tokens_seen": 2051538944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019111334002006017, + "loss": 1.3915, + "theoretical_loss": 3.4231240883370724, + "tokens_seen": 2051670016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019109327983951854, + "loss": 1.235, + "theoretical_loss": 3.423105946665963, + "tokens_seen": 2051801088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019107321965897693, + "loss": 1.1524, + "theoretical_loss": 3.4230878064782058, + "tokens_seen": 2051932160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019105315947843532, + "loss": 1.1607, + "theoretical_loss": 3.4230696677735843, + "tokens_seen": 2052063232 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001910330992978937, + "loss": 1.3511, + "theoretical_loss": 3.4230515305518825, + "tokens_seen": 2052194304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019101303911735207, + "loss": 1.2387, + "theoretical_loss": 3.423033394812885, + "tokens_seen": 2052325376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019099297893681046, + "loss": 1.2372, + "theoretical_loss": 3.423015260556375, + "tokens_seen": 2052456448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019097291875626882, + "loss": 1.2182, + "theoretical_loss": 3.4229971277821374, + "tokens_seen": 2052587520 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001909528585757272, + "loss": 1.3404, + "theoretical_loss": 3.422978996489956, + "tokens_seen": 2052718592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019093279839518557, + "loss": 1.1858, + "theoretical_loss": 3.422960866679616, + "tokens_seen": 2052849664 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.25360459089279175, + "objective/train/avg_token_score": 0.030829988420009613, + "objective/train/docs_used": 1161710, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8343194127082825, + "objective/train/original_loss": 4.547183036804199, + "objective/train/theoretical_loss": 3.422951802330068, + "objective/train/tokens_used": 2052915200, + "objective/train/unlikelihood_loss": 0.14609354734420776, + "theoretical_loss": 3.422951802330068, + "tokens_seen": 2052915200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019091273821464393, + "loss": 1.1222, + "theoretical_loss": 3.4229427383508995, + "tokens_seen": 2052980736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019089267803410232, + "loss": 1.2972, + "theoretical_loss": 3.422924611503593, + "tokens_seen": 2053111808 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019087261785356068, + "loss": 1.2939, + "theoretical_loss": 3.4229064861374803, + "tokens_seen": 2053242880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019085255767301907, + "loss": 1.4669, + "theoretical_loss": 3.422888362252345, + "tokens_seen": 2053373952 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019083249749247744, + "loss": 1.239, + "theoretical_loss": 3.422870239847973, + "tokens_seen": 2053505024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019081243731193582, + "loss": 1.3654, + "theoretical_loss": 3.4228521189241476, + "tokens_seen": 2053636096 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907923771313942, + "loss": 1.1877, + "theoretical_loss": 3.4228339994806536, + "tokens_seen": 2053767168 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019077231695085258, + "loss": 1.2379, + "theoretical_loss": 3.422815881517276, + "tokens_seen": 2053898240 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019075225677031094, + "loss": 1.4069, + "theoretical_loss": 3.422797765033799, + "tokens_seen": 2054029312 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907321965897693, + "loss": 1.228, + "theoretical_loss": 3.422779650030008, + "tokens_seen": 2054160384 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907121364092277, + "loss": 1.2724, + "theoretical_loss": 3.422761536505687, + "tokens_seen": 2054291456 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019069207622868605, + "loss": 1.2132, + "theoretical_loss": 3.422743424460621, + "tokens_seen": 2054422528 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.4751954972743988, + "objective/train/avg_token_score": 0.010146962478756905, + "objective/train/docs_used": 1162341, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3571432828903198, + "objective/train/original_loss": 3.59368634223938, + "objective/train/theoretical_loss": 3.4227253138945946, + "objective/train/tokens_used": 2054553600, + "objective/train/unlikelihood_loss": 0.20798954367637634, + "theoretical_loss": 3.4227253138945946, + "tokens_seen": 2054553600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019067201604814444, + "loss": 1.3132, + "theoretical_loss": 3.4227253138945946, + "tokens_seen": 2054553600 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001906519558676028, + "loss": 1.1858, + "theoretical_loss": 3.422707204807393, + "tokens_seen": 2054684672 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001906318956870612, + "loss": 1.4478, + "theoretical_loss": 3.422689097198802, + "tokens_seen": 2054815744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019061183550651956, + "loss": 1.4707, + "theoretical_loss": 3.422670991068605, + "tokens_seen": 2054946816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019059177532597794, + "loss": 1.4419, + "theoretical_loss": 3.4226528864165875, + "tokens_seen": 2055077888 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001905717151454363, + "loss": 1.3456, + "theoretical_loss": 3.422634783242535, + "tokens_seen": 2055208960 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019055165496489467, + "loss": 1.2471, + "theoretical_loss": 3.4226166815462316, + "tokens_seen": 2055340032 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019053159478435306, + "loss": 1.4009, + "theoretical_loss": 3.422598581327464, + "tokens_seen": 2055471104 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019051153460381142, + "loss": 1.2464, + "theoretical_loss": 3.4225804825860164, + "tokens_seen": 2055602176 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904914744232698, + "loss": 1.3075, + "theoretical_loss": 3.422562385321674, + "tokens_seen": 2055733248 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019047141424272817, + "loss": 1.3368, + "theoretical_loss": 3.422544289534222, + "tokens_seen": 2055864320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019045135406218656, + "loss": 1.3311, + "theoretical_loss": 3.4225261952234467, + "tokens_seen": 2055995392 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019043129388164492, + "loss": 1.4663, + "theoretical_loss": 3.4225081023891324, + "tokens_seen": 2056126464 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.3893572688102722, + "objective/train/avg_token_score": 0.04402134194970131, + "objective/train/docs_used": 1163822, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.163152813911438, + "objective/train/original_loss": 4.168545722961426, + "objective/train/theoretical_loss": 3.422499056525581, + "objective/train/tokens_used": 2056192000, + "objective/train/unlikelihood_loss": 0.18308132886886597, + "theoretical_loss": 3.422499056525581, + "tokens_seen": 2056192000 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904112337011033, + "loss": 1.1567, + "theoretical_loss": 3.4224900110310648, + "tokens_seen": 2056257536 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019039117352056168, + "loss": 1.2533, + "theoretical_loss": 3.4224719211490293, + "tokens_seen": 2056388608 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019037111334002007, + "loss": 1.2295, + "theoretical_loss": 3.4224538327428116, + "tokens_seen": 2056519680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019035105315947843, + "loss": 1.4224, + "theoretical_loss": 3.422435745812197, + "tokens_seen": 2056650752 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001903309929789368, + "loss": 1.3041, + "theoretical_loss": 3.422417660356972, + "tokens_seen": 2056781824 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001903109327983952, + "loss": 1.432, + "theoretical_loss": 3.422399576376921, + "tokens_seen": 2056912896 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019029087261785357, + "loss": 1.1512, + "theoretical_loss": 3.42238149387183, + "tokens_seen": 2057043968 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019027081243731196, + "loss": 1.3266, + "theoretical_loss": 3.4223634128414853, + "tokens_seen": 2057175040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019025075225677032, + "loss": 1.2752, + "theoretical_loss": 3.4223453332856724, + "tokens_seen": 2057306112 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001902306920762287, + "loss": 1.3016, + "theoretical_loss": 3.422327255204177, + "tokens_seen": 2057437184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019021063189568707, + "loss": 1.3568, + "theoretical_loss": 3.4223091785967847, + "tokens_seen": 2057568256 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019019057171514546, + "loss": 1.3513, + "theoretical_loss": 3.422291103463282, + "tokens_seen": 2057699328 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.39962121844291687, + "objective/train/avg_token_score": 0.008448837324976921, + "objective/train/docs_used": 1164494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2013311386108398, + "objective/train/original_loss": 4.044212341308594, + "objective/train/theoretical_loss": 3.4222730298034545, + "objective/train/tokens_used": 2057830400, + "objective/train/unlikelihood_loss": 0.11889484524726868, + "theoretical_loss": 3.4222730298034545, + "tokens_seen": 2057830400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019017051153460382, + "loss": 1.3079, + "theoretical_loss": 3.4222730298034545, + "tokens_seen": 2057830400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019015045135406219, + "loss": 1.3255, + "theoretical_loss": 3.4222549576170875, + "tokens_seen": 2057961472 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019013039117352057, + "loss": 1.4267, + "theoretical_loss": 3.4222368869039688, + "tokens_seen": 2058092544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019011033099297894, + "loss": 1.2469, + "theoretical_loss": 3.4222188176638833, + "tokens_seen": 2058223616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019009027081243733, + "loss": 1.3486, + "theoretical_loss": 3.422200749896617, + "tokens_seen": 2058354688 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001900702106318957, + "loss": 1.2776, + "theoretical_loss": 3.4221826836019567, + "tokens_seen": 2058485760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019005015045135408, + "loss": 1.2566, + "theoretical_loss": 3.422164618779688, + "tokens_seen": 2058616832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019003009027081244, + "loss": 1.4991, + "theoretical_loss": 3.4221465554295976, + "tokens_seen": 2058747904 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019001003009027083, + "loss": 1.3104, + "theoretical_loss": 3.4221284935514715, + "tokens_seen": 2058878976 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001899899699097292, + "loss": 1.4157, + "theoretical_loss": 3.4221104331450967, + "tokens_seen": 2059010048 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018996990972918758, + "loss": 1.1301, + "theoretical_loss": 3.4220923742102585, + "tokens_seen": 2059141120 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018994984954864594, + "loss": 1.496, + "theoretical_loss": 3.4220743167467442, + "tokens_seen": 2059272192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001899297893681043, + "loss": 1.123, + "theoretical_loss": 3.4220562607543403, + "tokens_seen": 2059403264 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.35453322529792786, + "objective/train/avg_token_score": 0.016934115439653397, + "objective/train/docs_used": 1165816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0977104902267456, + "objective/train/original_loss": 3.8269641399383545, + "objective/train/theoretical_loss": 3.422047233309738, + "objective/train/tokens_used": 2059468800, + "objective/train/unlikelihood_loss": 0.14981546998023987, + "theoretical_loss": 3.422047233309738, + "tokens_seen": 2059468800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001899097291875627, + "loss": 1.391, + "theoretical_loss": 3.422038206232833, + "tokens_seen": 2059534336 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018988966900702106, + "loss": 1.2238, + "theoretical_loss": 3.422020153182009, + "tokens_seen": 2059665408 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018986960882647945, + "loss": 1.3095, + "theoretical_loss": 3.4220021016016546, + "tokens_seen": 2059796480 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001898495486459378, + "loss": 1.2945, + "theoretical_loss": 3.421984051491557, + "tokens_seen": 2059927552 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001898294884653962, + "loss": 1.2047, + "theoretical_loss": 3.4219660028515024, + "tokens_seen": 2060058624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018980942828485456, + "loss": 1.443, + "theoretical_loss": 3.4219479556812784, + "tokens_seen": 2060189696 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018978936810431295, + "loss": 1.4449, + "theoretical_loss": 3.421929909980671, + "tokens_seen": 2060320768 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001897693079237713, + "loss": 1.2582, + "theoretical_loss": 3.4219118657494674, + "tokens_seen": 2060451840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018974924774322967, + "loss": 1.4219, + "theoretical_loss": 3.421893822987454, + "tokens_seen": 2060582912 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018972918756268806, + "loss": 1.3213, + "theoretical_loss": 3.4218757816944185, + "tokens_seen": 2060713984 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018970912738214643, + "loss": 1.393, + "theoretical_loss": 3.421857741870147, + "tokens_seen": 2060845056 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018968906720160481, + "loss": 1.3206, + "theoretical_loss": 3.4218397035144275, + "tokens_seen": 2060976128 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.43468964099884033, + "objective/train/avg_token_score": 0.003604997182264924, + "objective/train/docs_used": 1166525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2383164167404175, + "objective/train/original_loss": 3.453770637512207, + "objective/train/theoretical_loss": 3.421821666627046, + "objective/train/tokens_used": 2061107200, + "objective/train/unlikelihood_loss": 0.18989349901676178, + "theoretical_loss": 3.421821666627046, + "tokens_seen": 2061107200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018966900702106318, + "loss": 1.2834, + "theoretical_loss": 3.421821666627046, + "tokens_seen": 2061107200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018964894684052157, + "loss": 1.2077, + "theoretical_loss": 3.4218036312077906, + "tokens_seen": 2061238272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018962888665997993, + "loss": 1.326, + "theoretical_loss": 3.4217855972564477, + "tokens_seen": 2061369344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018960882647943832, + "loss": 1.2984, + "theoretical_loss": 3.4217675647728054, + "tokens_seen": 2061500416 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018958876629889668, + "loss": 1.284, + "theoretical_loss": 3.4217495337566497, + "tokens_seen": 2061631488 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018956870611835504, + "loss": 1.1998, + "theoretical_loss": 3.421731504207769, + "tokens_seen": 2061762560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018954864593781346, + "loss": 1.3292, + "theoretical_loss": 3.42171347612595, + "tokens_seen": 2061893632 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018952858575727182, + "loss": 1.3393, + "theoretical_loss": 3.42169544951098, + "tokens_seen": 2062024704 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001895085255767302, + "loss": 1.3153, + "theoretical_loss": 3.4216774243626475, + "tokens_seen": 2062155776 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018948846539618857, + "loss": 1.3193, + "theoretical_loss": 3.421659400680739, + "tokens_seen": 2062286848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018946840521564696, + "loss": 1.4018, + "theoretical_loss": 3.4216413784650417, + "tokens_seen": 2062417920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018944834503510532, + "loss": 1.4864, + "theoretical_loss": 3.421623357715344, + "tokens_seen": 2062548992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018942828485456371, + "loss": 1.1205, + "theoretical_loss": 3.4216053384314327, + "tokens_seen": 2062680064 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.5161290168762207, + "objective/train/avg_token_score": 0.007994516752660275, + "objective/train/docs_used": 1167825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.652220606803894, + "objective/train/original_loss": 3.448573350906372, + "objective/train/theoretical_loss": 3.4215963293390805, + "objective/train/tokens_used": 2062745600, + "objective/train/unlikelihood_loss": 0.14504040777683258, + "theoretical_loss": 3.4215963293390805, + "tokens_seen": 2062745600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018940822467402208, + "loss": 1.2733, + "theoretical_loss": 3.4215873206130962, + "tokens_seen": 2062811136 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018938816449348047, + "loss": 1.3975, + "theoretical_loss": 3.421569304260122, + "tokens_seen": 2062942208 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018936810431293883, + "loss": 1.254, + "theoretical_loss": 3.4215512893722972, + "tokens_seen": 2063073280 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001893480441323972, + "loss": 1.2454, + "theoretical_loss": 3.42153327594941, + "tokens_seen": 2063204352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018932798395185558, + "loss": 1.4226, + "theoretical_loss": 3.4215152639912487, + "tokens_seen": 2063335424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018930792377131394, + "loss": 1.3802, + "theoretical_loss": 3.421497253497601, + "tokens_seen": 2063466496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018928786359077233, + "loss": 1.222, + "theoretical_loss": 3.4214792444682542, + "tokens_seen": 2063597568 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001892678034102307, + "loss": 1.3975, + "theoretical_loss": 3.4214612369029966, + "tokens_seen": 2063728640 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018924774322968908, + "loss": 1.2832, + "theoretical_loss": 3.421443230801616, + "tokens_seen": 2063859712 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018922768304914744, + "loss": 1.3854, + "theoretical_loss": 3.421425226163901, + "tokens_seen": 2063990784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018920762286860583, + "loss": 1.1803, + "theoretical_loss": 3.4214072229896395, + "tokens_seen": 2064121856 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891875626880642, + "loss": 1.3863, + "theoretical_loss": 3.421389221278619, + "tokens_seen": 2064252928 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.4355449676513672, + "objective/train/avg_token_score": 0.0201333649456501, + "objective/train/docs_used": 1168450, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3724009990692139, + "objective/train/original_loss": 3.986499786376953, + "objective/train/theoretical_loss": 3.4213712210306286, + "objective/train/tokens_used": 2064384000, + "objective/train/unlikelihood_loss": 0.13897953927516937, + "theoretical_loss": 3.4213712210306286, + "tokens_seen": 2064384000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018916750250752256, + "loss": 1.3302, + "theoretical_loss": 3.4213712210306286, + "tokens_seen": 2064384000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018914744232698095, + "loss": 1.3339, + "theoretical_loss": 3.4213532222454557, + "tokens_seen": 2064515072 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891273821464393, + "loss": 1.2135, + "theoretical_loss": 3.421335224922889, + "tokens_seen": 2064646144 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891073219658977, + "loss": 1.3095, + "theoretical_loss": 3.4213172290627165, + "tokens_seen": 2064777216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018908726178535606, + "loss": 1.4311, + "theoretical_loss": 3.421299234664727, + "tokens_seen": 2064908288 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018906720160481445, + "loss": 1.3705, + "theoretical_loss": 3.4212812417287086, + "tokens_seen": 2065039360 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001890471414242728, + "loss": 1.1941, + "theoretical_loss": 3.42126325025445, + "tokens_seen": 2065170432 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001890270812437312, + "loss": 1.3348, + "theoretical_loss": 3.4212452602417396, + "tokens_seen": 2065301504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018900702106318956, + "loss": 1.4135, + "theoretical_loss": 3.4212272716903653, + "tokens_seen": 2065432576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018898696088264793, + "loss": 1.2275, + "theoretical_loss": 3.421209284600117, + "tokens_seen": 2065563648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018896690070210632, + "loss": 1.2035, + "theoretical_loss": 3.421191298970782, + "tokens_seen": 2065694720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018894684052156468, + "loss": 1.1667, + "theoretical_loss": 3.4211733148021493, + "tokens_seen": 2065825792 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018892678034102307, + "loss": 1.3231, + "theoretical_loss": 3.421155332094008, + "tokens_seen": 2065956864 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.27199414372444153, + "objective/train/avg_token_score": 0.043950822204351425, + "objective/train/docs_used": 1168888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8560883402824402, + "objective/train/original_loss": 4.299278736114502, + "objective/train/theoretical_loss": 3.4211463412875553, + "objective/train/tokens_used": 2066022400, + "objective/train/unlikelihood_loss": 0.13019482791423798, + "theoretical_loss": 3.4211463412875553, + "tokens_seen": 2066022400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018890672016048143, + "loss": 1.1621, + "theoretical_loss": 3.4211373508461462, + "tokens_seen": 2066087936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018888665997993982, + "loss": 1.2459, + "theoretical_loss": 3.421119371058354, + "tokens_seen": 2066219008 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018886659979939818, + "loss": 1.2158, + "theoretical_loss": 3.421101392730418, + "tokens_seen": 2066350080 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018884653961885657, + "loss": 1.2238, + "theoretical_loss": 3.421083415862129, + "tokens_seen": 2066481152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018882647943831493, + "loss": 1.2636, + "theoretical_loss": 3.421065440453276, + "tokens_seen": 2066612224 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018880641925777335, + "loss": 1.2494, + "theoretical_loss": 3.4210474665036466, + "tokens_seen": 2066743296 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001887863590772317, + "loss": 1.171, + "theoretical_loss": 3.421029494013031, + "tokens_seen": 2066874368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018876629889669007, + "loss": 1.3861, + "theoretical_loss": 3.421011522981217, + "tokens_seen": 2067005440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018874623871614846, + "loss": 1.2416, + "theoretical_loss": 3.420993553407995, + "tokens_seen": 2067136512 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018872617853560683, + "loss": 1.4338, + "theoretical_loss": 3.4209755852931534, + "tokens_seen": 2067267584 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018870611835506522, + "loss": 1.3154, + "theoretical_loss": 3.4209576186364816, + "tokens_seen": 2067398656 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018868605817452358, + "loss": 1.2193, + "theoretical_loss": 3.420939653437769, + "tokens_seen": 2067529728 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.4059750735759735, + "objective/train/avg_token_score": 0.02683071419596672, + "objective/train/docs_used": 1170265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2927608489990234, + "objective/train/original_loss": 3.9401214122772217, + "objective/train/theoretical_loss": 3.4209216896968044, + "objective/train/tokens_used": 2067660800, + "objective/train/unlikelihood_loss": 0.20445214211940765, + "theoretical_loss": 3.4209216896968044, + "tokens_seen": 2067660800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018866599799398197, + "loss": 1.5033, + "theoretical_loss": 3.4209216896968044, + "tokens_seen": 2067660800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018864593781344033, + "loss": 1.2224, + "theoretical_loss": 3.4209037274133776, + "tokens_seen": 2067791872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018862587763289872, + "loss": 1.3918, + "theoretical_loss": 3.4208857665872774, + "tokens_seen": 2067922944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018860581745235708, + "loss": 1.1601, + "theoretical_loss": 3.4208678072182943, + "tokens_seen": 2068054016 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018858575727181544, + "loss": 1.3995, + "theoretical_loss": 3.420849849306216, + "tokens_seen": 2068185088 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018856569709127383, + "loss": 1.3002, + "theoretical_loss": 3.420831892850834, + "tokens_seen": 2068316160 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001885456369107322, + "loss": 1.2646, + "theoretical_loss": 3.4208139378519364, + "tokens_seen": 2068447232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018852557673019058, + "loss": 1.3921, + "theoretical_loss": 3.420795984309313, + "tokens_seen": 2068578304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018850551654964895, + "loss": 1.1631, + "theoretical_loss": 3.4207780322227537, + "tokens_seen": 2068709376 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018848545636910734, + "loss": 1.3122, + "theoretical_loss": 3.4207600815920487, + "tokens_seen": 2068840448 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001884653961885657, + "loss": 1.2486, + "theoretical_loss": 3.4207421324169864, + "tokens_seen": 2068971520 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001884453360080241, + "loss": 1.313, + "theoretical_loss": 3.4207241846973577, + "tokens_seen": 2069102592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018842527582748245, + "loss": 1.1773, + "theoretical_loss": 3.4207062384329516, + "tokens_seen": 2069233664 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.35703811049461365, + "objective/train/avg_token_score": 0.00737313786521554, + "objective/train/docs_used": 1170942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2534971237182617, + "objective/train/original_loss": 4.055935382843018, + "objective/train/theoretical_loss": 3.4206972658463917, + "objective/train/tokens_used": 2069299200, + "objective/train/unlikelihood_loss": 0.1547859013080597, + "theoretical_loss": 3.4206972658463917, + "tokens_seen": 2069299200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018840521564694084, + "loss": 1.3069, + "theoretical_loss": 3.420688293623559, + "tokens_seen": 2069364736 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001883851554663992, + "loss": 1.3389, + "theoretical_loss": 3.420670350268969, + "tokens_seen": 2069495808 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018836509528585756, + "loss": 1.3169, + "theoretical_loss": 3.420652408368971, + "tokens_seen": 2069626880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018834503510531595, + "loss": 1.1793, + "theoretical_loss": 3.420634467923356, + "tokens_seen": 2069757952 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018832497492477431, + "loss": 1.2824, + "theoretical_loss": 3.4206165289319133, + "tokens_seen": 2069889024 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001883049147442327, + "loss": 1.2711, + "theoretical_loss": 3.420598591394433, + "tokens_seen": 2070020096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018828485456369107, + "loss": 1.2756, + "theoretical_loss": 3.4205806553107063, + "tokens_seen": 2070151168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018826479438314946, + "loss": 1.3344, + "theoretical_loss": 3.420562720680522, + "tokens_seen": 2070282240 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018824473420260782, + "loss": 1.2706, + "theoretical_loss": 3.4205447875036707, + "tokens_seen": 2070413312 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001882246740220662, + "loss": 1.2108, + "theoretical_loss": 3.4205268557799435, + "tokens_seen": 2070544384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018820461384152457, + "loss": 1.3234, + "theoretical_loss": 3.420508925509129, + "tokens_seen": 2070675456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018818455366098293, + "loss": 1.2246, + "theoretical_loss": 3.420490996691019, + "tokens_seen": 2070806528 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.4875977635383606, + "objective/train/avg_token_score": 0.006257314700633287, + "objective/train/docs_used": 1172086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5040123462677002, + "objective/train/original_loss": 3.4239742755889893, + "objective/train/theoretical_loss": 3.4204730693254035, + "objective/train/tokens_used": 2070937600, + "objective/train/unlikelihood_loss": 0.2167283445596695, + "theoretical_loss": 3.4204730693254035, + "tokens_seen": 2070937600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018816449348044132, + "loss": 1.411, + "theoretical_loss": 3.4204730693254035, + "tokens_seen": 2070937600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018814443329989968, + "loss": 1.3393, + "theoretical_loss": 3.420455143412072, + "tokens_seen": 2071068672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018812437311935807, + "loss": 1.3469, + "theoretical_loss": 3.4204372189508163, + "tokens_seen": 2071199744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018810431293881643, + "loss": 1.2745, + "theoretical_loss": 3.420419295941426, + "tokens_seen": 2071330816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018808425275827482, + "loss": 1.3648, + "theoretical_loss": 3.420401374383692, + "tokens_seen": 2071461888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018806419257773319, + "loss": 1.224, + "theoretical_loss": 3.4203834542774043, + "tokens_seen": 2071592960 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001880441323971916, + "loss": 1.3676, + "theoretical_loss": 3.420365535622355, + "tokens_seen": 2071724032 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018802407221664997, + "loss": 1.2991, + "theoretical_loss": 3.420347618418333, + "tokens_seen": 2071855104 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018800401203610833, + "loss": 1.283, + "theoretical_loss": 3.4203297026651303, + "tokens_seen": 2071986176 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018798395185556672, + "loss": 1.3405, + "theoretical_loss": 3.4203117883625365, + "tokens_seen": 2072117248 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018796389167502508, + "loss": 1.3409, + "theoretical_loss": 3.4202938755103434, + "tokens_seen": 2072248320 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018794383149448347, + "loss": 1.3129, + "theoretical_loss": 3.4202759641083413, + "tokens_seen": 2072379392 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018792377131394183, + "loss": 1.261, + "theoretical_loss": 3.4202580541563217, + "tokens_seen": 2072510464 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.39992669224739075, + "objective/train/avg_token_score": 0.05949433147907257, + "objective/train/docs_used": 1172787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1607714891433716, + "objective/train/original_loss": 3.956249475479126, + "objective/train/theoretical_loss": 3.4202490997239896, + "objective/train/tokens_used": 2072576000, + "objective/train/unlikelihood_loss": 0.17929115891456604, + "theoretical_loss": 3.4202490997239896, + "tokens_seen": 2072576000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018790371113340022, + "loss": 1.2062, + "theoretical_loss": 3.4202401456540747, + "tokens_seen": 2072641536 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018788365095285858, + "loss": 1.2122, + "theoretical_loss": 3.420222238601392, + "tokens_seen": 2072772608 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018786359077231697, + "loss": 1.2913, + "theoretical_loss": 3.420204332998064, + "tokens_seen": 2072903680 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018784353059177533, + "loss": 1.2187, + "theoretical_loss": 3.4201864288438824, + "tokens_seen": 2073034752 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018782347041123372, + "loss": 1.1955, + "theoretical_loss": 3.4201685261386374, + "tokens_seen": 2073165824 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018780341023069209, + "loss": 1.4538, + "theoretical_loss": 3.420150624882121, + "tokens_seen": 2073296896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018778335005015045, + "loss": 1.1774, + "theoretical_loss": 3.4201327250741245, + "tokens_seen": 2073427968 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018776328986960884, + "loss": 1.3332, + "theoretical_loss": 3.420114826714438, + "tokens_seen": 2073559040 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001877432296890672, + "loss": 1.1652, + "theoretical_loss": 3.4200969298028543, + "tokens_seen": 2073690112 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001877231695085256, + "loss": 1.2431, + "theoretical_loss": 3.420079034339163, + "tokens_seen": 2073821184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018770310932798395, + "loss": 1.24, + "theoretical_loss": 3.420061140323157, + "tokens_seen": 2073952256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018768304914744234, + "loss": 1.2653, + "theoretical_loss": 3.420043247754627, + "tokens_seen": 2074083328 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.2939271628856659, + "objective/train/avg_token_score": 0.006637068931013346, + "objective/train/docs_used": 1174027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9568867683410645, + "objective/train/original_loss": 4.3706254959106445, + "objective/train/theoretical_loss": 3.4200253566333645, + "objective/train/tokens_used": 2074214400, + "objective/train/unlikelihood_loss": 0.14363816380500793, + "theoretical_loss": 3.4200253566333645, + "tokens_seen": 2074214400 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001876629889669007, + "loss": 1.1139, + "theoretical_loss": 3.4200253566333645, + "tokens_seen": 2074214400 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001876429287863591, + "loss": 1.2366, + "theoretical_loss": 3.420007466959161, + "tokens_seen": 2074345472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018762286860581745, + "loss": 1.2872, + "theoretical_loss": 3.4199895787318084, + "tokens_seen": 2074476544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018760280842527582, + "loss": 1.132, + "theoretical_loss": 3.4199716919510976, + "tokens_seen": 2074607616 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001875827482447342, + "loss": 1.3708, + "theoretical_loss": 3.41995380661682, + "tokens_seen": 2074738688 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018756268806419257, + "loss": 1.1812, + "theoretical_loss": 3.4199359227287687, + "tokens_seen": 2074869760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018754262788365096, + "loss": 1.2875, + "theoretical_loss": 3.419918040286734, + "tokens_seen": 2075000832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018752256770310932, + "loss": 1.4444, + "theoretical_loss": 3.419900159290509, + "tokens_seen": 2075131904 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001875025075225677, + "loss": 1.2586, + "theoretical_loss": 3.419882279739884, + "tokens_seen": 2075262976 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018748244734202607, + "loss": 1.1756, + "theoretical_loss": 3.4198644016346513, + "tokens_seen": 2075394048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018746238716148446, + "loss": 1.336, + "theoretical_loss": 3.4198465249746035, + "tokens_seen": 2075525120 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018744232698094282, + "loss": 1.3633, + "theoretical_loss": 3.419828649759532, + "tokens_seen": 2075656192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018742226680040118, + "loss": 1.3576, + "theoretical_loss": 3.4198107759892284, + "tokens_seen": 2075787264 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.24346286058425903, + "objective/train/avg_token_score": 0.015438918955624104, + "objective/train/docs_used": 1174812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7827137112617493, + "objective/train/original_loss": 3.9111907482147217, + "objective/train/theoretical_loss": 3.4198018396457996, + "objective/train/tokens_used": 2075852800, + "objective/train/unlikelihood_loss": 0.2404288798570633, + "theoretical_loss": 3.4198018396457996, + "tokens_seen": 2075852800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018740220661985957, + "loss": 1.2095, + "theoretical_loss": 3.4197929036634855, + "tokens_seen": 2075918336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018738214643931794, + "loss": 1.3511, + "theoretical_loss": 3.419775032782095, + "tokens_seen": 2076049408 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018736208625877633, + "loss": 1.3441, + "theoretical_loss": 3.4197571633448485, + "tokens_seen": 2076180480 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001873420260782347, + "loss": 1.3092, + "theoretical_loss": 3.4197392953515386, + "tokens_seen": 2076311552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018732196589769308, + "loss": 1.4166, + "theoretical_loss": 3.4197214288019575, + "tokens_seen": 2076442624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018730190571715147, + "loss": 1.3313, + "theoretical_loss": 3.419703563695898, + "tokens_seen": 2076573696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018728184553660986, + "loss": 1.3269, + "theoretical_loss": 3.419685700033151, + "tokens_seen": 2076704768 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018726178535606822, + "loss": 1.3181, + "theoretical_loss": 3.41966783781351, + "tokens_seen": 2076835840 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001872417251755266, + "loss": 1.391, + "theoretical_loss": 3.4196499770367668, + "tokens_seen": 2076966912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018722166499498497, + "loss": 1.4277, + "theoretical_loss": 3.4196321177027142, + "tokens_seen": 2077097984 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018720160481444333, + "loss": 1.2416, + "theoretical_loss": 3.4196142598111443, + "tokens_seen": 2077229056 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018718154463390172, + "loss": 1.3772, + "theoretical_loss": 3.419596403361849, + "tokens_seen": 2077360128 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.5052541494369507, + "objective/train/avg_token_score": 0.016038062050938606, + "objective/train/docs_used": 1175335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.439935326576233, + "objective/train/original_loss": 3.5808002948760986, + "objective/train/theoretical_loss": 3.4195785483546226, + "objective/train/tokens_used": 2077491200, + "objective/train/unlikelihood_loss": 0.13271597027778625, + "theoretical_loss": 3.4195785483546226, + "tokens_seen": 2077491200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018716148445336008, + "loss": 1.3748, + "theoretical_loss": 3.4195785483546226, + "tokens_seen": 2077491200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018714142427281847, + "loss": 1.4595, + "theoretical_loss": 3.419560694789256, + "tokens_seen": 2077622272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018712136409227684, + "loss": 1.3524, + "theoretical_loss": 3.419542842665542, + "tokens_seen": 2077753344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018710130391173522, + "loss": 1.315, + "theoretical_loss": 3.4195249919832738, + "tokens_seen": 2077884416 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001870812437311936, + "loss": 1.5771, + "theoretical_loss": 3.419507142742244, + "tokens_seen": 2078015488 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018706118355065198, + "loss": 1.27, + "theoretical_loss": 3.4194892949422453, + "tokens_seen": 2078146560 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018704112337011034, + "loss": 1.2319, + "theoretical_loss": 3.419471448583071, + "tokens_seen": 2078277632 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001870210631895687, + "loss": 1.3263, + "theoretical_loss": 3.419453603664512, + "tokens_seen": 2078408704 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001870010030090271, + "loss": 1.4327, + "theoretical_loss": 3.4194357601863636, + "tokens_seen": 2078539776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018698094282848545, + "loss": 1.5074, + "theoretical_loss": 3.419417918148417, + "tokens_seen": 2078670848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018696088264794384, + "loss": 1.5649, + "theoretical_loss": 3.419400077550466, + "tokens_seen": 2078801920 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001869408224674022, + "loss": 1.2979, + "theoretical_loss": 3.419382238392304, + "tokens_seen": 2078932992 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001869207622868606, + "loss": 1.4072, + "theoretical_loss": 3.419364400673723, + "tokens_seen": 2079064064 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.46047165989875793, + "objective/train/avg_token_score": 0.0175623819231987, + "objective/train/docs_used": 1176572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.254642128944397, + "objective/train/original_loss": 3.667412519454956, + "objective/train/theoretical_loss": 3.4193554823542107, + "objective/train/tokens_used": 2079129600, + "objective/train/unlikelihood_loss": 0.19171510636806488, + "theoretical_loss": 3.4193554823542107, + "tokens_seen": 2079129600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018690070210631896, + "loss": 1.278, + "theoretical_loss": 3.419346564394516, + "tokens_seen": 2079195136 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018688064192577734, + "loss": 1.3727, + "theoretical_loss": 3.4193287295544774, + "tokens_seen": 2079326208 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001868605817452357, + "loss": 1.3079, + "theoretical_loss": 3.4193108961533993, + "tokens_seen": 2079457280 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018684052156469407, + "loss": 1.3122, + "theoretical_loss": 3.4192930641910753, + "tokens_seen": 2079588352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018682046138415246, + "loss": 1.3115, + "theoretical_loss": 3.4192752336672987, + "tokens_seen": 2079719424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018680040120361082, + "loss": 1.4292, + "theoretical_loss": 3.4192574045818622, + "tokens_seen": 2079850496 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867803410230692, + "loss": 1.3561, + "theoretical_loss": 3.41923957693456, + "tokens_seen": 2079981568 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018676028084252757, + "loss": 1.2901, + "theoretical_loss": 3.4192217507251854, + "tokens_seen": 2080112640 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018674022066198596, + "loss": 1.2138, + "theoretical_loss": 3.4192039259535307, + "tokens_seen": 2080243712 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018672016048144432, + "loss": 1.3804, + "theoretical_loss": 3.419186102619391, + "tokens_seen": 2080374784 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867001003009027, + "loss": 1.2516, + "theoretical_loss": 3.4191682807225585, + "tokens_seen": 2080505856 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018668004012036108, + "loss": 1.3537, + "theoretical_loss": 3.419150460262827, + "tokens_seen": 2080636928 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.37090665102005005, + "objective/train/avg_token_score": 0.022265885025262833, + "objective/train/docs_used": 1176843, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1730748414993286, + "objective/train/original_loss": 4.142400741577148, + "objective/train/theoretical_loss": 3.419132641239991, + "objective/train/tokens_used": 2080768000, + "objective/train/unlikelihood_loss": 0.18339000642299652, + "theoretical_loss": 3.419132641239991, + "tokens_seen": 2080768000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018665997993981946, + "loss": 1.3564, + "theoretical_loss": 3.419132641239991, + "tokens_seen": 2080768000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018663991975927783, + "loss": 1.3937, + "theoretical_loss": 3.4191148236538433, + "tokens_seen": 2080899072 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001866198595787362, + "loss": 1.2509, + "theoretical_loss": 3.4190970075041776, + "tokens_seen": 2081030144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018659979939819458, + "loss": 1.5066, + "theoretical_loss": 3.4190791927907878, + "tokens_seen": 2081161216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018657973921765294, + "loss": 1.375, + "theoretical_loss": 3.419061379513468, + "tokens_seen": 2081292288 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018655967903711136, + "loss": 1.2145, + "theoretical_loss": 3.419043567672011, + "tokens_seen": 2081423360 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018653961885656972, + "loss": 1.401, + "theoretical_loss": 3.419025757266212, + "tokens_seen": 2081554432 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001865195586760281, + "loss": 1.2952, + "theoretical_loss": 3.4190079482958637, + "tokens_seen": 2081685504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018649949849548647, + "loss": 1.3254, + "theoretical_loss": 3.4189901407607612, + "tokens_seen": 2081816576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018647943831494486, + "loss": 1.2546, + "theoretical_loss": 3.4189723346606975, + "tokens_seen": 2081947648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018645937813440322, + "loss": 1.3655, + "theoretical_loss": 3.4189545299954665, + "tokens_seen": 2082078720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018643931795386159, + "loss": 1.3213, + "theoretical_loss": 3.4189367267648634, + "tokens_seen": 2082209792 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018641925777331997, + "loss": 1.2001, + "theoretical_loss": 3.4189189249686813, + "tokens_seen": 2082340864 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.4824657738208771, + "objective/train/avg_token_score": 0.007272422779351473, + "objective/train/docs_used": 1177725, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6770548820495605, + "objective/train/original_loss": 3.825765371322632, + "objective/train/theoretical_loss": 3.418910024608434, + "objective/train/tokens_used": 2082406400, + "objective/train/unlikelihood_loss": 0.1262761503458023, + "theoretical_loss": 3.418910024608434, + "tokens_seen": 2082406400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018639919759277834, + "loss": 1.343, + "theoretical_loss": 3.4189011246067147, + "tokens_seen": 2082471936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018637913741223673, + "loss": 1.3978, + "theoretical_loss": 3.418883325678758, + "tokens_seen": 2082603008 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001863590772316951, + "loss": 1.2789, + "theoretical_loss": 3.4188655281846048, + "tokens_seen": 2082734080 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018633901705115348, + "loss": 1.3441, + "theoretical_loss": 3.41884773212405, + "tokens_seen": 2082865152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018631895687061184, + "loss": 1.3174, + "theoretical_loss": 3.4188299374968874, + "tokens_seen": 2082996224 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018629889669007023, + "loss": 1.3812, + "theoretical_loss": 3.418812144302912, + "tokens_seen": 2083127296 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001862788365095286, + "loss": 1.1415, + "theoretical_loss": 3.4187943525419175, + "tokens_seen": 2083258368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018625877632898698, + "loss": 1.3467, + "theoretical_loss": 3.418776562213699, + "tokens_seen": 2083389440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018623871614844534, + "loss": 1.1406, + "theoretical_loss": 3.418758773318051, + "tokens_seen": 2083520512 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001862186559679037, + "loss": 1.2008, + "theoretical_loss": 3.4187409858547673, + "tokens_seen": 2083651584 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001861985957873621, + "loss": 1.3482, + "theoretical_loss": 3.418723199823643, + "tokens_seen": 2083782656 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018617853560682046, + "loss": 1.3521, + "theoretical_loss": 3.4187054152244727, + "tokens_seen": 2083913728 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.31042277812957764, + "objective/train/avg_token_score": 0.035424478352069855, + "objective/train/docs_used": 1178158, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0148370265960693, + "objective/train/original_loss": 4.398836135864258, + "objective/train/theoretical_loss": 3.4186876320570505, + "objective/train/tokens_used": 2084044800, + "objective/train/unlikelihood_loss": 0.1635483205318451, + "theoretical_loss": 3.4186876320570505, + "tokens_seen": 2084044800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018615847542627885, + "loss": 1.304, + "theoretical_loss": 3.4186876320570505, + "tokens_seen": 2084044800 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001861384152457372, + "loss": 1.382, + "theoretical_loss": 3.418669850321172, + "tokens_seen": 2084175872 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001861183550651956, + "loss": 1.2747, + "theoretical_loss": 3.418652070016632, + "tokens_seen": 2084306944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018609829488465396, + "loss": 1.4771, + "theoretical_loss": 3.418634291143224, + "tokens_seen": 2084438016 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018607823470411235, + "loss": 1.3078, + "theoretical_loss": 3.4186165137007443, + "tokens_seen": 2084569088 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001860581745235707, + "loss": 1.343, + "theoretical_loss": 3.4185987376889866, + "tokens_seen": 2084700160 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018603811434302907, + "loss": 1.4216, + "theoretical_loss": 3.418580963107746, + "tokens_seen": 2084831232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018601805416248746, + "loss": 1.4797, + "theoretical_loss": 3.4185631899568185, + "tokens_seen": 2084962304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018599799398194583, + "loss": 1.3219, + "theoretical_loss": 3.4185454182359982, + "tokens_seen": 2085093376 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018597793380140421, + "loss": 1.2515, + "theoretical_loss": 3.41852764794508, + "tokens_seen": 2085224448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018595787362086258, + "loss": 1.3314, + "theoretical_loss": 3.4185098790838593, + "tokens_seen": 2085355520 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018593781344032097, + "loss": 1.3007, + "theoretical_loss": 3.4184921116521316, + "tokens_seen": 2085486592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018591775325977933, + "loss": 1.3326, + "theoretical_loss": 3.4184743456496918, + "tokens_seen": 2085617664 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.5334188938140869, + "objective/train/avg_token_score": 0.009126054123044014, + "objective/train/docs_used": 1179467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4946708679199219, + "objective/train/original_loss": 3.35902738571167, + "objective/train/theoretical_loss": 3.4184654631843907, + "objective/train/tokens_used": 2085683200, + "objective/train/unlikelihood_loss": 0.15892405807971954, + "theoretical_loss": 3.4184654631843907, + "tokens_seen": 2085683200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018589769307923772, + "loss": 1.4334, + "theoretical_loss": 3.4184565810763345, + "tokens_seen": 2085748736 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018587763289869608, + "loss": 1.3031, + "theoretical_loss": 3.4184388179318557, + "tokens_seen": 2085879808 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018585757271815444, + "loss": 1.4725, + "theoretical_loss": 3.4184210562160504, + "tokens_seen": 2086010880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018583751253761283, + "loss": 1.3268, + "theoretical_loss": 3.418403295928714, + "tokens_seen": 2086141952 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001858174523570712, + "loss": 1.2117, + "theoretical_loss": 3.418385537069642, + "tokens_seen": 2086273024 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001857973921765296, + "loss": 1.3446, + "theoretical_loss": 3.418367779638629, + "tokens_seen": 2086404096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018577733199598797, + "loss": 1.2718, + "theoretical_loss": 3.4183500236354716, + "tokens_seen": 2086535168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018575727181544636, + "loss": 1.4977, + "theoretical_loss": 3.418332269059965, + "tokens_seen": 2086666240 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018573721163490472, + "loss": 1.2827, + "theoretical_loss": 3.418314515911904, + "tokens_seen": 2086797312 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018571715145436311, + "loss": 1.2308, + "theoretical_loss": 3.4182967641910853, + "tokens_seen": 2086928384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018569709127382148, + "loss": 1.4275, + "theoretical_loss": 3.418279013897304, + "tokens_seen": 2087059456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018567703109327987, + "loss": 1.3351, + "theoretical_loss": 3.418261265030355, + "tokens_seen": 2087190528 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.479227751493454, + "objective/train/avg_token_score": 0.009189991280436516, + "objective/train/docs_used": 1180081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5690261125564575, + "objective/train/original_loss": 3.7264864444732666, + "objective/train/theoretical_loss": 3.418243517590036, + "objective/train/tokens_used": 2087321600, + "objective/train/unlikelihood_loss": 0.15629832446575165, + "theoretical_loss": 3.418243517590036, + "tokens_seen": 2087321600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018565697091273823, + "loss": 1.3278, + "theoretical_loss": 3.418243517590036, + "tokens_seen": 2087321600 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001856369107321966, + "loss": 1.3919, + "theoretical_loss": 3.4182257715761404, + "tokens_seen": 2087452672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018561685055165498, + "loss": 1.1987, + "theoretical_loss": 3.4182080269884656, + "tokens_seen": 2087583744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018559679037111334, + "loss": 1.4056, + "theoretical_loss": 3.4181902838268075, + "tokens_seen": 2087714816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018557673019057173, + "loss": 1.3837, + "theoretical_loss": 3.4181725420909608, + "tokens_seen": 2087845888 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001855566700100301, + "loss": 1.3315, + "theoretical_loss": 3.418154801780722, + "tokens_seen": 2087976960 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018553660982948848, + "loss": 1.3469, + "theoretical_loss": 3.4181370628958874, + "tokens_seen": 2088108032 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018551654964894684, + "loss": 1.3684, + "theoretical_loss": 3.418119325436253, + "tokens_seen": 2088239104 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018549648946840523, + "loss": 1.3766, + "theoretical_loss": 3.4181015894016142, + "tokens_seen": 2088370176 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001854764292878636, + "loss": 1.4362, + "theoretical_loss": 3.418083854791768, + "tokens_seen": 2088501248 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018545636910732196, + "loss": 1.3235, + "theoretical_loss": 3.4180661216065094, + "tokens_seen": 2088632320 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018543630892678035, + "loss": 1.3665, + "theoretical_loss": 3.4180483898456355, + "tokens_seen": 2088763392 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001854162487462387, + "loss": 1.4133, + "theoretical_loss": 3.418030659508942, + "tokens_seen": 2088894464 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.3742057681083679, + "objective/train/avg_token_score": 0.025276506319642067, + "objective/train/docs_used": 1181528, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2268763780593872, + "objective/train/original_loss": 3.964512825012207, + "objective/train/theoretical_loss": 3.4180217948745995, + "objective/train/tokens_used": 2088960000, + "objective/train/unlikelihood_loss": 0.18654951453208923, + "theoretical_loss": 3.4180217948745995, + "tokens_seen": 2088960000 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853961885656971, + "loss": 1.2727, + "theoretical_loss": 3.418012930596226, + "tokens_seen": 2089025536 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018537612838515546, + "loss": 1.4935, + "theoretical_loss": 3.4179952031072824, + "tokens_seen": 2089156608 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018535606820461385, + "loss": 1.3226, + "theoretical_loss": 3.4179774770419087, + "tokens_seen": 2089287680 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853360080240722, + "loss": 1.2966, + "theoretical_loss": 3.4179597523999012, + "tokens_seen": 2089418752 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853159478435306, + "loss": 1.3006, + "theoretical_loss": 3.417942029181056, + "tokens_seen": 2089549824 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018529588766298896, + "loss": 1.2346, + "theoretical_loss": 3.417924307385169, + "tokens_seen": 2089680896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018527582748244733, + "loss": 1.2238, + "theoretical_loss": 3.4179065870120375, + "tokens_seen": 2089811968 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018525576730190572, + "loss": 1.3456, + "theoretical_loss": 3.4178888680614583, + "tokens_seen": 2089943040 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018523570712136408, + "loss": 1.2429, + "theoretical_loss": 3.4178711505332275, + "tokens_seen": 2090074112 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018521564694082247, + "loss": 1.4127, + "theoretical_loss": 3.4178534344271414, + "tokens_seen": 2090205184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018519558676028083, + "loss": 1.3871, + "theoretical_loss": 3.417835719742998, + "tokens_seen": 2090336256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018517552657973922, + "loss": 1.2962, + "theoretical_loss": 3.417818006480592, + "tokens_seen": 2090467328 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.4141617715358734, + "objective/train/avg_token_score": 0.009824191220104694, + "objective/train/docs_used": 1182167, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4731876850128174, + "objective/train/original_loss": 3.736722946166992, + "objective/train/theoretical_loss": 3.4178002946397217, + "objective/train/tokens_used": 2090598400, + "objective/train/unlikelihood_loss": 0.1436697542667389, + "theoretical_loss": 3.4178002946397217, + "tokens_seen": 2090598400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018515546639919758, + "loss": 1.3674, + "theoretical_loss": 3.4178002946397217, + "tokens_seen": 2090598400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018513540621865597, + "loss": 1.319, + "theoretical_loss": 3.4177825842201837, + "tokens_seen": 2090729472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018511534603811433, + "loss": 1.3691, + "theoretical_loss": 3.417764875221774, + "tokens_seen": 2090860544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018509528585757272, + "loss": 1.3465, + "theoretical_loss": 3.4177471676442908, + "tokens_seen": 2090991616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018507522567703108, + "loss": 1.1749, + "theoretical_loss": 3.4177294614875304, + "tokens_seen": 2091122688 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018505516549648947, + "loss": 1.1564, + "theoretical_loss": 3.4177117567512894, + "tokens_seen": 2091253760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018503510531594786, + "loss": 1.3772, + "theoretical_loss": 3.4176940534353655, + "tokens_seen": 2091384832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018501504513540623, + "loss": 1.3558, + "theoretical_loss": 3.417676351539555, + "tokens_seen": 2091515904 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018499498495486462, + "loss": 1.4041, + "theoretical_loss": 3.417658651063655, + "tokens_seen": 2091646976 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018497492477432298, + "loss": 1.3376, + "theoretical_loss": 3.4176409520074635, + "tokens_seen": 2091778048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018495486459378137, + "loss": 1.3777, + "theoretical_loss": 3.417623254370777, + "tokens_seen": 2091909120 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018493480441323973, + "loss": 1.104, + "theoretical_loss": 3.4176055581533933, + "tokens_seen": 2092040192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018491474423269812, + "loss": 1.4438, + "theoretical_loss": 3.417587863355109, + "tokens_seen": 2092171264 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.5120967626571655, + "objective/train/avg_token_score": 0.008156620897352695, + "objective/train/docs_used": 1183291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7016239166259766, + "objective/train/original_loss": 3.6615970134735107, + "objective/train/theoretical_loss": 3.417579016488066, + "objective/train/tokens_used": 2092236800, + "objective/train/unlikelihood_loss": 0.15065442025661469, + "theoretical_loss": 3.417579016488066, + "tokens_seen": 2092236800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018489468405215648, + "loss": 1.3577, + "theoretical_loss": 3.417570169975722, + "tokens_seen": 2092302336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018487462387161484, + "loss": 1.3552, + "theoretical_loss": 3.417552478015029, + "tokens_seen": 2092433408 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018485456369107323, + "loss": 1.3201, + "theoretical_loss": 3.4175347874728277, + "tokens_seen": 2092564480 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001848345035105316, + "loss": 1.6667, + "theoretical_loss": 3.417517098348916, + "tokens_seen": 2092695552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018481444332998998, + "loss": 1.3502, + "theoretical_loss": 3.4174994106430905, + "tokens_seen": 2092826624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018479438314944835, + "loss": 1.2775, + "theoretical_loss": 3.4174817243551487, + "tokens_seen": 2092957696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018477432296890674, + "loss": 1.2962, + "theoretical_loss": 3.417464039484889, + "tokens_seen": 2093088768 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001847542627883651, + "loss": 1.3624, + "theoretical_loss": 3.417446356032109, + "tokens_seen": 2093219840 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001847342026078235, + "loss": 1.3202, + "theoretical_loss": 3.4174286739966053, + "tokens_seen": 2093350912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018471414242728185, + "loss": 1.4975, + "theoretical_loss": 3.417410993378177, + "tokens_seen": 2093481984 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018469408224674024, + "loss": 1.2645, + "theoretical_loss": 3.4173933141766204, + "tokens_seen": 2093613056 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001846740220661986, + "loss": 1.3948, + "theoretical_loss": 3.417375636391734, + "tokens_seen": 2093744128 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.43114614486694336, + "objective/train/avg_token_score": 0.015379317104816437, + "objective/train/docs_used": 1183994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4043447971343994, + "objective/train/original_loss": 3.7420334815979004, + "objective/train/theoretical_loss": 3.4173579600233155, + "objective/train/tokens_used": 2093875200, + "objective/train/unlikelihood_loss": 0.21826864778995514, + "theoretical_loss": 3.4173579600233155, + "tokens_seen": 2093875200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018465396188565696, + "loss": 1.3215, + "theoretical_loss": 3.4173579600233155, + "tokens_seen": 2093875200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018463390170511535, + "loss": 1.3616, + "theoretical_loss": 3.417340285071163, + "tokens_seen": 2094006272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018461384152457371, + "loss": 1.4327, + "theoretical_loss": 3.4173226115350737, + "tokens_seen": 2094137344 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001845937813440321, + "loss": 1.3948, + "theoretical_loss": 3.417304939414846, + "tokens_seen": 2094268416 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018457372116349047, + "loss": 1.2646, + "theoretical_loss": 3.417287268710278, + "tokens_seen": 2094399488 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018455366098294886, + "loss": 1.4171, + "theoretical_loss": 3.417269599421168, + "tokens_seen": 2094530560 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018453360080240722, + "loss": 1.229, + "theoretical_loss": 3.417251931547313, + "tokens_seen": 2094661632 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001845135406218656, + "loss": 1.3986, + "theoretical_loss": 3.4172342650885117, + "tokens_seen": 2094792704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018449348044132397, + "loss": 1.326, + "theoretical_loss": 3.4172166000445623, + "tokens_seen": 2094923776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018447342026078233, + "loss": 1.3634, + "theoretical_loss": 3.4171989364152626, + "tokens_seen": 2095054848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018445336008024072, + "loss": 1.4276, + "theoretical_loss": 3.4171812742004115, + "tokens_seen": 2095185920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018443329989969908, + "loss": 1.1947, + "theoretical_loss": 3.417163613399807, + "tokens_seen": 2095316992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018441323971915747, + "loss": 1.1497, + "theoretical_loss": 3.4171459540132467, + "tokens_seen": 2095448064 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.5014051795005798, + "objective/train/avg_token_score": 0.004601608030498028, + "objective/train/docs_used": 1184873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5879801511764526, + "objective/train/original_loss": 3.3373026847839355, + "objective/train/theoretical_loss": 3.4171371248501705, + "objective/train/tokens_used": 2095513600, + "objective/train/unlikelihood_loss": 0.20091430842876434, + "theoretical_loss": 3.4171371248501705, + "tokens_seen": 2095513600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018439317953861583, + "loss": 1.2735, + "theoretical_loss": 3.41712829604053, + "tokens_seen": 2095579136 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018437311935807422, + "loss": 1.2679, + "theoretical_loss": 3.4171106394814545, + "tokens_seen": 2095710208 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018435305917753259, + "loss": 1.2585, + "theoretical_loss": 3.4170929843358193, + "tokens_seen": 2095841280 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018433299899699098, + "loss": 1.2694, + "theoretical_loss": 3.4170753306034225, + "tokens_seen": 2095972352 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018431293881644934, + "loss": 1.3999, + "theoretical_loss": 3.4170576782840625, + "tokens_seen": 2096103424 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018429287863590773, + "loss": 1.3056, + "theoretical_loss": 3.4170400273775376, + "tokens_seen": 2096234496 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018427281845536612, + "loss": 1.4663, + "theoretical_loss": 3.4170223778836473, + "tokens_seen": 2096365568 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018425275827482448, + "loss": 1.4832, + "theoretical_loss": 3.4170047298021893, + "tokens_seen": 2096496640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018423269809428287, + "loss": 1.3982, + "theoretical_loss": 3.4169870831329625, + "tokens_seen": 2096627712 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018421263791374123, + "loss": 1.1542, + "theoretical_loss": 3.4169694378757662, + "tokens_seen": 2096758784 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018419257773319962, + "loss": 1.5225, + "theoretical_loss": 3.416951794030399, + "tokens_seen": 2096889856 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018417251755265798, + "loss": 1.1853, + "theoretical_loss": 3.4169341515966583, + "tokens_seen": 2097020928 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.41654446721076965, + "objective/train/avg_token_score": 0.007191215641796589, + "objective/train/docs_used": 1185597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1811728477478027, + "objective/train/original_loss": 3.41671085357666, + "objective/train/theoretical_loss": 3.416916510574345, + "objective/train/tokens_used": 2097152000, + "objective/train/unlikelihood_loss": 0.2363148033618927, + "theoretical_loss": 3.416916510574345, + "tokens_seen": 2097152000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018415245737211637, + "loss": 1.2874, + "theoretical_loss": 3.416916510574345, + "tokens_seen": 2097152000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018413239719157473, + "loss": 1.4832, + "theoretical_loss": 3.416898870963257, + "tokens_seen": 2097283072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018411233701103312, + "loss": 1.2455, + "theoretical_loss": 3.4168812327631928, + "tokens_seen": 2097414144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018409227683049149, + "loss": 1.318, + "theoretical_loss": 3.416863595973952, + "tokens_seen": 2097545216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018407221664994985, + "loss": 1.4259, + "theoretical_loss": 3.4168459605953334, + "tokens_seen": 2097676288 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018405215646940824, + "loss": 1.2973, + "theoretical_loss": 3.416828326627136, + "tokens_seen": 2097807360 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001840320962888666, + "loss": 1.3634, + "theoretical_loss": 3.4168106940691594, + "tokens_seen": 2097938432 + }, + { + "epoch": 0.64, + "learning_rate": 0.000184012036108325, + "loss": 1.3232, + "theoretical_loss": 3.4167930629212018, + "tokens_seen": 2098069504 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018399197592778335, + "loss": 1.3612, + "theoretical_loss": 3.416775433183063, + "tokens_seen": 2098200576 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018397191574724174, + "loss": 1.3266, + "theoretical_loss": 3.4167578048545417, + "tokens_seen": 2098331648 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001839518555667001, + "loss": 1.4939, + "theoretical_loss": 3.4167401779354383, + "tokens_seen": 2098462720 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001839317953861585, + "loss": 1.2383, + "theoretical_loss": 3.4167225524255507, + "tokens_seen": 2098593792 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018391173520561685, + "loss": 1.3529, + "theoretical_loss": 3.4167049283246786, + "tokens_seen": 2098724864 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.3090786933898926, + "objective/train/avg_token_score": 0.020390354096889496, + "objective/train/docs_used": 1186813, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0160932540893555, + "objective/train/original_loss": 4.679088115692139, + "objective/train/theoretical_loss": 3.4166961168025614, + "objective/train/tokens_used": 2098790400, + "objective/train/unlikelihood_loss": 0.15183141827583313, + "theoretical_loss": 3.4166961168025614, + "tokens_seen": 2098790400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018389167502507522, + "loss": 1.3413, + "theoretical_loss": 3.4166873056326224, + "tokens_seen": 2098855936 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001838716148445336, + "loss": 1.1888, + "theoretical_loss": 3.4166696843491806, + "tokens_seen": 2098987008 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018385155466399197, + "loss": 1.1913, + "theoretical_loss": 3.4166520644741523, + "tokens_seen": 2099118080 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018383149448345036, + "loss": 1.2555, + "theoretical_loss": 3.4166344460073375, + "tokens_seen": 2099249152 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018381143430290872, + "loss": 1.4566, + "theoretical_loss": 3.416616828948536, + "tokens_seen": 2099380224 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001837913741223671, + "loss": 1.3484, + "theoretical_loss": 3.416599213297547, + "tokens_seen": 2099511296 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018377131394182547, + "loss": 1.2476, + "theoretical_loss": 3.41658159905417, + "tokens_seen": 2099642368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018375125376128386, + "loss": 1.3104, + "theoretical_loss": 3.4165639862182053, + "tokens_seen": 2099773440 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018373119358074222, + "loss": 1.3116, + "theoretical_loss": 3.416546374789452, + "tokens_seen": 2099904512 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018371113340020058, + "loss": 1.1922, + "theoretical_loss": 3.41652876476771, + "tokens_seen": 2100035584 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018369107321965897, + "loss": 1.414, + "theoretical_loss": 3.416511156152779, + "tokens_seen": 2100166656 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018367101303911734, + "loss": 1.2058, + "theoretical_loss": 3.416493548944459, + "tokens_seen": 2100297728 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.3484237492084503, + "objective/train/avg_token_score": 0.015739643946290016, + "objective/train/docs_used": 1187470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9603504538536072, + "objective/train/original_loss": 3.325603485107422, + "objective/train/theoretical_loss": 3.4164759431425495, + "objective/train/tokens_used": 2100428800, + "objective/train/unlikelihood_loss": 0.19062767922878265, + "theoretical_loss": 3.4164759431425495, + "tokens_seen": 2100428800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018365095285857573, + "loss": 1.3048, + "theoretical_loss": 3.4164759431425495, + "tokens_seen": 2100428800 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001836308926780341, + "loss": 1.4155, + "theoretical_loss": 3.4164583387468515, + "tokens_seen": 2100559872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018361083249749248, + "loss": 1.3198, + "theoretical_loss": 3.4164407357571633, + "tokens_seen": 2100690944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018359077231695084, + "loss": 1.1795, + "theoretical_loss": 3.416423134173286, + "tokens_seen": 2100822016 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018357071213640923, + "loss": 1.1973, + "theoretical_loss": 3.4164055339950195, + "tokens_seen": 2100953088 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018355065195586762, + "loss": 1.3462, + "theoretical_loss": 3.416387935222164, + "tokens_seen": 2101084160 + }, + { + "epoch": 0.64, + "learning_rate": 0.000183530591775326, + "loss": 1.3923, + "theoretical_loss": 3.4163703378545187, + "tokens_seen": 2101215232 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018351053159478437, + "loss": 1.3432, + "theoretical_loss": 3.4163527418918846, + "tokens_seen": 2101346304 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018349047141424273, + "loss": 1.352, + "theoretical_loss": 3.416335147334062, + "tokens_seen": 2101477376 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018347041123370112, + "loss": 1.3369, + "theoretical_loss": 3.416317554180851, + "tokens_seen": 2101608448 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018345035105315948, + "loss": 1.1858, + "theoretical_loss": 3.416299962432051, + "tokens_seen": 2101739520 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018343029087261787, + "loss": 1.3492, + "theoretical_loss": 3.4162823720874633, + "tokens_seen": 2101870592 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018341023069207623, + "loss": 1.3366, + "theoretical_loss": 3.416264783146888, + "tokens_seen": 2102001664 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.3867301940917969, + "objective/train/avg_token_score": 0.007461334113031626, + "objective/train/docs_used": 1188148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.24367094039917, + "objective/train/original_loss": 3.793809413909912, + "objective/train/theoretical_loss": 3.4162559892030426, + "objective/train/tokens_used": 2102067200, + "objective/train/unlikelihood_loss": 0.25464192032814026, + "theoretical_loss": 3.4162559892030426, + "tokens_seen": 2102067200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018339017051153462, + "loss": 1.252, + "theoretical_loss": 3.4162471956101257, + "tokens_seen": 2102132736 + }, + { + "epoch": 0.64, + "learning_rate": 0.000183370110330993, + "loss": 1.3411, + "theoretical_loss": 3.4162296094769764, + "tokens_seen": 2102263808 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018335005015045138, + "loss": 1.5387, + "theoretical_loss": 3.416212024747241, + "tokens_seen": 2102394880 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018332998996990974, + "loss": 1.2929, + "theoretical_loss": 3.4161944414207195, + "tokens_seen": 2102525952 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001833099297893681, + "loss": 1.2764, + "theoretical_loss": 3.416176859497213, + "tokens_seen": 2102657024 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001832898696088265, + "loss": 1.2721, + "theoretical_loss": 3.416159278976522, + "tokens_seen": 2102788096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018326980942828485, + "loss": 1.2488, + "theoretical_loss": 3.4161416998584464, + "tokens_seen": 2102919168 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018324974924774324, + "loss": 1.3708, + "theoretical_loss": 3.4161241221427883, + "tokens_seen": 2103050240 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001832296890672016, + "loss": 1.3369, + "theoretical_loss": 3.416106545829347, + "tokens_seen": 2103181312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018320962888666, + "loss": 1.283, + "theoretical_loss": 3.4160889709179245, + "tokens_seen": 2103312384 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018318956870611836, + "loss": 1.5046, + "theoretical_loss": 3.4160713974083206, + "tokens_seen": 2103443456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018316950852557674, + "loss": 1.3194, + "theoretical_loss": 3.4160538253003367, + "tokens_seen": 2103574528 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.3603983521461487, + "objective/train/avg_token_score": 0.012275788933038712, + "objective/train/docs_used": 1189669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1371153593063354, + "objective/train/original_loss": 4.026082992553711, + "objective/train/theoretical_loss": 3.4160362545937732, + "objective/train/tokens_used": 2103705600, + "objective/train/unlikelihood_loss": 0.15194931626319885, + "theoretical_loss": 3.4160362545937732, + "tokens_seen": 2103705600 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001831494483450351, + "loss": 1.3378, + "theoretical_loss": 3.4160362545937732, + "tokens_seen": 2103705600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018312938816449347, + "loss": 1.422, + "theoretical_loss": 3.4160186852884316, + "tokens_seen": 2103836672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018310932798395186, + "loss": 1.3604, + "theoretical_loss": 3.4160011173841127, + "tokens_seen": 2103967744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018308926780341022, + "loss": 1.4728, + "theoretical_loss": 3.4159835508806173, + "tokens_seen": 2104098816 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001830692076228686, + "loss": 1.2898, + "theoretical_loss": 3.4159659857777465, + "tokens_seen": 2104229888 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018304914744232697, + "loss": 1.3357, + "theoretical_loss": 3.415948422075302, + "tokens_seen": 2104360960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018302908726178536, + "loss": 1.3898, + "theoretical_loss": 3.415930859773084, + "tokens_seen": 2104492032 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018300902708124372, + "loss": 1.2958, + "theoretical_loss": 3.415913298870894, + "tokens_seen": 2104623104 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001829889669007021, + "loss": 1.3621, + "theoretical_loss": 3.4158957393685334, + "tokens_seen": 2104754176 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018296890672016048, + "loss": 1.2246, + "theoretical_loss": 3.4158781812658034, + "tokens_seen": 2104885248 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018294884653961886, + "loss": 1.3592, + "theoretical_loss": 3.415860624562505, + "tokens_seen": 2105016320 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018292878635907723, + "loss": 1.4102, + "theoretical_loss": 3.4158430692584405, + "tokens_seen": 2105147392 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001829087261785356, + "loss": 1.4737, + "theoretical_loss": 3.4158255153534096, + "tokens_seen": 2105278464 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.51087486743927, + "objective/train/avg_token_score": 0.0051999702118337154, + "objective/train/docs_used": 1190193, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.737548589706421, + "objective/train/original_loss": 3.515737533569336, + "objective/train/theoretical_loss": 3.4158167389254706, + "objective/train/tokens_used": 2105344000, + "objective/train/unlikelihood_loss": 0.1452701836824417, + "theoretical_loss": 3.4158167389254706, + "tokens_seen": 2105344000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018288866599799398, + "loss": 1.3278, + "theoretical_loss": 3.4158079628472153, + "tokens_seen": 2105409536 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018286860581745234, + "loss": 1.334, + "theoretical_loss": 3.415790411739658, + "tokens_seen": 2105540608 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018284854563691073, + "loss": 1.3056, + "theoretical_loss": 3.41577286203054, + "tokens_seen": 2105671680 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001828284854563691, + "loss": 1.2851, + "theoretical_loss": 3.415755313719662, + "tokens_seen": 2105802752 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018280842527582748, + "loss": 1.2115, + "theoretical_loss": 3.415737766806826, + "tokens_seen": 2105933824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018278836509528587, + "loss": 1.3469, + "theoretical_loss": 3.415720221291834, + "tokens_seen": 2106064896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018276830491474426, + "loss": 1.1937, + "theoretical_loss": 3.4157026771744876, + "tokens_seen": 2106195968 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018274824473420262, + "loss": 1.1704, + "theoretical_loss": 3.4156851344545878, + "tokens_seen": 2106327040 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018272818455366098, + "loss": 1.226, + "theoretical_loss": 3.4156675931319365, + "tokens_seen": 2106458112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018270812437311937, + "loss": 1.3977, + "theoretical_loss": 3.415650053206336, + "tokens_seen": 2106589184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018268806419257774, + "loss": 1.449, + "theoretical_loss": 3.415632514677588, + "tokens_seen": 2106720256 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018266800401203613, + "loss": 1.3184, + "theoretical_loss": 3.415614977545494, + "tokens_seen": 2106851328 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.4002321660518646, + "objective/train/avg_token_score": 0.03367527946829796, + "objective/train/docs_used": 1191719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3049473762512207, + "objective/train/original_loss": 3.979520797729492, + "objective/train/theoretical_loss": 3.4155974418098562, + "objective/train/tokens_used": 2106982400, + "objective/train/unlikelihood_loss": 0.1676267683506012, + "theoretical_loss": 3.4155974418098562, + "tokens_seen": 2106982400 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826479438314945, + "loss": 1.3831, + "theoretical_loss": 3.4155974418098562, + "tokens_seen": 2106982400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018262788365095288, + "loss": 1.3258, + "theoretical_loss": 3.4155799074704762, + "tokens_seen": 2107113472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018260782347041124, + "loss": 1.2579, + "theoretical_loss": 3.4155623745271564, + "tokens_seen": 2107244544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018258776328986963, + "loss": 1.2664, + "theoretical_loss": 3.4155448429796986, + "tokens_seen": 2107375616 + }, + { + "epoch": 0.64, + "learning_rate": 0.000182567703109328, + "loss": 1.2563, + "theoretical_loss": 3.415527312827905, + "tokens_seen": 2107506688 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018254764292878638, + "loss": 1.3197, + "theoretical_loss": 3.4155097840715776, + "tokens_seen": 2107637760 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018252758274824474, + "loss": 1.3692, + "theoretical_loss": 3.415492256710518, + "tokens_seen": 2107768832 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001825075225677031, + "loss": 1.4321, + "theoretical_loss": 3.41547473074453, + "tokens_seen": 2107899904 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001824874623871615, + "loss": 1.3887, + "theoretical_loss": 3.4154572061734143, + "tokens_seen": 2108030976 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018246740220661986, + "loss": 1.1918, + "theoretical_loss": 3.4154396829969733, + "tokens_seen": 2108162048 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018244734202607825, + "loss": 1.289, + "theoretical_loss": 3.4154221612150097, + "tokens_seen": 2108293120 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001824272818455366, + "loss": 1.327, + "theoretical_loss": 3.415404640827326, + "tokens_seen": 2108424192 + }, + { + "epoch": 0.64, + "learning_rate": 0.000182407221664995, + "loss": 1.2546, + "theoretical_loss": 3.4153871218337244, + "tokens_seen": 2108555264 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.520405650138855, + "objective/train/avg_token_score": 0.012159794569015503, + "objective/train/docs_used": 1192553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6573365926742554, + "objective/train/original_loss": 3.823451280593872, + "objective/train/theoretical_loss": 3.4153783628596424, + "objective/train/tokens_used": 2108620800, + "objective/train/unlikelihood_loss": 0.1624690592288971, + "theoretical_loss": 3.4153783628596424, + "tokens_seen": 2108620800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018238716148445336, + "loss": 1.2767, + "theoretical_loss": 3.415369604234007, + "tokens_seen": 2108686336 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018236710130391175, + "loss": 1.2485, + "theoretical_loss": 3.415352088027977, + "tokens_seen": 2108817408 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001823470411233701, + "loss": 1.2867, + "theoretical_loss": 3.415334573215436, + "tokens_seen": 2108948480 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018232698094282847, + "loss": 1.3133, + "theoretical_loss": 3.4153170597961875, + "tokens_seen": 2109079552 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018230692076228686, + "loss": 1.4052, + "theoretical_loss": 3.415299547770033, + "tokens_seen": 2109210624 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018228686058174523, + "loss": 1.2412, + "theoretical_loss": 3.415282037136776, + "tokens_seen": 2109341696 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018226680040120361, + "loss": 1.2281, + "theoretical_loss": 3.415264527896219, + "tokens_seen": 2109472768 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018224674022066198, + "loss": 1.4198, + "theoretical_loss": 3.4152470200481644, + "tokens_seen": 2109603840 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018222668004012037, + "loss": 1.3649, + "theoretical_loss": 3.4152295135924153, + "tokens_seen": 2109734912 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018220661985957873, + "loss": 1.2599, + "theoretical_loss": 3.4152120085287745, + "tokens_seen": 2109865984 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018218655967903712, + "loss": 1.4204, + "theoretical_loss": 3.415194504857044, + "tokens_seen": 2109997056 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018216649949849548, + "loss": 1.1384, + "theoretical_loss": 3.415177002577028, + "tokens_seen": 2110128128 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.2896505296230316, + "objective/train/avg_token_score": 0.009733373299241066, + "objective/train/docs_used": 1193312, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9516772031784058, + "objective/train/original_loss": 4.552639961242676, + "objective/train/theoretical_loss": 3.415159501688528, + "objective/train/tokens_used": 2110259200, + "objective/train/unlikelihood_loss": 0.13014976680278778, + "theoretical_loss": 3.415159501688528, + "tokens_seen": 2110259200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018214643931795384, + "loss": 1.2939, + "theoretical_loss": 3.415159501688528, + "tokens_seen": 2110259200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018212637913741223, + "loss": 1.5138, + "theoretical_loss": 3.4151420021913483, + "tokens_seen": 2110390272 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001821063189568706, + "loss": 1.1708, + "theoretical_loss": 3.4151245040852913, + "tokens_seen": 2110521344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018208625877632898, + "loss": 1.1782, + "theoretical_loss": 3.4151070073701595, + "tokens_seen": 2110652416 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018206619859578735, + "loss": 1.253, + "theoretical_loss": 3.415089512045757, + "tokens_seen": 2110783488 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018204613841524576, + "loss": 1.4247, + "theoretical_loss": 3.415072018111886, + "tokens_seen": 2110914560 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018202607823470412, + "loss": 1.2282, + "theoretical_loss": 3.41505452556835, + "tokens_seen": 2111045632 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001820060180541625, + "loss": 1.5511, + "theoretical_loss": 3.4150370344149525, + "tokens_seen": 2111176704 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018198595787362088, + "loss": 1.2658, + "theoretical_loss": 3.4150195446514964, + "tokens_seen": 2111307776 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018196589769307926, + "loss": 1.2829, + "theoretical_loss": 3.4150020562777854, + "tokens_seen": 2111438848 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018194583751253763, + "loss": 1.2914, + "theoretical_loss": 3.4149845692936216, + "tokens_seen": 2111569920 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181925777331996, + "loss": 1.3112, + "theoretical_loss": 3.4149670836988095, + "tokens_seen": 2111700992 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018190571715145438, + "loss": 1.311, + "theoretical_loss": 3.414949599493152, + "tokens_seen": 2111832064 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.3622922897338867, + "objective/train/avg_token_score": 0.04146071523427963, + "objective/train/docs_used": 1194755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0483016967773438, + "objective/train/original_loss": 4.009406089782715, + "objective/train/theoretical_loss": 3.414940857911195, + "objective/train/tokens_used": 2111897600, + "objective/train/unlikelihood_loss": 0.21139970421791077, + "theoretical_loss": 3.414940857911195, + "tokens_seen": 2111897600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018188565697091274, + "loss": 1.3244, + "theoretical_loss": 3.4149321166764532, + "tokens_seen": 2111963136 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018186559679037113, + "loss": 1.323, + "theoretical_loss": 3.4149146352485156, + "tokens_seen": 2112094208 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001818455366098295, + "loss": 1.3871, + "theoretical_loss": 3.414897155209143, + "tokens_seen": 2112225280 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018182547642928788, + "loss": 1.3028, + "theoretical_loss": 3.4148796765581397, + "tokens_seen": 2112356352 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018180541624874624, + "loss": 1.3587, + "theoretical_loss": 3.4148621992953085, + "tokens_seen": 2112487424 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018178535606820463, + "loss": 1.2544, + "theoretical_loss": 3.414844723420453, + "tokens_seen": 2112618496 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181765295887663, + "loss": 1.4414, + "theoretical_loss": 3.4148272489333773, + "tokens_seen": 2112749568 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018174523570712136, + "loss": 1.3122, + "theoretical_loss": 3.414809775833885, + "tokens_seen": 2112880640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018172517552657975, + "loss": 1.3935, + "theoretical_loss": 3.4147923041217796, + "tokens_seen": 2113011712 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001817051153460381, + "loss": 1.2657, + "theoretical_loss": 3.414774833796865, + "tokens_seen": 2113142784 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001816850551654965, + "loss": 1.2806, + "theoretical_loss": 3.414757364858945, + "tokens_seen": 2113273856 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018166499498495486, + "loss": 1.2871, + "theoretical_loss": 3.414739897307824, + "tokens_seen": 2113404928 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.3913123309612274, + "objective/train/avg_token_score": 0.019222678616642952, + "objective/train/docs_used": 1195575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.179430603981018, + "objective/train/original_loss": 3.431609869003296, + "objective/train/theoretical_loss": 3.4147224311433053, + "objective/train/tokens_used": 2113536000, + "objective/train/unlikelihood_loss": 0.24719899892807007, + "theoretical_loss": 3.4147224311433053, + "tokens_seen": 2113536000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018164493480441325, + "loss": 1.3257, + "theoretical_loss": 3.4147224311433053, + "tokens_seen": 2113536000 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001816248746238716, + "loss": 1.2125, + "theoretical_loss": 3.4147049663651927, + "tokens_seen": 2113667072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018160481444333, + "loss": 1.4337, + "theoretical_loss": 3.4146875029732904, + "tokens_seen": 2113798144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018158475426278836, + "loss": 1.4584, + "theoretical_loss": 3.414670040967403, + "tokens_seen": 2113929216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018156469408224673, + "loss": 1.4468, + "theoretical_loss": 3.414652580347334, + "tokens_seen": 2114060288 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018154463390170512, + "loss": 1.2997, + "theoretical_loss": 3.4146351211128874, + "tokens_seen": 2114191360 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018152457372116348, + "loss": 1.3628, + "theoretical_loss": 3.4146176632638676, + "tokens_seen": 2114322432 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018150451354062187, + "loss": 1.3714, + "theoretical_loss": 3.4146002068000785, + "tokens_seen": 2114453504 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018148445336008023, + "loss": 1.2661, + "theoretical_loss": 3.414582751721325, + "tokens_seen": 2114584576 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018146439317953862, + "loss": 1.2863, + "theoretical_loss": 3.414565298027411, + "tokens_seen": 2114715648 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018144433299899698, + "loss": 1.3315, + "theoretical_loss": 3.41454784571814, + "tokens_seen": 2114846720 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018142427281845537, + "loss": 1.289, + "theoretical_loss": 3.414530394793318, + "tokens_seen": 2114977792 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018140421263791373, + "loss": 1.2481, + "theoretical_loss": 3.414512945252748, + "tokens_seen": 2115108864 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.37689393758773804, + "objective/train/avg_token_score": 0.015272802673280239, + "objective/train/docs_used": 1196855, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.239571452140808, + "objective/train/original_loss": 4.22519588470459, + "objective/train/theoretical_loss": 3.414504221001496, + "objective/train/tokens_used": 2115174400, + "objective/train/unlikelihood_loss": 0.12248257547616959, + "theoretical_loss": 3.414504221001496, + "tokens_seen": 2115174400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018138415245737212, + "loss": 1.3703, + "theoretical_loss": 3.414495497096235, + "tokens_seen": 2115239936 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018136409227683048, + "loss": 1.2354, + "theoretical_loss": 3.414478050323583, + "tokens_seen": 2115371008 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018134403209628885, + "loss": 1.1522, + "theoretical_loss": 3.414460604934597, + "tokens_seen": 2115502080 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018132397191574724, + "loss": 1.3487, + "theoretical_loss": 3.414443160929082, + "tokens_seen": 2115633152 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018130391173520563, + "loss": 1.3643, + "theoretical_loss": 3.414425718306841, + "tokens_seen": 2115764224 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018128385155466401, + "loss": 1.253, + "theoretical_loss": 3.4144082770676807, + "tokens_seen": 2115895296 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018126379137412238, + "loss": 1.345, + "theoretical_loss": 3.4143908372114042, + "tokens_seen": 2116026368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018124373119358077, + "loss": 1.3034, + "theoretical_loss": 3.4143733987378164, + "tokens_seen": 2116157440 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018122367101303913, + "loss": 1.3599, + "theoretical_loss": 3.4143559616467227, + "tokens_seen": 2116288512 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018120361083249752, + "loss": 1.3434, + "theoretical_loss": 3.4143385259379277, + "tokens_seen": 2116419584 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018118355065195588, + "loss": 1.3889, + "theoretical_loss": 3.4143210916112356, + "tokens_seen": 2116550656 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018116349047141424, + "loss": 1.2907, + "theoretical_loss": 3.414303658666452, + "tokens_seen": 2116681728 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.4720185697078705, + "objective/train/avg_token_score": 0.01116443332284689, + "objective/train/docs_used": 1197590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6493721008300781, + "objective/train/original_loss": 3.817815065383911, + "objective/train/theoretical_loss": 3.414286227103381, + "objective/train/tokens_used": 2116812800, + "objective/train/unlikelihood_loss": 0.13268323242664337, + "theoretical_loss": 3.414286227103381, + "tokens_seen": 2116812800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018114343029087263, + "loss": 1.3975, + "theoretical_loss": 3.414286227103381, + "tokens_seen": 2116812800 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181123370110331, + "loss": 1.1607, + "theoretical_loss": 3.414268796921829, + "tokens_seen": 2116943872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018110330992978938, + "loss": 1.2311, + "theoretical_loss": 3.414251368121599, + "tokens_seen": 2117074944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018108324974924775, + "loss": 1.2884, + "theoretical_loss": 3.414233940702498, + "tokens_seen": 2117206016 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018106318956870613, + "loss": 1.3741, + "theoretical_loss": 3.4142165146643295, + "tokens_seen": 2117337088 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001810431293881645, + "loss": 1.439, + "theoretical_loss": 3.4141990900068997, + "tokens_seen": 2117468160 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001810230692076229, + "loss": 1.2422, + "theoretical_loss": 3.4141816667300127, + "tokens_seen": 2117599232 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018100300902708125, + "loss": 1.2572, + "theoretical_loss": 3.414164244833475, + "tokens_seen": 2117730304 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001809829488465396, + "loss": 1.2942, + "theoretical_loss": 3.41414682431709, + "tokens_seen": 2117861376 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180962888665998, + "loss": 1.3227, + "theoretical_loss": 3.414129405180665, + "tokens_seen": 2117992448 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018094282848545636, + "loss": 1.2376, + "theoretical_loss": 3.4141119874240036, + "tokens_seen": 2118123520 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018092276830491475, + "loss": 1.1721, + "theoretical_loss": 3.4140945710469124, + "tokens_seen": 2118254592 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018090270812437311, + "loss": 1.2416, + "theoretical_loss": 3.4140771560491956, + "tokens_seen": 2118385664 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.33571603894233704, + "objective/train/avg_token_score": 0.01624106802046299, + "objective/train/docs_used": 1199039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0135148763656616, + "objective/train/original_loss": 3.51552414894104, + "objective/train/theoretical_loss": 3.4140684490675426, + "objective/train/tokens_used": 2118451200, + "objective/train/unlikelihood_loss": 0.2362351268529892, + "theoretical_loss": 3.4140684490675426, + "tokens_seen": 2118451200 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001808826479438315, + "loss": 1.1979, + "theoretical_loss": 3.41405974243066, + "tokens_seen": 2118516736 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018086258776328987, + "loss": 1.2957, + "theoretical_loss": 3.41404233019111, + "tokens_seen": 2118647808 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018084252758274826, + "loss": 1.476, + "theoretical_loss": 3.414024919330351, + "tokens_seen": 2118778880 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018082246740220662, + "loss": 1.3169, + "theoretical_loss": 3.4140075098481892, + "tokens_seen": 2118909952 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180802407221665, + "loss": 1.3736, + "theoretical_loss": 3.41399010174443, + "tokens_seen": 2119041024 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018078234704112337, + "loss": 1.3729, + "theoretical_loss": 3.413972695018879, + "tokens_seen": 2119172096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018076228686058173, + "loss": 1.1995, + "theoretical_loss": 3.4139552896713417, + "tokens_seen": 2119303168 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018074222668004012, + "loss": 1.1618, + "theoretical_loss": 3.413937885701624, + "tokens_seen": 2119434240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018072216649949848, + "loss": 1.2159, + "theoretical_loss": 3.413920483109531, + "tokens_seen": 2119565312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018070210631895687, + "loss": 1.4301, + "theoretical_loss": 3.41390308189487, + "tokens_seen": 2119696384 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018068204613841523, + "loss": 1.525, + "theoretical_loss": 3.413885682057445, + "tokens_seen": 2119827456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018066198595787362, + "loss": 1.2517, + "theoretical_loss": 3.4138682835970626, + "tokens_seen": 2119958528 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.4067693054676056, + "objective/train/avg_token_score": 0.006931984797120094, + "objective/train/docs_used": 1199714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2498054504394531, + "objective/train/original_loss": 3.665820360183716, + "objective/train/theoretical_loss": 3.413850886513529, + "objective/train/tokens_used": 2120089600, + "objective/train/unlikelihood_loss": 0.1692916303873062, + "theoretical_loss": 3.413850886513529, + "tokens_seen": 2120089600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018064192577733199, + "loss": 1.3361, + "theoretical_loss": 3.413850886513529, + "tokens_seen": 2120089600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018062186559679038, + "loss": 1.3888, + "theoretical_loss": 3.41383349080665, + "tokens_seen": 2120220672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018060180541624874, + "loss": 1.3628, + "theoretical_loss": 3.4138160964762307, + "tokens_seen": 2120351744 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001805817452357071, + "loss": 1.3999, + "theoretical_loss": 3.4137987035220787, + "tokens_seen": 2120482816 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001805616850551655, + "loss": 1.2053, + "theoretical_loss": 3.413781311943999, + "tokens_seen": 2120613888 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018054162487462388, + "loss": 1.3883, + "theoretical_loss": 3.413763921741797, + "tokens_seen": 2120744960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018052156469408227, + "loss": 1.3527, + "theoretical_loss": 3.4137465329152805, + "tokens_seen": 2120876032 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018050150451354063, + "loss": 1.2659, + "theoretical_loss": 3.413729145464255, + "tokens_seen": 2121007104 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018048144433299902, + "loss": 1.2713, + "theoretical_loss": 3.413711759388526, + "tokens_seen": 2121138176 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018046138415245738, + "loss": 1.3048, + "theoretical_loss": 3.4136943746879007, + "tokens_seen": 2121269248 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018044132397191577, + "loss": 1.3149, + "theoretical_loss": 3.413676991362185, + "tokens_seen": 2121400320 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018042126379137413, + "loss": 1.357, + "theoretical_loss": 3.413659609411185, + "tokens_seen": 2121531392 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018040120361083252, + "loss": 1.4546, + "theoretical_loss": 3.4136422288347075, + "tokens_seen": 2121662464 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.4218597114086151, + "objective/train/avg_token_score": 0.017544005066156387, + "objective/train/docs_used": 1200493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2635380029678345, + "objective/train/original_loss": 3.8949637413024902, + "objective/train/theoretical_loss": 3.4136335390618537, + "objective/train/tokens_used": 2121728000, + "objective/train/unlikelihood_loss": 0.1207747757434845, + "theoretical_loss": 3.4136335390618537, + "tokens_seen": 2121728000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018038114343029088, + "loss": 1.3263, + "theoretical_loss": 3.4136248496325585, + "tokens_seen": 2121793536 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018036108324974925, + "loss": 1.348, + "theoretical_loss": 3.4136074718045446, + "tokens_seen": 2121924608 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018034102306920764, + "loss": 1.2462, + "theoretical_loss": 3.4135900953504725, + "tokens_seen": 2122055680 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180320962888666, + "loss": 1.2383, + "theoretical_loss": 3.4135727202701487, + "tokens_seen": 2122186752 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001803009027081244, + "loss": 1.2341, + "theoretical_loss": 3.4135553465633794, + "tokens_seen": 2122317824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018028084252758275, + "loss": 1.2815, + "theoretical_loss": 3.413537974229971, + "tokens_seen": 2122448896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018026078234704114, + "loss": 1.2921, + "theoretical_loss": 3.4135206032697307, + "tokens_seen": 2122579968 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001802407221664995, + "loss": 1.3845, + "theoretical_loss": 3.413503233682465, + "tokens_seen": 2122711040 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001802206619859579, + "loss": 1.2987, + "theoretical_loss": 3.4134858654679805, + "tokens_seen": 2122842112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018020060180541625, + "loss": 1.3066, + "theoretical_loss": 3.4134684986260844, + "tokens_seen": 2122973184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018018054162487462, + "loss": 1.3155, + "theoretical_loss": 3.413451133156583, + "tokens_seen": 2123104256 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180160481444333, + "loss": 1.2515, + "theoretical_loss": 3.4134337690592833, + "tokens_seen": 2123235328 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.3167155385017395, + "objective/train/avg_token_score": 0.008843827061355114, + "objective/train/docs_used": 1201963, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8512635827064514, + "objective/train/original_loss": 3.853912353515625, + "objective/train/theoretical_loss": 3.413416406333992, + "objective/train/tokens_used": 2123366400, + "objective/train/unlikelihood_loss": 0.18007661402225494, + "theoretical_loss": 3.413416406333992, + "tokens_seen": 2123366400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018014042126379137, + "loss": 1.2743, + "theoretical_loss": 3.413416406333992, + "tokens_seen": 2123366400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018012036108324976, + "loss": 1.3422, + "theoretical_loss": 3.4133990449805163, + "tokens_seen": 2123497472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018010030090270812, + "loss": 1.4024, + "theoretical_loss": 3.413381684998663, + "tokens_seen": 2123628544 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001800802407221665, + "loss": 1.308, + "theoretical_loss": 3.4133643263882387, + "tokens_seen": 2123759616 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018006018054162487, + "loss": 1.2436, + "theoretical_loss": 3.4133469691490514, + "tokens_seen": 2123890688 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018004012036108326, + "loss": 1.3158, + "theoretical_loss": 3.4133296132809074, + "tokens_seen": 2124021760 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018002006018054162, + "loss": 1.2669, + "theoretical_loss": 3.413312258783614, + "tokens_seen": 2124152832 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017999999999999998, + "loss": 1.392, + "theoretical_loss": 3.4132949056569784, + "tokens_seen": 2124283904 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017997993981945837, + "loss": 1.3222, + "theoretical_loss": 3.413277553900808, + "tokens_seen": 2124414976 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017995987963891674, + "loss": 1.1883, + "theoretical_loss": 3.413260203514909, + "tokens_seen": 2124546048 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017993981945837513, + "loss": 1.2331, + "theoretical_loss": 3.4132428544990896, + "tokens_seen": 2124677120 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001799197592778335, + "loss": 1.3749, + "theoretical_loss": 3.4132255068531574, + "tokens_seen": 2124808192 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017989969909729188, + "loss": 1.2987, + "theoretical_loss": 3.413208160576919, + "tokens_seen": 2124939264 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.5472874045372009, + "objective/train/avg_token_score": 0.017387334257364273, + "objective/train/docs_used": 1202675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7423763275146484, + "objective/train/original_loss": 3.6742799282073975, + "objective/train/theoretical_loss": 3.413199487952374, + "objective/train/tokens_used": 2125004800, + "objective/train/unlikelihood_loss": 0.16289794445037842, + "theoretical_loss": 3.413199487952374, + "tokens_seen": 2125004800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017987963891675024, + "loss": 1.3907, + "theoretical_loss": 3.4131908156701813, + "tokens_seen": 2125070336 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017985957873620863, + "loss": 1.3758, + "theoretical_loss": 3.4131734721327533, + "tokens_seen": 2125201408 + }, + { + "epoch": 0.64, + "learning_rate": 0.000179839518555667, + "loss": 1.4194, + "theoretical_loss": 3.4131561299644417, + "tokens_seen": 2125332480 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017981945837512538, + "loss": 1.2886, + "theoretical_loss": 3.4131387891650533, + "tokens_seen": 2125463552 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017979939819458377, + "loss": 1.21, + "theoretical_loss": 3.4131214497343967, + "tokens_seen": 2125594624 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017977933801404213, + "loss": 1.3581, + "theoretical_loss": 3.4131041116722787, + "tokens_seen": 2125725696 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017975927783350052, + "loss": 1.2162, + "theoretical_loss": 3.4130867749785074, + "tokens_seen": 2125856768 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017973921765295888, + "loss": 1.3151, + "theoretical_loss": 3.4130694396528902, + "tokens_seen": 2125987840 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017971915747241727, + "loss": 1.3112, + "theoretical_loss": 3.4130521056952348, + "tokens_seen": 2126118912 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017969909729187563, + "loss": 1.2179, + "theoretical_loss": 3.413034773105349, + "tokens_seen": 2126249984 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017967903711133402, + "loss": 1.332, + "theoretical_loss": 3.4130174418830403, + "tokens_seen": 2126381056 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017965897693079239, + "loss": 1.4187, + "theoretical_loss": 3.4130001120281173, + "tokens_seen": 2126512128 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.39540567994117737, + "objective/train/avg_token_score": 0.027513986453413963, + "objective/train/docs_used": 1203810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0922682285308838, + "objective/train/original_loss": 3.7994725704193115, + "objective/train/theoretical_loss": 3.412982783540387, + "objective/train/tokens_used": 2126643200, + "objective/train/unlikelihood_loss": 0.17327545583248138, + "theoretical_loss": 3.412982783540387, + "tokens_seen": 2126643200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017963891675025078, + "loss": 1.2209, + "theoretical_loss": 3.412982783540387, + "tokens_seen": 2126643200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017961885656970914, + "loss": 1.5956, + "theoretical_loss": 3.4129654564196574, + "tokens_seen": 2126774272 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001795987963891675, + "loss": 1.3355, + "theoretical_loss": 3.412948130665737, + "tokens_seen": 2126905344 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001795787362086259, + "loss": 1.3193, + "theoretical_loss": 3.4129308062784336, + "tokens_seen": 2127036416 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017955867602808425, + "loss": 1.3726, + "theoretical_loss": 3.4129134832575545, + "tokens_seen": 2127167488 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017953861584754264, + "loss": 1.4465, + "theoretical_loss": 3.4128961616029083, + "tokens_seen": 2127298560 + }, + { + "epoch": 0.64, + "learning_rate": 0.000179518555667001, + "loss": 1.2916, + "theoretical_loss": 3.4128788413143027, + "tokens_seen": 2127429632 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001794984954864594, + "loss": 1.3134, + "theoretical_loss": 3.412861522391547, + "tokens_seen": 2127560704 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017947843530591775, + "loss": 1.1469, + "theoretical_loss": 3.412844204834448, + "tokens_seen": 2127691776 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017945837512537614, + "loss": 1.3043, + "theoretical_loss": 3.412826888642814, + "tokens_seen": 2127822848 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001794383149448345, + "loss": 1.3264, + "theoretical_loss": 3.4128095738164537, + "tokens_seen": 2127953920 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017941825476429287, + "loss": 1.1717, + "theoretical_loss": 3.4127922603551752, + "tokens_seen": 2128084992 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017939819458375126, + "loss": 1.2462, + "theoretical_loss": 3.4127749482587872, + "tokens_seen": 2128216064 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.3288123309612274, + "objective/train/avg_token_score": 0.0060279532335698605, + "objective/train/docs_used": 1204521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0280143022537231, + "objective/train/original_loss": 3.4813432693481445, + "objective/train/theoretical_loss": 3.412766292722367, + "objective/train/tokens_used": 2128281600, + "objective/train/unlikelihood_loss": 0.18704842031002045, + "theoretical_loss": 3.412766292722367, + "tokens_seen": 2128281600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017937813440320962, + "loss": 1.2232, + "theoretical_loss": 3.412757637527098, + "tokens_seen": 2128347136 + }, + { + "epoch": 0.64, + "learning_rate": 0.000179358074222668, + "loss": 1.279, + "theoretical_loss": 3.412740328159915, + "tokens_seen": 2128478208 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017933801404212637, + "loss": 1.3975, + "theoretical_loss": 3.4127230201570478, + "tokens_seen": 2128609280 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017931795386158476, + "loss": 1.5079, + "theoretical_loss": 3.4127057135183043, + "tokens_seen": 2128740352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017929789368104312, + "loss": 1.2242, + "theoretical_loss": 3.4126884082434934, + "tokens_seen": 2128871424 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001792778335005015, + "loss": 1.3733, + "theoretical_loss": 3.4126711043324227, + "tokens_seen": 2129002496 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017925777331995988, + "loss": 1.1531, + "theoretical_loss": 3.4126538017849017, + "tokens_seen": 2129133568 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017923771313941826, + "loss": 1.2513, + "theoretical_loss": 3.4126365006007386, + "tokens_seen": 2129264640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017921765295887663, + "loss": 1.4709, + "theoretical_loss": 3.412619200779743, + "tokens_seen": 2129395712 + }, + { + "epoch": 0.65, + "learning_rate": 0.000179197592778335, + "loss": 1.4739, + "theoretical_loss": 3.4126019023217222, + "tokens_seen": 2129526784 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017917753259779338, + "loss": 1.2115, + "theoretical_loss": 3.4125846052264857, + "tokens_seen": 2129657856 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017915747241725174, + "loss": 1.3492, + "theoretical_loss": 3.412567309493842, + "tokens_seen": 2129788928 + }, + { + "debugging/length": 543.3333333333334, + "debugging/num_segments": 9, + "epoch": 0.65, + "objective/train/aligned_token_count": 0.3570992052555084, + "objective/train/avg_token_score": 0.01960843987762928, + "objective/train/docs_used": 1206012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0887759923934937, + "objective/train/original_loss": 3.8267877101898193, + "objective/train/theoretical_loss": 3.4125500151236, + "objective/train/tokens_used": 2129920000, + "objective/train/unlikelihood_loss": 0.140985369682312, + "theoretical_loss": 3.4125500151236, + "tokens_seen": 2129920000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017913741223671013, + "loss": 1.3775, + "theoretical_loss": 3.4125500151236, + "tokens_seen": 2129920000 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001791173520561685, + "loss": 1.2837, + "theoretical_loss": 3.412532722115569, + "tokens_seen": 2130051072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017909729187562688, + "loss": 1.3587, + "theoretical_loss": 3.412515430469557, + "tokens_seen": 2130182144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017907723169508524, + "loss": 1.3948, + "theoretical_loss": 3.4124981401853733, + "tokens_seen": 2130313216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017905717151454363, + "loss": 1.2623, + "theoretical_loss": 3.4124808512628277, + "tokens_seen": 2130444288 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017903711133400202, + "loss": 1.227, + "theoretical_loss": 3.4124635637017278, + "tokens_seen": 2130575360 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017901705115346038, + "loss": 1.3261, + "theoretical_loss": 3.412446277501884, + "tokens_seen": 2130706432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017899699097291877, + "loss": 1.2006, + "theoretical_loss": 3.412428992663104, + "tokens_seen": 2130837504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017897693079237714, + "loss": 1.2365, + "theoretical_loss": 3.4124117091851978, + "tokens_seen": 2130968576 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017895687061183553, + "loss": 1.3529, + "theoretical_loss": 3.4123944270679742, + "tokens_seen": 2131099648 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001789368104312939, + "loss": 1.3209, + "theoretical_loss": 3.4123771463112433, + "tokens_seen": 2131230720 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017891675025075228, + "loss": 1.2477, + "theoretical_loss": 3.412359866914813, + "tokens_seen": 2131361792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017889669007021064, + "loss": 1.1956, + "theoretical_loss": 3.4123425888784933, + "tokens_seen": 2131492864 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.4004765450954437, + "objective/train/avg_token_score": 0.008303572423756123, + "objective/train/docs_used": 1206751, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2522740364074707, + "objective/train/original_loss": 3.931788682937622, + "objective/train/theoretical_loss": 3.4123339503703156, + "objective/train/tokens_used": 2131558400, + "objective/train/unlikelihood_loss": 0.12630677223205566, + "theoretical_loss": 3.4123339503703156, + "tokens_seen": 2131558400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017887662988966903, + "loss": 1.3939, + "theoretical_loss": 3.4123253122020936, + "tokens_seen": 2131623936 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001788565697091274, + "loss": 1.231, + "theoretical_loss": 3.412308036885423, + "tokens_seen": 2131755008 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017883650952858578, + "loss": 1.288, + "theoretical_loss": 3.4122907629282913, + "tokens_seen": 2131886080 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017881644934804414, + "loss": 1.285, + "theoretical_loss": 3.412273490330507, + "tokens_seen": 2132017152 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001787963891675025, + "loss": 1.4497, + "theoretical_loss": 3.412256219091881, + "tokens_seen": 2132148224 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001787763289869609, + "loss": 1.2, + "theoretical_loss": 3.412238949212221, + "tokens_seen": 2132279296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017875626880641926, + "loss": 1.2709, + "theoretical_loss": 3.412221680691338, + "tokens_seen": 2132410368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017873620862587765, + "loss": 1.4303, + "theoretical_loss": 3.4122044135290412, + "tokens_seen": 2132541440 + }, + { + "epoch": 0.65, + "learning_rate": 0.000178716148445336, + "loss": 1.3734, + "theoretical_loss": 3.41218714772514, + "tokens_seen": 2132672512 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001786960882647944, + "loss": 1.4136, + "theoretical_loss": 3.412169883279444, + "tokens_seen": 2132803584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017867602808425276, + "loss": 1.2979, + "theoretical_loss": 3.4121526201917636, + "tokens_seen": 2132934656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017865596790371115, + "loss": 1.2193, + "theoretical_loss": 3.4121353584619074, + "tokens_seen": 2133065728 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.32154202461242676, + "objective/train/avg_token_score": 0.00944567285478115, + "objective/train/docs_used": 1207326, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9050133228302002, + "objective/train/original_loss": 3.288184881210327, + "objective/train/theoretical_loss": 3.412118098089686, + "objective/train/tokens_used": 2133196800, + "objective/train/unlikelihood_loss": 0.25312477350234985, + "theoretical_loss": 3.412118098089686, + "tokens_seen": 2133196800 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001786359077231695, + "loss": 1.2968, + "theoretical_loss": 3.412118098089686, + "tokens_seen": 2133196800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017861584754262787, + "loss": 1.3344, + "theoretical_loss": 3.412100839074909, + "tokens_seen": 2133327872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017859578736208626, + "loss": 1.2687, + "theoretical_loss": 3.412083581417386, + "tokens_seen": 2133458944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017857572718154462, + "loss": 1.2265, + "theoretical_loss": 3.412066325116928, + "tokens_seen": 2133590016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017855566700100301, + "loss": 1.2981, + "theoretical_loss": 3.4120490701733432, + "tokens_seen": 2133721088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017853560682046138, + "loss": 1.272, + "theoretical_loss": 3.4120318165864427, + "tokens_seen": 2133852160 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017851554663991977, + "loss": 1.5378, + "theoretical_loss": 3.4120145643560367, + "tokens_seen": 2133983232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017849548645937813, + "loss": 1.2651, + "theoretical_loss": 3.411997313481934, + "tokens_seen": 2134114304 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017847542627883652, + "loss": 1.2477, + "theoretical_loss": 3.411980063963946, + "tokens_seen": 2134245376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017845536609829488, + "loss": 1.2535, + "theoretical_loss": 3.4119628158018824, + "tokens_seen": 2134376448 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017843530591775324, + "loss": 1.2857, + "theoretical_loss": 3.411945568995553, + "tokens_seen": 2134507520 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017841524573721163, + "loss": 1.3396, + "theoretical_loss": 3.4119283235447684, + "tokens_seen": 2134638592 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017839518555667, + "loss": 1.5025, + "theoretical_loss": 3.4119110794493386, + "tokens_seen": 2134769664 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.38159823417663574, + "objective/train/avg_token_score": 0.006760616786777973, + "objective/train/docs_used": 1208575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1852684020996094, + "objective/train/original_loss": 3.3930459022521973, + "objective/train/theoretical_loss": 3.4119024579098225, + "objective/train/tokens_used": 2134835200, + "objective/train/unlikelihood_loss": 0.195537731051445, + "theoretical_loss": 3.4119024579098225, + "tokens_seen": 2134835200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017837512537612838, + "loss": 1.2981, + "theoretical_loss": 3.4118938367090736, + "tokens_seen": 2134900736 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017835506519558675, + "loss": 1.2546, + "theoretical_loss": 3.4118765953237844, + "tokens_seen": 2135031808 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017833500501504513, + "loss": 1.3504, + "theoretical_loss": 3.411859355293281, + "tokens_seen": 2135162880 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001783149448345035, + "loss": 1.4692, + "theoretical_loss": 3.4118421166173736, + "tokens_seen": 2135293952 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001782948846539619, + "loss": 1.4417, + "theoretical_loss": 3.4118248792958736, + "tokens_seen": 2135425024 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017827482447342028, + "loss": 1.2754, + "theoretical_loss": 3.41180764332859, + "tokens_seen": 2135556096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017825476429287866, + "loss": 1.4433, + "theoretical_loss": 3.411790408715334, + "tokens_seen": 2135687168 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017823470411233703, + "loss": 1.3523, + "theoretical_loss": 3.411773175455916, + "tokens_seen": 2135818240 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001782146439317954, + "loss": 1.3068, + "theoretical_loss": 3.411755943550147, + "tokens_seen": 2135949312 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017819458375125378, + "loss": 1.222, + "theoretical_loss": 3.4117387129978374, + "tokens_seen": 2136080384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017817452357071214, + "loss": 1.3863, + "theoretical_loss": 3.4117214837987975, + "tokens_seen": 2136211456 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017815446339017053, + "loss": 1.3469, + "theoretical_loss": 3.4117042559528388, + "tokens_seen": 2136342528 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.41660556197166443, + "objective/train/avg_token_score": 0.04863067716360092, + "objective/train/docs_used": 1209281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2820112705230713, + "objective/train/original_loss": 3.8600919246673584, + "objective/train/theoretical_loss": 3.411687029459771, + "objective/train/tokens_used": 2136473600, + "objective/train/unlikelihood_loss": 0.13130347430706024, + "theoretical_loss": 3.411687029459771, + "tokens_seen": 2136473600 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001781344032096289, + "loss": 1.3189, + "theoretical_loss": 3.411687029459771, + "tokens_seen": 2136473600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017811434302908728, + "loss": 1.2574, + "theoretical_loss": 3.4116698043194056, + "tokens_seen": 2136604672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017809428284854564, + "loss": 1.3606, + "theoretical_loss": 3.4116525805315527, + "tokens_seen": 2136735744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017807422266800403, + "loss": 1.3927, + "theoretical_loss": 3.4116353580960244, + "tokens_seen": 2136866816 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001780541624874624, + "loss": 1.5035, + "theoretical_loss": 3.4116181370126304, + "tokens_seen": 2136997888 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017803410230692076, + "loss": 1.3257, + "theoretical_loss": 3.4116009172811816, + "tokens_seen": 2137128960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017801404212637915, + "loss": 1.2791, + "theoretical_loss": 3.41158369890149, + "tokens_seen": 2137260032 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001779939819458375, + "loss": 1.3907, + "theoretical_loss": 3.4115664818733658, + "tokens_seen": 2137391104 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001779739217652959, + "loss": 1.2146, + "theoretical_loss": 3.4115492661966202, + "tokens_seen": 2137522176 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017795386158475426, + "loss": 1.2554, + "theoretical_loss": 3.411532051871064, + "tokens_seen": 2137653248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017793380140421265, + "loss": 1.3832, + "theoretical_loss": 3.411514838896509, + "tokens_seen": 2137784320 + }, + { + "epoch": 0.65, + "learning_rate": 0.000177913741223671, + "loss": 1.2802, + "theoretical_loss": 3.411497627272766, + "tokens_seen": 2137915392 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001778936810431294, + "loss": 1.222, + "theoretical_loss": 3.4114804169996464, + "tokens_seen": 2138046464 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.4525904059410095, + "objective/train/avg_token_score": 0.017068156972527504, + "objective/train/docs_used": 1210838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1511309146881104, + "objective/train/original_loss": 3.1748669147491455, + "objective/train/theoretical_loss": 3.411471812369511, + "objective/train/tokens_used": 2138112000, + "objective/train/unlikelihood_loss": 0.24462713301181793, + "theoretical_loss": 3.411471812369511, + "tokens_seen": 2138112000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017787362086258776, + "loss": 1.2752, + "theoretical_loss": 3.4114632080769605, + "tokens_seen": 2138177536 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017785356068204613, + "loss": 1.5449, + "theoretical_loss": 3.4114460005045206, + "tokens_seen": 2138308608 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017783350050150452, + "loss": 1.1579, + "theoretical_loss": 3.4114287942821377, + "tokens_seen": 2138439680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017781344032096288, + "loss": 1.1861, + "theoretical_loss": 3.4114115894096226, + "tokens_seen": 2138570752 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017779338014042127, + "loss": 1.4027, + "theoretical_loss": 3.4113943858867875, + "tokens_seen": 2138701824 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017777331995987963, + "loss": 1.3614, + "theoretical_loss": 3.4113771837134435, + "tokens_seen": 2138832896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017775325977933802, + "loss": 1.4402, + "theoretical_loss": 3.411359982889402, + "tokens_seen": 2138963968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017773319959879638, + "loss": 1.3531, + "theoretical_loss": 3.411342783414474, + "tokens_seen": 2139095040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017771313941825477, + "loss": 1.243, + "theoretical_loss": 3.411325585288472, + "tokens_seen": 2139226112 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017769307923771313, + "loss": 1.2508, + "theoretical_loss": 3.411308388511207, + "tokens_seen": 2139357184 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017767301905717152, + "loss": 1.1447, + "theoretical_loss": 3.4112911930824907, + "tokens_seen": 2139488256 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017765295887662988, + "loss": 1.469, + "theoretical_loss": 3.411273999002135, + "tokens_seen": 2139619328 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.32178640365600586, + "objective/train/avg_token_score": 0.02578609436750412, + "objective/train/docs_used": 1211569, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9785780906677246, + "objective/train/original_loss": 4.122771739959717, + "objective/train/theoretical_loss": 3.411256806269951, + "objective/train/tokens_used": 2139750400, + "objective/train/unlikelihood_loss": 0.1451168656349182, + "theoretical_loss": 3.411256806269951, + "tokens_seen": 2139750400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017763289869608825, + "loss": 1.3435, + "theoretical_loss": 3.411256806269951, + "tokens_seen": 2139750400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017761283851554664, + "loss": 1.4001, + "theoretical_loss": 3.4112396148857504, + "tokens_seen": 2139881472 + }, + { + "epoch": 0.65, + "learning_rate": 0.000177592778335005, + "loss": 1.3342, + "theoretical_loss": 3.411222424849346, + "tokens_seen": 2140012544 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001775727181544634, + "loss": 1.3604, + "theoretical_loss": 3.4112052361605487, + "tokens_seen": 2140143616 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017755265797392175, + "loss": 1.2607, + "theoretical_loss": 3.41118804881917, + "tokens_seen": 2140274688 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017753259779338017, + "loss": 1.4027, + "theoretical_loss": 3.411170862825023, + "tokens_seen": 2140405760 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017751253761283853, + "loss": 1.4596, + "theoretical_loss": 3.4111536781779184, + "tokens_seen": 2140536832 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017749247743229692, + "loss": 1.3141, + "theoretical_loss": 3.4111364948776686, + "tokens_seen": 2140667904 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017747241725175528, + "loss": 1.3635, + "theoretical_loss": 3.411119312924086, + "tokens_seen": 2140798976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017745235707121364, + "loss": 1.2205, + "theoretical_loss": 3.411102132316982, + "tokens_seen": 2140930048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017743229689067203, + "loss": 1.4575, + "theoretical_loss": 3.411084953056169, + "tokens_seen": 2141061120 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001774122367101304, + "loss": 1.409, + "theoretical_loss": 3.411067775141459, + "tokens_seen": 2141192192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017739217652958878, + "loss": 1.3429, + "theoretical_loss": 3.411050598572664, + "tokens_seen": 2141323264 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.49718964099884033, + "objective/train/avg_token_score": 0.01187342219054699, + "objective/train/docs_used": 1213144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.72148859500885, + "objective/train/original_loss": 3.8219923973083496, + "objective/train/theoretical_loss": 3.4110420107929262, + "objective/train/tokens_used": 2141388800, + "objective/train/unlikelihood_loss": 0.1743890941143036, + "theoretical_loss": 3.4110420107929262, + "tokens_seen": 2141388800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017737211634904715, + "loss": 1.3595, + "theoretical_loss": 3.4110334233495965, + "tokens_seen": 2141454336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017735205616850553, + "loss": 1.2239, + "theoretical_loss": 3.4110162494720684, + "tokens_seen": 2141585408 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001773319959879639, + "loss": 1.3925, + "theoretical_loss": 3.410999076939892, + "tokens_seen": 2141716480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017731193580742229, + "loss": 1.2806, + "theoretical_loss": 3.4109819057528794, + "tokens_seen": 2141847552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017729187562688065, + "loss": 1.43, + "theoretical_loss": 3.4109647359108437, + "tokens_seen": 2141978624 + }, + { + "epoch": 0.65, + "learning_rate": 0.000177271815446339, + "loss": 1.212, + "theoretical_loss": 3.4109475674135963, + "tokens_seen": 2142109696 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001772517552657974, + "loss": 1.3039, + "theoretical_loss": 3.41093040026095, + "tokens_seen": 2142240768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017723169508525576, + "loss": 1.2357, + "theoretical_loss": 3.4109132344527175, + "tokens_seen": 2142371840 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017721163490471415, + "loss": 1.3271, + "theoretical_loss": 3.410896069988711, + "tokens_seen": 2142502912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017719157472417251, + "loss": 1.317, + "theoretical_loss": 3.410878906868743, + "tokens_seen": 2142633984 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001771715145436309, + "loss": 1.2215, + "theoretical_loss": 3.4108617450926255, + "tokens_seen": 2142765056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017715145436308927, + "loss": 1.1113, + "theoretical_loss": 3.4108445846601727, + "tokens_seen": 2142896128 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.4610826075077057, + "objective/train/avg_token_score": 0.01991514302790165, + "objective/train/docs_used": 1213818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5608515739440918, + "objective/train/original_loss": 3.720557451248169, + "objective/train/theoretical_loss": 3.4108274255711954, + "objective/train/tokens_used": 2143027200, + "objective/train/unlikelihood_loss": 0.1177854984998703, + "theoretical_loss": 3.4108274255711954, + "tokens_seen": 2143027200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017713139418254765, + "loss": 1.3068, + "theoretical_loss": 3.4108274255711954, + "tokens_seen": 2143027200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017711133400200602, + "loss": 1.2925, + "theoretical_loss": 3.4108102678255072, + "tokens_seen": 2143158272 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001770912738214644, + "loss": 1.3102, + "theoretical_loss": 3.4107931114229206, + "tokens_seen": 2143289344 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017707121364092277, + "loss": 1.4538, + "theoretical_loss": 3.4107759563632487, + "tokens_seen": 2143420416 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017705115346038113, + "loss": 1.3444, + "theoretical_loss": 3.4107588026463036, + "tokens_seen": 2143551488 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017703109327983952, + "loss": 1.2649, + "theoretical_loss": 3.4107416502718984, + "tokens_seen": 2143682560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017701103309929788, + "loss": 1.3357, + "theoretical_loss": 3.4107244992398464, + "tokens_seen": 2143813632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017699097291875627, + "loss": 1.2774, + "theoretical_loss": 3.41070734954996, + "tokens_seen": 2143944704 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017697091273821463, + "loss": 1.3713, + "theoretical_loss": 3.410690201202052, + "tokens_seen": 2144075776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017695085255767302, + "loss": 1.2283, + "theoretical_loss": 3.410673054195936, + "tokens_seen": 2144206848 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017693079237713139, + "loss": 1.555, + "theoretical_loss": 3.4106559085314236, + "tokens_seen": 2144337920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017691073219658978, + "loss": 1.4507, + "theoretical_loss": 3.41063876420833, + "tokens_seen": 2144468992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017689067201604814, + "loss": 1.2991, + "theoretical_loss": 3.4106216212264666, + "tokens_seen": 2144600064 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.44904690980911255, + "objective/train/avg_token_score": 0.00305955414660275, + "objective/train/docs_used": 1215022, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2880713939666748, + "objective/train/original_loss": 3.3166396617889404, + "objective/train/theoretical_loss": 3.4106130502384375, + "objective/train/tokens_used": 2144665600, + "objective/train/unlikelihood_loss": 0.25235339999198914, + "theoretical_loss": 3.4106130502384375, + "tokens_seen": 2144665600 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001768706118355065, + "loss": 1.2788, + "theoretical_loss": 3.410604479585647, + "tokens_seen": 2144731136 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001768505516549649, + "loss": 1.209, + "theoretical_loss": 3.410587339285684, + "tokens_seen": 2144862208 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017683049147442325, + "loss": 1.1759, + "theoretical_loss": 3.4105702003263914, + "tokens_seen": 2144993280 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017681043129388164, + "loss": 1.3465, + "theoretical_loss": 3.410553062707582, + "tokens_seen": 2145124352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017679037111334003, + "loss": 1.3775, + "theoretical_loss": 3.4105359264290693, + "tokens_seen": 2145255424 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017677031093279842, + "loss": 1.1824, + "theoretical_loss": 3.4105187914906665, + "tokens_seen": 2145386496 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017675025075225678, + "loss": 1.2153, + "theoretical_loss": 3.410501657892187, + "tokens_seen": 2145517568 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017673019057171517, + "loss": 1.1627, + "theoretical_loss": 3.410484525633444, + "tokens_seen": 2145648640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017671013039117353, + "loss": 1.2444, + "theoretical_loss": 3.410467394714251, + "tokens_seen": 2145779712 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017669007021063192, + "loss": 1.4113, + "theoretical_loss": 3.410450265134422, + "tokens_seen": 2145910784 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017667001003009028, + "loss": 1.2583, + "theoretical_loss": 3.4104331368937695, + "tokens_seen": 2146041856 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017664994984954865, + "loss": 1.2818, + "theoretical_loss": 3.4104160099921077, + "tokens_seen": 2146172928 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.4820992052555084, + "objective/train/avg_token_score": 0.029476068913936615, + "objective/train/docs_used": 1215651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5438436269760132, + "objective/train/original_loss": 4.010006427764893, + "objective/train/theoretical_loss": 3.4103988844292497, + "objective/train/tokens_used": 2146304000, + "objective/train/unlikelihood_loss": 0.13768695294857025, + "theoretical_loss": 3.4103988844292497, + "tokens_seen": 2146304000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017662988966900704, + "loss": 1.2681, + "theoretical_loss": 3.4103988844292497, + "tokens_seen": 2146304000 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001766098294884654, + "loss": 1.2515, + "theoretical_loss": 3.4103817602050093, + "tokens_seen": 2146435072 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765897693079238, + "loss": 1.2171, + "theoretical_loss": 3.4103646373192005, + "tokens_seen": 2146566144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017656970912738215, + "loss": 1.2498, + "theoretical_loss": 3.4103475157716363, + "tokens_seen": 2146697216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017654964894684054, + "loss": 1.293, + "theoretical_loss": 3.4103303955621307, + "tokens_seen": 2146828288 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765295887662989, + "loss": 1.3044, + "theoretical_loss": 3.4103132766904976, + "tokens_seen": 2146959360 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765095285857573, + "loss": 1.3971, + "theoretical_loss": 3.410296159156551, + "tokens_seen": 2147090432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017648946840521565, + "loss": 1.2436, + "theoretical_loss": 3.410279042960104, + "tokens_seen": 2147221504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017646940822467402, + "loss": 1.4638, + "theoretical_loss": 3.4102619281009714, + "tokens_seen": 2147352576 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001764493480441324, + "loss": 1.4005, + "theoretical_loss": 3.4102448145789657, + "tokens_seen": 2147483648 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017642928786359077, + "loss": 1.316, + "theoretical_loss": 3.4102277023939025, + "tokens_seen": 2147614720 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017640922768304916, + "loss": 1.5689, + "theoretical_loss": 3.4102105915455945, + "tokens_seen": 2147745792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017638916750250752, + "loss": 1.3771, + "theoretical_loss": 3.410193482033856, + "tokens_seen": 2147876864 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.39913246035575867, + "objective/train/avg_token_score": 0.009204434230923653, + "objective/train/docs_used": 1216251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3201860189437866, + "objective/train/original_loss": 3.8831491470336914, + "objective/train/theoretical_loss": 3.4101849277791425, + "objective/train/tokens_used": 2147942400, + "objective/train/unlikelihood_loss": 0.18366695940494537, + "theoretical_loss": 3.4101849277791425, + "tokens_seen": 2147942400 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001763691073219659, + "loss": 1.2406, + "theoretical_loss": 3.4101763738585014, + "tokens_seen": 2148007936 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017634904714142427, + "loss": 1.3015, + "theoretical_loss": 3.4101592670193446, + "tokens_seen": 2148139008 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017632898696088266, + "loss": 1.2872, + "theoretical_loss": 3.4101421615162, + "tokens_seen": 2148270080 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017630892678034102, + "loss": 1.3431, + "theoretical_loss": 3.410125057348881, + "tokens_seen": 2148401152 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017628886659979938, + "loss": 1.2931, + "theoretical_loss": 3.4101079545172026, + "tokens_seen": 2148532224 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017626880641925777, + "loss": 1.2539, + "theoretical_loss": 3.4100908530209786, + "tokens_seen": 2148663296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017624874623871614, + "loss": 1.431, + "theoretical_loss": 3.410073752860023, + "tokens_seen": 2148794368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017622868605817452, + "loss": 1.137, + "theoretical_loss": 3.410056654034151, + "tokens_seen": 2148925440 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001762086258776329, + "loss": 1.3359, + "theoretical_loss": 3.4100395565431763, + "tokens_seen": 2149056512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017618856569709128, + "loss": 1.2961, + "theoretical_loss": 3.410022460386913, + "tokens_seen": 2149187584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017616850551654964, + "loss": 1.186, + "theoretical_loss": 3.4100053655651763, + "tokens_seen": 2149318656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017614844533600803, + "loss": 1.4793, + "theoretical_loss": 3.4099882720777797, + "tokens_seen": 2149449728 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.43866080045700073, + "objective/train/avg_token_score": 0.026488972827792168, + "objective/train/docs_used": 1217415, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3665950298309326, + "objective/train/original_loss": 4.008810997009277, + "objective/train/theoretical_loss": 3.4099711799245385, + "objective/train/tokens_used": 2149580800, + "objective/train/unlikelihood_loss": 0.12495262920856476, + "theoretical_loss": 3.4099711799245385, + "tokens_seen": 2149580800 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001761283851554664, + "loss": 1.2696, + "theoretical_loss": 3.4099711799245385, + "tokens_seen": 2149580800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017610832497492478, + "loss": 1.2942, + "theoretical_loss": 3.409954089105267, + "tokens_seen": 2149711872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017608826479438314, + "loss": 1.4118, + "theoretical_loss": 3.40993699961978, + "tokens_seen": 2149842944 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001760682046138415, + "loss": 1.1904, + "theoretical_loss": 3.4099199114678918, + "tokens_seen": 2149974016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017604814443329992, + "loss": 1.2263, + "theoretical_loss": 3.4099028246494165, + "tokens_seen": 2150105088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017602808425275828, + "loss": 1.3065, + "theoretical_loss": 3.40988573916417, + "tokens_seen": 2150236160 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017600802407221667, + "loss": 1.2711, + "theoretical_loss": 3.409868655011966, + "tokens_seen": 2150367232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017598796389167503, + "loss": 1.3939, + "theoretical_loss": 3.4098515721926193, + "tokens_seen": 2150498304 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017596790371113342, + "loss": 1.3262, + "theoretical_loss": 3.4098344907059452, + "tokens_seen": 2150629376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017594784353059179, + "loss": 1.2395, + "theoretical_loss": 3.4098174105517582, + "tokens_seen": 2150760448 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017592778335005018, + "loss": 1.3556, + "theoretical_loss": 3.4098003317298735, + "tokens_seen": 2150891520 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017590772316950854, + "loss": 1.3217, + "theoretical_loss": 3.4097832542401063, + "tokens_seen": 2151022592 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758876629889669, + "loss": 1.3782, + "theoretical_loss": 3.40976617808227, + "tokens_seen": 2151153664 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.42766374349594116, + "objective/train/avg_token_score": 0.013860506936907768, + "objective/train/docs_used": 1217907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3609411716461182, + "objective/train/original_loss": 3.6805081367492676, + "objective/train/theoretical_loss": 3.409757640502769, + "objective/train/tokens_used": 2151219200, + "objective/train/unlikelihood_loss": 0.1662524938583374, + "theoretical_loss": 3.409757640502769, + "tokens_seen": 2151219200 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758676028084253, + "loss": 1.3046, + "theoretical_loss": 3.409749103256181, + "tokens_seen": 2151284736 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017584754262788365, + "loss": 1.3981, + "theoretical_loss": 3.4097320297616545, + "tokens_seen": 2151415808 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017582748244734204, + "loss": 1.2349, + "theoretical_loss": 3.4097149575985037, + "tokens_seen": 2151546880 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758074222668004, + "loss": 1.3219, + "theoretical_loss": 3.409697886766546, + "tokens_seen": 2151677952 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001757873620862588, + "loss": 1.2505, + "theoretical_loss": 3.409680817265595, + "tokens_seen": 2151809024 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017576730190571715, + "loss": 1.2685, + "theoretical_loss": 3.409663749095466, + "tokens_seen": 2151940096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017574724172517554, + "loss": 1.2936, + "theoretical_loss": 3.409646682255975, + "tokens_seen": 2152071168 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001757271815446339, + "loss": 1.3852, + "theoretical_loss": 3.4096296167469364, + "tokens_seen": 2152202240 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017570712136409227, + "loss": 1.387, + "theoretical_loss": 3.409612552568166, + "tokens_seen": 2152333312 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017568706118355066, + "loss": 1.413, + "theoretical_loss": 3.4095954897194787, + "tokens_seen": 2152464384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017566700100300902, + "loss": 1.1201, + "theoretical_loss": 3.40957842820069, + "tokens_seen": 2152595456 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001756469408224674, + "loss": 1.4285, + "theoretical_loss": 3.4095613680116155, + "tokens_seen": 2152726528 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.46340420842170715, + "objective/train/avg_token_score": 0.009474211372435093, + "objective/train/docs_used": 1219413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.546663522720337, + "objective/train/original_loss": 3.567782402038574, + "objective/train/theoretical_loss": 3.4095443091520705, + "objective/train/tokens_used": 2152857600, + "objective/train/unlikelihood_loss": 0.20949453115463257, + "theoretical_loss": 3.4095443091520705, + "tokens_seen": 2152857600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017562688064192577, + "loss": 1.403, + "theoretical_loss": 3.4095443091520705, + "tokens_seen": 2152857600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017560682046138416, + "loss": 1.2775, + "theoretical_loss": 3.4095272516218698, + "tokens_seen": 2152988672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017558676028084252, + "loss": 1.2916, + "theoretical_loss": 3.40951019542083, + "tokens_seen": 2153119744 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001755667001003009, + "loss": 1.3179, + "theoretical_loss": 3.409493140548766, + "tokens_seen": 2153250816 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017554663991975927, + "loss": 1.3172, + "theoretical_loss": 3.409476087005493, + "tokens_seen": 2153381888 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017552657973921766, + "loss": 1.1942, + "theoretical_loss": 3.4094590347908276, + "tokens_seen": 2153512960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017550651955867603, + "loss": 1.2796, + "theoretical_loss": 3.4094419839045846, + "tokens_seen": 2153644032 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001754864593781344, + "loss": 1.2299, + "theoretical_loss": 3.40942493434658, + "tokens_seen": 2153775104 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017546639919759278, + "loss": 1.1643, + "theoretical_loss": 3.4094078861166297, + "tokens_seen": 2153906176 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017544633901705114, + "loss": 1.2413, + "theoretical_loss": 3.4093908392145487, + "tokens_seen": 2154037248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017542627883650953, + "loss": 1.3461, + "theoretical_loss": 3.4093737936401536, + "tokens_seen": 2154168320 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001754062186559679, + "loss": 1.2416, + "theoretical_loss": 3.4093567493932597, + "tokens_seen": 2154299392 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017538615847542628, + "loss": 1.303, + "theoretical_loss": 3.4093397064736832, + "tokens_seen": 2154430464 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.4808773100376129, + "objective/train/avg_token_score": 0.02487337402999401, + "objective/train/docs_used": 1220074, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5643993616104126, + "objective/train/original_loss": 3.455756664276123, + "objective/train/theoretical_loss": 3.4093311855115815, + "objective/train/tokens_used": 2154496000, + "objective/train/unlikelihood_loss": 0.20628929138183594, + "theoretical_loss": 3.4093311855115815, + "tokens_seen": 2154496000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017536609829488464, + "loss": 1.4143, + "theoretical_loss": 3.4093226648812403, + "tokens_seen": 2154561536 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017534603811434303, + "loss": 1.3795, + "theoretical_loss": 3.4093056246157456, + "tokens_seen": 2154692608 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001753259779338014, + "loss": 1.1726, + "theoretical_loss": 3.4092885856770168, + "tokens_seen": 2154823680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017530591775325976, + "loss": 1.3017, + "theoretical_loss": 3.4092715480648685, + "tokens_seen": 2154954752 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017528585757271817, + "loss": 1.3831, + "theoretical_loss": 3.4092545117791175, + "tokens_seen": 2155085824 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017526579739217654, + "loss": 1.2616, + "theoretical_loss": 3.4092374768195794, + "tokens_seen": 2155216896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017524573721163493, + "loss": 1.3501, + "theoretical_loss": 3.4092204431860713, + "tokens_seen": 2155347968 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001752256770310933, + "loss": 1.5118, + "theoretical_loss": 3.409203410878408, + "tokens_seen": 2155479040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017520561685055168, + "loss": 1.3295, + "theoretical_loss": 3.4091863798964064, + "tokens_seen": 2155610112 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017518555667001004, + "loss": 1.3558, + "theoretical_loss": 3.4091693502398828, + "tokens_seen": 2155741184 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017516549648946843, + "loss": 1.2415, + "theoretical_loss": 3.4091523219086532, + "tokens_seen": 2155872256 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001751454363089268, + "loss": 1.2336, + "theoretical_loss": 3.4091352949025344, + "tokens_seen": 2156003328 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.5097140669822693, + "objective/train/avg_token_score": 0.010697956196963787, + "objective/train/docs_used": 1221515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6273577213287354, + "objective/train/original_loss": 3.5189735889434814, + "objective/train/theoretical_loss": 3.4091182692213415, + "objective/train/tokens_used": 2156134400, + "objective/train/unlikelihood_loss": 0.10906907916069031, + "theoretical_loss": 3.4091182692213415, + "tokens_seen": 2156134400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017512537612838518, + "loss": 1.3908, + "theoretical_loss": 3.4091182692213415, + "tokens_seen": 2156134400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017510531594784354, + "loss": 1.2873, + "theoretical_loss": 3.4091012448648925, + "tokens_seen": 2156265472 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001750852557673019, + "loss": 1.2182, + "theoretical_loss": 3.4090842218330026, + "tokens_seen": 2156396544 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001750651955867603, + "loss": 1.3982, + "theoretical_loss": 3.409067200125489, + "tokens_seen": 2156527616 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017504513540621866, + "loss": 1.1247, + "theoretical_loss": 3.4090501797421675, + "tokens_seen": 2156658688 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017502507522567705, + "loss": 1.1813, + "theoretical_loss": 3.409033160682855, + "tokens_seen": 2156789760 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001750050150451354, + "loss": 1.2581, + "theoretical_loss": 3.409016142947368, + "tokens_seen": 2156920832 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001749849548645938, + "loss": 1.5579, + "theoretical_loss": 3.408999126535523, + "tokens_seen": 2157051904 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017496489468405216, + "loss": 1.2282, + "theoretical_loss": 3.408982111447137, + "tokens_seen": 2157182976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017494483450351055, + "loss": 1.258, + "theoretical_loss": 3.4089650976820263, + "tokens_seen": 2157314048 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001749247743229689, + "loss": 1.4226, + "theoretical_loss": 3.4089480852400076, + "tokens_seen": 2157445120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017490471414242727, + "loss": 1.2698, + "theoretical_loss": 3.408931074120898, + "tokens_seen": 2157576192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017488465396188566, + "loss": 1.3469, + "theoretical_loss": 3.4089140643245135, + "tokens_seen": 2157707264 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.5538856387138367, + "objective/train/avg_token_score": 0.019045038148760796, + "objective/train/docs_used": 1222265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8272037506103516, + "objective/train/original_loss": 3.911555290222168, + "objective/train/theoretical_loss": 3.408905559922286, + "objective/train/tokens_used": 2157772800, + "objective/train/unlikelihood_loss": 0.12278693169355392, + "theoretical_loss": 3.408905559922286, + "tokens_seen": 2157772800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017486459378134402, + "loss": 1.5488, + "theoretical_loss": 3.4088970558506717, + "tokens_seen": 2157838336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017484453360080241, + "loss": 1.3203, + "theoretical_loss": 3.408880048699189, + "tokens_seen": 2157969408 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017482447342026078, + "loss": 1.2905, + "theoretical_loss": 3.4088630428698825, + "tokens_seen": 2158100480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017480441323971917, + "loss": 1.3223, + "theoretical_loss": 3.408846038362569, + "tokens_seen": 2158231552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017478435305917753, + "loss": 1.1928, + "theoretical_loss": 3.4088290351770656, + "tokens_seen": 2158362624 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017476429287863592, + "loss": 1.4067, + "theoretical_loss": 3.4088120333131893, + "tokens_seen": 2158493696 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017474423269809428, + "loss": 1.2207, + "theoretical_loss": 3.408795032770757, + "tokens_seen": 2158624768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017472417251755264, + "loss": 1.3498, + "theoretical_loss": 3.408778033549585, + "tokens_seen": 2158755840 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017470411233701103, + "loss": 1.3098, + "theoretical_loss": 3.4087610356494924, + "tokens_seen": 2158886912 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746840521564694, + "loss": 1.4563, + "theoretical_loss": 3.408744039070294, + "tokens_seen": 2159017984 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017466399197592778, + "loss": 1.3644, + "theoretical_loss": 3.408727043811809, + "tokens_seen": 2159149056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017464393179538614, + "loss": 1.4266, + "theoretical_loss": 3.4087100498738536, + "tokens_seen": 2159280128 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.35428884625434875, + "objective/train/avg_token_score": 0.029898177832365036, + "objective/train/docs_used": 1223379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2096444368362427, + "objective/train/original_loss": 3.956677198410034, + "objective/train/theoretical_loss": 3.4086930572562446, + "objective/train/tokens_used": 2159411200, + "objective/train/unlikelihood_loss": 0.2211945503950119, + "theoretical_loss": 3.4086930572562446, + "tokens_seen": 2159411200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017462387161484453, + "loss": 1.3568, + "theoretical_loss": 3.4086930572562446, + "tokens_seen": 2159411200 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746038114343029, + "loss": 1.3255, + "theoretical_loss": 3.4086760659588, + "tokens_seen": 2159542272 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017458375125376129, + "loss": 1.4518, + "theoretical_loss": 3.408659075981337, + "tokens_seen": 2159673344 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017456369107321965, + "loss": 1.3341, + "theoretical_loss": 3.408642087323673, + "tokens_seen": 2159804416 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017454363089267806, + "loss": 1.2872, + "theoretical_loss": 3.4086250999856254, + "tokens_seen": 2159935488 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017452357071213643, + "loss": 1.4177, + "theoretical_loss": 3.408608113967011, + "tokens_seen": 2160066560 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001745035105315948, + "loss": 1.4047, + "theoretical_loss": 3.4085911292676485, + "tokens_seen": 2160197632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017448345035105318, + "loss": 1.3895, + "theoretical_loss": 3.4085741458873544, + "tokens_seen": 2160328704 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017446339017051154, + "loss": 1.2428, + "theoretical_loss": 3.408557163825946, + "tokens_seen": 2160459776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017444332998996993, + "loss": 1.2527, + "theoretical_loss": 3.4085401830832422, + "tokens_seen": 2160590848 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001744232698094283, + "loss": 1.3627, + "theoretical_loss": 3.408523203659059, + "tokens_seen": 2160721920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017440320962888668, + "loss": 1.3164, + "theoretical_loss": 3.4085062255532157, + "tokens_seen": 2160852992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017438314944834504, + "loss": 1.3231, + "theoretical_loss": 3.4084892487655285, + "tokens_seen": 2160984064 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.3732282519340515, + "objective/train/avg_token_score": 0.005723316688090563, + "objective/train/docs_used": 1223894, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9199358224868774, + "objective/train/original_loss": 3.185774087905884, + "objective/train/theoretical_loss": 3.408480760865937, + "objective/train/tokens_used": 2161049600, + "objective/train/unlikelihood_loss": 0.24825607240200043, + "theoretical_loss": 3.408480760865937, + "tokens_seen": 2161049600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017436308926780343, + "loss": 1.3548, + "theoretical_loss": 3.408472273295816, + "tokens_seen": 2161115136 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001743430290872618, + "loss": 1.3811, + "theoretical_loss": 3.408455299143896, + "tokens_seen": 2161246208 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017432296890672016, + "loss": 1.2697, + "theoretical_loss": 3.4084383263095854, + "tokens_seen": 2161377280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017430290872617855, + "loss": 1.3175, + "theoretical_loss": 3.408421354792703, + "tokens_seen": 2161508352 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001742828485456369, + "loss": 1.4249, + "theoretical_loss": 3.408404384593066, + "tokens_seen": 2161639424 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001742627883650953, + "loss": 1.3611, + "theoretical_loss": 3.408387415710493, + "tokens_seen": 2161770496 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017424272818455366, + "loss": 1.1308, + "theoretical_loss": 3.4083704481448014, + "tokens_seen": 2161901568 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017422266800401205, + "loss": 1.2083, + "theoretical_loss": 3.408353481895809, + "tokens_seen": 2162032640 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001742026078234704, + "loss": 1.2701, + "theoretical_loss": 3.408336516963334, + "tokens_seen": 2162163712 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001741825476429288, + "loss": 1.3555, + "theoretical_loss": 3.4083195533471953, + "tokens_seen": 2162294784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017416248746238716, + "loss": 1.3721, + "theoretical_loss": 3.4083025910472093, + "tokens_seen": 2162425856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017414242728184553, + "loss": 1.3304, + "theoretical_loss": 3.4082856300631956, + "tokens_seen": 2162556928 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.43285679817199707, + "objective/train/avg_token_score": 0.026902787387371063, + "objective/train/docs_used": 1225280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2378101348876953, + "objective/train/original_loss": 3.457395076751709, + "objective/train/theoretical_loss": 3.4082686703949716, + "objective/train/tokens_used": 2162688000, + "objective/train/unlikelihood_loss": 0.20558080077171326, + "theoretical_loss": 3.4082686703949716, + "tokens_seen": 2162688000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017412236710130392, + "loss": 1.2733, + "theoretical_loss": 3.4082686703949716, + "tokens_seen": 2162688000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017410230692076228, + "loss": 1.3331, + "theoretical_loss": 3.4082517120423557, + "tokens_seen": 2162819072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017408224674022067, + "loss": 1.4498, + "theoretical_loss": 3.408234755005166, + "tokens_seen": 2162950144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017406218655967903, + "loss": 1.3132, + "theoretical_loss": 3.4082177992832214, + "tokens_seen": 2163081216 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017404212637913742, + "loss": 1.2865, + "theoretical_loss": 3.4082008448763395, + "tokens_seen": 2163212288 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017402206619859578, + "loss": 1.4343, + "theoretical_loss": 3.408183891784338, + "tokens_seen": 2163343360 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017400200601805417, + "loss": 1.1992, + "theoretical_loss": 3.408166940007037, + "tokens_seen": 2163474432 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017398194583751253, + "loss": 1.3406, + "theoretical_loss": 3.4081499895442535, + "tokens_seen": 2163605504 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017396188565697092, + "loss": 1.2534, + "theoretical_loss": 3.4081330403958066, + "tokens_seen": 2163736576 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017394182547642928, + "loss": 1.1525, + "theoretical_loss": 3.4081160925615146, + "tokens_seen": 2163867648 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017392176529588765, + "loss": 1.3454, + "theoretical_loss": 3.408099146041196, + "tokens_seen": 2163998720 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017390170511534604, + "loss": 1.3325, + "theoretical_loss": 3.408082200834669, + "tokens_seen": 2164129792 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738816449348044, + "loss": 1.3592, + "theoretical_loss": 3.408065256941753, + "tokens_seen": 2164260864 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.43420088291168213, + "objective/train/avg_token_score": 0.01722126454114914, + "objective/train/docs_used": 1225734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3030797243118286, + "objective/train/original_loss": 3.7955288887023926, + "objective/train/theoretical_loss": 3.408056785487842, + "objective/train/tokens_used": 2164326400, + "objective/train/unlikelihood_loss": 0.16144044697284698, + "theoretical_loss": 3.408056785487842, + "tokens_seen": 2164326400 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738615847542628, + "loss": 1.2658, + "theoretical_loss": 3.408048314362266, + "tokens_seen": 2164391936 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017384152457372115, + "loss": 1.231, + "theoretical_loss": 3.4080313730960268, + "tokens_seen": 2164523008 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017382146439317954, + "loss": 1.3056, + "theoretical_loss": 3.408014433142854, + "tokens_seen": 2164654080 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738014042126379, + "loss": 1.3479, + "theoretical_loss": 3.407997494502567, + "tokens_seen": 2164785152 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017378134403209632, + "loss": 1.3066, + "theoretical_loss": 3.4079805571749833, + "tokens_seen": 2164916224 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017376128385155468, + "loss": 1.3294, + "theoretical_loss": 3.4079636211599227, + "tokens_seen": 2165047296 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017374122367101304, + "loss": 1.1185, + "theoretical_loss": 3.407946686457204, + "tokens_seen": 2165178368 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017372116349047143, + "loss": 1.4631, + "theoretical_loss": 3.4079297530666457, + "tokens_seen": 2165309440 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001737011033099298, + "loss": 1.361, + "theoretical_loss": 3.4079128209880665, + "tokens_seen": 2165440512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017368104312938818, + "loss": 1.1929, + "theoretical_loss": 3.407895890221286, + "tokens_seen": 2165571584 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017366098294884655, + "loss": 1.43, + "theoretical_loss": 3.407878960766123, + "tokens_seen": 2165702656 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017364092276830493, + "loss": 1.2749, + "theoretical_loss": 3.4078620326223956, + "tokens_seen": 2165833728 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.4650537669658661, + "objective/train/avg_token_score": 0.02273428998887539, + "objective/train/docs_used": 1226526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.480191946029663, + "objective/train/original_loss": 3.7012853622436523, + "objective/train/theoretical_loss": 3.4078451057899244, + "objective/train/tokens_used": 2165964800, + "objective/train/unlikelihood_loss": 0.17883025109767914, + "theoretical_loss": 3.4078451057899244, + "tokens_seen": 2165964800 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001736208625877633, + "loss": 1.2491, + "theoretical_loss": 3.4078451057899244, + "tokens_seen": 2165964800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017360080240722169, + "loss": 1.3335, + "theoretical_loss": 3.4078281802685275, + "tokens_seen": 2166095872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017358074222668005, + "loss": 1.1507, + "theoretical_loss": 3.4078112560580243, + "tokens_seen": 2166226944 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001735606820461384, + "loss": 1.3822, + "theoretical_loss": 3.407794333158234, + "tokens_seen": 2166358016 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001735406218655968, + "loss": 1.4586, + "theoretical_loss": 3.407777411568975, + "tokens_seen": 2166489088 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017352056168505516, + "loss": 1.2911, + "theoretical_loss": 3.407760491290068, + "tokens_seen": 2166620160 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017350050150451355, + "loss": 1.2629, + "theoretical_loss": 3.4077435723213316, + "tokens_seen": 2166751232 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017348044132397191, + "loss": 1.2847, + "theoretical_loss": 3.407726654662585, + "tokens_seen": 2166882304 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001734603811434303, + "loss": 1.4367, + "theoretical_loss": 3.4077097383136468, + "tokens_seen": 2167013376 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017344032096288867, + "loss": 1.3421, + "theoretical_loss": 3.4076928232743375, + "tokens_seen": 2167144448 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017342026078234705, + "loss": 1.4449, + "theoretical_loss": 3.4076759095444764, + "tokens_seen": 2167275520 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017340020060180542, + "loss": 1.3136, + "theoretical_loss": 3.4076589971238826, + "tokens_seen": 2167406592 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001733801404212638, + "loss": 1.1794, + "theoretical_loss": 3.4076420860123755, + "tokens_seen": 2167537664 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.46041056513786316, + "objective/train/avg_token_score": 0.010373012162744999, + "objective/train/docs_used": 1227923, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2640165090560913, + "objective/train/original_loss": 3.443406343460083, + "objective/train/theoretical_loss": 3.4076336309474735, + "objective/train/tokens_used": 2167603200, + "objective/train/unlikelihood_loss": 0.15692183375358582, + "theoretical_loss": 3.4076336309474735, + "tokens_seen": 2167603200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017336008024072217, + "loss": 1.3659, + "theoretical_loss": 3.407625176209775, + "tokens_seen": 2167668736 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017334002006018053, + "loss": 1.2898, + "theoretical_loss": 3.4076082677159003, + "tokens_seen": 2167799808 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017331995987963892, + "loss": 1.3017, + "theoretical_loss": 3.407591360530571, + "tokens_seen": 2167930880 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017329989969909728, + "loss": 1.3416, + "theoretical_loss": 3.4075744546536075, + "tokens_seen": 2168061952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017327983951855567, + "loss": 1.3, + "theoretical_loss": 3.407557550084828, + "tokens_seen": 2168193024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017325977933801403, + "loss": 1.3942, + "theoretical_loss": 3.4075406468240534, + "tokens_seen": 2168324096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017323971915747242, + "loss": 1.2178, + "theoretical_loss": 3.4075237448711033, + "tokens_seen": 2168455168 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017321965897693079, + "loss": 1.2784, + "theoretical_loss": 3.407506844225797, + "tokens_seen": 2168586240 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017319959879638917, + "loss": 1.157, + "theoretical_loss": 3.4074899448879545, + "tokens_seen": 2168717312 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017317953861584754, + "loss": 1.1948, + "theoretical_loss": 3.4074730468573957, + "tokens_seen": 2168848384 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001731594784353059, + "loss": 1.3618, + "theoretical_loss": 3.4074561501339407, + "tokens_seen": 2168979456 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001731394182547643, + "loss": 1.1718, + "theoretical_loss": 3.407439254717409, + "tokens_seen": 2169110528 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.3870967626571655, + "objective/train/avg_token_score": 0.02737361565232277, + "objective/train/docs_used": 1228616, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0638030767440796, + "objective/train/original_loss": 3.77616548538208, + "objective/train/theoretical_loss": 3.4074223606076206, + "objective/train/tokens_used": 2169241600, + "objective/train/unlikelihood_loss": 0.12934909760951996, + "theoretical_loss": 3.4074223606076206, + "tokens_seen": 2169241600 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017311935807422265, + "loss": 1.1739, + "theoretical_loss": 3.4074223606076206, + "tokens_seen": 2169241600 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017309929789368104, + "loss": 1.3491, + "theoretical_loss": 3.407405467804396, + "tokens_seen": 2169372672 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001730792377131394, + "loss": 1.3258, + "theoretical_loss": 3.407388576307554, + "tokens_seen": 2169503744 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001730591775325978, + "loss": 1.3335, + "theoretical_loss": 3.407371686116916, + "tokens_seen": 2169634816 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017303911735205618, + "loss": 1.3535, + "theoretical_loss": 3.4073547972323017, + "tokens_seen": 2169765888 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017301905717151457, + "loss": 1.2512, + "theoretical_loss": 3.4073379096535312, + "tokens_seen": 2169896960 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017299899699097293, + "loss": 1.383, + "theoretical_loss": 3.4073210233804243, + "tokens_seen": 2170028032 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017297893681043132, + "loss": 1.3647, + "theoretical_loss": 3.407304138412801, + "tokens_seen": 2170159104 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017295887662988968, + "loss": 1.2218, + "theoretical_loss": 3.4072872547504827, + "tokens_seen": 2170290176 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017293881644934805, + "loss": 1.3233, + "theoretical_loss": 3.407270372393289, + "tokens_seen": 2170421248 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017291875626880644, + "loss": 1.2799, + "theoretical_loss": 3.4072534913410397, + "tokens_seen": 2170552320 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728986960882648, + "loss": 1.425, + "theoretical_loss": 3.407236611593556, + "tokens_seen": 2170683392 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728786359077232, + "loss": 1.2989, + "theoretical_loss": 3.4072197331506575, + "tokens_seen": 2170814464 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.3898460268974304, + "objective/train/avg_token_score": 0.01922510750591755, + "objective/train/docs_used": 1230033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2772955894470215, + "objective/train/original_loss": 3.9439916610717773, + "objective/train/theoretical_loss": 3.407211294418372, + "objective/train/tokens_used": 2170880000, + "objective/train/unlikelihood_loss": 0.1885448396205902, + "theoretical_loss": 3.407211294418372, + "tokens_seen": 2170880000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017285857572718155, + "loss": 1.2609, + "theoretical_loss": 3.4072028560121654, + "tokens_seen": 2170945536 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017283851554663994, + "loss": 1.4562, + "theoretical_loss": 3.4071859801778994, + "tokens_seen": 2171076608 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728184553660983, + "loss": 1.2647, + "theoretical_loss": 3.4071691056476805, + "tokens_seen": 2171207680 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001727983951855567, + "loss": 1.2637, + "theoretical_loss": 3.4071522324213293, + "tokens_seen": 2171338752 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017277833500501505, + "loss": 1.3197, + "theoretical_loss": 3.407135360498666, + "tokens_seen": 2171469824 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017275827482447342, + "loss": 1.2499, + "theoretical_loss": 3.407118489879511, + "tokens_seen": 2171600896 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001727382146439318, + "loss": 1.3756, + "theoretical_loss": 3.407101620563685, + "tokens_seen": 2171731968 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017271815446339017, + "loss": 1.1178, + "theoretical_loss": 3.4070847525510093, + "tokens_seen": 2171863040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017269809428284856, + "loss": 1.2985, + "theoretical_loss": 3.407067885841304, + "tokens_seen": 2171994112 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017267803410230692, + "loss": 1.3954, + "theoretical_loss": 3.4070510204343902, + "tokens_seen": 2172125184 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001726579739217653, + "loss": 1.2322, + "theoretical_loss": 3.407034156330088, + "tokens_seen": 2172256256 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017263791374122367, + "loss": 1.3734, + "theoretical_loss": 3.4070172935282192, + "tokens_seen": 2172387328 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.33693793416023254, + "objective/train/avg_token_score": 0.016742536798119545, + "objective/train/docs_used": 1230759, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1851741075515747, + "objective/train/original_loss": 4.242486476898193, + "objective/train/theoretical_loss": 3.4070004320286036, + "objective/train/tokens_used": 2172518400, + "objective/train/unlikelihood_loss": 0.15599969029426575, + "theoretical_loss": 3.4070004320286036, + "tokens_seen": 2172518400 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017261785356068206, + "loss": 1.1844, + "theoretical_loss": 3.4070004320286036, + "tokens_seen": 2172518400 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017259779338014042, + "loss": 1.488, + "theoretical_loss": 3.4069835718310633, + "tokens_seen": 2172649472 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017257773319959878, + "loss": 1.4679, + "theoretical_loss": 3.406966712935418, + "tokens_seen": 2172780544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017255767301905717, + "loss": 1.2884, + "theoretical_loss": 3.406949855341489, + "tokens_seen": 2172911616 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017253761283851554, + "loss": 1.3148, + "theoretical_loss": 3.4069329990490975, + "tokens_seen": 2173042688 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017251755265797392, + "loss": 1.2329, + "theoretical_loss": 3.406916144058065, + "tokens_seen": 2173173760 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001724974924774323, + "loss": 1.3969, + "theoretical_loss": 3.4068992903682114, + "tokens_seen": 2173304832 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017247743229689068, + "loss": 1.2926, + "theoretical_loss": 3.406882437979358, + "tokens_seen": 2173435904 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017245737211634904, + "loss": 1.2558, + "theoretical_loss": 3.4068655868913273, + "tokens_seen": 2173566976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017243731193580743, + "loss": 1.2866, + "theoretical_loss": 3.406848737103939, + "tokens_seen": 2173698048 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001724172517552658, + "loss": 1.2997, + "theoretical_loss": 3.406831888617014, + "tokens_seen": 2173829120 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017239719157472418, + "loss": 1.2159, + "theoretical_loss": 3.4068150414303755, + "tokens_seen": 2173960192 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017237713139418254, + "loss": 1.3316, + "theoretical_loss": 3.4067981955438427, + "tokens_seen": 2174091264 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.45741692185401917, + "objective/train/avg_token_score": 0.012394599616527557, + "objective/train/docs_used": 1232037, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3559364080429077, + "objective/train/original_loss": 3.705355405807495, + "objective/train/theoretical_loss": 3.4067897730880605, + "objective/train/tokens_used": 2174156800, + "objective/train/unlikelihood_loss": 0.1495666652917862, + "theoretical_loss": 3.4067897730880605, + "tokens_seen": 2174156800 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001723570712136409, + "loss": 1.1852, + "theoretical_loss": 3.406781350957238, + "tokens_seen": 2174222336 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001723370110330993, + "loss": 1.228, + "theoretical_loss": 3.4067645076703825, + "tokens_seen": 2174353408 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017231695085255766, + "loss": 1.3639, + "theoretical_loss": 3.4067476656830973, + "tokens_seen": 2174484480 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017229689067201607, + "loss": 1.2062, + "theoretical_loss": 3.406730824995204, + "tokens_seen": 2174615552 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017227683049147443, + "loss": 1.2788, + "theoretical_loss": 3.4067139856065247, + "tokens_seen": 2174746624 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017225677031093282, + "loss": 1.3769, + "theoretical_loss": 3.40669714751688, + "tokens_seen": 2174877696 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017223671013039119, + "loss": 1.403, + "theoretical_loss": 3.4066803107260912, + "tokens_seen": 2175008768 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017221664994984958, + "loss": 1.3084, + "theoretical_loss": 3.4066634752339806, + "tokens_seen": 2175139840 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017219658976930794, + "loss": 1.504, + "theoretical_loss": 3.4066466410403695, + "tokens_seen": 2175270912 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001721765295887663, + "loss": 1.4397, + "theoretical_loss": 3.4066298081450794, + "tokens_seen": 2175401984 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001721564694082247, + "loss": 1.3121, + "theoretical_loss": 3.4066129765479323, + "tokens_seen": 2175533056 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017213640922768305, + "loss": 1.2232, + "theoretical_loss": 3.406596146248749, + "tokens_seen": 2175664128 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.4040811359882355, + "objective/train/avg_token_score": 0.023202180862426758, + "objective/train/docs_used": 1232648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4107543230056763, + "objective/train/original_loss": 3.813439130783081, + "objective/train/theoretical_loss": 3.4065793172473526, + "objective/train/tokens_used": 2175795200, + "objective/train/unlikelihood_loss": 0.1789805293083191, + "theoretical_loss": 3.4065793172473526, + "tokens_seen": 2175795200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017211634904714144, + "loss": 1.3061, + "theoretical_loss": 3.4065793172473526, + "tokens_seen": 2175795200 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001720962888665998, + "loss": 1.0943, + "theoretical_loss": 3.4065624895435636, + "tokens_seen": 2175926272 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001720762286860582, + "loss": 1.2065, + "theoretical_loss": 3.406545663137204, + "tokens_seen": 2176057344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017205616850551655, + "loss": 1.1685, + "theoretical_loss": 3.406528838028097, + "tokens_seen": 2176188416 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017203610832497494, + "loss": 1.2999, + "theoretical_loss": 3.4065120142160628, + "tokens_seen": 2176319488 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001720160481444333, + "loss": 1.4225, + "theoretical_loss": 3.406495191700924, + "tokens_seen": 2176450560 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017199598796389167, + "loss": 1.3271, + "theoretical_loss": 3.4064783704825023, + "tokens_seen": 2176581632 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017197592778335006, + "loss": 1.4312, + "theoretical_loss": 3.4064615505606195, + "tokens_seen": 2176712704 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017195586760280842, + "loss": 1.4172, + "theoretical_loss": 3.4064447319350983, + "tokens_seen": 2176843776 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001719358074222668, + "loss": 1.417, + "theoretical_loss": 3.4064279146057603, + "tokens_seen": 2176974848 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017191574724172517, + "loss": 1.3214, + "theoretical_loss": 3.406411098572428, + "tokens_seen": 2177105920 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017189568706118356, + "loss": 1.4142, + "theoretical_loss": 3.4063942838349224, + "tokens_seen": 2177236992 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017187562688064192, + "loss": 1.117, + "theoretical_loss": 3.4063774703930667, + "tokens_seen": 2177368064 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.41703322529792786, + "objective/train/avg_token_score": 0.012862755917012691, + "objective/train/docs_used": 1233682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3110846281051636, + "objective/train/original_loss": 3.496626377105713, + "objective/train/theoretical_loss": 3.4063690641579516, + "objective/train/tokens_used": 2177433600, + "objective/train/unlikelihood_loss": 0.2019149363040924, + "theoretical_loss": 3.4063690641579516, + "tokens_seen": 2177433600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001718555667001003, + "loss": 1.4441, + "theoretical_loss": 3.4063606582466823, + "tokens_seen": 2177499136 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017183550651955867, + "loss": 1.2825, + "theoretical_loss": 3.4063438473955925, + "tokens_seen": 2177630208 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017181544633901706, + "loss": 1.318, + "theoretical_loss": 3.4063270378396187, + "tokens_seen": 2177761280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017179538615847543, + "loss": 1.3673, + "theoretical_loss": 3.406310229578583, + "tokens_seen": 2177892352 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001717753259779338, + "loss": 1.4086, + "theoretical_loss": 3.4062934226123085, + "tokens_seen": 2178023424 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017175526579739218, + "loss": 1.3547, + "theoretical_loss": 3.406276616940617, + "tokens_seen": 2178154496 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017173520561685054, + "loss": 1.1886, + "theoretical_loss": 3.406259812563331, + "tokens_seen": 2178285568 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017171514543630893, + "loss": 1.2992, + "theoretical_loss": 3.406243009480273, + "tokens_seen": 2178416640 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001716950852557673, + "loss": 1.1921, + "theoretical_loss": 3.4062262076912653, + "tokens_seen": 2178547712 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017167502507522568, + "loss": 1.2509, + "theoretical_loss": 3.406209407196131, + "tokens_seen": 2178678784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017165496489468404, + "loss": 1.2378, + "theoretical_loss": 3.406192607994692, + "tokens_seen": 2178809856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017163490471414243, + "loss": 1.2281, + "theoretical_loss": 3.4061758100867707, + "tokens_seen": 2178940928 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.5032380223274231, + "objective/train/avg_token_score": 0.014877188950777054, + "objective/train/docs_used": 1234346, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6239598989486694, + "objective/train/original_loss": 3.3975419998168945, + "objective/train/theoretical_loss": 3.40615901347219, + "objective/train/tokens_used": 2179072000, + "objective/train/unlikelihood_loss": 0.14759010076522827, + "theoretical_loss": 3.40615901347219, + "tokens_seen": 2179072000 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001716148445336008, + "loss": 1.3003, + "theoretical_loss": 3.40615901347219, + "tokens_seen": 2179072000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017159478435305916, + "loss": 1.3106, + "theoretical_loss": 3.406142218150773, + "tokens_seen": 2179203072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017157472417251755, + "loss": 1.3408, + "theoretical_loss": 3.4061254241223415, + "tokens_seen": 2179334144 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715546639919759, + "loss": 1.4039, + "theoretical_loss": 3.406108631386719, + "tokens_seen": 2179465216 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017153460381143433, + "loss": 1.3399, + "theoretical_loss": 3.406091839943728, + "tokens_seen": 2179596288 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715145436308927, + "loss": 1.3792, + "theoretical_loss": 3.4060750497931904, + "tokens_seen": 2179727360 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017149448345035108, + "loss": 1.2687, + "theoretical_loss": 3.4060582609349304, + "tokens_seen": 2179858432 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017147442326980944, + "loss": 1.4468, + "theoretical_loss": 3.4060414733687696, + "tokens_seen": 2179989504 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017145436308926783, + "loss": 1.3573, + "theoretical_loss": 3.406024687094532, + "tokens_seen": 2180120576 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001714343029087262, + "loss": 1.2873, + "theoretical_loss": 3.40600790211204, + "tokens_seen": 2180251648 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017141424272818455, + "loss": 1.3265, + "theoretical_loss": 3.405991118421116, + "tokens_seen": 2180382720 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017139418254764294, + "loss": 1.3537, + "theoretical_loss": 3.405974336021584, + "tokens_seen": 2180513792 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001713741223671013, + "loss": 1.364, + "theoretical_loss": 3.4059575549132663, + "tokens_seen": 2180644864 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.423631489276886, + "objective/train/avg_token_score": 0.010067375376820564, + "objective/train/docs_used": 1235717, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3751492500305176, + "objective/train/original_loss": 3.78859543800354, + "objective/train/theoretical_loss": 3.4059491648432574, + "objective/train/tokens_used": 2180710400, + "objective/train/unlikelihood_loss": 0.1462312787771225, + "theoretical_loss": 3.4059491648432574, + "tokens_seen": 2180710400 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001713540621865597, + "loss": 1.307, + "theoretical_loss": 3.4059407750959863, + "tokens_seen": 2180775936 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017133400200601806, + "loss": 1.4388, + "theoretical_loss": 3.405923996569567, + "tokens_seen": 2180907008 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017131394182547645, + "loss": 1.1594, + "theoretical_loss": 3.405907219333831, + "tokens_seen": 2181038080 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001712938816449348, + "loss": 1.2846, + "theoretical_loss": 3.4058904433886026, + "tokens_seen": 2181169152 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001712738214643932, + "loss": 1.3023, + "theoretical_loss": 3.4058736687337046, + "tokens_seen": 2181300224 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017125376128385156, + "loss": 1.2776, + "theoretical_loss": 3.4058568953689594, + "tokens_seen": 2181431296 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017123370110330995, + "loss": 1.3087, + "theoretical_loss": 3.405840123294191, + "tokens_seen": 2181562368 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001712136409227683, + "loss": 1.3328, + "theoretical_loss": 3.4058233525092225, + "tokens_seen": 2181693440 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017119358074222667, + "loss": 1.387, + "theoretical_loss": 3.4058065830138773, + "tokens_seen": 2181824512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017117352056168506, + "loss": 1.2958, + "theoretical_loss": 3.405789814807979, + "tokens_seen": 2181955584 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017115346038114342, + "loss": 1.3389, + "theoretical_loss": 3.4057730478913504, + "tokens_seen": 2182086656 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017113340020060181, + "loss": 1.4835, + "theoretical_loss": 3.405756282263815, + "tokens_seen": 2182217728 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.41287878155708313, + "objective/train/avg_token_score": 0.00480588199570775, + "objective/train/docs_used": 1236411, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2892358303070068, + "objective/train/original_loss": 3.617506504058838, + "objective/train/theoretical_loss": 3.405739517925197, + "objective/train/tokens_used": 2182348800, + "objective/train/unlikelihood_loss": 0.12441212683916092, + "theoretical_loss": 3.405739517925197, + "tokens_seen": 2182348800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017111334002006018, + "loss": 1.2163, + "theoretical_loss": 3.405739517925197, + "tokens_seen": 2182348800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017109327983951857, + "loss": 1.1824, + "theoretical_loss": 3.4057227548753195, + "tokens_seen": 2182479872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017107321965897693, + "loss": 1.3938, + "theoretical_loss": 3.405705993114006, + "tokens_seen": 2182610944 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017105315947843532, + "loss": 1.4208, + "theoretical_loss": 3.40568923264108, + "tokens_seen": 2182742016 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017103309929789368, + "loss": 1.3781, + "theoretical_loss": 3.405672473456365, + "tokens_seen": 2182873088 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017101303911735204, + "loss": 1.3948, + "theoretical_loss": 3.405655715559685, + "tokens_seen": 2183004160 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017099297893681043, + "loss": 1.3869, + "theoretical_loss": 3.4056389589508633, + "tokens_seen": 2183135232 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001709729187562688, + "loss": 1.5484, + "theoretical_loss": 3.4056222036297243, + "tokens_seen": 2183266304 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017095285857572718, + "loss": 1.2972, + "theoretical_loss": 3.405605449596091, + "tokens_seen": 2183397376 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017093279839518554, + "loss": 1.3619, + "theoretical_loss": 3.4055886968497875, + "tokens_seen": 2183528448 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017091273821464393, + "loss": 1.3508, + "theoretical_loss": 3.4055719453906375, + "tokens_seen": 2183659520 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708926780341023, + "loss": 1.3115, + "theoretical_loss": 3.405555195218465, + "tokens_seen": 2183790592 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017087261785356069, + "loss": 1.4591, + "theoretical_loss": 3.405538446333094, + "tokens_seen": 2183921664 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.40860214829444885, + "objective/train/avg_token_score": 0.017220061272382736, + "objective/train/docs_used": 1237677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4479116201400757, + "objective/train/original_loss": 4.242138385772705, + "objective/train/theoretical_loss": 3.405530072372904, + "objective/train/tokens_used": 2183987200, + "objective/train/unlikelihood_loss": 0.10991041362285614, + "theoretical_loss": 3.405530072372904, + "tokens_seen": 2183987200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017085255767301905, + "loss": 1.2962, + "theoretical_loss": 3.405521698734348, + "tokens_seen": 2184052736 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708324974924774, + "loss": 1.3785, + "theoretical_loss": 3.405504952422051, + "tokens_seen": 2184183808 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708124373119358, + "loss": 1.2792, + "theoretical_loss": 3.4054882073960275, + "tokens_seen": 2184314880 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707923771313942, + "loss": 1.3924, + "theoretical_loss": 3.4054714636561014, + "tokens_seen": 2184445952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017077231695085258, + "loss": 1.2057, + "theoretical_loss": 3.405454721202096, + "tokens_seen": 2184577024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017075225677031094, + "loss": 1.3733, + "theoretical_loss": 3.4054379800338364, + "tokens_seen": 2184708096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017073219658976933, + "loss": 1.3108, + "theoretical_loss": 3.4054212401511466, + "tokens_seen": 2184839168 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707121364092277, + "loss": 1.4057, + "theoretical_loss": 3.40540450155385, + "tokens_seen": 2184970240 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017069207622868608, + "loss": 1.2548, + "theoretical_loss": 3.4053877642417714, + "tokens_seen": 2185101312 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017067201604814444, + "loss": 1.2974, + "theoretical_loss": 3.405371028214735, + "tokens_seen": 2185232384 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017065195586760283, + "loss": 1.3038, + "theoretical_loss": 3.405354293472565, + "tokens_seen": 2185363456 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001706318956870612, + "loss": 1.2533, + "theoretical_loss": 3.4053375600150853, + "tokens_seen": 2185494528 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.5099584460258484, + "objective/train/avg_token_score": 0.006493102293461561, + "objective/train/docs_used": 1238334, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6248334646224976, + "objective/train/original_loss": 3.7724416255950928, + "objective/train/theoretical_loss": 3.4053208278421208, + "objective/train/tokens_used": 2185625600, + "objective/train/unlikelihood_loss": 0.12906888127326965, + "theoretical_loss": 3.4053208278421208, + "tokens_seen": 2185625600 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017061183550651956, + "loss": 1.5047, + "theoretical_loss": 3.4053208278421208, + "tokens_seen": 2185625600 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017059177532597795, + "loss": 1.4046, + "theoretical_loss": 3.405304096953496, + "tokens_seen": 2185756672 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001705717151454363, + "loss": 1.3546, + "theoretical_loss": 3.4052873673490347, + "tokens_seen": 2185887744 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001705516549648947, + "loss": 1.3812, + "theoretical_loss": 3.405270639028562, + "tokens_seen": 2186018816 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017053159478435306, + "loss": 1.4485, + "theoretical_loss": 3.4052539119919016, + "tokens_seen": 2186149888 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017051153460381145, + "loss": 1.4035, + "theoretical_loss": 3.4052371862388786, + "tokens_seen": 2186280960 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001704914744232698, + "loss": 1.4851, + "theoretical_loss": 3.4052204617693174, + "tokens_seen": 2186412032 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001704714142427282, + "loss": 1.3955, + "theoretical_loss": 3.4052037385830425, + "tokens_seen": 2186543104 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017045135406218656, + "loss": 1.5042, + "theoretical_loss": 3.4051870166798786, + "tokens_seen": 2186674176 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017043129388164493, + "loss": 1.3503, + "theoretical_loss": 3.4051702960596506, + "tokens_seen": 2186805248 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017041123370110332, + "loss": 1.3497, + "theoretical_loss": 3.4051535767221823, + "tokens_seen": 2186936320 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017039117352056168, + "loss": 1.3484, + "theoretical_loss": 3.4051368586672996, + "tokens_seen": 2187067392 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017037111334002007, + "loss": 1.2025, + "theoretical_loss": 3.4051201418948263, + "tokens_seen": 2187198464 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.46163246035575867, + "objective/train/avg_token_score": 0.007555610965937376, + "objective/train/docs_used": 1239462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.292608618736267, + "objective/train/original_loss": 3.158935308456421, + "objective/train/theoretical_loss": 3.4051117839894385, + "objective/train/tokens_used": 2187264000, + "objective/train/unlikelihood_loss": 0.20322516560554504, + "theoretical_loss": 3.4051117839894385, + "tokens_seen": 2187264000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017035105315947843, + "loss": 1.2315, + "theoretical_loss": 3.4051034264045876, + "tokens_seen": 2187329536 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017033099297893682, + "loss": 1.2491, + "theoretical_loss": 3.4050867121964083, + "tokens_seen": 2187460608 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017031093279839518, + "loss": 1.3424, + "theoretical_loss": 3.4050699992701134, + "tokens_seen": 2187591680 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017029087261785357, + "loss": 1.2926, + "theoretical_loss": 3.4050532876255275, + "tokens_seen": 2187722752 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017027081243731193, + "loss": 1.3627, + "theoretical_loss": 3.405036577262476, + "tokens_seen": 2187853824 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017025075225677032, + "loss": 1.3678, + "theoretical_loss": 3.405019868180783, + "tokens_seen": 2187984896 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017023069207622868, + "loss": 1.1986, + "theoretical_loss": 3.405003160380274, + "tokens_seen": 2188115968 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017021063189568705, + "loss": 1.2302, + "theoretical_loss": 3.404986453860775, + "tokens_seen": 2188247040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017019057171514544, + "loss": 1.2939, + "theoretical_loss": 3.404969748622109, + "tokens_seen": 2188378112 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001701705115346038, + "loss": 1.4764, + "theoretical_loss": 3.4049530446641025, + "tokens_seen": 2188509184 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001701504513540622, + "loss": 1.3612, + "theoretical_loss": 3.4049363419865806, + "tokens_seen": 2188640256 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017013039117352055, + "loss": 1.2326, + "theoretical_loss": 3.404919640589368, + "tokens_seen": 2188771328 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.5454545617103577, + "objective/train/avg_token_score": 0.007881923578679562, + "objective/train/docs_used": 1240051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8245166540145874, + "objective/train/original_loss": 3.64921236038208, + "objective/train/theoretical_loss": 3.4049029404722897, + "objective/train/tokens_used": 2188902400, + "objective/train/unlikelihood_loss": 0.15708287060260773, + "theoretical_loss": 3.4049029404722897, + "tokens_seen": 2188902400 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017011033099297894, + "loss": 1.4413, + "theoretical_loss": 3.4049029404722897, + "tokens_seen": 2188902400 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001700902708124373, + "loss": 1.4071, + "theoretical_loss": 3.4048862416351717, + "tokens_seen": 2189033472 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001700702106318957, + "loss": 1.2286, + "theoretical_loss": 3.404869544077839, + "tokens_seen": 2189164544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017005015045135405, + "loss": 1.2779, + "theoretical_loss": 3.4048528478001168, + "tokens_seen": 2189295616 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017003009027081244, + "loss": 1.3644, + "theoretical_loss": 3.4048361528018303, + "tokens_seen": 2189426688 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017001003009027083, + "loss": 1.3968, + "theoretical_loss": 3.404819459082805, + "tokens_seen": 2189557760 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001699899699097292, + "loss": 1.1167, + "theoretical_loss": 3.404802766642866, + "tokens_seen": 2189688832 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016996990972918758, + "loss": 1.2923, + "theoretical_loss": 3.4047860754818395, + "tokens_seen": 2189819904 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016994984954864594, + "loss": 1.2128, + "theoretical_loss": 3.4047693855995504, + "tokens_seen": 2189950976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016992978936810433, + "loss": 1.4019, + "theoretical_loss": 3.404752696995824, + "tokens_seen": 2190082048 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001699097291875627, + "loss": 1.2576, + "theoretical_loss": 3.4047360096704864, + "tokens_seen": 2190213120 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016988966900702109, + "loss": 1.4169, + "theoretical_loss": 3.4047193236233633, + "tokens_seen": 2190344192 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016986960882647945, + "loss": 1.3603, + "theoretical_loss": 3.4047026388542796, + "tokens_seen": 2190475264 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.46768084168434143, + "objective/train/avg_token_score": 0.007777104154229164, + "objective/train/docs_used": 1241523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4028042554855347, + "objective/train/original_loss": 3.5941321849823, + "objective/train/theoretical_loss": 3.404694296948948, + "objective/train/tokens_used": 2190540800, + "objective/train/unlikelihood_loss": 0.18663804233074188, + "theoretical_loss": 3.404694296948948, + "tokens_seen": 2190540800 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001698495486459378, + "loss": 1.3949, + "theoretical_loss": 3.4046859553630613, + "tokens_seen": 2190606336 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001698294884653962, + "loss": 1.326, + "theoretical_loss": 3.4046692731495343, + "tokens_seen": 2190737408 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016980942828485456, + "loss": 1.4612, + "theoretical_loss": 3.404652592213524, + "tokens_seen": 2190868480 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016978936810431295, + "loss": 1.3189, + "theoretical_loss": 3.404635912554856, + "tokens_seen": 2190999552 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001697693079237713, + "loss": 1.3816, + "theoretical_loss": 3.4046192341733565, + "tokens_seen": 2191130624 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001697492477432297, + "loss": 1.287, + "theoretical_loss": 3.404602557068851, + "tokens_seen": 2191261696 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016972918756268807, + "loss": 1.3521, + "theoretical_loss": 3.4045858812411662, + "tokens_seen": 2191392768 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016970912738214645, + "loss": 1.3421, + "theoretical_loss": 3.4045692066901263, + "tokens_seen": 2191523840 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016968906720160482, + "loss": 1.3704, + "theoretical_loss": 3.4045525334155586, + "tokens_seen": 2191654912 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001696690070210632, + "loss": 1.3112, + "theoretical_loss": 3.4045358614172887, + "tokens_seen": 2191785984 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016964894684052157, + "loss": 1.2974, + "theoretical_loss": 3.404519190695143, + "tokens_seen": 2191917056 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016962888665997993, + "loss": 1.2319, + "theoretical_loss": 3.404502521248946, + "tokens_seen": 2192048128 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.4858260154724121, + "objective/train/avg_token_score": 0.024061625823378563, + "objective/train/docs_used": 1242176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5639647245407104, + "objective/train/original_loss": 3.799980640411377, + "objective/train/theoretical_loss": 3.4044858530785254, + "objective/train/tokens_used": 2192179200, + "objective/train/unlikelihood_loss": 0.13133184611797333, + "theoretical_loss": 3.4044858530785254, + "tokens_seen": 2192179200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016960882647943832, + "loss": 1.3489, + "theoretical_loss": 3.4044858530785254, + "tokens_seen": 2192179200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016958876629889668, + "loss": 1.3609, + "theoretical_loss": 3.404469186183707, + "tokens_seen": 2192310272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016956870611835507, + "loss": 1.4739, + "theoretical_loss": 3.404452520564316, + "tokens_seen": 2192441344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016954864593781343, + "loss": 1.3201, + "theoretical_loss": 3.4044358562201795, + "tokens_seen": 2192572416 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016952858575727182, + "loss": 1.2281, + "theoretical_loss": 3.4044191931511234, + "tokens_seen": 2192703488 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016950852557673019, + "loss": 1.2745, + "theoretical_loss": 3.4044025313569737, + "tokens_seen": 2192834560 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016948846539618857, + "loss": 1.4227, + "theoretical_loss": 3.404385870837557, + "tokens_seen": 2192965632 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016946840521564694, + "loss": 1.2751, + "theoretical_loss": 3.4043692115926993, + "tokens_seen": 2193096704 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694483450351053, + "loss": 1.3555, + "theoretical_loss": 3.4043525536222274, + "tokens_seen": 2193227776 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694282848545637, + "loss": 1.3186, + "theoretical_loss": 3.404335896925967, + "tokens_seen": 2193358848 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016940822467402205, + "loss": 1.4034, + "theoretical_loss": 3.404319241503745, + "tokens_seen": 2193489920 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016938816449348044, + "loss": 1.2863, + "theoretical_loss": 3.404302587355388, + "tokens_seen": 2193620992 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001693681043129388, + "loss": 1.2743, + "theoretical_loss": 3.4042859344807215, + "tokens_seen": 2193752064 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.4784946143627167, + "objective/train/avg_token_score": 0.005486602894961834, + "objective/train/docs_used": 1243185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3290934562683105, + "objective/train/original_loss": 3.3570291996002197, + "objective/train/theoretical_loss": 3.4042776085209683, + "objective/train/tokens_used": 2193817600, + "objective/train/unlikelihood_loss": 0.17965403199195862, + "theoretical_loss": 3.4042776085209683, + "tokens_seen": 2193817600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001693480441323972, + "loss": 1.1621, + "theoretical_loss": 3.404269282879573, + "tokens_seen": 2193883136 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016932798395185555, + "loss": 1.3265, + "theoretical_loss": 3.404252632551769, + "tokens_seen": 2194014208 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016930792377131394, + "loss": 1.3213, + "theoretical_loss": 3.4042359834971347, + "tokens_seen": 2194145280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016928786359077233, + "loss": 1.2762, + "theoretical_loss": 3.4042193357154984, + "tokens_seen": 2194276352 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016926780341023072, + "loss": 1.2638, + "theoretical_loss": 3.404202689206686, + "tokens_seen": 2194407424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016924774322968908, + "loss": 1.2388, + "theoretical_loss": 3.4041860439705243, + "tokens_seen": 2194538496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016922768304914745, + "loss": 1.4141, + "theoretical_loss": 3.4041694000068397, + "tokens_seen": 2194669568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016920762286860584, + "loss": 1.3864, + "theoretical_loss": 3.4041527573154595, + "tokens_seen": 2194800640 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691875626880642, + "loss": 1.3493, + "theoretical_loss": 3.40413611589621, + "tokens_seen": 2194931712 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691675025075226, + "loss": 1.3936, + "theoretical_loss": 3.4041194757489177, + "tokens_seen": 2195062784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016914744232698095, + "loss": 1.2666, + "theoretical_loss": 3.4041028368734105, + "tokens_seen": 2195193856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016912738214643934, + "loss": 1.3767, + "theoretical_loss": 3.404086199269514, + "tokens_seen": 2195324928 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.48423752188682556, + "objective/train/avg_token_score": 0.004468325525522232, + "objective/train/docs_used": 1243800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2411506175994873, + "objective/train/original_loss": 3.145841121673584, + "objective/train/theoretical_loss": 3.4040695629370563, + "objective/train/tokens_used": 2195456000, + "objective/train/unlikelihood_loss": 0.2387589067220688, + "theoretical_loss": 3.4040695629370563, + "tokens_seen": 2195456000 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691073219658977, + "loss": 1.2899, + "theoretical_loss": 3.4040695629370563, + "tokens_seen": 2195456000 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001690872617853561, + "loss": 1.5129, + "theoretical_loss": 3.4040529278758638, + "tokens_seen": 2195587072 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016906720160481445, + "loss": 1.4151, + "theoretical_loss": 3.4040362940857634, + "tokens_seen": 2195718144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016904714142427281, + "loss": 1.3946, + "theoretical_loss": 3.404019661566582, + "tokens_seen": 2195849216 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001690270812437312, + "loss": 1.3638, + "theoretical_loss": 3.4040030303181465, + "tokens_seen": 2195980288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016900702106318957, + "loss": 1.2695, + "theoretical_loss": 3.403986400340285, + "tokens_seen": 2196111360 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016898696088264796, + "loss": 1.2959, + "theoretical_loss": 3.4039697716328234, + "tokens_seen": 2196242432 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016896690070210632, + "loss": 1.3886, + "theoretical_loss": 3.4039531441955897, + "tokens_seen": 2196373504 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001689468405215647, + "loss": 1.2454, + "theoretical_loss": 3.4039365180284102, + "tokens_seen": 2196504576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016892678034102307, + "loss": 1.4816, + "theoretical_loss": 3.403919893131113, + "tokens_seen": 2196635648 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016890672016048146, + "loss": 1.4635, + "theoretical_loss": 3.403903269503525, + "tokens_seen": 2196766720 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016888665997993982, + "loss": 1.3089, + "theoretical_loss": 3.4038866471454736, + "tokens_seen": 2196897792 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016886659979939818, + "loss": 1.3474, + "theoretical_loss": 3.4038700260567856, + "tokens_seen": 2197028864 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.38031524419784546, + "objective/train/avg_token_score": 0.023482317104935646, + "objective/train/docs_used": 1244898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1626790761947632, + "objective/train/original_loss": 3.843801975250244, + "objective/train/theoretical_loss": 3.4038617159883993, + "objective/train/tokens_used": 2197094400, + "objective/train/unlikelihood_loss": 0.20273272693157196, + "theoretical_loss": 3.4038617159883993, + "tokens_seen": 2197094400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016884653961885657, + "loss": 1.2869, + "theoretical_loss": 3.403853406237289, + "tokens_seen": 2197159936 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016882647943831494, + "loss": 1.3113, + "theoretical_loss": 3.4038367876868105, + "tokens_seen": 2197291008 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016880641925777332, + "loss": 1.3896, + "theoretical_loss": 3.403820170405178, + "tokens_seen": 2197422080 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001687863590772317, + "loss": 1.4448, + "theoretical_loss": 3.403803554392219, + "tokens_seen": 2197553152 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016876629889669008, + "loss": 1.2679, + "theoretical_loss": 3.4037869396477607, + "tokens_seen": 2197684224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016874623871614844, + "loss": 1.3359, + "theoretical_loss": 3.4037703261716308, + "tokens_seen": 2197815296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016872617853560683, + "loss": 1.2943, + "theoretical_loss": 3.4037537139636567, + "tokens_seen": 2197946368 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001687061183550652, + "loss": 1.4669, + "theoretical_loss": 3.403737103023666, + "tokens_seen": 2198077440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016868605817452355, + "loss": 1.4592, + "theoretical_loss": 3.4037204933514866, + "tokens_seen": 2198208512 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016866599799398194, + "loss": 1.3655, + "theoretical_loss": 3.403703884946946, + "tokens_seen": 2198339584 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001686459378134403, + "loss": 1.3573, + "theoretical_loss": 3.4036872778098717, + "tokens_seen": 2198470656 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001686258776328987, + "loss": 1.3386, + "theoretical_loss": 3.403670671940091, + "tokens_seen": 2198601728 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.4037145674228668, + "objective/train/avg_token_score": 0.020199256017804146, + "objective/train/docs_used": 1245692, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3007694482803345, + "objective/train/original_loss": 4.149553298950195, + "objective/train/theoretical_loss": 3.403654067337433, + "objective/train/tokens_used": 2198732800, + "objective/train/unlikelihood_loss": 0.12533389031887054, + "theoretical_loss": 3.403654067337433, + "tokens_seen": 2198732800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016860581745235706, + "loss": 1.4427, + "theoretical_loss": 3.403654067337433, + "tokens_seen": 2198732800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016858575727181544, + "loss": 1.31, + "theoretical_loss": 3.403637464001724, + "tokens_seen": 2198863872 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001685656970912738, + "loss": 1.4076, + "theoretical_loss": 3.403620861932793, + "tokens_seen": 2198994944 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001685456369107322, + "loss": 1.3996, + "theoretical_loss": 3.403604261130467, + "tokens_seen": 2199126016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016852557673019059, + "loss": 1.1897, + "theoretical_loss": 3.4035876615945746, + "tokens_seen": 2199257088 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016850551654964897, + "loss": 1.4633, + "theoretical_loss": 3.403571063324943, + "tokens_seen": 2199388160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016848545636910734, + "loss": 1.3604, + "theoretical_loss": 3.4035544663214004, + "tokens_seen": 2199519232 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001684653961885657, + "loss": 1.3191, + "theoretical_loss": 3.403537870583775, + "tokens_seen": 2199650304 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001684453360080241, + "loss": 1.3009, + "theoretical_loss": 3.4035212761118947, + "tokens_seen": 2199781376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016842527582748245, + "loss": 1.3558, + "theoretical_loss": 3.403504682905588, + "tokens_seen": 2199912448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016840521564694084, + "loss": 1.3386, + "theoretical_loss": 3.403488090964682, + "tokens_seen": 2200043520 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683851554663992, + "loss": 1.3735, + "theoretical_loss": 3.403471500289005, + "tokens_seen": 2200174592 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683650952858576, + "loss": 1.3156, + "theoretical_loss": 3.403454910878386, + "tokens_seen": 2200305664 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.46841397881507874, + "objective/train/avg_token_score": 0.021130314096808434, + "objective/train/docs_used": 1246987, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4613827466964722, + "objective/train/original_loss": 3.726722002029419, + "objective/train/theoretical_loss": 3.4034466166474195, + "objective/train/tokens_used": 2200371200, + "objective/train/unlikelihood_loss": 0.14814133942127228, + "theoretical_loss": 3.4034466166474195, + "tokens_seen": 2200371200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016834503510531595, + "loss": 1.3738, + "theoretical_loss": 3.4034383227326526, + "tokens_seen": 2200436736 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016832497492477434, + "loss": 1.2657, + "theoretical_loss": 3.403421735851633, + "tokens_seen": 2200567808 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683049147442327, + "loss": 1.2456, + "theoretical_loss": 3.403405150235156, + "tokens_seen": 2200698880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016828485456369107, + "loss": 1.2108, + "theoretical_loss": 3.4033885658830494, + "tokens_seen": 2200829952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016826479438314946, + "loss": 1.1511, + "theoretical_loss": 3.403371982795141, + "tokens_seen": 2200961024 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016824473420260782, + "loss": 1.3011, + "theoretical_loss": 3.40335540097126, + "tokens_seen": 2201092096 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001682246740220662, + "loss": 1.2938, + "theoretical_loss": 3.4033388204112347, + "tokens_seen": 2201223168 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016820461384152457, + "loss": 1.4874, + "theoretical_loss": 3.403322241114893, + "tokens_seen": 2201354240 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016818455366098296, + "loss": 1.3041, + "theoretical_loss": 3.403305663082064, + "tokens_seen": 2201485312 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016816449348044132, + "loss": 1.2735, + "theoretical_loss": 3.4032890863125758, + "tokens_seen": 2201616384 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001681444332998997, + "loss": 1.3251, + "theoretical_loss": 3.403272510806257, + "tokens_seen": 2201747456 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016812437311935807, + "loss": 1.3591, + "theoretical_loss": 3.4032559365629362, + "tokens_seen": 2201878528 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.405303031206131, + "objective/train/avg_token_score": 0.01647990196943283, + "objective/train/docs_used": 1247624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2952309846878052, + "objective/train/original_loss": 3.8539247512817383, + "objective/train/theoretical_loss": 3.4032393635824416, + "objective/train/tokens_used": 2202009600, + "objective/train/unlikelihood_loss": 0.18945452570915222, + "theoretical_loss": 3.4032393635824416, + "tokens_seen": 2202009600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016810431293881646, + "loss": 1.3063, + "theoretical_loss": 3.4032393635824416, + "tokens_seen": 2202009600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016808425275827483, + "loss": 1.33, + "theoretical_loss": 3.4032227918646027, + "tokens_seen": 2202140672 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001680641925777332, + "loss": 1.3252, + "theoretical_loss": 3.4032062214092473, + "tokens_seen": 2202271744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016804413239719158, + "loss": 1.2828, + "theoretical_loss": 3.403189652216205, + "tokens_seen": 2202402816 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016802407221664994, + "loss": 1.3331, + "theoretical_loss": 3.4031730842853034, + "tokens_seen": 2202533888 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016800401203610833, + "loss": 1.3655, + "theoretical_loss": 3.4031565176163716, + "tokens_seen": 2202664960 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001679839518555667, + "loss": 1.3372, + "theoretical_loss": 3.403139952209239, + "tokens_seen": 2202796032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016796389167502508, + "loss": 1.4471, + "theoretical_loss": 3.403123388063734, + "tokens_seen": 2202927104 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016794383149448344, + "loss": 1.4065, + "theoretical_loss": 3.403106825179685, + "tokens_seen": 2203058176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016792377131394183, + "loss": 1.3666, + "theoretical_loss": 3.403090263556922, + "tokens_seen": 2203189248 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001679037111334002, + "loss": 1.3072, + "theoretical_loss": 3.403073703195273, + "tokens_seen": 2203320320 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016788365095285856, + "loss": 1.2702, + "theoretical_loss": 3.4030571440945674, + "tokens_seen": 2203451392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016786359077231695, + "loss": 1.2371, + "theoretical_loss": 3.4030405862546336, + "tokens_seen": 2203582464 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.3909457325935364, + "objective/train/avg_token_score": 0.012943543493747711, + "objective/train/docs_used": 1248832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1878594160079956, + "objective/train/original_loss": 4.3220367431640625, + "objective/train/theoretical_loss": 3.403032307807403, + "objective/train/tokens_used": 2203648000, + "objective/train/unlikelihood_loss": 0.09069668501615524, + "theoretical_loss": 3.403032307807403, + "tokens_seen": 2203648000 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001678435305917753, + "loss": 1.3381, + "theoretical_loss": 3.4030240296753016, + "tokens_seen": 2203713536 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001678234704112337, + "loss": 1.1745, + "theoretical_loss": 3.4030074743563996, + "tokens_seen": 2203844608 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016780341023069206, + "loss": 1.227, + "theoretical_loss": 3.402990920297757, + "tokens_seen": 2203975680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016778335005015048, + "loss": 1.2623, + "theoretical_loss": 3.402974367499203, + "tokens_seen": 2204106752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016776328986960884, + "loss": 1.2862, + "theoretical_loss": 3.4029578159605665, + "tokens_seen": 2204237824 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016774322968906723, + "loss": 1.4104, + "theoretical_loss": 3.4029412656816773, + "tokens_seen": 2204368896 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001677231695085256, + "loss": 1.3981, + "theoretical_loss": 3.4029247166623637, + "tokens_seen": 2204499968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016770310932798395, + "loss": 1.5347, + "theoretical_loss": 3.402908168902456, + "tokens_seen": 2204631040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016768304914744234, + "loss": 1.4097, + "theoretical_loss": 3.4028916224017824, + "tokens_seen": 2204762112 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676629889669007, + "loss": 1.1747, + "theoretical_loss": 3.4028750771601732, + "tokens_seen": 2204893184 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676429287863591, + "loss": 1.2521, + "theoretical_loss": 3.402858533177457, + "tokens_seen": 2205024256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016762286860581746, + "loss": 1.2294, + "theoretical_loss": 3.4028419904534637, + "tokens_seen": 2205155328 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.4778836667537689, + "objective/train/avg_token_score": 0.0047411746345460415, + "objective/train/docs_used": 1249474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.476658582687378, + "objective/train/original_loss": 3.251304864883423, + "objective/train/theoretical_loss": 3.4028254489880227, + "objective/train/tokens_used": 2205286400, + "objective/train/unlikelihood_loss": 0.27285274863243103, + "theoretical_loss": 3.4028254489880227, + "tokens_seen": 2205286400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016760280842527584, + "loss": 1.3334, + "theoretical_loss": 3.4028254489880227, + "tokens_seen": 2205286400 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001675827482447342, + "loss": 1.4274, + "theoretical_loss": 3.402808908780963, + "tokens_seen": 2205417472 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001675626880641926, + "loss": 1.3466, + "theoretical_loss": 3.4027923698321145, + "tokens_seen": 2205548544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016754262788365096, + "loss": 1.335, + "theoretical_loss": 3.402775832141307, + "tokens_seen": 2205679616 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016752256770310935, + "loss": 1.2027, + "theoretical_loss": 3.402759295708369, + "tokens_seen": 2205810688 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001675025075225677, + "loss": 1.3163, + "theoretical_loss": 3.402742760533131, + "tokens_seen": 2205941760 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016748244734202607, + "loss": 1.4498, + "theoretical_loss": 3.402726226615423, + "tokens_seen": 2206072832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016746238716148446, + "loss": 1.4069, + "theoretical_loss": 3.402709693955073, + "tokens_seen": 2206203904 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016744232698094282, + "loss": 1.3845, + "theoretical_loss": 3.402693162551912, + "tokens_seen": 2206334976 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001674222668004012, + "loss": 1.2343, + "theoretical_loss": 3.40267663240577, + "tokens_seen": 2206466048 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016740220661985958, + "loss": 1.3476, + "theoretical_loss": 3.402660103516476, + "tokens_seen": 2206597120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016738214643931797, + "loss": 1.3733, + "theoretical_loss": 3.4026435758838596, + "tokens_seen": 2206728192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016736208625877633, + "loss": 1.5242, + "theoretical_loss": 3.4026270495077515, + "tokens_seen": 2206859264 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.44831377267837524, + "objective/train/avg_token_score": 0.007722725626081228, + "objective/train/docs_used": 1250567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2805020809173584, + "objective/train/original_loss": 3.282392978668213, + "objective/train/theoretical_loss": 3.4026187867908346, + "objective/train/tokens_used": 2206924800, + "objective/train/unlikelihood_loss": 0.1872881054878235, + "theoretical_loss": 3.4026187867908346, + "tokens_seen": 2206924800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016734202607823472, + "loss": 1.3599, + "theoretical_loss": 3.4026105243879807, + "tokens_seen": 2206990336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016732196589769308, + "loss": 1.2463, + "theoretical_loss": 3.4025940005243775, + "tokens_seen": 2207121408 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016730190571715144, + "loss": 1.3081, + "theoretical_loss": 3.402577477916772, + "tokens_seen": 2207252480 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016728184553660983, + "loss": 1.4109, + "theoretical_loss": 3.4025609565649937, + "tokens_seen": 2207383552 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001672617853560682, + "loss": 1.3592, + "theoretical_loss": 3.402544436468873, + "tokens_seen": 2207514624 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016724172517552658, + "loss": 1.3603, + "theoretical_loss": 3.4025279176282397, + "tokens_seen": 2207645696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016722166499498494, + "loss": 1.4517, + "theoretical_loss": 3.4025114000429237, + "tokens_seen": 2207776768 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016720160481444333, + "loss": 1.3132, + "theoretical_loss": 3.4024948837127558, + "tokens_seen": 2207907840 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001671815446339017, + "loss": 1.4681, + "theoretical_loss": 3.4024783686375653, + "tokens_seen": 2208038912 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016716148445336009, + "loss": 1.3917, + "theoretical_loss": 3.4024618548171826, + "tokens_seen": 2208169984 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016714142427281845, + "loss": 1.3145, + "theoretical_loss": 3.402445342251438, + "tokens_seen": 2208301056 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001671213640922768, + "loss": 1.3155, + "theoretical_loss": 3.4024288309401616, + "tokens_seen": 2208432128 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.43560606241226196, + "objective/train/avg_token_score": 0.010403500869870186, + "objective/train/docs_used": 1251065, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4363981485366821, + "objective/train/original_loss": 3.8681387901306152, + "objective/train/theoretical_loss": 3.4024123208831836, + "objective/train/tokens_used": 2208563200, + "objective/train/unlikelihood_loss": 0.12989090383052826, + "theoretical_loss": 3.4024123208831836, + "tokens_seen": 2208563200 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001671013039117352, + "loss": 1.189, + "theoretical_loss": 3.4024123208831836, + "tokens_seen": 2208563200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016708124373119356, + "loss": 1.4683, + "theoretical_loss": 3.402395812080335, + "tokens_seen": 2208694272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016706118355065195, + "loss": 1.2314, + "theoretical_loss": 3.4023793045314448, + "tokens_seen": 2208825344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016704112337011034, + "loss": 1.258, + "theoretical_loss": 3.4023627982363442, + "tokens_seen": 2208956416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016702106318956873, + "loss": 1.2553, + "theoretical_loss": 3.4023462931948636, + "tokens_seen": 2209087488 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001670010030090271, + "loss": 1.2429, + "theoretical_loss": 3.402329789406833, + "tokens_seen": 2209218560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016698094282848548, + "loss": 1.4816, + "theoretical_loss": 3.402313286872084, + "tokens_seen": 2209349632 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016696088264794384, + "loss": 1.264, + "theoretical_loss": 3.4022967855904453, + "tokens_seen": 2209480704 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016694082246740223, + "loss": 1.2108, + "theoretical_loss": 3.4022802855617487, + "tokens_seen": 2209611776 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001669207622868606, + "loss": 1.478, + "theoretical_loss": 3.402263786785824, + "tokens_seen": 2209742848 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016690070210631896, + "loss": 1.2338, + "theoretical_loss": 3.402247289262503, + "tokens_seen": 2209873920 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016688064192577735, + "loss": 1.4542, + "theoretical_loss": 3.4022307929916145, + "tokens_seen": 2210004992 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001668605817452357, + "loss": 1.3757, + "theoretical_loss": 3.4022142979729906, + "tokens_seen": 2210136064 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.4881476163864136, + "objective/train/avg_token_score": 0.012473991140723228, + "objective/train/docs_used": 1252332, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5845160484313965, + "objective/train/original_loss": 3.871239423751831, + "objective/train/theoretical_loss": 3.402206050933225, + "objective/train/tokens_used": 2210201600, + "objective/train/unlikelihood_loss": 0.1416260451078415, + "theoretical_loss": 3.402206050933225, + "tokens_seen": 2210201600 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001668405215646941, + "loss": 1.4702, + "theoretical_loss": 3.4021978042064616, + "tokens_seen": 2210267136 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016682046138415246, + "loss": 1.2275, + "theoretical_loss": 3.402181311691858, + "tokens_seen": 2210398208 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016680040120361085, + "loss": 1.241, + "theoretical_loss": 3.4021648204290105, + "tokens_seen": 2210529280 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001667803410230692, + "loss": 1.322, + "theoretical_loss": 3.40214833041775, + "tokens_seen": 2210660352 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001667602808425276, + "loss": 1.2714, + "theoretical_loss": 3.4021318416579076, + "tokens_seen": 2210791424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016674022066198596, + "loss": 1.2642, + "theoretical_loss": 3.4021153541493137, + "tokens_seen": 2210922496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016672016048144433, + "loss": 1.3596, + "theoretical_loss": 3.4020988678918, + "tokens_seen": 2211053568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016670010030090271, + "loss": 1.3659, + "theoretical_loss": 3.4020823828851956, + "tokens_seen": 2211184640 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016668004012036108, + "loss": 1.2672, + "theoretical_loss": 3.4020658991293335, + "tokens_seen": 2211315712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016665997993981947, + "loss": 1.4722, + "theoretical_loss": 3.4020494166240436, + "tokens_seen": 2211446784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016663991975927783, + "loss": 1.22, + "theoretical_loss": 3.4020329353691574, + "tokens_seen": 2211577856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016661985957873622, + "loss": 1.2139, + "theoretical_loss": 3.4020164553645054, + "tokens_seen": 2211708928 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.48765885829925537, + "objective/train/avg_token_score": 0.012732814997434616, + "objective/train/docs_used": 1253043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5017952919006348, + "objective/train/original_loss": 3.8243677616119385, + "objective/train/theoretical_loss": 3.4019999766099187, + "objective/train/tokens_used": 2211840000, + "objective/train/unlikelihood_loss": 0.09291806071996689, + "theoretical_loss": 3.4019999766099187, + "tokens_seen": 2211840000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016659979939819458, + "loss": 1.057, + "theoretical_loss": 3.4019999766099187, + "tokens_seen": 2211840000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016657973921765297, + "loss": 1.3161, + "theoretical_loss": 3.401983499105229, + "tokens_seen": 2211971072 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016655967903711133, + "loss": 1.2587, + "theoretical_loss": 3.401967022850267, + "tokens_seen": 2212102144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016653961885656972, + "loss": 1.3588, + "theoretical_loss": 3.401950547844864, + "tokens_seen": 2212233216 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016651955867602808, + "loss": 1.3605, + "theoretical_loss": 3.4019340740888513, + "tokens_seen": 2212364288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016649949849548645, + "loss": 1.3614, + "theoretical_loss": 3.40191760158206, + "tokens_seen": 2212495360 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016647943831494484, + "loss": 1.2973, + "theoretical_loss": 3.4019011303243207, + "tokens_seen": 2212626432 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001664593781344032, + "loss": 1.2394, + "theoretical_loss": 3.401884660315466, + "tokens_seen": 2212757504 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001664393179538616, + "loss": 1.2773, + "theoretical_loss": 3.4018681915553266, + "tokens_seen": 2212888576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016641925777331995, + "loss": 1.3385, + "theoretical_loss": 3.4018517240437336, + "tokens_seen": 2213019648 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016639919759277834, + "loss": 1.3592, + "theoretical_loss": 3.401835257780519, + "tokens_seen": 2213150720 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001663791374122367, + "loss": 1.2236, + "theoretical_loss": 3.401818792765514, + "tokens_seen": 2213281792 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001663590772316951, + "loss": 1.4548, + "theoretical_loss": 3.40180232899855, + "tokens_seen": 2213412864 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.4152003824710846, + "objective/train/avg_token_score": 0.009559571743011475, + "objective/train/docs_used": 1254214, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1770663261413574, + "objective/train/original_loss": 4.0069050788879395, + "objective/train/theoretical_loss": 3.4017940975830308, + "objective/train/tokens_used": 2213478400, + "objective/train/unlikelihood_loss": 0.12187237292528152, + "theoretical_loss": 3.4017940975830308, + "tokens_seen": 2213478400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016633901705115345, + "loss": 1.2211, + "theoretical_loss": 3.4017858664794582, + "tokens_seen": 2213543936 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016631895687061181, + "loss": 1.411, + "theoretical_loss": 3.4017694052080705, + "tokens_seen": 2213675008 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001662988966900702, + "loss": 1.374, + "theoretical_loss": 3.401752945184219, + "tokens_seen": 2213806080 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001662788365095286, + "loss": 1.1545, + "theoretical_loss": 3.401736486407734, + "tokens_seen": 2213937152 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016625877632898698, + "loss": 1.3528, + "theoretical_loss": 3.401720028878448, + "tokens_seen": 2214068224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016623871614844534, + "loss": 1.3447, + "theoretical_loss": 3.4017035725961926, + "tokens_seen": 2214199296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016621865596790373, + "loss": 1.3284, + "theoretical_loss": 3.4016871175607997, + "tokens_seen": 2214330368 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661985957873621, + "loss": 1.2119, + "theoretical_loss": 3.4016706637721006, + "tokens_seen": 2214461440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016617853560682049, + "loss": 1.2025, + "theoretical_loss": 3.401654211229927, + "tokens_seen": 2214592512 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016615847542627885, + "loss": 1.4218, + "theoretical_loss": 3.4016377599341108, + "tokens_seen": 2214723584 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661384152457372, + "loss": 1.1813, + "theoretical_loss": 3.4016213098844843, + "tokens_seen": 2214854656 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661183550651956, + "loss": 1.2868, + "theoretical_loss": 3.4016048610808785, + "tokens_seen": 2214985728 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.5463098883628845, + "objective/train/avg_token_score": 0.02969295158982277, + "objective/train/docs_used": 1254739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5483824014663696, + "objective/train/original_loss": 3.6490139961242676, + "objective/train/theoretical_loss": 3.4015884135231262, + "objective/train/tokens_used": 2215116800, + "objective/train/unlikelihood_loss": 0.1078660786151886, + "theoretical_loss": 3.4015884135231262, + "tokens_seen": 2215116800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016609829488465396, + "loss": 1.5094, + "theoretical_loss": 3.4015884135231262, + "tokens_seen": 2215116800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016607823470411235, + "loss": 1.2836, + "theoretical_loss": 3.401571967211059, + "tokens_seen": 2215247872 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001660581745235707, + "loss": 1.2652, + "theoretical_loss": 3.4015555221445086, + "tokens_seen": 2215378944 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001660381143430291, + "loss": 1.3368, + "theoretical_loss": 3.401539078323307, + "tokens_seen": 2215510016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016601805416248746, + "loss": 1.2666, + "theoretical_loss": 3.4015226357472867, + "tokens_seen": 2215641088 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016599799398194585, + "loss": 1.2797, + "theoretical_loss": 3.401506194416279, + "tokens_seen": 2215772160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016597793380140422, + "loss": 1.3346, + "theoretical_loss": 3.401489754330117, + "tokens_seen": 2215903232 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001659578736208626, + "loss": 1.2401, + "theoretical_loss": 3.401473315488632, + "tokens_seen": 2216034304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016593781344032097, + "loss": 1.3115, + "theoretical_loss": 3.401456877891656, + "tokens_seen": 2216165376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016591775325977933, + "loss": 1.3306, + "theoretical_loss": 3.401440441539022, + "tokens_seen": 2216296448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016589769307923772, + "loss": 1.2418, + "theoretical_loss": 3.401424006430562, + "tokens_seen": 2216427520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016587763289869608, + "loss": 1.3881, + "theoretical_loss": 3.4014075725661077, + "tokens_seen": 2216558592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016585757271815447, + "loss": 1.4995, + "theoretical_loss": 3.401391139945492, + "tokens_seen": 2216689664 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.46169355511665344, + "objective/train/avg_token_score": 0.010330517776310444, + "objective/train/docs_used": 1256282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4510587453842163, + "objective/train/original_loss": 3.8294265270233154, + "objective/train/theoretical_loss": 3.401382924101571, + "objective/train/tokens_used": 2216755200, + "objective/train/unlikelihood_loss": 0.17375747859477997, + "theoretical_loss": 3.401382924101571, + "tokens_seen": 2216755200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016583751253761283, + "loss": 1.2375, + "theoretical_loss": 3.4013747085685466, + "tokens_seen": 2216820736 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016581745235707122, + "loss": 1.1945, + "theoretical_loss": 3.4013582784351044, + "tokens_seen": 2216951808 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016579739217652959, + "loss": 1.4845, + "theoretical_loss": 3.401341849544998, + "tokens_seen": 2217082880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016577733199598797, + "loss": 1.3438, + "theoretical_loss": 3.401325421898059, + "tokens_seen": 2217213952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016575727181544634, + "loss": 1.3507, + "theoretical_loss": 3.4013089954941202, + "tokens_seen": 2217345024 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001657372116349047, + "loss": 1.4065, + "theoretical_loss": 3.4012925703330144, + "tokens_seen": 2217476096 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001657171514543631, + "loss": 1.2709, + "theoretical_loss": 3.401276146414574, + "tokens_seen": 2217607168 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016569709127382145, + "loss": 1.3838, + "theoretical_loss": 3.4012597237386313, + "tokens_seen": 2217738240 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016567703109327984, + "loss": 1.4113, + "theoretical_loss": 3.4012433023050193, + "tokens_seen": 2217869312 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001656569709127382, + "loss": 1.2018, + "theoretical_loss": 3.4012268821135696, + "tokens_seen": 2218000384 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001656369107321966, + "loss": 1.121, + "theoretical_loss": 3.401210463164116, + "tokens_seen": 2218131456 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016561685055165495, + "loss": 1.2303, + "theoretical_loss": 3.4011940454564904, + "tokens_seen": 2218262528 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.25953078269958496, + "objective/train/avg_token_score": 0.04934045672416687, + "objective/train/docs_used": 1256744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8042884469032288, + "objective/train/original_loss": 4.709519863128662, + "objective/train/theoretical_loss": 3.401177628990526, + "objective/train/tokens_used": 2218393600, + "objective/train/unlikelihood_loss": 0.1682342290878296, + "theoretical_loss": 3.401177628990526, + "tokens_seen": 2218393600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016559679037111334, + "loss": 1.2585, + "theoretical_loss": 3.401177628990526, + "tokens_seen": 2218393600 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001655767301905717, + "loss": 1.435, + "theoretical_loss": 3.401161213766055, + "tokens_seen": 2218524672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016555667001003007, + "loss": 1.3749, + "theoretical_loss": 3.4011447997829114, + "tokens_seen": 2218655744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016553660982948848, + "loss": 1.3497, + "theoretical_loss": 3.4011283870409263, + "tokens_seen": 2218786816 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016551654964894685, + "loss": 1.2621, + "theoretical_loss": 3.4011119755399335, + "tokens_seen": 2218917888 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016549648946840524, + "loss": 1.2761, + "theoretical_loss": 3.4010955652797663, + "tokens_seen": 2219048960 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001654764292878636, + "loss": 1.3206, + "theoretical_loss": 3.4010791562602565, + "tokens_seen": 2219180032 + }, + { + "epoch": 0.67, + "learning_rate": 0.000165456369107322, + "loss": 1.3418, + "theoretical_loss": 3.401062748481238, + "tokens_seen": 2219311104 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016543630892678035, + "loss": 1.3234, + "theoretical_loss": 3.4010463419425427, + "tokens_seen": 2219442176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016541624874623874, + "loss": 1.4089, + "theoretical_loss": 3.401029936644005, + "tokens_seen": 2219573248 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653961885656971, + "loss": 1.3123, + "theoretical_loss": 3.4010135325854565, + "tokens_seen": 2219704320 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653761283851555, + "loss": 1.4613, + "theoretical_loss": 3.4009971297667314, + "tokens_seen": 2219835392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016535606820461385, + "loss": 1.3927, + "theoretical_loss": 3.4009807281876623, + "tokens_seen": 2219966464 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.46212121844291687, + "objective/train/avg_token_score": 0.008695484139025211, + "objective/train/docs_used": 1258010, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4167969226837158, + "objective/train/original_loss": 3.415731430053711, + "objective/train/theoretical_loss": 3.4009725278629466, + "objective/train/tokens_used": 2220032000, + "objective/train/unlikelihood_loss": 0.17007163166999817, + "theoretical_loss": 3.4009725278629466, + "tokens_seen": 2220032000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016533600802407221, + "loss": 1.4184, + "theoretical_loss": 3.400964327848082, + "tokens_seen": 2220097536 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653159478435306, + "loss": 1.1284, + "theoretical_loss": 3.4009479287478244, + "tokens_seen": 2220228608 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016529588766298897, + "loss": 1.2749, + "theoretical_loss": 3.400931530886722, + "tokens_seen": 2220359680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016527582748244736, + "loss": 1.2249, + "theoretical_loss": 3.4009151342646087, + "tokens_seen": 2220490752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016525576730190572, + "loss": 1.3084, + "theoretical_loss": 3.400898738881317, + "tokens_seen": 2220621824 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001652357071213641, + "loss": 1.2365, + "theoretical_loss": 3.400882344736681, + "tokens_seen": 2220752896 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016521564694082247, + "loss": 1.2822, + "theoretical_loss": 3.4008659518305335, + "tokens_seen": 2220883968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016519558676028086, + "loss": 1.2891, + "theoretical_loss": 3.4008495601627082, + "tokens_seen": 2221015040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016517552657973922, + "loss": 1.4069, + "theoretical_loss": 3.400833169733038, + "tokens_seen": 2221146112 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016515546639919758, + "loss": 1.403, + "theoretical_loss": 3.4008167805413563, + "tokens_seen": 2221277184 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016513540621865597, + "loss": 1.4563, + "theoretical_loss": 3.400800392587497, + "tokens_seen": 2221408256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016511534603811433, + "loss": 1.2983, + "theoretical_loss": 3.4007840058712935, + "tokens_seen": 2221539328 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.37988758087158203, + "objective/train/avg_token_score": 0.0075633409433066845, + "objective/train/docs_used": 1258716, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.258331060409546, + "objective/train/original_loss": 3.947659730911255, + "objective/train/theoretical_loss": 3.400767620392579, + "objective/train/tokens_used": 2221670400, + "objective/train/unlikelihood_loss": 0.19193151593208313, + "theoretical_loss": 3.400767620392579, + "tokens_seen": 2221670400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016509528585757272, + "loss": 1.3491, + "theoretical_loss": 3.400767620392579, + "tokens_seen": 2221670400 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001650752256770311, + "loss": 1.4798, + "theoretical_loss": 3.4007512361511876, + "tokens_seen": 2221801472 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016505516549648948, + "loss": 1.3252, + "theoretical_loss": 3.400734853146952, + "tokens_seen": 2221932544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016503510531594784, + "loss": 1.4093, + "theoretical_loss": 3.4007184713797067, + "tokens_seen": 2222063616 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016501504513540623, + "loss": 1.4518, + "theoretical_loss": 3.4007020908492853, + "tokens_seen": 2222194688 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001649949849548646, + "loss": 1.465, + "theoretical_loss": 3.400685711555521, + "tokens_seen": 2222325760 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016497492477432295, + "loss": 1.4209, + "theoretical_loss": 3.4006693334982474, + "tokens_seen": 2222456832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016495486459378134, + "loss": 1.2473, + "theoretical_loss": 3.4006529566772987, + "tokens_seen": 2222587904 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001649348044132397, + "loss": 1.2714, + "theoretical_loss": 3.4006365810925088, + "tokens_seen": 2222718976 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001649147442326981, + "loss": 1.2288, + "theoretical_loss": 3.4006202067437106, + "tokens_seen": 2222850048 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016489468405215646, + "loss": 1.337, + "theoretical_loss": 3.400603833630739, + "tokens_seen": 2222981120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016487462387161484, + "loss": 1.3761, + "theoretical_loss": 3.4005874617534273, + "tokens_seen": 2223112192 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001648545636910732, + "loss": 1.1387, + "theoretical_loss": 3.400571091111609, + "tokens_seen": 2223243264 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.3425586521625519, + "objective/train/avg_token_score": 0.013387668877840042, + "objective/train/docs_used": 1259970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.037359595298767, + "objective/train/original_loss": 3.9253785610198975, + "objective/train/theoretical_loss": 3.400562906253959, + "objective/train/tokens_used": 2223308800, + "objective/train/unlikelihood_loss": 0.16111186146736145, + "theoretical_loss": 3.400562906253959, + "tokens_seen": 2223308800 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001648345035105316, + "loss": 1.306, + "theoretical_loss": 3.4005547217051193, + "tokens_seen": 2223374336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016481444332998996, + "loss": 1.2708, + "theoretical_loss": 3.400538353533791, + "tokens_seen": 2223505408 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016479438314944835, + "loss": 1.2705, + "theoretical_loss": 3.4005219865974587, + "tokens_seen": 2223636480 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016477432296890674, + "loss": 1.4405, + "theoretical_loss": 3.4005056208959563, + "tokens_seen": 2223767552 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001647542627883651, + "loss": 1.2412, + "theoretical_loss": 3.400489256429118, + "tokens_seen": 2223898624 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001647342026078235, + "loss": 1.2461, + "theoretical_loss": 3.400472893196777, + "tokens_seen": 2224029696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016471414242728185, + "loss": 1.4058, + "theoretical_loss": 3.4004565311987687, + "tokens_seen": 2224160768 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016469408224674024, + "loss": 1.4777, + "theoretical_loss": 3.4004401704349263, + "tokens_seen": 2224291840 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001646740220661986, + "loss": 1.3968, + "theoretical_loss": 3.400423810905085, + "tokens_seen": 2224422912 + }, + { + "epoch": 0.67, + "learning_rate": 0.000164653961885657, + "loss": 1.3392, + "theoretical_loss": 3.4004074526090777, + "tokens_seen": 2224553984 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016463390170511535, + "loss": 1.3241, + "theoretical_loss": 3.4003910955467394, + "tokens_seen": 2224685056 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016461384152457374, + "loss": 1.3757, + "theoretical_loss": 3.400374739717905, + "tokens_seen": 2224816128 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.25531524419784546, + "objective/train/avg_token_score": 0.013964212499558926, + "objective/train/docs_used": 1260594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8548713326454163, + "objective/train/original_loss": 4.094290733337402, + "objective/train/theoretical_loss": 3.4003583851224075, + "objective/train/tokens_used": 2224947200, + "objective/train/unlikelihood_loss": 0.15806782245635986, + "theoretical_loss": 3.4003583851224075, + "tokens_seen": 2224947200 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001645937813440321, + "loss": 1.2343, + "theoretical_loss": 3.4003583851224075, + "tokens_seen": 2224947200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016457372116349047, + "loss": 1.2945, + "theoretical_loss": 3.400342031760082, + "tokens_seen": 2225078272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016455366098294886, + "loss": 1.3363, + "theoretical_loss": 3.400325679630763, + "tokens_seen": 2225209344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016453360080240722, + "loss": 1.2436, + "theoretical_loss": 3.4003093287342843, + "tokens_seen": 2225340416 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001645135406218656, + "loss": 1.2725, + "theoretical_loss": 3.400292979070481, + "tokens_seen": 2225471488 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016449348044132397, + "loss": 1.3023, + "theoretical_loss": 3.4002766306391874, + "tokens_seen": 2225602560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016447342026078236, + "loss": 1.2494, + "theoretical_loss": 3.400260283440238, + "tokens_seen": 2225733632 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016445336008024072, + "loss": 1.3268, + "theoretical_loss": 3.400243937473467, + "tokens_seen": 2225864704 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001644332998996991, + "loss": 1.5268, + "theoretical_loss": 3.4002275927387093, + "tokens_seen": 2225995776 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016441323971915747, + "loss": 1.1377, + "theoretical_loss": 3.4002112492357996, + "tokens_seen": 2226126848 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016439317953861586, + "loss": 1.166, + "theoretical_loss": 3.4001949069645723, + "tokens_seen": 2226257920 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016437311935807423, + "loss": 1.3356, + "theoretical_loss": 3.400178565924862, + "tokens_seen": 2226388992 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001643530591775326, + "loss": 1.3263, + "theoretical_loss": 3.400162226116504, + "tokens_seen": 2226520064 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.5647605061531067, + "objective/train/avg_token_score": 0.010996363125741482, + "objective/train/docs_used": 1261463, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9950264692306519, + "objective/train/original_loss": 4.130087375640869, + "objective/train/theoretical_loss": 3.40015405667403, + "objective/train/tokens_used": 2226585600, + "objective/train/unlikelihood_loss": 0.14971353113651276, + "theoretical_loss": 3.40015405667403, + "tokens_seen": 2226585600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016433299899699098, + "loss": 1.3544, + "theoretical_loss": 3.4001458875393324, + "tokens_seen": 2226651136 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016431293881644934, + "loss": 1.1626, + "theoretical_loss": 3.4001295501931823, + "tokens_seen": 2226782208 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016429287863590773, + "loss": 1.3736, + "theoretical_loss": 3.400113214077888, + "tokens_seen": 2226913280 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001642728184553661, + "loss": 1.327, + "theoretical_loss": 3.400096879193285, + "tokens_seen": 2227044352 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016425275827482448, + "loss": 1.1864, + "theoretical_loss": 3.4000805455392076, + "tokens_seen": 2227175424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016423269809428284, + "loss": 1.2456, + "theoretical_loss": 3.400064213115491, + "tokens_seen": 2227306496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016421263791374123, + "loss": 1.3671, + "theoretical_loss": 3.40004788192197, + "tokens_seen": 2227437568 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001641925777331996, + "loss": 1.271, + "theoretical_loss": 3.40003155195848, + "tokens_seen": 2227568640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016417251755265796, + "loss": 1.3685, + "theoretical_loss": 3.4000152232248553, + "tokens_seen": 2227699712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016415245737211635, + "loss": 1.3942, + "theoretical_loss": 3.3999988957209313, + "tokens_seen": 2227830784 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001641323971915747, + "loss": 1.2445, + "theoretical_loss": 3.399982569446543, + "tokens_seen": 2227961856 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001641123370110331, + "loss": 1.3723, + "theoretical_loss": 3.3999662444015257, + "tokens_seen": 2228092928 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.6069159507751465, + "objective/train/avg_token_score": 0.006504898890852928, + "objective/train/docs_used": 1262743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.0266976356506348, + "objective/train/original_loss": 3.562269449234009, + "objective/train/theoretical_loss": 3.3999499205857138, + "objective/train/tokens_used": 2228224000, + "objective/train/unlikelihood_loss": 0.1684316098690033, + "theoretical_loss": 3.3999499205857138, + "tokens_seen": 2228224000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016409227683049146, + "loss": 1.5281, + "theoretical_loss": 3.3999499205857138, + "tokens_seen": 2228224000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016407221664994985, + "loss": 1.3974, + "theoretical_loss": 3.399933597998943, + "tokens_seen": 2228355072 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001640521564694082, + "loss": 1.2545, + "theoretical_loss": 3.399917276641049, + "tokens_seen": 2228486144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016403209628886663, + "loss": 1.1355, + "theoretical_loss": 3.399900956511866, + "tokens_seen": 2228617216 + }, + { + "epoch": 0.68, + "learning_rate": 0.000164012036108325, + "loss": 1.2747, + "theoretical_loss": 3.3998846376112297, + "tokens_seen": 2228748288 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016399197592778335, + "loss": 1.3881, + "theoretical_loss": 3.3998683199389754, + "tokens_seen": 2228879360 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016397191574724174, + "loss": 1.3991, + "theoretical_loss": 3.3998520034949387, + "tokens_seen": 2229010432 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001639518555667001, + "loss": 1.4172, + "theoretical_loss": 3.3998356882789538, + "tokens_seen": 2229141504 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001639317953861585, + "loss": 1.1923, + "theoretical_loss": 3.3998193742908573, + "tokens_seen": 2229272576 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016391173520561686, + "loss": 1.1931, + "theoretical_loss": 3.399803061530484, + "tokens_seen": 2229403648 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016389167502507524, + "loss": 1.4277, + "theoretical_loss": 3.39978674999767, + "tokens_seen": 2229534720 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001638716148445336, + "loss": 1.3397, + "theoretical_loss": 3.39977043969225, + "tokens_seen": 2229665792 + }, + { + "epoch": 0.68, + "learning_rate": 0.000163851554663992, + "loss": 1.2224, + "theoretical_loss": 3.3997541306140597, + "tokens_seen": 2229796864 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.46584799885749817, + "objective/train/avg_token_score": 0.008627312257885933, + "objective/train/docs_used": 1263359, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4600435495376587, + "objective/train/original_loss": 3.7039074897766113, + "objective/train/theoretical_loss": 3.399745976535124, + "objective/train/tokens_used": 2229862400, + "objective/train/unlikelihood_loss": 0.17117580771446228, + "theoretical_loss": 3.399745976535124, + "tokens_seen": 2229862400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016383149448345036, + "loss": 1.3814, + "theoretical_loss": 3.3997378227629347, + "tokens_seen": 2229927936 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016381143430290875, + "loss": 1.3324, + "theoretical_loss": 3.3997215161387104, + "tokens_seen": 2230059008 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001637913741223671, + "loss": 1.2241, + "theoretical_loss": 3.3997052107412227, + "tokens_seen": 2230190080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016377131394182547, + "loss": 1.284, + "theoretical_loss": 3.399688906570307, + "tokens_seen": 2230321152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016375125376128386, + "loss": 1.413, + "theoretical_loss": 3.3996726036257994, + "tokens_seen": 2230452224 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016373119358074222, + "loss": 1.2952, + "theoretical_loss": 3.3996563019075348, + "tokens_seen": 2230583296 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001637111334002006, + "loss": 1.2233, + "theoretical_loss": 3.3996400014153494, + "tokens_seen": 2230714368 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016369107321965898, + "loss": 1.2518, + "theoretical_loss": 3.399623702149079, + "tokens_seen": 2230845440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016367101303911736, + "loss": 1.1707, + "theoretical_loss": 3.399607404108559, + "tokens_seen": 2230976512 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016365095285857573, + "loss": 1.2223, + "theoretical_loss": 3.3995911072936256, + "tokens_seen": 2231107584 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016363089267803412, + "loss": 1.3194, + "theoretical_loss": 3.399574811704115, + "tokens_seen": 2231238656 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016361083249749248, + "loss": 1.4532, + "theoretical_loss": 3.399558517339862, + "tokens_seen": 2231369728 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.33993157744407654, + "objective/train/avg_token_score": 0.0383659191429615, + "objective/train/docs_used": 1264337, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9079421758651733, + "objective/train/original_loss": 4.009981155395508, + "objective/train/theoretical_loss": 3.3995422242007036, + "objective/train/tokens_used": 2231500800, + "objective/train/unlikelihood_loss": 0.16685305535793304, + "theoretical_loss": 3.3995422242007036, + "tokens_seen": 2231500800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016359077231695084, + "loss": 1.2948, + "theoretical_loss": 3.3995422242007036, + "tokens_seen": 2231500800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016357071213640923, + "loss": 1.3002, + "theoretical_loss": 3.399525932286475, + "tokens_seen": 2231631872 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001635506519558676, + "loss": 1.4834, + "theoretical_loss": 3.399509641597012, + "tokens_seen": 2231762944 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016353059177532598, + "loss": 1.354, + "theoretical_loss": 3.3994933521321515, + "tokens_seen": 2231894016 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016351053159478434, + "loss": 1.4026, + "theoretical_loss": 3.3994770638917293, + "tokens_seen": 2232025088 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016349047141424273, + "loss": 1.445, + "theoretical_loss": 3.3994607768755807, + "tokens_seen": 2232156160 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001634704112337011, + "loss": 1.1169, + "theoretical_loss": 3.3994444910835426, + "tokens_seen": 2232287232 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016345035105315949, + "loss": 1.3716, + "theoretical_loss": 3.399428206515451, + "tokens_seen": 2232418304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016343029087261785, + "loss": 1.3243, + "theoretical_loss": 3.3994119231711415, + "tokens_seen": 2232549376 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001634102306920762, + "loss": 1.3445, + "theoretical_loss": 3.399395641050451, + "tokens_seen": 2232680448 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001633901705115346, + "loss": 1.4971, + "theoretical_loss": 3.3993793601532154, + "tokens_seen": 2232811520 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016337011033099296, + "loss": 1.1699, + "theoretical_loss": 3.399363080479271, + "tokens_seen": 2232942592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016335005015045135, + "loss": 1.3468, + "theoretical_loss": 3.399346802028454, + "tokens_seen": 2233073664 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.3577101528644562, + "objective/train/avg_token_score": 0.01836022175848484, + "objective/train/docs_used": 1265060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1864618062973022, + "objective/train/original_loss": 4.090343475341797, + "objective/train/theoretical_loss": 3.3993386632616676, + "objective/train/tokens_used": 2233139200, + "objective/train/unlikelihood_loss": 0.16447505354881287, + "theoretical_loss": 3.3993386632616676, + "tokens_seen": 2233139200 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001633299899699097, + "loss": 1.1805, + "theoretical_loss": 3.399330524800601, + "tokens_seen": 2233204736 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001633099297893681, + "loss": 1.1746, + "theoretical_loss": 3.399314248795548, + "tokens_seen": 2233335808 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016328986960882646, + "loss": 1.2304, + "theoretical_loss": 3.3992979740131317, + "tokens_seen": 2233466880 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016326980942828488, + "loss": 1.2672, + "theoretical_loss": 3.399281700453188, + "tokens_seen": 2233597952 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016324974924774324, + "loss": 1.3815, + "theoretical_loss": 3.3992654281155543, + "tokens_seen": 2233729024 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016322968906720163, + "loss": 1.2255, + "theoretical_loss": 3.399249157000066, + "tokens_seen": 2233860096 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016320962888666, + "loss": 1.2156, + "theoretical_loss": 3.3992328871065602, + "tokens_seen": 2233991168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016318956870611836, + "loss": 1.2466, + "theoretical_loss": 3.3992166184348735, + "tokens_seen": 2234122240 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016316950852557675, + "loss": 1.3342, + "theoretical_loss": 3.399200350984842, + "tokens_seen": 2234253312 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001631494483450351, + "loss": 1.4483, + "theoretical_loss": 3.399184084756303, + "tokens_seen": 2234384384 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001631293881644935, + "loss": 1.6087, + "theoretical_loss": 3.399167819749092, + "tokens_seen": 2234515456 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016310932798395186, + "loss": 1.223, + "theoretical_loss": 3.399151555963047, + "tokens_seen": 2234646528 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.505437433719635, + "objective/train/avg_token_score": 0.010132663883268833, + "objective/train/docs_used": 1266280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.620944857597351, + "objective/train/original_loss": 3.6274094581604004, + "objective/train/theoretical_loss": 3.399135293398004, + "objective/train/tokens_used": 2234777600, + "objective/train/unlikelihood_loss": 0.1456189900636673, + "theoretical_loss": 3.399135293398004, + "tokens_seen": 2234777600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016308926780341025, + "loss": 1.308, + "theoretical_loss": 3.399135293398004, + "tokens_seen": 2234777600 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001630692076228686, + "loss": 1.4127, + "theoretical_loss": 3.3991190320538, + "tokens_seen": 2234908672 + }, + { + "epoch": 0.68, + "learning_rate": 0.000163049147442327, + "loss": 1.0384, + "theoretical_loss": 3.3991027719302713, + "tokens_seen": 2235039744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016302908726178536, + "loss": 1.2323, + "theoretical_loss": 3.399086513027255, + "tokens_seen": 2235170816 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016300902708124373, + "loss": 1.379, + "theoretical_loss": 3.3990702553445877, + "tokens_seen": 2235301888 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016298896690070211, + "loss": 1.226, + "theoretical_loss": 3.3990539988821067, + "tokens_seen": 2235432960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016296890672016048, + "loss": 1.1853, + "theoretical_loss": 3.399037743639649, + "tokens_seen": 2235564032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016294884653961887, + "loss": 1.3718, + "theoretical_loss": 3.3990214896170503, + "tokens_seen": 2235695104 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016292878635907723, + "loss": 1.353, + "theoretical_loss": 3.399005236814149, + "tokens_seen": 2235826176 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016290872617853562, + "loss": 1.4294, + "theoretical_loss": 3.3989889852307815, + "tokens_seen": 2235957248 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016288866599799398, + "loss": 1.2225, + "theoretical_loss": 3.3989727348667844, + "tokens_seen": 2236088320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016286860581745237, + "loss": 1.362, + "theoretical_loss": 3.3989564857219956, + "tokens_seen": 2236219392 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016284854563691073, + "loss": 1.342, + "theoretical_loss": 3.398940237796251, + "tokens_seen": 2236350464 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.4514296054840088, + "objective/train/avg_token_score": 0.016081299632787704, + "objective/train/docs_used": 1266935, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.569617509841919, + "objective/train/original_loss": 4.050370693206787, + "objective/train/theoretical_loss": 3.3989321142904703, + "objective/train/tokens_used": 2236416000, + "objective/train/unlikelihood_loss": 0.13412082195281982, + "theoretical_loss": 3.3989321142904703, + "tokens_seen": 2236416000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016282848545636912, + "loss": 1.4267, + "theoretical_loss": 3.398923991089389, + "tokens_seen": 2236481536 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016280842527582748, + "loss": 1.3849, + "theoretical_loss": 3.3989077456012464, + "tokens_seen": 2236612608 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016278836509528585, + "loss": 1.3368, + "theoretical_loss": 3.3988915013316596, + "tokens_seen": 2236743680 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016276830491474423, + "loss": 1.4461, + "theoretical_loss": 3.3988752582804667, + "tokens_seen": 2236874752 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001627482447342026, + "loss": 1.3699, + "theoretical_loss": 3.398859016447504, + "tokens_seen": 2237005824 + }, + { + "epoch": 0.68, + "learning_rate": 0.000162728184553661, + "loss": 1.2507, + "theoretical_loss": 3.39884277583261, + "tokens_seen": 2237136896 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016270812437311935, + "loss": 1.2794, + "theoretical_loss": 3.3988265364356205, + "tokens_seen": 2237267968 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016268806419257774, + "loss": 1.3222, + "theoretical_loss": 3.3988102982563744, + "tokens_seen": 2237399040 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001626680040120361, + "loss": 1.3738, + "theoretical_loss": 3.3987940612947076, + "tokens_seen": 2237530112 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001626479438314945, + "loss": 1.2744, + "theoretical_loss": 3.3987778255504586, + "tokens_seen": 2237661184 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016262788365095285, + "loss": 1.2757, + "theoretical_loss": 3.3987615910234643, + "tokens_seen": 2237792256 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016260782347041121, + "loss": 1.4747, + "theoretical_loss": 3.398745357713562, + "tokens_seen": 2237923328 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.5367180109024048, + "objective/train/avg_token_score": 0.007801317144185305, + "objective/train/docs_used": 1268187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6846973896026611, + "objective/train/original_loss": 3.591951847076416, + "objective/train/theoretical_loss": 3.39872912562059, + "objective/train/tokens_used": 2238054400, + "objective/train/unlikelihood_loss": 0.11725018918514252, + "theoretical_loss": 3.39872912562059, + "tokens_seen": 2238054400 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001625877632898696, + "loss": 1.3157, + "theoretical_loss": 3.39872912562059, + "tokens_seen": 2238054400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016256770310932797, + "loss": 1.4619, + "theoretical_loss": 3.3987128947443845, + "tokens_seen": 2238185472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016254764292878636, + "loss": 1.5413, + "theoretical_loss": 3.398696665084784, + "tokens_seen": 2238316544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016252758274824474, + "loss": 1.2528, + "theoretical_loss": 3.398680436641626, + "tokens_seen": 2238447616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016250752256770313, + "loss": 1.2714, + "theoretical_loss": 3.398664209414748, + "tokens_seen": 2238578688 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001624874623871615, + "loss": 1.1969, + "theoretical_loss": 3.398647983403987, + "tokens_seen": 2238709760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016246740220661989, + "loss": 1.3061, + "theoretical_loss": 3.3986317586091817, + "tokens_seen": 2238840832 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016244734202607825, + "loss": 1.3717, + "theoretical_loss": 3.3986155350301694, + "tokens_seen": 2238971904 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001624272818455366, + "loss": 1.2875, + "theoretical_loss": 3.3985993126667875, + "tokens_seen": 2239102976 + }, + { + "epoch": 0.68, + "learning_rate": 0.000162407221664995, + "loss": 1.3443, + "theoretical_loss": 3.3985830915188746, + "tokens_seen": 2239234048 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016238716148445336, + "loss": 1.4029, + "theoretical_loss": 3.3985668715862674, + "tokens_seen": 2239365120 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016236710130391175, + "loss": 1.2103, + "theoretical_loss": 3.3985506528688045, + "tokens_seen": 2239496192 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001623470411233701, + "loss": 1.448, + "theoretical_loss": 3.398534435366323, + "tokens_seen": 2239627264 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.3566104471683502, + "objective/train/avg_token_score": 0.01457633450627327, + "objective/train/docs_used": 1268791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0363723039627075, + "objective/train/original_loss": 3.7639737129211426, + "objective/train/theoretical_loss": 3.3985263270706505, + "objective/train/tokens_used": 2239692800, + "objective/train/unlikelihood_loss": 0.19765157997608185, + "theoretical_loss": 3.3985263270706505, + "tokens_seen": 2239692800 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001623269809428285, + "loss": 1.3695, + "theoretical_loss": 3.398518219078662, + "tokens_seen": 2239758336 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016230692076228686, + "loss": 1.3325, + "theoretical_loss": 3.3985020040056586, + "tokens_seen": 2239889408 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016228686058174525, + "loss": 1.1689, + "theoretical_loss": 3.3984857901471504, + "tokens_seen": 2240020480 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016226680040120362, + "loss": 1.3872, + "theoretical_loss": 3.398469577502976, + "tokens_seen": 2240151552 + }, + { + "epoch": 0.68, + "learning_rate": 0.000162246740220662, + "loss": 1.3852, + "theoretical_loss": 3.398453366072973, + "tokens_seen": 2240282624 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016222668004012037, + "loss": 1.3388, + "theoretical_loss": 3.39843715585698, + "tokens_seen": 2240413696 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016220661985957873, + "loss": 1.5498, + "theoretical_loss": 3.398420946854835, + "tokens_seen": 2240544768 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016218655967903712, + "loss": 1.533, + "theoretical_loss": 3.3984047390663754, + "tokens_seen": 2240675840 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016216649949849548, + "loss": 1.2887, + "theoretical_loss": 3.39838853249144, + "tokens_seen": 2240806912 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016214643931795387, + "loss": 1.3034, + "theoretical_loss": 3.398372327129867, + "tokens_seen": 2240937984 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016212637913741223, + "loss": 1.215, + "theoretical_loss": 3.398356122981494, + "tokens_seen": 2241069056 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016210631895687062, + "loss": 1.2811, + "theoretical_loss": 3.3983399200461597, + "tokens_seen": 2241200128 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.43139052391052246, + "objective/train/avg_token_score": 0.027375759556889534, + "objective/train/docs_used": 1270246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3165286779403687, + "objective/train/original_loss": 3.7604568004608154, + "objective/train/theoretical_loss": 3.3983237183237023, + "objective/train/tokens_used": 2241331200, + "objective/train/unlikelihood_loss": 0.20954500138759613, + "theoretical_loss": 3.3983237183237023, + "tokens_seen": 2241331200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016208625877632898, + "loss": 1.1623, + "theoretical_loss": 3.3983237183237023, + "tokens_seen": 2241331200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016206619859578737, + "loss": 1.2474, + "theoretical_loss": 3.3983075178139597, + "tokens_seen": 2241462272 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016204613841524574, + "loss": 1.2788, + "theoretical_loss": 3.3982913185167707, + "tokens_seen": 2241593344 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001620260782347041, + "loss": 1.3872, + "theoretical_loss": 3.3982751204319737, + "tokens_seen": 2241724416 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001620060180541625, + "loss": 1.2537, + "theoretical_loss": 3.398258923559407, + "tokens_seen": 2241855488 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016198595787362085, + "loss": 1.4475, + "theoretical_loss": 3.398242727898908, + "tokens_seen": 2241986560 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016196589769307924, + "loss": 1.3924, + "theoretical_loss": 3.398226533450317, + "tokens_seen": 2242117632 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001619458375125376, + "loss": 1.3176, + "theoretical_loss": 3.3982103402134713, + "tokens_seen": 2242248704 + }, + { + "epoch": 0.68, + "learning_rate": 0.000161925777331996, + "loss": 1.3897, + "theoretical_loss": 3.3981941481882094, + "tokens_seen": 2242379776 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016190571715145435, + "loss": 1.3646, + "theoretical_loss": 3.39817795737437, + "tokens_seen": 2242510848 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016188565697091274, + "loss": 1.2667, + "theoretical_loss": 3.398161767771792, + "tokens_seen": 2242641920 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001618655967903711, + "loss": 1.242, + "theoretical_loss": 3.3981455793803135, + "tokens_seen": 2242772992 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016184553660982947, + "loss": 1.2826, + "theoretical_loss": 3.3981293921997735, + "tokens_seen": 2242904064 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.4218597114086151, + "objective/train/avg_token_score": 0.03158407658338547, + "objective/train/docs_used": 1270875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3631649017333984, + "objective/train/original_loss": 3.95414662361145, + "objective/train/theoretical_loss": 3.398121299063555, + "objective/train/tokens_used": 2242969600, + "objective/train/unlikelihood_loss": 0.1576191484928131, + "theoretical_loss": 3.398121299063555, + "tokens_seen": 2242969600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016182547642928786, + "loss": 1.3801, + "theoretical_loss": 3.3981132062300103, + "tokens_seen": 2243035136 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016180541624874622, + "loss": 1.2566, + "theoretical_loss": 3.398097021470863, + "tokens_seen": 2243166208 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016178535606820464, + "loss": 1.2727, + "theoretical_loss": 3.39808083792217, + "tokens_seen": 2243297280 + }, + { + "epoch": 0.68, + "learning_rate": 0.000161765295887663, + "loss": 1.4716, + "theoretical_loss": 3.39806465558377, + "tokens_seen": 2243428352 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001617452357071214, + "loss": 1.2263, + "theoretical_loss": 3.3980484744555026, + "tokens_seen": 2243559424 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016172517552657975, + "loss": 1.2579, + "theoretical_loss": 3.3980322945372055, + "tokens_seen": 2243690496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016170511534603814, + "loss": 1.3619, + "theoretical_loss": 3.3980161158287183, + "tokens_seen": 2243821568 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001616850551654965, + "loss": 1.3684, + "theoretical_loss": 3.397999938329879, + "tokens_seen": 2243952640 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001616649949849549, + "loss": 1.1672, + "theoretical_loss": 3.3979837620405275, + "tokens_seen": 2244083712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016164493480441325, + "loss": 1.2092, + "theoretical_loss": 3.3979675869605024, + "tokens_seen": 2244214784 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016162487462387161, + "loss": 1.3249, + "theoretical_loss": 3.3979514130896424, + "tokens_seen": 2244345856 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016160481444333, + "loss": 1.2763, + "theoretical_loss": 3.3979352404277874, + "tokens_seen": 2244476928 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.40591397881507874, + "objective/train/avg_token_score": 0.015817703679203987, + "objective/train/docs_used": 1272103, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3362866640090942, + "objective/train/original_loss": 3.6313579082489014, + "objective/train/theoretical_loss": 3.397919068974775, + "objective/train/tokens_used": 2244608000, + "objective/train/unlikelihood_loss": 0.2006363421678543, + "theoretical_loss": 3.397919068974775, + "tokens_seen": 2244608000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016158475426278837, + "loss": 1.2247, + "theoretical_loss": 3.397919068974775, + "tokens_seen": 2244608000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016156469408224676, + "loss": 1.2977, + "theoretical_loss": 3.3979028987304454, + "tokens_seen": 2244739072 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016154463390170512, + "loss": 1.3219, + "theoretical_loss": 3.397886729694637, + "tokens_seen": 2244870144 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001615245737211635, + "loss": 1.1621, + "theoretical_loss": 3.3978705618671894, + "tokens_seen": 2245001216 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016150451354062187, + "loss": 1.4109, + "theoretical_loss": 3.3978543952479416, + "tokens_seen": 2245132288 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016148445336008026, + "loss": 1.304, + "theoretical_loss": 3.397838229836733, + "tokens_seen": 2245263360 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016146439317953862, + "loss": 1.2642, + "theoretical_loss": 3.3978220656334024, + "tokens_seen": 2245394432 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016144433299899698, + "loss": 1.2782, + "theoretical_loss": 3.397805902637789, + "tokens_seen": 2245525504 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016142427281845537, + "loss": 1.3638, + "theoretical_loss": 3.397789740849732, + "tokens_seen": 2245656576 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016140421263791373, + "loss": 1.2959, + "theoretical_loss": 3.397773580269072, + "tokens_seen": 2245787648 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016138415245737212, + "loss": 1.432, + "theoretical_loss": 3.3977574208956467, + "tokens_seen": 2245918720 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016136409227683049, + "loss": 1.2048, + "theoretical_loss": 3.3977412627292964, + "tokens_seen": 2246049792 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016134403209628888, + "loss": 1.2732, + "theoretical_loss": 3.39772510576986, + "tokens_seen": 2246180864 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.44458699226379395, + "objective/train/avg_token_score": 0.018207818269729614, + "objective/train/docs_used": 1272482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2546980381011963, + "objective/train/original_loss": 3.6830179691314697, + "objective/train/theoretical_loss": 3.3977170277426847, + "objective/train/tokens_used": 2246246400, + "objective/train/unlikelihood_loss": 0.17190679907798767, + "theoretical_loss": 3.3977170277426847, + "tokens_seen": 2246246400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016132397191574724, + "loss": 1.3907, + "theoretical_loss": 3.3977089500171775, + "tokens_seen": 2246311936 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016130391173520563, + "loss": 1.4543, + "theoretical_loss": 3.3976927954710874, + "tokens_seen": 2246443008 + }, + { + "epoch": 0.68, + "learning_rate": 0.000161283851554664, + "loss": 1.2397, + "theoretical_loss": 3.3976766421314304, + "tokens_seen": 2246574080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016126379137412235, + "loss": 1.2494, + "theoretical_loss": 3.3976604899980454, + "tokens_seen": 2246705152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016124373119358074, + "loss": 1.2575, + "theoretical_loss": 3.3976443390707716, + "tokens_seen": 2246836224 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001612236710130391, + "loss": 1.2443, + "theoretical_loss": 3.397628189349449, + "tokens_seen": 2246967296 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001612036108324975, + "loss": 1.4106, + "theoretical_loss": 3.3976120408339177, + "tokens_seen": 2247098368 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016118355065195585, + "loss": 1.2874, + "theoretical_loss": 3.3975958935240165, + "tokens_seen": 2247229440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016116349047141424, + "loss": 1.3107, + "theoretical_loss": 3.3975797474195857, + "tokens_seen": 2247360512 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001611434302908726, + "loss": 1.4279, + "theoretical_loss": 3.3975636025204645, + "tokens_seen": 2247491584 + }, + { + "epoch": 0.68, + "learning_rate": 0.000161123370110331, + "loss": 1.2831, + "theoretical_loss": 3.3975474588264927, + "tokens_seen": 2247622656 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016110330992978936, + "loss": 1.4067, + "theoretical_loss": 3.3975313163375107, + "tokens_seen": 2247753728 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.3189760446548462, + "objective/train/avg_token_score": 0.018239615485072136, + "objective/train/docs_used": 1273740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.002007246017456, + "objective/train/original_loss": 3.9686782360076904, + "objective/train/theoretical_loss": 3.3975151750533574, + "objective/train/tokens_used": 2247884800, + "objective/train/unlikelihood_loss": 0.15150760114192963, + "theoretical_loss": 3.3975151750533574, + "tokens_seen": 2247884800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016108324974924775, + "loss": 1.3919, + "theoretical_loss": 3.3975151750533574, + "tokens_seen": 2247884800 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001610631895687061, + "loss": 1.2931, + "theoretical_loss": 3.397499034973873, + "tokens_seen": 2248015872 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016104312938816447, + "loss": 1.236, + "theoretical_loss": 3.397482896098898, + "tokens_seen": 2248146944 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001610230692076229, + "loss": 1.2228, + "theoretical_loss": 3.397466758428271, + "tokens_seen": 2248278016 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016100300902708125, + "loss": 1.2852, + "theoretical_loss": 3.3974506219618332, + "tokens_seen": 2248409088 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016098294884653964, + "loss": 1.3618, + "theoretical_loss": 3.3974344866994235, + "tokens_seen": 2248540160 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160962888665998, + "loss": 1.4318, + "theoretical_loss": 3.3974183526408828, + "tokens_seen": 2248671232 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001609428284854564, + "loss": 1.3493, + "theoretical_loss": 3.3974022197860503, + "tokens_seen": 2248802304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016092276830491475, + "loss": 1.2664, + "theoretical_loss": 3.397386088134767, + "tokens_seen": 2248933376 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016090270812437314, + "loss": 1.3732, + "theoretical_loss": 3.397369957686872, + "tokens_seen": 2249064448 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001608826479438315, + "loss": 1.439, + "theoretical_loss": 3.397353828442206, + "tokens_seen": 2249195520 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016086258776328987, + "loss": 1.2823, + "theoretical_loss": 3.397337700400609, + "tokens_seen": 2249326592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016084252758274826, + "loss": 1.3105, + "theoretical_loss": 3.397321573561921, + "tokens_seen": 2249457664 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.3000977635383606, + "objective/train/avg_token_score": 0.03202424198389053, + "objective/train/docs_used": 1274460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9710298180580139, + "objective/train/original_loss": 4.151874542236328, + "objective/train/theoretical_loss": 3.3973135105936176, + "objective/train/tokens_used": 2249523200, + "objective/train/unlikelihood_loss": 0.20692919194698334, + "theoretical_loss": 3.3973135105936176, + "tokens_seen": 2249523200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016082246740220662, + "loss": 1.2014, + "theoretical_loss": 3.397305447925982, + "tokens_seen": 2249588736 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160802407221665, + "loss": 1.3173, + "theoretical_loss": 3.397289323492633, + "tokens_seen": 2249719808 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016078234704112337, + "loss": 1.2921, + "theoretical_loss": 3.397273200261713, + "tokens_seen": 2249850880 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016076228686058176, + "loss": 1.3641, + "theoretical_loss": 3.397257078233064, + "tokens_seen": 2249981952 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016074222668004012, + "loss": 1.3642, + "theoretical_loss": 3.397240957406525, + "tokens_seen": 2250113024 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001607221664994985, + "loss": 1.3527, + "theoretical_loss": 3.397224837781937, + "tokens_seen": 2250244096 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016070210631895687, + "loss": 1.3875, + "theoretical_loss": 3.3972087193591403, + "tokens_seen": 2250375168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016068204613841526, + "loss": 1.4818, + "theoretical_loss": 3.397192602137975, + "tokens_seen": 2250506240 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016066198595787363, + "loss": 1.4921, + "theoretical_loss": 3.3971764861182816, + "tokens_seen": 2250637312 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160641925777332, + "loss": 1.3354, + "theoretical_loss": 3.3971603712999006, + "tokens_seen": 2250768384 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016062186559679038, + "loss": 1.264, + "theoretical_loss": 3.3971442576826725, + "tokens_seen": 2250899456 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016060180541624874, + "loss": 1.2762, + "theoretical_loss": 3.3971281452664384, + "tokens_seen": 2251030528 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.3972996175289154, + "objective/train/avg_token_score": 0.018043529242277145, + "objective/train/docs_used": 1275535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0833194255828857, + "objective/train/original_loss": 4.042899131774902, + "objective/train/theoretical_loss": 3.3971120340510375, + "objective/train/tokens_used": 2251161600, + "objective/train/unlikelihood_loss": 0.09438111633062363, + "theoretical_loss": 3.3971120340510375, + "tokens_seen": 2251161600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016058174523570713, + "loss": 1.2803, + "theoretical_loss": 3.3971120340510375, + "tokens_seen": 2251161600 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001605616850551655, + "loss": 1.336, + "theoretical_loss": 3.3970959240363117, + "tokens_seen": 2251292672 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016054162487462388, + "loss": 1.1939, + "theoretical_loss": 3.3970798152221016, + "tokens_seen": 2251423744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016052156469408224, + "loss": 1.4203, + "theoretical_loss": 3.397063707608247, + "tokens_seen": 2251554816 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016050150451354063, + "loss": 1.276, + "theoretical_loss": 3.397047601194589, + "tokens_seen": 2251685888 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160481444332999, + "loss": 1.2477, + "theoretical_loss": 3.3970314959809684, + "tokens_seen": 2251816960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016046138415245736, + "loss": 1.1647, + "theoretical_loss": 3.397015391967226, + "tokens_seen": 2251948032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016044132397191575, + "loss": 1.3711, + "theoretical_loss": 3.3969992891532024, + "tokens_seen": 2252079104 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001604212637913741, + "loss": 1.4624, + "theoretical_loss": 3.396983187538739, + "tokens_seen": 2252210176 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001604012036108325, + "loss": 1.2074, + "theoretical_loss": 3.396967087123675, + "tokens_seen": 2252341248 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016038114343029086, + "loss": 1.4947, + "theoretical_loss": 3.3969509879078528, + "tokens_seen": 2252472320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016036108324974925, + "loss": 1.3975, + "theoretical_loss": 3.396934889891113, + "tokens_seen": 2252603392 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001603410230692076, + "loss": 1.0833, + "theoretical_loss": 3.3969187930732967, + "tokens_seen": 2252734464 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.5001221895217896, + "objective/train/avg_token_score": 0.008072170428931713, + "objective/train/docs_used": 1276002, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5848617553710938, + "objective/train/original_loss": 3.540743827819824, + "objective/train/theoretical_loss": 3.396910745113935, + "objective/train/tokens_used": 2252800000, + "objective/train/unlikelihood_loss": 0.15157487988471985, + "theoretical_loss": 3.396910745113935, + "tokens_seen": 2252800000 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160320962888666, + "loss": 1.2974, + "theoretical_loss": 3.3969026974542444, + "tokens_seen": 2252865536 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016030090270812436, + "loss": 1.2608, + "theoretical_loss": 3.396886603033797, + "tokens_seen": 2252996608 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016028084252758275, + "loss": 1.4583, + "theoretical_loss": 3.396870509811796, + "tokens_seen": 2253127680 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016026078234704114, + "loss": 1.3421, + "theoretical_loss": 3.3968544177880817, + "tokens_seen": 2253258752 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001602407221664995, + "loss": 1.3435, + "theoretical_loss": 3.3968383269624964, + "tokens_seen": 2253389824 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001602206619859579, + "loss": 1.3257, + "theoretical_loss": 3.3968222373348804, + "tokens_seen": 2253520896 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016020060180541626, + "loss": 1.2737, + "theoretical_loss": 3.3968061489050747, + "tokens_seen": 2253651968 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016018054162487464, + "loss": 1.3669, + "theoretical_loss": 3.3967900616729207, + "tokens_seen": 2253783040 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160160481444333, + "loss": 1.2004, + "theoretical_loss": 3.3967739756382596, + "tokens_seen": 2253914112 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001601404212637914, + "loss": 1.2616, + "theoretical_loss": 3.396757890800933, + "tokens_seen": 2254045184 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016012036108324976, + "loss": 1.3183, + "theoretical_loss": 3.3967418071607813, + "tokens_seen": 2254176256 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016010030090270815, + "loss": 1.317, + "theoretical_loss": 3.3967257247176468, + "tokens_seen": 2254307328 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.5912756323814392, + "objective/train/avg_token_score": 0.006867651361972094, + "objective/train/docs_used": 1277060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9138622283935547, + "objective/train/original_loss": 3.4892263412475586, + "objective/train/theoretical_loss": 3.39670964347137, + "objective/train/tokens_used": 2254438400, + "objective/train/unlikelihood_loss": 0.14271540939807892, + "theoretical_loss": 3.39670964347137, + "tokens_seen": 2254438400 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001600802407221665, + "loss": 1.4615, + "theoretical_loss": 3.39670964347137, + "tokens_seen": 2254438400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016006018054162487, + "loss": 1.3595, + "theoretical_loss": 3.3966935634217927, + "tokens_seen": 2254569472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016004012036108326, + "loss": 1.3835, + "theoretical_loss": 3.396677484568756, + "tokens_seen": 2254700544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016002006018054162, + "loss": 1.3935, + "theoretical_loss": 3.3966614069121017, + "tokens_seen": 2254831616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016, + "loss": 1.4954, + "theoretical_loss": 3.3966453304516713, + "tokens_seen": 2254962688 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015997993981945838, + "loss": 1.4013, + "theoretical_loss": 3.3966292551873054, + "tokens_seen": 2255093760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015995987963891676, + "loss": 1.3073, + "theoretical_loss": 3.3966131811188465, + "tokens_seen": 2255224832 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015993981945837513, + "loss": 1.2147, + "theoretical_loss": 3.3965971082461355, + "tokens_seen": 2255355904 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015991975927783352, + "loss": 1.3847, + "theoretical_loss": 3.3965810365690143, + "tokens_seen": 2255486976 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015989969909729188, + "loss": 1.3186, + "theoretical_loss": 3.3965649660873245, + "tokens_seen": 2255618048 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015987963891675024, + "loss": 1.2532, + "theoretical_loss": 3.3965488968009074, + "tokens_seen": 2255749120 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015985957873620863, + "loss": 1.407, + "theoretical_loss": 3.3965328287096046, + "tokens_seen": 2255880192 + }, + { + "epoch": 0.68, + "learning_rate": 0.000159839518555667, + "loss": 1.2682, + "theoretical_loss": 3.396516761813259, + "tokens_seen": 2256011264 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.5083088874816895, + "objective/train/avg_token_score": 0.018798891454935074, + "objective/train/docs_used": 1277665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5365872383117676, + "objective/train/original_loss": 3.7172746658325195, + "objective/train/theoretical_loss": 3.396508728813145, + "objective/train/tokens_used": 2256076800, + "objective/train/unlikelihood_loss": 0.22049881517887115, + "theoretical_loss": 3.396508728813145, + "tokens_seen": 2256076800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015981945837512538, + "loss": 1.3217, + "theoretical_loss": 3.3965006961117106, + "tokens_seen": 2256142336 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015979939819458374, + "loss": 1.328, + "theoretical_loss": 3.396484631604802, + "tokens_seen": 2256273408 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015977933801404213, + "loss": 1.4153, + "theoretical_loss": 3.396468568292375, + "tokens_seen": 2256404480 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001597592778335005, + "loss": 1.3093, + "theoretical_loss": 3.3964525061742714, + "tokens_seen": 2256535552 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015973921765295888, + "loss": 1.27, + "theoretical_loss": 3.3964364452503326, + "tokens_seen": 2256666624 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015971915747241725, + "loss": 1.2102, + "theoretical_loss": 3.396420385520401, + "tokens_seen": 2256797696 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001596990972918756, + "loss": 1.2795, + "theoretical_loss": 3.3964043269843183, + "tokens_seen": 2256928768 + }, + { + "epoch": 0.68, + "learning_rate": 0.000159679037111334, + "loss": 1.4027, + "theoretical_loss": 3.3963882696419265, + "tokens_seen": 2257059840 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015965897693079236, + "loss": 1.5436, + "theoretical_loss": 3.396372213493067, + "tokens_seen": 2257190912 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015963891675025075, + "loss": 1.3351, + "theoretical_loss": 3.396356158537583, + "tokens_seen": 2257321984 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001596188565697091, + "loss": 1.3573, + "theoretical_loss": 3.3963401047753154, + "tokens_seen": 2257453056 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001595987963891675, + "loss": 1.1508, + "theoretical_loss": 3.3963240522061064, + "tokens_seen": 2257584128 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.5305473804473877, + "objective/train/avg_token_score": 0.003977807238698006, + "objective/train/docs_used": 1279074, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6115937232971191, + "objective/train/original_loss": 3.621623992919922, + "objective/train/theoretical_loss": 3.396308000829799, + "objective/train/tokens_used": 2257715200, + "objective/train/unlikelihood_loss": 0.12437203526496887, + "theoretical_loss": 3.396308000829799, + "tokens_seen": 2257715200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015957873620862586, + "loss": 1.3809, + "theoretical_loss": 3.396308000829799, + "tokens_seen": 2257715200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015955867602808425, + "loss": 1.362, + "theoretical_loss": 3.396291950646234, + "tokens_seen": 2257846272 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015953861584754262, + "loss": 1.3295, + "theoretical_loss": 3.396275901655254, + "tokens_seen": 2257977344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015951855566700103, + "loss": 1.3588, + "theoretical_loss": 3.396259853856702, + "tokens_seen": 2258108416 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001594984954864594, + "loss": 1.3123, + "theoretical_loss": 3.396243807250419, + "tokens_seen": 2258239488 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015947843530591776, + "loss": 1.2971, + "theoretical_loss": 3.396227761836248, + "tokens_seen": 2258370560 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015945837512537615, + "loss": 1.3979, + "theoretical_loss": 3.3962117176140305, + "tokens_seen": 2258501632 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001594383149448345, + "loss": 1.2939, + "theoretical_loss": 3.39619567458361, + "tokens_seen": 2258632704 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001594182547642929, + "loss": 1.2973, + "theoretical_loss": 3.3961796327448277, + "tokens_seen": 2258763776 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015939819458375126, + "loss": 1.3309, + "theoretical_loss": 3.3961635920975266, + "tokens_seen": 2258894848 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015937813440320965, + "loss": 1.4059, + "theoretical_loss": 3.3961475526415486, + "tokens_seen": 2259025920 + }, + { + "epoch": 0.68, + "learning_rate": 0.000159358074222668, + "loss": 1.2407, + "theoretical_loss": 3.3961315143767363, + "tokens_seen": 2259156992 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001593380140421264, + "loss": 1.3062, + "theoretical_loss": 3.3961154773029323, + "tokens_seen": 2259288064 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.47110214829444885, + "objective/train/avg_token_score": 0.009308700449764729, + "objective/train/docs_used": 1279794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4832922220230103, + "objective/train/original_loss": 3.6377556324005127, + "objective/train/theoretical_loss": 3.396107459212609, + "objective/train/tokens_used": 2259353600, + "objective/train/unlikelihood_loss": 0.2077137678861618, + "theoretical_loss": 3.396107459212609, + "tokens_seen": 2259353600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015931795386158476, + "loss": 1.3531, + "theoretical_loss": 3.3960994414199783, + "tokens_seen": 2259419136 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015929789368104313, + "loss": 1.3454, + "theoretical_loss": 3.396083406727718, + "tokens_seen": 2259550208 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015927783350050151, + "loss": 1.2298, + "theoretical_loss": 3.3960673732259936, + "tokens_seen": 2259681280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015925777331995988, + "loss": 1.3196, + "theoretical_loss": 3.396051340914647, + "tokens_seen": 2259812352 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015923771313941827, + "loss": 1.4227, + "theoretical_loss": 3.3960353097935214, + "tokens_seen": 2259943424 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015921765295887663, + "loss": 1.2506, + "theoretical_loss": 3.3960192798624593, + "tokens_seen": 2260074496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015919759277833502, + "loss": 1.3742, + "theoretical_loss": 3.396003251121303, + "tokens_seen": 2260205568 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015917753259779338, + "loss": 1.2898, + "theoretical_loss": 3.3959872235698954, + "tokens_seen": 2260336640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015915747241725177, + "loss": 1.3647, + "theoretical_loss": 3.3959711972080795, + "tokens_seen": 2260467712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015913741223671013, + "loss": 1.2294, + "theoretical_loss": 3.3959551720356975, + "tokens_seen": 2260598784 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001591173520561685, + "loss": 1.3864, + "theoretical_loss": 3.3959391480525927, + "tokens_seen": 2260729856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015909729187562688, + "loss": 1.2094, + "theoretical_loss": 3.3959231252586077, + "tokens_seen": 2260860928 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.3951002061367035, + "objective/train/avg_token_score": 0.025016045197844505, + "objective/train/docs_used": 1281070, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2654213905334473, + "objective/train/original_loss": 4.072766304016113, + "objective/train/theoretical_loss": 3.395907103653585, + "objective/train/tokens_used": 2260992000, + "objective/train/unlikelihood_loss": 0.16731466352939606, + "theoretical_loss": 3.395907103653585, + "tokens_seen": 2260992000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015907723169508525, + "loss": 1.27, + "theoretical_loss": 3.395907103653585, + "tokens_seen": 2260992000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015905717151454363, + "loss": 1.3768, + "theoretical_loss": 3.395891083237368, + "tokens_seen": 2261123072 + }, + { + "epoch": 0.69, + "learning_rate": 0.000159037111334002, + "loss": 1.3644, + "theoretical_loss": 3.3958750640097994, + "tokens_seen": 2261254144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015901705115346039, + "loss": 1.4168, + "theoretical_loss": 3.3958590459707216, + "tokens_seen": 2261385216 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015899699097291875, + "loss": 1.3515, + "theoretical_loss": 3.395843029119978, + "tokens_seen": 2261516288 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015897693079237714, + "loss": 1.2281, + "theoretical_loss": 3.395827013457412, + "tokens_seen": 2261647360 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001589568706118355, + "loss": 1.3539, + "theoretical_loss": 3.3958109989828658, + "tokens_seen": 2261778432 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001589368104312939, + "loss": 1.409, + "theoretical_loss": 3.395794985696183, + "tokens_seen": 2261909504 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015891675025075225, + "loss": 1.3381, + "theoretical_loss": 3.3957789735972064, + "tokens_seen": 2262040576 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015889669007021061, + "loss": 1.2642, + "theoretical_loss": 3.395762962685779, + "tokens_seen": 2262171648 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158876629889669, + "loss": 1.4091, + "theoretical_loss": 3.395746952961744, + "tokens_seen": 2262302720 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015885656970912737, + "loss": 1.1551, + "theoretical_loss": 3.395730944424945, + "tokens_seen": 2262433792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015883650952858575, + "loss": 1.3543, + "theoretical_loss": 3.3957149370752244, + "tokens_seen": 2262564864 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.4364613890647888, + "objective/train/avg_token_score": 0.01746309921145439, + "objective/train/docs_used": 1281765, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4334213733673096, + "objective/train/original_loss": 3.7984652519226074, + "objective/train/theoretical_loss": 3.39570693384547, + "objective/train/tokens_used": 2262630400, + "objective/train/unlikelihood_loss": 0.1767033338546753, + "theoretical_loss": 3.39570693384547, + "tokens_seen": 2262630400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015881644934804412, + "loss": 1.4468, + "theoretical_loss": 3.3956989309124257, + "tokens_seen": 2262695936 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001587963891675025, + "loss": 1.3288, + "theoretical_loss": 3.3956829259363928, + "tokens_seen": 2262827008 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001587763289869609, + "loss": 1.2433, + "theoretical_loss": 3.395666922146968, + "tokens_seen": 2262958080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015875626880641929, + "loss": 1.2669, + "theoretical_loss": 3.3956509195439954, + "tokens_seen": 2263089152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015873620862587765, + "loss": 1.3163, + "theoretical_loss": 3.395634918127318, + "tokens_seen": 2263220224 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158716148445336, + "loss": 1.2506, + "theoretical_loss": 3.395618917896779, + "tokens_seen": 2263351296 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586960882647944, + "loss": 1.3528, + "theoretical_loss": 3.395602918852222, + "tokens_seen": 2263482368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015867602808425276, + "loss": 1.3713, + "theoretical_loss": 3.39558692099349, + "tokens_seen": 2263613440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015865596790371115, + "loss": 1.2417, + "theoretical_loss": 3.3955709243204275, + "tokens_seen": 2263744512 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586359077231695, + "loss": 1.5222, + "theoretical_loss": 3.3955549288328766, + "tokens_seen": 2263875584 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586158475426279, + "loss": 1.1576, + "theoretical_loss": 3.395538934530682, + "tokens_seen": 2264006656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015859578736208626, + "loss": 1.2391, + "theoretical_loss": 3.3955229414136863, + "tokens_seen": 2264137728 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.407746821641922, + "objective/train/avg_token_score": 0.016632545739412308, + "objective/train/docs_used": 1282394, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2163424491882324, + "objective/train/original_loss": 3.767566442489624, + "objective/train/theoretical_loss": 3.395506949481734, + "objective/train/tokens_used": 2264268800, + "objective/train/unlikelihood_loss": 0.13916462659835815, + "theoretical_loss": 3.395506949481734, + "tokens_seen": 2264268800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015857572718154465, + "loss": 1.2013, + "theoretical_loss": 3.395506949481734, + "tokens_seen": 2264268800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015855566700100302, + "loss": 1.2247, + "theoretical_loss": 3.395490958734668, + "tokens_seen": 2264399872 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001585356068204614, + "loss": 1.2645, + "theoretical_loss": 3.395474969172332, + "tokens_seen": 2264530944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015851554663991977, + "loss": 1.5449, + "theoretical_loss": 3.39545898079457, + "tokens_seen": 2264662016 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015849548645937813, + "loss": 1.3609, + "theoretical_loss": 3.3954429936012254, + "tokens_seen": 2264793088 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015847542627883652, + "loss": 1.2119, + "theoretical_loss": 3.3954270075921418, + "tokens_seen": 2264924160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015845536609829488, + "loss": 1.307, + "theoretical_loss": 3.3954110227671634, + "tokens_seen": 2265055232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015843530591775327, + "loss": 1.3169, + "theoretical_loss": 3.395395039126134, + "tokens_seen": 2265186304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015841524573721163, + "loss": 1.2801, + "theoretical_loss": 3.395379056668897, + "tokens_seen": 2265317376 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015839518555667002, + "loss": 1.3463, + "theoretical_loss": 3.395363075395296, + "tokens_seen": 2265448448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015837512537612838, + "loss": 1.2853, + "theoretical_loss": 3.395347095305176, + "tokens_seen": 2265579520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015835506519558677, + "loss": 1.4571, + "theoretical_loss": 3.3953311163983795, + "tokens_seen": 2265710592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015833500501504514, + "loss": 1.2914, + "theoretical_loss": 3.3953151386747513, + "tokens_seen": 2265841664 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.4001099765300751, + "objective/train/avg_token_score": 0.017674226313829422, + "objective/train/docs_used": 1283537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2806695699691772, + "objective/train/original_loss": 3.7781784534454346, + "objective/train/theoretical_loss": 3.3953071502565764, + "objective/train/tokens_used": 2265907200, + "objective/train/unlikelihood_loss": 0.18527039885520935, + "theoretical_loss": 3.3953071502565764, + "tokens_seen": 2265907200 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001583149448345035, + "loss": 1.3611, + "theoretical_loss": 3.395299162134135, + "tokens_seen": 2265972736 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001582948846539619, + "loss": 1.2671, + "theoretical_loss": 3.395283186776375, + "tokens_seen": 2266103808 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015827482447342025, + "loss": 1.2551, + "theoretical_loss": 3.395267212601315, + "tokens_seen": 2266234880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015825476429287864, + "loss": 1.1942, + "theoretical_loss": 3.395251239608799, + "tokens_seen": 2266365952 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158234704112337, + "loss": 1.3433, + "theoretical_loss": 3.3952352677986712, + "tokens_seen": 2266497024 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001582146439317954, + "loss": 1.2852, + "theoretical_loss": 3.3952192971707755, + "tokens_seen": 2266628096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015819458375125375, + "loss": 1.1977, + "theoretical_loss": 3.3952033277249565, + "tokens_seen": 2266759168 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015817452357071214, + "loss": 1.2493, + "theoretical_loss": 3.3951873594610578, + "tokens_seen": 2266890240 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001581544633901705, + "loss": 1.2587, + "theoretical_loss": 3.395171392378924, + "tokens_seen": 2267021312 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015813440320962887, + "loss": 1.1861, + "theoretical_loss": 3.395155426478399, + "tokens_seen": 2267152384 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015811434302908726, + "loss": 1.4082, + "theoretical_loss": 3.395139461759327, + "tokens_seen": 2267283456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015809428284854562, + "loss": 1.3167, + "theoretical_loss": 3.3951234982215532, + "tokens_seen": 2267414528 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.306512713432312, + "objective/train/avg_token_score": 0.03380957990884781, + "objective/train/docs_used": 1284224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8985409140586853, + "objective/train/original_loss": 4.5698723793029785, + "objective/train/theoretical_loss": 3.3951075358649208, + "objective/train/tokens_used": 2267545600, + "objective/train/unlikelihood_loss": 0.1777840107679367, + "theoretical_loss": 3.3951075358649208, + "tokens_seen": 2267545600 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158074222668004, + "loss": 1.3073, + "theoretical_loss": 3.3951075358649208, + "tokens_seen": 2267545600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015805416248746237, + "loss": 1.2214, + "theoretical_loss": 3.3950915746892747, + "tokens_seen": 2267676672 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015803410230692076, + "loss": 1.1705, + "theoretical_loss": 3.395075614694459, + "tokens_seen": 2267807744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015801404212637915, + "loss": 1.2901, + "theoretical_loss": 3.3950596558803183, + "tokens_seen": 2267938816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015799398194583754, + "loss": 1.238, + "theoretical_loss": 3.395043698246697, + "tokens_seen": 2268069888 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001579739217652959, + "loss": 1.3796, + "theoretical_loss": 3.39502774179344, + "tokens_seen": 2268200960 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001579538615847543, + "loss": 1.3253, + "theoretical_loss": 3.3950117865203904, + "tokens_seen": 2268332032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015793380140421265, + "loss": 1.4569, + "theoretical_loss": 3.3949958324273943, + "tokens_seen": 2268463104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015791374122367101, + "loss": 1.2345, + "theoretical_loss": 3.3949798795142954, + "tokens_seen": 2268594176 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001578936810431294, + "loss": 1.3555, + "theoretical_loss": 3.394963927780938, + "tokens_seen": 2268725248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015787362086258777, + "loss": 1.1882, + "theoretical_loss": 3.394947977227168, + "tokens_seen": 2268856320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015785356068204616, + "loss": 1.3151, + "theoretical_loss": 3.3949320278528283, + "tokens_seen": 2268987392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015783350050150452, + "loss": 1.465, + "theoretical_loss": 3.394916079657765, + "tokens_seen": 2269118464 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.32728493213653564, + "objective/train/avg_token_score": 0.01970667392015457, + "objective/train/docs_used": 1285503, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9133962988853455, + "objective/train/original_loss": 3.9655141830444336, + "objective/train/theoretical_loss": 3.394908106002413, + "objective/train/tokens_used": 2269184000, + "objective/train/unlikelihood_loss": 0.13970309495925903, + "theoretical_loss": 3.394908106002413, + "tokens_seen": 2269184000 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001578134403209629, + "loss": 1.2782, + "theoretical_loss": 3.394900132641822, + "tokens_seen": 2269249536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015779338014042127, + "loss": 1.1807, + "theoretical_loss": 3.3948841868048447, + "tokens_seen": 2269380608 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015777331995987966, + "loss": 1.1872, + "theoretical_loss": 3.394868242146677, + "tokens_seen": 2269511680 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015775325977933802, + "loss": 1.4122, + "theoretical_loss": 3.3948522986671645, + "tokens_seen": 2269642752 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015773319959879638, + "loss": 1.334, + "theoretical_loss": 3.3948363563661514, + "tokens_seen": 2269773824 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015771313941825477, + "loss": 1.3069, + "theoretical_loss": 3.394820415243483, + "tokens_seen": 2269904896 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015769307923771313, + "loss": 1.1601, + "theoretical_loss": 3.3948044752990034, + "tokens_seen": 2270035968 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015767301905717152, + "loss": 1.4015, + "theoretical_loss": 3.3947885365325585, + "tokens_seen": 2270167040 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015765295887662989, + "loss": 1.2708, + "theoretical_loss": 3.3947725989439927, + "tokens_seen": 2270298112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015763289869608828, + "loss": 1.1797, + "theoretical_loss": 3.394756662533151, + "tokens_seen": 2270429184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015761283851554664, + "loss": 1.3081, + "theoretical_loss": 3.394740727299878, + "tokens_seen": 2270560256 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015759277833500503, + "loss": 1.1947, + "theoretical_loss": 3.39472479324402, + "tokens_seen": 2270691328 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.5219941139221191, + "objective/train/avg_token_score": 0.008759761229157448, + "objective/train/docs_used": 1286076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5947712659835815, + "objective/train/original_loss": 3.716681718826294, + "objective/train/theoretical_loss": 3.3947088603654203, + "objective/train/tokens_used": 2270822400, + "objective/train/unlikelihood_loss": 0.15071004629135132, + "theoretical_loss": 3.3947088603654203, + "tokens_seen": 2270822400 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001575727181544634, + "loss": 1.4143, + "theoretical_loss": 3.3947088603654203, + "tokens_seen": 2270822400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015755265797392175, + "loss": 1.2574, + "theoretical_loss": 3.3946929286639254, + "tokens_seen": 2270953472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015753259779338014, + "loss": 1.3305, + "theoretical_loss": 3.3946769981393796, + "tokens_seen": 2271084544 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001575125376128385, + "loss": 1.3654, + "theoretical_loss": 3.3946610687916285, + "tokens_seen": 2271215616 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001574924774322969, + "loss": 1.2553, + "theoretical_loss": 3.3946451406205167, + "tokens_seen": 2271346688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015747241725175525, + "loss": 1.1552, + "theoretical_loss": 3.3946292136258904, + "tokens_seen": 2271477760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015745235707121364, + "loss": 1.2424, + "theoretical_loss": 3.3946132878075934, + "tokens_seen": 2271608832 + }, + { + "epoch": 0.69, + "learning_rate": 0.000157432296890672, + "loss": 1.2725, + "theoretical_loss": 3.3945973631654724, + "tokens_seen": 2271739904 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001574122367101304, + "loss": 1.4056, + "theoretical_loss": 3.3945814396993716, + "tokens_seen": 2271870976 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015739217652958876, + "loss": 1.2126, + "theoretical_loss": 3.394565517409137, + "tokens_seen": 2272002048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015737211634904715, + "loss": 1.1834, + "theoretical_loss": 3.3945495962946137, + "tokens_seen": 2272133120 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001573520561685055, + "loss": 1.1319, + "theoretical_loss": 3.394533676355647, + "tokens_seen": 2272264192 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015733199598796387, + "loss": 1.1757, + "theoretical_loss": 3.394517757592082, + "tokens_seen": 2272395264 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.4074413478374481, + "objective/train/avg_token_score": 0.0109179075807333, + "objective/train/docs_used": 1287335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.23031485080719, + "objective/train/original_loss": 3.9669594764709473, + "objective/train/theoretical_loss": 3.3945097986510273, + "objective/train/tokens_used": 2272460800, + "objective/train/unlikelihood_loss": 0.16893677413463593, + "theoretical_loss": 3.3945097986510273, + "tokens_seen": 2272460800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015731193580742226, + "loss": 1.3681, + "theoretical_loss": 3.3945018400037648, + "tokens_seen": 2272526336 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015729187562688062, + "loss": 1.4169, + "theoretical_loss": 3.3944859235905405, + "tokens_seen": 2272657408 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015727181544633904, + "loss": 1.0703, + "theoretical_loss": 3.394470008352254, + "tokens_seen": 2272788480 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001572517552657974, + "loss": 1.3046, + "theoretical_loss": 3.3944540942887524, + "tokens_seen": 2272919552 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001572316950852558, + "loss": 1.3586, + "theoretical_loss": 3.3944381813998796, + "tokens_seen": 2273050624 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015721163490471415, + "loss": 1.3671, + "theoretical_loss": 3.394422269685482, + "tokens_seen": 2273181696 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015719157472417254, + "loss": 1.3002, + "theoretical_loss": 3.394406359145405, + "tokens_seen": 2273312768 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001571715145436309, + "loss": 1.3295, + "theoretical_loss": 3.3943904497794946, + "tokens_seen": 2273443840 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015715145436308927, + "loss": 1.3775, + "theoretical_loss": 3.3943745415875957, + "tokens_seen": 2273574912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015713139418254766, + "loss": 1.2976, + "theoretical_loss": 3.3943586345695547, + "tokens_seen": 2273705984 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015711133400200602, + "loss": 1.0558, + "theoretical_loss": 3.394342728725217, + "tokens_seen": 2273837056 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001570912738214644, + "loss": 1.276, + "theoretical_loss": 3.394326824054428, + "tokens_seen": 2273968128 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.4348118305206299, + "objective/train/avg_token_score": 0.006308425217866898, + "objective/train/docs_used": 1288139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2677479982376099, + "objective/train/original_loss": 3.6132524013519287, + "objective/train/theoretical_loss": 3.394310920557034, + "objective/train/tokens_used": 2274099200, + "objective/train/unlikelihood_loss": 0.1992749571800232, + "theoretical_loss": 3.394310920557034, + "tokens_seen": 2274099200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015707121364092277, + "loss": 1.2963, + "theoretical_loss": 3.394310920557034, + "tokens_seen": 2274099200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015705115346038116, + "loss": 1.2768, + "theoretical_loss": 3.394295018232881, + "tokens_seen": 2274230272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015703109327983952, + "loss": 1.2814, + "theoretical_loss": 3.3942791170818145, + "tokens_seen": 2274361344 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001570110330992979, + "loss": 1.2968, + "theoretical_loss": 3.3942632171036804, + "tokens_seen": 2274492416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015699097291875627, + "loss": 1.3423, + "theoretical_loss": 3.3942473182983246, + "tokens_seen": 2274623488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015697091273821466, + "loss": 1.3031, + "theoretical_loss": 3.3942314206655926, + "tokens_seen": 2274754560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015695085255767303, + "loss": 1.3213, + "theoretical_loss": 3.394215524205331, + "tokens_seen": 2274885632 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001569307923771314, + "loss": 1.2336, + "theoretical_loss": 3.3941996289173857, + "tokens_seen": 2275016704 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015691073219658978, + "loss": 1.4513, + "theoretical_loss": 3.3941837348016026, + "tokens_seen": 2275147776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015689067201604814, + "loss": 1.3687, + "theoretical_loss": 3.3941678418578274, + "tokens_seen": 2275278848 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015687061183550653, + "loss": 1.3819, + "theoretical_loss": 3.3941519500859063, + "tokens_seen": 2275409920 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001568505516549649, + "loss": 1.2918, + "theoretical_loss": 3.394136059485686, + "tokens_seen": 2275540992 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015683049147442328, + "loss": 1.1577, + "theoretical_loss": 3.394120170057012, + "tokens_seen": 2275672064 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.41385629773139954, + "objective/train/avg_token_score": 0.02416197955608368, + "objective/train/docs_used": 1288752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2829045057296753, + "objective/train/original_loss": 4.160581588745117, + "objective/train/theoretical_loss": 3.394112225781957, + "objective/train/tokens_used": 2275737600, + "objective/train/unlikelihood_loss": 0.1890370100736618, + "theoretical_loss": 3.394112225781957, + "tokens_seen": 2275737600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015681043129388164, + "loss": 1.2136, + "theoretical_loss": 3.3941042817997307, + "tokens_seen": 2275803136 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015679037111334003, + "loss": 1.2113, + "theoretical_loss": 3.394088394713688, + "tokens_seen": 2275934208 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001567703109327984, + "loss": 1.2063, + "theoretical_loss": 3.3940725087987307, + "tokens_seen": 2276065280 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015675025075225676, + "loss": 1.3087, + "theoretical_loss": 3.3940566240547043, + "tokens_seen": 2276196352 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015673019057171515, + "loss": 1.2504, + "theoretical_loss": 3.3940407404814557, + "tokens_seen": 2276327424 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001567101303911735, + "loss": 1.1234, + "theoretical_loss": 3.394024858078831, + "tokens_seen": 2276458496 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001566900702106319, + "loss": 1.2927, + "theoretical_loss": 3.394008976846676, + "tokens_seen": 2276589568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015667001003009026, + "loss": 1.2239, + "theoretical_loss": 3.393993096784838, + "tokens_seen": 2276720640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015664994984954865, + "loss": 1.2185, + "theoretical_loss": 3.393977217893163, + "tokens_seen": 2276851712 + }, + { + "epoch": 0.69, + "learning_rate": 0.000156629889669007, + "loss": 1.1959, + "theoretical_loss": 3.393961340171497, + "tokens_seen": 2276982784 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001566098294884654, + "loss": 1.1577, + "theoretical_loss": 3.393945463619687, + "tokens_seen": 2277113856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015658976930792376, + "loss": 1.2336, + "theoretical_loss": 3.3939295882375795, + "tokens_seen": 2277244928 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.39821603894233704, + "objective/train/avg_token_score": 0.019360875710844994, + "objective/train/docs_used": 1289790, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2406386137008667, + "objective/train/original_loss": 3.9693188667297363, + "objective/train/theoretical_loss": 3.393913714025021, + "objective/train/tokens_used": 2277376000, + "objective/train/unlikelihood_loss": 0.16577592492103577, + "theoretical_loss": 3.393913714025021, + "tokens_seen": 2277376000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015656970912738212, + "loss": 1.0606, + "theoretical_loss": 3.393913714025021, + "tokens_seen": 2277376000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015654964894684051, + "loss": 1.2797, + "theoretical_loss": 3.393897840981857, + "tokens_seen": 2277507072 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001565295887662989, + "loss": 1.2734, + "theoretical_loss": 3.3938819691079356, + "tokens_seen": 2277638144 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001565095285857573, + "loss": 1.4634, + "theoretical_loss": 3.393866098403102, + "tokens_seen": 2277769216 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015648946840521565, + "loss": 0.9412, + "theoretical_loss": 3.393850228867204, + "tokens_seen": 2277900288 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015646940822467404, + "loss": 1.2821, + "theoretical_loss": 3.3938343605000876, + "tokens_seen": 2278031360 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001564493480441324, + "loss": 1.2541, + "theoretical_loss": 3.3938184933015996, + "tokens_seen": 2278162432 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001564292878635908, + "loss": 1.1962, + "theoretical_loss": 3.3938026272715867, + "tokens_seen": 2278293504 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015640922768304916, + "loss": 1.2824, + "theoretical_loss": 3.393786762409896, + "tokens_seen": 2278424576 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015638916750250755, + "loss": 1.3624, + "theoretical_loss": 3.3937708987163733, + "tokens_seen": 2278555648 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001563691073219659, + "loss": 1.1213, + "theoretical_loss": 3.393755036190867, + "tokens_seen": 2278686720 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015634904714142427, + "loss": 1.2363, + "theoretical_loss": 3.393739174833222, + "tokens_seen": 2278817792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015632898696088266, + "loss": 1.3565, + "theoretical_loss": 3.393723314643286, + "tokens_seen": 2278948864 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.4977394938468933, + "objective/train/avg_token_score": 0.012306621298193932, + "objective/train/docs_used": 1290345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4051812887191772, + "objective/train/original_loss": 3.4367287158966064, + "objective/train/theoretical_loss": 3.393715384986162, + "objective/train/tokens_used": 2279014400, + "objective/train/unlikelihood_loss": 0.19915375113487244, + "theoretical_loss": 3.393715384986162, + "tokens_seen": 2279014400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015630892678034102, + "loss": 1.204, + "theoretical_loss": 3.393707455620907, + "tokens_seen": 2279079936 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001562888665997994, + "loss": 1.3291, + "theoretical_loss": 3.3936915977659305, + "tokens_seen": 2279211008 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015626880641925778, + "loss": 1.1225, + "theoretical_loss": 3.3936757410782032, + "tokens_seen": 2279342080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015624874623871616, + "loss": 1.1999, + "theoretical_loss": 3.3936598855575735, + "tokens_seen": 2279473152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015622868605817453, + "loss": 1.2367, + "theoretical_loss": 3.393644031203887, + "tokens_seen": 2279604224 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015620862587763292, + "loss": 1.3238, + "theoretical_loss": 3.393628178016992, + "tokens_seen": 2279735296 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015618856569709128, + "loss": 1.1403, + "theoretical_loss": 3.393612325996734, + "tokens_seen": 2279866368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015616850551654964, + "loss": 1.2064, + "theoretical_loss": 3.3935964751429615, + "tokens_seen": 2279997440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015614844533600803, + "loss": 1.2217, + "theoretical_loss": 3.393580625455521, + "tokens_seen": 2280128512 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001561283851554664, + "loss": 1.2135, + "theoretical_loss": 3.3935647769342596, + "tokens_seen": 2280259584 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015610832497492478, + "loss": 1.354, + "theoretical_loss": 3.393548929579025, + "tokens_seen": 2280390656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015608826479438314, + "loss": 1.2811, + "theoretical_loss": 3.3935330833896638, + "tokens_seen": 2280521728 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.28286901116371155, + "objective/train/avg_token_score": 0.023816438391804695, + "objective/train/docs_used": 1291619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.892422080039978, + "objective/train/original_loss": 4.1542277336120605, + "objective/train/theoretical_loss": 3.393517238366023, + "objective/train/tokens_used": 2280652800, + "objective/train/unlikelihood_loss": 0.16759559512138367, + "theoretical_loss": 3.393517238366023, + "tokens_seen": 2280652800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015606820461384153, + "loss": 1.1759, + "theoretical_loss": 3.393517238366023, + "tokens_seen": 2280652800 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001560481444332999, + "loss": 1.259, + "theoretical_loss": 3.3935013945079504, + "tokens_seen": 2280783872 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015602808425275828, + "loss": 1.1599, + "theoretical_loss": 3.3934855518152935, + "tokens_seen": 2280914944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015600802407221665, + "loss": 1.1494, + "theoretical_loss": 3.393469710287899, + "tokens_seen": 2281046016 + }, + { + "epoch": 0.69, + "learning_rate": 0.000155987963891675, + "loss": 1.1286, + "theoretical_loss": 3.3934538699256143, + "tokens_seen": 2281177088 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001559679037111334, + "loss": 1.238, + "theoretical_loss": 3.393438030728287, + "tokens_seen": 2281308160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015594784353059176, + "loss": 1.3027, + "theoretical_loss": 3.3934221926957653, + "tokens_seen": 2281439232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015592778335005015, + "loss": 1.3049, + "theoretical_loss": 3.393406355827895, + "tokens_seen": 2281570304 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001559077231695085, + "loss": 1.3455, + "theoretical_loss": 3.3933905201245245, + "tokens_seen": 2281701376 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001558876629889669, + "loss": 1.4095, + "theoretical_loss": 3.3933746855855014, + "tokens_seen": 2281832448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015586760280842526, + "loss": 1.3072, + "theoretical_loss": 3.393358852210673, + "tokens_seen": 2281963520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015584754262788365, + "loss": 1.221, + "theoretical_loss": 3.393343019999886, + "tokens_seen": 2282094592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015582748244734202, + "loss": 1.3229, + "theoretical_loss": 3.3933271889529895, + "tokens_seen": 2282225664 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.37212854623794556, + "objective/train/avg_token_score": 0.02387051470577717, + "objective/train/docs_used": 1292123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1078745126724243, + "objective/train/original_loss": 4.084591865539551, + "objective/train/theoretical_loss": 3.393319273865952, + "objective/train/tokens_used": 2282291200, + "objective/train/unlikelihood_loss": 0.12449561804533005, + "theoretical_loss": 3.393319273865952, + "tokens_seen": 2282291200 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001558074222668004, + "loss": 1.2663, + "theoretical_loss": 3.3933113590698305, + "tokens_seen": 2282356736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015578736208625877, + "loss": 1.305, + "theoretical_loss": 3.393295530350256, + "tokens_seen": 2282487808 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015576730190571716, + "loss": 1.2655, + "theoretical_loss": 3.3932797027941146, + "tokens_seen": 2282618880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015574724172517555, + "loss": 1.3152, + "theoretical_loss": 3.393263876401253, + "tokens_seen": 2282749952 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001557271815446339, + "loss": 1.1681, + "theoretical_loss": 3.39324805117152, + "tokens_seen": 2282881024 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001557071213640923, + "loss": 1.2745, + "theoretical_loss": 3.393232227104763, + "tokens_seen": 2283012096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015568706118355066, + "loss": 1.2937, + "theoretical_loss": 3.393216404200829, + "tokens_seen": 2283143168 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015566700100300905, + "loss": 1.1902, + "theoretical_loss": 3.3932005824595666, + "tokens_seen": 2283274240 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001556469408224674, + "loss": 1.3406, + "theoretical_loss": 3.3931847618808235, + "tokens_seen": 2283405312 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001556268806419258, + "loss": 1.3483, + "theoretical_loss": 3.393168942464447, + "tokens_seen": 2283536384 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015560682046138416, + "loss": 1.2823, + "theoretical_loss": 3.3931531242102864, + "tokens_seen": 2283667456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015558676028084252, + "loss": 1.3481, + "theoretical_loss": 3.393137307118188, + "tokens_seen": 2283798528 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.5161290168762207, + "objective/train/avg_token_score": 0.03730396553874016, + "objective/train/docs_used": 1293247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8039666414260864, + "objective/train/original_loss": 3.97208571434021, + "objective/train/theoretical_loss": 3.3931214911880008, + "objective/train/tokens_used": 2283929600, + "objective/train/unlikelihood_loss": 0.14698567986488342, + "theoretical_loss": 3.3931214911880008, + "tokens_seen": 2283929600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015556670010030091, + "loss": 1.3308, + "theoretical_loss": 3.3931214911880008, + "tokens_seen": 2283929600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015554663991975928, + "loss": 1.2179, + "theoretical_loss": 3.393105676419572, + "tokens_seen": 2284060672 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015552657973921767, + "loss": 1.1962, + "theoretical_loss": 3.3930898628127504, + "tokens_seen": 2284191744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015550651955867603, + "loss": 1.4125, + "theoretical_loss": 3.3930740503673835, + "tokens_seen": 2284322816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015548645937813442, + "loss": 1.3192, + "theoretical_loss": 3.3930582390833193, + "tokens_seen": 2284453888 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015546639919759278, + "loss": 1.3023, + "theoretical_loss": 3.393042428960406, + "tokens_seen": 2284584960 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015544633901705117, + "loss": 1.3162, + "theoretical_loss": 3.3930266199984924, + "tokens_seen": 2284716032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015542627883650953, + "loss": 1.2406, + "theoretical_loss": 3.393010812197425, + "tokens_seen": 2284847104 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001554062186559679, + "loss": 1.3326, + "theoretical_loss": 3.3929950055570544, + "tokens_seen": 2284978176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015538615847542628, + "loss": 1.1496, + "theoretical_loss": 3.3929792000772263, + "tokens_seen": 2285109248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015536609829488465, + "loss": 1.1815, + "theoretical_loss": 3.3929633957577905, + "tokens_seen": 2285240320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015534603811434303, + "loss": 1.1984, + "theoretical_loss": 3.392947592598595, + "tokens_seen": 2285371392 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001553259779338014, + "loss": 1.2495, + "theoretical_loss": 3.3929317905994876, + "tokens_seen": 2285502464 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.3415811359882355, + "objective/train/avg_token_score": 0.01826086826622486, + "objective/train/docs_used": 1293760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.008630394935608, + "objective/train/original_loss": 3.8879079818725586, + "objective/train/theoretical_loss": 3.3929238900349197, + "objective/train/tokens_used": 2285568000, + "objective/train/unlikelihood_loss": 0.24365819990634918, + "theoretical_loss": 3.3929238900349197, + "tokens_seen": 2285568000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015530591775325979, + "loss": 1.2369, + "theoretical_loss": 3.3929159897603167, + "tokens_seen": 2285633536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015528585757271815, + "loss": 1.4772, + "theoretical_loss": 3.3929001900809315, + "tokens_seen": 2285764608 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015526579739217654, + "loss": 1.1975, + "theoretical_loss": 3.392884391561179, + "tokens_seen": 2285895680 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001552457372116349, + "loss": 1.2212, + "theoretical_loss": 3.392868594200909, + "tokens_seen": 2286026752 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001552256770310933, + "loss": 1.1618, + "theoretical_loss": 3.3928527979999688, + "tokens_seen": 2286157824 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015520561685055165, + "loss": 1.392, + "theoretical_loss": 3.3928370029582076, + "tokens_seen": 2286288896 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015518555667001001, + "loss": 1.187, + "theoretical_loss": 3.3928212090754735, + "tokens_seen": 2286419968 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001551654964894684, + "loss": 1.3856, + "theoretical_loss": 3.3928054163516155, + "tokens_seen": 2286551040 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015514543630892677, + "loss": 1.2295, + "theoretical_loss": 3.3927896247864813, + "tokens_seen": 2286682112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015512537612838515, + "loss": 1.1165, + "theoretical_loss": 3.39277383437992, + "tokens_seen": 2286813184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015510531594784352, + "loss": 1.5195, + "theoretical_loss": 3.39275804513178, + "tokens_seen": 2286944256 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001550852557673019, + "loss": 1.2402, + "theoretical_loss": 3.3927422570419106, + "tokens_seen": 2287075328 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.44855815172195435, + "objective/train/avg_token_score": 0.014770084992051125, + "objective/train/docs_used": 1294930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4360640048980713, + "objective/train/original_loss": 3.890127182006836, + "objective/train/theoretical_loss": 3.3927264701101594, + "objective/train/tokens_used": 2287206400, + "objective/train/unlikelihood_loss": 0.14880019426345825, + "theoretical_loss": 3.3927264701101594, + "tokens_seen": 2287206400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015506519558676027, + "loss": 1.2245, + "theoretical_loss": 3.3927264701101594, + "tokens_seen": 2287206400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015504513540621866, + "loss": 1.5046, + "theoretical_loss": 3.392710684336376, + "tokens_seen": 2287337472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015502507522567705, + "loss": 1.2773, + "theoretical_loss": 3.3926948997204085, + "tokens_seen": 2287468544 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001550050150451354, + "loss": 1.3277, + "theoretical_loss": 3.392679116262106, + "tokens_seen": 2287599616 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001549849548645938, + "loss": 1.2928, + "theoretical_loss": 3.392663333961317, + "tokens_seen": 2287730688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015496489468405216, + "loss": 1.2928, + "theoretical_loss": 3.3926475528178908, + "tokens_seen": 2287861760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015494483450351055, + "loss": 1.2115, + "theoretical_loss": 3.3926317728316757, + "tokens_seen": 2287992832 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001549247743229689, + "loss": 1.3244, + "theoretical_loss": 3.3926159940025205, + "tokens_seen": 2288123904 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001549047141424273, + "loss": 1.3738, + "theoretical_loss": 3.392600216330275, + "tokens_seen": 2288254976 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015488465396188566, + "loss": 1.3583, + "theoretical_loss": 3.3925844398147866, + "tokens_seen": 2288386048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015486459378134405, + "loss": 1.2612, + "theoretical_loss": 3.3925686644559057, + "tokens_seen": 2288517120 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015484453360080242, + "loss": 1.3949, + "theoretical_loss": 3.39255289025348, + "tokens_seen": 2288648192 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001548244734202608, + "loss": 1.1806, + "theoretical_loss": 3.3925371172073597, + "tokens_seen": 2288779264 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.40139296650886536, + "objective/train/avg_token_score": 0.010853451676666737, + "objective/train/docs_used": 1295606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1416370868682861, + "objective/train/original_loss": 3.7313756942749023, + "objective/train/theoretical_loss": 3.3925292311178668, + "objective/train/tokens_used": 2288844800, + "objective/train/unlikelihood_loss": 0.17426209151744843, + "theoretical_loss": 3.3925292311178668, + "tokens_seen": 2288844800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015480441323971917, + "loss": 1.2948, + "theoretical_loss": 3.3925213453173932, + "tokens_seen": 2288910336 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015478435305917753, + "loss": 1.2514, + "theoretical_loss": 3.3925055745834296, + "tokens_seen": 2289041408 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015476429287863592, + "loss": 1.3779, + "theoretical_loss": 3.3924898050053183, + "tokens_seen": 2289172480 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015474423269809428, + "loss": 1.3536, + "theoretical_loss": 3.392474036582908, + "tokens_seen": 2289303552 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015472417251755267, + "loss": 1.1954, + "theoretical_loss": 3.3924582693160477, + "tokens_seen": 2289434624 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015470411233701103, + "loss": 1.3898, + "theoretical_loss": 3.392442503204587, + "tokens_seen": 2289565696 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015468405215646942, + "loss": 1.2693, + "theoretical_loss": 3.392426738248375, + "tokens_seen": 2289696768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015466399197592778, + "loss": 1.3999, + "theoretical_loss": 3.3924109744472606, + "tokens_seen": 2289827840 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015464393179538617, + "loss": 1.2508, + "theoretical_loss": 3.3923952118010936, + "tokens_seen": 2289958912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015462387161484454, + "loss": 1.3709, + "theoretical_loss": 3.392379450309723, + "tokens_seen": 2290089984 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001546038114343029, + "loss": 1.2537, + "theoretical_loss": 3.392363689972998, + "tokens_seen": 2290221056 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545837512537613, + "loss": 1.2884, + "theoretical_loss": 3.392347930790768, + "tokens_seen": 2290352128 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.35581621527671814, + "objective/train/avg_token_score": 0.015282261185348034, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1861110925674438, + "objective/train/original_loss": 4.165960311889648, + "objective/train/theoretical_loss": 3.392332172762883, + "objective/train/tokens_used": 2290483200, + "objective/train/unlikelihood_loss": 0.1550193727016449, + "theoretical_loss": 3.392332172762883, + "tokens_seen": 2290483200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015456369107321965, + "loss": 1.3196, + "theoretical_loss": 3.392332172762883, + "tokens_seen": 2290483200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015454363089267804, + "loss": 1.3288, + "theoretical_loss": 3.3923164158891908, + "tokens_seen": 2290614272 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545235707121364, + "loss": 1.3394, + "theoretical_loss": 3.3923006601695427, + "tokens_seen": 2290745344 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545035105315948, + "loss": 1.2416, + "theoretical_loss": 3.3922849056037867, + "tokens_seen": 2290876416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015448345035105315, + "loss": 1.2124, + "theoretical_loss": 3.3922691521917736, + "tokens_seen": 2291007488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015446339017051154, + "loss": 1.2804, + "theoretical_loss": 3.3922533999333515, + "tokens_seen": 2291138560 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001544433299899699, + "loss": 1.3638, + "theoretical_loss": 3.392237648828371, + "tokens_seen": 2291269632 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015442326980942827, + "loss": 1.2293, + "theoretical_loss": 3.3922218988766812, + "tokens_seen": 2291400704 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015440320962888666, + "loss": 1.2912, + "theoretical_loss": 3.3922061500781315, + "tokens_seen": 2291531776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015438314944834502, + "loss": 1.3158, + "theoretical_loss": 3.392190402432572, + "tokens_seen": 2291662848 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001543630892678034, + "loss": 1.2349, + "theoretical_loss": 3.3921746559398525, + "tokens_seen": 2291793920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015434302908726177, + "loss": 1.1386, + "theoretical_loss": 3.392158910599822, + "tokens_seen": 2291924992 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015432296890672016, + "loss": 1.3139, + "theoretical_loss": 3.3921431664123305, + "tokens_seen": 2292056064 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.32508552074432373, + "objective/train/avg_token_score": 0.01706630550324917, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9923251867294312, + "objective/train/original_loss": 4.053013324737549, + "objective/train/theoretical_loss": 3.39213529475074, + "objective/train/tokens_used": 2292121600, + "objective/train/unlikelihood_loss": 0.14606963098049164, + "theoretical_loss": 3.39213529475074, + "tokens_seen": 2292121600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015430290872617852, + "loss": 1.3089, + "theoretical_loss": 3.392127423377228, + "tokens_seen": 2292187136 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001542828485456369, + "loss": 1.0561, + "theoretical_loss": 3.392111681494364, + "tokens_seen": 2292318208 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001542627883650953, + "loss": 1.2185, + "theoretical_loss": 3.3920959407635882, + "tokens_seen": 2292449280 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001542427281845537, + "loss": 1.3724, + "theoretical_loss": 3.3920802011847506, + "tokens_seen": 2292580352 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015422266800401205, + "loss": 1.3135, + "theoretical_loss": 3.392064462757701, + "tokens_seen": 2292711424 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015420260782347041, + "loss": 1.4327, + "theoretical_loss": 3.3920487254822893, + "tokens_seen": 2292842496 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001541825476429288, + "loss": 1.3045, + "theoretical_loss": 3.392032989358365, + "tokens_seen": 2292973568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015416248746238717, + "loss": 1.3306, + "theoretical_loss": 3.3920172543857787, + "tokens_seen": 2293104640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015414242728184555, + "loss": 1.2164, + "theoretical_loss": 3.39200152056438, + "tokens_seen": 2293235712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015412236710130392, + "loss": 1.2097, + "theoretical_loss": 3.3919857878940194, + "tokens_seen": 2293366784 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001541023069207623, + "loss": 1.4924, + "theoretical_loss": 3.3919700563745456, + "tokens_seen": 2293497856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015408224674022067, + "loss": 1.4293, + "theoretical_loss": 3.39195432600581, + "tokens_seen": 2293628928 + }, + { + "debugging/length": 477.3333333333333, + "debugging/num_segments": 3, + "epoch": 0.7, + "objective/train/aligned_token_count": 0.5938416123390198, + "objective/train/avg_token_score": 0.012000557966530323, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8890172243118286, + "objective/train/original_loss": 3.8090944290161133, + "objective/train/theoretical_loss": 3.3919385967876625, + "objective/train/tokens_used": 2293760000, + "objective/train/unlikelihood_loss": 0.11794061213731766, + "theoretical_loss": 3.3919385967876625, + "tokens_seen": 2293760000 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015406218655967906, + "loss": 1.4083, + "theoretical_loss": 3.3919385967876625, + "tokens_seen": 2293760000 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015404212637913742, + "loss": 1.3769, + "theoretical_loss": 3.3919228687199525, + "tokens_seen": 2293891072 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015402206619859578, + "loss": 1.245, + "theoretical_loss": 3.3919071418025304, + "tokens_seen": 2294022144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015400200601805417, + "loss": 1.1833, + "theoretical_loss": 3.391891416035247, + "tokens_seen": 2294153216 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015398194583751253, + "loss": 1.3976, + "theoretical_loss": 3.3918756914179515, + "tokens_seen": 2294284288 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015396188565697092, + "loss": 1.3864, + "theoretical_loss": 3.3918599679504946, + "tokens_seen": 2294415360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015394182547642929, + "loss": 1.342, + "theoretical_loss": 3.3918442456327265, + "tokens_seen": 2294546432 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015392176529588768, + "loss": 1.379, + "theoretical_loss": 3.3918285244644975, + "tokens_seen": 2294677504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015390170511534604, + "loss": 1.2739, + "theoretical_loss": 3.391812804445658, + "tokens_seen": 2294808576 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015388164493480443, + "loss": 1.2072, + "theoretical_loss": 3.391797085576058, + "tokens_seen": 2294939648 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001538615847542628, + "loss": 1.2644, + "theoretical_loss": 3.391781367855548, + "tokens_seen": 2295070720 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015384152457372115, + "loss": 1.2329, + "theoretical_loss": 3.3917656512839787, + "tokens_seen": 2295201792 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015382146439317954, + "loss": 1.4549, + "theoretical_loss": 3.3917499358612, + "tokens_seen": 2295332864 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.3189760446548462, + "objective/train/avg_token_score": 0.011348468251526356, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1041885614395142, + "objective/train/original_loss": 3.8123679161071777, + "objective/train/theoretical_loss": 3.391742078580561, + "objective/train/tokens_used": 2295398400, + "objective/train/unlikelihood_loss": 0.17808495461940765, + "theoretical_loss": 3.391742078580561, + "tokens_seen": 2295398400 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001538014042126379, + "loss": 1.2783, + "theoretical_loss": 3.391734221587063, + "tokens_seen": 2295463936 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537813440320963, + "loss": 1.4617, + "theoretical_loss": 3.3917185084614174, + "tokens_seen": 2295595008 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015376128385155465, + "loss": 1.3435, + "theoretical_loss": 3.391702796484114, + "tokens_seen": 2295726080 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015374122367101304, + "loss": 1.4182, + "theoretical_loss": 3.3916870856550037, + "tokens_seen": 2295857152 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537211634904714, + "loss": 1.2913, + "theoretical_loss": 3.391671375973936, + "tokens_seen": 2295988224 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537011033099298, + "loss": 1.292, + "theoretical_loss": 3.391655667440763, + "tokens_seen": 2296119296 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015368104312938816, + "loss": 1.3783, + "theoretical_loss": 3.391639960055334, + "tokens_seen": 2296250368 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015366098294884655, + "loss": 1.379, + "theoretical_loss": 3.3916242538175005, + "tokens_seen": 2296381440 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001536409227683049, + "loss": 1.2971, + "theoretical_loss": 3.3916085487271124, + "tokens_seen": 2296512512 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015362086258776327, + "loss": 1.2402, + "theoretical_loss": 3.3915928447840207, + "tokens_seen": 2296643584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015360080240722166, + "loss": 1.2628, + "theoretical_loss": 3.3915771419880762, + "tokens_seen": 2296774656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015358074222668002, + "loss": 1.2856, + "theoretical_loss": 3.39156144033913, + "tokens_seen": 2296905728 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.39943793416023254, + "objective/train/avg_token_score": 0.016543062403798103, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3580049276351929, + "objective/train/original_loss": 4.0356550216674805, + "objective/train/theoretical_loss": 3.3915457398370323, + "objective/train/tokens_used": 2297036800, + "objective/train/unlikelihood_loss": 0.16320449113845825, + "theoretical_loss": 3.3915457398370323, + "tokens_seen": 2297036800 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001535606820461384, + "loss": 1.2728, + "theoretical_loss": 3.3915457398370323, + "tokens_seen": 2297036800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015354062186559677, + "loss": 1.2755, + "theoretical_loss": 3.3915300404816344, + "tokens_seen": 2297167872 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001535205616850552, + "loss": 1.2894, + "theoretical_loss": 3.391514342272786, + "tokens_seen": 2297298944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015350050150451355, + "loss": 1.2113, + "theoretical_loss": 3.3914986452103397, + "tokens_seen": 2297430016 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015348044132397194, + "loss": 1.1411, + "theoretical_loss": 3.391482949294145, + "tokens_seen": 2297561088 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001534603811434303, + "loss": 1.1712, + "theoretical_loss": 3.3914672545240534, + "tokens_seen": 2297692160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015344032096288867, + "loss": 1.155, + "theoretical_loss": 3.391451560899916, + "tokens_seen": 2297823232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015342026078234706, + "loss": 1.1266, + "theoretical_loss": 3.391435868421583, + "tokens_seen": 2297954304 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015340020060180542, + "loss": 1.1913, + "theoretical_loss": 3.391420177088906, + "tokens_seen": 2298085376 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001533801404212638, + "loss": 1.2238, + "theoretical_loss": 3.391404486901736, + "tokens_seen": 2298216448 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015336008024072217, + "loss": 1.0721, + "theoretical_loss": 3.391388797859924, + "tokens_seen": 2298347520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015334002006018056, + "loss": 1.2181, + "theoretical_loss": 3.391373109963321, + "tokens_seen": 2298478592 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015331995987963892, + "loss": 1.2412, + "theoretical_loss": 3.3913574232117782, + "tokens_seen": 2298609664 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.2785312831401825, + "objective/train/avg_token_score": 0.016774887219071388, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9727440476417542, + "objective/train/original_loss": 4.828059673309326, + "objective/train/theoretical_loss": 3.3913495802653575, + "objective/train/tokens_used": 2298675200, + "objective/train/unlikelihood_loss": 0.12017910182476044, + "theoretical_loss": 3.3913495802653575, + "tokens_seen": 2298675200 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001532998996990973, + "loss": 1.1756, + "theoretical_loss": 3.3913417376051465, + "tokens_seen": 2298740736 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015327983951855567, + "loss": 1.215, + "theoretical_loss": 3.3913260531432776, + "tokens_seen": 2298871808 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015325977933801406, + "loss": 1.3479, + "theoretical_loss": 3.391310369826022, + "tokens_seen": 2299002880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015323971915747243, + "loss": 1.0875, + "theoretical_loss": 3.3912946876532315, + "tokens_seen": 2299133952 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001532196589769308, + "loss": 1.0671, + "theoretical_loss": 3.3912790066247567, + "tokens_seen": 2299265024 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015319959879638918, + "loss": 1.0464, + "theoretical_loss": 3.3912633267404497, + "tokens_seen": 2299396096 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015317953861584754, + "loss": 0.9929, + "theoretical_loss": 3.3912476480001614, + "tokens_seen": 2299527168 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015315947843530593, + "loss": 0.9807, + "theoretical_loss": 3.391231970403743, + "tokens_seen": 2299658240 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001531394182547643, + "loss": 1.3145, + "theoretical_loss": 3.3912162939510457, + "tokens_seen": 2299789312 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015311935807422268, + "loss": 1.0757, + "theoretical_loss": 3.3912006186419217, + "tokens_seen": 2299920384 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015309929789368104, + "loss": 1.071, + "theoretical_loss": 3.3911849444762217, + "tokens_seen": 2300051456 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015307923771313943, + "loss": 1.149, + "theoretical_loss": 3.391169271453797, + "tokens_seen": 2300182528 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.3147605061531067, + "objective/train/avg_token_score": 0.01919655315577984, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0847728252410889, + "objective/train/original_loss": 4.208125114440918, + "objective/train/theoretical_loss": 3.3911535995744995, + "objective/train/tokens_used": 2300313600, + "objective/train/unlikelihood_loss": 0.1455707848072052, + "theoretical_loss": 3.3911535995744995, + "tokens_seen": 2300313600 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001530591775325978, + "loss": 1.0089, + "theoretical_loss": 3.3911535995744995, + "tokens_seen": 2300313600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015303911735205616, + "loss": 1.2732, + "theoretical_loss": 3.3911379288381807, + "tokens_seen": 2300444672 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015301905717151455, + "loss": 1.0252, + "theoretical_loss": 3.391122259244692, + "tokens_seen": 2300575744 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001529989969909729, + "loss": 1.2289, + "theoretical_loss": 3.3911065907938847, + "tokens_seen": 2300706816 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001529789368104313, + "loss": 1.0846, + "theoretical_loss": 3.391090923485611, + "tokens_seen": 2300837888 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015295887662988966, + "loss": 1.0233, + "theoretical_loss": 3.391075257319722, + "tokens_seen": 2300968960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015293881644934805, + "loss": 1.1637, + "theoretical_loss": 3.3910595922960693, + "tokens_seen": 2301100032 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001529187562688064, + "loss": 1.149, + "theoretical_loss": 3.391043928414505, + "tokens_seen": 2301231104 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001528986960882648, + "loss": 1.3313, + "theoretical_loss": 3.3910282656748807, + "tokens_seen": 2301362176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015287863590772316, + "loss": 1.1213, + "theoretical_loss": 3.3910126040770474, + "tokens_seen": 2301493248 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015285857572718152, + "loss": 1.1031, + "theoretical_loss": 3.390996943620858, + "tokens_seen": 2301624320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015283851554663991, + "loss": 1.1092, + "theoretical_loss": 3.390981284306163, + "tokens_seen": 2301755392 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015281845536609828, + "loss": 1.1228, + "theoretical_loss": 3.3909656261328154, + "tokens_seen": 2301886464 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.4206378161907196, + "objective/train/avg_token_score": 0.006139714270830154, + "objective/train/docs_used": 1297386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4460867643356323, + "objective/train/original_loss": 3.8704869747161865, + "objective/train/theoretical_loss": 3.3909577974741003, + "objective/train/tokens_used": 2301952000, + "objective/train/unlikelihood_loss": 0.1468256115913391, + "theoretical_loss": 3.3909577974741003, + "tokens_seen": 2301952000 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015279839518555667, + "loss": 1.1996, + "theoretical_loss": 3.390949969100666, + "tokens_seen": 2302017536 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015277833500501503, + "loss": 1.3005, + "theoretical_loss": 3.3909343132095673, + "tokens_seen": 2302148608 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015275827482447344, + "loss": 1.226, + "theoretical_loss": 3.3909186584593716, + "tokens_seen": 2302279680 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001527382146439318, + "loss": 0.934, + "theoretical_loss": 3.3909030048499296, + "tokens_seen": 2302410752 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001527181544633902, + "loss": 1.0482, + "theoretical_loss": 3.3908873523810943, + "tokens_seen": 2302541824 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015269809428284856, + "loss": 1.1551, + "theoretical_loss": 3.390871701052717, + "tokens_seen": 2302672896 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015267803410230695, + "loss": 1.0932, + "theoretical_loss": 3.3908560508646497, + "tokens_seen": 2302803968 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001526579739217653, + "loss": 1.1576, + "theoretical_loss": 3.390840401816745, + "tokens_seen": 2302935040 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015263791374122367, + "loss": 1.1309, + "theoretical_loss": 3.3908247539088547, + "tokens_seen": 2303066112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015261785356068206, + "loss": 1.208, + "theoretical_loss": 3.390809107140831, + "tokens_seen": 2303197184 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015259779338014042, + "loss": 1.1744, + "theoretical_loss": 3.3907934615125255, + "tokens_seen": 2303328256 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001525777331995988, + "loss": 1.0488, + "theoretical_loss": 3.3907778170237908, + "tokens_seen": 2303459328 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.27718719840049744, + "objective/train/avg_token_score": 0.034820862114429474, + "objective/train/docs_used": 1298119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8426401615142822, + "objective/train/original_loss": 4.728061676025391, + "objective/train/theoretical_loss": 3.390762173674479, + "objective/train/tokens_used": 2303590400, + "objective/train/unlikelihood_loss": 0.10134782642126083, + "theoretical_loss": 3.390762173674479, + "tokens_seen": 2303590400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015255767301905717, + "loss": 1.0895, + "theoretical_loss": 3.390762173674479, + "tokens_seen": 2303590400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015253761283851556, + "loss": 1.1381, + "theoretical_loss": 3.3907465314644414, + "tokens_seen": 2303721472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015251755265797393, + "loss": 1.1995, + "theoretical_loss": 3.390730890393532, + "tokens_seen": 2303852544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015249749247743232, + "loss": 1.2165, + "theoretical_loss": 3.3907152504616014, + "tokens_seen": 2303983616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015247743229689068, + "loss": 1.0658, + "theoretical_loss": 3.390699611668503, + "tokens_seen": 2304114688 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015245737211634904, + "loss": 1.0852, + "theoretical_loss": 3.390683974014088, + "tokens_seen": 2304245760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015243731193580743, + "loss": 1.2842, + "theoretical_loss": 3.39066833749821, + "tokens_seen": 2304376832 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001524172517552658, + "loss": 1.245, + "theoretical_loss": 3.3906527021207205, + "tokens_seen": 2304507904 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015239719157472418, + "loss": 1.108, + "theoretical_loss": 3.390637067881472, + "tokens_seen": 2304638976 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015237713139418254, + "loss": 1.217, + "theoretical_loss": 3.390621434780317, + "tokens_seen": 2304770048 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015235707121364093, + "loss": 1.2438, + "theoretical_loss": 3.3906058028171078, + "tokens_seen": 2304901120 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001523370110330993, + "loss": 1.0916, + "theoretical_loss": 3.3905901719916973, + "tokens_seen": 2305032192 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015231695085255768, + "loss": 1.1525, + "theoretical_loss": 3.3905745423039377, + "tokens_seen": 2305163264 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.42674732208251953, + "objective/train/avg_token_score": 0.03267257288098335, + "objective/train/docs_used": 1299212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.422577142715454, + "objective/train/original_loss": 4.615384578704834, + "objective/train/theoretical_loss": 3.3905667278866307, + "objective/train/tokens_used": 2305228800, + "objective/train/unlikelihood_loss": 0.13162735104560852, + "theoretical_loss": 3.3905667278866307, + "tokens_seen": 2305228800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015229689067201605, + "loss": 1.3446, + "theoretical_loss": 3.390558913753681, + "tokens_seen": 2305294336 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001522768304914744, + "loss": 1.1919, + "theoretical_loss": 3.3905432863407805, + "tokens_seen": 2305425408 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001522567703109328, + "loss": 1.1459, + "theoretical_loss": 3.390527660065089, + "tokens_seen": 2305556480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015223671013039116, + "loss": 1.2372, + "theoretical_loss": 3.3905120349264584, + "tokens_seen": 2305687552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015221664994984955, + "loss": 1.1327, + "theoretical_loss": 3.390496410924741, + "tokens_seen": 2305818624 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001521965897693079, + "loss": 1.1471, + "theoretical_loss": 3.3904807880597905, + "tokens_seen": 2305949696 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001521765295887663, + "loss": 1.331, + "theoretical_loss": 3.3904651663314587, + "tokens_seen": 2306080768 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015215646940822466, + "loss": 1.1325, + "theoretical_loss": 3.390449545739599, + "tokens_seen": 2306211840 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015213640922768305, + "loss": 1.3184, + "theoretical_loss": 3.390433926284064, + "tokens_seen": 2306342912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015211634904714142, + "loss": 1.2533, + "theoretical_loss": 3.390418307964706, + "tokens_seen": 2306473984 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001520962888665998, + "loss": 1.1168, + "theoretical_loss": 3.3904026907813787, + "tokens_seen": 2306605056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015207622868605817, + "loss": 1.0948, + "theoretical_loss": 3.3903870747339333, + "tokens_seen": 2306736128 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.29227760434150696, + "objective/train/avg_token_score": 0.012246023863554, + "objective/train/docs_used": 1299822, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9791136980056763, + "objective/train/original_loss": 4.273101806640625, + "objective/train/theoretical_loss": 3.3903714598222243, + "objective/train/tokens_used": 2306867200, + "objective/train/unlikelihood_loss": 0.13253124058246613, + "theoretical_loss": 3.3903714598222243, + "tokens_seen": 2306867200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015205616850551653, + "loss": 1.1752, + "theoretical_loss": 3.3903714598222243, + "tokens_seen": 2306867200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015203610832497492, + "loss": 1.1089, + "theoretical_loss": 3.390355846046104, + "tokens_seen": 2306998272 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001520160481444333, + "loss": 1.2902, + "theoretical_loss": 3.3903402334054253, + "tokens_seen": 2307129344 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001519959879638917, + "loss": 1.1113, + "theoretical_loss": 3.390324621900041, + "tokens_seen": 2307260416 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015197592778335006, + "loss": 1.1787, + "theoretical_loss": 3.390309011529804, + "tokens_seen": 2307391488 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015195586760280845, + "loss": 1.2626, + "theoretical_loss": 3.390293402294567, + "tokens_seen": 2307522560 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001519358074222668, + "loss": 1.3492, + "theoretical_loss": 3.3902777941941844, + "tokens_seen": 2307653632 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001519157472417252, + "loss": 1.1541, + "theoretical_loss": 3.3902621872285077, + "tokens_seen": 2307784704 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015189568706118356, + "loss": 1.2239, + "theoretical_loss": 3.3902465813973905, + "tokens_seen": 2307915776 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015187562688064192, + "loss": 1.2436, + "theoretical_loss": 3.3902309767006864, + "tokens_seen": 2308046848 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015185556670010031, + "loss": 1.3766, + "theoretical_loss": 3.390215373138248, + "tokens_seen": 2308177920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015183550651955868, + "loss": 1.2004, + "theoretical_loss": 3.390199770709928, + "tokens_seen": 2308308992 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015181544633901707, + "loss": 1.1044, + "theoretical_loss": 3.3901841694155803, + "tokens_seen": 2308440064 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.38599705696105957, + "objective/train/avg_token_score": 0.018179545179009438, + "objective/train/docs_used": 1301383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2420426607131958, + "objective/train/original_loss": 3.9889068603515625, + "objective/train/theoretical_loss": 3.3901763691936, + "objective/train/tokens_used": 2308505600, + "objective/train/unlikelihood_loss": 0.1694166511297226, + "theoretical_loss": 3.3901763691936, + "tokens_seen": 2308505600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015179538615847543, + "loss": 1.0292, + "theoretical_loss": 3.390168569255058, + "tokens_seen": 2308571136 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015177532597793382, + "loss": 1.3666, + "theoretical_loss": 3.390152970228214, + "tokens_seen": 2308702208 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015175526579739218, + "loss": 1.1302, + "theoretical_loss": 3.390137372334902, + "tokens_seen": 2308833280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015173520561685057, + "loss": 1.2237, + "theoretical_loss": 3.390121775574975, + "tokens_seen": 2308964352 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015171514543630893, + "loss": 1.2344, + "theoretical_loss": 3.390106179948287, + "tokens_seen": 2309095424 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516950852557673, + "loss": 1.1303, + "theoretical_loss": 3.39009058545469, + "tokens_seen": 2309226496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015167502507522568, + "loss": 1.1836, + "theoretical_loss": 3.3900749920940383, + "tokens_seen": 2309357568 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015165496489468404, + "loss": 1.1869, + "theoretical_loss": 3.390059399866185, + "tokens_seen": 2309488640 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015163490471414243, + "loss": 1.1073, + "theoretical_loss": 3.3900438087709834, + "tokens_seen": 2309619712 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516148445336008, + "loss": 1.0937, + "theoretical_loss": 3.3900282188082875, + "tokens_seen": 2309750784 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015159478435305919, + "loss": 1.1581, + "theoretical_loss": 3.3900126299779503, + "tokens_seen": 2309881856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015157472417251755, + "loss": 1.3931, + "theoretical_loss": 3.3899970422798256, + "tokens_seen": 2310012928 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.5120356678962708, + "objective/train/avg_token_score": 0.015221510082483292, + "objective/train/docs_used": 1301708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.630745768547058, + "objective/train/original_loss": 3.7996633052825928, + "objective/train/theoretical_loss": 3.389981455713767, + "objective/train/tokens_used": 2310144000, + "objective/train/unlikelihood_loss": 0.11564325541257858, + "theoretical_loss": 3.389981455713767, + "tokens_seen": 2310144000 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015155466399197594, + "loss": 1.3278, + "theoretical_loss": 3.389981455713767, + "tokens_seen": 2310144000 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001515346038114343, + "loss": 1.0925, + "theoretical_loss": 3.389965870279627, + "tokens_seen": 2310275072 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001515145436308927, + "loss": 1.1713, + "theoretical_loss": 3.3899502859772612, + "tokens_seen": 2310406144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015149448345035105, + "loss": 1.1019, + "theoretical_loss": 3.389934702806521, + "tokens_seen": 2310537216 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001514744232698094, + "loss": 1.4476, + "theoretical_loss": 3.389919120767262, + "tokens_seen": 2310668288 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001514543630892678, + "loss": 1.2057, + "theoretical_loss": 3.3899035398593362, + "tokens_seen": 2310799360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015143430290872617, + "loss": 1.1903, + "theoretical_loss": 3.389887960082598, + "tokens_seen": 2310930432 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015141424272818455, + "loss": 1.3384, + "theoretical_loss": 3.3898723814369016, + "tokens_seen": 2311061504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015139418254764292, + "loss": 1.274, + "theoretical_loss": 3.3898568039221004, + "tokens_seen": 2311192576 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001513741223671013, + "loss": 1.2368, + "theoretical_loss": 3.3898412275380476, + "tokens_seen": 2311323648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015135406218655967, + "loss": 1.0319, + "theoretical_loss": 3.389825652284598, + "tokens_seen": 2311454720 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015133400200601806, + "loss": 1.2091, + "theoretical_loss": 3.389810078161605, + "tokens_seen": 2311585792 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015131394182547642, + "loss": 1.15, + "theoretical_loss": 3.389794505168922, + "tokens_seen": 2311716864 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.5171065330505371, + "objective/train/avg_token_score": 0.01907159760594368, + "objective/train/docs_used": 1302992, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6192337274551392, + "objective/train/original_loss": 3.909627914428711, + "objective/train/theoretical_loss": 3.389786719096401, + "objective/train/tokens_used": 2311782400, + "objective/train/unlikelihood_loss": 0.13028784096240997, + "theoretical_loss": 3.389786719096401, + "tokens_seen": 2311782400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015129388164493478, + "loss": 1.2603, + "theoretical_loss": 3.389778933306403, + "tokens_seen": 2311847936 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001512738214643932, + "loss": 1.2569, + "theoretical_loss": 3.3897633625739028, + "tokens_seen": 2311979008 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015125376128385156, + "loss": 1.1009, + "theoretical_loss": 3.3897477929712743, + "tokens_seen": 2312110080 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015123370110330995, + "loss": 1.3572, + "theoretical_loss": 3.3897322244983723, + "tokens_seen": 2312241152 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001512136409227683, + "loss": 1.3046, + "theoretical_loss": 3.3897166571550503, + "tokens_seen": 2312372224 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001511935807422267, + "loss": 1.3263, + "theoretical_loss": 3.3897010909411622, + "tokens_seen": 2312503296 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015117352056168506, + "loss": 1.3456, + "theoretical_loss": 3.3896855258565624, + "tokens_seen": 2312634368 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015115346038114345, + "loss": 1.3373, + "theoretical_loss": 3.3896699619011046, + "tokens_seen": 2312765440 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015113340020060182, + "loss": 1.2073, + "theoretical_loss": 3.3896543990746437, + "tokens_seen": 2312896512 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001511133400200602, + "loss": 1.2497, + "theoretical_loss": 3.3896388373770328, + "tokens_seen": 2313027584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015109327983951857, + "loss": 1.2341, + "theoretical_loss": 3.389623276808127, + "tokens_seen": 2313158656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015107321965897693, + "loss": 1.154, + "theoretical_loss": 3.389607717367779, + "tokens_seen": 2313289728 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.33449414372444153, + "objective/train/avg_token_score": 0.03228846937417984, + "objective/train/docs_used": 1303654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0336333513259888, + "objective/train/original_loss": 4.27305269241333, + "objective/train/theoretical_loss": 3.389592159055845, + "objective/train/tokens_used": 2313420800, + "objective/train/unlikelihood_loss": 0.17926278710365295, + "theoretical_loss": 3.389592159055845, + "tokens_seen": 2313420800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015105315947843532, + "loss": 1.2419, + "theoretical_loss": 3.389592159055845, + "tokens_seen": 2313420800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015103309929789368, + "loss": 1.2054, + "theoretical_loss": 3.389576601872178, + "tokens_seen": 2313551872 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015101303911735207, + "loss": 1.1099, + "theoretical_loss": 3.3895610458166323, + "tokens_seen": 2313682944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015099297893681043, + "loss": 1.2508, + "theoretical_loss": 3.3895454908890628, + "tokens_seen": 2313814016 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015097291875626882, + "loss": 1.1921, + "theoretical_loss": 3.389529937089323, + "tokens_seen": 2313945088 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015095285857572718, + "loss": 1.2226, + "theoretical_loss": 3.389514384417268, + "tokens_seen": 2314076160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015093279839518557, + "loss": 1.3339, + "theoretical_loss": 3.3894988328727518, + "tokens_seen": 2314207232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015091273821464394, + "loss": 1.43, + "theoretical_loss": 3.389483282455629, + "tokens_seen": 2314338304 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001508926780341023, + "loss": 1.4147, + "theoretical_loss": 3.3894677331657537, + "tokens_seen": 2314469376 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001508726178535607, + "loss": 1.2321, + "theoretical_loss": 3.3894521850029804, + "tokens_seen": 2314600448 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015085255767301905, + "loss": 1.1812, + "theoretical_loss": 3.389436637967164, + "tokens_seen": 2314731520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015083249749247744, + "loss": 1.1552, + "theoretical_loss": 3.389421092058159, + "tokens_seen": 2314862592 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001508124373119358, + "loss": 1.2333, + "theoretical_loss": 3.389405547275819, + "tokens_seen": 2314993664 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.4375610947608948, + "objective/train/avg_token_score": 0.022290382534265518, + "objective/train/docs_used": 1304892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.330048680305481, + "objective/train/original_loss": 4.390745162963867, + "objective/train/theoretical_loss": 3.3893977753071036, + "objective/train/tokens_used": 2315059200, + "objective/train/unlikelihood_loss": 0.09900231659412384, + "theoretical_loss": 3.3893977753071036, + "tokens_seen": 2315059200 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001507923771313942, + "loss": 1.3096, + "theoretical_loss": 3.3893900036199995, + "tokens_seen": 2315124736 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015077231695085255, + "loss": 1.16, + "theoretical_loss": 3.389374461090555, + "tokens_seen": 2315255808 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015075225677031094, + "loss": 1.1174, + "theoretical_loss": 3.38935891968734, + "tokens_seen": 2315386880 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001507321965897693, + "loss": 1.1904, + "theoretical_loss": 3.389343379410209, + "tokens_seen": 2315517952 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015071213640922767, + "loss": 1.1528, + "theoretical_loss": 3.389327840259017, + "tokens_seen": 2315649024 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015069207622868606, + "loss": 1.2629, + "theoretical_loss": 3.3893123022336176, + "tokens_seen": 2315780096 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015067201604814442, + "loss": 1.161, + "theoretical_loss": 3.389296765333867, + "tokens_seen": 2315911168 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001506519558676028, + "loss": 1.0345, + "theoretical_loss": 3.3892812295596193, + "tokens_seen": 2316042240 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015063189568706117, + "loss": 1.4091, + "theoretical_loss": 3.3892656949107294, + "tokens_seen": 2316173312 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015061183550651956, + "loss": 1.273, + "theoretical_loss": 3.389250161387052, + "tokens_seen": 2316304384 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015059177532597792, + "loss": 1.2156, + "theoretical_loss": 3.389234628988442, + "tokens_seen": 2316435456 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001505717151454363, + "loss": 1.0732, + "theoretical_loss": 3.389219097714754, + "tokens_seen": 2316566528 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.4662756621837616, + "objective/train/avg_token_score": 0.014157815836369991, + "objective/train/docs_used": 1305637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4977900981903076, + "objective/train/original_loss": 3.613152503967285, + "objective/train/theoretical_loss": 3.389203567565843, + "objective/train/tokens_used": 2316697600, + "objective/train/unlikelihood_loss": 0.15379591286182404, + "theoretical_loss": 3.389203567565843, + "tokens_seen": 2316697600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015055165496489467, + "loss": 1.2949, + "theoretical_loss": 3.389203567565843, + "tokens_seen": 2316697600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015053159478435304, + "loss": 1.2978, + "theoretical_loss": 3.389188038541564, + "tokens_seen": 2316828672 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015051153460381145, + "loss": 1.337, + "theoretical_loss": 3.3891725106417723, + "tokens_seen": 2316959744 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015049147442326981, + "loss": 1.2487, + "theoretical_loss": 3.3891569838663225, + "tokens_seen": 2317090816 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001504714142427282, + "loss": 1.2149, + "theoretical_loss": 3.389141458215069, + "tokens_seen": 2317221888 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015045135406218657, + "loss": 1.2251, + "theoretical_loss": 3.3891259336878683, + "tokens_seen": 2317352960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015043129388164495, + "loss": 1.3521, + "theoretical_loss": 3.389110410284574, + "tokens_seen": 2317484032 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015041123370110332, + "loss": 1.3349, + "theoretical_loss": 3.389094888005042, + "tokens_seen": 2317615104 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001503911735205617, + "loss": 1.3135, + "theoretical_loss": 3.389079366849127, + "tokens_seen": 2317746176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015037111334002007, + "loss": 1.2909, + "theoretical_loss": 3.3890638468166845, + "tokens_seen": 2317877248 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015035105315947846, + "loss": 1.2901, + "theoretical_loss": 3.3890483279075694, + "tokens_seen": 2318008320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015033099297893682, + "loss": 1.1989, + "theoretical_loss": 3.3890328101216367, + "tokens_seen": 2318139392 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015031093279839518, + "loss": 1.228, + "theoretical_loss": 3.389017293458742, + "tokens_seen": 2318270464 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.3092619776725769, + "objective/train/avg_token_score": 0.03551428020000458, + "objective/train/docs_used": 1306410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.023398995399475, + "objective/train/original_loss": 4.712287425994873, + "objective/train/theoretical_loss": 3.389009535548389, + "objective/train/tokens_used": 2318336000, + "objective/train/unlikelihood_loss": 0.09409680217504501, + "theoretical_loss": 3.389009535548389, + "tokens_seen": 2318336000 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015029087261785357, + "loss": 1.3869, + "theoretical_loss": 3.3890017779187405, + "tokens_seen": 2318401536 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015027081243731193, + "loss": 1.3653, + "theoretical_loss": 3.3889862635014874, + "tokens_seen": 2318532608 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015025075225677032, + "loss": 1.3226, + "theoretical_loss": 3.3889707502068376, + "tokens_seen": 2318663680 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015023069207622869, + "loss": 1.0641, + "theoretical_loss": 3.3889552380346473, + "tokens_seen": 2318794752 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015021063189568707, + "loss": 1.2574, + "theoretical_loss": 3.3889397269847708, + "tokens_seen": 2318925824 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015019057171514544, + "loss": 1.2734, + "theoretical_loss": 3.388924217057064, + "tokens_seen": 2319056896 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015017051153460383, + "loss": 1.3303, + "theoretical_loss": 3.3889087082513827, + "tokens_seen": 2319187968 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001501504513540622, + "loss": 1.2534, + "theoretical_loss": 3.388893200567581, + "tokens_seen": 2319319040 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015013039117352055, + "loss": 1.2255, + "theoretical_loss": 3.388877694005516, + "tokens_seen": 2319450112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015011033099297894, + "loss": 1.3364, + "theoretical_loss": 3.3888621885650423, + "tokens_seen": 2319581184 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001500902708124373, + "loss": 1.1709, + "theoretical_loss": 3.3888466842460154, + "tokens_seen": 2319712256 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001500702106318957, + "loss": 1.0734, + "theoretical_loss": 3.3888311810482907, + "tokens_seen": 2319843328 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.5023216009140015, + "objective/train/avg_token_score": 0.008982245810329914, + "objective/train/docs_used": 1307847, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.43809974193573, + "objective/train/original_loss": 3.6708669662475586, + "objective/train/theoretical_loss": 3.3888156789717243, + "objective/train/tokens_used": 2319974400, + "objective/train/unlikelihood_loss": 0.17630194127559662, + "theoretical_loss": 3.3888156789717243, + "tokens_seen": 2319974400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015005015045135405, + "loss": 1.2485, + "theoretical_loss": 3.3888156789717243, + "tokens_seen": 2319974400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015003009027081244, + "loss": 1.1597, + "theoretical_loss": 3.3888001780161714, + "tokens_seen": 2320105472 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001500100300902708, + "loss": 1.2929, + "theoretical_loss": 3.388784678181488, + "tokens_seen": 2320236544 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001499899699097292, + "loss": 1.2242, + "theoretical_loss": 3.388769179467529, + "tokens_seen": 2320367616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014996990972918756, + "loss": 1.3634, + "theoretical_loss": 3.388753681874151, + "tokens_seen": 2320498688 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014994984954864595, + "loss": 1.107, + "theoretical_loss": 3.3887381854012086, + "tokens_seen": 2320629760 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001499297893681043, + "loss": 1.25, + "theoretical_loss": 3.3887226900485583, + "tokens_seen": 2320760832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014990972918756267, + "loss": 1.2168, + "theoretical_loss": 3.388707195816056, + "tokens_seen": 2320891904 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014988966900702106, + "loss": 1.2639, + "theoretical_loss": 3.3886917027035572, + "tokens_seen": 2321022976 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014986960882647942, + "loss": 1.4035, + "theoretical_loss": 3.3886762107109174, + "tokens_seen": 2321154048 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001498495486459378, + "loss": 1.2669, + "theoretical_loss": 3.3886607198379926, + "tokens_seen": 2321285120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014982948846539617, + "loss": 1.1742, + "theoretical_loss": 3.3886452300846384, + "tokens_seen": 2321416192 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014980942828485456, + "loss": 1.2581, + "theoretical_loss": 3.3886297414507114, + "tokens_seen": 2321547264 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.3601539731025696, + "objective/train/avg_token_score": 0.01737206056714058, + "objective/train/docs_used": 1308577, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1356550455093384, + "objective/train/original_loss": 4.025541305541992, + "objective/train/theoretical_loss": 3.388621997553488, + "objective/train/tokens_used": 2321612800, + "objective/train/unlikelihood_loss": 0.17812199890613556, + "theoretical_loss": 3.388621997553488, + "tokens_seen": 2321612800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014978936810431293, + "loss": 1.2755, + "theoretical_loss": 3.388614253936067, + "tokens_seen": 2321678336 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014976930792377134, + "loss": 1.2719, + "theoretical_loss": 3.3885987675405613, + "tokens_seen": 2321809408 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001497492477432297, + "loss": 1.1548, + "theoretical_loss": 3.38858328226405, + "tokens_seen": 2321940480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014972918756268807, + "loss": 1.2679, + "theoretical_loss": 3.388567798106389, + "tokens_seen": 2322071552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014970912738214646, + "loss": 1.2221, + "theoretical_loss": 3.3885523150674355, + "tokens_seen": 2322202624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014968906720160482, + "loss": 1.2933, + "theoretical_loss": 3.388536833147044, + "tokens_seen": 2322333696 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001496690070210632, + "loss": 1.3089, + "theoretical_loss": 3.3885213523450712, + "tokens_seen": 2322464768 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014964894684052157, + "loss": 1.1543, + "theoretical_loss": 3.3885058726613733, + "tokens_seen": 2322595840 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014962888665997996, + "loss": 1.2715, + "theoretical_loss": 3.388490394095806, + "tokens_seen": 2322726912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014960882647943832, + "loss": 1.291, + "theoretical_loss": 3.388474916648226, + "tokens_seen": 2322857984 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001495887662988967, + "loss": 1.2624, + "theoretical_loss": 3.3884594403184893, + "tokens_seen": 2322989056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014956870611835507, + "loss": 1.1806, + "theoretical_loss": 3.388443965106452, + "tokens_seen": 2323120128 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.43194037675857544, + "objective/train/avg_token_score": 0.021422499790787697, + "objective/train/docs_used": 1309762, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1087015867233276, + "objective/train/original_loss": 3.9813621044158936, + "objective/train/theoretical_loss": 3.3884284910119704, + "objective/train/tokens_used": 2323251200, + "objective/train/unlikelihood_loss": 0.1111120656132698, + "theoretical_loss": 3.3884284910119704, + "tokens_seen": 2323251200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014954864593781344, + "loss": 1.2314, + "theoretical_loss": 3.3884284910119704, + "tokens_seen": 2323251200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014952858575727182, + "loss": 1.314, + "theoretical_loss": 3.3884130180349006, + "tokens_seen": 2323382272 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001495085255767302, + "loss": 1.3133, + "theoretical_loss": 3.388397546175099, + "tokens_seen": 2323513344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014948846539618858, + "loss": 1.4786, + "theoretical_loss": 3.3883820754324216, + "tokens_seen": 2323644416 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014946840521564694, + "loss": 1.4132, + "theoretical_loss": 3.388366605806725, + "tokens_seen": 2323775488 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014944834503510533, + "loss": 1.2676, + "theoretical_loss": 3.388351137297866, + "tokens_seen": 2323906560 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001494282848545637, + "loss": 1.1052, + "theoretical_loss": 3.3883356699057003, + "tokens_seen": 2324037632 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014940822467402208, + "loss": 1.1788, + "theoretical_loss": 3.3883202036300846, + "tokens_seen": 2324168704 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014938816449348044, + "loss": 1.2494, + "theoretical_loss": 3.3883047384708753, + "tokens_seen": 2324299776 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014936810431293883, + "loss": 1.2347, + "theoretical_loss": 3.3882892744279287, + "tokens_seen": 2324430848 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001493480441323972, + "loss": 1.3033, + "theoretical_loss": 3.3882738115011017, + "tokens_seen": 2324561920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014932798395185556, + "loss": 1.2709, + "theoretical_loss": 3.388258349690251, + "tokens_seen": 2324692992 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014930792377131394, + "loss": 1.2611, + "theoretical_loss": 3.388242888995232, + "tokens_seen": 2324824064 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.4223484992980957, + "objective/train/avg_token_score": 0.013356111012399197, + "objective/train/docs_used": 1310527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.422232985496521, + "objective/train/original_loss": 3.8524045944213867, + "objective/train/theoretical_loss": 3.388235159066115, + "objective/train/tokens_used": 2324889600, + "objective/train/unlikelihood_loss": 0.15602673590183258, + "theoretical_loss": 3.388235159066115, + "tokens_seen": 2324889600 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001492878635907723, + "loss": 1.3509, + "theoretical_loss": 3.388227429415902, + "tokens_seen": 2324955136 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001492678034102307, + "loss": 1.3054, + "theoretical_loss": 3.388211970952118, + "tokens_seen": 2325086208 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014924774322968906, + "loss": 1.2514, + "theoretical_loss": 3.388196513603736, + "tokens_seen": 2325217280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014922768304914745, + "loss": 1.0801, + "theoretical_loss": 3.3881810573706126, + "tokens_seen": 2325348352 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001492076228686058, + "loss": 1.3704, + "theoretical_loss": 3.388165602252605, + "tokens_seen": 2325479424 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001491875626880642, + "loss": 1.2887, + "theoretical_loss": 3.38815014824957, + "tokens_seen": 2325610496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014916750250752256, + "loss": 1.2962, + "theoretical_loss": 3.3881346953613636, + "tokens_seen": 2325741568 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014914744232698092, + "loss": 1.3035, + "theoretical_loss": 3.388119243587843, + "tokens_seen": 2325872640 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001491273821464393, + "loss": 1.2352, + "theoretical_loss": 3.388103792928865, + "tokens_seen": 2326003712 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014910732196589768, + "loss": 1.2848, + "theoretical_loss": 3.388088343384286, + "tokens_seen": 2326134784 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014908726178535607, + "loss": 1.3727, + "theoretical_loss": 3.388072894953963, + "tokens_seen": 2326265856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014906720160481443, + "loss": 1.1771, + "theoretical_loss": 3.3880574476377534, + "tokens_seen": 2326396928 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.3762829899787903, + "objective/train/avg_token_score": 0.022464554756879807, + "objective/train/docs_used": 1311908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.070949912071228, + "objective/train/original_loss": 3.6455185413360596, + "objective/train/theoretical_loss": 3.3880420014355135, + "objective/train/tokens_used": 2326528000, + "objective/train/unlikelihood_loss": 0.17161016166210175, + "theoretical_loss": 3.3880420014355135, + "tokens_seen": 2326528000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014904714142427282, + "loss": 1.1832, + "theoretical_loss": 3.3880420014355135, + "tokens_seen": 2326528000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014902708124373118, + "loss": 1.1651, + "theoretical_loss": 3.388026556347101, + "tokens_seen": 2326659072 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001490070210631896, + "loss": 1.2617, + "theoretical_loss": 3.3880111123723715, + "tokens_seen": 2326790144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014898696088264796, + "loss": 1.3008, + "theoretical_loss": 3.387995669511183, + "tokens_seen": 2326921216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014896690070210635, + "loss": 1.2108, + "theoretical_loss": 3.3879802277633924, + "tokens_seen": 2327052288 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001489468405215647, + "loss": 1.2185, + "theoretical_loss": 3.3879647871288565, + "tokens_seen": 2327183360 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014892678034102307, + "loss": 1.3308, + "theoretical_loss": 3.387949347607432, + "tokens_seen": 2327314432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014890672016048146, + "loss": 1.1574, + "theoretical_loss": 3.387933909198977, + "tokens_seen": 2327445504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014888665997993982, + "loss": 1.299, + "theoretical_loss": 3.3879184719033475, + "tokens_seen": 2327576576 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001488665997993982, + "loss": 1.2003, + "theoretical_loss": 3.3879030357204014, + "tokens_seen": 2327707648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014884653961885657, + "loss": 1.2267, + "theoretical_loss": 3.387887600649995, + "tokens_seen": 2327838720 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014882647943831496, + "loss": 1.2695, + "theoretical_loss": 3.387872166691986, + "tokens_seen": 2327969792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014880641925777333, + "loss": 1.3379, + "theoretical_loss": 3.3878567338462324, + "tokens_seen": 2328100864 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.5959188938140869, + "objective/train/avg_token_score": 0.004629809875041246, + "objective/train/docs_used": 1312576, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8958364725112915, + "objective/train/original_loss": 3.5964369773864746, + "objective/train/theoretical_loss": 3.387849017840406, + "objective/train/tokens_used": 2328166400, + "objective/train/unlikelihood_loss": 0.13021941483020782, + "theoretical_loss": 3.387849017840406, + "tokens_seen": 2328166400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014878635907723172, + "loss": 1.3168, + "theoretical_loss": 3.3878413021125904, + "tokens_seen": 2328231936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014876629889669008, + "loss": 1.1649, + "theoretical_loss": 3.387825871490917, + "tokens_seen": 2328363008 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014874623871614844, + "loss": 1.2664, + "theoretical_loss": 3.387810441981071, + "tokens_seen": 2328494080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014872617853560683, + "loss": 1.162, + "theoretical_loss": 3.387795013582908, + "tokens_seen": 2328625152 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001487061183550652, + "loss": 1.2299, + "theoretical_loss": 3.387779586296286, + "tokens_seen": 2328756224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014868605817452358, + "loss": 1.2981, + "theoretical_loss": 3.3877641601210624, + "tokens_seen": 2328887296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014866599799398194, + "loss": 1.1314, + "theoretical_loss": 3.3877487350570945, + "tokens_seen": 2329018368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014864593781344033, + "loss": 1.3521, + "theoretical_loss": 3.3877333111042405, + "tokens_seen": 2329149440 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001486258776328987, + "loss": 1.1761, + "theoretical_loss": 3.3877178882623564, + "tokens_seen": 2329280512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014860581745235708, + "loss": 1.1494, + "theoretical_loss": 3.3877024665313007, + "tokens_seen": 2329411584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014858575727181545, + "loss": 1.2172, + "theoretical_loss": 3.3876870459109307, + "tokens_seen": 2329542656 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001485656970912738, + "loss": 1.2937, + "theoretical_loss": 3.387671626401104, + "tokens_seen": 2329673728 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.4272971749305725, + "objective/train/avg_token_score": 0.01036697905510664, + "objective/train/docs_used": 1313793, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.380463719367981, + "objective/train/original_loss": 3.895826578140259, + "objective/train/theoretical_loss": 3.387656208001678, + "objective/train/tokens_used": 2329804800, + "objective/train/unlikelihood_loss": 0.1721668243408203, + "theoretical_loss": 3.387656208001678, + "tokens_seen": 2329804800 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001485456369107322, + "loss": 1.3092, + "theoretical_loss": 3.387656208001678, + "tokens_seen": 2329804800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014852557673019056, + "loss": 1.1651, + "theoretical_loss": 3.38764079071251, + "tokens_seen": 2329935872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014850551654964895, + "loss": 1.3605, + "theoretical_loss": 3.387625374533458, + "tokens_seen": 2330066944 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484854563691073, + "loss": 1.2308, + "theoretical_loss": 3.3876099594643794, + "tokens_seen": 2330198016 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484653961885657, + "loss": 1.1672, + "theoretical_loss": 3.3875945455051317, + "tokens_seen": 2330329088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014844533600802406, + "loss": 1.2543, + "theoretical_loss": 3.3875791326555733, + "tokens_seen": 2330460160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014842527582748245, + "loss": 1.1944, + "theoretical_loss": 3.387563720915561, + "tokens_seen": 2330591232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014840521564694081, + "loss": 1.2634, + "theoretical_loss": 3.3875483102849535, + "tokens_seen": 2330722304 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001483851554663992, + "loss": 1.1762, + "theoretical_loss": 3.387532900763608, + "tokens_seen": 2330853376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014836509528585757, + "loss": 1.229, + "theoretical_loss": 3.3875174923513818, + "tokens_seen": 2330984448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014834503510531593, + "loss": 1.3165, + "theoretical_loss": 3.3875020850481334, + "tokens_seen": 2331115520 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014832497492477432, + "loss": 1.2889, + "theoretical_loss": 3.387486678853721, + "tokens_seen": 2331246592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014830491474423268, + "loss": 1.3843, + "theoretical_loss": 3.387471273768001, + "tokens_seen": 2331377664 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.5370234847068787, + "objective/train/avg_token_score": 0.016726333647966385, + "objective/train/docs_used": 1314556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7179009914398193, + "objective/train/original_loss": 3.9776666164398193, + "objective/train/theoretical_loss": 3.3874635716408568, + "objective/train/tokens_used": 2331443200, + "objective/train/unlikelihood_loss": 0.09187538176774979, + "theoretical_loss": 3.3874635716408568, + "tokens_seen": 2331443200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014828485456369107, + "loss": 1.2531, + "theoretical_loss": 3.3874558697908324, + "tokens_seen": 2331508736 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014826479438314946, + "loss": 1.2353, + "theoretical_loss": 3.3874404669220732, + "tokens_seen": 2331639808 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014824473420260785, + "loss": 1.1159, + "theoretical_loss": 3.3874250651615814, + "tokens_seen": 2331770880 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001482246740220662, + "loss": 1.1374, + "theoretical_loss": 3.3874096645092138, + "tokens_seen": 2331901952 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001482046138415246, + "loss": 1.1693, + "theoretical_loss": 3.3873942649648296, + "tokens_seen": 2332033024 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014818455366098296, + "loss": 1.185, + "theoretical_loss": 3.3873788665282865, + "tokens_seen": 2332164096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014816449348044132, + "loss": 1.4177, + "theoretical_loss": 3.3873634691994425, + "tokens_seen": 2332295168 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014814443329989971, + "loss": 1.2556, + "theoretical_loss": 3.3873480729781558, + "tokens_seen": 2332426240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014812437311935808, + "loss": 1.2372, + "theoretical_loss": 3.3873326778642845, + "tokens_seen": 2332557312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014810431293881647, + "loss": 1.3526, + "theoretical_loss": 3.387317283857686, + "tokens_seen": 2332688384 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014808425275827483, + "loss": 1.3743, + "theoretical_loss": 3.387301890958219, + "tokens_seen": 2332819456 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014806419257773322, + "loss": 1.349, + "theoretical_loss": 3.387286499165742, + "tokens_seen": 2332950528 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.3667522072792053, + "objective/train/avg_token_score": 0.020929569378495216, + "objective/train/docs_used": 1315240, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2170908451080322, + "objective/train/original_loss": 4.361345291137695, + "objective/train/theoretical_loss": 3.387271108480113, + "objective/train/tokens_used": 2333081600, + "objective/train/unlikelihood_loss": 0.11804627627134323, + "theoretical_loss": 3.387271108480113, + "tokens_seen": 2333081600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014804413239719158, + "loss": 1.2401, + "theoretical_loss": 3.387271108480113, + "tokens_seen": 2333081600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014802407221664997, + "loss": 1.1723, + "theoretical_loss": 3.3872557189011903, + "tokens_seen": 2333212672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014800401203610833, + "loss": 1.15, + "theoretical_loss": 3.3872403304288317, + "tokens_seen": 2333343744 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001479839518555667, + "loss": 1.2803, + "theoretical_loss": 3.387224943062896, + "tokens_seen": 2333474816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014796389167502508, + "loss": 1.2876, + "theoretical_loss": 3.387209556803241, + "tokens_seen": 2333605888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014794383149448344, + "loss": 1.1541, + "theoretical_loss": 3.3871941716497256, + "tokens_seen": 2333736960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014792377131394183, + "loss": 1.2247, + "theoretical_loss": 3.387178787602208, + "tokens_seen": 2333868032 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001479037111334002, + "loss": 1.3022, + "theoretical_loss": 3.387163404660546, + "tokens_seen": 2333999104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014788365095285859, + "loss": 1.1314, + "theoretical_loss": 3.3871480228245985, + "tokens_seen": 2334130176 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014786359077231695, + "loss": 1.143, + "theoretical_loss": 3.3871326420942243, + "tokens_seen": 2334261248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014784353059177534, + "loss": 1.4147, + "theoretical_loss": 3.387117262469281, + "tokens_seen": 2334392320 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001478234704112337, + "loss": 1.219, + "theoretical_loss": 3.387101883949628, + "tokens_seen": 2334523392 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001478034102306921, + "loss": 1.2408, + "theoretical_loss": 3.387086506535123, + "tokens_seen": 2334654464 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.32532989978790283, + "objective/train/avg_token_score": 0.00966570619493723, + "objective/train/docs_used": 1316418, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9401884078979492, + "objective/train/original_loss": 4.018411636352539, + "objective/train/theoretical_loss": 3.3870788182422573, + "objective/train/tokens_used": 2334720000, + "objective/train/unlikelihood_loss": 0.13074660301208496, + "theoretical_loss": 3.3870788182422573, + "tokens_seen": 2334720000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014778335005015045, + "loss": 1.1023, + "theoretical_loss": 3.3870711302256256, + "tokens_seen": 2334785536 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001477632898696088, + "loss": 1.0599, + "theoretical_loss": 3.387055755020993, + "tokens_seen": 2334916608 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001477432296890672, + "loss": 1.2081, + "theoretical_loss": 3.3870403809210847, + "tokens_seen": 2335047680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014772316950852556, + "loss": 1.287, + "theoretical_loss": 3.387025007925759, + "tokens_seen": 2335178752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014770310932798395, + "loss": 1.327, + "theoretical_loss": 3.387009636034875, + "tokens_seen": 2335309824 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014768304914744232, + "loss": 1.2523, + "theoretical_loss": 3.386994265248291, + "tokens_seen": 2335440896 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001476629889669007, + "loss": 1.2129, + "theoretical_loss": 3.3869788955658655, + "tokens_seen": 2335571968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014764292878635907, + "loss": 1.4052, + "theoretical_loss": 3.3869635269874574, + "tokens_seen": 2335703040 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014762286860581746, + "loss": 1.3922, + "theoretical_loss": 3.386948159512926, + "tokens_seen": 2335834112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014760280842527582, + "loss": 1.2948, + "theoretical_loss": 3.3869327931421287, + "tokens_seen": 2335965184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014758274824473418, + "loss": 1.2134, + "theoretical_loss": 3.3869174278749257, + "tokens_seen": 2336096256 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014756268806419257, + "loss": 1.2164, + "theoretical_loss": 3.386902063711175, + "tokens_seen": 2336227328 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.29447701573371887, + "objective/train/avg_token_score": 0.012515685521066189, + "objective/train/docs_used": 1317114, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9837108254432678, + "objective/train/original_loss": 4.397698402404785, + "objective/train/theoretical_loss": 3.386886700650736, + "objective/train/tokens_used": 2336358400, + "objective/train/unlikelihood_loss": 0.1556367129087448, + "theoretical_loss": 3.386886700650736, + "tokens_seen": 2336358400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014754262788365093, + "loss": 1.332, + "theoretical_loss": 3.386886700650736, + "tokens_seen": 2336358400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014752256770310932, + "loss": 1.344, + "theoretical_loss": 3.3868713386934677, + "tokens_seen": 2336489472 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001475025075225677, + "loss": 1.2004, + "theoretical_loss": 3.386855977839228, + "tokens_seen": 2336620544 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001474824473420261, + "loss": 1.2414, + "theoretical_loss": 3.386840618087877, + "tokens_seen": 2336751616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014746238716148446, + "loss": 1.3525, + "theoretical_loss": 3.386825259439273, + "tokens_seen": 2336882688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014744232698094285, + "loss": 1.3827, + "theoretical_loss": 3.386809901893275, + "tokens_seen": 2337013760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014742226680040122, + "loss": 1.0784, + "theoretical_loss": 3.3867945454497423, + "tokens_seen": 2337144832 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001474022066198596, + "loss": 1.2669, + "theoretical_loss": 3.3867791901085336, + "tokens_seen": 2337275904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014738214643931797, + "loss": 1.2117, + "theoretical_loss": 3.386763835869508, + "tokens_seen": 2337406976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014736208625877633, + "loss": 1.3497, + "theoretical_loss": 3.386748482732525, + "tokens_seen": 2337538048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014734202607823472, + "loss": 1.2635, + "theoretical_loss": 3.3867331306974435, + "tokens_seen": 2337669120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014732196589769308, + "loss": 1.2542, + "theoretical_loss": 3.386717779764122, + "tokens_seen": 2337800192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014730190571715147, + "loss": 1.2333, + "theoretical_loss": 3.386702429932421, + "tokens_seen": 2337931264 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.3459188640117645, + "objective/train/avg_token_score": 0.012717272154986858, + "objective/train/docs_used": 1318542, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1052815914154053, + "objective/train/original_loss": 4.142512798309326, + "objective/train/theoretical_loss": 3.386694755429634, + "objective/train/tokens_used": 2337996800, + "objective/train/unlikelihood_loss": 0.1224541962146759, + "theoretical_loss": 3.386694755429634, + "tokens_seen": 2337996800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014728184553660983, + "loss": 1.3087, + "theoretical_loss": 3.3866870812021985, + "tokens_seen": 2338062336 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014726178535606822, + "loss": 1.2393, + "theoretical_loss": 3.3866717335733143, + "tokens_seen": 2338193408 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014724172517552658, + "loss": 1.1799, + "theoretical_loss": 3.3866563870456274, + "tokens_seen": 2338324480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014722166499498497, + "loss": 1.2125, + "theoretical_loss": 3.386641041618997, + "tokens_seen": 2338455552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014720160481444334, + "loss": 1.3498, + "theoretical_loss": 3.3866256972932827, + "tokens_seen": 2338586624 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471815446339017, + "loss": 1.2376, + "theoretical_loss": 3.3866103540683437, + "tokens_seen": 2338717696 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471614844533601, + "loss": 1.2157, + "theoretical_loss": 3.3865950119440393, + "tokens_seen": 2338848768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014714142427281845, + "loss": 1.3835, + "theoretical_loss": 3.386579670920229, + "tokens_seen": 2338979840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014712136409227684, + "loss": 1.0684, + "theoretical_loss": 3.386564330996772, + "tokens_seen": 2339110912 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471013039117352, + "loss": 1.2187, + "theoretical_loss": 3.3865489921735277, + "tokens_seen": 2339241984 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001470812437311936, + "loss": 1.2773, + "theoretical_loss": 3.386533654450356, + "tokens_seen": 2339373056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014706118355065195, + "loss": 1.3611, + "theoretical_loss": 3.3865183178271154, + "tokens_seen": 2339504128 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.43933284282684326, + "objective/train/avg_token_score": 0.005854027811437845, + "objective/train/docs_used": 1319266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.297311544418335, + "objective/train/original_loss": 3.3922276496887207, + "objective/train/theoretical_loss": 3.386502982303667, + "objective/train/tokens_used": 2339635200, + "objective/train/unlikelihood_loss": 0.19320271909236908, + "theoretical_loss": 3.386502982303667, + "tokens_seen": 2339635200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014704112337011034, + "loss": 1.2589, + "theoretical_loss": 3.386502982303667, + "tokens_seen": 2339635200 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001470210631895687, + "loss": 1.306, + "theoretical_loss": 3.3864876478798687, + "tokens_seen": 2339766272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014700100300902707, + "loss": 1.163, + "theoretical_loss": 3.3864723145555806, + "tokens_seen": 2339897344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014698094282848546, + "loss": 1.2844, + "theoretical_loss": 3.386456982330663, + "tokens_seen": 2340028416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014696088264794382, + "loss": 1.2379, + "theoretical_loss": 3.3864416512049744, + "tokens_seen": 2340159488 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001469408224674022, + "loss": 1.3499, + "theoretical_loss": 3.386426321178375, + "tokens_seen": 2340290560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014692076228686057, + "loss": 1.3988, + "theoretical_loss": 3.3864109922507244, + "tokens_seen": 2340421632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014690070210631896, + "loss": 1.2656, + "theoretical_loss": 3.3863956644218822, + "tokens_seen": 2340552704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014688064192577732, + "loss": 1.3058, + "theoretical_loss": 3.3863803376917083, + "tokens_seen": 2340683776 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001468605817452357, + "loss": 1.2797, + "theoretical_loss": 3.386365012060063, + "tokens_seen": 2340814848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014684052156469407, + "loss": 1.2114, + "theoretical_loss": 3.3863496875268044, + "tokens_seen": 2340945920 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014682046138415243, + "loss": 1.2335, + "theoretical_loss": 3.386334364091794, + "tokens_seen": 2341076992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014680040120361082, + "loss": 1.2834, + "theoretical_loss": 3.38631904175489, + "tokens_seen": 2341208064 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.45869991183280945, + "objective/train/avg_token_score": 0.027463437989354134, + "objective/train/docs_used": 1320683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3013819456100464, + "objective/train/original_loss": 4.03317403793335, + "objective/train/theoretical_loss": 3.386311380998185, + "objective/train/tokens_used": 2341273600, + "objective/train/unlikelihood_loss": 0.1340867578983307, + "theoretical_loss": 3.386311380998185, + "tokens_seen": 2341273600 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001467803410230692, + "loss": 1.2757, + "theoretical_loss": 3.386303720515954, + "tokens_seen": 2341339136 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001467602808425276, + "loss": 1.3833, + "theoretical_loss": 3.3862884003748444, + "tokens_seen": 2341470208 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014674022066198597, + "loss": 1.4286, + "theoretical_loss": 3.3862730813314217, + "tokens_seen": 2341601280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014672016048144435, + "loss": 1.2255, + "theoretical_loss": 3.386257763385546, + "tokens_seen": 2341732352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014670010030090272, + "loss": 1.3098, + "theoretical_loss": 3.386242446537077, + "tokens_seen": 2341863424 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001466800401203611, + "loss": 1.3882, + "theoretical_loss": 3.3862271307858745, + "tokens_seen": 2341994496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014665997993981947, + "loss": 1.2, + "theoretical_loss": 3.386211816131799, + "tokens_seen": 2342125568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014663991975927786, + "loss": 1.1943, + "theoretical_loss": 3.38619650257471, + "tokens_seen": 2342256640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014661985957873622, + "loss": 1.1421, + "theoretical_loss": 3.3861811901144674, + "tokens_seen": 2342387712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014659979939819458, + "loss": 1.3893, + "theoretical_loss": 3.3861658787509317, + "tokens_seen": 2342518784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014657973921765297, + "loss": 1.1191, + "theoretical_loss": 3.3861505684839632, + "tokens_seen": 2342649856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014655967903711133, + "loss": 1.2496, + "theoretical_loss": 3.3861352593134217, + "tokens_seen": 2342780928 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.3646138906478882, + "objective/train/avg_token_score": 0.01468344870954752, + "objective/train/docs_used": 1321245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0848995447158813, + "objective/train/original_loss": 3.958277463912964, + "objective/train/theoretical_loss": 3.3861199512391673, + "objective/train/tokens_used": 2342912000, + "objective/train/unlikelihood_loss": 0.11130140721797943, + "theoretical_loss": 3.3861199512391673, + "tokens_seen": 2342912000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014653961885656972, + "loss": 1.4238, + "theoretical_loss": 3.3861199512391673, + "tokens_seen": 2342912000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014651955867602809, + "loss": 1.2151, + "theoretical_loss": 3.38610464426106, + "tokens_seen": 2343043072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014649949849548647, + "loss": 1.3007, + "theoretical_loss": 3.3860893383789605, + "tokens_seen": 2343174144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014647943831494484, + "loss": 1.3365, + "theoretical_loss": 3.386074033592729, + "tokens_seen": 2343305216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014645937813440323, + "loss": 1.3553, + "theoretical_loss": 3.386058729902225, + "tokens_seen": 2343436288 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001464393179538616, + "loss": 1.25, + "theoretical_loss": 3.3860434273073095, + "tokens_seen": 2343567360 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014641925777331995, + "loss": 1.1947, + "theoretical_loss": 3.3860281258078424, + "tokens_seen": 2343698432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014639919759277834, + "loss": 1.2555, + "theoretical_loss": 3.3860128254036845, + "tokens_seen": 2343829504 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001463791374122367, + "loss": 1.2196, + "theoretical_loss": 3.3859975260946955, + "tokens_seen": 2343960576 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001463590772316951, + "loss": 1.3273, + "theoretical_loss": 3.3859822278807363, + "tokens_seen": 2344091648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014633901705115345, + "loss": 1.3282, + "theoretical_loss": 3.385966930761667, + "tokens_seen": 2344222720 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014631895687061184, + "loss": 1.2934, + "theoretical_loss": 3.3859516347373484, + "tokens_seen": 2344353792 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001462988966900702, + "loss": 1.3365, + "theoretical_loss": 3.3859363398076407, + "tokens_seen": 2344484864 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.2599584460258484, + "objective/train/avg_token_score": 0.022275226190686226, + "objective/train/docs_used": 1321833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.805070161819458, + "objective/train/original_loss": 4.368040084838867, + "objective/train/theoretical_loss": 3.3859286927532217, + "objective/train/tokens_used": 2344550400, + "objective/train/unlikelihood_loss": 0.13490186631679535, + "theoretical_loss": 3.3859286927532217, + "tokens_seen": 2344550400 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001462788365095286, + "loss": 1.1412, + "theoretical_loss": 3.385921045972404, + "tokens_seen": 2344615936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014625877632898696, + "loss": 1.397, + "theoretical_loss": 3.3859057532314996, + "tokens_seen": 2344747008 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014623871614844535, + "loss": 1.3204, + "theoretical_loss": 3.385890461584787, + "tokens_seen": 2344878080 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001462186559679037, + "loss": 1.3527, + "theoretical_loss": 3.385875171032128, + "tokens_seen": 2345009152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014619859578736207, + "loss": 1.2955, + "theoretical_loss": 3.3858598815733822, + "tokens_seen": 2345140224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014617853560682046, + "loss": 1.2297, + "theoretical_loss": 3.385844593208411, + "tokens_seen": 2345271296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014615847542627882, + "loss": 1.1684, + "theoretical_loss": 3.385829305937074, + "tokens_seen": 2345402368 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001461384152457372, + "loss": 1.1969, + "theoretical_loss": 3.385814019759233, + "tokens_seen": 2345533440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014611835506519557, + "loss": 1.3552, + "theoretical_loss": 3.3857987346747476, + "tokens_seen": 2345664512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014609829488465396, + "loss": 1.2898, + "theoretical_loss": 3.3857834506834794, + "tokens_seen": 2345795584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014607823470411233, + "loss": 1.1435, + "theoretical_loss": 3.3857681677852884, + "tokens_seen": 2345926656 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014605817452357072, + "loss": 1.3085, + "theoretical_loss": 3.385752885980036, + "tokens_seen": 2346057728 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.3234359622001648, + "objective/train/avg_token_score": 0.08222930133342743, + "objective/train/docs_used": 1323122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.006677508354187, + "objective/train/original_loss": 4.702110290527344, + "objective/train/theoretical_loss": 3.385737605267583, + "objective/train/tokens_used": 2346188800, + "objective/train/unlikelihood_loss": 0.12849514186382294, + "theoretical_loss": 3.385737605267583, + "tokens_seen": 2346188800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014603811434302908, + "loss": 1.2703, + "theoretical_loss": 3.385737605267583, + "tokens_seen": 2346188800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014601805416248747, + "loss": 1.2451, + "theoretical_loss": 3.3857223256477895, + "tokens_seen": 2346319872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014599799398194586, + "loss": 1.4869, + "theoretical_loss": 3.385707047120517, + "tokens_seen": 2346450944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014597793380140422, + "loss": 1.1946, + "theoretical_loss": 3.385691769685626, + "tokens_seen": 2346582016 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001459578736208626, + "loss": 1.3251, + "theoretical_loss": 3.3856764933429773, + "tokens_seen": 2346713088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014593781344032097, + "loss": 1.0841, + "theoretical_loss": 3.3856612180924324, + "tokens_seen": 2346844160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014591775325977936, + "loss": 1.3366, + "theoretical_loss": 3.3856459439338513, + "tokens_seen": 2346975232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014589769307923772, + "loss": 1.2187, + "theoretical_loss": 3.385630670867096, + "tokens_seen": 2347106304 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001458776328986961, + "loss": 1.3514, + "theoretical_loss": 3.385615398892027, + "tokens_seen": 2347237376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014585757271815447, + "loss": 1.3035, + "theoretical_loss": 3.3856001280085053, + "tokens_seen": 2347368448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014583751253761284, + "loss": 1.1709, + "theoretical_loss": 3.385584858216392, + "tokens_seen": 2347499520 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014581745235707122, + "loss": 1.1957, + "theoretical_loss": 3.3855695895155478, + "tokens_seen": 2347630592 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001457973921765296, + "loss": 1.276, + "theoretical_loss": 3.3855543219058344, + "tokens_seen": 2347761664 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.3486681282520294, + "objective/train/avg_token_score": 0.013085623271763325, + "objective/train/docs_used": 1323884, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0539867877960205, + "objective/train/original_loss": 3.7258827686309814, + "objective/train/theoretical_loss": 3.385546688510108, + "objective/train/tokens_used": 2347827200, + "objective/train/unlikelihood_loss": 0.21776022017002106, + "theoretical_loss": 3.385546688510108, + "tokens_seen": 2347827200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014577733199598798, + "loss": 1.3016, + "theoretical_loss": 3.3855390553871128, + "tokens_seen": 2347892736 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014575727181544634, + "loss": 1.2899, + "theoretical_loss": 3.385523789959244, + "tokens_seen": 2348023808 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014573721163490473, + "loss": 1.1407, + "theoretical_loss": 3.3855085256220887, + "tokens_seen": 2348154880 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001457171514543631, + "loss": 1.2394, + "theoretical_loss": 3.3854932623755087, + "tokens_seen": 2348285952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014569709127382148, + "loss": 1.414, + "theoretical_loss": 3.3854780002193654, + "tokens_seen": 2348417024 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014567703109327984, + "loss": 1.2124, + "theoretical_loss": 3.3854627391535193, + "tokens_seen": 2348548096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014565697091273823, + "loss": 1.3979, + "theoretical_loss": 3.385447479177832, + "tokens_seen": 2348679168 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001456369107321966, + "loss": 1.3363, + "theoretical_loss": 3.385432220292165, + "tokens_seen": 2348810240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014561685055165496, + "loss": 1.4221, + "theoretical_loss": 3.3854169624963797, + "tokens_seen": 2348941312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014559679037111334, + "loss": 1.2587, + "theoretical_loss": 3.385401705790337, + "tokens_seen": 2349072384 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001455767301905717, + "loss": 1.1741, + "theoretical_loss": 3.3853864501738986, + "tokens_seen": 2349203456 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001455566700100301, + "loss": 1.2612, + "theoretical_loss": 3.385371195646926, + "tokens_seen": 2349334528 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.38947945833206177, + "objective/train/avg_token_score": 0.014553938992321491, + "objective/train/docs_used": 1325195, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.333880066871643, + "objective/train/original_loss": 3.964092969894409, + "objective/train/theoretical_loss": 3.38535594220928, + "objective/train/tokens_used": 2349465600, + "objective/train/unlikelihood_loss": 0.16539306938648224, + "theoretical_loss": 3.38535594220928, + "tokens_seen": 2349465600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014553660982948846, + "loss": 1.1432, + "theoretical_loss": 3.38535594220928, + "tokens_seen": 2349465600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014551654964894685, + "loss": 1.254, + "theoretical_loss": 3.3853406898608225, + "tokens_seen": 2349596672 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454964894684052, + "loss": 1.2223, + "theoretical_loss": 3.385325438601415, + "tokens_seen": 2349727744 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454764292878636, + "loss": 1.1795, + "theoretical_loss": 3.385310188430919, + "tokens_seen": 2349858816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014545636910732196, + "loss": 1.235, + "theoretical_loss": 3.385294939349196, + "tokens_seen": 2349989888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014543630892678032, + "loss": 1.3284, + "theoretical_loss": 3.385279691356107, + "tokens_seen": 2350120960 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454162487462387, + "loss": 1.2728, + "theoretical_loss": 3.385264444451515, + "tokens_seen": 2350252032 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014539618856569708, + "loss": 1.2121, + "theoretical_loss": 3.38524919863528, + "tokens_seen": 2350383104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014537612838515546, + "loss": 1.3035, + "theoretical_loss": 3.3852339539072647, + "tokens_seen": 2350514176 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014535606820461383, + "loss": 1.2557, + "theoretical_loss": 3.38521871026733, + "tokens_seen": 2350645248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014533600802407222, + "loss": 1.048, + "theoretical_loss": 3.385203467715338, + "tokens_seen": 2350776320 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014531594784353058, + "loss": 1.2861, + "theoretical_loss": 3.38518822625115, + "tokens_seen": 2350907392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014529588766298897, + "loss": 1.145, + "theoretical_loss": 3.3851729858746284, + "tokens_seen": 2351038464 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.4478861093521118, + "objective/train/avg_token_score": 0.004579076077789068, + "objective/train/docs_used": 1325921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3135968446731567, + "objective/train/original_loss": 3.6563305854797363, + "objective/train/theoretical_loss": 3.385165366094199, + "objective/train/tokens_used": 2351104000, + "objective/train/unlikelihood_loss": 0.21085220575332642, + "theoretical_loss": 3.385165366094199, + "tokens_seen": 2351104000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014527582748244733, + "loss": 1.3117, + "theoretical_loss": 3.3851577465856346, + "tokens_seen": 2351169536 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014525576730190575, + "loss": 1.2243, + "theoretical_loss": 3.38514250838403, + "tokens_seen": 2351300608 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001452357071213641, + "loss": 1.3269, + "theoretical_loss": 3.3851272712696776, + "tokens_seen": 2351431680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014521564694082247, + "loss": 1.3939, + "theoretical_loss": 3.3851120352424378, + "tokens_seen": 2351562752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014519558676028086, + "loss": 1.3175, + "theoretical_loss": 3.385096800302173, + "tokens_seen": 2351693824 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014517552657973922, + "loss": 1.234, + "theoretical_loss": 3.3850815664487452, + "tokens_seen": 2351824896 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001451554663991976, + "loss": 1.3119, + "theoretical_loss": 3.3850663336820164, + "tokens_seen": 2351955968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014513540621865597, + "loss": 1.319, + "theoretical_loss": 3.385051102001848, + "tokens_seen": 2352087040 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014511534603811436, + "loss": 1.3374, + "theoretical_loss": 3.3850358714081024, + "tokens_seen": 2352218112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014509528585757273, + "loss": 1.2566, + "theoretical_loss": 3.385020641900642, + "tokens_seen": 2352349184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014507522567703112, + "loss": 1.1881, + "theoretical_loss": 3.3850054134793277, + "tokens_seen": 2352480256 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014505516549648948, + "loss": 1.1345, + "theoretical_loss": 3.384990186144022, + "tokens_seen": 2352611328 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.27938660979270935, + "objective/train/avg_token_score": 0.011007433757185936, + "objective/train/docs_used": 1327039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8740823864936829, + "objective/train/original_loss": 3.8632776737213135, + "objective/train/theoretical_loss": 3.3849749598945875, + "objective/train/tokens_used": 2352742400, + "objective/train/unlikelihood_loss": 0.22852568328380585, + "theoretical_loss": 3.3849749598945875, + "tokens_seen": 2352742400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014503510531594784, + "loss": 1.3322, + "theoretical_loss": 3.3849749598945875, + "tokens_seen": 2352742400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014501504513540623, + "loss": 1.4113, + "theoretical_loss": 3.3849597347308853, + "tokens_seen": 2352873472 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001449949849548646, + "loss": 1.2253, + "theoretical_loss": 3.3849445106527787, + "tokens_seen": 2353004544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014497492477432298, + "loss": 1.2882, + "theoretical_loss": 3.3849292876601287, + "tokens_seen": 2353135616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014495486459378134, + "loss": 1.3742, + "theoretical_loss": 3.384914065752798, + "tokens_seen": 2353266688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014493480441323973, + "loss": 1.3279, + "theoretical_loss": 3.3848988449306487, + "tokens_seen": 2353397760 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001449147442326981, + "loss": 1.3415, + "theoretical_loss": 3.384883625193543, + "tokens_seen": 2353528832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014489468405215648, + "loss": 1.2946, + "theoretical_loss": 3.3848684065413437, + "tokens_seen": 2353659904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014487462387161485, + "loss": 1.3151, + "theoretical_loss": 3.3848531889739117, + "tokens_seen": 2353790976 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001448545636910732, + "loss": 1.3157, + "theoretical_loss": 3.3848379724911104, + "tokens_seen": 2353922048 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001448345035105316, + "loss": 1.237, + "theoretical_loss": 3.3848227570928016, + "tokens_seen": 2354053120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014481444332998996, + "loss": 1.2919, + "theoretical_loss": 3.384807542778848, + "tokens_seen": 2354184192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014479438314944835, + "loss": 1.1732, + "theoretical_loss": 3.3847923295491116, + "tokens_seen": 2354315264 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.4960899353027344, + "objective/train/avg_token_score": 0.03330947458744049, + "objective/train/docs_used": 1327759, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6360046863555908, + "objective/train/original_loss": 3.9464986324310303, + "objective/train/theoretical_loss": 3.384784723340782, + "objective/train/tokens_used": 2354380800, + "objective/train/unlikelihood_loss": 0.15787065029144287, + "theoretical_loss": 3.384784723340782, + "tokens_seen": 2354380800 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447743229689067, + "loss": 1.6028, + "theoretical_loss": 3.384777117403455, + "tokens_seen": 2354446336 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447542627883651, + "loss": 1.3254, + "theoretical_loss": 3.384761906341741, + "tokens_seen": 2354577408 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014473420260782346, + "loss": 1.3796, + "theoretical_loss": 3.384746696363831, + "tokens_seen": 2354708480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014471414242728185, + "loss": 1.1389, + "theoretical_loss": 3.3847314874695877, + "tokens_seen": 2354839552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014469408224674021, + "loss": 1.2198, + "theoretical_loss": 3.384716279658875, + "tokens_seen": 2354970624 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001446740220661986, + "loss": 1.141, + "theoretical_loss": 3.3847010729315534, + "tokens_seen": 2355101696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014465396188565697, + "loss": 1.2347, + "theoretical_loss": 3.3846858672874864, + "tokens_seen": 2355232768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014463390170511533, + "loss": 1.2263, + "theoretical_loss": 3.384670662726537, + "tokens_seen": 2355363840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014461384152457372, + "loss": 1.3337, + "theoretical_loss": 3.384655459248567, + "tokens_seen": 2355494912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014459378134403208, + "loss": 1.228, + "theoretical_loss": 3.3846402568534395, + "tokens_seen": 2355625984 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014457372116349047, + "loss": 1.2085, + "theoretical_loss": 3.384625055541017, + "tokens_seen": 2355757056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014455366098294883, + "loss": 1.4902, + "theoretical_loss": 3.3846098553111617, + "tokens_seen": 2355888128 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.40683040022850037, + "objective/train/avg_token_score": 0.026728538796305656, + "objective/train/docs_used": 1329159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.244175672531128, + "objective/train/original_loss": 3.6951820850372314, + "objective/train/theoretical_loss": 3.3845946561637366, + "objective/train/tokens_used": 2356019200, + "objective/train/unlikelihood_loss": 0.20782817900180817, + "theoretical_loss": 3.3845946561637366, + "tokens_seen": 2356019200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014453360080240722, + "loss": 1.1872, + "theoretical_loss": 3.3845946561637366, + "tokens_seen": 2356019200 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001445135406218656, + "loss": 1.4583, + "theoretical_loss": 3.384579458098605, + "tokens_seen": 2356150272 + }, + { + "epoch": 0.71, + "learning_rate": 0.000144493480441324, + "loss": 1.2184, + "theoretical_loss": 3.3845642611156284, + "tokens_seen": 2356281344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014447342026078236, + "loss": 1.2708, + "theoretical_loss": 3.384549065214671, + "tokens_seen": 2356412416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014445336008024072, + "loss": 1.342, + "theoretical_loss": 3.384533870395594, + "tokens_seen": 2356543488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014443329989969911, + "loss": 1.3672, + "theoretical_loss": 3.384518676658262, + "tokens_seen": 2356674560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014441323971915748, + "loss": 1.4732, + "theoretical_loss": 3.384503484002536, + "tokens_seen": 2356805632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014439317953861587, + "loss": 1.3677, + "theoretical_loss": 3.38448829242828, + "tokens_seen": 2356936704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014437311935807423, + "loss": 1.4457, + "theoretical_loss": 3.384473101935357, + "tokens_seen": 2357067776 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014435305917753262, + "loss": 1.2357, + "theoretical_loss": 3.384457912523629, + "tokens_seen": 2357198848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014433299899699098, + "loss": 1.1466, + "theoretical_loss": 3.3844427241929598, + "tokens_seen": 2357329920 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014431293881644937, + "loss": 1.1395, + "theoretical_loss": 3.3844275369432113, + "tokens_seen": 2357460992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014429287863590773, + "loss": 1.3592, + "theoretical_loss": 3.384412350774248, + "tokens_seen": 2357592064 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.439943790435791, + "objective/train/avg_token_score": 0.007430693134665489, + "objective/train/docs_used": 1329871, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3364849090576172, + "objective/train/original_loss": 3.4891281127929688, + "objective/train/theoretical_loss": 3.384404758095018, + "objective/train/tokens_used": 2357657600, + "objective/train/unlikelihood_loss": 0.2261323183774948, + "theoretical_loss": 3.384404758095018, + "tokens_seen": 2357657600 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001442728184553661, + "loss": 1.3993, + "theoretical_loss": 3.3843971656859315, + "tokens_seen": 2357723136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014425275827482448, + "loss": 1.3267, + "theoretical_loss": 3.384381981678126, + "tokens_seen": 2357854208 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014423269809428284, + "loss": 1.2963, + "theoretical_loss": 3.384366798750694, + "tokens_seen": 2357985280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014421263791374123, + "loss": 1.2351, + "theoretical_loss": 3.384351616903498, + "tokens_seen": 2358116352 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001441925777331996, + "loss": 1.1498, + "theoretical_loss": 3.3843364361364023, + "tokens_seen": 2358247424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014417251755265799, + "loss": 1.4604, + "theoretical_loss": 3.384321256449269, + "tokens_seen": 2358378496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014415245737211635, + "loss": 1.193, + "theoretical_loss": 3.384306077841962, + "tokens_seen": 2358509568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014413239719157474, + "loss": 1.2227, + "theoretical_loss": 3.3842909003143435, + "tokens_seen": 2358640640 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001441123370110331, + "loss": 1.1076, + "theoretical_loss": 3.3842757238662777, + "tokens_seen": 2358771712 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440922768304915, + "loss": 1.3452, + "theoretical_loss": 3.3842605484976276, + "tokens_seen": 2358902784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014407221664994985, + "loss": 1.2619, + "theoretical_loss": 3.3842453742082563, + "tokens_seen": 2359033856 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440521564694082, + "loss": 1.1528, + "theoretical_loss": 3.3842302009980267, + "tokens_seen": 2359164928 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.5124022364616394, + "objective/train/avg_token_score": 0.009839991107583046, + "objective/train/docs_used": 1331053, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5842231512069702, + "objective/train/original_loss": 3.6717052459716797, + "objective/train/theoretical_loss": 3.384215028866803, + "objective/train/tokens_used": 2359296000, + "objective/train/unlikelihood_loss": 0.15382184088230133, + "theoretical_loss": 3.384215028866803, + "tokens_seen": 2359296000 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440320962888666, + "loss": 1.3236, + "theoretical_loss": 3.384215028866803, + "tokens_seen": 2359296000 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014401203610832496, + "loss": 1.3693, + "theoretical_loss": 3.384199857814448, + "tokens_seen": 2359427072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014399197592778335, + "loss": 1.2356, + "theoretical_loss": 3.384184687840825, + "tokens_seen": 2359558144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014397191574724172, + "loss": 1.1228, + "theoretical_loss": 3.3841695189457974, + "tokens_seen": 2359689216 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001439518555667001, + "loss": 1.3163, + "theoretical_loss": 3.384154351129229, + "tokens_seen": 2359820288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014393179538615847, + "loss": 1.3398, + "theoretical_loss": 3.3841391843909827, + "tokens_seen": 2359951360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014391173520561686, + "loss": 1.1676, + "theoretical_loss": 3.384124018730922, + "tokens_seen": 2360082432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014389167502507522, + "loss": 1.2713, + "theoretical_loss": 3.384108854148911, + "tokens_seen": 2360213504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014387161484453358, + "loss": 1.3219, + "theoretical_loss": 3.3840936906448125, + "tokens_seen": 2360344576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014385155466399197, + "loss": 1.2529, + "theoretical_loss": 3.38407852821849, + "tokens_seen": 2360475648 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014383149448345033, + "loss": 1.214, + "theoretical_loss": 3.3840633668698077, + "tokens_seen": 2360606720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014381143430290872, + "loss": 1.3357, + "theoretical_loss": 3.384048206598629, + "tokens_seen": 2360737792 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014379137412236708, + "loss": 1.3329, + "theoretical_loss": 3.384033047404817, + "tokens_seen": 2360868864 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.41361191868782043, + "objective/train/avg_token_score": 0.006495154928416014, + "objective/train/docs_used": 1331743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2481850385665894, + "objective/train/original_loss": 3.721306562423706, + "objective/train/theoretical_loss": 3.384025468211881, + "objective/train/tokens_used": 2360934400, + "objective/train/unlikelihood_loss": 0.138692706823349, + "theoretical_loss": 3.384025468211881, + "tokens_seen": 2360934400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014377131394182547, + "loss": 1.3294, + "theoretical_loss": 3.384017889288236, + "tokens_seen": 2360999936 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014375125376128386, + "loss": 1.2858, + "theoretical_loss": 3.384002732248749, + "tokens_seen": 2361131008 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014373119358074225, + "loss": 1.1871, + "theoretical_loss": 3.38398757628622, + "tokens_seen": 2361262080 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014371113340020062, + "loss": 1.2444, + "theoretical_loss": 3.3839724214005127, + "tokens_seen": 2361393152 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143691073219659, + "loss": 1.168, + "theoretical_loss": 3.383957267591491, + "tokens_seen": 2361524224 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014367101303911737, + "loss": 1.354, + "theoretical_loss": 3.3839421148590185, + "tokens_seen": 2361655296 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014365095285857573, + "loss": 1.3547, + "theoretical_loss": 3.383926963202959, + "tokens_seen": 2361786368 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014363089267803412, + "loss": 1.2831, + "theoretical_loss": 3.3839118126231758, + "tokens_seen": 2361917440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014361083249749248, + "loss": 1.3337, + "theoretical_loss": 3.3838966631195335, + "tokens_seen": 2362048512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014359077231695087, + "loss": 1.3413, + "theoretical_loss": 3.383881514691896, + "tokens_seen": 2362179584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014357071213640923, + "loss": 1.3521, + "theoretical_loss": 3.383866367340126, + "tokens_seen": 2362310656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014355065195586762, + "loss": 1.2994, + "theoretical_loss": 3.3838512210640888, + "tokens_seen": 2362441728 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.40261486172676086, + "objective/train/avg_token_score": 0.0218084417283535, + "objective/train/docs_used": 1333010, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3427903652191162, + "objective/train/original_loss": 3.8764054775238037, + "objective/train/theoretical_loss": 3.3838360758636474, + "objective/train/tokens_used": 2362572800, + "objective/train/unlikelihood_loss": 0.13625091314315796, + "theoretical_loss": 3.3838360758636474, + "tokens_seen": 2362572800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014353059177532598, + "loss": 1.2701, + "theoretical_loss": 3.3838360758636474, + "tokens_seen": 2362572800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014351053159478437, + "loss": 1.2463, + "theoretical_loss": 3.3838209317386667, + "tokens_seen": 2362703872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014349047141424274, + "loss": 1.2236, + "theoretical_loss": 3.3838057886890094, + "tokens_seen": 2362834944 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001434704112337011, + "loss": 1.2061, + "theoretical_loss": 3.3837906467145404, + "tokens_seen": 2362966016 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001434503510531595, + "loss": 1.263, + "theoretical_loss": 3.383775505815124, + "tokens_seen": 2363097088 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014343029087261785, + "loss": 1.1771, + "theoretical_loss": 3.3837603659906232, + "tokens_seen": 2363228160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014341023069207624, + "loss": 1.253, + "theoretical_loss": 3.3837452272409028, + "tokens_seen": 2363359232 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001433901705115346, + "loss": 1.3266, + "theoretical_loss": 3.3837300895658267, + "tokens_seen": 2363490304 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143370110330993, + "loss": 1.3074, + "theoretical_loss": 3.3837149529652586, + "tokens_seen": 2363621376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014335005015045135, + "loss": 1.225, + "theoretical_loss": 3.383699817439064, + "tokens_seen": 2363752448 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014332998996990974, + "loss": 1.2403, + "theoretical_loss": 3.3836846829871057, + "tokens_seen": 2363883520 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001433099297893681, + "loss": 1.1998, + "theoretical_loss": 3.383669549609248, + "tokens_seen": 2364014592 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014328986960882647, + "loss": 1.3682, + "theoretical_loss": 3.383654417305356, + "tokens_seen": 2364145664 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.5059261918067932, + "objective/train/avg_token_score": 0.020548617467284203, + "objective/train/docs_used": 1333580, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6175297498703003, + "objective/train/original_loss": 3.450058698654175, + "objective/train/theoretical_loss": 3.3836468515561045, + "objective/train/tokens_used": 2364211200, + "objective/train/unlikelihood_loss": 0.1951136440038681, + "theoretical_loss": 3.3836468515561045, + "tokens_seen": 2364211200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014326980942828486, + "loss": 1.3238, + "theoretical_loss": 3.3836392860752933, + "tokens_seen": 2364276736 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014324974924774322, + "loss": 1.2946, + "theoretical_loss": 3.383624155918924, + "tokens_seen": 2364407808 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001432296890672016, + "loss": 1.2953, + "theoretical_loss": 3.3836090268361136, + "tokens_seen": 2364538880 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014320962888665997, + "loss": 1.1691, + "theoretical_loss": 3.383593898826725, + "tokens_seen": 2364669952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014318956870611836, + "loss": 1.2836, + "theoretical_loss": 3.383578771890623, + "tokens_seen": 2364801024 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014316950852557672, + "loss": 1.3337, + "theoretical_loss": 3.383563646027672, + "tokens_seen": 2364932096 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001431494483450351, + "loss": 1.327, + "theoretical_loss": 3.3835485212377368, + "tokens_seen": 2365063168 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014312938816449347, + "loss": 1.2018, + "theoretical_loss": 3.383533397520681, + "tokens_seen": 2365194240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014310932798395183, + "loss": 1.3984, + "theoretical_loss": 3.38351827487637, + "tokens_seen": 2365325312 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014308926780341022, + "loss": 1.3616, + "theoretical_loss": 3.3835031533046678, + "tokens_seen": 2365456384 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014306920762286859, + "loss": 1.4252, + "theoretical_loss": 3.3834880328054386, + "tokens_seen": 2365587456 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014304914744232698, + "loss": 1.4481, + "theoretical_loss": 3.3834729133785473, + "tokens_seen": 2365718528 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.4157502353191376, + "objective/train/avg_token_score": 0.015357684344053268, + "objective/train/docs_used": 1334672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.222555160522461, + "objective/train/original_loss": 3.7802464962005615, + "objective/train/theoretical_loss": 3.3834577950238582, + "objective/train/tokens_used": 2365849600, + "objective/train/unlikelihood_loss": 0.14460502564907074, + "theoretical_loss": 3.3834577950238582, + "tokens_seen": 2365849600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014302908726178534, + "loss": 1.2568, + "theoretical_loss": 3.3834577950238582, + "tokens_seen": 2365849600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014300902708124375, + "loss": 1.2807, + "theoretical_loss": 3.383442677741236, + "tokens_seen": 2365980672 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014298896690070212, + "loss": 1.2866, + "theoretical_loss": 3.383427561530546, + "tokens_seen": 2366111744 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001429689067201605, + "loss": 1.3964, + "theoretical_loss": 3.3834124463916515, + "tokens_seen": 2366242816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014294884653961887, + "loss": 1.4133, + "theoretical_loss": 3.3833973323244178, + "tokens_seen": 2366373888 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014292878635907726, + "loss": 1.4816, + "theoretical_loss": 3.3833822193287095, + "tokens_seen": 2366504960 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014290872617853562, + "loss": 1.2742, + "theoretical_loss": 3.3833671074043914, + "tokens_seen": 2366636032 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014288866599799398, + "loss": 1.2083, + "theoretical_loss": 3.3833519965513283, + "tokens_seen": 2366767104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014286860581745237, + "loss": 1.4535, + "theoretical_loss": 3.3833368867693845, + "tokens_seen": 2366898176 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014284854563691073, + "loss": 1.3432, + "theoretical_loss": 3.3833217780584253, + "tokens_seen": 2367029248 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014282848545636912, + "loss": 1.2758, + "theoretical_loss": 3.3833066704183152, + "tokens_seen": 2367160320 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014280842527582749, + "loss": 1.4377, + "theoretical_loss": 3.383291563848919, + "tokens_seen": 2367291392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014278836509528587, + "loss": 1.36, + "theoretical_loss": 3.383276458350102, + "tokens_seen": 2367422464 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.36870723962783813, + "objective/train/avg_token_score": 0.04109158366918564, + "objective/train/docs_used": 1335172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.139473557472229, + "objective/train/original_loss": 4.196728229522705, + "objective/train/theoretical_loss": 3.383268906002118, + "objective/train/tokens_used": 2367488000, + "objective/train/unlikelihood_loss": 0.11203251779079437, + "theoretical_loss": 3.383268906002118, + "tokens_seen": 2367488000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014276830491474424, + "loss": 1.2822, + "theoretical_loss": 3.383261353921728, + "tokens_seen": 2367553536 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014274824473420263, + "loss": 1.2262, + "theoretical_loss": 3.383246250563663, + "tokens_seen": 2367684608 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142728184553661, + "loss": 1.3317, + "theoretical_loss": 3.383231148275771, + "tokens_seen": 2367815680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014270812437311935, + "loss": 1.5461, + "theoretical_loss": 3.383216047057918, + "tokens_seen": 2367946752 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014268806419257774, + "loss": 1.4171, + "theoretical_loss": 3.3832009469099678, + "tokens_seen": 2368077824 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001426680040120361, + "loss": 1.2509, + "theoretical_loss": 3.383185847831786, + "tokens_seen": 2368208896 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001426479438314945, + "loss": 1.3957, + "theoretical_loss": 3.383170749823238, + "tokens_seen": 2368339968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014262788365095285, + "loss": 1.1521, + "theoretical_loss": 3.3831556528841884, + "tokens_seen": 2368471040 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014260782347041124, + "loss": 1.3748, + "theoretical_loss": 3.383140557014502, + "tokens_seen": 2368602112 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425877632898696, + "loss": 1.313, + "theoretical_loss": 3.383125462214044, + "tokens_seen": 2368733184 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142567703109328, + "loss": 1.283, + "theoretical_loss": 3.3831103684826798, + "tokens_seen": 2368864256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014254764292878636, + "loss": 1.3501, + "theoretical_loss": 3.3830952758202746, + "tokens_seen": 2368995328 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.501099705696106, + "objective/train/avg_token_score": 0.008159796707332134, + "objective/train/docs_used": 1335842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5793068408966064, + "objective/train/original_loss": 3.3050968647003174, + "objective/train/theoretical_loss": 3.383080184226693, + "objective/train/tokens_used": 2369126400, + "objective/train/unlikelihood_loss": 0.18719884753227234, + "theoretical_loss": 3.383080184226693, + "tokens_seen": 2369126400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014252758274824475, + "loss": 1.2295, + "theoretical_loss": 3.383080184226693, + "tokens_seen": 2369126400 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425075225677031, + "loss": 1.3004, + "theoretical_loss": 3.3830650937018008, + "tokens_seen": 2369257472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014248746238716147, + "loss": 1.4841, + "theoretical_loss": 3.3830500042454625, + "tokens_seen": 2369388544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014246740220661986, + "loss": 1.2337, + "theoretical_loss": 3.383034915857544, + "tokens_seen": 2369519616 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014244734202607822, + "loss": 1.2152, + "theoretical_loss": 3.3830198285379103, + "tokens_seen": 2369650688 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001424272818455366, + "loss": 1.26, + "theoretical_loss": 3.3830047422864267, + "tokens_seen": 2369781760 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014240722166499497, + "loss": 1.4068, + "theoretical_loss": 3.382989657102959, + "tokens_seen": 2369912832 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014238716148445336, + "loss": 1.2739, + "theoretical_loss": 3.3829745729873713, + "tokens_seen": 2370043904 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014236710130391173, + "loss": 1.2727, + "theoretical_loss": 3.38295948993953, + "tokens_seen": 2370174976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014234704112337011, + "loss": 1.4192, + "theoretical_loss": 3.3829444079593003, + "tokens_seen": 2370306048 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014232698094282848, + "loss": 1.4422, + "theoretical_loss": 3.382929327046547, + "tokens_seen": 2370437120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014230692076228684, + "loss": 1.1618, + "theoretical_loss": 3.3829142472011364, + "tokens_seen": 2370568192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014228686058174523, + "loss": 1.2085, + "theoretical_loss": 3.382899168422933, + "tokens_seen": 2370699264 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.2762707769870758, + "objective/train/avg_token_score": 0.0214913971722126, + "objective/train/docs_used": 1336773, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8089417219161987, + "objective/train/original_loss": 4.383663177490234, + "objective/train/theoretical_loss": 3.382891629433993, + "objective/train/tokens_used": 2370764800, + "objective/train/unlikelihood_loss": 0.11337920278310776, + "theoretical_loss": 3.382891629433993, + "tokens_seen": 2370764800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014226680040120362, + "loss": 1.1985, + "theoretical_loss": 3.382884090711803, + "tokens_seen": 2370830336 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142246740220662, + "loss": 1.2297, + "theoretical_loss": 3.3828690140676123, + "tokens_seen": 2370961408 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014222668004012037, + "loss": 1.3571, + "theoretical_loss": 3.3828539384902254, + "tokens_seen": 2371092480 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014220661985957876, + "loss": 1.2645, + "theoretical_loss": 3.3828388639795084, + "tokens_seen": 2371223552 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014218655967903712, + "loss": 1.3279, + "theoretical_loss": 3.3828237905353267, + "tokens_seen": 2371354624 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001421664994984955, + "loss": 1.3236, + "theoretical_loss": 3.3828087181575457, + "tokens_seen": 2371485696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014214643931795387, + "loss": 1.2337, + "theoretical_loss": 3.382793646846032, + "tokens_seen": 2371616768 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014212637913741223, + "loss": 1.4107, + "theoretical_loss": 3.3827785766006495, + "tokens_seen": 2371747840 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014210631895687062, + "loss": 1.3942, + "theoretical_loss": 3.3827635074212656, + "tokens_seen": 2371878912 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142086258776329, + "loss": 1.186, + "theoretical_loss": 3.382748439307745, + "tokens_seen": 2372009984 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014206619859578738, + "loss": 1.194, + "theoretical_loss": 3.382733372259954, + "tokens_seen": 2372141056 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014204613841524574, + "loss": 1.2382, + "theoretical_loss": 3.3827183062777575, + "tokens_seen": 2372272128 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.4258919954299927, + "objective/train/avg_token_score": 0.01168939657509327, + "objective/train/docs_used": 1337535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4055068492889404, + "objective/train/original_loss": 3.6695053577423096, + "objective/train/theoretical_loss": 3.3827032413610225, + "objective/train/tokens_used": 2372403200, + "objective/train/unlikelihood_loss": 0.15995487570762634, + "theoretical_loss": 3.3827032413610225, + "tokens_seen": 2372403200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014202607823470413, + "loss": 1.3403, + "theoretical_loss": 3.3827032413610225, + "tokens_seen": 2372403200 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001420060180541625, + "loss": 1.3697, + "theoretical_loss": 3.3826881775096136, + "tokens_seen": 2372534272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014198595787362088, + "loss": 1.2389, + "theoretical_loss": 3.3826731147233975, + "tokens_seen": 2372665344 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014196589769307924, + "loss": 1.2737, + "theoretical_loss": 3.382658053002239, + "tokens_seen": 2372796416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014194583751253763, + "loss": 1.2647, + "theoretical_loss": 3.3826429923460055, + "tokens_seen": 2372927488 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141925777331996, + "loss": 1.3392, + "theoretical_loss": 3.382627932754562, + "tokens_seen": 2373058560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014190571715145436, + "loss": 1.2554, + "theoretical_loss": 3.3826128742277737, + "tokens_seen": 2373189632 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014188565697091274, + "loss": 1.3719, + "theoretical_loss": 3.3825978167655077, + "tokens_seen": 2373320704 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001418655967903711, + "loss": 1.3188, + "theoretical_loss": 3.3825827603676295, + "tokens_seen": 2373451776 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001418455366098295, + "loss": 1.1542, + "theoretical_loss": 3.382567705034005, + "tokens_seen": 2373582848 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014182547642928786, + "loss": 1.3693, + "theoretical_loss": 3.3825526507645005, + "tokens_seen": 2373713920 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014180541624874625, + "loss": 1.2993, + "theoretical_loss": 3.3825375975589815, + "tokens_seen": 2373844992 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001417853560682046, + "loss": 1.2939, + "theoretical_loss": 3.3825225454173147, + "tokens_seen": 2373976064 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.4134897291660309, + "objective/train/avg_token_score": 0.010809668339788914, + "objective/train/docs_used": 1338943, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3046631813049316, + "objective/train/original_loss": 3.622081995010376, + "objective/train/theoretical_loss": 3.382515019745384, + "objective/train/tokens_used": 2374041600, + "objective/train/unlikelihood_loss": 0.18257510662078857, + "theoretical_loss": 3.382515019745384, + "tokens_seen": 2374041600 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141765295887663, + "loss": 1.2026, + "theoretical_loss": 3.3825074943393663, + "tokens_seen": 2374107136 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014174523570712136, + "loss": 1.2896, + "theoretical_loss": 3.3824924443250013, + "tokens_seen": 2374238208 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014172517552657972, + "loss": 1.2236, + "theoretical_loss": 3.382477395374087, + "tokens_seen": 2374369280 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001417051153460381, + "loss": 1.1905, + "theoretical_loss": 3.3824623474864888, + "tokens_seen": 2374500352 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014168505516549648, + "loss": 1.3299, + "theoretical_loss": 3.3824473006620734, + "tokens_seen": 2374631424 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014166499498495486, + "loss": 1.2886, + "theoretical_loss": 3.382432254900707, + "tokens_seen": 2374762496 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014164493480441323, + "loss": 1.2438, + "theoretical_loss": 3.382417210202255, + "tokens_seen": 2374893568 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014162487462387162, + "loss": 1.346, + "theoretical_loss": 3.382402166566585, + "tokens_seen": 2375024640 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014160481444332998, + "loss": 1.2322, + "theoretical_loss": 3.382387123993562, + "tokens_seen": 2375155712 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014158475426278837, + "loss": 1.348, + "theoretical_loss": 3.382372082483053, + "tokens_seen": 2375286784 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014156469408224673, + "loss": 1.2501, + "theoretical_loss": 3.3823570420349243, + "tokens_seen": 2375417856 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001415446339017051, + "loss": 1.3974, + "theoretical_loss": 3.3823420026490423, + "tokens_seen": 2375548928 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.46419844031333923, + "objective/train/avg_token_score": 0.013501446694135666, + "objective/train/docs_used": 1339545, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4769748449325562, + "objective/train/original_loss": 3.4556756019592285, + "objective/train/theoretical_loss": 3.382326964325273, + "objective/train/tokens_used": 2375680000, + "objective/train/unlikelihood_loss": 0.2115408480167389, + "theoretical_loss": 3.382326964325273, + "tokens_seen": 2375680000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014152457372116348, + "loss": 1.2378, + "theoretical_loss": 3.382326964325273, + "tokens_seen": 2375680000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014150451354062187, + "loss": 1.3754, + "theoretical_loss": 3.3823119270634825, + "tokens_seen": 2375811072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014148445336008026, + "loss": 1.3735, + "theoretical_loss": 3.3822968908635382, + "tokens_seen": 2375942144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014146439317953862, + "loss": 1.418, + "theoretical_loss": 3.382281855725306, + "tokens_seen": 2376073216 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141444332998997, + "loss": 1.2976, + "theoretical_loss": 3.3822668216486527, + "tokens_seen": 2376204288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014142427281845537, + "loss": 1.2061, + "theoretical_loss": 3.382251788633444, + "tokens_seen": 2376335360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014140421263791376, + "loss": 1.3505, + "theoretical_loss": 3.3822367566795473, + "tokens_seen": 2376466432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014138415245737213, + "loss": 1.2537, + "theoretical_loss": 3.382221725786829, + "tokens_seen": 2376597504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014136409227683052, + "loss": 1.2163, + "theoretical_loss": 3.382206695955155, + "tokens_seen": 2376728576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014134403209628888, + "loss": 1.2021, + "theoretical_loss": 3.3821916671843923, + "tokens_seen": 2376859648 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014132397191574724, + "loss": 1.1587, + "theoretical_loss": 3.382176639474408, + "tokens_seen": 2376990720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014130391173520563, + "loss": 1.2709, + "theoretical_loss": 3.382161612825068, + "tokens_seen": 2377121792 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141283851554664, + "loss": 1.351, + "theoretical_loss": 3.3821465872362397, + "tokens_seen": 2377252864 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.36663001775741577, + "objective/train/avg_token_score": 0.04448649287223816, + "objective/train/docs_used": 1340666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2378889322280884, + "objective/train/original_loss": 4.173747539520264, + "objective/train/theoretical_loss": 3.3821390748394755, + "objective/train/tokens_used": 2377318400, + "objective/train/unlikelihood_loss": 0.13599048554897308, + "theoretical_loss": 3.3821390748394755, + "tokens_seen": 2377318400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014126379137412238, + "loss": 1.3276, + "theoretical_loss": 3.382131562707789, + "tokens_seen": 2377383936 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014124373119358074, + "loss": 1.3442, + "theoretical_loss": 3.382116539239583, + "tokens_seen": 2377515008 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014122367101303913, + "loss": 1.4356, + "theoretical_loss": 3.3821015168314883, + "tokens_seen": 2377646080 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001412036108324975, + "loss": 1.4446, + "theoretical_loss": 3.382086495483372, + "tokens_seen": 2377777152 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014118355065195588, + "loss": 1.3089, + "theoretical_loss": 3.3820714751951004, + "tokens_seen": 2377908224 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014116349047141425, + "loss": 1.2475, + "theoretical_loss": 3.382056455966541, + "tokens_seen": 2378039296 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001411434302908726, + "loss": 1.2365, + "theoretical_loss": 3.38204143779756, + "tokens_seen": 2378170368 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141123370110331, + "loss": 1.236, + "theoretical_loss": 3.382026420688024, + "tokens_seen": 2378301440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014110330992978936, + "loss": 1.2671, + "theoretical_loss": 3.382011404637801, + "tokens_seen": 2378432512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014108324974924775, + "loss": 1.1525, + "theoretical_loss": 3.381996389646757, + "tokens_seen": 2378563584 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001410631895687061, + "loss": 1.3646, + "theoretical_loss": 3.381981375714759, + "tokens_seen": 2378694656 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001410431293881645, + "loss": 1.431, + "theoretical_loss": 3.3819663628416743, + "tokens_seen": 2378825728 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.37188416719436646, + "objective/train/avg_token_score": 0.029728475958108902, + "objective/train/docs_used": 1341274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1770116090774536, + "objective/train/original_loss": 4.1920623779296875, + "objective/train/theoretical_loss": 3.3819513510273698, + "objective/train/tokens_used": 2378956800, + "objective/train/unlikelihood_loss": 0.15678761899471283, + "theoretical_loss": 3.3819513510273698, + "tokens_seen": 2378956800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014102306920762286, + "loss": 1.2869, + "theoretical_loss": 3.3819513510273698, + "tokens_seen": 2378956800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014100300902708125, + "loss": 1.3915, + "theoretical_loss": 3.3819363402717117, + "tokens_seen": 2379087872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014098294884653961, + "loss": 1.2035, + "theoretical_loss": 3.3819213305745683, + "tokens_seen": 2379218944 + }, + { + "epoch": 0.72, + "learning_rate": 0.000140962888665998, + "loss": 1.3032, + "theoretical_loss": 3.3819063219358063, + "tokens_seen": 2379350016 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014094282848545637, + "loss": 1.3351, + "theoretical_loss": 3.381891314355292, + "tokens_seen": 2379481088 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014092276830491473, + "loss": 1.327, + "theoretical_loss": 3.3818763078328935, + "tokens_seen": 2379612160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014090270812437312, + "loss": 1.2953, + "theoretical_loss": 3.381861302368477, + "tokens_seen": 2379743232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014088264794383148, + "loss": 1.2518, + "theoretical_loss": 3.3818462979619106, + "tokens_seen": 2379874304 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014086258776328987, + "loss": 1.3773, + "theoretical_loss": 3.3818312946130606, + "tokens_seen": 2380005376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014084252758274823, + "loss": 1.2115, + "theoretical_loss": 3.381816292321795, + "tokens_seen": 2380136448 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014082246740220662, + "loss": 1.1089, + "theoretical_loss": 3.3818012910879798, + "tokens_seen": 2380267520 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014080240722166498, + "loss": 1.3484, + "theoretical_loss": 3.3817862909114837, + "tokens_seen": 2380398592 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014078234704112337, + "loss": 1.2687, + "theoretical_loss": 3.381771291792173, + "tokens_seen": 2380529664 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.4067082107067108, + "objective/train/avg_token_score": 0.008125965483486652, + "objective/train/docs_used": 1342636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.214895248413086, + "objective/train/original_loss": 3.5503406524658203, + "objective/train/theoretical_loss": 3.381763792628921, + "objective/train/tokens_used": 2380595200, + "objective/train/unlikelihood_loss": 0.20627997815608978, + "theoretical_loss": 3.381763792628921, + "tokens_seen": 2380595200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014076228686058176, + "loss": 1.376, + "theoretical_loss": 3.3817562937299153, + "tokens_seen": 2380660736 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014074222668004012, + "loss": 1.3824, + "theoretical_loss": 3.381741296724578, + "tokens_seen": 2380791808 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001407221664994985, + "loss": 1.1423, + "theoretical_loss": 3.3817263007760285, + "tokens_seen": 2380922880 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014070210631895688, + "loss": 1.3408, + "theoretical_loss": 3.3817113058841337, + "tokens_seen": 2381053952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014068204613841526, + "loss": 1.3696, + "theoretical_loss": 3.3816963120487613, + "tokens_seen": 2381185024 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014066198595787363, + "loss": 1.2874, + "theoretical_loss": 3.381681319269779, + "tokens_seen": 2381316096 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014064192577733202, + "loss": 1.2418, + "theoretical_loss": 3.3816663275470535, + "tokens_seen": 2381447168 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014062186559679038, + "loss": 1.3527, + "theoretical_loss": 3.381651336880453, + "tokens_seen": 2381578240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014060180541624877, + "loss": 1.2865, + "theoretical_loss": 3.3816363472698443, + "tokens_seen": 2381709312 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014058174523570713, + "loss": 1.231, + "theoretical_loss": 3.3816213587150954, + "tokens_seen": 2381840384 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001405616850551655, + "loss": 1.2103, + "theoretical_loss": 3.3816063712160735, + "tokens_seen": 2381971456 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014054162487462388, + "loss": 1.3658, + "theoretical_loss": 3.381591384772647, + "tokens_seen": 2382102528 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.4178885519504547, + "objective/train/avg_token_score": 0.01622309722006321, + "objective/train/docs_used": 1343061, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2539981603622437, + "objective/train/original_loss": 3.801119565963745, + "objective/train/theoretical_loss": 3.381576399384682, + "objective/train/tokens_used": 2382233600, + "objective/train/unlikelihood_loss": 0.1286291778087616, + "theoretical_loss": 3.381576399384682, + "tokens_seen": 2382233600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014052156469408224, + "loss": 1.347, + "theoretical_loss": 3.381576399384682, + "tokens_seen": 2382233600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014050150451354063, + "loss": 1.139, + "theoretical_loss": 3.3815614150520474, + "tokens_seen": 2382364672 + }, + { + "epoch": 0.72, + "learning_rate": 0.000140481444332999, + "loss": 1.2033, + "theoretical_loss": 3.3815464317746105, + "tokens_seen": 2382495744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014046138415245739, + "loss": 1.1622, + "theoretical_loss": 3.3815314495522384, + "tokens_seen": 2382626816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014044132397191575, + "loss": 1.3518, + "theoretical_loss": 3.381516468384799, + "tokens_seen": 2382757888 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014042126379137414, + "loss": 1.2522, + "theoretical_loss": 3.3815014882721606, + "tokens_seen": 2382888960 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001404012036108325, + "loss": 1.2427, + "theoretical_loss": 3.38148650921419, + "tokens_seen": 2383020032 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001403811434302909, + "loss": 1.2552, + "theoretical_loss": 3.381471531210756, + "tokens_seen": 2383151104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014036108324974925, + "loss": 1.1516, + "theoretical_loss": 3.3814565542617254, + "tokens_seen": 2383282176 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001403410230692076, + "loss": 1.2628, + "theoretical_loss": 3.3814415783669665, + "tokens_seen": 2383413248 + }, + { + "epoch": 0.72, + "learning_rate": 0.000140320962888666, + "loss": 1.1232, + "theoretical_loss": 3.381426603526347, + "tokens_seen": 2383544320 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014030090270812436, + "loss": 1.3737, + "theoretical_loss": 3.381411629739735, + "tokens_seen": 2383675392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014028084252758275, + "loss": 1.4509, + "theoretical_loss": 3.3813966570069973, + "tokens_seen": 2383806464 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.386424720287323, + "objective/train/avg_token_score": 0.01430264301598072, + "objective/train/docs_used": 1344107, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.106664776802063, + "objective/train/original_loss": 3.4108388423919678, + "objective/train/theoretical_loss": 3.381389171035791, + "objective/train/tokens_used": 2383872000, + "objective/train/unlikelihood_loss": 0.2657824456691742, + "theoretical_loss": 3.381389171035791, + "tokens_seen": 2383872000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014026078234704112, + "loss": 1.1966, + "theoretical_loss": 3.3813816853280034, + "tokens_seen": 2383937536 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001402407221664995, + "loss": 1.2319, + "theoretical_loss": 3.3813667147026196, + "tokens_seen": 2384068608 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014022066198595787, + "loss": 1.5143, + "theoretical_loss": 3.3813517451307153, + "tokens_seen": 2384199680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014020060180541626, + "loss": 1.4293, + "theoretical_loss": 3.381336776612158, + "tokens_seen": 2384330752 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014018054162487462, + "loss": 1.2062, + "theoretical_loss": 3.3813218091468147, + "tokens_seen": 2384461824 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014016048144433298, + "loss": 1.2009, + "theoretical_loss": 3.3813068427345545, + "tokens_seen": 2384592896 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014014042126379137, + "loss": 1.4295, + "theoretical_loss": 3.381291877375245, + "tokens_seen": 2384723968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014012036108324973, + "loss": 1.2309, + "theoretical_loss": 3.3812769130687546, + "tokens_seen": 2384855040 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014010030090270812, + "loss": 1.3753, + "theoretical_loss": 3.3812619498149505, + "tokens_seen": 2384986112 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014008024072216648, + "loss": 1.1868, + "theoretical_loss": 3.381246987613702, + "tokens_seen": 2385117184 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014006018054162487, + "loss": 1.1888, + "theoretical_loss": 3.3812320264648767, + "tokens_seen": 2385248256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014004012036108324, + "loss": 1.252, + "theoretical_loss": 3.3812170663683423, + "tokens_seen": 2385379328 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.354227751493454, + "objective/train/avg_token_score": 0.030031898990273476, + "objective/train/docs_used": 1344624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.067555546760559, + "objective/train/original_loss": 4.029555320739746, + "objective/train/theoretical_loss": 3.3812021073239675, + "objective/train/tokens_used": 2385510400, + "objective/train/unlikelihood_loss": 0.1548645943403244, + "theoretical_loss": 3.3812021073239675, + "tokens_seen": 2385510400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014002006018054163, + "loss": 1.1906, + "theoretical_loss": 3.3812021073239675, + "tokens_seen": 2385510400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014000000000000001, + "loss": 1.1365, + "theoretical_loss": 3.3811871493316206, + "tokens_seen": 2385641472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013997993981945838, + "loss": 1.1245, + "theoretical_loss": 3.3811721923911695, + "tokens_seen": 2385772544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013995987963891677, + "loss": 1.2418, + "theoretical_loss": 3.3811572365024825, + "tokens_seen": 2385903616 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013993981945837513, + "loss": 1.5161, + "theoretical_loss": 3.381142281665428, + "tokens_seen": 2386034688 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013991975927783352, + "loss": 1.4028, + "theoretical_loss": 3.3811273278798737, + "tokens_seen": 2386165760 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013989969909729188, + "loss": 1.3259, + "theoretical_loss": 3.3811123751456886, + "tokens_seen": 2386296832 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013987963891675027, + "loss": 1.2532, + "theoretical_loss": 3.3810974234627413, + "tokens_seen": 2386427904 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013985957873620863, + "loss": 1.2988, + "theoretical_loss": 3.3810824728308995, + "tokens_seen": 2386558976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013983951855566702, + "loss": 1.1237, + "theoretical_loss": 3.3810675232500316, + "tokens_seen": 2386690048 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013981945837512538, + "loss": 1.2995, + "theoretical_loss": 3.381052574720006, + "tokens_seen": 2386821120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013979939819458377, + "loss": 1.1235, + "theoretical_loss": 3.3810376272406915, + "tokens_seen": 2386952192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013977933801404214, + "loss": 1.2417, + "theoretical_loss": 3.381022680811957, + "tokens_seen": 2387083264 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.49560117721557617, + "objective/train/avg_token_score": 0.005265652202069759, + "objective/train/docs_used": 1344624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5241023302078247, + "objective/train/original_loss": 3.4595277309417725, + "objective/train/theoretical_loss": 3.3810152079915152, + "objective/train/tokens_used": 2387148800, + "objective/train/unlikelihood_loss": 0.14777123928070068, + "theoretical_loss": 3.3810152079915152, + "tokens_seen": 2387148800 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001397592778335005, + "loss": 1.3359, + "theoretical_loss": 3.3810077354336694, + "tokens_seen": 2387214336 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001397392176529589, + "loss": 1.4259, + "theoretical_loss": 3.3809927911056987, + "tokens_seen": 2387345408 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013971915747241725, + "loss": 1.17, + "theoretical_loss": 3.380977847827913, + "tokens_seen": 2387476480 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013969909729187564, + "loss": 1.2454, + "theoretical_loss": 3.3809629056001804, + "tokens_seen": 2387607552 + }, + { + "epoch": 0.72, + "learning_rate": 0.000139679037111334, + "loss": 1.3263, + "theoretical_loss": 3.38094796442237, + "tokens_seen": 2387738624 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001396589769307924, + "loss": 1.3816, + "theoretical_loss": 3.38093302429435, + "tokens_seen": 2387869696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013963891675025075, + "loss": 1.2876, + "theoretical_loss": 3.3809180852159892, + "tokens_seen": 2388000768 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013961885656970914, + "loss": 1.4261, + "theoretical_loss": 3.3809031471871567, + "tokens_seen": 2388131840 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001395987963891675, + "loss": 1.224, + "theoretical_loss": 3.3808882102077207, + "tokens_seen": 2388262912 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013957873620862587, + "loss": 1.2815, + "theoretical_loss": 3.38087327427755, + "tokens_seen": 2388393984 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013955867602808426, + "loss": 1.2909, + "theoretical_loss": 3.380858339396513, + "tokens_seen": 2388525056 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013953861584754262, + "loss": 1.3916, + "theoretical_loss": 3.3808434055644785, + "tokens_seen": 2388656128 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.4174608886241913, + "objective/train/avg_token_score": 0.04830110818147659, + "objective/train/docs_used": 1345276, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1620757579803467, + "objective/train/original_loss": 4.006445407867432, + "objective/train/theoretical_loss": 3.380828472781316, + "objective/train/tokens_used": 2388787200, + "objective/train/unlikelihood_loss": 0.1252376139163971, + "theoretical_loss": 3.380828472781316, + "tokens_seen": 2388787200 + }, + { + "epoch": 0.72, + "learning_rate": 0.000139518555667001, + "loss": 1.1649, + "theoretical_loss": 3.380828472781316, + "tokens_seen": 2388787200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013949849548645937, + "loss": 1.3964, + "theoretical_loss": 3.3808135410468934, + "tokens_seen": 2388918272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013947843530591776, + "loss": 1.4002, + "theoretical_loss": 3.3807986103610803, + "tokens_seen": 2389049344 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013945837512537612, + "loss": 1.2729, + "theoretical_loss": 3.3807836807237446, + "tokens_seen": 2389180416 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001394383149448345, + "loss": 1.3104, + "theoretical_loss": 3.380768752134756, + "tokens_seen": 2389311488 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013941825476429287, + "loss": 1.3348, + "theoretical_loss": 3.3807538245939828, + "tokens_seen": 2389442560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013939819458375123, + "loss": 1.33, + "theoretical_loss": 3.3807388981012942, + "tokens_seen": 2389573632 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013937813440320962, + "loss": 1.1748, + "theoretical_loss": 3.3807239726565594, + "tokens_seen": 2389704704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013935807422266799, + "loss": 1.1701, + "theoretical_loss": 3.380709048259647, + "tokens_seen": 2389835776 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013933801404212638, + "loss": 1.3415, + "theoretical_loss": 3.3806941249104256, + "tokens_seen": 2389966848 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013931795386158474, + "loss": 1.3269, + "theoretical_loss": 3.380679202608765, + "tokens_seen": 2390097920 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013929789368104313, + "loss": 1.2355, + "theoretical_loss": 3.380664281354534, + "tokens_seen": 2390228992 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001392778335005015, + "loss": 1.3102, + "theoretical_loss": 3.3806493611476007, + "tokens_seen": 2390360064 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.2961876690387726, + "objective/train/avg_token_score": 0.030879981815814972, + "objective/train/docs_used": 1345792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.965428352355957, + "objective/train/original_loss": 4.115960121154785, + "objective/train/theoretical_loss": 3.3806419014368303, + "objective/train/tokens_used": 2390425600, + "objective/train/unlikelihood_loss": 0.16118139028549194, + "theoretical_loss": 3.3806419014368303, + "tokens_seen": 2390425600 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001392577733199599, + "loss": 1.0271, + "theoretical_loss": 3.3806344419878354, + "tokens_seen": 2390491136 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013923771313941827, + "loss": 1.1023, + "theoretical_loss": 3.3806195238751067, + "tokens_seen": 2390622208 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013921765295887666, + "loss": 1.3464, + "theoretical_loss": 3.3806046068092837, + "tokens_seen": 2390753280 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013919759277833502, + "loss": 1.3096, + "theoretical_loss": 3.380589690790236, + "tokens_seen": 2390884352 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013917753259779338, + "loss": 1.4361, + "theoretical_loss": 3.380574775817832, + "tokens_seen": 2391015424 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013915747241725177, + "loss": 1.286, + "theoretical_loss": 3.3805598618919412, + "tokens_seen": 2391146496 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013913741223671013, + "loss": 1.3794, + "theoretical_loss": 3.3805449490124335, + "tokens_seen": 2391277568 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013911735205616852, + "loss": 1.2783, + "theoretical_loss": 3.3805300371791764, + "tokens_seen": 2391408640 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013909729187562688, + "loss": 1.2313, + "theoretical_loss": 3.380515126392041, + "tokens_seen": 2391539712 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013907723169508527, + "loss": 1.4562, + "theoretical_loss": 3.380500216650896, + "tokens_seen": 2391670784 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013905717151454364, + "loss": 1.2648, + "theoretical_loss": 3.3804853079556105, + "tokens_seen": 2391801856 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013903711133400203, + "loss": 1.1889, + "theoretical_loss": 3.3804704003060535, + "tokens_seen": 2391932928 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.35575512051582336, + "objective/train/avg_token_score": 0.02449936792254448, + "objective/train/docs_used": 1347158, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2638897895812988, + "objective/train/original_loss": 4.604652404785156, + "objective/train/theoretical_loss": 3.380455493702095, + "objective/train/tokens_used": 2392064000, + "objective/train/unlikelihood_loss": 0.1377403438091278, + "theoretical_loss": 3.380455493702095, + "tokens_seen": 2392064000 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001390170511534604, + "loss": 1.2015, + "theoretical_loss": 3.380455493702095, + "tokens_seen": 2392064000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013899699097291875, + "loss": 1.2474, + "theoretical_loss": 3.380440588143604, + "tokens_seen": 2392195072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013897693079237714, + "loss": 1.1676, + "theoretical_loss": 3.3804256836304503, + "tokens_seen": 2392326144 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001389568706118355, + "loss": 1.11, + "theoretical_loss": 3.380410780162503, + "tokens_seen": 2392457216 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001389368104312939, + "loss": 1.365, + "theoretical_loss": 3.3803958777396312, + "tokens_seen": 2392588288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013891675025075225, + "loss": 1.1681, + "theoretical_loss": 3.380380976361705, + "tokens_seen": 2392719360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013889669007021064, + "loss": 1.346, + "theoretical_loss": 3.3803660760285936, + "tokens_seen": 2392850432 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138876629889669, + "loss": 1.2418, + "theoretical_loss": 3.3803511767401666, + "tokens_seen": 2392981504 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001388565697091274, + "loss": 1.241, + "theoretical_loss": 3.3803362784962934, + "tokens_seen": 2393112576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013883650952858576, + "loss": 1.113, + "theoretical_loss": 3.380321381296844, + "tokens_seen": 2393243648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013881644934804415, + "loss": 1.3287, + "theoretical_loss": 3.380306485141687, + "tokens_seen": 2393374720 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001387963891675025, + "loss": 1.3576, + "theoretical_loss": 3.3802915900306933, + "tokens_seen": 2393505792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013877632898696087, + "loss": 1.4214, + "theoretical_loss": 3.3802766959637314, + "tokens_seen": 2393636864 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.33070626854896545, + "objective/train/avg_token_score": 0.03765522316098213, + "objective/train/docs_used": 1347750, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1774699687957764, + "objective/train/original_loss": 4.0373454093933105, + "objective/train/theoretical_loss": 3.380269249321722, + "objective/train/tokens_used": 2393702400, + "objective/train/unlikelihood_loss": 0.1498364508152008, + "theoretical_loss": 3.380269249321722, + "tokens_seen": 2393702400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013875626880641926, + "loss": 1.3784, + "theoretical_loss": 3.3802618029406717, + "tokens_seen": 2393767936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013873620862587762, + "loss": 1.1616, + "theoretical_loss": 3.3802469109613837, + "tokens_seen": 2393899008 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138716148445336, + "loss": 1.3127, + "theoretical_loss": 3.380232020025737, + "tokens_seen": 2394030080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013869608826479437, + "loss": 1.3096, + "theoretical_loss": 3.3802171301336017, + "tokens_seen": 2394161152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013867602808425276, + "loss": 1.329, + "theoretical_loss": 3.380202241284847, + "tokens_seen": 2394292224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013865596790371113, + "loss": 1.4099, + "theoretical_loss": 3.3801873534793425, + "tokens_seen": 2394423296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013863590772316951, + "loss": 1.1969, + "theoretical_loss": 3.380172466716959, + "tokens_seen": 2394554368 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013861584754262788, + "loss": 1.218, + "theoretical_loss": 3.3801575809975652, + "tokens_seen": 2394685440 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013859578736208624, + "loss": 1.4392, + "theoretical_loss": 3.380142696321032, + "tokens_seen": 2394816512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013857572718154463, + "loss": 1.1715, + "theoretical_loss": 3.3801278126872285, + "tokens_seen": 2394947584 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138555667001003, + "loss": 1.2575, + "theoretical_loss": 3.3801129300960246, + "tokens_seen": 2395078656 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013853560682046138, + "loss": 1.3059, + "theoretical_loss": 3.380098048547291, + "tokens_seen": 2395209728 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.4141617715358734, + "objective/train/avg_token_score": 0.012660033069550991, + "objective/train/docs_used": 1348886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4426941871643066, + "objective/train/original_loss": 4.044859886169434, + "objective/train/theoretical_loss": 3.3800831680408963, + "objective/train/tokens_used": 2395340800, + "objective/train/unlikelihood_loss": 0.14499318599700928, + "theoretical_loss": 3.3800831680408963, + "tokens_seen": 2395340800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013851554663991974, + "loss": 1.3526, + "theoretical_loss": 3.3800831680408963, + "tokens_seen": 2395340800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013849548645937816, + "loss": 1.1226, + "theoretical_loss": 3.380068288576712, + "tokens_seen": 2395471872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013847542627883652, + "loss": 1.2625, + "theoretical_loss": 3.3800534101546065, + "tokens_seen": 2395602944 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001384553660982949, + "loss": 1.3145, + "theoretical_loss": 3.380038532774451, + "tokens_seen": 2395734016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013843530591775327, + "loss": 1.2491, + "theoretical_loss": 3.3800236564361157, + "tokens_seen": 2395865088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013841524573721163, + "loss": 1.5214, + "theoretical_loss": 3.3800087811394697, + "tokens_seen": 2395996160 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013839518555667002, + "loss": 1.2384, + "theoretical_loss": 3.3799939068843834, + "tokens_seen": 2396127232 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013837512537612839, + "loss": 1.343, + "theoretical_loss": 3.379979033670727, + "tokens_seen": 2396258304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013835506519558678, + "loss": 1.3667, + "theoretical_loss": 3.379964161498371, + "tokens_seen": 2396389376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013833500501504514, + "loss": 1.2226, + "theoretical_loss": 3.3799492903671844, + "tokens_seen": 2396520448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013831494483450353, + "loss": 1.3373, + "theoretical_loss": 3.379934420277039, + "tokens_seen": 2396651520 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001382948846539619, + "loss": 1.3363, + "theoretical_loss": 3.379919551227804, + "tokens_seen": 2396782592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013827482447342028, + "loss": 1.3817, + "theoretical_loss": 3.3799046832193493, + "tokens_seen": 2396913664 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.47232404351234436, + "objective/train/avg_token_score": 0.025202132761478424, + "objective/train/docs_used": 1349689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5090183019638062, + "objective/train/original_loss": 3.8714871406555176, + "objective/train/theoretical_loss": 3.3798972496053743, + "objective/train/tokens_used": 2396979200, + "objective/train/unlikelihood_loss": 0.1394972801208496, + "theoretical_loss": 3.3798972496053743, + "tokens_seen": 2396979200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013825476429287864, + "loss": 1.3372, + "theoretical_loss": 3.379889816251546, + "tokens_seen": 2397044736 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013823470411233703, + "loss": 1.255, + "theoretical_loss": 3.3798749503242638, + "tokens_seen": 2397175808 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001382146439317954, + "loss": 1.2671, + "theoretical_loss": 3.379860085437373, + "tokens_seen": 2397306880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013819458375125375, + "loss": 1.2878, + "theoretical_loss": 3.379845221590744, + "tokens_seen": 2397437952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013817452357071214, + "loss": 1.2364, + "theoretical_loss": 3.3798303587842473, + "tokens_seen": 2397569024 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001381544633901705, + "loss": 1.3541, + "theoretical_loss": 3.3798154970177534, + "tokens_seen": 2397700096 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001381344032096289, + "loss": 1.3044, + "theoretical_loss": 3.379800636291132, + "tokens_seen": 2397831168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013811434302908726, + "loss": 1.2182, + "theoretical_loss": 3.3797857766042547, + "tokens_seen": 2397962240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013809428284854565, + "loss": 1.2884, + "theoretical_loss": 3.3797709179569906, + "tokens_seen": 2398093312 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138074222668004, + "loss": 1.2322, + "theoretical_loss": 3.3797560603492105, + "tokens_seen": 2398224384 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001380541624874624, + "loss": 1.2491, + "theoretical_loss": 3.3797412037807857, + "tokens_seen": 2398355456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013803410230692076, + "loss": 1.3125, + "theoretical_loss": 3.3797263482515856, + "tokens_seen": 2398486528 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.38330888748168945, + "objective/train/avg_token_score": 0.010782254859805107, + "objective/train/docs_used": 1350650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2439827919006348, + "objective/train/original_loss": 3.611248731613159, + "objective/train/theoretical_loss": 3.3797114937614814, + "objective/train/tokens_used": 2398617600, + "objective/train/unlikelihood_loss": 0.20027384161949158, + "theoretical_loss": 3.3797114937614814, + "tokens_seen": 2398617600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013801404212637912, + "loss": 1.1576, + "theoretical_loss": 3.3797114937614814, + "tokens_seen": 2398617600 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001379939819458375, + "loss": 1.192, + "theoretical_loss": 3.3796966403103434, + "tokens_seen": 2398748672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013797392176529588, + "loss": 1.3496, + "theoretical_loss": 3.3796817878980425, + "tokens_seen": 2398879744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013795386158475426, + "loss": 1.2812, + "theoretical_loss": 3.3796669365244485, + "tokens_seen": 2399010816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013793380140421263, + "loss": 1.4199, + "theoretical_loss": 3.379652086189433, + "tokens_seen": 2399141888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013791374122367102, + "loss": 1.2277, + "theoretical_loss": 3.379637236892866, + "tokens_seen": 2399272960 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013789368104312938, + "loss": 1.3465, + "theoretical_loss": 3.379622388634618, + "tokens_seen": 2399404032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013787362086258777, + "loss": 1.3014, + "theoretical_loss": 3.3796075414145603, + "tokens_seen": 2399535104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013785356068204613, + "loss": 1.1865, + "theoretical_loss": 3.3795926952325637, + "tokens_seen": 2399666176 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001378335005015045, + "loss": 1.3318, + "theoretical_loss": 3.3795778500884976, + "tokens_seen": 2399797248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013781344032096288, + "loss": 1.2304, + "theoretical_loss": 3.3795630059822344, + "tokens_seen": 2399928320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013779338014042124, + "loss": 1.1091, + "theoretical_loss": 3.379548162913644, + "tokens_seen": 2400059392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013777331995987963, + "loss": 1.2845, + "theoretical_loss": 3.3795333208825973, + "tokens_seen": 2400190464 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.4886363744735718, + "objective/train/avg_token_score": 0.01498680841177702, + "objective/train/docs_used": 1352096, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5392920970916748, + "objective/train/original_loss": 3.771796703338623, + "objective/train/theoretical_loss": 3.3795259002561124, + "objective/train/tokens_used": 2400256000, + "objective/train/unlikelihood_loss": 0.1335362046957016, + "theoretical_loss": 3.3795259002561124, + "tokens_seen": 2400256000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013775325977933802, + "loss": 1.3765, + "theoretical_loss": 3.379518479888965, + "tokens_seen": 2400321536 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001377331995987964, + "loss": 1.381, + "theoretical_loss": 3.3795036399326186, + "tokens_seen": 2400452608 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013771313941825477, + "loss": 1.435, + "theoretical_loss": 3.3794888010134283, + "tokens_seen": 2400583680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013769307923771316, + "loss": 1.3651, + "theoretical_loss": 3.379473963131265, + "tokens_seen": 2400714752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013767301905717153, + "loss": 1.2693, + "theoretical_loss": 3.379459126286, + "tokens_seen": 2400845824 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013765295887662991, + "loss": 1.224, + "theoretical_loss": 3.3794442904775037, + "tokens_seen": 2400976896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013763289869608828, + "loss": 1.4051, + "theoretical_loss": 3.3794294557056475, + "tokens_seen": 2401107968 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013761283851554664, + "loss": 1.3136, + "theoretical_loss": 3.3794146219703025, + "tokens_seen": 2401239040 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013759277833500503, + "loss": 1.3396, + "theoretical_loss": 3.379399789271339, + "tokens_seen": 2401370112 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001375727181544634, + "loss": 1.3256, + "theoretical_loss": 3.3793849576086297, + "tokens_seen": 2401501184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013755265797392178, + "loss": 1.3734, + "theoretical_loss": 3.3793701269820433, + "tokens_seen": 2401632256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013753259779338014, + "loss": 1.3084, + "theoretical_loss": 3.3793552973914522, + "tokens_seen": 2401763328 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.4265640377998352, + "objective/train/avg_token_score": 0.021169526502490044, + "objective/train/docs_used": 1352871, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3740012645721436, + "objective/train/original_loss": 3.69340181350708, + "objective/train/theoretical_loss": 3.3793404688367277, + "objective/train/tokens_used": 2401894400, + "objective/train/unlikelihood_loss": 0.16596078872680664, + "theoretical_loss": 3.3793404688367277, + "tokens_seen": 2401894400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013751253761283853, + "loss": 1.383, + "theoretical_loss": 3.3793404688367277, + "tokens_seen": 2401894400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001374924774322969, + "loss": 1.2536, + "theoretical_loss": 3.3793256413177404, + "tokens_seen": 2402025472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013747241725175528, + "loss": 1.4312, + "theoretical_loss": 3.379310814834362, + "tokens_seen": 2402156544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013745235707121365, + "loss": 1.3545, + "theoretical_loss": 3.3792959893864625, + "tokens_seen": 2402287616 + }, + { + "epoch": 0.73, + "learning_rate": 0.000137432296890672, + "loss": 1.3108, + "theoretical_loss": 3.3792811649739143, + "tokens_seen": 2402418688 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001374122367101304, + "loss": 1.2975, + "theoretical_loss": 3.3792663415965882, + "tokens_seen": 2402549760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013739217652958876, + "loss": 1.1327, + "theoretical_loss": 3.3792515192543555, + "tokens_seen": 2402680832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013737211634904715, + "loss": 1.3336, + "theoretical_loss": 3.3792366979470874, + "tokens_seen": 2402811904 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001373520561685055, + "loss": 1.3808, + "theoretical_loss": 3.379221877674655, + "tokens_seen": 2402942976 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001373319959879639, + "loss": 1.4761, + "theoretical_loss": 3.3792070584369296, + "tokens_seen": 2403074048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013731193580742226, + "loss": 1.4282, + "theoretical_loss": 3.379192240233783, + "tokens_seen": 2403205120 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013729187562688065, + "loss": 1.282, + "theoretical_loss": 3.379177423065086, + "tokens_seen": 2403336192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013727181544633901, + "loss": 1.2301, + "theoretical_loss": 3.37916260693071, + "tokens_seen": 2403467264 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.5302419066429138, + "objective/train/avg_token_score": 0.009037159383296967, + "objective/train/docs_used": 1354213, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7096360921859741, + "objective/train/original_loss": 3.3960864543914795, + "objective/train/theoretical_loss": 3.3791551992513527, + "objective/train/tokens_used": 2403532800, + "objective/train/unlikelihood_loss": 0.14058798551559448, + "theoretical_loss": 3.3791551992513527, + "tokens_seen": 2403532800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013725175526579738, + "loss": 1.3382, + "theoretical_loss": 3.3791477918305275, + "tokens_seen": 2403598336 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013723169508525577, + "loss": 1.2202, + "theoretical_loss": 3.3791329777644084, + "tokens_seen": 2403729408 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013721163490471413, + "loss": 1.3654, + "theoretical_loss": 3.379118164732225, + "tokens_seen": 2403860480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013719157472417252, + "loss": 1.2659, + "theoretical_loss": 3.379103352733848, + "tokens_seen": 2403991552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013717151454363088, + "loss": 1.273, + "theoretical_loss": 3.3790885417691503, + "tokens_seen": 2404122624 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013715145436308927, + "loss": 1.3091, + "theoretical_loss": 3.379073731838002, + "tokens_seen": 2404253696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013713139418254763, + "loss": 1.3874, + "theoretical_loss": 3.3790589229402754, + "tokens_seen": 2404384768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013711133400200602, + "loss": 1.2891, + "theoretical_loss": 3.3790441150758417, + "tokens_seen": 2404515840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013709127382146438, + "loss": 1.5005, + "theoretical_loss": 3.379029308244573, + "tokens_seen": 2404646912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013707121364092277, + "loss": 1.1674, + "theoretical_loss": 3.3790145024463403, + "tokens_seen": 2404777984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013705115346038113, + "loss": 1.1763, + "theoretical_loss": 3.3789996976810155, + "tokens_seen": 2404909056 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001370310932798395, + "loss": 1.319, + "theoretical_loss": 3.37898489394847, + "tokens_seen": 2405040128 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.3079789876937866, + "objective/train/avg_token_score": 0.027728047221899033, + "objective/train/docs_used": 1354954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9509592652320862, + "objective/train/original_loss": 3.7793796062469482, + "objective/train/theoretical_loss": 3.378970091248576, + "objective/train/tokens_used": 2405171200, + "objective/train/unlikelihood_loss": 0.2637613117694855, + "theoretical_loss": 3.378970091248576, + "tokens_seen": 2405171200 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001370110330992979, + "loss": 1.3326, + "theoretical_loss": 3.378970091248576, + "tokens_seen": 2405171200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013699097291875628, + "loss": 1.1769, + "theoretical_loss": 3.3789552895812047, + "tokens_seen": 2405302272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013697091273821466, + "loss": 1.376, + "theoretical_loss": 3.378940488946228, + "tokens_seen": 2405433344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013695085255767303, + "loss": 1.2218, + "theoretical_loss": 3.378925689343518, + "tokens_seen": 2405564416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013693079237713142, + "loss": 1.283, + "theoretical_loss": 3.3789108907729455, + "tokens_seen": 2405695488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013691073219658978, + "loss": 1.1732, + "theoretical_loss": 3.3788960932343834, + "tokens_seen": 2405826560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013689067201604817, + "loss": 1.2373, + "theoretical_loss": 3.3788812967277027, + "tokens_seen": 2405957632 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013687061183550653, + "loss": 1.2993, + "theoretical_loss": 3.3788665012527757, + "tokens_seen": 2406088704 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001368505516549649, + "loss": 1.3625, + "theoretical_loss": 3.378851706809474, + "tokens_seen": 2406219776 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013683049147442328, + "loss": 1.2731, + "theoretical_loss": 3.37883691339767, + "tokens_seen": 2406350848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013681043129388164, + "loss": 1.3433, + "theoretical_loss": 3.378822121017235, + "tokens_seen": 2406481920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013679037111334003, + "loss": 1.2862, + "theoretical_loss": 3.378807329668041, + "tokens_seen": 2406612992 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001367703109327984, + "loss": 1.0632, + "theoretical_loss": 3.37879253934996, + "tokens_seen": 2406744064 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.3121945261955261, + "objective/train/avg_token_score": 0.00792326033115387, + "objective/train/docs_used": 1355619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.901352047920227, + "objective/train/original_loss": 3.7000503540039062, + "objective/train/theoretical_loss": 3.378785144577547, + "objective/train/tokens_used": 2406809600, + "objective/train/unlikelihood_loss": 0.15240934491157532, + "theoretical_loss": 3.378785144577547, + "tokens_seen": 2406809600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013675025075225678, + "loss": 1.2908, + "theoretical_loss": 3.3787777500628646, + "tokens_seen": 2406875136 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013673019057171515, + "loss": 1.3555, + "theoretical_loss": 3.3787629618066255, + "tokens_seen": 2407006208 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013671013039117354, + "loss": 1.39, + "theoretical_loss": 3.3787481745811156, + "tokens_seen": 2407137280 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001366900702106319, + "loss": 1.2048, + "theoretical_loss": 3.378733388386207, + "tokens_seen": 2407268352 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001366700100300903, + "loss": 1.287, + "theoretical_loss": 3.3787186032217718, + "tokens_seen": 2407399424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013664994984954865, + "loss": 1.2053, + "theoretical_loss": 3.3787038190876815, + "tokens_seen": 2407530496 + }, + { + "epoch": 0.73, + "learning_rate": 0.000136629889669007, + "loss": 1.2326, + "theoretical_loss": 3.3786890359838084, + "tokens_seen": 2407661568 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001366098294884654, + "loss": 1.1807, + "theoretical_loss": 3.378674253910025, + "tokens_seen": 2407792640 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013658976930792376, + "loss": 1.0997, + "theoretical_loss": 3.3786594728662034, + "tokens_seen": 2407923712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013656970912738215, + "loss": 1.2078, + "theoretical_loss": 3.378644692852215, + "tokens_seen": 2408054784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013654964894684052, + "loss": 1.2148, + "theoretical_loss": 3.3786299138679334, + "tokens_seen": 2408185856 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001365295887662989, + "loss": 1.3043, + "theoretical_loss": 3.3786151359132295, + "tokens_seen": 2408316928 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.4818548262119293, + "objective/train/avg_token_score": 0.009630024433135986, + "objective/train/docs_used": 1356770, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.542802095413208, + "objective/train/original_loss": 3.5953686237335205, + "objective/train/theoretical_loss": 3.3786003589879763, + "objective/train/tokens_used": 2408448000, + "objective/train/unlikelihood_loss": 0.13244184851646423, + "theoretical_loss": 3.3786003589879763, + "tokens_seen": 2408448000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013650952858575727, + "loss": 1.3486, + "theoretical_loss": 3.3786003589879763, + "tokens_seen": 2408448000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013648946840521566, + "loss": 1.2467, + "theoretical_loss": 3.378585583092046, + "tokens_seen": 2408579072 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013646940822467402, + "loss": 1.2437, + "theoretical_loss": 3.3785708082253105, + "tokens_seen": 2408710144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013644934804413238, + "loss": 1.314, + "theoretical_loss": 3.378556034387642, + "tokens_seen": 2408841216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013642928786359077, + "loss": 1.2382, + "theoretical_loss": 3.378541261578914, + "tokens_seen": 2408972288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013640922768304913, + "loss": 1.2287, + "theoretical_loss": 3.3785264897989977, + "tokens_seen": 2409103360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013638916750250752, + "loss": 1.3692, + "theoretical_loss": 3.378511719047766, + "tokens_seen": 2409234432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013636910732196588, + "loss": 1.3045, + "theoretical_loss": 3.3784969493250907, + "tokens_seen": 2409365504 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013634904714142427, + "loss": 1.4512, + "theoretical_loss": 3.378482180630845, + "tokens_seen": 2409496576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013632898696088264, + "loss": 1.2365, + "theoretical_loss": 3.378467412964901, + "tokens_seen": 2409627648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013630892678034103, + "loss": 1.2453, + "theoretical_loss": 3.3784526463271307, + "tokens_seen": 2409758720 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001362888665997994, + "loss": 1.4183, + "theoretical_loss": 3.378437880717408, + "tokens_seen": 2409889792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013626880641925775, + "loss": 1.324, + "theoretical_loss": 3.378423116135604, + "tokens_seen": 2410020864 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.369562566280365, + "objective/train/avg_token_score": 0.04130237177014351, + "objective/train/docs_used": 1357404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1985323429107666, + "objective/train/original_loss": 4.038417339324951, + "objective/train/theoretical_loss": 3.3784157342301318, + "objective/train/tokens_used": 2410086400, + "objective/train/unlikelihood_loss": 0.17888520658016205, + "theoretical_loss": 3.3784157342301318, + "tokens_seen": 2410086400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013624874623871617, + "loss": 1.4122, + "theoretical_loss": 3.378408352581592, + "tokens_seen": 2410151936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013622868605817453, + "loss": 1.2542, + "theoretical_loss": 3.3783935900552438, + "tokens_seen": 2410283008 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013620862587763292, + "loss": 1.4701, + "theoretical_loss": 3.378378828556433, + "tokens_seen": 2410414080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013618856569709128, + "loss": 1.1654, + "theoretical_loss": 3.3783640680850313, + "tokens_seen": 2410545152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013616850551654967, + "loss": 1.3692, + "theoretical_loss": 3.378349308640912, + "tokens_seen": 2410676224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013614844533600803, + "loss": 1.3523, + "theoretical_loss": 3.378334550223947, + "tokens_seen": 2410807296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013612838515546642, + "loss": 1.4315, + "theoretical_loss": 3.3783197928340103, + "tokens_seen": 2410938368 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013610832497492478, + "loss": 1.3673, + "theoretical_loss": 3.378305036470973, + "tokens_seen": 2411069440 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013608826479438317, + "loss": 1.3499, + "theoretical_loss": 3.3782902811347095, + "tokens_seen": 2411200512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013606820461384153, + "loss": 1.2888, + "theoretical_loss": 3.378275526825091, + "tokens_seen": 2411331584 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001360481444332999, + "loss": 1.4112, + "theoretical_loss": 3.378260773541991, + "tokens_seen": 2411462656 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013602808425275829, + "loss": 1.3245, + "theoretical_loss": 3.3782460212852827, + "tokens_seen": 2411593728 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.38159823417663574, + "objective/train/avg_token_score": 0.020865801721811295, + "objective/train/docs_used": 1358876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1793745756149292, + "objective/train/original_loss": 4.011548042297363, + "objective/train/theoretical_loss": 3.3782312700548376, + "objective/train/tokens_used": 2411724800, + "objective/train/unlikelihood_loss": 0.13719308376312256, + "theoretical_loss": 3.3782312700548376, + "tokens_seen": 2411724800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013600802407221665, + "loss": 1.1892, + "theoretical_loss": 3.3782312700548376, + "tokens_seen": 2411724800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013598796389167504, + "loss": 1.4532, + "theoretical_loss": 3.37821651985053, + "tokens_seen": 2411855872 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001359679037111334, + "loss": 1.1685, + "theoretical_loss": 3.3782017706722316, + "tokens_seen": 2411986944 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001359478435305918, + "loss": 1.3718, + "theoretical_loss": 3.3781870225198163, + "tokens_seen": 2412118016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013592778335005015, + "loss": 1.3545, + "theoretical_loss": 3.3781722753931565, + "tokens_seen": 2412249088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013590772316950854, + "loss": 1.0448, + "theoretical_loss": 3.3781575292921247, + "tokens_seen": 2412380160 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001358876629889669, + "loss": 1.2985, + "theoretical_loss": 3.3781427842165948, + "tokens_seen": 2412511232 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013586760280842527, + "loss": 1.2828, + "theoretical_loss": 3.378128040166439, + "tokens_seen": 2412642304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013584754262788365, + "loss": 1.3145, + "theoretical_loss": 3.3781132971415304, + "tokens_seen": 2412773376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013582748244734202, + "loss": 1.2928, + "theoretical_loss": 3.3780985551417424, + "tokens_seen": 2412904448 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001358074222668004, + "loss": 1.4648, + "theoretical_loss": 3.378083814166948, + "tokens_seen": 2413035520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013578736208625877, + "loss": 1.2895, + "theoretical_loss": 3.378069074217019, + "tokens_seen": 2413166592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013576730190571716, + "loss": 1.3293, + "theoretical_loss": 3.378054335291831, + "tokens_seen": 2413297664 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.3558773100376129, + "objective/train/avg_token_score": 0.019469160586595535, + "objective/train/docs_used": 1359530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1706666946411133, + "objective/train/original_loss": 4.531891345977783, + "objective/train/theoretical_loss": 3.378046966213474, + "objective/train/tokens_used": 2413363200, + "objective/train/unlikelihood_loss": 0.10529115051031113, + "theoretical_loss": 3.378046966213474, + "tokens_seen": 2413363200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013574724172517552, + "loss": 1.2914, + "theoretical_loss": 3.378039597391255, + "tokens_seen": 2413428736 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001357271815446339, + "loss": 1.3145, + "theoretical_loss": 3.378024860515165, + "tokens_seen": 2413559808 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013570712136409227, + "loss": 1.434, + "theoretical_loss": 3.3780101246634335, + "tokens_seen": 2413690880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013568706118355063, + "loss": 1.3422, + "theoretical_loss": 3.3779953898359345, + "tokens_seen": 2413821952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013566700100300902, + "loss": 1.2612, + "theoretical_loss": 3.3779806560325407, + "tokens_seen": 2413953024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013564694082246739, + "loss": 1.2928, + "theoretical_loss": 3.3779659232531256, + "tokens_seen": 2414084096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013562688064192578, + "loss": 1.4661, + "theoretical_loss": 3.377951191497562, + "tokens_seen": 2414215168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013560682046138414, + "loss": 1.3223, + "theoretical_loss": 3.377936460765724, + "tokens_seen": 2414346240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013558676028084253, + "loss": 1.3047, + "theoretical_loss": 3.3779217310574836, + "tokens_seen": 2414477312 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001355667001003009, + "loss": 1.1998, + "theoretical_loss": 3.3779070023727154, + "tokens_seen": 2414608384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013554663991975928, + "loss": 1.3644, + "theoretical_loss": 3.377892274711292, + "tokens_seen": 2414739456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013552657973921764, + "loss": 1.3152, + "theoretical_loss": 3.377877548073087, + "tokens_seen": 2414870528 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.4919965863227844, + "objective/train/avg_token_score": 0.03518358990550041, + "objective/train/docs_used": 1361019, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3892899751663208, + "objective/train/original_loss": 3.405484437942505, + "objective/train/theoretical_loss": 3.377862822457974, + "objective/train/tokens_used": 2415001600, + "objective/train/unlikelihood_loss": 0.18315108120441437, + "theoretical_loss": 3.377862822457974, + "tokens_seen": 2415001600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013550651955867606, + "loss": 1.4214, + "theoretical_loss": 3.377862822457974, + "tokens_seen": 2415001600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013548645937813442, + "loss": 1.4336, + "theoretical_loss": 3.377848097865826, + "tokens_seen": 2415132672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013546639919759278, + "loss": 1.3256, + "theoretical_loss": 3.3778333742965163, + "tokens_seen": 2415263744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013544633901705117, + "loss": 1.2949, + "theoretical_loss": 3.377818651749919, + "tokens_seen": 2415394816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013542627883650953, + "loss": 1.5724, + "theoretical_loss": 3.377803930225907, + "tokens_seen": 2415525888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013540621865596792, + "loss": 1.2861, + "theoretical_loss": 3.377789209724354, + "tokens_seen": 2415656960 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013538615847542628, + "loss": 1.3422, + "theoretical_loss": 3.3777744902451334, + "tokens_seen": 2415788032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013536609829488467, + "loss": 1.4542, + "theoretical_loss": 3.377759771788119, + "tokens_seen": 2415919104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013534603811434304, + "loss": 1.2959, + "theoretical_loss": 3.3777450543531837, + "tokens_seen": 2416050176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013532597793380143, + "loss": 1.3188, + "theoretical_loss": 3.377730337940202, + "tokens_seen": 2416181248 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001353059177532598, + "loss": 1.1773, + "theoretical_loss": 3.3777156225490472, + "tokens_seen": 2416312320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013528585757271815, + "loss": 1.2565, + "theoretical_loss": 3.3777009081795923, + "tokens_seen": 2416443392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013526579739217654, + "loss": 1.347, + "theoretical_loss": 3.377686194831712, + "tokens_seen": 2416574464 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.3494623601436615, + "objective/train/avg_token_score": 0.02256671153008938, + "objective/train/docs_used": 1361609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0390946865081787, + "objective/train/original_loss": 4.615906238555908, + "objective/train/theoretical_loss": 3.377678838540822, + "objective/train/tokens_used": 2416640000, + "objective/train/unlikelihood_loss": 0.15202991664409637, + "theoretical_loss": 3.377678838540822, + "tokens_seen": 2416640000 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001352457372116349, + "loss": 1.3302, + "theoretical_loss": 3.3776714825052787, + "tokens_seen": 2416705536 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001352256770310933, + "loss": 1.3593, + "theoretical_loss": 3.3776567712001673, + "tokens_seen": 2416836608 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013520561685055165, + "loss": 1.1417, + "theoretical_loss": 3.377642060916251, + "tokens_seen": 2416967680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013518555667001004, + "loss": 1.45, + "theoretical_loss": 3.3776273516534037, + "tokens_seen": 2417098752 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001351654964894684, + "loss": 1.2589, + "theoretical_loss": 3.377612643411499, + "tokens_seen": 2417229824 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001351454363089268, + "loss": 1.336, + "theoretical_loss": 3.3775979361904103, + "tokens_seen": 2417360896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013512537612838516, + "loss": 1.396, + "theoretical_loss": 3.3775832299900124, + "tokens_seen": 2417491968 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013510531594784355, + "loss": 1.3996, + "theoretical_loss": 3.377568524810178, + "tokens_seen": 2417623040 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001350852557673019, + "loss": 1.3285, + "theoretical_loss": 3.377553820650782, + "tokens_seen": 2417754112 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013506519558676027, + "loss": 1.4306, + "theoretical_loss": 3.3775391175116973, + "tokens_seen": 2417885184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013504513540621866, + "loss": 1.2118, + "theoretical_loss": 3.3775244153927986, + "tokens_seen": 2418016256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013502507522567702, + "loss": 1.3538, + "theoretical_loss": 3.3775097142939594, + "tokens_seen": 2418147328 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.39839932322502136, + "objective/train/avg_token_score": 0.01079068798571825, + "objective/train/docs_used": 1362527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2327979803085327, + "objective/train/original_loss": 3.745481252670288, + "objective/train/theoretical_loss": 3.3774950142150537, + "objective/train/tokens_used": 2418278400, + "objective/train/unlikelihood_loss": 0.17116564512252808, + "theoretical_loss": 3.3774950142150537, + "tokens_seen": 2418278400 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001350050150451354, + "loss": 1.3489, + "theoretical_loss": 3.3774950142150537, + "tokens_seen": 2418278400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013498495486459377, + "loss": 1.3217, + "theoretical_loss": 3.3774803151559554, + "tokens_seen": 2418409472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013496489468405216, + "loss": 1.2217, + "theoretical_loss": 3.3774656171165387, + "tokens_seen": 2418540544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013494483450351052, + "loss": 1.3885, + "theoretical_loss": 3.3774509200966776, + "tokens_seen": 2418671616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013492477432296891, + "loss": 1.2119, + "theoretical_loss": 3.377436224096246, + "tokens_seen": 2418802688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013490471414242728, + "loss": 1.4224, + "theoretical_loss": 3.377421529115118, + "tokens_seen": 2418933760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013488465396188564, + "loss": 1.2379, + "theoretical_loss": 3.377406835153167, + "tokens_seen": 2419064832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013486459378134403, + "loss": 1.1802, + "theoretical_loss": 3.377392142210269, + "tokens_seen": 2419195904 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001348445336008024, + "loss": 1.1746, + "theoretical_loss": 3.377377450286296, + "tokens_seen": 2419326976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013482447342026078, + "loss": 1.3302, + "theoretical_loss": 3.377362759381123, + "tokens_seen": 2419458048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013480441323971914, + "loss": 1.2914, + "theoretical_loss": 3.3773480694946243, + "tokens_seen": 2419589120 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013478435305917753, + "loss": 1.2493, + "theoretical_loss": 3.3773333806266743, + "tokens_seen": 2419720192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013478435305917753, + "loss": 1.3033, + "theoretical_loss": 3.3773186927771466, + "tokens_seen": 2419851264 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.41709432005882263, + "objective/train/avg_token_score": 0.007591762579977512, + "objective/train/docs_used": 1363161, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3133161067962646, + "objective/train/original_loss": 3.8388216495513916, + "objective/train/theoretical_loss": 3.377311349234252, + "objective/train/tokens_used": 2419916800, + "objective/train/unlikelihood_loss": 0.17207969725131989, + "theoretical_loss": 3.377311349234252, + "tokens_seen": 2419916800 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001347642928786359, + "loss": 1.1464, + "theoretical_loss": 3.3773040059459154, + "tokens_seen": 2419982336 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001347442326980943, + "loss": 1.3034, + "theoretical_loss": 3.3772893201328555, + "tokens_seen": 2420113408 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013472417251755267, + "loss": 1.4433, + "theoretical_loss": 3.3772746353378413, + "tokens_seen": 2420244480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013470411233701103, + "loss": 1.344, + "theoretical_loss": 3.3772599515607467, + "tokens_seen": 2420375552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013468405215646942, + "loss": 1.2259, + "theoretical_loss": 3.3772452688014454, + "tokens_seen": 2420506624 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013466399197592779, + "loss": 1.4517, + "theoretical_loss": 3.377230587059813, + "tokens_seen": 2420637696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013464393179538618, + "loss": 1.314, + "theoretical_loss": 3.3772159063357234, + "tokens_seen": 2420768768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013462387161484454, + "loss": 1.3886, + "theoretical_loss": 3.37720122662905, + "tokens_seen": 2420899840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013460381143430293, + "loss": 1.2229, + "theoretical_loss": 3.3771865479396688, + "tokens_seen": 2421030912 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001345837512537613, + "loss": 1.2807, + "theoretical_loss": 3.3771718702674534, + "tokens_seen": 2421161984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013456369107321968, + "loss": 1.2992, + "theoretical_loss": 3.3771571936122786, + "tokens_seen": 2421293056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013454363089267804, + "loss": 1.2709, + "theoretical_loss": 3.3771425179740184, + "tokens_seen": 2421424128 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.47330155968666077, + "objective/train/avg_token_score": 0.01892470195889473, + "objective/train/docs_used": 1364650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4332988262176514, + "objective/train/original_loss": 3.761988639831543, + "objective/train/theoretical_loss": 3.3771278433525476, + "objective/train/tokens_used": 2421555200, + "objective/train/unlikelihood_loss": 0.13350285589694977, + "theoretical_loss": 3.3771278433525476, + "tokens_seen": 2421555200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013452357071213643, + "loss": 1.3387, + "theoretical_loss": 3.3771278433525476, + "tokens_seen": 2421555200 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001345035105315948, + "loss": 1.4575, + "theoretical_loss": 3.3771131697477403, + "tokens_seen": 2421686272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013448345035105315, + "loss": 1.4032, + "theoretical_loss": 3.3770984971594715, + "tokens_seen": 2421817344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013446339017051154, + "loss": 1.1746, + "theoretical_loss": 3.377083825587616, + "tokens_seen": 2421948416 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001344433299899699, + "loss": 1.2335, + "theoretical_loss": 3.3770691550320477, + "tokens_seen": 2422079488 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001344232698094283, + "loss": 1.1689, + "theoretical_loss": 3.377054485492642, + "tokens_seen": 2422210560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013440320962888666, + "loss": 1.3894, + "theoretical_loss": 3.3770398169692726, + "tokens_seen": 2422341632 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013438314944834505, + "loss": 1.1635, + "theoretical_loss": 3.377025149461815, + "tokens_seen": 2422472704 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001343630892678034, + "loss": 1.4611, + "theoretical_loss": 3.3770104829701437, + "tokens_seen": 2422603776 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001343430290872618, + "loss": 1.3411, + "theoretical_loss": 3.376995817494133, + "tokens_seen": 2422734848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013432296890672016, + "loss": 1.4199, + "theoretical_loss": 3.3769811530336575, + "tokens_seen": 2422865920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013430290872617852, + "loss": 1.4394, + "theoretical_loss": 3.3769664895885927, + "tokens_seen": 2422996992 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001342828485456369, + "loss": 1.3203, + "theoretical_loss": 3.376951827158813, + "tokens_seen": 2423128064 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.29429373145103455, + "objective/train/avg_token_score": 0.014065215364098549, + "objective/train/docs_used": 1365365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9421887993812561, + "objective/train/original_loss": 4.114762306213379, + "objective/train/theoretical_loss": 3.376944496324616, + "objective/train/tokens_used": 2423193600, + "objective/train/unlikelihood_loss": 0.19028885662555695, + "theoretical_loss": 3.376944496324616, + "tokens_seen": 2423193600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013426278836509527, + "loss": 1.1816, + "theoretical_loss": 3.3769371657441933, + "tokens_seen": 2423259136 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013424272818455366, + "loss": 1.3118, + "theoretical_loss": 3.376922505344608, + "tokens_seen": 2423390208 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013422266800401203, + "loss": 1.4945, + "theoretical_loss": 3.3769078459599324, + "tokens_seen": 2423521280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013420260782347042, + "loss": 1.2817, + "theoretical_loss": 3.376893187590041, + "tokens_seen": 2423652352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013418254764292878, + "loss": 1.3675, + "theoretical_loss": 3.376878530234809, + "tokens_seen": 2423783424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013416248746238717, + "loss": 1.3696, + "theoretical_loss": 3.376863873894111, + "tokens_seen": 2423914496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013414242728184553, + "loss": 1.2813, + "theoretical_loss": 3.376849218567822, + "tokens_seen": 2424045568 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001341223671013039, + "loss": 1.3383, + "theoretical_loss": 3.3768345642558173, + "tokens_seen": 2424176640 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013410230692076228, + "loss": 1.2008, + "theoretical_loss": 3.3768199109579715, + "tokens_seen": 2424307712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013408224674022064, + "loss": 1.1821, + "theoretical_loss": 3.3768052586741595, + "tokens_seen": 2424438784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013406218655967903, + "loss": 1.3794, + "theoretical_loss": 3.3767906074042564, + "tokens_seen": 2424569856 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001340421263791374, + "loss": 1.1597, + "theoretical_loss": 3.376775957148138, + "tokens_seen": 2424700928 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.33547165989875793, + "objective/train/avg_token_score": 0.010109266266226768, + "objective/train/docs_used": 1366807, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0341312885284424, + "objective/train/original_loss": 3.8938887119293213, + "objective/train/theoretical_loss": 3.376761307905678, + "objective/train/tokens_used": 2424832000, + "objective/train/unlikelihood_loss": 0.25114792585372925, + "theoretical_loss": 3.376761307905678, + "tokens_seen": 2424832000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013402206619859578, + "loss": 1.4685, + "theoretical_loss": 3.376761307905678, + "tokens_seen": 2424832000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013400200601805417, + "loss": 1.2627, + "theoretical_loss": 3.376746659676752, + "tokens_seen": 2424963072 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013398194583751256, + "loss": 1.3397, + "theoretical_loss": 3.3767320124612357, + "tokens_seen": 2425094144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013396188565697093, + "loss": 1.4265, + "theoretical_loss": 3.3767173662590038, + "tokens_seen": 2425225216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013394182547642931, + "loss": 1.1966, + "theoretical_loss": 3.3767027210699307, + "tokens_seen": 2425356288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013392176529588768, + "loss": 1.2905, + "theoretical_loss": 3.376688076893893, + "tokens_seen": 2425487360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013390170511534604, + "loss": 1.3182, + "theoretical_loss": 3.376673433730765, + "tokens_seen": 2425618432 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013388164493480443, + "loss": 1.2985, + "theoretical_loss": 3.376658791580422, + "tokens_seen": 2425749504 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001338615847542628, + "loss": 1.2653, + "theoretical_loss": 3.3766441504427394, + "tokens_seen": 2425880576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013384152457372118, + "loss": 1.4211, + "theoretical_loss": 3.376629510317592, + "tokens_seen": 2426011648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013382146439317954, + "loss": 1.4273, + "theoretical_loss": 3.3766148712048563, + "tokens_seen": 2426142720 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013380140421263793, + "loss": 1.3713, + "theoretical_loss": 3.376600233104406, + "tokens_seen": 2426273792 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001337813440320963, + "loss": 1.2626, + "theoretical_loss": 3.3765855960161173, + "tokens_seen": 2426404864 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.48894184827804565, + "objective/train/avg_token_score": 0.003422300796955824, + "objective/train/docs_used": 1367391, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5913094282150269, + "objective/train/original_loss": 3.814455270767212, + "objective/train/theoretical_loss": 3.3765782778514946, + "objective/train/tokens_used": 2426470400, + "objective/train/unlikelihood_loss": 0.14930018782615662, + "theoretical_loss": 3.3765782778514946, + "tokens_seen": 2426470400 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013376128385155468, + "loss": 1.3033, + "theoretical_loss": 3.3765709599398654, + "tokens_seen": 2426535936 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013374122367101305, + "loss": 1.3381, + "theoretical_loss": 3.376556324875526, + "tokens_seen": 2426667008 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001337211634904714, + "loss": 1.132, + "theoretical_loss": 3.3765416908229735, + "tokens_seen": 2426798080 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001337011033099298, + "loss": 1.3287, + "theoretical_loss": 3.3765270577820847, + "tokens_seen": 2426929152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013368104312938816, + "loss": 1.3468, + "theoretical_loss": 3.376512425752734, + "tokens_seen": 2427060224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013366098294884655, + "loss": 1.2837, + "theoretical_loss": 3.3764977947347967, + "tokens_seen": 2427191296 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001336409227683049, + "loss": 1.448, + "theoretical_loss": 3.3764831647281492, + "tokens_seen": 2427322368 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001336208625877633, + "loss": 1.273, + "theoretical_loss": 3.3764685357326667, + "tokens_seen": 2427453440 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013360080240722166, + "loss": 1.3539, + "theoretical_loss": 3.376453907748224, + "tokens_seen": 2427584512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013358074222668005, + "loss": 1.2461, + "theoretical_loss": 3.3764392807746977, + "tokens_seen": 2427715584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013356068204613841, + "loss": 1.2365, + "theoretical_loss": 3.3764246548119625, + "tokens_seen": 2427846656 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013354062186559678, + "loss": 1.3999, + "theoretical_loss": 3.3764100298598945, + "tokens_seen": 2427977728 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.48148825764656067, + "objective/train/avg_token_score": 0.006856139283627272, + "objective/train/docs_used": 1368496, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.458189845085144, + "objective/train/original_loss": 3.2424471378326416, + "objective/train/theoretical_loss": 3.376395405918369, + "objective/train/tokens_used": 2428108800, + "objective/train/unlikelihood_loss": 0.17081989347934723, + "theoretical_loss": 3.376395405918369, + "tokens_seen": 2428108800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013352056168505517, + "loss": 1.4441, + "theoretical_loss": 3.376395405918369, + "tokens_seen": 2428108800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013350050150451353, + "loss": 1.407, + "theoretical_loss": 3.376380782987262, + "tokens_seen": 2428239872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013348044132397192, + "loss": 1.3307, + "theoretical_loss": 3.376366161066449, + "tokens_seen": 2428370944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013346038114343028, + "loss": 1.4108, + "theoretical_loss": 3.376351540155805, + "tokens_seen": 2428502016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013344032096288867, + "loss": 1.2776, + "theoretical_loss": 3.376336920255207, + "tokens_seen": 2428633088 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013342026078234703, + "loss": 1.3598, + "theoretical_loss": 3.37632230136453, + "tokens_seen": 2428764160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013340020060180542, + "loss": 1.3157, + "theoretical_loss": 3.376307683483649, + "tokens_seen": 2428895232 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013338014042126378, + "loss": 1.4849, + "theoretical_loss": 3.376293066612441, + "tokens_seen": 2429026304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013336008024072217, + "loss": 1.322, + "theoretical_loss": 3.3762784507507813, + "tokens_seen": 2429157376 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013334002006018053, + "loss": 1.3143, + "theoretical_loss": 3.3762638358985457, + "tokens_seen": 2429288448 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001333199598796389, + "loss": 1.1727, + "theoretical_loss": 3.37624922205561, + "tokens_seen": 2429419520 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013329989969909729, + "loss": 1.325, + "theoretical_loss": 3.37623460922185, + "tokens_seen": 2429550592 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013327983951855565, + "loss": 1.4703, + "theoretical_loss": 3.3762199973971416, + "tokens_seen": 2429681664 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.41410067677497864, + "objective/train/avg_token_score": 0.013785120099782944, + "objective/train/docs_used": 1368772, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2090646028518677, + "objective/train/original_loss": 4.004479885101318, + "objective/train/theoretical_loss": 3.3762126918631434, + "objective/train/tokens_used": 2429747200, + "objective/train/unlikelihood_loss": 0.1300012171268463, + "theoretical_loss": 3.3762126918631434, + "tokens_seen": 2429747200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013325977933801404, + "loss": 1.1549, + "theoretical_loss": 3.376205386581361, + "tokens_seen": 2429812736 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013323971915747243, + "loss": 1.4134, + "theoretical_loss": 3.3761907767743837, + "tokens_seen": 2429943808 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013321965897693082, + "loss": 1.4239, + "theoretical_loss": 3.376176167976086, + "tokens_seen": 2430074880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013319959879638918, + "loss": 1.1565, + "theoretical_loss": 3.3761615601863433, + "tokens_seen": 2430205952 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013317953861584757, + "loss": 1.329, + "theoretical_loss": 3.376146953405032, + "tokens_seen": 2430337024 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013315947843530593, + "loss": 1.2539, + "theoretical_loss": 3.376132347632028, + "tokens_seen": 2430468096 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001331394182547643, + "loss": 1.3343, + "theoretical_loss": 3.376117742867208, + "tokens_seen": 2430599168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013311935807422268, + "loss": 1.505, + "theoretical_loss": 3.376103139110447, + "tokens_seen": 2430730240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013309929789368104, + "loss": 1.2312, + "theoretical_loss": 3.376088536361621, + "tokens_seen": 2430861312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013307923771313943, + "loss": 1.2467, + "theoretical_loss": 3.3760739346206075, + "tokens_seen": 2430992384 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001330591775325978, + "loss": 1.3434, + "theoretical_loss": 3.376059333887281, + "tokens_seen": 2431123456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013303911735205618, + "loss": 1.2382, + "theoretical_loss": 3.3760447341615185, + "tokens_seen": 2431254528 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.42247068881988525, + "objective/train/avg_token_score": 0.010578609071671963, + "objective/train/docs_used": 1369369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3805270195007324, + "objective/train/original_loss": 4.568303108215332, + "objective/train/theoretical_loss": 3.3760301354431963, + "objective/train/tokens_used": 2431385600, + "objective/train/unlikelihood_loss": 0.10472545772790909, + "theoretical_loss": 3.3760301354431963, + "tokens_seen": 2431385600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013301905717151455, + "loss": 1.2558, + "theoretical_loss": 3.3760301354431963, + "tokens_seen": 2431385600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013299899699097294, + "loss": 1.3096, + "theoretical_loss": 3.37601553773219, + "tokens_seen": 2431516672 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001329789368104313, + "loss": 1.5194, + "theoretical_loss": 3.3760009410283756, + "tokens_seen": 2431647744 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001329588766298897, + "loss": 1.4072, + "theoretical_loss": 3.3759863453316306, + "tokens_seen": 2431778816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013293881644934805, + "loss": 1.3576, + "theoretical_loss": 3.37597175064183, + "tokens_seen": 2431909888 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001329187562688064, + "loss": 1.4122, + "theoretical_loss": 3.3759571569588505, + "tokens_seen": 2432040960 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001328986960882648, + "loss": 1.3219, + "theoretical_loss": 3.3759425642825684, + "tokens_seen": 2432172032 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013287863590772316, + "loss": 1.3634, + "theoretical_loss": 3.37592797261286, + "tokens_seen": 2432303104 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013285857572718155, + "loss": 1.3948, + "theoretical_loss": 3.3759133819496014, + "tokens_seen": 2432434176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013283851554663992, + "loss": 1.3338, + "theoretical_loss": 3.3758987922926695, + "tokens_seen": 2432565248 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001328184553660983, + "loss": 1.3846, + "theoretical_loss": 3.3758842036419408, + "tokens_seen": 2432696320 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013279839518555667, + "loss": 1.3139, + "theoretical_loss": 3.3758696159972903, + "tokens_seen": 2432827392 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013277833500501506, + "loss": 1.2604, + "theoretical_loss": 3.375855029358596, + "tokens_seen": 2432958464 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.4818548262119293, + "objective/train/avg_token_score": 0.005415834486484528, + "objective/train/docs_used": 1370835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4493337869644165, + "objective/train/original_loss": 3.819040298461914, + "objective/train/theoretical_loss": 3.3758477364164436, + "objective/train/tokens_used": 2433024000, + "objective/train/unlikelihood_loss": 0.14338652789592743, + "theoretical_loss": 3.3758477364164436, + "tokens_seen": 2433024000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013275827482447342, + "loss": 1.357, + "theoretical_loss": 3.3758404437257337, + "tokens_seen": 2433089536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013273821464393178, + "loss": 1.2161, + "theoretical_loss": 3.3758258590985797, + "tokens_seen": 2433220608 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013271815446339017, + "loss": 1.3791, + "theoretical_loss": 3.3758112754770107, + "tokens_seen": 2433351680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013269809428284853, + "loss": 1.2783, + "theoretical_loss": 3.375796692860903, + "tokens_seen": 2433482752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013267803410230692, + "loss": 1.3004, + "theoretical_loss": 3.375782111250133, + "tokens_seen": 2433613824 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013265797392176528, + "loss": 1.3997, + "theoretical_loss": 3.3757675306445782, + "tokens_seen": 2433744896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013263791374122367, + "loss": 1.3863, + "theoretical_loss": 3.3757529510441144, + "tokens_seen": 2433875968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013261785356068204, + "loss": 1.4347, + "theoretical_loss": 3.375738372448618, + "tokens_seen": 2434007040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013259779338014043, + "loss": 1.3051, + "theoretical_loss": 3.375723794857966, + "tokens_seen": 2434138112 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001325777331995988, + "loss": 1.3751, + "theoretical_loss": 3.375709218272035, + "tokens_seen": 2434269184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013255767301905715, + "loss": 1.3645, + "theoretical_loss": 3.3756946426907017, + "tokens_seen": 2434400256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013253761283851554, + "loss": 1.4205, + "theoretical_loss": 3.375680068113843, + "tokens_seen": 2434531328 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.33889296650886536, + "objective/train/avg_token_score": 0.0121343694627285, + "objective/train/docs_used": 1371381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0578525066375732, + "objective/train/original_loss": 3.891307830810547, + "objective/train/theoretical_loss": 3.375665494541335, + "objective/train/tokens_used": 2434662400, + "objective/train/unlikelihood_loss": 0.20551219582557678, + "theoretical_loss": 3.375665494541335, + "tokens_seen": 2434662400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001325175526579739, + "loss": 1.5311, + "theoretical_loss": 3.375665494541335, + "tokens_seen": 2434662400 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013249749247743232, + "loss": 1.3645, + "theoretical_loss": 3.3756509219730546, + "tokens_seen": 2434793472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013247743229689068, + "loss": 1.3306, + "theoretical_loss": 3.375636350408879, + "tokens_seen": 2434924544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013245737211634907, + "loss": 1.1864, + "theoretical_loss": 3.3756217798486845, + "tokens_seen": 2435055616 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013243731193580743, + "loss": 1.2333, + "theoretical_loss": 3.375607210292348, + "tokens_seen": 2435186688 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013241725175526582, + "loss": 1.3036, + "theoretical_loss": 3.3755926417397464, + "tokens_seen": 2435317760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013239719157472418, + "loss": 1.1844, + "theoretical_loss": 3.375578074190756, + "tokens_seen": 2435448832 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013237713139418257, + "loss": 1.4226, + "theoretical_loss": 3.375563507645255, + "tokens_seen": 2435579904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013235707121364093, + "loss": 1.2598, + "theoretical_loss": 3.375548942103119, + "tokens_seen": 2435710976 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001323370110330993, + "loss": 1.324, + "theoretical_loss": 3.3755343775642253, + "tokens_seen": 2435842048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013231695085255769, + "loss": 1.3284, + "theoretical_loss": 3.3755198140284506, + "tokens_seen": 2435973120 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013229689067201605, + "loss": 1.3595, + "theoretical_loss": 3.3755052514956723, + "tokens_seen": 2436104192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013227683049147444, + "loss": 1.2353, + "theoretical_loss": 3.3754906899657673, + "tokens_seen": 2436235264 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.37836021184921265, + "objective/train/avg_token_score": 0.009234017692506313, + "objective/train/docs_used": 1372668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1662172079086304, + "objective/train/original_loss": 3.5548665523529053, + "objective/train/theoretical_loss": 3.3754834095768533, + "objective/train/tokens_used": 2436300800, + "objective/train/unlikelihood_loss": 0.1837824136018753, + "theoretical_loss": 3.3754834095768533, + "tokens_seen": 2436300800 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001322567703109328, + "loss": 1.3053, + "theoretical_loss": 3.375476129438612, + "tokens_seen": 2436366336 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001322367101303912, + "loss": 1.3524, + "theoretical_loss": 3.3754615699140844, + "tokens_seen": 2436497408 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013221664994984955, + "loss": 1.3896, + "theoretical_loss": 3.3754470113920605, + "tokens_seen": 2436628480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013219658976930794, + "loss": 1.372, + "theoretical_loss": 3.375432453872418, + "tokens_seen": 2436759552 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001321765295887663, + "loss": 1.3653, + "theoretical_loss": 3.3754178973550335, + "tokens_seen": 2436890624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013215646940822467, + "loss": 1.4546, + "theoretical_loss": 3.375403341839785, + "tokens_seen": 2437021696 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013213640922768305, + "loss": 1.2899, + "theoretical_loss": 3.3753887873265485, + "tokens_seen": 2437152768 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013211634904714142, + "loss": 1.2444, + "theoretical_loss": 3.3753742338152017, + "tokens_seen": 2437283840 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001320962888665998, + "loss": 1.3361, + "theoretical_loss": 3.3753596813056213, + "tokens_seen": 2437414912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013207622868605817, + "loss": 1.2751, + "theoretical_loss": 3.3753451297976858, + "tokens_seen": 2437545984 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013205616850551656, + "loss": 1.2675, + "theoretical_loss": 3.375330579291271, + "tokens_seen": 2437677056 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013203610832497492, + "loss": 1.3274, + "theoretical_loss": 3.375316029786254, + "tokens_seen": 2437808128 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.33339443802833557, + "objective/train/avg_token_score": 0.01571497693657875, + "objective/train/docs_used": 1373513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0080534219741821, + "objective/train/original_loss": 3.8283193111419678, + "objective/train/theoretical_loss": 3.3753014812825133, + "objective/train/tokens_used": 2437939200, + "objective/train/unlikelihood_loss": 0.17019516229629517, + "theoretical_loss": 3.3753014812825133, + "tokens_seen": 2437939200 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001320160481444333, + "loss": 1.3594, + "theoretical_loss": 3.3753014812825133, + "tokens_seen": 2437939200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013199598796389167, + "loss": 1.4317, + "theoretical_loss": 3.3752869337799254, + "tokens_seen": 2438070272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013197592778335003, + "loss": 1.3924, + "theoretical_loss": 3.375272387278368, + "tokens_seen": 2438201344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013195586760280842, + "loss": 1.3852, + "theoretical_loss": 3.3752578417777177, + "tokens_seen": 2438332416 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013193580742226679, + "loss": 1.2368, + "theoretical_loss": 3.3752432972778523, + "tokens_seen": 2438463488 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013191574724172517, + "loss": 1.3877, + "theoretical_loss": 3.375228753778649, + "tokens_seen": 2438594560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013189568706118354, + "loss": 1.4091, + "theoretical_loss": 3.375214211279985, + "tokens_seen": 2438725632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013187562688064193, + "loss": 1.4453, + "theoretical_loss": 3.375199669781739, + "tokens_seen": 2438856704 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318555667001003, + "loss": 1.2212, + "theoretical_loss": 3.3751851292837864, + "tokens_seen": 2438987776 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013183550651955868, + "loss": 1.2036, + "theoretical_loss": 3.375170589786006, + "tokens_seen": 2439118848 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013181544633901704, + "loss": 1.2338, + "theoretical_loss": 3.3751560512882746, + "tokens_seen": 2439249920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013179538615847543, + "loss": 1.2933, + "theoretical_loss": 3.3751415137904703, + "tokens_seen": 2439380992 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001317753259779338, + "loss": 1.394, + "theoretical_loss": 3.3751269772924704, + "tokens_seen": 2439512064 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.3566104471683502, + "objective/train/avg_token_score": 0.010680005885660648, + "objective/train/docs_used": 1374810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1794753074645996, + "objective/train/original_loss": 3.72882080078125, + "objective/train/theoretical_loss": 3.3751197094183585, + "objective/train/tokens_used": 2439577600, + "objective/train/unlikelihood_loss": 0.200412780046463, + "theoretical_loss": 3.3751197094183585, + "tokens_seen": 2439577600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013175526579739218, + "loss": 1.2493, + "theoretical_loss": 3.375112441794152, + "tokens_seen": 2439643136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013173520561685057, + "loss": 1.2353, + "theoretical_loss": 3.375097907295393, + "tokens_seen": 2439774208 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013171514543630893, + "loss": 1.4767, + "theoretical_loss": 3.375083373796071, + "tokens_seen": 2439905280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013169508525576732, + "loss": 1.4075, + "theoretical_loss": 3.3750688412960637, + "tokens_seen": 2440036352 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013167502507522568, + "loss": 1.4567, + "theoretical_loss": 3.375054309795248, + "tokens_seen": 2440167424 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013165496489468407, + "loss": 1.1279, + "theoretical_loss": 3.3750397792935023, + "tokens_seen": 2440298496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013163490471414244, + "loss": 1.4226, + "theoretical_loss": 3.375025249790704, + "tokens_seen": 2440429568 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013161484453360083, + "loss": 1.2568, + "theoretical_loss": 3.375010721286731, + "tokens_seen": 2440560640 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001315947843530592, + "loss": 1.1519, + "theoretical_loss": 3.3749961937814605, + "tokens_seen": 2440691712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013157472417251755, + "loss": 1.4602, + "theoretical_loss": 3.3749816672747706, + "tokens_seen": 2440822784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013155466399197594, + "loss": 1.3216, + "theoretical_loss": 3.374967141766539, + "tokens_seen": 2440953856 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001315346038114343, + "loss": 1.3697, + "theoretical_loss": 3.3749526172566435, + "tokens_seen": 2441084928 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.3008309006690979, + "objective/train/avg_token_score": 0.005307936109602451, + "objective/train/docs_used": 1375482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9629269242286682, + "objective/train/original_loss": 3.8463449478149414, + "objective/train/theoretical_loss": 3.374938093744962, + "objective/train/tokens_used": 2441216000, + "objective/train/unlikelihood_loss": 0.1930529624223709, + "theoretical_loss": 3.374938093744962, + "tokens_seen": 2441216000 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001315145436308927, + "loss": 1.1971, + "theoretical_loss": 3.374938093744962, + "tokens_seen": 2441216000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013149448345035105, + "loss": 1.2778, + "theoretical_loss": 3.3749235712313714, + "tokens_seen": 2441347072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013147442326980944, + "loss": 1.3408, + "theoretical_loss": 3.374909049715751, + "tokens_seen": 2441478144 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001314543630892678, + "loss": 1.3583, + "theoretical_loss": 3.3748945291979773, + "tokens_seen": 2441609216 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001314343029087262, + "loss": 1.284, + "theoretical_loss": 3.374880009677929, + "tokens_seen": 2441740288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013141424272818456, + "loss": 1.2244, + "theoretical_loss": 3.3748654911554836, + "tokens_seen": 2441871360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013139418254764295, + "loss": 1.3214, + "theoretical_loss": 3.3748509736305192, + "tokens_seen": 2442002432 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001313741223671013, + "loss": 1.451, + "theoretical_loss": 3.374836457102914, + "tokens_seen": 2442133504 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013135406218655967, + "loss": 1.285, + "theoretical_loss": 3.374821941572545, + "tokens_seen": 2442264576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013133400200601806, + "loss": 1.2247, + "theoretical_loss": 3.3748074270392916, + "tokens_seen": 2442395648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013131394182547642, + "loss": 1.2027, + "theoretical_loss": 3.374792913503031, + "tokens_seen": 2442526720 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001312938816449348, + "loss": 1.1805, + "theoretical_loss": 3.3747784009636406, + "tokens_seen": 2442657792 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013127382146439317, + "loss": 1.3919, + "theoretical_loss": 3.3747638894209997, + "tokens_seen": 2442788864 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.4065249264240265, + "objective/train/avg_token_score": 0.013941793702542782, + "objective/train/docs_used": 1376516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3988875150680542, + "objective/train/original_loss": 4.280553340911865, + "objective/train/theoretical_loss": 3.374756634023422, + "objective/train/tokens_used": 2442854400, + "objective/train/unlikelihood_loss": 0.14194297790527344, + "theoretical_loss": 3.374756634023422, + "tokens_seen": 2442854400 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013125376128385156, + "loss": 1.3299, + "theoretical_loss": 3.374749378874985, + "tokens_seen": 2442919936 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013123370110330992, + "loss": 1.274, + "theoretical_loss": 3.374734869325476, + "tokens_seen": 2443051008 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013121364092276831, + "loss": 1.3491, + "theoretical_loss": 3.3747203607723506, + "tokens_seen": 2443182080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013119358074222668, + "loss": 1.3869, + "theoretical_loss": 3.3747058532154854, + "tokens_seen": 2443313152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013117352056168504, + "loss": 1.2974, + "theoretical_loss": 3.3746913466547603, + "tokens_seen": 2443444224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013115346038114343, + "loss": 1.2175, + "theoretical_loss": 3.3746768410900527, + "tokens_seen": 2443575296 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001311334002006018, + "loss": 1.3801, + "theoretical_loss": 3.374662336521241, + "tokens_seen": 2443706368 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013111334002006018, + "loss": 1.2599, + "theoretical_loss": 3.3746478329482033, + "tokens_seen": 2443837440 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013109327983951854, + "loss": 1.2202, + "theoretical_loss": 3.3746333303708176, + "tokens_seen": 2443968512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013107321965897693, + "loss": 1.147, + "theoretical_loss": 3.3746188287889627, + "tokens_seen": 2444099584 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001310531594784353, + "loss": 1.5219, + "theoretical_loss": 3.3746043282025164, + "tokens_seen": 2444230656 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013103309929789368, + "loss": 1.2184, + "theoretical_loss": 3.3745898286113576, + "tokens_seen": 2444361728 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.4359726309776306, + "objective/train/avg_token_score": 0.014014572836458683, + "objective/train/docs_used": 1376516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3405327796936035, + "objective/train/original_loss": 3.4046225547790527, + "objective/train/theoretical_loss": 3.3745753300153636, + "objective/train/tokens_used": 2444492800, + "objective/train/unlikelihood_loss": 0.16560351848602295, + "theoretical_loss": 3.3745753300153636, + "tokens_seen": 2444492800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013101303911735204, + "loss": 1.4563, + "theoretical_loss": 3.3745753300153636, + "tokens_seen": 2444492800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013099297893681043, + "loss": 1.1889, + "theoretical_loss": 3.3745608324144136, + "tokens_seen": 2444623872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013097291875626882, + "loss": 1.3563, + "theoretical_loss": 3.374546335808386, + "tokens_seen": 2444754944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013095285857572719, + "loss": 1.3633, + "theoretical_loss": 3.3745318401971582, + "tokens_seen": 2444886016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013093279839518558, + "loss": 1.301, + "theoretical_loss": 3.37451734558061, + "tokens_seen": 2445017088 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013091273821464394, + "loss": 1.5559, + "theoretical_loss": 3.374502851958619, + "tokens_seen": 2445148160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013089267803410233, + "loss": 1.3837, + "theoretical_loss": 3.3744883593310635, + "tokens_seen": 2445279232 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001308726178535607, + "loss": 1.2584, + "theoretical_loss": 3.3744738676978225, + "tokens_seen": 2445410304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013085255767301908, + "loss": 1.358, + "theoretical_loss": 3.3744593770587743, + "tokens_seen": 2445541376 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013083249749247744, + "loss": 1.3756, + "theoretical_loss": 3.374444887413797, + "tokens_seen": 2445672448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013081243731193583, + "loss": 1.2421, + "theoretical_loss": 3.37443039876277, + "tokens_seen": 2445803520 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001307923771313942, + "loss": 1.1644, + "theoretical_loss": 3.3744159111055714, + "tokens_seen": 2445934592 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013077231695085255, + "loss": 1.4324, + "theoretical_loss": 3.3744014244420795, + "tokens_seen": 2446065664 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.3693181872367859, + "objective/train/avg_token_score": 0.01604434847831726, + "objective/train/docs_used": 1377094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1149852275848389, + "objective/train/original_loss": 3.8539233207702637, + "objective/train/theoretical_loss": 3.3743941814829355, + "objective/train/tokens_used": 2446131200, + "objective/train/unlikelihood_loss": 0.1914929449558258, + "theoretical_loss": 3.3743941814829355, + "tokens_seen": 2446131200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013075225677031094, + "loss": 1.446, + "theoretical_loss": 3.374386938772173, + "tokens_seen": 2446196736 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001307321965897693, + "loss": 1.3269, + "theoretical_loss": 3.374372454095731, + "tokens_seen": 2446327808 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001307121364092277, + "loss": 1.3935, + "theoretical_loss": 3.3743579704126314, + "tokens_seen": 2446458880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013069207622868606, + "loss": 1.3187, + "theoretical_loss": 3.3743434877227534, + "tokens_seen": 2446589952 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013067201604814445, + "loss": 1.291, + "theoretical_loss": 3.3743290060259756, + "tokens_seen": 2446721024 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001306519558676028, + "loss": 1.3666, + "theoretical_loss": 3.374314525322177, + "tokens_seen": 2446852096 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001306318956870612, + "loss": 1.3959, + "theoretical_loss": 3.3743000456112355, + "tokens_seen": 2446983168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013061183550651956, + "loss": 1.3152, + "theoretical_loss": 3.3742855668930307, + "tokens_seen": 2447114240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013059177532597792, + "loss": 1.306, + "theoretical_loss": 3.374271089167441, + "tokens_seen": 2447245312 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001305717151454363, + "loss": 1.248, + "theoretical_loss": 3.3742566124343445, + "tokens_seen": 2447376384 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013055165496489467, + "loss": 1.2142, + "theoretical_loss": 3.3742421366936215, + "tokens_seen": 2447507456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013053159478435306, + "loss": 1.2775, + "theoretical_loss": 3.3742276619451497, + "tokens_seen": 2447638528 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.40169844031333923, + "objective/train/avg_token_score": 0.005674814805388451, + "objective/train/docs_used": 1378304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2345160245895386, + "objective/train/original_loss": 3.657505989074707, + "objective/train/theoretical_loss": 3.374213188188808, + "objective/train/tokens_used": 2447769600, + "objective/train/unlikelihood_loss": 0.18412935733795166, + "theoretical_loss": 3.374213188188808, + "tokens_seen": 2447769600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013051153460381143, + "loss": 1.2835, + "theoretical_loss": 3.374213188188808, + "tokens_seen": 2447769600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013049147442326982, + "loss": 1.3481, + "theoretical_loss": 3.3741987154244764, + "tokens_seen": 2447900672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013047141424272818, + "loss": 1.1954, + "theoretical_loss": 3.3741842436520324, + "tokens_seen": 2448031744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013045135406218657, + "loss": 1.3575, + "theoretical_loss": 3.3741697728713556, + "tokens_seen": 2448162816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013043129388164493, + "loss": 1.2736, + "theoretical_loss": 3.374155303082325, + "tokens_seen": 2448293888 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001304112337011033, + "loss": 1.3694, + "theoretical_loss": 3.374140834284819, + "tokens_seen": 2448424960 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013039117352056168, + "loss": 1.6081, + "theoretical_loss": 3.3741263664787176, + "tokens_seen": 2448556032 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013037111334002004, + "loss": 1.1498, + "theoretical_loss": 3.3741118996638986, + "tokens_seen": 2448687104 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013035105315947843, + "loss": 1.4478, + "theoretical_loss": 3.374097433840242, + "tokens_seen": 2448818176 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001303309929789368, + "loss": 1.3839, + "theoretical_loss": 3.374082969007626, + "tokens_seen": 2448949248 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013031093279839518, + "loss": 1.3322, + "theoretical_loss": 3.3740685051659307, + "tokens_seen": 2449080320 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013029087261785355, + "loss": 1.3292, + "theoretical_loss": 3.3740540423150343, + "tokens_seen": 2449211392 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013027081243731194, + "loss": 1.2356, + "theoretical_loss": 3.374039580454816, + "tokens_seen": 2449342464 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.484665185213089, + "objective/train/avg_token_score": 0.006117618642747402, + "objective/train/docs_used": 1378955, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3923017978668213, + "objective/train/original_loss": 3.772844076156616, + "objective/train/theoretical_loss": 3.374032349896174, + "objective/train/tokens_used": 2449408000, + "objective/train/unlikelihood_loss": 0.14212283492088318, + "theoretical_loss": 3.374032349896174, + "tokens_seen": 2449408000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013025075225677033, + "loss": 1.2412, + "theoretical_loss": 3.3740251195851556, + "tokens_seen": 2449473536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013023069207622871, + "loss": 1.3424, + "theoretical_loss": 3.3740106597059314, + "tokens_seen": 2449604608 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013021063189568708, + "loss": 1.2891, + "theoretical_loss": 3.3739962008170234, + "tokens_seen": 2449735680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013019057171514544, + "loss": 1.287, + "theoretical_loss": 3.37398174291831, + "tokens_seen": 2449866752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013017051153460383, + "loss": 1.2117, + "theoretical_loss": 3.373967286009671, + "tokens_seen": 2449997824 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001301504513540622, + "loss": 1.2492, + "theoretical_loss": 3.3739528300909853, + "tokens_seen": 2450128896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013013039117352058, + "loss": 1.2975, + "theoretical_loss": 3.3739383751621324, + "tokens_seen": 2450259968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013011033099297894, + "loss": 1.3631, + "theoretical_loss": 3.3739239212229912, + "tokens_seen": 2450391040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013009027081243733, + "loss": 1.2351, + "theoretical_loss": 3.3739094682734416, + "tokens_seen": 2450522112 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001300702106318957, + "loss": 1.3486, + "theoretical_loss": 3.3738950163133623, + "tokens_seen": 2450653184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013005015045135408, + "loss": 1.2018, + "theoretical_loss": 3.3738805653426334, + "tokens_seen": 2450784256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013003009027081245, + "loss": 1.261, + "theoretical_loss": 3.3738661153611336, + "tokens_seen": 2450915328 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.2721163332462311, + "objective/train/avg_token_score": 0.015441839583218098, + "objective/train/docs_used": 1380357, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8002740144729614, + "objective/train/original_loss": 4.215646743774414, + "objective/train/theoretical_loss": 3.373851666368742, + "objective/train/tokens_used": 2451046400, + "objective/train/unlikelihood_loss": 0.13252310454845428, + "theoretical_loss": 3.373851666368742, + "tokens_seen": 2451046400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001300100300902708, + "loss": 1.335, + "theoretical_loss": 3.373851666368742, + "tokens_seen": 2451046400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001299899699097292, + "loss": 1.2985, + "theoretical_loss": 3.373837218365339, + "tokens_seen": 2451177472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012996990972918756, + "loss": 1.345, + "theoretical_loss": 3.3738227713508038, + "tokens_seen": 2451308544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012994984954864595, + "loss": 1.3251, + "theoretical_loss": 3.373808325325015, + "tokens_seen": 2451439616 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001299297893681043, + "loss": 1.3062, + "theoretical_loss": 3.373793880287853, + "tokens_seen": 2451570688 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001299097291875627, + "loss": 1.235, + "theoretical_loss": 3.3737794362391966, + "tokens_seen": 2451701760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012988966900702106, + "loss": 1.5853, + "theoretical_loss": 3.3737649931789258, + "tokens_seen": 2451832832 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012986960882647945, + "loss": 1.3259, + "theoretical_loss": 3.37375055110692, + "tokens_seen": 2451963904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012984954864593781, + "loss": 1.4574, + "theoretical_loss": 3.3737361100230587, + "tokens_seen": 2452094976 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012982948846539618, + "loss": 1.3506, + "theoretical_loss": 3.3737216699272214, + "tokens_seen": 2452226048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012980942828485457, + "loss": 1.2082, + "theoretical_loss": 3.373707230819288, + "tokens_seen": 2452357120 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012978936810431293, + "loss": 1.3168, + "theoretical_loss": 3.3736927926991376, + "tokens_seen": 2452488192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012976930792377132, + "loss": 1.3119, + "theoretical_loss": 3.3736783555666507, + "tokens_seen": 2452619264 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.45821115374565125, + "objective/train/avg_token_score": 0.030401069670915604, + "objective/train/docs_used": 1380887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.493085265159607, + "objective/train/original_loss": 3.8985280990600586, + "objective/train/theoretical_loss": 3.373671137370743, + "objective/train/tokens_used": 2452684800, + "objective/train/unlikelihood_loss": 0.1450464278459549, + "theoretical_loss": 3.373671137370743, + "tokens_seen": 2452684800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012974924774322968, + "loss": 1.4232, + "theoretical_loss": 3.373663919421706, + "tokens_seen": 2452750336 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012972918756268807, + "loss": 1.3402, + "theoretical_loss": 3.3736494842641838, + "tokens_seen": 2452881408 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012970912738214643, + "loss": 1.2579, + "theoretical_loss": 3.373635050093964, + "tokens_seen": 2453012480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012968906720160482, + "loss": 1.4098, + "theoretical_loss": 3.373620616910925, + "tokens_seen": 2453143552 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012966900702106318, + "loss": 1.4252, + "theoretical_loss": 3.373606184714948, + "tokens_seen": 2453274624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012964894684052157, + "loss": 1.3393, + "theoretical_loss": 3.373591753505912, + "tokens_seen": 2453405696 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012962888665997993, + "loss": 1.2808, + "theoretical_loss": 3.373577323283697, + "tokens_seen": 2453536768 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001296088264794383, + "loss": 1.3188, + "theoretical_loss": 3.3735628940481837, + "tokens_seen": 2453667840 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012958876629889669, + "loss": 1.2857, + "theoretical_loss": 3.3735484657992503, + "tokens_seen": 2453798912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012956870611835505, + "loss": 1.0152, + "theoretical_loss": 3.373534038536777, + "tokens_seen": 2453929984 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012954864593781344, + "loss": 1.2665, + "theoretical_loss": 3.373519612260645, + "tokens_seen": 2454061056 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001295285857572718, + "loss": 1.2472, + "theoretical_loss": 3.373505186970733, + "tokens_seen": 2454192128 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.5084921717643738, + "objective/train/avg_token_score": 0.021491076797246933, + "objective/train/docs_used": 1382209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5006868839263916, + "objective/train/original_loss": 3.621232032775879, + "objective/train/theoretical_loss": 3.373490762666921, + "objective/train/tokens_used": 2454323200, + "objective/train/unlikelihood_loss": 0.14579971134662628, + "theoretical_loss": 3.373490762666921, + "tokens_seen": 2454323200 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001295085255767302, + "loss": 1.2029, + "theoretical_loss": 3.373490762666921, + "tokens_seen": 2454323200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012948846539618858, + "loss": 1.3979, + "theoretical_loss": 3.3734763393490894, + "tokens_seen": 2454454272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012946840521564697, + "loss": 1.4315, + "theoretical_loss": 3.3734619170171176, + "tokens_seen": 2454585344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012944834503510533, + "loss": 1.3205, + "theoretical_loss": 3.3734474956708858, + "tokens_seen": 2454716416 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001294282848545637, + "loss": 1.0839, + "theoretical_loss": 3.3734330753102744, + "tokens_seen": 2454847488 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012940822467402208, + "loss": 1.297, + "theoretical_loss": 3.3734186559351627, + "tokens_seen": 2454978560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012938816449348044, + "loss": 1.2662, + "theoretical_loss": 3.373404237545431, + "tokens_seen": 2455109632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012936810431293883, + "loss": 1.3705, + "theoretical_loss": 3.37338982014096, + "tokens_seen": 2455240704 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001293480441323972, + "loss": 1.296, + "theoretical_loss": 3.3733754037216293, + "tokens_seen": 2455371776 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012932798395185558, + "loss": 1.354, + "theoretical_loss": 3.3733609882873186, + "tokens_seen": 2455502848 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012930792377131395, + "loss": 1.4474, + "theoretical_loss": 3.3733465738379085, + "tokens_seen": 2455633920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012928786359077234, + "loss": 1.2018, + "theoretical_loss": 3.3733321603732787, + "tokens_seen": 2455764992 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001292678034102307, + "loss": 1.2925, + "theoretical_loss": 3.37331774789331, + "tokens_seen": 2455896064 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.40139296650886536, + "objective/train/avg_token_score": 0.012949761003255844, + "objective/train/docs_used": 1382688, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3653579950332642, + "objective/train/original_loss": 4.146922588348389, + "objective/train/theoretical_loss": 3.373310542022536, + "objective/train/tokens_used": 2455961600, + "objective/train/unlikelihood_loss": 0.13372966647148132, + "theoretical_loss": 3.373310542022536, + "tokens_seen": 2455961600 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001292477432296891, + "loss": 1.2787, + "theoretical_loss": 3.3733033363978824, + "tokens_seen": 2456027136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012922768304914745, + "loss": 1.3231, + "theoretical_loss": 3.3732889258868757, + "tokens_seen": 2456158208 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001292076228686058, + "loss": 1.3498, + "theoretical_loss": 3.3732745163601705, + "tokens_seen": 2456289280 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001291875626880642, + "loss": 1.3553, + "theoretical_loss": 3.373260107817647, + "tokens_seen": 2456420352 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012916750250752256, + "loss": 1.3004, + "theoretical_loss": 3.3732457002591856, + "tokens_seen": 2456551424 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012914744232698095, + "loss": 1.397, + "theoretical_loss": 3.3732312936846665, + "tokens_seen": 2456682496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012912738214643932, + "loss": 1.2505, + "theoretical_loss": 3.3732168880939692, + "tokens_seen": 2456813568 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001291073219658977, + "loss": 1.221, + "theoretical_loss": 3.3732024834869754, + "tokens_seen": 2456944640 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012908726178535607, + "loss": 1.3687, + "theoretical_loss": 3.3731880798635645, + "tokens_seen": 2457075712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012906720160481446, + "loss": 1.2685, + "theoretical_loss": 3.3731736772236176, + "tokens_seen": 2457206784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012904714142427282, + "loss": 1.4036, + "theoretical_loss": 3.3731592755670143, + "tokens_seen": 2457337856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012902708124373118, + "loss": 1.4156, + "theoretical_loss": 3.3731448748936357, + "tokens_seen": 2457468928 + }, + { + "debugging/length": 495.3333333333333, + "debugging/num_segments": 3, + "epoch": 0.74, + "objective/train/aligned_token_count": 0.23570381104946136, + "objective/train/avg_token_score": 0.02651287615299225, + "objective/train/docs_used": 1383210, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8070111870765686, + "objective/train/original_loss": 4.41131067276001, + "objective/train/theoretical_loss": 3.3731304752033617, + "objective/train/tokens_used": 2457600000, + "objective/train/unlikelihood_loss": 0.13992837071418762, + "theoretical_loss": 3.3731304752033617, + "tokens_seen": 2457600000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012900702106318957, + "loss": 1.2909, + "theoretical_loss": 3.3731304752033617, + "tokens_seen": 2457600000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012898696088264793, + "loss": 1.2613, + "theoretical_loss": 3.373116076496073, + "tokens_seen": 2457731072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012896690070210632, + "loss": 1.277, + "theoretical_loss": 3.3731016787716506, + "tokens_seen": 2457862144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012894684052156468, + "loss": 1.3733, + "theoretical_loss": 3.373087282029974, + "tokens_seen": 2457993216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012892678034102307, + "loss": 1.3881, + "theoretical_loss": 3.3730728862709243, + "tokens_seen": 2458124288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012890672016048144, + "loss": 1.3965, + "theoretical_loss": 3.373058491494382, + "tokens_seen": 2458255360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012888665997993982, + "loss": 1.4049, + "theoretical_loss": 3.3730440977002276, + "tokens_seen": 2458386432 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001288665997993982, + "loss": 1.2586, + "theoretical_loss": 3.3730297048883413, + "tokens_seen": 2458517504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012884653961885655, + "loss": 1.3069, + "theoretical_loss": 3.3730153130586045, + "tokens_seen": 2458648576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012882647943831494, + "loss": 1.3131, + "theoretical_loss": 3.373000922210897, + "tokens_seen": 2458779648 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001288064192577733, + "loss": 1.3311, + "theoretical_loss": 3.3729865323451005, + "tokens_seen": 2458910720 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001287863590772317, + "loss": 1.2847, + "theoretical_loss": 3.372972143461095, + "tokens_seen": 2459041792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012876629889669005, + "loss": 1.3569, + "theoretical_loss": 3.372957755558761, + "tokens_seen": 2459172864 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.47806695103645325, + "objective/train/avg_token_score": 0.032264579087495804, + "objective/train/docs_used": 1384412, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4370862245559692, + "objective/train/original_loss": 3.8491570949554443, + "objective/train/theoretical_loss": 3.3729505619756837, + "objective/train/tokens_used": 2459238400, + "objective/train/unlikelihood_loss": 0.14841726422309875, + "theoretical_loss": 3.3729505619756837, + "tokens_seen": 2459238400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012874623871614847, + "loss": 1.2715, + "theoretical_loss": 3.372943368637979, + "tokens_seen": 2459303936 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012872617853560683, + "loss": 1.2639, + "theoretical_loss": 3.372928982698631, + "tokens_seen": 2459435008 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012870611835506522, + "loss": 1.2035, + "theoretical_loss": 3.3729145977405963, + "tokens_seen": 2459566080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012868605817452358, + "loss": 1.1417, + "theoretical_loss": 3.3729002137637565, + "tokens_seen": 2459697152 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012866599799398197, + "loss": 1.3733, + "theoretical_loss": 3.3728858307679923, + "tokens_seen": 2459828224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012864593781344033, + "loss": 1.2288, + "theoretical_loss": 3.3728714487531843, + "tokens_seen": 2459959296 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001286258776328987, + "loss": 1.3214, + "theoretical_loss": 3.3728570677192136, + "tokens_seen": 2460090368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012860581745235709, + "loss": 1.2199, + "theoretical_loss": 3.3728426876659605, + "tokens_seen": 2460221440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012858575727181545, + "loss": 1.2906, + "theoretical_loss": 3.3728283085933066, + "tokens_seen": 2460352512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012856569709127384, + "loss": 1.2939, + "theoretical_loss": 3.3728139305011324, + "tokens_seen": 2460483584 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001285456369107322, + "loss": 1.2586, + "theoretical_loss": 3.372799553389319, + "tokens_seen": 2460614656 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001285255767301906, + "loss": 1.4299, + "theoretical_loss": 3.3727851772577466, + "tokens_seen": 2460745728 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.45692816376686096, + "objective/train/avg_token_score": 0.015916263684630394, + "objective/train/docs_used": 1385122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4990222454071045, + "objective/train/original_loss": 3.9778804779052734, + "objective/train/theoretical_loss": 3.372770802106297, + "objective/train/tokens_used": 2460876800, + "objective/train/unlikelihood_loss": 0.1581452190876007, + "theoretical_loss": 3.372770802106297, + "tokens_seen": 2460876800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012850551654964895, + "loss": 1.1745, + "theoretical_loss": 3.372770802106297, + "tokens_seen": 2460876800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012848545636910734, + "loss": 1.3293, + "theoretical_loss": 3.3727564279348514, + "tokens_seen": 2461007872 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001284653961885657, + "loss": 1.2183, + "theoretical_loss": 3.37274205474329, + "tokens_seen": 2461138944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012844533600802407, + "loss": 1.3114, + "theoretical_loss": 3.3727276825314942, + "tokens_seen": 2461270016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012842527582748245, + "loss": 1.3046, + "theoretical_loss": 3.372713311299345, + "tokens_seen": 2461401088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012840521564694082, + "loss": 1.2766, + "theoretical_loss": 3.3726989410467234, + "tokens_seen": 2461532160 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001283851554663992, + "loss": 1.2353, + "theoretical_loss": 3.3726845717735108, + "tokens_seen": 2461663232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012836509528585757, + "loss": 1.2025, + "theoretical_loss": 3.372670203479588, + "tokens_seen": 2461794304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012834503510531596, + "loss": 1.2704, + "theoretical_loss": 3.372655836164836, + "tokens_seen": 2461925376 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012832497492477432, + "loss": 1.2717, + "theoretical_loss": 3.3726414698291363, + "tokens_seen": 2462056448 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001283049147442327, + "loss": 1.3856, + "theoretical_loss": 3.37262710447237, + "tokens_seen": 2462187520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012828485456369107, + "loss": 1.286, + "theoretical_loss": 3.3726127400944184, + "tokens_seen": 2462318592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012826479438314943, + "loss": 1.2922, + "theoretical_loss": 3.372598376695162, + "tokens_seen": 2462449664 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.34756842255592346, + "objective/train/avg_token_score": 0.022292688488960266, + "objective/train/docs_used": 1386383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0406132936477661, + "objective/train/original_loss": 3.6909749507904053, + "objective/train/theoretical_loss": 3.3725911953625074, + "objective/train/tokens_used": 2462515200, + "objective/train/unlikelihood_loss": 0.22240091860294342, + "theoretical_loss": 3.3725911953625074, + "tokens_seen": 2462515200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012824473420260782, + "loss": 1.2861, + "theoretical_loss": 3.3725840142744827, + "tokens_seen": 2462580736 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012822467402206619, + "loss": 1.1771, + "theoretical_loss": 3.3725696528322615, + "tokens_seen": 2462711808 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012820461384152457, + "loss": 1.1981, + "theoretical_loss": 3.37255529236838, + "tokens_seen": 2462842880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012818455366098294, + "loss": 1.423, + "theoretical_loss": 3.372540932882719, + "tokens_seen": 2462973952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012816449348044133, + "loss": 1.3739, + "theoretical_loss": 3.37252657437516, + "tokens_seen": 2463105024 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001281444332998997, + "loss": 1.262, + "theoretical_loss": 3.3725122168455846, + "tokens_seen": 2463236096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012812437311935808, + "loss": 1.3306, + "theoretical_loss": 3.372497860293874, + "tokens_seen": 2463367168 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012810431293881644, + "loss": 1.3364, + "theoretical_loss": 3.3724835047199093, + "tokens_seen": 2463498240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012808425275827483, + "loss": 1.3598, + "theoretical_loss": 3.372469150123572, + "tokens_seen": 2463629312 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001280641925777332, + "loss": 1.1452, + "theoretical_loss": 3.372454796504744, + "tokens_seen": 2463760384 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012804413239719155, + "loss": 1.2965, + "theoretical_loss": 3.372440443863306, + "tokens_seen": 2463891456 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012802407221664994, + "loss": 1.1862, + "theoretical_loss": 3.37242609219914, + "tokens_seen": 2464022528 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.3353494703769684, + "objective/train/avg_token_score": 0.017477696761488914, + "objective/train/docs_used": 1387153, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0820199251174927, + "objective/train/original_loss": 4.468991279602051, + "objective/train/theoretical_loss": 3.3724117415121273, + "objective/train/tokens_used": 2464153600, + "objective/train/unlikelihood_loss": 0.12739039957523346, + "theoretical_loss": 3.3724117415121273, + "tokens_seen": 2464153600 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001280040120361083, + "loss": 1.2821, + "theoretical_loss": 3.3724117415121273, + "tokens_seen": 2464153600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012798395185556672, + "loss": 1.4974, + "theoretical_loss": 3.3723973918021493, + "tokens_seen": 2464284672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012796389167502508, + "loss": 1.3576, + "theoretical_loss": 3.3723830430690875, + "tokens_seen": 2464415744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012794383149448347, + "loss": 1.2398, + "theoretical_loss": 3.372368695312824, + "tokens_seen": 2464546816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012792377131394184, + "loss": 1.3477, + "theoretical_loss": 3.3723543485332392, + "tokens_seen": 2464677888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012790371113340023, + "loss": 1.3943, + "theoretical_loss": 3.3723400027302164, + "tokens_seen": 2464808960 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001278836509528586, + "loss": 1.3528, + "theoretical_loss": 3.3723256579036356, + "tokens_seen": 2464940032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012786359077231695, + "loss": 1.3453, + "theoretical_loss": 3.3723113140533787, + "tokens_seen": 2465071104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012784353059177534, + "loss": 1.2864, + "theoretical_loss": 3.372296971179328, + "tokens_seen": 2465202176 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001278234704112337, + "loss": 1.2556, + "theoretical_loss": 3.372282629281365, + "tokens_seen": 2465333248 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001278034102306921, + "loss": 1.1421, + "theoretical_loss": 3.372268288359371, + "tokens_seen": 2465464320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012778335005015045, + "loss": 1.3302, + "theoretical_loss": 3.3722539484132277, + "tokens_seen": 2465595392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012776328986960884, + "loss": 1.3761, + "theoretical_loss": 3.3722396094428175, + "tokens_seen": 2465726464 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.5560239553451538, + "objective/train/avg_token_score": 0.006050017196685076, + "objective/train/docs_used": 1387695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5503959655761719, + "objective/train/original_loss": 3.2147905826568604, + "objective/train/theoretical_loss": 3.372232440323475, + "objective/train/tokens_used": 2465792000, + "objective/train/unlikelihood_loss": 0.26668158173561096, + "theoretical_loss": 3.372232440323475, + "tokens_seen": 2465792000 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001277432296890672, + "loss": 1.4193, + "theoretical_loss": 3.372225271448021, + "tokens_seen": 2465857536 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001277231695085256, + "loss": 1.3144, + "theoretical_loss": 3.3722109344287214, + "tokens_seen": 2465988608 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012770310932798396, + "loss": 1.3251, + "theoretical_loss": 3.3721965983847992, + "tokens_seen": 2466119680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012768304914744232, + "loss": 1.3116, + "theoretical_loss": 3.3721822633161374, + "tokens_seen": 2466250752 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001276629889669007, + "loss": 1.4296, + "theoretical_loss": 3.3721679292226163, + "tokens_seen": 2466381824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012764292878635907, + "loss": 1.3336, + "theoretical_loss": 3.3721535961041194, + "tokens_seen": 2466512896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012762286860581746, + "loss": 1.3599, + "theoretical_loss": 3.3721392639605274, + "tokens_seen": 2466643968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012760280842527582, + "loss": 1.333, + "theoretical_loss": 3.3721249327917224, + "tokens_seen": 2466775040 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001275827482447342, + "loss": 1.314, + "theoretical_loss": 3.372110602597587, + "tokens_seen": 2466906112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012756268806419257, + "loss": 1.3825, + "theoretical_loss": 3.372096273378002, + "tokens_seen": 2467037184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012754262788365096, + "loss": 1.3522, + "theoretical_loss": 3.3720819451328508, + "tokens_seen": 2467168256 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012752256770310932, + "loss": 1.2939, + "theoretical_loss": 3.372067617862014, + "tokens_seen": 2467299328 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.41807183623313904, + "objective/train/avg_token_score": 0.007097250781953335, + "objective/train/docs_used": 1388592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2504676580429077, + "objective/train/original_loss": 3.958369731903076, + "objective/train/theoretical_loss": 3.372053291565374, + "objective/train/tokens_used": 2467430400, + "objective/train/unlikelihood_loss": 0.1409129798412323, + "theoretical_loss": 3.372053291565374, + "tokens_seen": 2467430400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012750250752256771, + "loss": 1.3105, + "theoretical_loss": 3.372053291565374, + "tokens_seen": 2467430400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012748244734202608, + "loss": 1.4033, + "theoretical_loss": 3.372038966242813, + "tokens_seen": 2467561472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012746238716148444, + "loss": 1.2956, + "theoretical_loss": 3.372024641894213, + "tokens_seen": 2467692544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012744232698094283, + "loss": 1.239, + "theoretical_loss": 3.3720103185194565, + "tokens_seen": 2467823616 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001274222668004012, + "loss": 1.3259, + "theoretical_loss": 3.3719959961184247, + "tokens_seen": 2467954688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012740220661985958, + "loss": 1.2419, + "theoretical_loss": 3.3719816746910003, + "tokens_seen": 2468085760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012738214643931794, + "loss": 1.3544, + "theoretical_loss": 3.371967354237065, + "tokens_seen": 2468216832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012736208625877633, + "loss": 1.2787, + "theoretical_loss": 3.371953034756501, + "tokens_seen": 2468347904 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001273420260782347, + "loss": 1.2893, + "theoretical_loss": 3.371938716249191, + "tokens_seen": 2468478976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012732196589769308, + "loss": 1.5047, + "theoretical_loss": 3.3719243987150174, + "tokens_seen": 2468610048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012730190571715144, + "loss": 1.2672, + "theoretical_loss": 3.371910082153861, + "tokens_seen": 2468741120 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001272818455366098, + "loss": 1.2276, + "theoretical_loss": 3.3718957665656046, + "tokens_seen": 2468872192 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001272617853560682, + "loss": 1.2122, + "theoretical_loss": 3.3718814519501312, + "tokens_seen": 2469003264 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.3228861093521118, + "objective/train/avg_token_score": 0.008408852852880955, + "objective/train/docs_used": 1389376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0473883152008057, + "objective/train/original_loss": 3.887521505355835, + "objective/train/theoretical_loss": 3.371874295007151, + "objective/train/tokens_used": 2469068800, + "objective/train/unlikelihood_loss": 0.24553132057189941, + "theoretical_loss": 3.371874295007151, + "tokens_seen": 2469068800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012724172517552659, + "loss": 1.3302, + "theoretical_loss": 3.371867138307322, + "tokens_seen": 2469134336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012722166499498497, + "loss": 1.3805, + "theoretical_loss": 3.3718528256370606, + "tokens_seen": 2469265408 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012720160481444334, + "loss": 1.3785, + "theoretical_loss": 3.371838513939228, + "tokens_seen": 2469396480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012718154463390173, + "loss": 1.2993, + "theoretical_loss": 3.371824203213707, + "tokens_seen": 2469527552 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001271614844533601, + "loss": 1.2564, + "theoretical_loss": 3.3718098934603797, + "tokens_seen": 2469658624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012714142427281848, + "loss": 1.329, + "theoretical_loss": 3.371795584679129, + "tokens_seen": 2469789696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012712136409227684, + "loss": 1.1409, + "theoretical_loss": 3.371781276869837, + "tokens_seen": 2469920768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012710130391173523, + "loss": 1.3397, + "theoretical_loss": 3.3717669700323865, + "tokens_seen": 2470051840 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001270812437311936, + "loss": 1.2126, + "theoretical_loss": 3.3717526641666593, + "tokens_seen": 2470182912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012706118355065195, + "loss": 1.194, + "theoretical_loss": 3.3717383592725376, + "tokens_seen": 2470313984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012704112337011034, + "loss": 1.1276, + "theoretical_loss": 3.371724055349905, + "tokens_seen": 2470445056 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001270210631895687, + "loss": 1.3298, + "theoretical_loss": 3.371709752398643, + "tokens_seen": 2470576128 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.4007820188999176, + "objective/train/avg_token_score": 0.01472628116607666, + "objective/train/docs_used": 1390574, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.342423677444458, + "objective/train/original_loss": 3.723189115524292, + "objective/train/theoretical_loss": 3.3716954504186347, + "objective/train/tokens_used": 2470707200, + "objective/train/unlikelihood_loss": 0.18125538527965546, + "theoretical_loss": 3.3716954504186347, + "tokens_seen": 2470707200 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001270010030090271, + "loss": 1.3292, + "theoretical_loss": 3.3716954504186347, + "tokens_seen": 2470707200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012698094282848546, + "loss": 1.528, + "theoretical_loss": 3.371681149409762, + "tokens_seen": 2470838272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012696088264794385, + "loss": 1.3151, + "theoretical_loss": 3.371666849371908, + "tokens_seen": 2470969344 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001269408224674022, + "loss": 1.4479, + "theoretical_loss": 3.371652550304955, + "tokens_seen": 2471100416 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001269207622868606, + "loss": 1.4344, + "theoretical_loss": 3.3716382522087858, + "tokens_seen": 2471231488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012690070210631896, + "loss": 1.3582, + "theoretical_loss": 3.3716239550832827, + "tokens_seen": 2471362560 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012688064192577732, + "loss": 1.3057, + "theoretical_loss": 3.3716096589283286, + "tokens_seen": 2471493632 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001268605817452357, + "loss": 1.299, + "theoretical_loss": 3.371595363743806, + "tokens_seen": 2471624704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012684052156469407, + "loss": 1.3585, + "theoretical_loss": 3.3715810695295976, + "tokens_seen": 2471755776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012682046138415246, + "loss": 1.3616, + "theoretical_loss": 3.371566776285586, + "tokens_seen": 2471886848 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012680040120361083, + "loss": 1.407, + "theoretical_loss": 3.371552484011654, + "tokens_seen": 2472017920 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012678034102306922, + "loss": 1.3715, + "theoretical_loss": 3.3715381927076846, + "tokens_seen": 2472148992 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012676028084252758, + "loss": 1.4952, + "theoretical_loss": 3.3715239023735606, + "tokens_seen": 2472280064 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.34408602118492126, + "objective/train/avg_token_score": 0.01500159502029419, + "objective/train/docs_used": 1391138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0186907052993774, + "objective/train/original_loss": 4.163264274597168, + "objective/train/theoretical_loss": 3.3715167575701535, + "objective/train/tokens_used": 2472345600, + "objective/train/unlikelihood_loss": 0.17216773331165314, + "theoretical_loss": 3.3715167575701535, + "tokens_seen": 2472345600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012674022066198597, + "loss": 1.1566, + "theoretical_loss": 3.371509613009164, + "tokens_seen": 2472411136 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012672016048144433, + "loss": 1.329, + "theoretical_loss": 3.371495324614378, + "tokens_seen": 2472542208 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001267001003009027, + "loss": 1.2488, + "theoretical_loss": 3.3714810371890858, + "tokens_seen": 2472673280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012668004012036108, + "loss": 1.2658, + "theoretical_loss": 3.3714667507331697, + "tokens_seen": 2472804352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012665997993981944, + "loss": 1.273, + "theoretical_loss": 3.3714524652465125, + "tokens_seen": 2472935424 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012663991975927783, + "loss": 1.2609, + "theoretical_loss": 3.3714381807289975, + "tokens_seen": 2473066496 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001266198595787362, + "loss": 1.3423, + "theoretical_loss": 3.3714238971805077, + "tokens_seen": 2473197568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012659979939819458, + "loss": 1.4718, + "theoretical_loss": 3.3714096146009256, + "tokens_seen": 2473328640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012657973921765295, + "loss": 1.2306, + "theoretical_loss": 3.3713953329901343, + "tokens_seen": 2473459712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012655967903711134, + "loss": 1.2083, + "theoretical_loss": 3.3713810523480165, + "tokens_seen": 2473590784 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001265396188565697, + "loss": 1.2379, + "theoretical_loss": 3.3713667726744556, + "tokens_seen": 2473721856 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001265195586760281, + "loss": 1.3626, + "theoretical_loss": 3.3713524939693347, + "tokens_seen": 2473852928 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.4122678339481354, + "objective/train/avg_token_score": 0.007274851202964783, + "objective/train/docs_used": 1392460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2815887928009033, + "objective/train/original_loss": 3.558070182800293, + "objective/train/theoretical_loss": 3.371338216232536, + "objective/train/tokens_used": 2473984000, + "objective/train/unlikelihood_loss": 0.24350346624851227, + "theoretical_loss": 3.371338216232536, + "tokens_seen": 2473984000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012649949849548648, + "loss": 1.3213, + "theoretical_loss": 3.371338216232536, + "tokens_seen": 2473984000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012647943831494484, + "loss": 1.3291, + "theoretical_loss": 3.371323939463943, + "tokens_seen": 2474115072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012645937813440323, + "loss": 1.3915, + "theoretical_loss": 3.371309663663439, + "tokens_seen": 2474246144 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001264393179538616, + "loss": 1.3356, + "theoretical_loss": 3.371295388830907, + "tokens_seen": 2474377216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012641925777331998, + "loss": 1.1965, + "theoretical_loss": 3.3712811149662296, + "tokens_seen": 2474508288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012639919759277834, + "loss": 1.3133, + "theoretical_loss": 3.371266842069291, + "tokens_seen": 2474639360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012637913741223673, + "loss": 1.3674, + "theoretical_loss": 3.371252570139973, + "tokens_seen": 2474770432 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001263590772316951, + "loss": 1.4988, + "theoretical_loss": 3.3712382991781595, + "tokens_seen": 2474901504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012633901705115348, + "loss": 1.2402, + "theoretical_loss": 3.3712240291837334, + "tokens_seen": 2475032576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012631895687061185, + "loss": 1.3523, + "theoretical_loss": 3.3712097601565785, + "tokens_seen": 2475163648 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001262988966900702, + "loss": 1.3451, + "theoretical_loss": 3.3711954920965774, + "tokens_seen": 2475294720 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001262788365095286, + "loss": 1.4078, + "theoretical_loss": 3.3711812250036135, + "tokens_seen": 2475425792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012625877632898696, + "loss": 1.1534, + "theoretical_loss": 3.37116695887757, + "tokens_seen": 2475556864 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.5370234847068787, + "objective/train/avg_token_score": 0.007418331690132618, + "objective/train/docs_used": 1393060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7353415489196777, + "objective/train/original_loss": 3.4139907360076904, + "objective/train/theoretical_loss": 3.371159826177107, + "objective/train/tokens_used": 2475622400, + "objective/train/unlikelihood_loss": 0.13810770213603973, + "theoretical_loss": 3.371159826177107, + "tokens_seen": 2475622400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012623871614844535, + "loss": 1.4043, + "theoretical_loss": 3.3711526937183303, + "tokens_seen": 2475687936 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001262186559679037, + "loss": 1.359, + "theoretical_loss": 3.371138429525778, + "tokens_seen": 2475819008 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001261985957873621, + "loss": 1.4313, + "theoretical_loss": 3.3711241662997957, + "tokens_seen": 2475950080 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012617853560682046, + "loss": 1.3381, + "theoretical_loss": 3.3711099040402672, + "tokens_seen": 2476081152 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012615847542627885, + "loss": 1.3491, + "theoretical_loss": 3.3710956427470755, + "tokens_seen": 2476212224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012613841524573721, + "loss": 1.4008, + "theoretical_loss": 3.3710813824201047, + "tokens_seen": 2476343296 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012611835506519558, + "loss": 1.4266, + "theoretical_loss": 3.3710671230592375, + "tokens_seen": 2476474368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012609829488465397, + "loss": 1.3662, + "theoretical_loss": 3.3710528646643576, + "tokens_seen": 2476605440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012607823470411233, + "loss": 1.3852, + "theoretical_loss": 3.3710386072353486, + "tokens_seen": 2476736512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012605817452357072, + "loss": 1.3584, + "theoretical_loss": 3.371024350772094, + "tokens_seen": 2476867584 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012603811434302908, + "loss": 1.3827, + "theoretical_loss": 3.3710100952744764, + "tokens_seen": 2476998656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012601805416248747, + "loss": 1.4267, + "theoretical_loss": 3.3709958407423803, + "tokens_seen": 2477129728 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.4154447615146637, + "objective/train/avg_token_score": 0.01760588400065899, + "objective/train/docs_used": 1394031, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2257088422775269, + "objective/train/original_loss": 3.882392406463623, + "objective/train/theoretical_loss": 3.370981587175689, + "objective/train/tokens_used": 2477260800, + "objective/train/unlikelihood_loss": 0.1426595151424408, + "theoretical_loss": 3.370981587175689, + "tokens_seen": 2477260800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012599799398194583, + "loss": 1.2255, + "theoretical_loss": 3.370981587175689, + "tokens_seen": 2477260800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012597793380140422, + "loss": 1.2025, + "theoretical_loss": 3.370967334574286, + "tokens_seen": 2477391872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012595787362086258, + "loss": 1.3186, + "theoretical_loss": 3.3709530829380543, + "tokens_seen": 2477522944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012593781344032097, + "loss": 1.5086, + "theoretical_loss": 3.3709388322668783, + "tokens_seen": 2477654016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012591775325977933, + "loss": 1.2815, + "theoretical_loss": 3.3709245825606415, + "tokens_seen": 2477785088 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001258976930792377, + "loss": 1.4119, + "theoretical_loss": 3.370910333819227, + "tokens_seen": 2477916160 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012587763289869609, + "loss": 1.3392, + "theoretical_loss": 3.370896086042519, + "tokens_seen": 2478047232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012585757271815445, + "loss": 1.3791, + "theoretical_loss": 3.3708818392304005, + "tokens_seen": 2478178304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012583751253761284, + "loss": 1.3409, + "theoretical_loss": 3.370867593382756, + "tokens_seen": 2478309376 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001258174523570712, + "loss": 1.2913, + "theoretical_loss": 3.3708533484994687, + "tokens_seen": 2478440448 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001257973921765296, + "loss": 1.2822, + "theoretical_loss": 3.3708391045804222, + "tokens_seen": 2478571520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012577733199598795, + "loss": 1.2454, + "theoretical_loss": 3.3708248616255005, + "tokens_seen": 2478702592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012575727181544634, + "loss": 1.4405, + "theoretical_loss": 3.3708106196345877, + "tokens_seen": 2478833664 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.34439149498939514, + "objective/train/avg_token_score": 0.02218431606888771, + "objective/train/docs_used": 1394307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0432597398757935, + "objective/train/original_loss": 3.831913709640503, + "objective/train/theoretical_loss": 3.370803499000598, + "objective/train/tokens_used": 2478899200, + "objective/train/unlikelihood_loss": 0.25531435012817383, + "theoretical_loss": 3.370803499000598, + "tokens_seen": 2478899200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012573721163490473, + "loss": 1.4973, + "theoretical_loss": 3.370796378607567, + "tokens_seen": 2478964736 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001257171514543631, + "loss": 1.3067, + "theoretical_loss": 3.3707821385443224, + "tokens_seen": 2479095808 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012569709127382148, + "loss": 1.346, + "theoretical_loss": 3.3707678994447376, + "tokens_seen": 2479226880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012567703109327984, + "loss": 1.4615, + "theoretical_loss": 3.370753661308697, + "tokens_seen": 2479357952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012565697091273823, + "loss": 1.485, + "theoretical_loss": 3.3707394241360835, + "tokens_seen": 2479489024 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001256369107321966, + "loss": 1.4044, + "theoretical_loss": 3.370725187926782, + "tokens_seen": 2479620096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012561685055165498, + "loss": 1.3155, + "theoretical_loss": 3.3707109526806756, + "tokens_seen": 2479751168 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012559679037111335, + "loss": 1.3149, + "theoretical_loss": 3.3706967183976486, + "tokens_seen": 2479882240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012557673019057174, + "loss": 1.3274, + "theoretical_loss": 3.370682485077585, + "tokens_seen": 2480013312 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001255566700100301, + "loss": 1.5245, + "theoretical_loss": 3.3706682527203693, + "tokens_seen": 2480144384 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001255366098294885, + "loss": 1.495, + "theoretical_loss": 3.370654021325884, + "tokens_seen": 2480275456 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012551654964894685, + "loss": 1.3791, + "theoretical_loss": 3.3706397908940144, + "tokens_seen": 2480406528 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.37237292528152466, + "objective/train/avg_token_score": 0.016657637432217598, + "objective/train/docs_used": 1394994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.144840121269226, + "objective/train/original_loss": 3.9098727703094482, + "objective/train/theoretical_loss": 3.370625561424644, + "objective/train/tokens_used": 2480537600, + "objective/train/unlikelihood_loss": 0.11969485133886337, + "theoretical_loss": 3.370625561424644, + "tokens_seen": 2480537600 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001254964894684052, + "loss": 1.2515, + "theoretical_loss": 3.370625561424644, + "tokens_seen": 2480537600 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001254764292878636, + "loss": 1.3587, + "theoretical_loss": 3.3706113329176572, + "tokens_seen": 2480668672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012545636910732196, + "loss": 1.1207, + "theoretical_loss": 3.3705971053729376, + "tokens_seen": 2480799744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012543630892678035, + "loss": 1.4183, + "theoretical_loss": 3.370582878790369, + "tokens_seen": 2480930816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012541624874623872, + "loss": 1.2449, + "theoretical_loss": 3.370568653169837, + "tokens_seen": 2481061888 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001253961885656971, + "loss": 1.2597, + "theoretical_loss": 3.370554428511224, + "tokens_seen": 2481192960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012537612838515547, + "loss": 1.3296, + "theoretical_loss": 3.3705402048144153, + "tokens_seen": 2481324032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012535606820461386, + "loss": 1.4254, + "theoretical_loss": 3.370525982079294, + "tokens_seen": 2481455104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012533600802407222, + "loss": 1.3029, + "theoretical_loss": 3.3705117603057455, + "tokens_seen": 2481586176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012531594784353058, + "loss": 1.3671, + "theoretical_loss": 3.3704975394936536, + "tokens_seen": 2481717248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012529588766298897, + "loss": 1.3115, + "theoretical_loss": 3.3704833196429016, + "tokens_seen": 2481848320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012527582748244733, + "loss": 1.1641, + "theoretical_loss": 3.3704691007533754, + "tokens_seen": 2481979392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012525576730190572, + "loss": 1.3319, + "theoretical_loss": 3.370454882824958, + "tokens_seen": 2482110464 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.36223119497299194, + "objective/train/avg_token_score": 0.01683439128100872, + "objective/train/docs_used": 1396325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2178186178207397, + "objective/train/original_loss": 4.549417972564697, + "objective/train/theoretical_loss": 3.370447774221129, + "objective/train/tokens_used": 2482176000, + "objective/train/unlikelihood_loss": 0.1356949359178543, + "theoretical_loss": 3.370447774221129, + "tokens_seen": 2482176000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012523570712136408, + "loss": 1.2289, + "theoretical_loss": 3.370440665857534, + "tokens_seen": 2482241536 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012521564694082247, + "loss": 1.3607, + "theoretical_loss": 3.3704264498509877, + "tokens_seen": 2482372608 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012519558676028084, + "loss": 1.303, + "theoretical_loss": 3.3704122348052037, + "tokens_seen": 2482503680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012517552657973922, + "loss": 1.3592, + "theoretical_loss": 3.3703980207200663, + "tokens_seen": 2482634752 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001251554663991976, + "loss": 1.3183, + "theoretical_loss": 3.3703838075954593, + "tokens_seen": 2482765824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012513540621865595, + "loss": 1.5035, + "theoretical_loss": 3.3703695954312676, + "tokens_seen": 2482896896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012511534603811434, + "loss": 1.4133, + "theoretical_loss": 3.3703553842273757, + "tokens_seen": 2483027968 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001250952858575727, + "loss": 1.2805, + "theoretical_loss": 3.3703411739836677, + "tokens_seen": 2483159040 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001250752256770311, + "loss": 1.305, + "theoretical_loss": 3.3703269647000282, + "tokens_seen": 2483290112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012505516549648945, + "loss": 1.2206, + "theoretical_loss": 3.3703127563763418, + "tokens_seen": 2483421184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012503510531594784, + "loss": 1.4225, + "theoretical_loss": 3.370298549012493, + "tokens_seen": 2483552256 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001250150451354062, + "loss": 1.3435, + "theoretical_loss": 3.370284342608366, + "tokens_seen": 2483683328 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.5081866979598999, + "objective/train/avg_token_score": 0.02093549817800522, + "objective/train/docs_used": 1397049, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5106844902038574, + "objective/train/original_loss": 3.5384230613708496, + "objective/train/theoretical_loss": 3.3702701371638453, + "objective/train/tokens_used": 2483814400, + "objective/train/unlikelihood_loss": 0.14648538827896118, + "theoretical_loss": 3.3702701371638453, + "tokens_seen": 2483814400 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001249949849548646, + "loss": 1.4033, + "theoretical_loss": 3.3702701371638453, + "tokens_seen": 2483814400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012497492477432298, + "loss": 1.2166, + "theoretical_loss": 3.3702559326788157, + "tokens_seen": 2483945472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012495486459378134, + "loss": 1.3581, + "theoretical_loss": 3.3702417291531614, + "tokens_seen": 2484076544 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001249348044132397, + "loss": 1.3606, + "theoretical_loss": 3.370227526586768, + "tokens_seen": 2484207616 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001249147442326981, + "loss": 1.3769, + "theoretical_loss": 3.3702133249795185, + "tokens_seen": 2484338688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012489468405215646, + "loss": 1.3386, + "theoretical_loss": 3.370199124331299, + "tokens_seen": 2484469760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012487462387161485, + "loss": 1.3414, + "theoretical_loss": 3.3701849246419937, + "tokens_seen": 2484600832 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001248545636910732, + "loss": 1.2874, + "theoretical_loss": 3.370170725911487, + "tokens_seen": 2484731904 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001248345035105316, + "loss": 1.4086, + "theoretical_loss": 3.3701565281396633, + "tokens_seen": 2484862976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012481444332998996, + "loss": 1.3288, + "theoretical_loss": 3.370142331326408, + "tokens_seen": 2484994048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012479438314944835, + "loss": 1.3229, + "theoretical_loss": 3.3701281354716057, + "tokens_seen": 2485125120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012477432296890674, + "loss": 1.4162, + "theoretical_loss": 3.370113940575141, + "tokens_seen": 2485256192 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001247542627883651, + "loss": 1.0533, + "theoretical_loss": 3.3700997466368983, + "tokens_seen": 2485387264 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.48130497336387634, + "objective/train/avg_token_score": 0.014580986462533474, + "objective/train/docs_used": 1398115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4551233053207397, + "objective/train/original_loss": 3.6515283584594727, + "objective/train/theoretical_loss": 3.3700926500270745, + "objective/train/tokens_used": 2485452800, + "objective/train/unlikelihood_loss": 0.13650338351726532, + "theoretical_loss": 3.3700926500270745, + "tokens_seen": 2485452800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012473420260782346, + "loss": 1.2965, + "theoretical_loss": 3.3700855536567627, + "tokens_seen": 2485518336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012471414242728185, + "loss": 1.4081, + "theoretical_loss": 3.3700713616346194, + "tokens_seen": 2485649408 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012469408224674022, + "loss": 1.3659, + "theoretical_loss": 3.370057170570353, + "tokens_seen": 2485780480 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001246740220661986, + "loss": 1.3678, + "theoretical_loss": 3.370042980463848, + "tokens_seen": 2485911552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012465396188565697, + "loss": 1.3584, + "theoretical_loss": 3.3700287913149896, + "tokens_seen": 2486042624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012463390170511536, + "loss": 1.2626, + "theoretical_loss": 3.3700146031236624, + "tokens_seen": 2486173696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012461384152457372, + "loss": 1.4119, + "theoretical_loss": 3.3700004158897516, + "tokens_seen": 2486304768 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001245937813440321, + "loss": 1.2848, + "theoretical_loss": 3.3699862296131418, + "tokens_seen": 2486435840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012457372116349047, + "loss": 1.318, + "theoretical_loss": 3.3699720442937187, + "tokens_seen": 2486566912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012455366098294883, + "loss": 1.3423, + "theoretical_loss": 3.3699578599313664, + "tokens_seen": 2486697984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012453360080240722, + "loss": 1.3206, + "theoretical_loss": 3.36994367652597, + "tokens_seen": 2486829056 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012451354062186559, + "loss": 1.3932, + "theoretical_loss": 3.3699294940774154, + "tokens_seen": 2486960128 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.41654446721076965, + "objective/train/avg_token_score": 0.022394584491848946, + "objective/train/docs_used": 1398737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3324507474899292, + "objective/train/original_loss": 3.748929738998413, + "objective/train/theoretical_loss": 3.369915312585586, + "objective/train/tokens_used": 2487091200, + "objective/train/unlikelihood_loss": 0.20001628994941711, + "theoretical_loss": 3.369915312585586, + "tokens_seen": 2487091200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012449348044132397, + "loss": 1.3609, + "theoretical_loss": 3.369915312585586, + "tokens_seen": 2487091200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012447342026078234, + "loss": 1.482, + "theoretical_loss": 3.3699011320503685, + "tokens_seen": 2487222272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012445336008024073, + "loss": 1.2417, + "theoretical_loss": 3.369886952471647, + "tokens_seen": 2487353344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012443329989969912, + "loss": 1.3398, + "theoretical_loss": 3.369872773849307, + "tokens_seen": 2487484416 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012441323971915748, + "loss": 1.3166, + "theoretical_loss": 3.3698585961832332, + "tokens_seen": 2487615488 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012439317953861587, + "loss": 1.3139, + "theoretical_loss": 3.369844419473311, + "tokens_seen": 2487746560 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012437311935807423, + "loss": 1.3353, + "theoretical_loss": 3.3698302437194254, + "tokens_seen": 2487877632 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001243530591775326, + "loss": 1.3506, + "theoretical_loss": 3.369816068921462, + "tokens_seen": 2488008704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012433299899699098, + "loss": 1.3332, + "theoretical_loss": 3.3698018950793056, + "tokens_seen": 2488139776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012431293881644934, + "loss": 1.4976, + "theoretical_loss": 3.369787722192841, + "tokens_seen": 2488270848 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012429287863590773, + "loss": 1.3935, + "theoretical_loss": 3.3697735502619546, + "tokens_seen": 2488401920 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001242728184553661, + "loss": 1.3178, + "theoretical_loss": 3.369759379286531, + "tokens_seen": 2488532992 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012425275827482448, + "loss": 1.2682, + "theoretical_loss": 3.369745209266455, + "tokens_seen": 2488664064 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.5237658619880676, + "objective/train/avg_token_score": 0.006544805597513914, + "objective/train/docs_used": 1399945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.565650463104248, + "objective/train/original_loss": 3.455615758895874, + "objective/train/theoretical_loss": 3.3697381246146363, + "objective/train/tokens_used": 2488729600, + "objective/train/unlikelihood_loss": 0.161449134349823, + "theoretical_loss": 3.3697381246146363, + "tokens_seen": 2488729600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012423269809428285, + "loss": 1.1601, + "theoretical_loss": 3.369731040201612, + "tokens_seen": 2488795136 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012421263791374124, + "loss": 1.3432, + "theoretical_loss": 3.369716872091888, + "tokens_seen": 2488926208 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241925777331996, + "loss": 1.4711, + "theoretical_loss": 3.3697027049371675, + "tokens_seen": 2489057280 + }, + { + "epoch": 0.75, + "learning_rate": 0.000124172517552658, + "loss": 1.3372, + "theoretical_loss": 3.369688538737337, + "tokens_seen": 2489188352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012415245737211635, + "loss": 1.2472, + "theoretical_loss": 3.3696743734922805, + "tokens_seen": 2489319424 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241323971915747, + "loss": 1.2942, + "theoretical_loss": 3.369660209201884, + "tokens_seen": 2489450496 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241123370110331, + "loss": 1.2608, + "theoretical_loss": 3.369646045866033, + "tokens_seen": 2489581568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012409227683049146, + "loss": 1.2933, + "theoretical_loss": 3.369631883484613, + "tokens_seen": 2489712640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012407221664994985, + "loss": 1.2205, + "theoretical_loss": 3.3696177220575096, + "tokens_seen": 2489843712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012405215646940824, + "loss": 1.2829, + "theoretical_loss": 3.369603561584607, + "tokens_seen": 2489974784 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001240320962888666, + "loss": 1.2497, + "theoretical_loss": 3.3695894020657926, + "tokens_seen": 2490105856 + }, + { + "epoch": 0.75, + "learning_rate": 0.000124012036108325, + "loss": 1.279, + "theoretical_loss": 3.3695752435009503, + "tokens_seen": 2490236928 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.4575391113758087, + "objective/train/avg_token_score": 0.019898781552910805, + "objective/train/docs_used": 1400696, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.312430739402771, + "objective/train/original_loss": 3.5190389156341553, + "objective/train/theoretical_loss": 3.3695610858899663, + "objective/train/tokens_used": 2490368000, + "objective/train/unlikelihood_loss": 0.1576598882675171, + "theoretical_loss": 3.3695610858899663, + "tokens_seen": 2490368000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012399197592778336, + "loss": 1.4521, + "theoretical_loss": 3.3695610858899663, + "tokens_seen": 2490368000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012397191574724172, + "loss": 1.4334, + "theoretical_loss": 3.369546929232726, + "tokens_seen": 2490499072 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001239518555667001, + "loss": 1.3938, + "theoretical_loss": 3.369532773529115, + "tokens_seen": 2490630144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012393179538615847, + "loss": 1.3909, + "theoretical_loss": 3.3695186187790194, + "tokens_seen": 2490761216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012391173520561686, + "loss": 1.2324, + "theoretical_loss": 3.369504464982324, + "tokens_seen": 2490892288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012389167502507522, + "loss": 1.493, + "theoretical_loss": 3.3694903121389146, + "tokens_seen": 2491023360 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001238716148445336, + "loss": 1.3041, + "theoretical_loss": 3.369476160248677, + "tokens_seen": 2491154432 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012385155466399197, + "loss": 1.3052, + "theoretical_loss": 3.369462009311497, + "tokens_seen": 2491285504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012383149448345036, + "loss": 1.4479, + "theoretical_loss": 3.36944785932726, + "tokens_seen": 2491416576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012381143430290872, + "loss": 1.3447, + "theoretical_loss": 3.369433710295852, + "tokens_seen": 2491547648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012379137412236711, + "loss": 1.3334, + "theoretical_loss": 3.3694195622171583, + "tokens_seen": 2491678720 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012377131394182548, + "loss": 1.2146, + "theoretical_loss": 3.369405415091065, + "tokens_seen": 2491809792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012375125376128384, + "loss": 1.3326, + "theoretical_loss": 3.3693912689174574, + "tokens_seen": 2491940864 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.43762218952178955, + "objective/train/avg_token_score": 0.01231518667191267, + "objective/train/docs_used": 1402287, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3051884174346924, + "objective/train/original_loss": 3.733487606048584, + "objective/train/theoretical_loss": 3.3693841961878004, + "objective/train/tokens_used": 2492006400, + "objective/train/unlikelihood_loss": 0.18112695217132568, + "theoretical_loss": 3.3693841961878004, + "tokens_seen": 2492006400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012373119358074223, + "loss": 1.2401, + "theoretical_loss": 3.3693771236962218, + "tokens_seen": 2492071936 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001237111334002006, + "loss": 1.1883, + "theoretical_loss": 3.369362979427244, + "tokens_seen": 2492203008 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012369107321965898, + "loss": 1.324, + "theoretical_loss": 3.3693488361104094, + "tokens_seen": 2492334080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012367101303911737, + "loss": 1.238, + "theoretical_loss": 3.3693346937456043, + "tokens_seen": 2492465152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012365095285857573, + "loss": 1.2079, + "theoretical_loss": 3.369320552332714, + "tokens_seen": 2492596224 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012363089267803412, + "loss": 1.3178, + "theoretical_loss": 3.3693064118716247, + "tokens_seen": 2492727296 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012361083249749248, + "loss": 1.3464, + "theoretical_loss": 3.3692922723622223, + "tokens_seen": 2492858368 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012359077231695087, + "loss": 1.3809, + "theoretical_loss": 3.369278133804393, + "tokens_seen": 2492989440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012357071213640923, + "loss": 1.3394, + "theoretical_loss": 3.369263996198022, + "tokens_seen": 2493120512 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001235506519558676, + "loss": 1.3504, + "theoretical_loss": 3.3692498595429963, + "tokens_seen": 2493251584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012353059177532599, + "loss": 1.4324, + "theoretical_loss": 3.369235723839201, + "tokens_seen": 2493382656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012351053159478435, + "loss": 1.2321, + "theoretical_loss": 3.3692215890865223, + "tokens_seen": 2493513728 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.4002321660518646, + "objective/train/avg_token_score": 0.01936599612236023, + "objective/train/docs_used": 1402946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3052077293395996, + "objective/train/original_loss": 3.350410223007202, + "objective/train/theoretical_loss": 3.3692074552848457, + "objective/train/tokens_used": 2493644800, + "objective/train/unlikelihood_loss": 0.22861124575138092, + "theoretical_loss": 3.3692074552848457, + "tokens_seen": 2493644800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012349047141424274, + "loss": 1.2207, + "theoretical_loss": 3.3692074552848457, + "tokens_seen": 2493644800 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001234704112337011, + "loss": 1.2499, + "theoretical_loss": 3.3691933224340587, + "tokens_seen": 2493775872 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001234503510531595, + "loss": 1.2955, + "theoretical_loss": 3.369179190534046, + "tokens_seen": 2493906944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012343029087261785, + "loss": 1.2816, + "theoretical_loss": 3.3691650595846943, + "tokens_seen": 2494038016 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012341023069207624, + "loss": 1.1975, + "theoretical_loss": 3.3691509295858895, + "tokens_seen": 2494169088 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001233901705115346, + "loss": 1.2288, + "theoretical_loss": 3.3691368005375173, + "tokens_seen": 2494300160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012337011033099296, + "loss": 1.4744, + "theoretical_loss": 3.369122672439465, + "tokens_seen": 2494431232 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012335005015045135, + "loss": 1.0449, + "theoretical_loss": 3.3691085452916174, + "tokens_seen": 2494562304 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012332998996990972, + "loss": 1.1664, + "theoretical_loss": 3.3690944190938614, + "tokens_seen": 2494693376 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001233099297893681, + "loss": 1.2958, + "theoretical_loss": 3.3690802938460833, + "tokens_seen": 2494824448 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001232898696088265, + "loss": 1.3024, + "theoretical_loss": 3.369066169548169, + "tokens_seen": 2494955520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012326980942828486, + "loss": 1.2968, + "theoretical_loss": 3.3690520462000046, + "tokens_seen": 2495086592 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012324974924774325, + "loss": 1.4667, + "theoretical_loss": 3.369037923801477, + "tokens_seen": 2495217664 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.5110581517219543, + "objective/train/avg_token_score": 0.006341453175991774, + "objective/train/docs_used": 1404179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7831273078918457, + "objective/train/original_loss": 3.659313917160034, + "objective/train/theoretical_loss": 3.369030862958291, + "objective/train/tokens_used": 2495283200, + "objective/train/unlikelihood_loss": 0.16347527503967285, + "theoretical_loss": 3.369030862958291, + "tokens_seen": 2495283200 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001232296890672016, + "loss": 1.3092, + "theoretical_loss": 3.369023802352472, + "tokens_seen": 2495348736 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012320962888666, + "loss": 1.3301, + "theoretical_loss": 3.3690096818528756, + "tokens_seen": 2495479808 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012318956870611836, + "loss": 1.2872, + "theoretical_loss": 3.3689955623025747, + "tokens_seen": 2495610880 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012316950852557672, + "loss": 1.4031, + "theoretical_loss": 3.368981443701455, + "tokens_seen": 2495741952 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001231494483450351, + "loss": 1.3173, + "theoretical_loss": 3.3689673260494035, + "tokens_seen": 2495873024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012312938816449347, + "loss": 1.2086, + "theoretical_loss": 3.368953209346306, + "tokens_seen": 2496004096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012310932798395186, + "loss": 1.34, + "theoretical_loss": 3.36893909359205, + "tokens_seen": 2496135168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012308926780341023, + "loss": 1.3468, + "theoretical_loss": 3.36892497878652, + "tokens_seen": 2496266240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012306920762286862, + "loss": 1.3982, + "theoretical_loss": 3.368910864929604, + "tokens_seen": 2496397312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012304914744232698, + "loss": 1.4021, + "theoretical_loss": 3.368896752021188, + "tokens_seen": 2496528384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012302908726178537, + "loss": 1.1435, + "theoretical_loss": 3.368882640061158, + "tokens_seen": 2496659456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012300902708124373, + "loss": 1.3719, + "theoretical_loss": 3.368868529049401, + "tokens_seen": 2496790528 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.5071480870246887, + "objective/train/avg_token_score": 0.02931123413145542, + "objective/train/docs_used": 1405043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5585694313049316, + "objective/train/original_loss": 3.770049810409546, + "objective/train/theoretical_loss": 3.368854418985804, + "objective/train/tokens_used": 2496921600, + "objective/train/unlikelihood_loss": 0.1658545881509781, + "theoretical_loss": 3.368854418985804, + "tokens_seen": 2496921600 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001229889669007021, + "loss": 1.4325, + "theoretical_loss": 3.368854418985804, + "tokens_seen": 2496921600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012296890672016048, + "loss": 1.3206, + "theoretical_loss": 3.368840309870252, + "tokens_seen": 2497052672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012294884653961884, + "loss": 1.336, + "theoretical_loss": 3.368826201702633, + "tokens_seen": 2497183744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012292878635907723, + "loss": 1.4392, + "theoretical_loss": 3.3688120944828333, + "tokens_seen": 2497314816 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012290872617853562, + "loss": 1.2841, + "theoretical_loss": 3.368797988210739, + "tokens_seen": 2497445888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012288866599799398, + "loss": 1.3203, + "theoretical_loss": 3.3687838828862366, + "tokens_seen": 2497576960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012286860581745237, + "loss": 1.2951, + "theoretical_loss": 3.368769778509213, + "tokens_seen": 2497708032 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012284854563691074, + "loss": 1.2443, + "theoretical_loss": 3.368755675079555, + "tokens_seen": 2497839104 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012282848545636912, + "loss": 1.38, + "theoretical_loss": 3.3687415725971492, + "tokens_seen": 2497970176 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001228084252758275, + "loss": 1.1717, + "theoretical_loss": 3.368727471061882, + "tokens_seen": 2498101248 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012278836509528585, + "loss": 1.1661, + "theoretical_loss": 3.3687133704736407, + "tokens_seen": 2498232320 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012276830491474424, + "loss": 1.2672, + "theoretical_loss": 3.3686992708323116, + "tokens_seen": 2498363392 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001227482447342026, + "loss": 1.197, + "theoretical_loss": 3.368685172137781, + "tokens_seen": 2498494464 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.418621689081192, + "objective/train/avg_token_score": 0.0220972690731287, + "objective/train/docs_used": 1405716, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.211168646812439, + "objective/train/original_loss": 4.010618209838867, + "objective/train/theoretical_loss": 3.3686781231455303, + "objective/train/tokens_used": 2498560000, + "objective/train/unlikelihood_loss": 0.12707215547561646, + "theoretical_loss": 3.3686781231455303, + "tokens_seen": 2498560000 + }, + { + "epoch": 0.76, + "learning_rate": 0.000122728184553661, + "loss": 1.3162, + "theoretical_loss": 3.3686710743899364, + "tokens_seen": 2498625536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012270812437311935, + "loss": 1.3653, + "theoretical_loss": 3.3686569775886643, + "tokens_seen": 2498756608 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012268806419257774, + "loss": 1.4111, + "theoretical_loss": 3.3686428817338516, + "tokens_seen": 2498887680 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226680040120361, + "loss": 1.2641, + "theoretical_loss": 3.3686287868253846, + "tokens_seen": 2499018752 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226479438314945, + "loss": 1.4362, + "theoretical_loss": 3.368614692863151, + "tokens_seen": 2499149824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012262788365095286, + "loss": 1.3247, + "theoretical_loss": 3.3686005998470367, + "tokens_seen": 2499280896 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012260782347041122, + "loss": 1.2878, + "theoretical_loss": 3.3685865077769295, + "tokens_seen": 2499411968 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001225877632898696, + "loss": 1.3276, + "theoretical_loss": 3.368572416652716, + "tokens_seen": 2499543040 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012256770310932797, + "loss": 1.4032, + "theoretical_loss": 3.3685583264742824, + "tokens_seen": 2499674112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012254764292878636, + "loss": 1.3656, + "theoretical_loss": 3.3685442372415166, + "tokens_seen": 2499805184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012252758274824475, + "loss": 1.3099, + "theoretical_loss": 3.368530148954305, + "tokens_seen": 2499936256 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001225075225677031, + "loss": 1.3824, + "theoretical_loss": 3.3685160616125347, + "tokens_seen": 2500067328 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.40866324305534363, + "objective/train/avg_token_score": 0.032216910272836685, + "objective/train/docs_used": 1406694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2770309448242188, + "objective/train/original_loss": 3.9760046005249023, + "objective/train/theoretical_loss": 3.368501975216093, + "objective/train/tokens_used": 2500198400, + "objective/train/unlikelihood_loss": 0.19266390800476074, + "theoretical_loss": 3.368501975216093, + "tokens_seen": 2500198400 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001224874623871615, + "loss": 1.1663, + "theoretical_loss": 3.368501975216093, + "tokens_seen": 2500198400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012246740220661986, + "loss": 1.4659, + "theoretical_loss": 3.3684878897648662, + "tokens_seen": 2500329472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012244734202607825, + "loss": 1.3429, + "theoretical_loss": 3.368473805258742, + "tokens_seen": 2500460544 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001224272818455366, + "loss": 1.3464, + "theoretical_loss": 3.3684597216976075, + "tokens_seen": 2500591616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012240722166499498, + "loss": 1.4709, + "theoretical_loss": 3.368445639081349, + "tokens_seen": 2500722688 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012238716148445336, + "loss": 1.2114, + "theoretical_loss": 3.3684315574098544, + "tokens_seen": 2500853760 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012236710130391173, + "loss": 1.2982, + "theoretical_loss": 3.3684174766830104, + "tokens_seen": 2500984832 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012234704112337012, + "loss": 1.2968, + "theoretical_loss": 3.368403396900704, + "tokens_seen": 2501115904 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012232698094282848, + "loss": 1.2868, + "theoretical_loss": 3.368389318062823, + "tokens_seen": 2501246976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012230692076228687, + "loss": 1.4209, + "theoretical_loss": 3.3683752401692537, + "tokens_seen": 2501378048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012228686058174523, + "loss": 1.2954, + "theoretical_loss": 3.368361163219884, + "tokens_seen": 2501509120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012226680040120362, + "loss": 1.3747, + "theoretical_loss": 3.368347087214601, + "tokens_seen": 2501640192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012224674022066198, + "loss": 1.0974, + "theoretical_loss": 3.368333012153291, + "tokens_seen": 2501771264 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.49169111251831055, + "objective/train/avg_token_score": 0.021189110353589058, + "objective/train/docs_used": 1407476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4792695045471191, + "objective/train/original_loss": 3.889026403427124, + "objective/train/theoretical_loss": 3.368325974976591, + "objective/train/tokens_used": 2501836800, + "objective/train/unlikelihood_loss": 0.13166335225105286, + "theoretical_loss": 3.368325974976591, + "tokens_seen": 2501836800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012222668004012037, + "loss": 1.4145, + "theoretical_loss": 3.3683189380358423, + "tokens_seen": 2501902336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012220661985957873, + "loss": 1.3284, + "theoretical_loss": 3.368304864862142, + "tokens_seen": 2502033408 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001221865596790371, + "loss": 1.3194, + "theoretical_loss": 3.3682907926320773, + "tokens_seen": 2502164480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012216649949849549, + "loss": 1.425, + "theoretical_loss": 3.368276721345535, + "tokens_seen": 2502295552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012214643931795387, + "loss": 1.2903, + "theoretical_loss": 3.368262651002403, + "tokens_seen": 2502426624 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012212637913741224, + "loss": 1.2684, + "theoretical_loss": 3.3682485816025687, + "tokens_seen": 2502557696 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012210631895687063, + "loss": 1.3193, + "theoretical_loss": 3.3682345131459193, + "tokens_seen": 2502688768 + }, + { + "epoch": 0.76, + "learning_rate": 0.000122086258776329, + "loss": 1.1729, + "theoretical_loss": 3.3682204456323417, + "tokens_seen": 2502819840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012206619859578736, + "loss": 1.2458, + "theoretical_loss": 3.368206379061724, + "tokens_seen": 2502950912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012204613841524574, + "loss": 1.3626, + "theoretical_loss": 3.3681923134339535, + "tokens_seen": 2503081984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012202607823470412, + "loss": 1.3816, + "theoretical_loss": 3.368178248748917, + "tokens_seen": 2503213056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012200601805416249, + "loss": 1.3242, + "theoretical_loss": 3.3681641850065027, + "tokens_seen": 2503344128 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.5177785754203796, + "objective/train/avg_token_score": 0.00706838583573699, + "objective/train/docs_used": 1408714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6336101293563843, + "objective/train/original_loss": 3.4894723892211914, + "objective/train/theoretical_loss": 3.3681501222065977, + "objective/train/tokens_used": 2503475200, + "objective/train/unlikelihood_loss": 0.15001827478408813, + "theoretical_loss": 3.3681501222065977, + "tokens_seen": 2503475200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012198595787362087, + "loss": 1.4657, + "theoretical_loss": 3.3681501222065977, + "tokens_seen": 2503475200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012196589769307924, + "loss": 1.3405, + "theoretical_loss": 3.36813606034909, + "tokens_seen": 2503606272 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012194583751253762, + "loss": 1.2499, + "theoretical_loss": 3.3681219994338663, + "tokens_seen": 2503737344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012192577733199598, + "loss": 1.1952, + "theoretical_loss": 3.3681079394608147, + "tokens_seen": 2503868416 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012190571715145436, + "loss": 1.4193, + "theoretical_loss": 3.3680938804298224, + "tokens_seen": 2503999488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012188565697091273, + "loss": 1.4495, + "theoretical_loss": 3.3680798223407775, + "tokens_seen": 2504130560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012186559679037111, + "loss": 1.2888, + "theoretical_loss": 3.368065765193567, + "tokens_seen": 2504261632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012184553660982948, + "loss": 1.3995, + "theoretical_loss": 3.3680517089880784, + "tokens_seen": 2504392704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012182547642928786, + "loss": 1.4275, + "theoretical_loss": 3.3680376537242003, + "tokens_seen": 2504523776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012180541624874625, + "loss": 1.3674, + "theoretical_loss": 3.3680235994018197, + "tokens_seen": 2504654848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012178535606820463, + "loss": 1.379, + "theoretical_loss": 3.3680095460208244, + "tokens_seen": 2504785920 + }, + { + "epoch": 0.76, + "learning_rate": 0.000121765295887663, + "loss": 1.2602, + "theoretical_loss": 3.367995493581102, + "tokens_seen": 2504916992 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012174523570712138, + "loss": 1.2441, + "theoretical_loss": 3.36798144208254, + "tokens_seen": 2505048064 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.26900050044059753, + "objective/train/avg_token_score": 0.026772432029247284, + "objective/train/docs_used": 1409243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7958700656890869, + "objective/train/original_loss": 3.9889986515045166, + "objective/train/theoretical_loss": 3.3679744166861587, + "objective/train/tokens_used": 2505113600, + "objective/train/unlikelihood_loss": 0.2634621560573578, + "theoretical_loss": 3.3679744166861587, + "tokens_seen": 2505113600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012172517552657974, + "loss": 1.1711, + "theoretical_loss": 3.367967391525026, + "tokens_seen": 2505179136 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012170511534603811, + "loss": 1.2774, + "theoretical_loss": 3.3679533419084486, + "tokens_seen": 2505310208 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012168505516549649, + "loss": 1.3966, + "theoretical_loss": 3.367939293232695, + "tokens_seen": 2505441280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012166499498495487, + "loss": 1.4546, + "theoretical_loss": 3.367925245497653, + "tokens_seen": 2505572352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012164493480441324, + "loss": 1.2514, + "theoretical_loss": 3.36791119870321, + "tokens_seen": 2505703424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012162487462387162, + "loss": 1.2377, + "theoretical_loss": 3.3678971528492547, + "tokens_seen": 2505834496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012160481444333, + "loss": 1.2227, + "theoretical_loss": 3.3678831079356746, + "tokens_seen": 2505965568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012158475426278837, + "loss": 1.3381, + "theoretical_loss": 3.367869063962357, + "tokens_seen": 2506096640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012156469408224675, + "loss": 1.3896, + "theoretical_loss": 3.367855020929191, + "tokens_seen": 2506227712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012154463390170512, + "loss": 1.3939, + "theoretical_loss": 3.367840978836063, + "tokens_seen": 2506358784 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012152457372116348, + "loss": 1.4279, + "theoretical_loss": 3.367826937682862, + "tokens_seen": 2506489856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012150451354062186, + "loss": 1.4266, + "theoretical_loss": 3.3678128974694754, + "tokens_seen": 2506620928 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.40609726309776306, + "objective/train/avg_token_score": 0.010531141422688961, + "objective/train/docs_used": 1410261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2536122798919678, + "objective/train/original_loss": 4.0127787590026855, + "objective/train/theoretical_loss": 3.367798858195792, + "objective/train/tokens_used": 2506752000, + "objective/train/unlikelihood_loss": 0.12238708138465881, + "theoretical_loss": 3.367798858195792, + "tokens_seen": 2506752000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012148445336008024, + "loss": 1.4029, + "theoretical_loss": 3.367798858195792, + "tokens_seen": 2506752000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012146439317953861, + "loss": 1.4293, + "theoretical_loss": 3.3677848198616984, + "tokens_seen": 2506883072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012144433299899699, + "loss": 1.2584, + "theoretical_loss": 3.3677707824670837, + "tokens_seen": 2507014144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012142427281845538, + "loss": 1.2278, + "theoretical_loss": 3.3677567460118354, + "tokens_seen": 2507145216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012140421263791375, + "loss": 1.2701, + "theoretical_loss": 3.3677427104958415, + "tokens_seen": 2507276288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012138415245737213, + "loss": 1.4219, + "theoretical_loss": 3.3677286759189906, + "tokens_seen": 2507407360 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001213640922768305, + "loss": 1.219, + "theoretical_loss": 3.3677146422811703, + "tokens_seen": 2507538432 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012134403209628888, + "loss": 1.1112, + "theoretical_loss": 3.367700609582269, + "tokens_seen": 2507669504 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012132397191574724, + "loss": 1.2366, + "theoretical_loss": 3.367686577822174, + "tokens_seen": 2507800576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012130391173520562, + "loss": 1.1016, + "theoretical_loss": 3.3676725470007747, + "tokens_seen": 2507931648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012128385155466399, + "loss": 1.4495, + "theoretical_loss": 3.3676585171179583, + "tokens_seen": 2508062720 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012126379137412237, + "loss": 1.414, + "theoretical_loss": 3.367644488173613, + "tokens_seen": 2508193792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012124373119358074, + "loss": 1.4721, + "theoretical_loss": 3.3676304601676277, + "tokens_seen": 2508324864 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.3645527958869934, + "objective/train/avg_token_score": 0.016059426590800285, + "objective/train/docs_used": 1410976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1856598854064941, + "objective/train/original_loss": 3.5911905765533447, + "objective/train/theoretical_loss": 3.3676234465164847, + "objective/train/tokens_used": 2508390400, + "objective/train/unlikelihood_loss": 0.17621493339538574, + "theoretical_loss": 3.3676234465164847, + "tokens_seen": 2508390400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012122367101303912, + "loss": 1.2899, + "theoretical_loss": 3.3676164330998897, + "tokens_seen": 2508455936 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001212036108324975, + "loss": 1.4172, + "theoretical_loss": 3.367602406970288, + "tokens_seen": 2508587008 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012118355065195587, + "loss": 1.3637, + "theoretical_loss": 3.3675883817787104, + "tokens_seen": 2508718080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012116349047141425, + "loss": 1.393, + "theoretical_loss": 3.3675743575250454, + "tokens_seen": 2508849152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012114343029087261, + "loss": 1.3559, + "theoretical_loss": 3.3675603342091813, + "tokens_seen": 2508980224 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012112337011033099, + "loss": 1.3026, + "theoretical_loss": 3.367546311831006, + "tokens_seen": 2509111296 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012110330992978936, + "loss": 1.2718, + "theoretical_loss": 3.367532290390408, + "tokens_seen": 2509242368 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012108324974924774, + "loss": 1.3983, + "theoretical_loss": 3.3675182698872757, + "tokens_seen": 2509373440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012106318956870611, + "loss": 1.3759, + "theoretical_loss": 3.367504250321498, + "tokens_seen": 2509504512 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001210431293881645, + "loss": 1.2546, + "theoretical_loss": 3.3674902316929622, + "tokens_seen": 2509635584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012102306920762288, + "loss": 1.2269, + "theoretical_loss": 3.3674762140015577, + "tokens_seen": 2509766656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012100300902708125, + "loss": 1.2642, + "theoretical_loss": 3.367462197247172, + "tokens_seen": 2509897728 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.369012713432312, + "objective/train/avg_token_score": 0.014518202282488346, + "objective/train/docs_used": 1412115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0317844152450562, + "objective/train/original_loss": 3.7621877193450928, + "objective/train/theoretical_loss": 3.367448181429695, + "objective/train/tokens_used": 2510028800, + "objective/train/unlikelihood_loss": 0.15590694546699524, + "theoretical_loss": 3.367448181429695, + "tokens_seen": 2510028800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012098294884653963, + "loss": 1.2675, + "theoretical_loss": 3.367448181429695, + "tokens_seen": 2510028800 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120962888665998, + "loss": 1.2423, + "theoretical_loss": 3.367434166549013, + "tokens_seen": 2510159872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012094282848545637, + "loss": 1.354, + "theoretical_loss": 3.3674201526050163, + "tokens_seen": 2510290944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012092276830491474, + "loss": 1.1446, + "theoretical_loss": 3.3674061395975925, + "tokens_seen": 2510422016 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012090270812437312, + "loss": 1.336, + "theoretical_loss": 3.3673921275266303, + "tokens_seen": 2510553088 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001208826479438315, + "loss": 1.2964, + "theoretical_loss": 3.3673781163920187, + "tokens_seen": 2510684160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012086258776328987, + "loss": 1.4286, + "theoretical_loss": 3.3673641061936457, + "tokens_seen": 2510815232 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012084252758274825, + "loss": 1.411, + "theoretical_loss": 3.3673500969314, + "tokens_seen": 2510946304 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012082246740220662, + "loss": 1.1981, + "theoretical_loss": 3.3673360886051698, + "tokens_seen": 2511077376 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120802407221665, + "loss": 1.2811, + "theoretical_loss": 3.3673220812148443, + "tokens_seen": 2511208448 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012078234704112337, + "loss": 1.2604, + "theoretical_loss": 3.367308074760312, + "tokens_seen": 2511339520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012076228686058175, + "loss": 1.3664, + "theoretical_loss": 3.3672940692414617, + "tokens_seen": 2511470592 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012074222668004011, + "loss": 1.2749, + "theoretical_loss": 3.367280064658182, + "tokens_seen": 2511601664 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.4873533844947815, + "objective/train/avg_token_score": 0.013335516676306725, + "objective/train/docs_used": 1412775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5741328001022339, + "objective/train/original_loss": 3.815882921218872, + "objective/train/theoretical_loss": 3.367273062717346, + "objective/train/tokens_used": 2511667200, + "objective/train/unlikelihood_loss": 0.13685227930545807, + "theoretical_loss": 3.367273062717346, + "tokens_seen": 2511667200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012072216649949849, + "loss": 1.3356, + "theoretical_loss": 3.367266061010361, + "tokens_seen": 2511732736 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012070210631895686, + "loss": 1.2277, + "theoretical_loss": 3.367252058297888, + "tokens_seen": 2511863808 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012068204613841525, + "loss": 1.2684, + "theoretical_loss": 3.3672380565206517, + "tokens_seen": 2511994880 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012066198595787363, + "loss": 1.4096, + "theoretical_loss": 3.36722405567854, + "tokens_seen": 2512125952 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120641925777332, + "loss": 1.3926, + "theoretical_loss": 3.367210055771443, + "tokens_seen": 2512257024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012062186559679038, + "loss": 1.3373, + "theoretical_loss": 3.3671960567992487, + "tokens_seen": 2512388096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012060180541624876, + "loss": 1.3527, + "theoretical_loss": 3.367182058761846, + "tokens_seen": 2512519168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012058174523570713, + "loss": 1.3985, + "theoretical_loss": 3.367168061659124, + "tokens_seen": 2512650240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012056168505516551, + "loss": 1.3584, + "theoretical_loss": 3.367154065490971, + "tokens_seen": 2512781312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012054162487462387, + "loss": 1.2884, + "theoretical_loss": 3.3671400702572765, + "tokens_seen": 2512912384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012052156469408225, + "loss": 1.4536, + "theoretical_loss": 3.3671260759579287, + "tokens_seen": 2513043456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012050150451354062, + "loss": 1.4198, + "theoretical_loss": 3.3671120825928167, + "tokens_seen": 2513174528 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.32581865787506104, + "objective/train/avg_token_score": 0.010318056680262089, + "objective/train/docs_used": 1413257, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9999253749847412, + "objective/train/original_loss": 3.968043088912964, + "objective/train/theoretical_loss": 3.36709809016183, + "objective/train/tokens_used": 2513305600, + "objective/train/unlikelihood_loss": 0.16103196144104004, + "theoretical_loss": 3.36709809016183, + "tokens_seen": 2513305600 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120481444332999, + "loss": 1.255, + "theoretical_loss": 3.36709809016183, + "tokens_seen": 2513305600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012046138415245737, + "loss": 1.3316, + "theoretical_loss": 3.367084098664857, + "tokens_seen": 2513436672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012044132397191575, + "loss": 1.2165, + "theoretical_loss": 3.367070108101786, + "tokens_seen": 2513567744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012042126379137412, + "loss": 1.1433, + "theoretical_loss": 3.3670561184725076, + "tokens_seen": 2513698816 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001204012036108325, + "loss": 1.2244, + "theoretical_loss": 3.3670421297769093, + "tokens_seen": 2513829888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012038114343029088, + "loss": 1.2802, + "theoretical_loss": 3.367028142014881, + "tokens_seen": 2513960960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012036108324974924, + "loss": 1.3297, + "theoretical_loss": 3.367014155186311, + "tokens_seen": 2514092032 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012034102306920761, + "loss": 1.179, + "theoretical_loss": 3.367000169291089, + "tokens_seen": 2514223104 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012032096288866599, + "loss": 1.372, + "theoretical_loss": 3.3669861843291042, + "tokens_seen": 2514354176 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012030090270812438, + "loss": 1.2185, + "theoretical_loss": 3.366972200300245, + "tokens_seen": 2514485248 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012028084252758276, + "loss": 1.3373, + "theoretical_loss": 3.3669582172044006, + "tokens_seen": 2514616320 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012026078234704113, + "loss": 1.2368, + "theoretical_loss": 3.3669442350414602, + "tokens_seen": 2514747392 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012024072216649951, + "loss": 1.1932, + "theoretical_loss": 3.3669302538113137, + "tokens_seen": 2514878464 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.5368402004241943, + "objective/train/avg_token_score": 0.020285589620471, + "objective/train/docs_used": 1414570, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6437259912490845, + "objective/train/original_loss": 3.7912018299102783, + "objective/train/theoretical_loss": 3.366923263546003, + "objective/train/tokens_used": 2514944000, + "objective/train/unlikelihood_loss": 0.12309518456459045, + "theoretical_loss": 3.366923263546003, + "tokens_seen": 2514944000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012022066198595788, + "loss": 1.2574, + "theoretical_loss": 3.366916273513849, + "tokens_seen": 2515009536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012020060180541626, + "loss": 1.1479, + "theoretical_loss": 3.3669022941489564, + "tokens_seen": 2515140608 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012018054162487463, + "loss": 1.2052, + "theoretical_loss": 3.3668883157165244, + "tokens_seen": 2515271680 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120160481444333, + "loss": 1.4141, + "theoretical_loss": 3.366874338216442, + "tokens_seen": 2515402752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012014042126379137, + "loss": 1.3262, + "theoretical_loss": 3.3668603616485995, + "tokens_seen": 2515533824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012012036108324975, + "loss": 1.2519, + "theoretical_loss": 3.366846386012885, + "tokens_seen": 2515664896 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012010030090270812, + "loss": 1.3188, + "theoretical_loss": 3.3668324113091885, + "tokens_seen": 2515795968 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001200802407221665, + "loss": 1.3594, + "theoretical_loss": 3.366818437537399, + "tokens_seen": 2515927040 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012006018054162488, + "loss": 1.3321, + "theoretical_loss": 3.366804464697406, + "tokens_seen": 2516058112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012004012036108325, + "loss": 1.2987, + "theoretical_loss": 3.366790492789099, + "tokens_seen": 2516189184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012002006018054163, + "loss": 1.3274, + "theoretical_loss": 3.3667765218123664, + "tokens_seen": 2516320256 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012, + "loss": 1.2377, + "theoretical_loss": 3.3667625517670987, + "tokens_seen": 2516451328 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.4563172161579132, + "objective/train/avg_token_score": 0.009295858442783356, + "objective/train/docs_used": 1415224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5211631059646606, + "objective/train/original_loss": 3.5299036502838135, + "objective/train/theoretical_loss": 3.3667485826531847, + "objective/train/tokens_used": 2516582400, + "objective/train/unlikelihood_loss": 0.15351314842700958, + "theoretical_loss": 3.3667485826531847, + "tokens_seen": 2516582400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011997993981945838, + "loss": 1.2543, + "theoretical_loss": 3.3667485826531847, + "tokens_seen": 2516582400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011995987963891674, + "loss": 1.2568, + "theoretical_loss": 3.3667346144705137, + "tokens_seen": 2516713472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011993981945837512, + "loss": 1.2891, + "theoretical_loss": 3.3667206472189757, + "tokens_seen": 2516844544 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001199197592778335, + "loss": 1.2987, + "theoretical_loss": 3.3667066808984596, + "tokens_seen": 2516975616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011989969909729188, + "loss": 1.3654, + "theoretical_loss": 3.3666927155088553, + "tokens_seen": 2517106688 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011987963891675026, + "loss": 1.2967, + "theoretical_loss": 3.3666787510500518, + "tokens_seen": 2517237760 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011985957873620863, + "loss": 1.4215, + "theoretical_loss": 3.366664787521939, + "tokens_seen": 2517368832 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011983951855566701, + "loss": 1.3349, + "theoretical_loss": 3.3666508249244056, + "tokens_seen": 2517499904 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011981945837512539, + "loss": 1.3313, + "theoretical_loss": 3.3666368632573427, + "tokens_seen": 2517630976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011979939819458376, + "loss": 1.1902, + "theoretical_loss": 3.3666229025206382, + "tokens_seen": 2517762048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011977933801404214, + "loss": 1.4145, + "theoretical_loss": 3.3666089427141825, + "tokens_seen": 2517893120 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001197592778335005, + "loss": 1.3239, + "theoretical_loss": 3.3665949838378655, + "tokens_seen": 2518024192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011973921765295887, + "loss": 1.2844, + "theoretical_loss": 3.3665810258915765, + "tokens_seen": 2518155264 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.3715786933898926, + "objective/train/avg_token_score": 0.013253060169517994, + "objective/train/docs_used": 1416375, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1498441696166992, + "objective/train/original_loss": 4.45274543762207, + "objective/train/theoretical_loss": 3.3665740472671577, + "objective/train/tokens_used": 2518220800, + "objective/train/unlikelihood_loss": 0.1139402985572815, + "theoretical_loss": 3.3665740472671577, + "tokens_seen": 2518220800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011971915747241725, + "loss": 1.2577, + "theoretical_loss": 3.366567068875205, + "tokens_seen": 2518286336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011969909729187563, + "loss": 1.1993, + "theoretical_loss": 3.3665531127886403, + "tokens_seen": 2518417408 + }, + { + "epoch": 0.76, + "learning_rate": 0.000119679037111334, + "loss": 1.3113, + "theoretical_loss": 3.3665391576317725, + "tokens_seen": 2518548480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011965897693079238, + "loss": 1.2046, + "theoretical_loss": 3.3665252034044917, + "tokens_seen": 2518679552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011963891675025075, + "loss": 1.3099, + "theoretical_loss": 3.366511250106687, + "tokens_seen": 2518810624 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011961885656970913, + "loss": 1.2393, + "theoretical_loss": 3.366497297738248, + "tokens_seen": 2518941696 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001195987963891675, + "loss": 1.1937, + "theoretical_loss": 3.366483346299065, + "tokens_seen": 2519072768 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011957873620862587, + "loss": 1.1652, + "theoretical_loss": 3.3664693957890277, + "tokens_seen": 2519203840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011955867602808424, + "loss": 1.4174, + "theoretical_loss": 3.3664554462080254, + "tokens_seen": 2519334912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011953861584754263, + "loss": 1.3413, + "theoretical_loss": 3.3664414975559485, + "tokens_seen": 2519465984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011951855566700101, + "loss": 1.2436, + "theoretical_loss": 3.366427549832686, + "tokens_seen": 2519597056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011949849548645938, + "loss": 1.3223, + "theoretical_loss": 3.3664136030381284, + "tokens_seen": 2519728128 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.433956503868103, + "objective/train/avg_token_score": 0.010076232254505157, + "objective/train/docs_used": 1417108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.435458779335022, + "objective/train/original_loss": 3.9024646282196045, + "objective/train/theoretical_loss": 3.366399657172166, + "objective/train/tokens_used": 2519859200, + "objective/train/unlikelihood_loss": 0.14210116863250732, + "theoretical_loss": 3.366399657172166, + "tokens_seen": 2519859200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011947843530591776, + "loss": 1.4118, + "theoretical_loss": 3.366399657172166, + "tokens_seen": 2519859200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011945837512537614, + "loss": 1.2954, + "theoretical_loss": 3.3663857122346874, + "tokens_seen": 2519990272 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011943831494483451, + "loss": 1.2399, + "theoretical_loss": 3.3663717682255836, + "tokens_seen": 2520121344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011941825476429289, + "loss": 1.4124, + "theoretical_loss": 3.3663578251447435, + "tokens_seen": 2520252416 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011939819458375126, + "loss": 1.2522, + "theoretical_loss": 3.3663438829920582, + "tokens_seen": 2520383488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011937813440320963, + "loss": 1.1933, + "theoretical_loss": 3.366329941767417, + "tokens_seen": 2520514560 + }, + { + "epoch": 0.76, + "learning_rate": 0.000119358074222668, + "loss": 1.375, + "theoretical_loss": 3.3663160014707096, + "tokens_seen": 2520645632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011933801404212638, + "loss": 1.4402, + "theoretical_loss": 3.366302062101827, + "tokens_seen": 2520776704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011931795386158475, + "loss": 1.3507, + "theoretical_loss": 3.366288123660658, + "tokens_seen": 2520907776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011929789368104313, + "loss": 1.236, + "theoretical_loss": 3.366274186147093, + "tokens_seen": 2521038848 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001192778335005015, + "loss": 1.3749, + "theoretical_loss": 3.366260249561023, + "tokens_seen": 2521169920 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011925777331995988, + "loss": 1.1295, + "theoretical_loss": 3.3662463139023364, + "tokens_seen": 2521300992 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011923771313941826, + "loss": 1.316, + "theoretical_loss": 3.366232379170925, + "tokens_seen": 2521432064 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.3462854325771332, + "objective/train/avg_token_score": 0.007644950412213802, + "objective/train/docs_used": 1417967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1187459230422974, + "objective/train/original_loss": 3.4458887577056885, + "objective/train/theoretical_loss": 3.3662254121529123, + "objective/train/tokens_used": 2521497600, + "objective/train/unlikelihood_loss": 0.2601977586746216, + "theoretical_loss": 3.3662254121529123, + "tokens_seen": 2521497600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011921765295887663, + "loss": 1.2999, + "theoretical_loss": 3.3662184453666777, + "tokens_seen": 2521563136 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011919759277833501, + "loss": 1.1128, + "theoretical_loss": 3.3662045124894844, + "tokens_seen": 2521694208 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011917753259779338, + "loss": 1.1524, + "theoretical_loss": 3.3661905805392367, + "tokens_seen": 2521825280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011915747241725176, + "loss": 1.2966, + "theoretical_loss": 3.3661766495158236, + "tokens_seen": 2521956352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011913741223671014, + "loss": 1.1496, + "theoretical_loss": 3.3661627194191355, + "tokens_seen": 2522087424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011911735205616851, + "loss": 1.0357, + "theoretical_loss": 3.3661487902490626, + "tokens_seen": 2522218496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011909729187562689, + "loss": 1.2476, + "theoretical_loss": 3.366134862005495, + "tokens_seen": 2522349568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011907723169508526, + "loss": 1.4056, + "theoretical_loss": 3.366120934688323, + "tokens_seen": 2522480640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011905717151454364, + "loss": 1.2437, + "theoretical_loss": 3.366107008297438, + "tokens_seen": 2522611712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011903711133400201, + "loss": 1.2165, + "theoretical_loss": 3.366093082832728, + "tokens_seen": 2522742784 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011901705115346039, + "loss": 1.3092, + "theoretical_loss": 3.3660791582940854, + "tokens_seen": 2522873856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011899699097291877, + "loss": 1.3109, + "theoretical_loss": 3.366065234681399, + "tokens_seen": 2523004928 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.4617546498775482, + "objective/train/avg_token_score": 0.022248832508921623, + "objective/train/docs_used": 1418616, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.327524185180664, + "objective/train/original_loss": 3.4304778575897217, + "objective/train/theoretical_loss": 3.36605131199456, + "objective/train/tokens_used": 2523136000, + "objective/train/unlikelihood_loss": 0.19657643139362335, + "theoretical_loss": 3.36605131199456, + "tokens_seen": 2523136000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011897693079237713, + "loss": 1.3818, + "theoretical_loss": 3.36605131199456, + "tokens_seen": 2523136000 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001189568706118355, + "loss": 1.2152, + "theoretical_loss": 3.3660373902334584, + "tokens_seen": 2523267072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011893681043129388, + "loss": 1.3301, + "theoretical_loss": 3.3660234693979847, + "tokens_seen": 2523398144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011891675025075226, + "loss": 1.3069, + "theoretical_loss": 3.3660095494880293, + "tokens_seen": 2523529216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011889669007021063, + "loss": 1.2388, + "theoretical_loss": 3.3659956305034826, + "tokens_seen": 2523660288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011887662988966901, + "loss": 1.242, + "theoretical_loss": 3.365981712444235, + "tokens_seen": 2523791360 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011885656970912738, + "loss": 1.2883, + "theoretical_loss": 3.3659677953101763, + "tokens_seen": 2523922432 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011883650952858576, + "loss": 1.2543, + "theoretical_loss": 3.3659538791011983, + "tokens_seen": 2524053504 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011881644934804413, + "loss": 1.0895, + "theoretical_loss": 3.3659399638171905, + "tokens_seen": 2524184576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011879638916750251, + "loss": 1.2419, + "theoretical_loss": 3.3659260494580434, + "tokens_seen": 2524315648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011877632898696089, + "loss": 1.102, + "theoretical_loss": 3.365912136023648, + "tokens_seen": 2524446720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011875626880641926, + "loss": 1.3102, + "theoretical_loss": 3.365898223513894, + "tokens_seen": 2524577792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011873620862587764, + "loss": 1.3509, + "theoretical_loss": 3.3658843119286734, + "tokens_seen": 2524708864 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.4032258093357086, + "objective/train/avg_token_score": 0.014578872360289097, + "objective/train/docs_used": 1419316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4343980550765991, + "objective/train/original_loss": 4.250813961029053, + "objective/train/theoretical_loss": 3.365877356482728, + "objective/train/tokens_used": 2524774400, + "objective/train/unlikelihood_loss": 0.13600173592567444, + "theoretical_loss": 3.365877356482728, + "tokens_seen": 2524774400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011871614844533601, + "loss": 1.2602, + "theoretical_loss": 3.365870401267875, + "tokens_seen": 2524839936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011869608826479439, + "loss": 1.1524, + "theoretical_loss": 3.365856491531391, + "tokens_seen": 2524971008 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011867602808425276, + "loss": 1.2816, + "theoretical_loss": 3.365842582719111, + "tokens_seen": 2525102080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011865596790371114, + "loss": 1.1405, + "theoretical_loss": 3.365828674830926, + "tokens_seen": 2525233152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011863590772316952, + "loss": 1.3008, + "theoretical_loss": 3.3658147678667265, + "tokens_seen": 2525364224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011861584754262789, + "loss": 1.2457, + "theoretical_loss": 3.3658008618264033, + "tokens_seen": 2525495296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011859578736208625, + "loss": 1.327, + "theoretical_loss": 3.3657869567098464, + "tokens_seen": 2525626368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011857572718154463, + "loss": 1.2415, + "theoretical_loss": 3.3657730525169476, + "tokens_seen": 2525757440 + }, + { + "epoch": 0.77, + "learning_rate": 0.000118555667001003, + "loss": 1.1309, + "theoretical_loss": 3.365759149247597, + "tokens_seen": 2525888512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011853560682046138, + "loss": 1.0442, + "theoretical_loss": 3.3657452469016853, + "tokens_seen": 2526019584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011851554663991976, + "loss": 1.3396, + "theoretical_loss": 3.365731345479104, + "tokens_seen": 2526150656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011849548645937813, + "loss": 1.3016, + "theoretical_loss": 3.365717444979743, + "tokens_seen": 2526281728 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.35459432005882263, + "objective/train/avg_token_score": 0.017100641503930092, + "objective/train/docs_used": 1420528, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2136473655700684, + "objective/train/original_loss": 4.0429277420043945, + "objective/train/theoretical_loss": 3.365703545403493, + "objective/train/tokens_used": 2526412800, + "objective/train/unlikelihood_loss": 0.1731574833393097, + "theoretical_loss": 3.365703545403493, + "tokens_seen": 2526412800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011847542627883651, + "loss": 1.3127, + "theoretical_loss": 3.365703545403493, + "tokens_seen": 2526412800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011845536609829488, + "loss": 1.2449, + "theoretical_loss": 3.365689646750245, + "tokens_seen": 2526543872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011843530591775326, + "loss": 1.2433, + "theoretical_loss": 3.3656757490198905, + "tokens_seen": 2526674944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011841524573721165, + "loss": 1.3777, + "theoretical_loss": 3.3656618522123196, + "tokens_seen": 2526806016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011839518555667001, + "loss": 1.3912, + "theoretical_loss": 3.365647956327424, + "tokens_seen": 2526937088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011837512537612839, + "loss": 1.3493, + "theoretical_loss": 3.3656340613650935, + "tokens_seen": 2527068160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011835506519558676, + "loss": 1.2328, + "theoretical_loss": 3.3656201673252193, + "tokens_seen": 2527199232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011833500501504514, + "loss": 1.291, + "theoretical_loss": 3.3656062742076926, + "tokens_seen": 2527330304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011831494483450352, + "loss": 1.2439, + "theoretical_loss": 3.3655923820124047, + "tokens_seen": 2527461376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011829488465396189, + "loss": 1.2864, + "theoretical_loss": 3.365578490739246, + "tokens_seen": 2527592448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011827482447342027, + "loss": 1.1648, + "theoretical_loss": 3.3655646003881072, + "tokens_seen": 2527723520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011825476429287864, + "loss": 1.1557, + "theoretical_loss": 3.36555071095888, + "tokens_seen": 2527854592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011823470411233702, + "loss": 1.3381, + "theoretical_loss": 3.365536822451455, + "tokens_seen": 2527985664 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.4533846378326416, + "objective/train/avg_token_score": 0.014376137405633926, + "objective/train/docs_used": 1420951, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.407084345817566, + "objective/train/original_loss": 3.564441204071045, + "objective/train/theoretical_loss": 3.3655298785433847, + "objective/train/tokens_used": 2528051200, + "objective/train/unlikelihood_loss": 0.15810038149356842, + "theoretical_loss": 3.3655298785433847, + "tokens_seen": 2528051200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011821464393179538, + "loss": 1.2142, + "theoretical_loss": 3.365522934865724, + "tokens_seen": 2528116736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011819458375125376, + "loss": 1.3367, + "theoretical_loss": 3.3655090482015764, + "tokens_seen": 2528247808 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011817452357071213, + "loss": 1.2755, + "theoretical_loss": 3.3654951624589047, + "tokens_seen": 2528378880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011815446339017051, + "loss": 1.3533, + "theoretical_loss": 3.3654812776376, + "tokens_seen": 2528509952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011813440320962888, + "loss": 1.3775, + "theoretical_loss": 3.365467393737552, + "tokens_seen": 2528641024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011811434302908726, + "loss": 1.1711, + "theoretical_loss": 3.3654535107586536, + "tokens_seen": 2528772096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011809428284854564, + "loss": 1.1967, + "theoretical_loss": 3.365439628700795, + "tokens_seen": 2528903168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011807422266800401, + "loss": 1.3171, + "theoretical_loss": 3.3654257475638674, + "tokens_seen": 2529034240 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001180541624874624, + "loss": 1.3692, + "theoretical_loss": 3.365411867347762, + "tokens_seen": 2529165312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011803410230692078, + "loss": 1.2254, + "theoretical_loss": 3.36539798805237, + "tokens_seen": 2529296384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011801404212637914, + "loss": 1.2068, + "theoretical_loss": 3.365384109677583, + "tokens_seen": 2529427456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011799398194583751, + "loss": 1.2967, + "theoretical_loss": 3.3653702322232917, + "tokens_seen": 2529558528 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.38856303691864014, + "objective/train/avg_token_score": 0.007778517436236143, + "objective/train/docs_used": 1422170, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2797482013702393, + "objective/train/original_loss": 3.903369665145874, + "objective/train/theoretical_loss": 3.3653563556893875, + "objective/train/tokens_used": 2529689600, + "objective/train/unlikelihood_loss": 0.17671747505664825, + "theoretical_loss": 3.3653563556893875, + "tokens_seen": 2529689600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011797392176529589, + "loss": 1.3044, + "theoretical_loss": 3.3653563556893875, + "tokens_seen": 2529689600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011795386158475427, + "loss": 1.2601, + "theoretical_loss": 3.3653424800757623, + "tokens_seen": 2529820672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011793380140421264, + "loss": 1.3252, + "theoretical_loss": 3.3653286053823064, + "tokens_seen": 2529951744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011791374122367102, + "loss": 1.2969, + "theoretical_loss": 3.365314731608912, + "tokens_seen": 2530082816 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001178936810431294, + "loss": 1.1851, + "theoretical_loss": 3.3653008587554694, + "tokens_seen": 2530213888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011787362086258777, + "loss": 1.2988, + "theoretical_loss": 3.3652869868218707, + "tokens_seen": 2530344960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011785356068204615, + "loss": 1.2572, + "theoretical_loss": 3.3652731158080074, + "tokens_seen": 2530476032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011783350050150452, + "loss": 0.9875, + "theoretical_loss": 3.36525924571377, + "tokens_seen": 2530607104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011781344032096288, + "loss": 1.1988, + "theoretical_loss": 3.3652453765390513, + "tokens_seen": 2530738176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011779338014042126, + "loss": 1.4308, + "theoretical_loss": 3.3652315082837414, + "tokens_seen": 2530869248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011777331995987963, + "loss": 1.1786, + "theoretical_loss": 3.3652176409477326, + "tokens_seen": 2531000320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011775325977933801, + "loss": 1.3716, + "theoretical_loss": 3.3652037745309156, + "tokens_seen": 2531131392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011773319959879639, + "loss": 1.3236, + "theoretical_loss": 3.3651899090331825, + "tokens_seen": 2531262464 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.49217987060546875, + "objective/train/avg_token_score": 0.010771811939775944, + "objective/train/docs_used": 1422669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3786225318908691, + "objective/train/original_loss": 3.382481575012207, + "objective/train/theoretical_loss": 3.3651829766289385, + "objective/train/tokens_used": 2531328000, + "objective/train/unlikelihood_loss": 0.14217449724674225, + "theoretical_loss": 3.3651829766289385, + "tokens_seen": 2531328000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011771313941825476, + "loss": 1.2917, + "theoretical_loss": 3.365176044454425, + "tokens_seen": 2531393536 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011769307923771314, + "loss": 1.3249, + "theoretical_loss": 3.3651621807945338, + "tokens_seen": 2531524608 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011767301905717153, + "loss": 1.2645, + "theoretical_loss": 3.365148318053401, + "tokens_seen": 2531655680 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001176529588766299, + "loss": 1.1946, + "theoretical_loss": 3.365134456230918, + "tokens_seen": 2531786752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011763289869608828, + "loss": 1.2043, + "theoretical_loss": 3.365120595326976, + "tokens_seen": 2531917824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011761283851554664, + "loss": 1.3343, + "theoretical_loss": 3.365106735341467, + "tokens_seen": 2532048896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011759277833500502, + "loss": 1.161, + "theoretical_loss": 3.365092876274283, + "tokens_seen": 2532179968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011757271815446339, + "loss": 1.2299, + "theoretical_loss": 3.3650790181253147, + "tokens_seen": 2532311040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011755265797392177, + "loss": 1.3824, + "theoretical_loss": 3.3650651608944546, + "tokens_seen": 2532442112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011753259779338014, + "loss": 1.3017, + "theoretical_loss": 3.365051304581594, + "tokens_seen": 2532573184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011751253761283852, + "loss": 1.3271, + "theoretical_loss": 3.365037449186624, + "tokens_seen": 2532704256 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001174924774322969, + "loss": 1.1495, + "theoretical_loss": 3.365023594709437, + "tokens_seen": 2532835328 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.3851417303085327, + "objective/train/avg_token_score": 0.01154994871467352, + "objective/train/docs_used": 1423215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.049639344215393, + "objective/train/original_loss": 3.776392936706543, + "objective/train/theoretical_loss": 3.365009741149925, + "objective/train/tokens_used": 2532966400, + "objective/train/unlikelihood_loss": 0.17148828506469727, + "theoretical_loss": 3.365009741149925, + "tokens_seen": 2532966400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011747241725175527, + "loss": 1.042, + "theoretical_loss": 3.365009741149925, + "tokens_seen": 2532966400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011745235707121365, + "loss": 1.1647, + "theoretical_loss": 3.3649958885079796, + "tokens_seen": 2533097472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011743229689067201, + "loss": 1.1576, + "theoretical_loss": 3.3649820367834917, + "tokens_seen": 2533228544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011741223671013039, + "loss": 1.3261, + "theoretical_loss": 3.3649681859763536, + "tokens_seen": 2533359616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011739217652958876, + "loss": 1.3341, + "theoretical_loss": 3.3649543360864573, + "tokens_seen": 2533490688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011737211634904714, + "loss": 1.4489, + "theoretical_loss": 3.3649404871136945, + "tokens_seen": 2533621760 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011735205616850551, + "loss": 1.2218, + "theoretical_loss": 3.364926639057957, + "tokens_seen": 2533752832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011733199598796389, + "loss": 1.1332, + "theoretical_loss": 3.364912791919136, + "tokens_seen": 2533883904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011731193580742226, + "loss": 1.2361, + "theoretical_loss": 3.3648989456971243, + "tokens_seen": 2534014976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011729187562688065, + "loss": 1.19, + "theoretical_loss": 3.364885100391814, + "tokens_seen": 2534146048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011727181544633903, + "loss": 1.1861, + "theoretical_loss": 3.364871256003096, + "tokens_seen": 2534277120 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001172517552657974, + "loss": 1.2542, + "theoretical_loss": 3.3648574125308626, + "tokens_seen": 2534408192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011723169508525577, + "loss": 1.1063, + "theoretical_loss": 3.364843569975006, + "tokens_seen": 2534539264 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.484115332365036, + "objective/train/avg_token_score": 0.003912466112524271, + "objective/train/docs_used": 1424480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6015217304229736, + "objective/train/original_loss": 3.780318260192871, + "objective/train/theoretical_loss": 3.3648366490406847, + "objective/train/tokens_used": 2534604800, + "objective/train/unlikelihood_loss": 0.1595519781112671, + "theoretical_loss": 3.3648366490406847, + "tokens_seen": 2534604800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011721163490471414, + "loss": 1.3382, + "theoretical_loss": 3.3648297283354176, + "tokens_seen": 2534670336 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011719157472417252, + "loss": 1.1416, + "theoretical_loss": 3.3648158876119902, + "tokens_seen": 2534801408 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001171715145436309, + "loss": 1.2726, + "theoretical_loss": 3.364802047804615, + "tokens_seen": 2534932480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011715145436308927, + "loss": 1.2568, + "theoretical_loss": 3.3647882089131844, + "tokens_seen": 2535063552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011713139418254765, + "loss": 1.2841, + "theoretical_loss": 3.36477437093759, + "tokens_seen": 2535194624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011711133400200602, + "loss": 1.1906, + "theoretical_loss": 3.3647605338777247, + "tokens_seen": 2535325696 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001170912738214644, + "loss": 1.3206, + "theoretical_loss": 3.36474669773348, + "tokens_seen": 2535456768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011707121364092277, + "loss": 1.342, + "theoretical_loss": 3.364732862504748, + "tokens_seen": 2535587840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011705115346038115, + "loss": 1.3216, + "theoretical_loss": 3.3647190281914208, + "tokens_seen": 2535718912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011703109327983951, + "loss": 1.3766, + "theoretical_loss": 3.364705194793391, + "tokens_seen": 2535849984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011701103309929789, + "loss": 1.3471, + "theoretical_loss": 3.36469136231055, + "tokens_seen": 2535981056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011699097291875626, + "loss": 1.2966, + "theoretical_loss": 3.36467753074279, + "tokens_seen": 2536112128 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.5023216009140015, + "objective/train/avg_token_score": 0.005013773683458567, + "objective/train/docs_used": 1425012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4955847263336182, + "objective/train/original_loss": 3.389505386352539, + "objective/train/theoretical_loss": 3.364663700090004, + "objective/train/tokens_used": 2536243200, + "objective/train/unlikelihood_loss": 0.15175245702266693, + "theoretical_loss": 3.364663700090004, + "tokens_seen": 2536243200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011697091273821464, + "loss": 1.2295, + "theoretical_loss": 3.364663700090004, + "tokens_seen": 2536243200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011695085255767302, + "loss": 1.1648, + "theoretical_loss": 3.364649870352083, + "tokens_seen": 2536374272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011693079237713139, + "loss": 1.1902, + "theoretical_loss": 3.36463604152892, + "tokens_seen": 2536505344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011691073219658978, + "loss": 1.0956, + "theoretical_loss": 3.3646222136204074, + "tokens_seen": 2536636416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011689067201604816, + "loss": 1.2839, + "theoretical_loss": 3.364608386626437, + "tokens_seen": 2536767488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011687061183550653, + "loss": 1.2267, + "theoretical_loss": 3.3645945605469008, + "tokens_seen": 2536898560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011685055165496491, + "loss": 1.2336, + "theoretical_loss": 3.364580735381692, + "tokens_seen": 2537029632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011683049147442327, + "loss": 1.2165, + "theoretical_loss": 3.3645669111307024, + "tokens_seen": 2537160704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011681043129388165, + "loss": 1.3336, + "theoretical_loss": 3.364553087793824, + "tokens_seen": 2537291776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011679037111334002, + "loss": 1.1421, + "theoretical_loss": 3.3645392653709494, + "tokens_seen": 2537422848 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001167703109327984, + "loss": 1.1843, + "theoretical_loss": 3.364525443861971, + "tokens_seen": 2537553920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011675025075225677, + "loss": 1.2036, + "theoretical_loss": 3.3645116232667815, + "tokens_seen": 2537684992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011673019057171515, + "loss": 1.1643, + "theoretical_loss": 3.3644978035852726, + "tokens_seen": 2537816064 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.5631109476089478, + "objective/train/avg_token_score": 0.012946546077728271, + "objective/train/docs_used": 1426240, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7018793821334839, + "objective/train/original_loss": 3.508789300918579, + "objective/train/theoretical_loss": 3.364490894087115, + "objective/train/tokens_used": 2537881600, + "objective/train/unlikelihood_loss": 0.16983002424240112, + "theoretical_loss": 3.364490894087115, + "tokens_seen": 2537881600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011671013039117352, + "loss": 1.3454, + "theoretical_loss": 3.3644839848173373, + "tokens_seen": 2537947136 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001166900702106319, + "loss": 1.2877, + "theoretical_loss": 3.364470166962868, + "tokens_seen": 2538078208 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011667001003009028, + "loss": 1.2948, + "theoretical_loss": 3.364456350021756, + "tokens_seen": 2538209280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011664994984954864, + "loss": 1.3997, + "theoretical_loss": 3.3644425339938957, + "tokens_seen": 2538340352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011662988966900701, + "loss": 1.2369, + "theoretical_loss": 3.3644287188791786, + "tokens_seen": 2538471424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011660982948846539, + "loss": 1.2606, + "theoretical_loss": 3.3644149046774965, + "tokens_seen": 2538602496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011658976930792377, + "loss": 1.3124, + "theoretical_loss": 3.364401091388743, + "tokens_seen": 2538733568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011656970912738214, + "loss": 1.2023, + "theoretical_loss": 3.3643872790128104, + "tokens_seen": 2538864640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011654964894684053, + "loss": 1.1682, + "theoretical_loss": 3.364373467549591, + "tokens_seen": 2538995712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011652958876629891, + "loss": 1.4327, + "theoretical_loss": 3.3643596569989778, + "tokens_seen": 2539126784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011650952858575728, + "loss": 1.2994, + "theoretical_loss": 3.3643458473608625, + "tokens_seen": 2539257856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011648946840521566, + "loss": 1.2725, + "theoretical_loss": 3.3643320386351387, + "tokens_seen": 2539388928 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.2950879633426666, + "objective/train/avg_token_score": 0.02515227161347866, + "objective/train/docs_used": 1426896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8092211484909058, + "objective/train/original_loss": 4.643178462982178, + "objective/train/theoretical_loss": 3.3643182308216986, + "objective/train/tokens_used": 2539520000, + "objective/train/unlikelihood_loss": 0.1352221965789795, + "theoretical_loss": 3.3643182308216986, + "tokens_seen": 2539520000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011646940822467403, + "loss": 1.225, + "theoretical_loss": 3.3643182308216986, + "tokens_seen": 2539520000 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001164493480441324, + "loss": 1.2154, + "theoretical_loss": 3.3643044239204345, + "tokens_seen": 2539651072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011642928786359077, + "loss": 1.3108, + "theoretical_loss": 3.36429061793124, + "tokens_seen": 2539782144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011640922768304915, + "loss": 1.2286, + "theoretical_loss": 3.364276812854007, + "tokens_seen": 2539913216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011638916750250752, + "loss": 1.3124, + "theoretical_loss": 3.3642630086886287, + "tokens_seen": 2540044288 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001163691073219659, + "loss": 1.2102, + "theoretical_loss": 3.364249205434997, + "tokens_seen": 2540175360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011634904714142428, + "loss": 1.308, + "theoretical_loss": 3.3642354030930055, + "tokens_seen": 2540306432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011632898696088265, + "loss": 1.2438, + "theoretical_loss": 3.3642216016625466, + "tokens_seen": 2540437504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011630892678034103, + "loss": 1.2332, + "theoretical_loss": 3.3642078011435133, + "tokens_seen": 2540568576 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001162888665997994, + "loss": 1.1454, + "theoretical_loss": 3.364194001535798, + "tokens_seen": 2540699648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011626880641925778, + "loss": 1.2504, + "theoretical_loss": 3.364180202839294, + "tokens_seen": 2540830720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011624874623871614, + "loss": 1.2151, + "theoretical_loss": 3.3641664050538935, + "tokens_seen": 2540961792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011622868605817452, + "loss": 1.3876, + "theoretical_loss": 3.36415260817949, + "tokens_seen": 2541092864 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.38062071800231934, + "objective/train/avg_token_score": 0.01354947965592146, + "objective/train/docs_used": 1428293, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2237597703933716, + "objective/train/original_loss": 4.065069675445557, + "objective/train/theoretical_loss": 3.3641457100838785, + "objective/train/tokens_used": 2541158400, + "objective/train/unlikelihood_loss": 0.20355582237243652, + "theoretical_loss": 3.3641457100838785, + "tokens_seen": 2541158400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011620862587763289, + "loss": 1.2829, + "theoretical_loss": 3.3641388122159763, + "tokens_seen": 2541223936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011618856569709127, + "loss": 1.2447, + "theoretical_loss": 3.3641250171632446, + "tokens_seen": 2541355008 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011616850551654966, + "loss": 1.3201, + "theoretical_loss": 3.3641112230211885, + "tokens_seen": 2541486080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011614844533600803, + "loss": 1.2507, + "theoretical_loss": 3.3640974297897004, + "tokens_seen": 2541617152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011612838515546641, + "loss": 1.2914, + "theoretical_loss": 3.364083637468674, + "tokens_seen": 2541748224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011610832497492478, + "loss": 1.2636, + "theoretical_loss": 3.364069846058001, + "tokens_seen": 2541879296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011608826479438316, + "loss": 1.1242, + "theoretical_loss": 3.364056055557576, + "tokens_seen": 2542010368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011606820461384154, + "loss": 1.3687, + "theoretical_loss": 3.364042265967291, + "tokens_seen": 2542141440 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001160481444332999, + "loss": 1.2676, + "theoretical_loss": 3.364028477287039, + "tokens_seen": 2542272512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011602808425275827, + "loss": 1.2865, + "theoretical_loss": 3.364014689516713, + "tokens_seen": 2542403584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011600802407221665, + "loss": 1.4192, + "theoretical_loss": 3.3640009026562065, + "tokens_seen": 2542534656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011598796389167503, + "loss": 1.2058, + "theoretical_loss": 3.3639871167054123, + "tokens_seen": 2542665728 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.3112170100212097, + "objective/train/avg_token_score": 0.021357087418437004, + "objective/train/docs_used": 1429050, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9328053593635559, + "objective/train/original_loss": 3.6355385780334473, + "objective/train/theoretical_loss": 3.3639733316642233, + "objective/train/tokens_used": 2542796800, + "objective/train/unlikelihood_loss": 0.22579777240753174, + "theoretical_loss": 3.3639733316642233, + "tokens_seen": 2542796800 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001159679037111334, + "loss": 1.2, + "theoretical_loss": 3.3639733316642233, + "tokens_seen": 2542796800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011594784353059178, + "loss": 1.2527, + "theoretical_loss": 3.363959547532533, + "tokens_seen": 2542927872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011592778335005015, + "loss": 1.2563, + "theoretical_loss": 3.363945764310234, + "tokens_seen": 2543058944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011590772316950853, + "loss": 1.1623, + "theoretical_loss": 3.36393198199722, + "tokens_seen": 2543190016 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001158876629889669, + "loss": 1.2451, + "theoretical_loss": 3.3639182005933836, + "tokens_seen": 2543321088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011586760280842527, + "loss": 1.2974, + "theoretical_loss": 3.363904420098619, + "tokens_seen": 2543452160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011584754262788364, + "loss": 1.2509, + "theoretical_loss": 3.363890640512818, + "tokens_seen": 2543583232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011582748244734202, + "loss": 1.1381, + "theoretical_loss": 3.3638768618358745, + "tokens_seen": 2543714304 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001158074222668004, + "loss": 1.2271, + "theoretical_loss": 3.363863084067682, + "tokens_seen": 2543845376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011578736208625878, + "loss": 1.2318, + "theoretical_loss": 3.3638493072081332, + "tokens_seen": 2543976448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011576730190571716, + "loss": 1.2095, + "theoretical_loss": 3.3638355312571218, + "tokens_seen": 2544107520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011574724172517554, + "loss": 1.2079, + "theoretical_loss": 3.363821756214541, + "tokens_seen": 2544238592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011572718154463391, + "loss": 1.2292, + "theoretical_loss": 3.3638079820802838, + "tokens_seen": 2544369664 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.30987292528152466, + "objective/train/avg_token_score": 0.0591476634144783, + "objective/train/docs_used": 1430428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8369715809822083, + "objective/train/original_loss": 3.886655330657959, + "objective/train/theoretical_loss": 3.363801095353743, + "objective/train/tokens_used": 2544435200, + "objective/train/unlikelihood_loss": 0.2005622386932373, + "theoretical_loss": 3.363801095353743, + "tokens_seen": 2544435200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011570712136409229, + "loss": 1.1178, + "theoretical_loss": 3.3637942088542436, + "tokens_seen": 2544500736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011568706118355066, + "loss": 1.2294, + "theoretical_loss": 3.363780436536314, + "tokens_seen": 2544631808 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011566700100300903, + "loss": 1.2512, + "theoretical_loss": 3.3637666651263882, + "tokens_seen": 2544762880 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001156469408224674, + "loss": 1.1552, + "theoretical_loss": 3.3637528946243593, + "tokens_seen": 2544893952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011562688064192578, + "loss": 1.344, + "theoretical_loss": 3.363739125030121, + "tokens_seen": 2545025024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011560682046138415, + "loss": 1.2635, + "theoretical_loss": 3.363725356343567, + "tokens_seen": 2545156096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011558676028084253, + "loss": 1.0563, + "theoretical_loss": 3.3637115885645903, + "tokens_seen": 2545287168 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001155667001003009, + "loss": 1.2595, + "theoretical_loss": 3.363697821693085, + "tokens_seen": 2545418240 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011554663991975928, + "loss": 1.1885, + "theoretical_loss": 3.3636840557289434, + "tokens_seen": 2545549312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011552657973921766, + "loss": 1.1767, + "theoretical_loss": 3.3636702906720592, + "tokens_seen": 2545680384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011550651955867603, + "loss": 1.2869, + "theoretical_loss": 3.363656526522327, + "tokens_seen": 2545811456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011548645937813441, + "loss": 1.3236, + "theoretical_loss": 3.363642763279639, + "tokens_seen": 2545942528 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.4377443790435791, + "objective/train/avg_token_score": 0.012132072821259499, + "objective/train/docs_used": 1431073, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3536723852157593, + "objective/train/original_loss": 3.782212495803833, + "objective/train/theoretical_loss": 3.36362900094389, + "objective/train/tokens_used": 2546073600, + "objective/train/unlikelihood_loss": 0.11916511505842209, + "theoretical_loss": 3.36362900094389, + "tokens_seen": 2546073600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011546639919759277, + "loss": 1.1176, + "theoretical_loss": 3.36362900094389, + "tokens_seen": 2546073600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011544633901705115, + "loss": 1.1943, + "theoretical_loss": 3.3636152395149725, + "tokens_seen": 2546204672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011542627883650953, + "loss": 1.1698, + "theoretical_loss": 3.36360147899278, + "tokens_seen": 2546335744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011540621865596791, + "loss": 1.1666, + "theoretical_loss": 3.3635877193772075, + "tokens_seen": 2546466816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011538615847542629, + "loss": 1.2255, + "theoretical_loss": 3.363573960668147, + "tokens_seen": 2546597888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011536609829488466, + "loss": 1.3703, + "theoretical_loss": 3.363560202865493, + "tokens_seen": 2546728960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011534603811434304, + "loss": 1.2306, + "theoretical_loss": 3.363546445969139, + "tokens_seen": 2546860032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011532597793380141, + "loss": 1.2228, + "theoretical_loss": 3.363532689978978, + "tokens_seen": 2546991104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011530591775325979, + "loss": 1.2882, + "theoretical_loss": 3.363518934894905, + "tokens_seen": 2547122176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011528585757271815, + "loss": 1.2499, + "theoretical_loss": 3.3635051807168126, + "tokens_seen": 2547253248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011526579739217653, + "loss": 1.2367, + "theoretical_loss": 3.363491427444595, + "tokens_seen": 2547384320 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001152457372116349, + "loss": 1.2718, + "theoretical_loss": 3.363477675078146, + "tokens_seen": 2547515392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011522567703109328, + "loss": 1.2792, + "theoretical_loss": 3.3634639236173585, + "tokens_seen": 2547646464 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.458822101354599, + "objective/train/avg_token_score": 0.006727050058543682, + "objective/train/docs_used": 1432125, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3695168495178223, + "objective/train/original_loss": 3.9162509441375732, + "objective/train/theoretical_loss": 3.363457048226555, + "objective/train/tokens_used": 2547712000, + "objective/train/unlikelihood_loss": 0.15617798268795013, + "theoretical_loss": 3.363457048226555, + "tokens_seen": 2547712000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011520561685055165, + "loss": 1.3372, + "theoretical_loss": 3.3634501730621276, + "tokens_seen": 2547777536 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011518555667001003, + "loss": 1.0631, + "theoretical_loss": 3.3634364234123457, + "tokens_seen": 2547908608 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001151654964894684, + "loss": 1.2839, + "theoretical_loss": 3.3634226746679077, + "tokens_seen": 2548039680 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011514543630892678, + "loss": 1.2798, + "theoretical_loss": 3.363408926828707, + "tokens_seen": 2548170752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011512537612838516, + "loss": 1.1504, + "theoretical_loss": 3.3633951798946375, + "tokens_seen": 2548301824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011510531594784353, + "loss": 1.2621, + "theoretical_loss": 3.3633814338655927, + "tokens_seen": 2548432896 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001150852557673019, + "loss": 1.1407, + "theoretical_loss": 3.363367688741467, + "tokens_seen": 2548563968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011506519558676027, + "loss": 1.2532, + "theoretical_loss": 3.3633539445221543, + "tokens_seen": 2548695040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011504513540621866, + "loss": 1.1103, + "theoretical_loss": 3.3633402012075484, + "tokens_seen": 2548826112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011502507522567704, + "loss": 1.4041, + "theoretical_loss": 3.363326458797543, + "tokens_seen": 2548957184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011500501504513541, + "loss": 1.3348, + "theoretical_loss": 3.3633127172920316, + "tokens_seen": 2549088256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011498495486459379, + "loss": 1.3436, + "theoretical_loss": 3.3632989766909094, + "tokens_seen": 2549219328 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.3320503532886505, + "objective/train/avg_token_score": 0.01613553799688816, + "objective/train/docs_used": 1432724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0899935960769653, + "objective/train/original_loss": 4.200340270996094, + "objective/train/theoretical_loss": 3.3632852369940696, + "objective/train/tokens_used": 2549350400, + "objective/train/unlikelihood_loss": 0.16815020143985748, + "theoretical_loss": 3.3632852369940696, + "tokens_seen": 2549350400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011496489468405216, + "loss": 1.2601, + "theoretical_loss": 3.3632852369940696, + "tokens_seen": 2549350400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011494483450351054, + "loss": 1.3665, + "theoretical_loss": 3.363271498201406, + "tokens_seen": 2549481472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011492477432296892, + "loss": 1.2691, + "theoretical_loss": 3.363257760312813, + "tokens_seen": 2549612544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011490471414242729, + "loss": 1.2268, + "theoretical_loss": 3.3632440233281846, + "tokens_seen": 2549743616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011488465396188565, + "loss": 1.3467, + "theoretical_loss": 3.3632302872474154, + "tokens_seen": 2549874688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011486459378134403, + "loss": 1.2559, + "theoretical_loss": 3.3632165520703983, + "tokens_seen": 2550005760 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001148445336008024, + "loss": 1.3139, + "theoretical_loss": 3.363202817797028, + "tokens_seen": 2550136832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011482447342026078, + "loss": 1.2111, + "theoretical_loss": 3.3631890844271988, + "tokens_seen": 2550267904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011480441323971916, + "loss": 1.1171, + "theoretical_loss": 3.3631753519608045, + "tokens_seen": 2550398976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011478435305917753, + "loss": 1.2246, + "theoretical_loss": 3.3631616203977392, + "tokens_seen": 2550530048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011476429287863591, + "loss": 1.2558, + "theoretical_loss": 3.3631478897378977, + "tokens_seen": 2550661120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011474423269809428, + "loss": 1.3923, + "theoretical_loss": 3.3631341599811737, + "tokens_seen": 2550792192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011472417251755266, + "loss": 1.3157, + "theoretical_loss": 3.3631204311274607, + "tokens_seen": 2550923264 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.4313294291496277, + "objective/train/avg_token_score": 0.028820564970374107, + "objective/train/docs_used": 1434145, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4242037534713745, + "objective/train/original_loss": 4.477665424346924, + "objective/train/theoretical_loss": 3.3631135670392007, + "objective/train/tokens_used": 2550988800, + "objective/train/unlikelihood_loss": 0.12131133675575256, + "theoretical_loss": 3.3631135670392007, + "tokens_seen": 2550988800 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011470411233701104, + "loss": 1.3029, + "theoretical_loss": 3.363106703176654, + "tokens_seen": 2551054336 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001146840521564694, + "loss": 1.405, + "theoretical_loss": 3.3630929761286477, + "tokens_seen": 2551185408 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011466399197592779, + "loss": 1.0712, + "theoretical_loss": 3.3630792499833357, + "tokens_seen": 2551316480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011464393179538616, + "loss": 1.2599, + "theoretical_loss": 3.363065524740612, + "tokens_seen": 2551447552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011462387161484454, + "loss": 1.2625, + "theoretical_loss": 3.3630518004003718, + "tokens_seen": 2551578624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011460381143430292, + "loss": 1.2766, + "theoretical_loss": 3.3630380769625083, + "tokens_seen": 2551709696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011458375125376129, + "loss": 1.2496, + "theoretical_loss": 3.3630243544269165, + "tokens_seen": 2551840768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011456369107321967, + "loss": 1.374, + "theoretical_loss": 3.363010632793491, + "tokens_seen": 2551971840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011454363089267804, + "loss": 1.3646, + "theoretical_loss": 3.3629969120621257, + "tokens_seen": 2552102912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011452357071213642, + "loss": 1.287, + "theoretical_loss": 3.362983192232715, + "tokens_seen": 2552233984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011450351053159478, + "loss": 1.1113, + "theoretical_loss": 3.362969473305153, + "tokens_seen": 2552365056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011448345035105316, + "loss": 1.2308, + "theoretical_loss": 3.3629557552793345, + "tokens_seen": 2552496128 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.4253421425819397, + "objective/train/avg_token_score": 0.015418900176882744, + "objective/train/docs_used": 1434830, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2559372186660767, + "objective/train/original_loss": 4.061153888702393, + "objective/train/theoretical_loss": 3.362942038155154, + "objective/train/tokens_used": 2552627200, + "objective/train/unlikelihood_loss": 0.10696766525506973, + "theoretical_loss": 3.362942038155154, + "tokens_seen": 2552627200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011446339017051153, + "loss": 1.3494, + "theoretical_loss": 3.362942038155154, + "tokens_seen": 2552627200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011444332998996991, + "loss": 1.2812, + "theoretical_loss": 3.3629283219325057, + "tokens_seen": 2552758272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011442326980942828, + "loss": 1.2594, + "theoretical_loss": 3.3629146066112847, + "tokens_seen": 2552889344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011440320962888666, + "loss": 1.4445, + "theoretical_loss": 3.3629008921913845, + "tokens_seen": 2553020416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011438314944834504, + "loss": 1.312, + "theoretical_loss": 3.3628871786727004, + "tokens_seen": 2553151488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011436308926780341, + "loss": 1.2564, + "theoretical_loss": 3.362873466055126, + "tokens_seen": 2553282560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011434302908726179, + "loss": 1.2583, + "theoretical_loss": 3.3628597543385568, + "tokens_seen": 2553413632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011432296890672016, + "loss": 1.2299, + "theoretical_loss": 3.362846043522887, + "tokens_seen": 2553544704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011430290872617853, + "loss": 1.3401, + "theoretical_loss": 3.362832333608011, + "tokens_seen": 2553675776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011428284854563691, + "loss": 1.1466, + "theoretical_loss": 3.3628186245938236, + "tokens_seen": 2553806848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011426278836509529, + "loss": 1.1934, + "theoretical_loss": 3.3628049164802194, + "tokens_seen": 2553937920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011424272818455367, + "loss": 1.3051, + "theoretical_loss": 3.362791209267093, + "tokens_seen": 2554068992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011422266800401204, + "loss": 1.3121, + "theoretical_loss": 3.362777502954339, + "tokens_seen": 2554200064 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.44611436128616333, + "objective/train/avg_token_score": 0.008004299364984035, + "objective/train/docs_used": 1435894, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3703253269195557, + "objective/train/original_loss": 3.3361337184906006, + "objective/train/theoretical_loss": 3.3627706501355688, + "objective/train/tokens_used": 2554265600, + "objective/train/unlikelihood_loss": 0.234457865357399, + "theoretical_loss": 3.3627706501355688, + "tokens_seen": 2554265600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011420260782347042, + "loss": 1.2038, + "theoretical_loss": 3.362763797541852, + "tokens_seen": 2554331136 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001141825476429288, + "loss": 1.3531, + "theoretical_loss": 3.3627500930295264, + "tokens_seen": 2554462208 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011416248746238717, + "loss": 1.3814, + "theoretical_loss": 3.3627363894172575, + "tokens_seen": 2554593280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011414242728184554, + "loss": 1.3584, + "theoretical_loss": 3.36272268670494, + "tokens_seen": 2554724352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011412236710130392, + "loss": 1.4349, + "theoretical_loss": 3.362708984892468, + "tokens_seen": 2554855424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011410230692076228, + "loss": 1.2807, + "theoretical_loss": 3.362695283979737, + "tokens_seen": 2554986496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011408224674022066, + "loss": 1.5041, + "theoretical_loss": 3.362681583966641, + "tokens_seen": 2555117568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011406218655967903, + "loss": 1.4035, + "theoretical_loss": 3.3626678848530753, + "tokens_seen": 2555248640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011404212637913741, + "loss": 1.3833, + "theoretical_loss": 3.3626541866389346, + "tokens_seen": 2555379712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011402206619859579, + "loss": 1.3698, + "theoretical_loss": 3.3626404893241135, + "tokens_seen": 2555510784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011400200601805416, + "loss": 1.2974, + "theoretical_loss": 3.3626267929085074, + "tokens_seen": 2555641856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011398194583751254, + "loss": 1.3531, + "theoretical_loss": 3.3626130973920105, + "tokens_seen": 2555772928 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.4608382284641266, + "objective/train/avg_token_score": 0.008119449950754642, + "objective/train/docs_used": 1436568, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4848119020462036, + "objective/train/original_loss": 3.593524217605591, + "objective/train/theoretical_loss": 3.362599402774518, + "objective/train/tokens_used": 2555904000, + "objective/train/unlikelihood_loss": 0.2283518761396408, + "theoretical_loss": 3.362599402774518, + "tokens_seen": 2555904000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011396188565697091, + "loss": 1.4655, + "theoretical_loss": 3.362599402774518, + "tokens_seen": 2555904000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011394182547642929, + "loss": 1.3357, + "theoretical_loss": 3.362585709055925, + "tokens_seen": 2556035072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011392176529588768, + "loss": 1.2627, + "theoretical_loss": 3.3625720162361254, + "tokens_seen": 2556166144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011390170511534604, + "loss": 1.1871, + "theoretical_loss": 3.3625583243150157, + "tokens_seen": 2556297216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011388164493480442, + "loss": 1.1405, + "theoretical_loss": 3.3625446332924893, + "tokens_seen": 2556428288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011386158475426279, + "loss": 1.2916, + "theoretical_loss": 3.3625309431684425, + "tokens_seen": 2556559360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011384152457372117, + "loss": 1.3154, + "theoretical_loss": 3.3625172539427695, + "tokens_seen": 2556690432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011382146439317954, + "loss": 1.2838, + "theoretical_loss": 3.362503565615365, + "tokens_seen": 2556821504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011380140421263792, + "loss": 1.2844, + "theoretical_loss": 3.3624898781861248, + "tokens_seen": 2556952576 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001137813440320963, + "loss": 1.4299, + "theoretical_loss": 3.3624761916549435, + "tokens_seen": 2557083648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011376128385155467, + "loss": 1.2393, + "theoretical_loss": 3.3624625060217164, + "tokens_seen": 2557214720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011374122367101305, + "loss": 1.33, + "theoretical_loss": 3.362448821286338, + "tokens_seen": 2557345792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011372116349047141, + "loss": 1.2364, + "theoretical_loss": 3.3624351374487045, + "tokens_seen": 2557476864 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.4802052676677704, + "objective/train/avg_token_score": 0.02305307611823082, + "objective/train/docs_used": 1437925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5449268817901611, + "objective/train/original_loss": 3.8500688076019287, + "objective/train/theoretical_loss": 3.3624282958665086, + "objective/train/tokens_used": 2557542400, + "objective/train/unlikelihood_loss": 0.15261954069137573, + "theoretical_loss": 3.3624282958665086, + "tokens_seen": 2557542400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011370110330992979, + "loss": 1.2717, + "theoretical_loss": 3.3624214545087097, + "tokens_seen": 2557607936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011368104312938816, + "loss": 1.4005, + "theoretical_loss": 3.3624077724662493, + "tokens_seen": 2557739008 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011366098294884654, + "loss": 1.1591, + "theoretical_loss": 3.3623940913212187, + "tokens_seen": 2557870080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011364092276830491, + "loss": 1.4041, + "theoretical_loss": 3.362380411073513, + "tokens_seen": 2558001152 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011362086258776329, + "loss": 1.2973, + "theoretical_loss": 3.362366731723027, + "tokens_seen": 2558132224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011360080240722166, + "loss": 1.3651, + "theoretical_loss": 3.3623530532696555, + "tokens_seen": 2558263296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011358074222668004, + "loss": 1.3999, + "theoretical_loss": 3.3623393757132947, + "tokens_seen": 2558394368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011356068204613842, + "loss": 1.4887, + "theoretical_loss": 3.362325699053839, + "tokens_seen": 2558525440 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001135406218655968, + "loss": 1.296, + "theoretical_loss": 3.3623120232911847, + "tokens_seen": 2558656512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011352056168505517, + "loss": 1.4198, + "theoretical_loss": 3.362298348425226, + "tokens_seen": 2558787584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011350050150451354, + "loss": 1.3958, + "theoretical_loss": 3.3622846744558585, + "tokens_seen": 2558918656 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011348044132397192, + "loss": 1.3397, + "theoretical_loss": 3.362271001382978, + "tokens_seen": 2559049728 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.38202589750289917, + "objective/train/avg_token_score": 0.01907994970679283, + "objective/train/docs_used": 1438686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1569806337356567, + "objective/train/original_loss": 3.9274489879608154, + "objective/train/theoretical_loss": 3.362257329206479, + "objective/train/tokens_used": 2559180800, + "objective/train/unlikelihood_loss": 0.1675667017698288, + "theoretical_loss": 3.362257329206479, + "tokens_seen": 2559180800 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001134603811434303, + "loss": 1.1916, + "theoretical_loss": 3.362257329206479, + "tokens_seen": 2559180800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011344032096288867, + "loss": 1.44, + "theoretical_loss": 3.362243657926257, + "tokens_seen": 2559311872 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011342026078234705, + "loss": 1.1915, + "theoretical_loss": 3.362229987542208, + "tokens_seen": 2559442944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011340020060180542, + "loss": 1.398, + "theoretical_loss": 3.3622163180542266, + "tokens_seen": 2559574016 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001133801404212638, + "loss": 1.2665, + "theoretical_loss": 3.3622026494622084, + "tokens_seen": 2559705088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011336008024072217, + "loss": 1.3519, + "theoretical_loss": 3.362188981766049, + "tokens_seen": 2559836160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011334002006018055, + "loss": 1.1996, + "theoretical_loss": 3.362175314965644, + "tokens_seen": 2559967232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011331995987963891, + "loss": 1.1581, + "theoretical_loss": 3.3621616490608885, + "tokens_seen": 2560098304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011329989969909729, + "loss": 1.3831, + "theoretical_loss": 3.3621479840516777, + "tokens_seen": 2560229376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011327983951855566, + "loss": 1.1293, + "theoretical_loss": 3.3621343199379075, + "tokens_seen": 2560360448 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011325977933801404, + "loss": 1.3313, + "theoretical_loss": 3.3621206567194735, + "tokens_seen": 2560491520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011323971915747241, + "loss": 1.2469, + "theoretical_loss": 3.3621069943962705, + "tokens_seen": 2560622592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011321965897693079, + "loss": 1.2886, + "theoretical_loss": 3.3620933329681946, + "tokens_seen": 2560753664 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.392778605222702, + "objective/train/avg_token_score": 0.011459741741418839, + "objective/train/docs_used": 1440126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1687343120574951, + "objective/train/original_loss": 3.931407928466797, + "objective/train/theoretical_loss": 3.362086502589797, + "objective/train/tokens_used": 2560819200, + "objective/train/unlikelihood_loss": 0.1513223946094513, + "theoretical_loss": 3.362086502589797, + "tokens_seen": 2560819200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011319959879638917, + "loss": 1.3766, + "theoretical_loss": 3.3620796724351414, + "tokens_seen": 2560884736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011317953861584754, + "loss": 1.3254, + "theoretical_loss": 3.3620660127970066, + "tokens_seen": 2561015808 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011315947843530593, + "loss": 1.2495, + "theoretical_loss": 3.362052354053685, + "tokens_seen": 2561146880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011313941825476431, + "loss": 1.3443, + "theoretical_loss": 3.3620386962050723, + "tokens_seen": 2561277952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011311935807422267, + "loss": 1.5355, + "theoretical_loss": 3.362025039251065, + "tokens_seen": 2561409024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011309929789368105, + "loss": 1.264, + "theoretical_loss": 3.3620113831915583, + "tokens_seen": 2561540096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011307923771313942, + "loss": 1.3107, + "theoretical_loss": 3.3619977280264477, + "tokens_seen": 2561671168 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001130591775325978, + "loss": 1.3796, + "theoretical_loss": 3.361984073755629, + "tokens_seen": 2561802240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011303911735205617, + "loss": 1.2421, + "theoretical_loss": 3.3619704203789973, + "tokens_seen": 2561933312 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011301905717151455, + "loss": 1.2398, + "theoretical_loss": 3.361956767896449, + "tokens_seen": 2562064384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011299899699097292, + "loss": 1.3973, + "theoretical_loss": 3.3619431163078795, + "tokens_seen": 2562195456 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001129789368104313, + "loss": 1.3142, + "theoretical_loss": 3.361929465613185, + "tokens_seen": 2562326528 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.34341397881507874, + "objective/train/avg_token_score": 0.006894922815263271, + "objective/train/docs_used": 1440814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9668260216712952, + "objective/train/original_loss": 3.4816386699676514, + "objective/train/theoretical_loss": 3.36191581581226, + "objective/train/tokens_used": 2562457600, + "objective/train/unlikelihood_loss": 0.3350115418434143, + "theoretical_loss": 3.36191581581226, + "tokens_seen": 2562457600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011295887662988968, + "loss": 1.1763, + "theoretical_loss": 3.36191581581226, + "tokens_seen": 2562457600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011293881644934804, + "loss": 1.1823, + "theoretical_loss": 3.3619021669050024, + "tokens_seen": 2562588672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011291875626880641, + "loss": 1.2737, + "theoretical_loss": 3.361888518891306, + "tokens_seen": 2562719744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011289869608826479, + "loss": 1.2541, + "theoretical_loss": 3.361874871771067, + "tokens_seen": 2562850816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011287863590772317, + "loss": 1.294, + "theoretical_loss": 3.3618612255441827, + "tokens_seen": 2562981888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011285857572718154, + "loss": 1.4005, + "theoretical_loss": 3.3618475802105467, + "tokens_seen": 2563112960 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011283851554663992, + "loss": 1.4537, + "theoretical_loss": 3.361833935770057, + "tokens_seen": 2563244032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011281845536609829, + "loss": 1.3466, + "theoretical_loss": 3.3618202922226073, + "tokens_seen": 2563375104 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011279839518555668, + "loss": 1.1788, + "theoretical_loss": 3.361806649568095, + "tokens_seen": 2563506176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011277833500501506, + "loss": 1.2748, + "theoretical_loss": 3.361793007806416, + "tokens_seen": 2563637248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011275827482447343, + "loss": 1.315, + "theoretical_loss": 3.3617793669374656, + "tokens_seen": 2563768320 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001127382146439318, + "loss": 1.383, + "theoretical_loss": 3.36176572696114, + "tokens_seen": 2563899392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011271815446339017, + "loss": 1.2828, + "theoretical_loss": 3.361752087877335, + "tokens_seen": 2564030464 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.3926564157009125, + "objective/train/avg_token_score": 0.020998133346438408, + "objective/train/docs_used": 1442376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1981052160263062, + "objective/train/original_loss": 3.8250691890716553, + "objective/train/theoretical_loss": 3.3617452686700955, + "objective/train/tokens_used": 2564096000, + "objective/train/unlikelihood_loss": 0.1467137485742569, + "theoretical_loss": 3.3617452686700955, + "tokens_seen": 2564096000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011269809428284855, + "loss": 1.2133, + "theoretical_loss": 3.361738449685947, + "tokens_seen": 2564161536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011267803410230692, + "loss": 1.3224, + "theoretical_loss": 3.3617248123868713, + "tokens_seen": 2564292608 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001126579739217653, + "loss": 1.3772, + "theoretical_loss": 3.361711175980005, + "tokens_seen": 2564423680 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011263791374122368, + "loss": 1.3359, + "theoretical_loss": 3.361697540465243, + "tokens_seen": 2564554752 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011261785356068205, + "loss": 1.2321, + "theoretical_loss": 3.361683905842482, + "tokens_seen": 2564685824 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011259779338014043, + "loss": 1.3667, + "theoretical_loss": 3.361670272111618, + "tokens_seen": 2564816896 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001125777331995988, + "loss": 1.3389, + "theoretical_loss": 3.3616566392725464, + "tokens_seen": 2564947968 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011255767301905718, + "loss": 1.3475, + "theoretical_loss": 3.3616430073251644, + "tokens_seen": 2565079040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011253761283851554, + "loss": 1.3685, + "theoretical_loss": 3.3616293762693674, + "tokens_seen": 2565210112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011251755265797392, + "loss": 1.3342, + "theoretical_loss": 3.361615746105052, + "tokens_seen": 2565341184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011249749247743229, + "loss": 1.408, + "theoretical_loss": 3.3616021168321137, + "tokens_seen": 2565472256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011247743229689067, + "loss": 1.3955, + "theoretical_loss": 3.361588488450449, + "tokens_seen": 2565603328 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.32697945833206177, + "objective/train/avg_token_score": 0.05175742506980896, + "objective/train/docs_used": 1443131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0460937023162842, + "objective/train/original_loss": 4.526967525482178, + "objective/train/theoretical_loss": 3.3615748609599545, + "objective/train/tokens_used": 2565734400, + "objective/train/unlikelihood_loss": 0.11355502158403397, + "theoretical_loss": 3.3615748609599545, + "tokens_seen": 2565734400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011245737211634904, + "loss": 1.3298, + "theoretical_loss": 3.3615748609599545, + "tokens_seen": 2565734400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011243731193580742, + "loss": 1.2283, + "theoretical_loss": 3.3615612343605257, + "tokens_seen": 2565865472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011241725175526581, + "loss": 1.2317, + "theoretical_loss": 3.3615476086520593, + "tokens_seen": 2565996544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011239719157472418, + "loss": 1.2201, + "theoretical_loss": 3.3615339838344513, + "tokens_seen": 2566127616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011237713139418256, + "loss": 1.2248, + "theoretical_loss": 3.3615203599075985, + "tokens_seen": 2566258688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011235707121364094, + "loss": 1.2893, + "theoretical_loss": 3.3615067368713962, + "tokens_seen": 2566389760 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001123370110330993, + "loss": 1.136, + "theoretical_loss": 3.361493114725741, + "tokens_seen": 2566520832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011231695085255767, + "loss": 1.2182, + "theoretical_loss": 3.36147949347053, + "tokens_seen": 2566651904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011229689067201605, + "loss": 1.3991, + "theoretical_loss": 3.361465873105659, + "tokens_seen": 2566782976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011227683049147443, + "loss": 1.2842, + "theoretical_loss": 3.3614522536310236, + "tokens_seen": 2566914048 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001122567703109328, + "loss": 1.3469, + "theoretical_loss": 3.3614386350465213, + "tokens_seen": 2567045120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011223671013039118, + "loss": 1.095, + "theoretical_loss": 3.361425017352048, + "tokens_seen": 2567176192 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011221664994984955, + "loss": 1.4883, + "theoretical_loss": 3.3614114005475004, + "tokens_seen": 2567307264 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.3519672453403473, + "objective/train/avg_token_score": 0.010499109514057636, + "objective/train/docs_used": 1444461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9673712253570557, + "objective/train/original_loss": 3.8915281295776367, + "objective/train/theoretical_loss": 3.361404592478916, + "objective/train/tokens_used": 2567372800, + "objective/train/unlikelihood_loss": 0.2168387472629547, + "theoretical_loss": 3.361404592478916, + "tokens_seen": 2567372800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011219658976930793, + "loss": 1.2302, + "theoretical_loss": 3.361397784632774, + "tokens_seen": 2567438336 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001121765295887663, + "loss": 1.3076, + "theoretical_loss": 3.361384169607766, + "tokens_seen": 2567569408 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011215646940822467, + "loss": 1.1942, + "theoretical_loss": 3.3613705554723734, + "tokens_seen": 2567700480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011213640922768304, + "loss": 1.3994, + "theoretical_loss": 3.3613569422264913, + "tokens_seen": 2567831552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011211634904714142, + "loss": 1.4909, + "theoretical_loss": 3.3613433298700173, + "tokens_seen": 2567962624 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001120962888665998, + "loss": 1.2244, + "theoretical_loss": 3.361329718402847, + "tokens_seen": 2568093696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011207622868605817, + "loss": 1.2718, + "theoretical_loss": 3.361316107824878, + "tokens_seen": 2568224768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011205616850551655, + "loss": 1.1067, + "theoretical_loss": 3.3613024981360056, + "tokens_seen": 2568355840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011203610832497494, + "loss": 1.2004, + "theoretical_loss": 3.3612888893361275, + "tokens_seen": 2568486912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011201604814443331, + "loss": 1.3654, + "theoretical_loss": 3.361275281425139, + "tokens_seen": 2568617984 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011199598796389169, + "loss": 1.4563, + "theoretical_loss": 3.361261674402938, + "tokens_seen": 2568749056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011197592778335006, + "loss": 1.2855, + "theoretical_loss": 3.3612480682694206, + "tokens_seen": 2568880128 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.5086143612861633, + "objective/train/avg_token_score": 0.007483049761503935, + "objective/train/docs_used": 1445073, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6197497844696045, + "objective/train/original_loss": 3.6387557983398438, + "objective/train/theoretical_loss": 3.361234463024483, + "objective/train/tokens_used": 2569011200, + "objective/train/unlikelihood_loss": 0.16943007707595825, + "theoretical_loss": 3.361234463024483, + "tokens_seen": 2569011200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011195586760280843, + "loss": 1.1472, + "theoretical_loss": 3.361234463024483, + "tokens_seen": 2569011200 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001119358074222668, + "loss": 1.3251, + "theoretical_loss": 3.3612208586680223, + "tokens_seen": 2569142272 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011191574724172518, + "loss": 1.2813, + "theoretical_loss": 3.3612072551999352, + "tokens_seen": 2569273344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011189568706118355, + "loss": 1.1977, + "theoretical_loss": 3.361193652620118, + "tokens_seen": 2569404416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011187562688064193, + "loss": 1.1587, + "theoretical_loss": 3.3611800509284677, + "tokens_seen": 2569535488 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001118555667001003, + "loss": 1.2189, + "theoretical_loss": 3.361166450124881, + "tokens_seen": 2569666560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011183550651955868, + "loss": 1.207, + "theoretical_loss": 3.361152850209254, + "tokens_seen": 2569797632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011181544633901706, + "loss": 1.2843, + "theoretical_loss": 3.361139251181484, + "tokens_seen": 2569928704 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011179538615847543, + "loss": 1.1545, + "theoretical_loss": 3.3611256530414684, + "tokens_seen": 2570059776 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011177532597793381, + "loss": 1.2742, + "theoretical_loss": 3.361112055789103, + "tokens_seen": 2570190848 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011175526579739217, + "loss": 1.3928, + "theoretical_loss": 3.361098459424285, + "tokens_seen": 2570321920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011173520561685055, + "loss": 1.276, + "theoretical_loss": 3.361084863946911, + "tokens_seen": 2570452992 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011171514543630892, + "loss": 1.2395, + "theoretical_loss": 3.361071269356878, + "tokens_seen": 2570584064 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.3152492642402649, + "objective/train/avg_token_score": 0.044343333691358566, + "objective/train/docs_used": 1445848, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8649482727050781, + "objective/train/original_loss": 3.6451451778411865, + "objective/train/theoretical_loss": 3.361064472394582, + "objective/train/tokens_used": 2570649600, + "objective/train/unlikelihood_loss": 0.20206168293952942, + "theoretical_loss": 3.361064472394582, + "tokens_seen": 2570649600 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001116950852557673, + "loss": 1.2447, + "theoretical_loss": 3.3610576756540826, + "tokens_seen": 2570715136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011167502507522567, + "loss": 1.2233, + "theoretical_loss": 3.361044082838422, + "tokens_seen": 2570846208 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011165496489468406, + "loss": 1.4482, + "theoretical_loss": 3.361030490909793, + "tokens_seen": 2570977280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011163490471414244, + "loss": 1.3575, + "theoretical_loss": 3.3610168998680923, + "tokens_seen": 2571108352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011161484453360081, + "loss": 1.3293, + "theoretical_loss": 3.3610033097132166, + "tokens_seen": 2571239424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011159478435305919, + "loss": 1.3539, + "theoretical_loss": 3.3609897204450636, + "tokens_seen": 2571370496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011157472417251755, + "loss": 1.355, + "theoretical_loss": 3.36097613206353, + "tokens_seen": 2571501568 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011155466399197593, + "loss": 1.5741, + "theoretical_loss": 3.3609625445685123, + "tokens_seen": 2571632640 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001115346038114343, + "loss": 1.4358, + "theoretical_loss": 3.3609489579599074, + "tokens_seen": 2571763712 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011151454363089268, + "loss": 1.2632, + "theoretical_loss": 3.360935372237613, + "tokens_seen": 2571894784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011149448345035105, + "loss": 1.3682, + "theoretical_loss": 3.3609217874015265, + "tokens_seen": 2572025856 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011147442326980943, + "loss": 1.2737, + "theoretical_loss": 3.360908203451543, + "tokens_seen": 2572156928 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.5008553266525269, + "objective/train/avg_token_score": 0.010067198425531387, + "objective/train/docs_used": 1447125, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5513017177581787, + "objective/train/original_loss": 3.54445743560791, + "objective/train/theoretical_loss": 3.3608946203875614, + "objective/train/tokens_used": 2572288000, + "objective/train/unlikelihood_loss": 0.1767706423997879, + "theoretical_loss": 3.3608946203875614, + "tokens_seen": 2572288000 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001114543630892678, + "loss": 1.3466, + "theoretical_loss": 3.3608946203875614, + "tokens_seen": 2572288000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011143430290872618, + "loss": 1.1549, + "theoretical_loss": 3.3608810382094783, + "tokens_seen": 2572419072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011141424272818456, + "loss": 1.5145, + "theoretical_loss": 3.36086745691719, + "tokens_seen": 2572550144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011139418254764293, + "loss": 1.2059, + "theoretical_loss": 3.360853876510595, + "tokens_seen": 2572681216 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001113741223671013, + "loss": 1.261, + "theoretical_loss": 3.360840296989589, + "tokens_seen": 2572812288 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011135406218655967, + "loss": 1.1153, + "theoretical_loss": 3.3608267183540703, + "tokens_seen": 2572943360 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011133400200601805, + "loss": 1.2929, + "theoretical_loss": 3.360813140603935, + "tokens_seen": 2573074432 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011131394182547642, + "loss": 1.2461, + "theoretical_loss": 3.3607995637390813, + "tokens_seen": 2573205504 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011129388164493481, + "loss": 1.3527, + "theoretical_loss": 3.3607859877594057, + "tokens_seen": 2573336576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011127382146439319, + "loss": 1.3209, + "theoretical_loss": 3.3607724126648058, + "tokens_seen": 2573467648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011125376128385156, + "loss": 1.2861, + "theoretical_loss": 3.3607588384551788, + "tokens_seen": 2573598720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011123370110330994, + "loss": 1.3001, + "theoretical_loss": 3.3607452651304213, + "tokens_seen": 2573729792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011121364092276832, + "loss": 1.4016, + "theoretical_loss": 3.360731692690431, + "tokens_seen": 2573860864 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.4059750735759735, + "objective/train/avg_token_score": 0.015989920124411583, + "objective/train/docs_used": 1447717, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.183982253074646, + "objective/train/original_loss": 3.9272639751434326, + "objective/train/theoretical_loss": 3.360724906802192, + "objective/train/tokens_used": 2573926400, + "objective/train/unlikelihood_loss": 0.17380347847938538, + "theoretical_loss": 3.360724906802192, + "tokens_seen": 2573926400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011119358074222669, + "loss": 1.3014, + "theoretical_loss": 3.3607181211351054, + "tokens_seen": 2573991936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011117352056168505, + "loss": 1.3797, + "theoretical_loss": 3.360704550464342, + "tokens_seen": 2574123008 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011115346038114343, + "loss": 1.474, + "theoretical_loss": 3.360690980678037, + "tokens_seen": 2574254080 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001111334002006018, + "loss": 1.2468, + "theoretical_loss": 3.3606774117760887, + "tokens_seen": 2574385152 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011111334002006018, + "loss": 1.3243, + "theoretical_loss": 3.3606638437583944, + "tokens_seen": 2574516224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011109327983951856, + "loss": 1.2334, + "theoretical_loss": 3.3606502766248507, + "tokens_seen": 2574647296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011107321965897693, + "loss": 1.1881, + "theoretical_loss": 3.3606367103753563, + "tokens_seen": 2574778368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011105315947843531, + "loss": 1.3015, + "theoretical_loss": 3.360623145009807, + "tokens_seen": 2574909440 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011103309929789368, + "loss": 1.2277, + "theoretical_loss": 3.3606095805281018, + "tokens_seen": 2575040512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011101303911735206, + "loss": 1.2762, + "theoretical_loss": 3.360596016930137, + "tokens_seen": 2575171584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011099297893681042, + "loss": 1.5014, + "theoretical_loss": 3.3605824542158107, + "tokens_seen": 2575302656 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001109729187562688, + "loss": 1.2975, + "theoretical_loss": 3.3605688923850194, + "tokens_seen": 2575433728 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.5045210123062134, + "objective/train/avg_token_score": 0.03916860744357109, + "objective/train/docs_used": 1449217, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4172333478927612, + "objective/train/original_loss": 4.122230529785156, + "objective/train/theoretical_loss": 3.360555331437662, + "objective/train/tokens_used": 2575564800, + "objective/train/unlikelihood_loss": 0.14812655746936798, + "theoretical_loss": 3.360555331437662, + "tokens_seen": 2575564800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011095285857572717, + "loss": 1.1578, + "theoretical_loss": 3.360555331437662, + "tokens_seen": 2575564800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011093279839518555, + "loss": 1.1771, + "theoretical_loss": 3.3605417713736347, + "tokens_seen": 2575695872 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011091273821464394, + "loss": 1.285, + "theoretical_loss": 3.3605282121928357, + "tokens_seen": 2575826944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011089267803410231, + "loss": 1.419, + "theoretical_loss": 3.3605146538951627, + "tokens_seen": 2575958016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011087261785356069, + "loss": 1.1942, + "theoretical_loss": 3.3605010964805127, + "tokens_seen": 2576089088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011085255767301907, + "loss": 1.1749, + "theoretical_loss": 3.3604875399487835, + "tokens_seen": 2576220160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011083249749247744, + "loss": 1.2854, + "theoretical_loss": 3.3604739842998725, + "tokens_seen": 2576351232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011081243731193582, + "loss": 1.0974, + "theoretical_loss": 3.3604604295336777, + "tokens_seen": 2576482304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011079237713139418, + "loss": 1.3964, + "theoretical_loss": 3.360446875650097, + "tokens_seen": 2576613376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011077231695085256, + "loss": 1.3453, + "theoretical_loss": 3.360433322649027, + "tokens_seen": 2576744448 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011075225677031093, + "loss": 1.1852, + "theoretical_loss": 3.360419770530366, + "tokens_seen": 2576875520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011073219658976931, + "loss": 1.1376, + "theoretical_loss": 3.3604062192940116, + "tokens_seen": 2577006592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011071213640922768, + "loss": 1.2318, + "theoretical_loss": 3.3603926689398613, + "tokens_seen": 2577137664 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.5092253088951111, + "objective/train/avg_token_score": 0.009099521674215794, + "objective/train/docs_used": 1449913, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.625362753868103, + "objective/train/original_loss": 3.6376075744628906, + "objective/train/theoretical_loss": 3.360385894093581, + "objective/train/tokens_used": 2577203200, + "objective/train/unlikelihood_loss": 0.16667605936527252, + "theoretical_loss": 3.360385894093581, + "tokens_seen": 2577203200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011069207622868606, + "loss": 1.2885, + "theoretical_loss": 3.360379119467813, + "tokens_seen": 2577268736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011067201604814444, + "loss": 1.2468, + "theoretical_loss": 3.3603655708777644, + "tokens_seen": 2577399808 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011065195586760281, + "loss": 1.2676, + "theoretical_loss": 3.3603520231696136, + "tokens_seen": 2577530880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011063189568706119, + "loss": 1.1971, + "theoretical_loss": 3.3603384763432578, + "tokens_seen": 2577661952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011061183550651956, + "loss": 1.2131, + "theoretical_loss": 3.3603249303985945, + "tokens_seen": 2577793024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011059177532597792, + "loss": 1.3485, + "theoretical_loss": 3.3603113853355224, + "tokens_seen": 2577924096 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001105717151454363, + "loss": 1.3375, + "theoretical_loss": 3.3602978411539386, + "tokens_seen": 2578055168 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011055165496489468, + "loss": 1.1413, + "theoretical_loss": 3.3602842978537413, + "tokens_seen": 2578186240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011053159478435307, + "loss": 1.2503, + "theoretical_loss": 3.3602707554348283, + "tokens_seen": 2578317312 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011051153460381144, + "loss": 1.2898, + "theoretical_loss": 3.360257213897097, + "tokens_seen": 2578448384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011049147442326982, + "loss": 1.3819, + "theoretical_loss": 3.360243673240446, + "tokens_seen": 2578579456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011047141424272819, + "loss": 1.302, + "theoretical_loss": 3.3602301334647726, + "tokens_seen": 2578710528 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.41684994101524353, + "objective/train/avg_token_score": 0.020269716158509254, + "objective/train/docs_used": 1450481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2873060703277588, + "objective/train/original_loss": 3.96406626701355, + "objective/train/theoretical_loss": 3.360216594569975, + "objective/train/tokens_used": 2578841600, + "objective/train/unlikelihood_loss": 0.1995953619480133, + "theoretical_loss": 3.360216594569975, + "tokens_seen": 2578841600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011045135406218657, + "loss": 1.36, + "theoretical_loss": 3.360216594569975, + "tokens_seen": 2578841600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011043129388164494, + "loss": 1.3138, + "theoretical_loss": 3.3602030565559513, + "tokens_seen": 2578972672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011041123370110332, + "loss": 1.1604, + "theoretical_loss": 3.360189519422599, + "tokens_seen": 2579103744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011039117352056168, + "loss": 1.1196, + "theoretical_loss": 3.3601759831698157, + "tokens_seen": 2579234816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011037111334002006, + "loss": 1.1831, + "theoretical_loss": 3.3601624477975007, + "tokens_seen": 2579365888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011035105315947843, + "loss": 1.2788, + "theoretical_loss": 3.360148913305551, + "tokens_seen": 2579496960 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011033099297893681, + "loss": 1.2186, + "theoretical_loss": 3.3601353796938644, + "tokens_seen": 2579628032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011031093279839519, + "loss": 1.1381, + "theoretical_loss": 3.36012184696234, + "tokens_seen": 2579759104 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011029087261785356, + "loss": 1.3314, + "theoretical_loss": 3.360108315110875, + "tokens_seen": 2579890176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011027081243731194, + "loss": 1.3649, + "theoretical_loss": 3.360094784139368, + "tokens_seen": 2580021248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011025075225677031, + "loss": 1.2391, + "theoretical_loss": 3.3600812540477163, + "tokens_seen": 2580152320 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011023069207622869, + "loss": 1.353, + "theoretical_loss": 3.3600677248358184, + "tokens_seen": 2580283392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011021063189568705, + "loss": 1.2713, + "theoretical_loss": 3.3600541965035724, + "tokens_seen": 2580414464 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.395772248506546, + "objective/train/avg_token_score": 0.010340644977986813, + "objective/train/docs_used": 1451835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.082930088043213, + "objective/train/original_loss": 3.481045722961426, + "objective/train/theoretical_loss": 3.360047432667287, + "objective/train/tokens_used": 2580480000, + "objective/train/unlikelihood_loss": 0.20052585005760193, + "theoretical_loss": 3.360047432667287, + "tokens_seen": 2580480000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011019057171514543, + "loss": 1.1879, + "theoretical_loss": 3.3600406690508766, + "tokens_seen": 2580545536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011017051153460382, + "loss": 1.1363, + "theoretical_loss": 3.360027142477629, + "tokens_seen": 2580676608 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011015045135406219, + "loss": 1.1835, + "theoretical_loss": 3.360013616783728, + "tokens_seen": 2580807680 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011013039117352057, + "loss": 1.1197, + "theoretical_loss": 3.3600000919690713, + "tokens_seen": 2580938752 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011011033099297894, + "loss": 1.2999, + "theoretical_loss": 3.359986568033557, + "tokens_seen": 2581069824 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011009027081243732, + "loss": 1.1988, + "theoretical_loss": 3.3599730449770844, + "tokens_seen": 2581200896 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001100702106318957, + "loss": 1.1322, + "theoretical_loss": 3.359959522799551, + "tokens_seen": 2581331968 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011005015045135407, + "loss": 1.4326, + "theoretical_loss": 3.359946001500855, + "tokens_seen": 2581463040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011003009027081245, + "loss": 1.2779, + "theoretical_loss": 3.3599324810808944, + "tokens_seen": 2581594112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011001003009027081, + "loss": 1.2457, + "theoretical_loss": 3.3599189615395675, + "tokens_seen": 2581725184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010998996990972919, + "loss": 1.3914, + "theoretical_loss": 3.359905442876774, + "tokens_seen": 2581856256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010996990972918756, + "loss": 1.3318, + "theoretical_loss": 3.35989192509241, + "tokens_seen": 2581987328 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.39962121844291687, + "objective/train/avg_token_score": 0.009787692688405514, + "objective/train/docs_used": 1452541, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2846267223358154, + "objective/train/original_loss": 3.5875895023345947, + "objective/train/theoretical_loss": 3.3598784081863755, + "objective/train/tokens_used": 2582118400, + "objective/train/unlikelihood_loss": 0.16958364844322205, + "theoretical_loss": 3.3598784081863755, + "tokens_seen": 2582118400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010994984954864594, + "loss": 1.267, + "theoretical_loss": 3.3598784081863755, + "tokens_seen": 2582118400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010992978936810431, + "loss": 1.3033, + "theoretical_loss": 3.359864892158568, + "tokens_seen": 2582249472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010990972918756269, + "loss": 1.3331, + "theoretical_loss": 3.3598513770088863, + "tokens_seen": 2582380544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010988966900702106, + "loss": 1.24, + "theoretical_loss": 3.3598378627372285, + "tokens_seen": 2582511616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010986960882647944, + "loss": 1.2994, + "theoretical_loss": 3.3598243493434934, + "tokens_seen": 2582642688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010984954864593782, + "loss": 1.2584, + "theoretical_loss": 3.359810836827579, + "tokens_seen": 2582773760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010982948846539619, + "loss": 1.3113, + "theoretical_loss": 3.3597973251893842, + "tokens_seen": 2582904832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010980942828485455, + "loss": 1.2481, + "theoretical_loss": 3.359783814428807, + "tokens_seen": 2583035904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010978936810431294, + "loss": 1.4662, + "theoretical_loss": 3.3597703045457457, + "tokens_seen": 2583166976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010976930792377132, + "loss": 1.3419, + "theoretical_loss": 3.359756795540099, + "tokens_seen": 2583298048 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001097492477432297, + "loss": 1.1711, + "theoretical_loss": 3.359743287411766, + "tokens_seen": 2583429120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010972918756268807, + "loss": 1.3321, + "theoretical_loss": 3.3597297801606443, + "tokens_seen": 2583560192 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010970912738214645, + "loss": 1.3385, + "theoretical_loss": 3.359716273786633, + "tokens_seen": 2583691264 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.34787389636039734, + "objective/train/avg_token_score": 0.012733491137623787, + "objective/train/docs_used": 1453825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.064862847328186, + "objective/train/original_loss": 3.439624786376953, + "objective/train/theoretical_loss": 3.3597095209285124, + "objective/train/tokens_used": 2583756800, + "objective/train/unlikelihood_loss": 0.21951137483119965, + "theoretical_loss": 3.3597095209285124, + "tokens_seen": 2583756800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010968906720160482, + "loss": 1.1249, + "theoretical_loss": 3.3597027682896305, + "tokens_seen": 2583822336 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001096690070210632, + "loss": 1.1405, + "theoretical_loss": 3.3596892636695355, + "tokens_seen": 2583953408 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010964894684052157, + "loss": 1.153, + "theoretical_loss": 3.3596757599262466, + "tokens_seen": 2584084480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010962888665997995, + "loss": 1.3493, + "theoretical_loss": 3.3596622570596617, + "tokens_seen": 2584215552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010960882647943831, + "loss": 1.3305, + "theoretical_loss": 3.3596487550696805, + "tokens_seen": 2584346624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010958876629889669, + "loss": 1.287, + "theoretical_loss": 3.359635253956201, + "tokens_seen": 2584477696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010956870611835506, + "loss": 1.2687, + "theoretical_loss": 3.3596217537191215, + "tokens_seen": 2584608768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010954864593781344, + "loss": 1.0293, + "theoretical_loss": 3.359608254358342, + "tokens_seen": 2584739840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010952858575727181, + "loss": 1.3937, + "theoretical_loss": 3.3595947558737596, + "tokens_seen": 2584870912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010950852557673019, + "loss": 1.1197, + "theoretical_loss": 3.359581258265274, + "tokens_seen": 2585001984 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010948846539618857, + "loss": 1.2347, + "theoretical_loss": 3.3595677615327837, + "tokens_seen": 2585133056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010946840521564694, + "loss": 1.3784, + "theoretical_loss": 3.359554265676187, + "tokens_seen": 2585264128 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.36895161867141724, + "objective/train/avg_token_score": 0.01553166564553976, + "objective/train/docs_used": 1454456, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1367270946502686, + "objective/train/original_loss": 4.041872978210449, + "objective/train/theoretical_loss": 3.3595407706953835, + "objective/train/tokens_used": 2585395200, + "objective/train/unlikelihood_loss": 0.18689784407615662, + "theoretical_loss": 3.3595407706953835, + "tokens_seen": 2585395200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010944834503510532, + "loss": 1.2276, + "theoretical_loss": 3.3595407706953835, + "tokens_seen": 2585395200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010942828485456368, + "loss": 1.2024, + "theoretical_loss": 3.3595272765902715, + "tokens_seen": 2585526272 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010940822467402207, + "loss": 1.0717, + "theoretical_loss": 3.3595137833607494, + "tokens_seen": 2585657344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010938816449348045, + "loss": 1.2206, + "theoretical_loss": 3.3595002910067167, + "tokens_seen": 2585788416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010936810431293882, + "loss": 1.1037, + "theoretical_loss": 3.3594867995280717, + "tokens_seen": 2585919488 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001093480441323972, + "loss": 1.0541, + "theoretical_loss": 3.3594733089247137, + "tokens_seen": 2586050560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010932798395185557, + "loss": 1.258, + "theoretical_loss": 3.359459819196541, + "tokens_seen": 2586181632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010930792377131395, + "loss": 1.2846, + "theoretical_loss": 3.359446330343453, + "tokens_seen": 2586312704 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010928786359077232, + "loss": 1.3151, + "theoretical_loss": 3.3594328423653486, + "tokens_seen": 2586443776 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001092678034102307, + "loss": 1.1808, + "theoretical_loss": 3.3594193552621263, + "tokens_seen": 2586574848 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010924774322968908, + "loss": 1.0946, + "theoretical_loss": 3.359405869033685, + "tokens_seen": 2586705920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010922768304914744, + "loss": 1.1533, + "theoretical_loss": 3.359392383679924, + "tokens_seen": 2586836992 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010920762286860581, + "loss": 1.3026, + "theoretical_loss": 3.3593788992007423, + "tokens_seen": 2586968064 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.5175341963768005, + "objective/train/avg_token_score": 0.007690670900046825, + "objective/train/docs_used": 1455631, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5911283493041992, + "objective/train/original_loss": 3.458420515060425, + "objective/train/theoretical_loss": 3.359372157289087, + "objective/train/tokens_used": 2587033600, + "objective/train/unlikelihood_loss": 0.1894473135471344, + "theoretical_loss": 3.359372157289087, + "tokens_seen": 2587033600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010918756268806419, + "loss": 1.3791, + "theoretical_loss": 3.3593654155960384, + "tokens_seen": 2587099136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010916750250752257, + "loss": 1.2659, + "theoretical_loss": 3.3593519328657115, + "tokens_seen": 2587230208 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010914744232698094, + "loss": 1.2363, + "theoretical_loss": 3.359338451009661, + "tokens_seen": 2587361280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010912738214643932, + "loss": 1.3706, + "theoretical_loss": 3.359324970027785, + "tokens_seen": 2587492352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010910732196589769, + "loss": 1.2612, + "theoretical_loss": 3.3593114899199836, + "tokens_seen": 2587623424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010908726178535607, + "loss": 1.2566, + "theoretical_loss": 3.359298010686156, + "tokens_seen": 2587754496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010906720160481444, + "loss": 1.3484, + "theoretical_loss": 3.3592845323261997, + "tokens_seen": 2587885568 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010904714142427282, + "loss": 1.3155, + "theoretical_loss": 3.359271054840015, + "tokens_seen": 2588016640 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001090270812437312, + "loss": 1.3295, + "theoretical_loss": 3.359257578227501, + "tokens_seen": 2588147712 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010900702106318957, + "loss": 1.281, + "theoretical_loss": 3.3592441024885558, + "tokens_seen": 2588278784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010898696088264795, + "loss": 1.2725, + "theoretical_loss": 3.35923062762308, + "tokens_seen": 2588409856 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010896690070210632, + "loss": 1.2568, + "theoretical_loss": 3.359217153630972, + "tokens_seen": 2588540928 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.41306206583976746, + "objective/train/avg_token_score": 0.014502700418233871, + "objective/train/docs_used": 1456190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2982553243637085, + "objective/train/original_loss": 3.931485414505005, + "objective/train/theoretical_loss": 3.359203680512131, + "objective/train/tokens_used": 2588672000, + "objective/train/unlikelihood_loss": 0.15496826171875, + "theoretical_loss": 3.359203680512131, + "tokens_seen": 2588672000 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001089468405215647, + "loss": 1.2343, + "theoretical_loss": 3.359203680512131, + "tokens_seen": 2588672000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010892678034102307, + "loss": 1.3425, + "theoretical_loss": 3.3591902082664564, + "tokens_seen": 2588803072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010890672016048145, + "loss": 1.162, + "theoretical_loss": 3.359176736893847, + "tokens_seen": 2588934144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010888665997993983, + "loss": 1.2154, + "theoretical_loss": 3.3591632663942024, + "tokens_seen": 2589065216 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001088665997993982, + "loss": 1.406, + "theoretical_loss": 3.3591497967674218, + "tokens_seen": 2589196288 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010884653961885658, + "loss": 1.366, + "theoretical_loss": 3.3591363280134043, + "tokens_seen": 2589327360 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010882647943831494, + "loss": 1.1351, + "theoretical_loss": 3.3591228601320493, + "tokens_seen": 2589458432 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010880641925777332, + "loss": 1.306, + "theoretical_loss": 3.359109393123256, + "tokens_seen": 2589589504 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010878635907723169, + "loss": 1.1429, + "theoretical_loss": 3.3590959269869236, + "tokens_seen": 2589720576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010876629889669007, + "loss": 1.3221, + "theoretical_loss": 3.3590824617229513, + "tokens_seen": 2589851648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010874623871614844, + "loss": 1.3948, + "theoretical_loss": 3.3590689973312395, + "tokens_seen": 2589982720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010872617853560682, + "loss": 1.3335, + "theoretical_loss": 3.359055533811686, + "tokens_seen": 2590113792 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001087061183550652, + "loss": 1.185, + "theoretical_loss": 3.3590420711641915, + "tokens_seen": 2590244864 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.5476539731025696, + "objective/train/avg_token_score": 0.005745676811784506, + "objective/train/docs_used": 1457612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6523079872131348, + "objective/train/original_loss": 3.4287960529327393, + "objective/train/theoretical_loss": 3.3590353401674347, + "objective/train/tokens_used": 2590310400, + "objective/train/unlikelihood_loss": 0.1287716180086136, + "theoretical_loss": 3.3590353401674347, + "tokens_seen": 2590310400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010868605817452357, + "loss": 1.3108, + "theoretical_loss": 3.3590286093886546, + "tokens_seen": 2590375936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010866599799398196, + "loss": 1.2629, + "theoretical_loss": 3.359015148484975, + "tokens_seen": 2590507008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010864593781344032, + "loss": 1.1298, + "theoretical_loss": 3.359001688453052, + "tokens_seen": 2590638080 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001086258776328987, + "loss": 1.4447, + "theoretical_loss": 3.3589882292927853, + "tokens_seen": 2590769152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010860581745235707, + "loss": 1.2258, + "theoretical_loss": 3.358974771004074, + "tokens_seen": 2590900224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010858575727181545, + "loss": 1.3204, + "theoretical_loss": 3.3589613135868177, + "tokens_seen": 2591031296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010856569709127383, + "loss": 1.5175, + "theoretical_loss": 3.358947857040916, + "tokens_seen": 2591162368 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001085456369107322, + "loss": 1.1911, + "theoretical_loss": 3.3589344013662683, + "tokens_seen": 2591293440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010852557673019058, + "loss": 1.2816, + "theoretical_loss": 3.3589209465627743, + "tokens_seen": 2591424512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010850551654964895, + "loss": 1.3031, + "theoretical_loss": 3.3589074926303333, + "tokens_seen": 2591555584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010848545636910733, + "loss": 1.2183, + "theoretical_loss": 3.358894039568845, + "tokens_seen": 2591686656 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001084653961885657, + "loss": 1.1808, + "theoretical_loss": 3.358880587378209, + "tokens_seen": 2591817728 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.35306695103645325, + "objective/train/avg_token_score": 0.011546467430889606, + "objective/train/docs_used": 1458174, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.044063687324524, + "objective/train/original_loss": 3.7962114810943604, + "objective/train/theoretical_loss": 3.3588671360583247, + "objective/train/tokens_used": 2591948800, + "objective/train/unlikelihood_loss": 0.1604221612215042, + "theoretical_loss": 3.3588671360583247, + "tokens_seen": 2591948800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010844533600802407, + "loss": 1.1855, + "theoretical_loss": 3.3588671360583247, + "tokens_seen": 2591948800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010842527582748244, + "loss": 1.1375, + "theoretical_loss": 3.358853685609092, + "tokens_seen": 2592079872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010840521564694082, + "loss": 1.2837, + "theoretical_loss": 3.35884023603041, + "tokens_seen": 2592210944 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001083851554663992, + "loss": 1.1384, + "theoretical_loss": 3.358826787322179, + "tokens_seen": 2592342016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010836509528585757, + "loss": 1.2695, + "theoretical_loss": 3.3588133394842985, + "tokens_seen": 2592473088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010834503510531595, + "loss": 1.3418, + "theoretical_loss": 3.3587998925166676, + "tokens_seen": 2592604160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010832497492477432, + "loss": 1.2765, + "theoretical_loss": 3.3587864464191868, + "tokens_seen": 2592735232 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001083049147442327, + "loss": 1.2882, + "theoretical_loss": 3.3587730011917554, + "tokens_seen": 2592866304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010828485456369109, + "loss": 1.2846, + "theoretical_loss": 3.358759556834273, + "tokens_seen": 2592997376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010826479438314946, + "loss": 1.2788, + "theoretical_loss": 3.3587461133466396, + "tokens_seen": 2593128448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010824473420260782, + "loss": 1.3581, + "theoretical_loss": 3.358732670728755, + "tokens_seen": 2593259520 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001082246740220662, + "loss": 1.2716, + "theoretical_loss": 3.358719228980519, + "tokens_seen": 2593390592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010820461384152458, + "loss": 1.3594, + "theoretical_loss": 3.3587057881018305, + "tokens_seen": 2593521664 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.5756964683532715, + "objective/train/avg_token_score": 0.018810542300343513, + "objective/train/docs_used": 1459433, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.877524971961975, + "objective/train/original_loss": 3.410343647003174, + "objective/train/theoretical_loss": 3.358699067988536, + "objective/train/tokens_used": 2593587200, + "objective/train/unlikelihood_loss": 0.18468831479549408, + "theoretical_loss": 3.358699067988536, + "tokens_seen": 2593587200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010818455366098295, + "loss": 1.3875, + "theoretical_loss": 3.3586923480925908, + "tokens_seen": 2593652736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010816449348044133, + "loss": 1.2128, + "theoretical_loss": 3.3586789089526983, + "tokens_seen": 2593783808 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001081444332998997, + "loss": 1.3106, + "theoretical_loss": 3.358665470682054, + "tokens_seen": 2593914880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010812437311935808, + "loss": 1.1568, + "theoretical_loss": 3.358652033280557, + "tokens_seen": 2594045952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010810431293881646, + "loss": 1.3059, + "theoretical_loss": 3.3586385967481074, + "tokens_seen": 2594177024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010808425275827483, + "loss": 1.1654, + "theoretical_loss": 3.358625161084605, + "tokens_seen": 2594308096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010806419257773321, + "loss": 1.3369, + "theoretical_loss": 3.3586117262899506, + "tokens_seen": 2594439168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010804413239719157, + "loss": 1.3427, + "theoretical_loss": 3.358598292364043, + "tokens_seen": 2594570240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010802407221664995, + "loss": 1.1099, + "theoretical_loss": 3.3585848593067817, + "tokens_seen": 2594701312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010800401203610832, + "loss": 1.2244, + "theoretical_loss": 3.3585714271180684, + "tokens_seen": 2594832384 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001079839518555667, + "loss": 1.2521, + "theoretical_loss": 3.358557995797802, + "tokens_seen": 2594963456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010796389167502507, + "loss": 1.2561, + "theoretical_loss": 3.3585445653458823, + "tokens_seen": 2595094528 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.44751954078674316, + "objective/train/avg_token_score": 0.03019137494266033, + "objective/train/docs_used": 1459948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4038395881652832, + "objective/train/original_loss": 4.0312652587890625, + "objective/train/theoretical_loss": 3.3585311357622096, + "objective/train/tokens_used": 2595225600, + "objective/train/unlikelihood_loss": 0.11123685538768768, + "theoretical_loss": 3.3585311357622096, + "tokens_seen": 2595225600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010794383149448345, + "loss": 1.2098, + "theoretical_loss": 3.3585311357622096, + "tokens_seen": 2595225600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010792377131394182, + "loss": 1.3743, + "theoretical_loss": 3.3585177070466843, + "tokens_seen": 2595356672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010790371113340021, + "loss": 1.3057, + "theoretical_loss": 3.358504279199206, + "tokens_seen": 2595487744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010788365095285859, + "loss": 1.2195, + "theoretical_loss": 3.358490852219674, + "tokens_seen": 2595618816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010786359077231695, + "loss": 1.3085, + "theoretical_loss": 3.3584774261079904, + "tokens_seen": 2595749888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010784353059177533, + "loss": 1.4292, + "theoretical_loss": 3.3584640008640534, + "tokens_seen": 2595880960 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001078234704112337, + "loss": 1.2915, + "theoretical_loss": 3.358450576487764, + "tokens_seen": 2596012032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010780341023069208, + "loss": 1.3972, + "theoretical_loss": 3.3584371529790222, + "tokens_seen": 2596143104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010778335005015045, + "loss": 1.3121, + "theoretical_loss": 3.3584237303377282, + "tokens_seen": 2596274176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010776328986960883, + "loss": 1.2515, + "theoretical_loss": 3.3584103085637818, + "tokens_seen": 2596405248 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001077432296890672, + "loss": 1.1843, + "theoretical_loss": 3.3583968876570833, + "tokens_seen": 2596536320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010772316950852558, + "loss": 1.3516, + "theoretical_loss": 3.3583834676175335, + "tokens_seen": 2596667392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010770310932798396, + "loss": 1.2234, + "theoretical_loss": 3.358370048445032, + "tokens_seen": 2596798464 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.40285924077033997, + "objective/train/avg_token_score": 0.022873464971780777, + "objective/train/docs_used": 1461323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1946017742156982, + "objective/train/original_loss": 3.6913986206054688, + "objective/train/theoretical_loss": 3.3583633391838927, + "objective/train/tokens_used": 2596864000, + "objective/train/unlikelihood_loss": 0.1705203652381897, + "theoretical_loss": 3.3583633391838927, + "tokens_seen": 2596864000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010768304914744233, + "loss": 1.1481, + "theoretical_loss": 3.3583566301394785, + "tokens_seen": 2596929536 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001076629889669007, + "loss": 1.1669, + "theoretical_loss": 3.358343212700774, + "tokens_seen": 2597060608 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010764292878635907, + "loss": 1.3996, + "theoretical_loss": 3.358329796128819, + "tokens_seen": 2597191680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010762286860581745, + "loss": 1.15, + "theoretical_loss": 3.358316380423513, + "tokens_seen": 2597322752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010760280842527582, + "loss": 1.2336, + "theoretical_loss": 3.358302965584757, + "tokens_seen": 2597453824 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001075827482447342, + "loss": 1.3447, + "theoretical_loss": 3.358289551612451, + "tokens_seen": 2597584896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010756268806419257, + "loss": 1.1592, + "theoretical_loss": 3.358276138506495, + "tokens_seen": 2597715968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010754262788365096, + "loss": 1.2787, + "theoretical_loss": 3.3582627262667897, + "tokens_seen": 2597847040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010752256770310934, + "loss": 1.1467, + "theoretical_loss": 3.3582493148932357, + "tokens_seen": 2597978112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010750250752256772, + "loss": 1.3824, + "theoretical_loss": 3.3582359043857326, + "tokens_seen": 2598109184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010748244734202609, + "loss": 1.2259, + "theoretical_loss": 3.3582224947441817, + "tokens_seen": 2598240256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010746238716148445, + "loss": 1.2816, + "theoretical_loss": 3.3582090859684826, + "tokens_seen": 2598371328 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.34243646264076233, + "objective/train/avg_token_score": 0.01695491001009941, + "objective/train/docs_used": 1462170, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1154985427856445, + "objective/train/original_loss": 4.052247524261475, + "objective/train/theoretical_loss": 3.358195678058536, + "objective/train/tokens_used": 2598502400, + "objective/train/unlikelihood_loss": 0.16911669075489044, + "theoretical_loss": 3.358195678058536, + "tokens_seen": 2598502400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010744232698094283, + "loss": 1.1814, + "theoretical_loss": 3.358195678058536, + "tokens_seen": 2598502400 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001074222668004012, + "loss": 1.3989, + "theoretical_loss": 3.3581822710142424, + "tokens_seen": 2598633472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010740220661985958, + "loss": 1.3387, + "theoretical_loss": 3.358168864835503, + "tokens_seen": 2598764544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010738214643931796, + "loss": 1.2033, + "theoretical_loss": 3.3581554595222167, + "tokens_seen": 2598895616 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010736208625877633, + "loss": 1.2802, + "theoretical_loss": 3.3581420550742846, + "tokens_seen": 2599026688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010734202607823471, + "loss": 1.4072, + "theoretical_loss": 3.358128651491608, + "tokens_seen": 2599157760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010732196589769308, + "loss": 1.4656, + "theoretical_loss": 3.358115248774087, + "tokens_seen": 2599288832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010730190571715146, + "loss": 1.3833, + "theoretical_loss": 3.3581018469216213, + "tokens_seen": 2599419904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010728184553660982, + "loss": 1.3118, + "theoretical_loss": 3.3580884459341123, + "tokens_seen": 2599550976 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001072617853560682, + "loss": 1.2522, + "theoretical_loss": 3.3580750458114608, + "tokens_seen": 2599682048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010724172517552657, + "loss": 1.2731, + "theoretical_loss": 3.3580616465535664, + "tokens_seen": 2599813120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010722166499498495, + "loss": 1.2439, + "theoretical_loss": 3.3580482481603307, + "tokens_seen": 2599944192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010720160481444333, + "loss": 1.2292, + "theoretical_loss": 3.358034850631654, + "tokens_seen": 2600075264 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.371456503868103, + "objective/train/avg_token_score": 0.020424455404281616, + "objective/train/docs_used": 1463432, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1938410997390747, + "objective/train/original_loss": 4.065286159515381, + "objective/train/theoretical_loss": 3.358028152191494, + "objective/train/tokens_used": 2600140800, + "objective/train/unlikelihood_loss": 0.18783530592918396, + "theoretical_loss": 3.358028152191494, + "tokens_seen": 2600140800 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001071815446339017, + "loss": 1.3969, + "theoretical_loss": 3.3580214539674365, + "tokens_seen": 2600206336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010716148445336009, + "loss": 1.117, + "theoretical_loss": 3.358008058167579, + "tokens_seen": 2600337408 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010714142427281847, + "loss": 1.3261, + "theoretical_loss": 3.357994663231983, + "tokens_seen": 2600468480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010712136409227684, + "loss": 1.2115, + "theoretical_loss": 3.3579812691605477, + "tokens_seen": 2600599552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010710130391173522, + "loss": 1.3469, + "theoretical_loss": 3.3579678759531753, + "tokens_seen": 2600730624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010708124373119358, + "loss": 1.3515, + "theoretical_loss": 3.357954483609765, + "tokens_seen": 2600861696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010706118355065196, + "loss": 1.2448, + "theoretical_loss": 3.3579410921302193, + "tokens_seen": 2600992768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010704112337011033, + "loss": 1.3561, + "theoretical_loss": 3.3579277015144378, + "tokens_seen": 2601123840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010702106318956871, + "loss": 1.2531, + "theoretical_loss": 3.357914311762321, + "tokens_seen": 2601254912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010700100300902708, + "loss": 1.2731, + "theoretical_loss": 3.3579009228737706, + "tokens_seen": 2601385984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010698094282848546, + "loss": 1.3748, + "theoretical_loss": 3.3578875348486865, + "tokens_seen": 2601517056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010696088264794383, + "loss": 1.1935, + "theoretical_loss": 3.3578741476869705, + "tokens_seen": 2601648128 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.4598607122898102, + "objective/train/avg_token_score": 0.03758081793785095, + "objective/train/docs_used": 1464290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2319188117980957, + "objective/train/original_loss": 3.277106523513794, + "objective/train/theoretical_loss": 3.3578607613885225, + "objective/train/tokens_used": 2601779200, + "objective/train/unlikelihood_loss": 0.23699873685836792, + "theoretical_loss": 3.3578607613885225, + "tokens_seen": 2601779200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010694082246740221, + "loss": 1.3491, + "theoretical_loss": 3.3578607613885225, + "tokens_seen": 2601779200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010692076228686059, + "loss": 1.2797, + "theoretical_loss": 3.357847375953244, + "tokens_seen": 2601910272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010690070210631896, + "loss": 1.3161, + "theoretical_loss": 3.3578339913810353, + "tokens_seen": 2602041344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010688064192577732, + "loss": 1.4219, + "theoretical_loss": 3.357820607671798, + "tokens_seen": 2602172416 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001068605817452357, + "loss": 1.3142, + "theoretical_loss": 3.3578072248254327, + "tokens_seen": 2602303488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010684052156469408, + "loss": 1.3363, + "theoretical_loss": 3.35779384284184, + "tokens_seen": 2602434560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010682046138415245, + "loss": 1.4116, + "theoretical_loss": 3.3577804617209206, + "tokens_seen": 2602565632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010680040120361083, + "loss": 1.2536, + "theoretical_loss": 3.357767081462576, + "tokens_seen": 2602696704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010678034102306922, + "loss": 1.2521, + "theoretical_loss": 3.3577537020667076, + "tokens_seen": 2602827776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010676028084252759, + "loss": 1.1895, + "theoretical_loss": 3.3577403235332155, + "tokens_seen": 2602958848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010674022066198597, + "loss": 1.3793, + "theoretical_loss": 3.3577269458620007, + "tokens_seen": 2603089920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010672016048144434, + "loss": 1.3768, + "theoretical_loss": 3.357713569052965, + "tokens_seen": 2603220992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010670010030090272, + "loss": 1.1632, + "theoretical_loss": 3.357700193106009, + "tokens_seen": 2603352064 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.4124511182308197, + "objective/train/avg_token_score": 0.03939834237098694, + "objective/train/docs_used": 1465580, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2412278652191162, + "objective/train/original_loss": 3.9381299018859863, + "objective/train/theoretical_loss": 3.3576935054557797, + "objective/train/tokens_used": 2603417600, + "objective/train/unlikelihood_loss": 0.17405633628368378, + "theoretical_loss": 3.3576935054557797, + "tokens_seen": 2603417600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010668004012036108, + "loss": 1.2572, + "theoretical_loss": 3.3576868180210333, + "tokens_seen": 2603483136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010665997993981946, + "loss": 1.2293, + "theoretical_loss": 3.3576734437979394, + "tokens_seen": 2603614208 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010663991975927783, + "loss": 1.2403, + "theoretical_loss": 3.357660070436628, + "tokens_seen": 2603745280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010661985957873621, + "loss": 1.3746, + "theoretical_loss": 3.3576466979370014, + "tokens_seen": 2603876352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010659979939819459, + "loss": 1.2965, + "theoretical_loss": 3.3576333262989593, + "tokens_seen": 2604007424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010657973921765296, + "loss": 1.3599, + "theoretical_loss": 3.3576199555224036, + "tokens_seen": 2604138496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010655967903711134, + "loss": 1.1708, + "theoretical_loss": 3.357606585607235, + "tokens_seen": 2604269568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010653961885656971, + "loss": 1.2226, + "theoretical_loss": 3.3575932165533544, + "tokens_seen": 2604400640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010651955867602809, + "loss": 1.4454, + "theoretical_loss": 3.357579848360664, + "tokens_seen": 2604531712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010649949849548645, + "loss": 1.184, + "theoretical_loss": 3.3575664810290644, + "tokens_seen": 2604662784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010647943831494483, + "loss": 1.2826, + "theoretical_loss": 3.3575531145584567, + "tokens_seen": 2604793856 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001064593781344032, + "loss": 1.3462, + "theoretical_loss": 3.357539748948742, + "tokens_seen": 2604924928 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.3405425250530243, + "objective/train/avg_token_score": 0.01118626818060875, + "objective/train/docs_used": 1466226, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0546391010284424, + "objective/train/original_loss": 4.077969074249268, + "objective/train/theoretical_loss": 3.357526384199822, + "objective/train/tokens_used": 2605056000, + "objective/train/unlikelihood_loss": 0.15148966014385223, + "theoretical_loss": 3.357526384199822, + "tokens_seen": 2605056000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010643931795386158, + "loss": 1.3577, + "theoretical_loss": 3.357526384199822, + "tokens_seen": 2605056000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010641925777331995, + "loss": 1.4034, + "theoretical_loss": 3.357513020311598, + "tokens_seen": 2605187072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010639919759277834, + "loss": 1.4582, + "theoretical_loss": 3.3574996572839706, + "tokens_seen": 2605318144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010637913741223672, + "loss": 1.2268, + "theoretical_loss": 3.357486295116842, + "tokens_seen": 2605449216 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001063590772316951, + "loss": 1.1888, + "theoretical_loss": 3.3574729338101124, + "tokens_seen": 2605580288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010633901705115347, + "loss": 1.301, + "theoretical_loss": 3.357459573363684, + "tokens_seen": 2605711360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010631895687061185, + "loss": 1.3357, + "theoretical_loss": 3.357446213777458, + "tokens_seen": 2605842432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010629889669007021, + "loss": 1.2278, + "theoretical_loss": 3.3574328550513353, + "tokens_seen": 2605973504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010627883650952858, + "loss": 1.3205, + "theoretical_loss": 3.3574194971852176, + "tokens_seen": 2606104576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010625877632898696, + "loss": 1.2881, + "theoretical_loss": 3.3574061401790067, + "tokens_seen": 2606235648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010623871614844534, + "loss": 1.2888, + "theoretical_loss": 3.357392784032603, + "tokens_seen": 2606366720 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010621865596790371, + "loss": 1.417, + "theoretical_loss": 3.357379428745909, + "tokens_seen": 2606497792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010619859578736209, + "loss": 1.1741, + "theoretical_loss": 3.357366074318825, + "tokens_seen": 2606628864 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.42967987060546875, + "objective/train/avg_token_score": 0.009982718154788017, + "objective/train/docs_used": 1466793, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2509106397628784, + "objective/train/original_loss": 3.5074076652526855, + "objective/train/theoretical_loss": 3.3573593974276066, + "objective/train/tokens_used": 2606694400, + "objective/train/unlikelihood_loss": 0.23092739284038544, + "theoretical_loss": 3.3573593974276066, + "tokens_seen": 2606694400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010617853560682046, + "loss": 1.2802, + "theoretical_loss": 3.357352720751254, + "tokens_seen": 2606759936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010615847542627884, + "loss": 1.2489, + "theoretical_loss": 3.3573393680430956, + "tokens_seen": 2606891008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010613841524573722, + "loss": 1.2082, + "theoretical_loss": 3.3573260161942526, + "tokens_seen": 2607022080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010611835506519559, + "loss": 1.323, + "theoretical_loss": 3.3573126652046263, + "tokens_seen": 2607153152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010609829488465395, + "loss": 1.2528, + "theoretical_loss": 3.357299315074118, + "tokens_seen": 2607284224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010607823470411233, + "loss": 1.3571, + "theoretical_loss": 3.357285965802629, + "tokens_seen": 2607415296 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001060581745235707, + "loss": 1.4952, + "theoretical_loss": 3.3572726173900613, + "tokens_seen": 2607546368 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001060381143430291, + "loss": 1.1969, + "theoretical_loss": 3.3572592698363164, + "tokens_seen": 2607677440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010601805416248747, + "loss": 1.3068, + "theoretical_loss": 3.3572459231412957, + "tokens_seen": 2607808512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010599799398194585, + "loss": 1.2731, + "theoretical_loss": 3.3572325773049005, + "tokens_seen": 2607939584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010597793380140422, + "loss": 1.4382, + "theoretical_loss": 3.3572192323270333, + "tokens_seen": 2608070656 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001059578736208626, + "loss": 1.3173, + "theoretical_loss": 3.357205888207595, + "tokens_seen": 2608201728 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.47061339020729065, + "objective/train/avg_token_score": 0.024435650557279587, + "objective/train/docs_used": 1468389, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.315218448638916, + "objective/train/original_loss": 4.269686222076416, + "objective/train/theoretical_loss": 3.3571925449464874, + "objective/train/tokens_used": 2608332800, + "objective/train/unlikelihood_loss": 0.09701746702194214, + "theoretical_loss": 3.3571925449464874, + "tokens_seen": 2608332800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010593781344032097, + "loss": 1.3779, + "theoretical_loss": 3.3571925449464874, + "tokens_seen": 2608332800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010591775325977935, + "loss": 1.3975, + "theoretical_loss": 3.357179202543612, + "tokens_seen": 2608463872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010589769307923771, + "loss": 1.3232, + "theoretical_loss": 3.357165860998871, + "tokens_seen": 2608594944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010587763289869609, + "loss": 1.3126, + "theoretical_loss": 3.3571525203121655, + "tokens_seen": 2608726016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010585757271815446, + "loss": 1.0788, + "theoretical_loss": 3.3571391804833977, + "tokens_seen": 2608857088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010583751253761284, + "loss": 1.3453, + "theoretical_loss": 3.357125841512469, + "tokens_seen": 2608988160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010581745235707121, + "loss": 1.1499, + "theoretical_loss": 3.3571125033992812, + "tokens_seen": 2609119232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010579739217652959, + "loss": 1.2883, + "theoretical_loss": 3.357099166143737, + "tokens_seen": 2609250304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010577733199598797, + "loss": 1.3571, + "theoretical_loss": 3.357085829745736, + "tokens_seen": 2609381376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010575727181544634, + "loss": 1.4292, + "theoretical_loss": 3.357072494205182, + "tokens_seen": 2609512448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010573721163490472, + "loss": 1.4245, + "theoretical_loss": 3.357059159521976, + "tokens_seen": 2609643520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010571715145436308, + "loss": 1.3225, + "theoretical_loss": 3.35704582569602, + "tokens_seen": 2609774592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010569709127382146, + "loss": 1.4811, + "theoretical_loss": 3.3570324927272157, + "tokens_seen": 2609905664 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.3370601236820221, + "objective/train/avg_token_score": 0.025279054418206215, + "objective/train/docs_used": 1468957, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0115149021148682, + "objective/train/original_loss": 4.597873210906982, + "objective/train/theoretical_loss": 3.3570258265642146, + "objective/train/tokens_used": 2609971200, + "objective/train/unlikelihood_loss": 0.12522122263908386, + "theoretical_loss": 3.3570258265642146, + "tokens_seen": 2609971200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010567703109327983, + "loss": 1.2691, + "theoretical_loss": 3.3570191606154647, + "tokens_seen": 2610036736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010565697091273822, + "loss": 1.2326, + "theoretical_loss": 3.3570058293606695, + "tokens_seen": 2610167808 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001056369107321966, + "loss": 1.2125, + "theoretical_loss": 3.3569924989627316, + "tokens_seen": 2610298880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010561685055165497, + "loss": 1.2907, + "theoretical_loss": 3.3569791694215527, + "tokens_seen": 2610429952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010559679037111335, + "loss": 1.388, + "theoretical_loss": 3.3569658407370353, + "tokens_seen": 2610561024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010557673019057172, + "loss": 1.3406, + "theoretical_loss": 3.356952512909081, + "tokens_seen": 2610692096 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001055566700100301, + "loss": 1.3477, + "theoretical_loss": 3.356939185937592, + "tokens_seen": 2610823168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010553660982948848, + "loss": 1.2185, + "theoretical_loss": 3.3569258598224696, + "tokens_seen": 2610954240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010551654964894684, + "loss": 1.4041, + "theoretical_loss": 3.356912534563617, + "tokens_seen": 2611085312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010549648946840521, + "loss": 1.2714, + "theoretical_loss": 3.3568992101609347, + "tokens_seen": 2611216384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010547642928786359, + "loss": 1.2826, + "theoretical_loss": 3.3568858866143256, + "tokens_seen": 2611347456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010545636910732197, + "loss": 1.3706, + "theoretical_loss": 3.356872563923692, + "tokens_seen": 2611478528 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.38361436128616333, + "objective/train/avg_token_score": 0.010130687616765499, + "objective/train/docs_used": 1469548, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2799991369247437, + "objective/train/original_loss": 4.778223991394043, + "objective/train/theoretical_loss": 3.356859242088935, + "objective/train/tokens_used": 2611609600, + "objective/train/unlikelihood_loss": 0.17249134182929993, + "theoretical_loss": 3.356859242088935, + "tokens_seen": 2611609600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010543630892678034, + "loss": 1.2473, + "theoretical_loss": 3.356859242088935, + "tokens_seen": 2611609600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010541624874623872, + "loss": 1.1915, + "theoretical_loss": 3.3568459211099575, + "tokens_seen": 2611740672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010539618856569709, + "loss": 1.2228, + "theoretical_loss": 3.356832600986661, + "tokens_seen": 2611871744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010537612838515547, + "loss": 1.2692, + "theoretical_loss": 3.3568192817189484, + "tokens_seen": 2612002816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010535606820461384, + "loss": 1.2156, + "theoretical_loss": 3.356805963306721, + "tokens_seen": 2612133888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010533600802407222, + "loss": 1.1714, + "theoretical_loss": 3.3567926457498816, + "tokens_seen": 2612264960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010531594784353058, + "loss": 1.3046, + "theoretical_loss": 3.356779329048331, + "tokens_seen": 2612396032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010529588766298896, + "loss": 1.3777, + "theoretical_loss": 3.356766013201973, + "tokens_seen": 2612527104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010527582748244735, + "loss": 1.4716, + "theoretical_loss": 3.35675269821071, + "tokens_seen": 2612658176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010525576730190572, + "loss": 1.253, + "theoretical_loss": 3.356739384074442, + "tokens_seen": 2612789248 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001052357071213641, + "loss": 1.2936, + "theoretical_loss": 3.356726070793073, + "tokens_seen": 2612920320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010521564694082247, + "loss": 1.2768, + "theoretical_loss": 3.3567127583665046, + "tokens_seen": 2613051392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010519558676028085, + "loss": 1.1711, + "theoretical_loss": 3.3566994467946394, + "tokens_seen": 2613182464 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.4310239553451538, + "objective/train/avg_token_score": 0.012206902727484703, + "objective/train/docs_used": 1470931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3414191007614136, + "objective/train/original_loss": 3.3898279666900635, + "objective/train/theoretical_loss": 3.35669279132919, + "objective/train/tokens_used": 2613248000, + "objective/train/unlikelihood_loss": 0.2382292002439499, + "theoretical_loss": 3.35669279132919, + "tokens_seen": 2613248000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010517552657973923, + "loss": 1.2465, + "theoretical_loss": 3.3566861360773794, + "tokens_seen": 2613313536 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001051554663991976, + "loss": 1.3898, + "theoretical_loss": 3.356672826214627, + "tokens_seen": 2613444608 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010513540621865598, + "loss": 1.248, + "theoretical_loss": 3.3566595172062845, + "tokens_seen": 2613575680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010511534603811434, + "loss": 1.2741, + "theoretical_loss": 3.356646209052254, + "tokens_seen": 2613706752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010509528585757272, + "loss": 1.3884, + "theoretical_loss": 3.356632901752438, + "tokens_seen": 2613837824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010507522567703109, + "loss": 1.1592, + "theoretical_loss": 3.356619595306739, + "tokens_seen": 2613968896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010505516549648947, + "loss": 1.2801, + "theoretical_loss": 3.3566062897150584, + "tokens_seen": 2614099968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010503510531594784, + "loss": 1.3205, + "theoretical_loss": 3.3565929849772997, + "tokens_seen": 2614231040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010501504513540622, + "loss": 1.3003, + "theoretical_loss": 3.3565796810933652, + "tokens_seen": 2614362112 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001049949849548646, + "loss": 1.3585, + "theoretical_loss": 3.356566378063157, + "tokens_seen": 2614493184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010497492477432297, + "loss": 1.3034, + "theoretical_loss": 3.3565530758865774, + "tokens_seen": 2614624256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010495486459378135, + "loss": 1.1808, + "theoretical_loss": 3.356539774563529, + "tokens_seen": 2614755328 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.34860703349113464, + "objective/train/avg_token_score": 0.011132567189633846, + "objective/train/docs_used": 1471590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0904946327209473, + "objective/train/original_loss": 3.866039752960205, + "objective/train/theoretical_loss": 3.356526474093914, + "objective/train/tokens_used": 2614886400, + "objective/train/unlikelihood_loss": 0.1654970794916153, + "theoretical_loss": 3.356526474093914, + "tokens_seen": 2614886400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010493480441323971, + "loss": 1.263, + "theoretical_loss": 3.356526474093914, + "tokens_seen": 2614886400 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001049147442326981, + "loss": 1.3674, + "theoretical_loss": 3.3565131744776355, + "tokens_seen": 2615017472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010489468405215647, + "loss": 1.236, + "theoretical_loss": 3.3564998757145954, + "tokens_seen": 2615148544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010487462387161485, + "loss": 1.4066, + "theoretical_loss": 3.356486577804696, + "tokens_seen": 2615279616 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010485456369107323, + "loss": 1.2291, + "theoretical_loss": 3.356473280747841, + "tokens_seen": 2615410688 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001048345035105316, + "loss": 1.1509, + "theoretical_loss": 3.3564599845439314, + "tokens_seen": 2615541760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010481444332998998, + "loss": 1.2246, + "theoretical_loss": 3.356446689192871, + "tokens_seen": 2615672832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010479438314944835, + "loss": 1.2524, + "theoretical_loss": 3.356433394694561, + "tokens_seen": 2615803904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010477432296890673, + "loss": 1.2373, + "theoretical_loss": 3.356420101048905, + "tokens_seen": 2615934976 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001047542627883651, + "loss": 1.337, + "theoretical_loss": 3.356406808255806, + "tokens_seen": 2616066048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010473420260782347, + "loss": 1.2965, + "theoretical_loss": 3.356393516315166, + "tokens_seen": 2616197120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010471414242728184, + "loss": 1.2713, + "theoretical_loss": 3.3563802252268875, + "tokens_seen": 2616328192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010469408224674022, + "loss": 1.2763, + "theoretical_loss": 3.356366934990873, + "tokens_seen": 2616459264 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.5306695699691772, + "objective/train/avg_token_score": 0.017060836777091026, + "objective/train/docs_used": 1472953, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.480187177658081, + "objective/train/original_loss": 3.5450522899627686, + "objective/train/theoretical_loss": 3.3563602901924345, + "objective/train/tokens_used": 2616524800, + "objective/train/unlikelihood_loss": 0.14672689139842987, + "theoretical_loss": 3.3563602901924345, + "tokens_seen": 2616524800 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001046740220661986, + "loss": 1.3437, + "theoretical_loss": 3.3563536456070255, + "tokens_seen": 2616590336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010465396188565697, + "loss": 1.2864, + "theoretical_loss": 3.356340357075248, + "tokens_seen": 2616721408 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010463390170511535, + "loss": 1.1973, + "theoretical_loss": 3.3563270693954426, + "tokens_seen": 2616852480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010461384152457372, + "loss": 1.1855, + "theoretical_loss": 3.3563137825675122, + "tokens_seen": 2616983552 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001045937813440321, + "loss": 1.1938, + "theoretical_loss": 3.3563004965913597, + "tokens_seen": 2617114624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010457372116349047, + "loss": 1.4617, + "theoretical_loss": 3.3562872114668876, + "tokens_seen": 2617245696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010455366098294885, + "loss": 1.3629, + "theoretical_loss": 3.3562739271939988, + "tokens_seen": 2617376768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010453360080240722, + "loss": 1.3546, + "theoretical_loss": 3.356260643772596, + "tokens_seen": 2617507840 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001045135406218656, + "loss": 1.3886, + "theoretical_loss": 3.356247361202582, + "tokens_seen": 2617638912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010449348044132398, + "loss": 1.2362, + "theoretical_loss": 3.356234079483859, + "tokens_seen": 2617769984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010447342026078235, + "loss": 1.3584, + "theoretical_loss": 3.3562207986163313, + "tokens_seen": 2617901056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010445336008024073, + "loss": 1.2047, + "theoretical_loss": 3.3562075185999007, + "tokens_seen": 2618032128 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.2993646264076233, + "objective/train/avg_token_score": 0.03319826349616051, + "objective/train/docs_used": 1473611, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0192813873291016, + "objective/train/original_loss": 3.6048808097839355, + "objective/train/theoretical_loss": 3.35619423943447, + "objective/train/tokens_used": 2618163200, + "objective/train/unlikelihood_loss": 0.22071292996406555, + "theoretical_loss": 3.35619423943447, + "tokens_seen": 2618163200 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001044332998996991, + "loss": 1.2191, + "theoretical_loss": 3.35619423943447, + "tokens_seen": 2618163200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010441323971915748, + "loss": 1.2156, + "theoretical_loss": 3.3561809611199425, + "tokens_seen": 2618294272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010439317953861586, + "loss": 1.1879, + "theoretical_loss": 3.3561676836562206, + "tokens_seen": 2618425344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010437311935807423, + "loss": 1.3121, + "theoretical_loss": 3.3561544070432077, + "tokens_seen": 2618556416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010435305917753259, + "loss": 1.4542, + "theoretical_loss": 3.3561411312808067, + "tokens_seen": 2618687488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010433299899699097, + "loss": 1.3551, + "theoretical_loss": 3.3561278563689196, + "tokens_seen": 2618818560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010431293881644934, + "loss": 1.3277, + "theoretical_loss": 3.3561145823074505, + "tokens_seen": 2618949632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010429287863590772, + "loss": 1.3262, + "theoretical_loss": 3.356101309096302, + "tokens_seen": 2619080704 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001042728184553661, + "loss": 1.3245, + "theoretical_loss": 3.356088036735377, + "tokens_seen": 2619211776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010425275827482447, + "loss": 1.3135, + "theoretical_loss": 3.3560747652245784, + "tokens_seen": 2619342848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010423269809428285, + "loss": 1.2274, + "theoretical_loss": 3.3560614945638094, + "tokens_seen": 2619473920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010421263791374122, + "loss": 1.3261, + "theoretical_loss": 3.356048224752973, + "tokens_seen": 2619604992 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001041925777331996, + "loss": 1.168, + "theoretical_loss": 3.356034955791972, + "tokens_seen": 2619736064 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.2864736020565033, + "objective/train/avg_token_score": 0.010581444017589092, + "objective/train/docs_used": 1474984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9224516749382019, + "objective/train/original_loss": 4.267470836639404, + "objective/train/theoretical_loss": 3.3560283216301294, + "objective/train/tokens_used": 2619801600, + "objective/train/unlikelihood_loss": 0.17698955535888672, + "theoretical_loss": 3.3560283216301294, + "tokens_seen": 2619801600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010417251755265798, + "loss": 1.1709, + "theoretical_loss": 3.35602168768071, + "tokens_seen": 2619867136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010415245737211635, + "loss": 1.3624, + "theoretical_loss": 3.356008420419089, + "tokens_seen": 2619998208 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010413239719157473, + "loss": 1.2385, + "theoretical_loss": 3.3559951540070134, + "tokens_seen": 2620129280 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001041123370110331, + "loss": 1.2757, + "theoretical_loss": 3.3559818884443855, + "tokens_seen": 2620260352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010409227683049148, + "loss": 1.3135, + "theoretical_loss": 3.355968623731109, + "tokens_seen": 2620391424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010407221664994985, + "loss": 1.3212, + "theoretical_loss": 3.355955359867086, + "tokens_seen": 2620522496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010405215646940823, + "loss": 1.331, + "theoretical_loss": 3.355942096852221, + "tokens_seen": 2620653568 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001040320962888666, + "loss": 1.413, + "theoretical_loss": 3.355928834686416, + "tokens_seen": 2620784640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010401203610832498, + "loss": 1.3511, + "theoretical_loss": 3.355915573369575, + "tokens_seen": 2620915712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010399197592778336, + "loss": 1.3351, + "theoretical_loss": 3.3559023129016006, + "tokens_seen": 2621046784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010397191574724173, + "loss": 1.1167, + "theoretical_loss": 3.3558890532823966, + "tokens_seen": 2621177856 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001039518555667001, + "loss": 1.3457, + "theoretical_loss": 3.355875794511866, + "tokens_seen": 2621308928 + }, + { + "debugging/length": 547.75, + "debugging/num_segments": 4, + "epoch": 0.79, + "objective/train/aligned_token_count": 0.45326244831085205, + "objective/train/avg_token_score": 0.012631071731448174, + "objective/train/docs_used": 1475467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5047022104263306, + "objective/train/original_loss": 3.7960872650146484, + "objective/train/theoretical_loss": 3.3558625365899117, + "objective/train/tokens_used": 2621440000, + "objective/train/unlikelihood_loss": 0.18778850138187408, + "theoretical_loss": 3.3558625365899117, + "tokens_seen": 2621440000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010393179538615847, + "loss": 1.4569, + "theoretical_loss": 3.3558625365899117, + "tokens_seen": 2621440000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010391173520561685, + "loss": 1.3771, + "theoretical_loss": 3.355849279516437, + "tokens_seen": 2621571072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010389167502507522, + "loss": 1.1677, + "theoretical_loss": 3.355836023291346, + "tokens_seen": 2621702144 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001038716148445336, + "loss": 1.2895, + "theoretical_loss": 3.355822767914541, + "tokens_seen": 2621833216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010385155466399197, + "loss": 1.4175, + "theoretical_loss": 3.3558095133859265, + "tokens_seen": 2621964288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010383149448345035, + "loss": 1.3691, + "theoretical_loss": 3.3557962597054045, + "tokens_seen": 2622095360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010381143430290873, + "loss": 1.3329, + "theoretical_loss": 3.355783006872879, + "tokens_seen": 2622226432 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001037913741223671, + "loss": 1.3398, + "theoretical_loss": 3.355769754888253, + "tokens_seen": 2622357504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010377131394182549, + "loss": 1.1426, + "theoretical_loss": 3.3557565037514303, + "tokens_seen": 2622488576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010375125376128385, + "loss": 1.3318, + "theoretical_loss": 3.3557432534623146, + "tokens_seen": 2622619648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010373119358074223, + "loss": 1.3326, + "theoretical_loss": 3.355730004020808, + "tokens_seen": 2622750720 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001037111334002006, + "loss": 1.4517, + "theoretical_loss": 3.3557167554268155, + "tokens_seen": 2622881792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010369107321965898, + "loss": 1.2141, + "theoretical_loss": 3.3557035076802397, + "tokens_seen": 2623012864 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.3963831961154938, + "objective/train/avg_token_score": 0.020231252536177635, + "objective/train/docs_used": 1476787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.159981608390808, + "objective/train/original_loss": 3.416165828704834, + "objective/train/theoretical_loss": 3.355696884124703, + "objective/train/tokens_used": 2623078400, + "objective/train/unlikelihood_loss": 0.2429461032152176, + "theoretical_loss": 3.355696884124703, + "tokens_seen": 2623078400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010367101303911736, + "loss": 1.2434, + "theoretical_loss": 3.355690260780984, + "tokens_seen": 2623143936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010365095285857573, + "loss": 1.3277, + "theoretical_loss": 3.3556770147289523, + "tokens_seen": 2623275008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010363089267803411, + "loss": 1.2872, + "theoretical_loss": 3.3556637695240474, + "tokens_seen": 2623406080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010361083249749248, + "loss": 1.2728, + "theoretical_loss": 3.3556505251661735, + "tokens_seen": 2623537152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010359077231695086, + "loss": 1.3539, + "theoretical_loss": 3.355637281655234, + "tokens_seen": 2623668224 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010357071213640922, + "loss": 1.1761, + "theoretical_loss": 3.3556240389911323, + "tokens_seen": 2623799296 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001035506519558676, + "loss": 1.0998, + "theoretical_loss": 3.355610797173772, + "tokens_seen": 2623930368 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010353059177532597, + "loss": 1.2915, + "theoretical_loss": 3.355597556203056, + "tokens_seen": 2624061440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010351053159478435, + "loss": 1.0451, + "theoretical_loss": 3.3555843160788887, + "tokens_seen": 2624192512 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010349047141424273, + "loss": 1.3516, + "theoretical_loss": 3.3555710768011737, + "tokens_seen": 2624323584 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001034704112337011, + "loss": 1.2313, + "theoretical_loss": 3.3555578383698146, + "tokens_seen": 2624454656 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010345035105315948, + "loss": 1.2637, + "theoretical_loss": 3.3555446007847145, + "tokens_seen": 2624585728 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.30193060636520386, + "objective/train/avg_token_score": 0.014339080080389977, + "objective/train/docs_used": 1477458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0011394023895264, + "objective/train/original_loss": 4.2552809715271, + "objective/train/theoretical_loss": 3.3555313640457776, + "objective/train/tokens_used": 2624716800, + "objective/train/unlikelihood_loss": 0.20481981337070465, + "theoretical_loss": 3.3555313640457776, + "tokens_seen": 2624716800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010343029087261785, + "loss": 1.233, + "theoretical_loss": 3.3555313640457776, + "tokens_seen": 2624716800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010341023069207624, + "loss": 1.3721, + "theoretical_loss": 3.355518128152907, + "tokens_seen": 2624847872 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010339017051153462, + "loss": 1.2787, + "theoretical_loss": 3.3555048931060067, + "tokens_seen": 2624978944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010337011033099298, + "loss": 1.1889, + "theoretical_loss": 3.355491658904981, + "tokens_seen": 2625110016 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010335005015045136, + "loss": 1.4058, + "theoretical_loss": 3.3554784255497325, + "tokens_seen": 2625241088 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010332998996990973, + "loss": 1.2522, + "theoretical_loss": 3.3554651930401658, + "tokens_seen": 2625372160 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010330992978936811, + "loss": 1.1933, + "theoretical_loss": 3.3554519613761835, + "tokens_seen": 2625503232 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010328986960882648, + "loss": 1.4602, + "theoretical_loss": 3.355438730557691, + "tokens_seen": 2625634304 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010326980942828486, + "loss": 1.2806, + "theoretical_loss": 3.355425500584591, + "tokens_seen": 2625765376 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010324974924774323, + "loss": 1.3135, + "theoretical_loss": 3.355412271456787, + "tokens_seen": 2625896448 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010322968906720161, + "loss": 1.2389, + "theoretical_loss": 3.355399043174184, + "tokens_seen": 2626027520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010320962888665999, + "loss": 1.0678, + "theoretical_loss": 3.3553858157366845, + "tokens_seen": 2626158592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010318956870611836, + "loss": 1.4601, + "theoretical_loss": 3.3553725891441935, + "tokens_seen": 2626289664 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.405303031206131, + "objective/train/avg_token_score": 0.03990193456411362, + "objective/train/docs_used": 1478565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3489686250686646, + "objective/train/original_loss": 4.580089569091797, + "objective/train/theoretical_loss": 3.3553659761647956, + "objective/train/tokens_used": 2626355200, + "objective/train/unlikelihood_loss": 0.11727843433618546, + "theoretical_loss": 3.3553659761647956, + "tokens_seen": 2626355200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010316950852557672, + "loss": 1.3201, + "theoretical_loss": 3.355359363396614, + "tokens_seen": 2626420736 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001031494483450351, + "loss": 1.4675, + "theoretical_loss": 3.35534613849385, + "tokens_seen": 2626551808 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010312938816449348, + "loss": 1.2779, + "theoretical_loss": 3.355332914435806, + "tokens_seen": 2626682880 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010310932798395185, + "loss": 1.2763, + "theoretical_loss": 3.355319691222385, + "tokens_seen": 2626813952 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010308926780341023, + "loss": 1.3645, + "theoretical_loss": 3.3553064688534913, + "tokens_seen": 2626945024 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001030692076228686, + "loss": 1.3378, + "theoretical_loss": 3.3552932473290293, + "tokens_seen": 2627076096 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010304914744232698, + "loss": 1.3587, + "theoretical_loss": 3.3552800266489022, + "tokens_seen": 2627207168 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010302908726178537, + "loss": 1.3515, + "theoretical_loss": 3.3552668068130145, + "tokens_seen": 2627338240 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010300902708124374, + "loss": 1.2677, + "theoretical_loss": 3.3552535878212697, + "tokens_seen": 2627469312 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010298896690070212, + "loss": 1.2456, + "theoretical_loss": 3.3552403696735724, + "tokens_seen": 2627600384 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010296890672016048, + "loss": 1.201, + "theoretical_loss": 3.355227152369826, + "tokens_seen": 2627731456 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010294884653961886, + "loss": 1.3033, + "theoretical_loss": 3.355213935909935, + "tokens_seen": 2627862528 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.34188660979270935, + "objective/train/avg_token_score": 0.008579139597713947, + "objective/train/docs_used": 1479263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.184167504310608, + "objective/train/original_loss": 3.85632061958313, + "objective/train/theoretical_loss": 3.355200720293803, + "objective/train/tokens_used": 2627993600, + "objective/train/unlikelihood_loss": 0.16483087837696075, + "theoretical_loss": 3.355200720293803, + "tokens_seen": 2627993600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010292878635907723, + "loss": 1.2471, + "theoretical_loss": 3.355200720293803, + "tokens_seen": 2627993600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010290872617853561, + "loss": 1.2346, + "theoretical_loss": 3.355187505521334, + "tokens_seen": 2628124672 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010288866599799399, + "loss": 1.3915, + "theoretical_loss": 3.3551742915924327, + "tokens_seen": 2628255744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010286860581745236, + "loss": 1.3062, + "theoretical_loss": 3.3551610785070034, + "tokens_seen": 2628386816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010284854563691074, + "loss": 1.4377, + "theoretical_loss": 3.3551478662649488, + "tokens_seen": 2628517888 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010282848545636911, + "loss": 1.3974, + "theoretical_loss": 3.3551346548661742, + "tokens_seen": 2628648960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010280842527582749, + "loss": 1.2124, + "theoretical_loss": 3.3551214443105835, + "tokens_seen": 2628780032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010278836509528585, + "loss": 1.2924, + "theoretical_loss": 3.35510823459808, + "tokens_seen": 2628911104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010276830491474423, + "loss": 1.3593, + "theoretical_loss": 3.3550950257285694, + "tokens_seen": 2629042176 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001027482447342026, + "loss": 1.1508, + "theoretical_loss": 3.355081817701955, + "tokens_seen": 2629173248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010272818455366098, + "loss": 1.3203, + "theoretical_loss": 3.3550686105181406, + "tokens_seen": 2629304320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010270812437311935, + "loss": 1.3247, + "theoretical_loss": 3.3550554041770315, + "tokens_seen": 2629435392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010268806419257773, + "loss": 1.2163, + "theoretical_loss": 3.3550421986785306, + "tokens_seen": 2629566464 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.47385141253471375, + "objective/train/avg_token_score": 0.01905849203467369, + "objective/train/docs_used": 1479932, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.522308588027954, + "objective/train/original_loss": 4.194167613983154, + "objective/train/theoretical_loss": 3.355035596245229, + "objective/train/tokens_used": 2629632000, + "objective/train/unlikelihood_loss": 0.12071707099676132, + "theoretical_loss": 3.355035596245229, + "tokens_seen": 2629632000 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001026680040120361, + "loss": 1.1119, + "theoretical_loss": 3.3550289940225433, + "tokens_seen": 2629697536 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001026479438314945, + "loss": 1.385, + "theoretical_loss": 3.3550157902089737, + "tokens_seen": 2629828608 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010262788365095287, + "loss": 1.2545, + "theoretical_loss": 3.3550025872377254, + "tokens_seen": 2629959680 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010260782347041125, + "loss": 1.277, + "theoretical_loss": 3.354989385108703, + "tokens_seen": 2630090752 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010258776328986961, + "loss": 1.3218, + "theoretical_loss": 3.354976183821811, + "tokens_seen": 2630221824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010256770310932798, + "loss": 1.1891, + "theoretical_loss": 3.3549629833769536, + "tokens_seen": 2630352896 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010254764292878636, + "loss": 1.126, + "theoretical_loss": 3.354949783774035, + "tokens_seen": 2630483968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010252758274824474, + "loss": 1.4666, + "theoretical_loss": 3.3549365850129598, + "tokens_seen": 2630615040 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010250752256770311, + "loss": 1.3538, + "theoretical_loss": 3.354923387093632, + "tokens_seen": 2630746112 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010248746238716149, + "loss": 1.2758, + "theoretical_loss": 3.354910190015957, + "tokens_seen": 2630877184 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010246740220661986, + "loss": 1.3384, + "theoretical_loss": 3.3548969937798376, + "tokens_seen": 2631008256 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010244734202607824, + "loss": 1.2778, + "theoretical_loss": 3.3548837983851794, + "tokens_seen": 2631139328 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.41929373145103455, + "objective/train/avg_token_score": 0.0060736918821930885, + "objective/train/docs_used": 1481256, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2601205110549927, + "objective/train/original_loss": 3.4791793823242188, + "objective/train/theoretical_loss": 3.3548706038318867, + "objective/train/tokens_used": 2631270400, + "objective/train/unlikelihood_loss": 0.23374409973621368, + "theoretical_loss": 3.3548706038318867, + "tokens_seen": 2631270400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010242728184553662, + "loss": 1.276, + "theoretical_loss": 3.3548706038318867, + "tokens_seen": 2631270400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010240722166499499, + "loss": 1.1943, + "theoretical_loss": 3.3548574101198634, + "tokens_seen": 2631401472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010238716148445335, + "loss": 1.3313, + "theoretical_loss": 3.3548442172490147, + "tokens_seen": 2631532544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010236710130391173, + "loss": 1.3626, + "theoretical_loss": 3.3548310252192444, + "tokens_seen": 2631663616 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001023470411233701, + "loss": 1.4407, + "theoretical_loss": 3.354817834030457, + "tokens_seen": 2631794688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010232698094282848, + "loss": 1.3991, + "theoretical_loss": 3.3548046436825576, + "tokens_seen": 2631925760 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010230692076228686, + "loss": 1.2547, + "theoretical_loss": 3.35479145417545, + "tokens_seen": 2632056832 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010228686058174525, + "loss": 1.3557, + "theoretical_loss": 3.3547782655090397, + "tokens_seen": 2632187904 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010226680040120362, + "loss": 1.1379, + "theoretical_loss": 3.3547650776832305, + "tokens_seen": 2632318976 + }, + { + "epoch": 0.8, + "learning_rate": 0.000102246740220662, + "loss": 1.3749, + "theoretical_loss": 3.3547518906979272, + "tokens_seen": 2632450048 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010222668004012037, + "loss": 1.2097, + "theoretical_loss": 3.354738704553034, + "tokens_seen": 2632581120 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010220661985957875, + "loss": 1.2255, + "theoretical_loss": 3.354725519248456, + "tokens_seen": 2632712192 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010218655967903711, + "loss": 1.2866, + "theoretical_loss": 3.354712334784098, + "tokens_seen": 2632843264 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.4868646264076233, + "objective/train/avg_token_score": 0.010815800167620182, + "objective/train/docs_used": 1481803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4314779043197632, + "objective/train/original_loss": 3.478360414505005, + "objective/train/theoretical_loss": 3.354705742866971, + "objective/train/tokens_used": 2632908800, + "objective/train/unlikelihood_loss": 0.17515312135219574, + "theoretical_loss": 3.354705742866971, + "tokens_seen": 2632908800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010216649949849549, + "loss": 1.5262, + "theoretical_loss": 3.354699151159864, + "tokens_seen": 2632974336 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010214643931795386, + "loss": 1.3546, + "theoretical_loss": 3.3546859683756587, + "tokens_seen": 2633105408 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010212637913741224, + "loss": 1.1937, + "theoretical_loss": 3.3546727864313874, + "tokens_seen": 2633236480 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010210631895687061, + "loss": 1.2709, + "theoretical_loss": 3.354659605326954, + "tokens_seen": 2633367552 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010208625877632899, + "loss": 1.2806, + "theoretical_loss": 3.354646425062264, + "tokens_seen": 2633498624 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010206619859578737, + "loss": 1.3829, + "theoretical_loss": 3.3546332456372214, + "tokens_seen": 2633629696 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010204613841524574, + "loss": 1.4786, + "theoretical_loss": 3.3546200670517314, + "tokens_seen": 2633760768 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010202607823470412, + "loss": 1.2532, + "theoretical_loss": 3.354606889305699, + "tokens_seen": 2633891840 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010200601805416248, + "loss": 1.2481, + "theoretical_loss": 3.3545937123990273, + "tokens_seen": 2634022912 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010198595787362086, + "loss": 1.2979, + "theoretical_loss": 3.354580536331623, + "tokens_seen": 2634153984 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010196589769307923, + "loss": 1.3231, + "theoretical_loss": 3.3545673611033906, + "tokens_seen": 2634285056 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010194583751253761, + "loss": 1.2514, + "theoretical_loss": 3.3545541867142337, + "tokens_seen": 2634416128 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.482221394777298, + "objective/train/avg_token_score": 0.009596087969839573, + "objective/train/docs_used": 1483155, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.460322618484497, + "objective/train/original_loss": 3.59324049949646, + "objective/train/theoretical_loss": 3.3545410131640585, + "objective/train/tokens_used": 2634547200, + "objective/train/unlikelihood_loss": 0.15561571717262268, + "theoretical_loss": 3.3545410131640585, + "tokens_seen": 2634547200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010192577733199598, + "loss": 1.2941, + "theoretical_loss": 3.3545410131640585, + "tokens_seen": 2634547200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010190571715145437, + "loss": 1.2312, + "theoretical_loss": 3.3545278404527687, + "tokens_seen": 2634678272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010188565697091275, + "loss": 1.3873, + "theoretical_loss": 3.3545146685802703, + "tokens_seen": 2634809344 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010186559679037112, + "loss": 1.2522, + "theoretical_loss": 3.354501497546467, + "tokens_seen": 2634940416 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001018455366098295, + "loss": 1.3005, + "theoretical_loss": 3.3544883273512647, + "tokens_seen": 2635071488 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010182547642928788, + "loss": 1.2859, + "theoretical_loss": 3.3544751579945675, + "tokens_seen": 2635202560 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010180541624874624, + "loss": 1.3253, + "theoretical_loss": 3.354461989476281, + "tokens_seen": 2635333632 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010178535606820461, + "loss": 1.3625, + "theoretical_loss": 3.3544488217963093, + "tokens_seen": 2635464704 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010176529588766299, + "loss": 1.2382, + "theoretical_loss": 3.354435654954558, + "tokens_seen": 2635595776 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010174523570712136, + "loss": 1.2147, + "theoretical_loss": 3.354422488950932, + "tokens_seen": 2635726848 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010172517552657974, + "loss": 1.3583, + "theoretical_loss": 3.3544093237853363, + "tokens_seen": 2635857920 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010170511534603812, + "loss": 1.195, + "theoretical_loss": 3.3543961594576754, + "tokens_seen": 2635988992 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010168505516549649, + "loss": 1.1896, + "theoretical_loss": 3.354382995967855, + "tokens_seen": 2636120064 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.5218108296394348, + "objective/train/avg_token_score": 0.010513165965676308, + "objective/train/docs_used": 1483578, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.676706075668335, + "objective/train/original_loss": 3.929753541946411, + "objective/train/theoretical_loss": 3.354376414537105, + "objective/train/tokens_used": 2636185600, + "objective/train/unlikelihood_loss": 0.1660400629043579, + "theoretical_loss": 3.354376414537105, + "tokens_seen": 2636185600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010166499498495487, + "loss": 1.5156, + "theoretical_loss": 3.354369833315779, + "tokens_seen": 2636251136 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010164493480441324, + "loss": 1.2979, + "theoretical_loss": 3.354356671501354, + "tokens_seen": 2636382208 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010162487462387162, + "loss": 1.1178, + "theoretical_loss": 3.354343510524484, + "tokens_seen": 2636513280 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010160481444332998, + "loss": 1.2518, + "theoretical_loss": 3.3543303503850748, + "tokens_seen": 2636644352 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010158475426278836, + "loss": 1.4142, + "theoretical_loss": 3.3543171910830303, + "tokens_seen": 2636775424 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010156469408224673, + "loss": 1.2031, + "theoretical_loss": 3.3543040326182565, + "tokens_seen": 2636906496 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010154463390170511, + "loss": 1.3078, + "theoretical_loss": 3.3542908749906584, + "tokens_seen": 2637037568 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001015245737211635, + "loss": 1.2805, + "theoretical_loss": 3.354277718200141, + "tokens_seen": 2637168640 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010150451354062187, + "loss": 1.3263, + "theoretical_loss": 3.3542645622466094, + "tokens_seen": 2637299712 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010148445336008025, + "loss": 1.0484, + "theoretical_loss": 3.354251407129969, + "tokens_seen": 2637430784 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010146439317953863, + "loss": 1.3967, + "theoretical_loss": 3.354238252850125, + "tokens_seen": 2637561856 + }, + { + "epoch": 0.8, + "learning_rate": 0.000101444332998997, + "loss": 1.3014, + "theoretical_loss": 3.3542250994069818, + "tokens_seen": 2637692928 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.5275537371635437, + "objective/train/avg_token_score": 0.003689316101372242, + "objective/train/docs_used": 1484926, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6575952768325806, + "objective/train/original_loss": 3.8952789306640625, + "objective/train/theoretical_loss": 3.354211946800446, + "objective/train/tokens_used": 2637824000, + "objective/train/unlikelihood_loss": 0.13646502792835236, + "theoretical_loss": 3.354211946800446, + "tokens_seen": 2637824000 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010142427281845536, + "loss": 1.3375, + "theoretical_loss": 3.354211946800446, + "tokens_seen": 2637824000 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010140421263791374, + "loss": 1.158, + "theoretical_loss": 3.3541987950304213, + "tokens_seen": 2637955072 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010138415245737212, + "loss": 1.3854, + "theoretical_loss": 3.3541856440968143, + "tokens_seen": 2638086144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010136409227683049, + "loss": 1.2382, + "theoretical_loss": 3.3541724939995294, + "tokens_seen": 2638217216 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010134403209628887, + "loss": 1.3753, + "theoretical_loss": 3.354159344738472, + "tokens_seen": 2638348288 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010132397191574724, + "loss": 1.2542, + "theoretical_loss": 3.3541461963135477, + "tokens_seen": 2638479360 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010130391173520562, + "loss": 1.3516, + "theoretical_loss": 3.3541330487246617, + "tokens_seen": 2638610432 + }, + { + "epoch": 0.8, + "learning_rate": 0.000101283851554664, + "loss": 1.0602, + "theoretical_loss": 3.354119901971719, + "tokens_seen": 2638741504 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010126379137412237, + "loss": 1.2924, + "theoretical_loss": 3.354106756054625, + "tokens_seen": 2638872576 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010124373119358075, + "loss": 1.4146, + "theoretical_loss": 3.3540936109732855, + "tokens_seen": 2639003648 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010122367101303911, + "loss": 1.163, + "theoretical_loss": 3.3540804667276056, + "tokens_seen": 2639134720 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010120361083249748, + "loss": 1.4175, + "theoretical_loss": 3.3540673233174902, + "tokens_seen": 2639265792 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010118355065195586, + "loss": 1.1526, + "theoretical_loss": 3.3540541807428452, + "tokens_seen": 2639396864 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.3516617715358734, + "objective/train/avg_token_score": 0.020531360059976578, + "objective/train/docs_used": 1485572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1850521564483643, + "objective/train/original_loss": 4.303853511810303, + "objective/train/theoretical_loss": 3.354047609768795, + "objective/train/tokens_used": 2639462400, + "objective/train/unlikelihood_loss": 0.128317192196846, + "theoretical_loss": 3.354047609768795, + "tokens_seen": 2639462400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010116349047141425, + "loss": 1.2186, + "theoretical_loss": 3.3540410390035764, + "tokens_seen": 2639527936 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010114343029087263, + "loss": 1.3357, + "theoretical_loss": 3.354027898099588, + "tokens_seen": 2639659008 + }, + { + "epoch": 0.8, + "learning_rate": 0.000101123370110331, + "loss": 1.4356, + "theoretical_loss": 3.354014758030787, + "tokens_seen": 2639790080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010110330992978938, + "loss": 1.1556, + "theoretical_loss": 3.3540016187970774, + "tokens_seen": 2639921152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010108324974924775, + "loss": 1.3625, + "theoretical_loss": 3.3539884803983657, + "tokens_seen": 2640052224 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010106318956870613, + "loss": 1.2537, + "theoretical_loss": 3.353975342834557, + "tokens_seen": 2640183296 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001010431293881645, + "loss": 1.1654, + "theoretical_loss": 3.3539622061055563, + "tokens_seen": 2640314368 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010102306920762287, + "loss": 1.2398, + "theoretical_loss": 3.3539490702112698, + "tokens_seen": 2640445440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010100300902708124, + "loss": 1.2235, + "theoretical_loss": 3.353935935151603, + "tokens_seen": 2640576512 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010098294884653962, + "loss": 1.186, + "theoretical_loss": 3.353922800926461, + "tokens_seen": 2640707584 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100962888665998, + "loss": 1.3944, + "theoretical_loss": 3.35390966753575, + "tokens_seen": 2640838656 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010094282848545637, + "loss": 1.4109, + "theoretical_loss": 3.3538965349793752, + "tokens_seen": 2640969728 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.4414711594581604, + "objective/train/avg_token_score": 0.01300759520381689, + "objective/train/docs_used": 1486345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3558359146118164, + "objective/train/original_loss": 3.8389227390289307, + "objective/train/theoretical_loss": 3.3538834032572424, + "objective/train/tokens_used": 2641100800, + "objective/train/unlikelihood_loss": 0.15223631262779236, + "theoretical_loss": 3.3538834032572424, + "tokens_seen": 2641100800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010092276830491475, + "loss": 1.2272, + "theoretical_loss": 3.3538834032572424, + "tokens_seen": 2641100800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010090270812437312, + "loss": 1.3736, + "theoretical_loss": 3.3538702723692566, + "tokens_seen": 2641231872 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001008826479438315, + "loss": 1.2467, + "theoretical_loss": 3.3538571423153236, + "tokens_seen": 2641362944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010086258776328987, + "loss": 1.2055, + "theoretical_loss": 3.3538440130953497, + "tokens_seen": 2641494016 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010084252758274825, + "loss": 1.2934, + "theoretical_loss": 3.3538308847092404, + "tokens_seen": 2641625088 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010082246740220661, + "loss": 1.3138, + "theoretical_loss": 3.3538177571569006, + "tokens_seen": 2641756160 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010080240722166499, + "loss": 1.3149, + "theoretical_loss": 3.3538046304382365, + "tokens_seen": 2641887232 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010078234704112338, + "loss": 1.1216, + "theoretical_loss": 3.353791504553154, + "tokens_seen": 2642018304 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010076228686058175, + "loss": 1.239, + "theoretical_loss": 3.3537783795015588, + "tokens_seen": 2642149376 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010074222668004013, + "loss": 1.348, + "theoretical_loss": 3.3537652552833563, + "tokens_seen": 2642280448 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001007221664994985, + "loss": 1.1744, + "theoretical_loss": 3.3537521318984522, + "tokens_seen": 2642411520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010070210631895688, + "loss": 1.2731, + "theoretical_loss": 3.3537390093467523, + "tokens_seen": 2642542592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010068204613841525, + "loss": 1.2416, + "theoretical_loss": 3.353725887628163, + "tokens_seen": 2642673664 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.49688416719436646, + "objective/train/avg_token_score": 0.0246419794857502, + "objective/train/docs_used": 1487881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.657961368560791, + "objective/train/original_loss": 3.7626101970672607, + "objective/train/theoretical_loss": 3.3537193270812544, + "objective/train/tokens_used": 2642739200, + "objective/train/unlikelihood_loss": 0.15107613801956177, + "theoretical_loss": 3.3537193270812544, + "tokens_seen": 2642739200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010066198595787363, + "loss": 1.2483, + "theoretical_loss": 3.3537127667425892, + "tokens_seen": 2642804736 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010064192577733199, + "loss": 1.21, + "theoretical_loss": 3.353699646689937, + "tokens_seen": 2642935808 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010062186559679037, + "loss": 1.1761, + "theoretical_loss": 3.3536865274701126, + "tokens_seen": 2643066880 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010060180541624874, + "loss": 1.1068, + "theoretical_loss": 3.3536734090830214, + "tokens_seen": 2643197952 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010058174523570712, + "loss": 1.0744, + "theoretical_loss": 3.35366029152857, + "tokens_seen": 2643329024 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001005616850551655, + "loss": 1.2807, + "theoretical_loss": 3.353647174806663, + "tokens_seen": 2643460096 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010054162487462387, + "loss": 1.249, + "theoretical_loss": 3.353634058917207, + "tokens_seen": 2643591168 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010052156469408225, + "loss": 1.2089, + "theoretical_loss": 3.353620943860108, + "tokens_seen": 2643722240 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010050150451354062, + "loss": 1.3896, + "theoretical_loss": 3.353607829635272, + "tokens_seen": 2643853312 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100481444332999, + "loss": 1.3176, + "theoretical_loss": 3.3535947162426045, + "tokens_seen": 2643984384 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010046138415245738, + "loss": 1.4229, + "theoretical_loss": 3.353581603682012, + "tokens_seen": 2644115456 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010044132397191574, + "loss": 1.239, + "theoretical_loss": 3.3535684919534, + "tokens_seen": 2644246528 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.3837365508079529, + "objective/train/avg_token_score": 0.019270477816462517, + "objective/train/docs_used": 1488290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2979968786239624, + "objective/train/original_loss": 4.704802513122559, + "objective/train/theoretical_loss": 3.3535553810566743, + "objective/train/tokens_used": 2644377600, + "objective/train/unlikelihood_loss": 0.11533823609352112, + "theoretical_loss": 3.3535553810566743, + "tokens_seen": 2644377600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010042126379137411, + "loss": 1.1827, + "theoretical_loss": 3.3535553810566743, + "tokens_seen": 2644377600 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001004012036108325, + "loss": 1.3757, + "theoretical_loss": 3.3535422709917415, + "tokens_seen": 2644508672 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010038114343029088, + "loss": 1.4245, + "theoretical_loss": 3.353529161758507, + "tokens_seen": 2644639744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010036108324974925, + "loss": 1.375, + "theoretical_loss": 3.353516053356877, + "tokens_seen": 2644770816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010034102306920763, + "loss": 1.0489, + "theoretical_loss": 3.353502945786758, + "tokens_seen": 2644901888 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100320962888666, + "loss": 1.1676, + "theoretical_loss": 3.3534898390480556, + "tokens_seen": 2645032960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010030090270812438, + "loss": 1.2159, + "theoretical_loss": 3.3534767331406763, + "tokens_seen": 2645164032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010028084252758276, + "loss": 1.2645, + "theoretical_loss": 3.3534636280645254, + "tokens_seen": 2645295104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010026078234704113, + "loss": 1.2226, + "theoretical_loss": 3.35345052381951, + "tokens_seen": 2645426176 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001002407221664995, + "loss": 1.3328, + "theoretical_loss": 3.353437420405535, + "tokens_seen": 2645557248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010022066198595787, + "loss": 1.3179, + "theoretical_loss": 3.3534243178225074, + "tokens_seen": 2645688320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010020060180541625, + "loss": 1.2647, + "theoretical_loss": 3.353411216070333, + "tokens_seen": 2645819392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010018054162487462, + "loss": 1.1479, + "theoretical_loss": 3.3533981151489187, + "tokens_seen": 2645950464 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.40860214829444885, + "objective/train/avg_token_score": 0.07499361038208008, + "objective/train/docs_used": 1489355, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2399685382843018, + "objective/train/original_loss": 3.993478775024414, + "objective/train/theoretical_loss": 3.3533915649997166, + "objective/train/tokens_used": 2646016000, + "objective/train/unlikelihood_loss": 0.15400739014148712, + "theoretical_loss": 3.3533915649997166, + "tokens_seen": 2646016000 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100160481444333, + "loss": 1.2969, + "theoretical_loss": 3.3533850150581697, + "tokens_seen": 2646081536 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010014042126379137, + "loss": 1.1985, + "theoretical_loss": 3.353371915797992, + "tokens_seen": 2646212608 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010012036108324975, + "loss": 1.3606, + "theoretical_loss": 3.353358817368293, + "tokens_seen": 2646343680 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010010030090270813, + "loss": 1.2452, + "theoretical_loss": 3.353345719768978, + "tokens_seen": 2646474752 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001000802407221665, + "loss": 1.3115, + "theoretical_loss": 3.353332622999954, + "tokens_seen": 2646605824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010006018054162486, + "loss": 1.479, + "theoretical_loss": 3.3533195270611262, + "tokens_seen": 2646736896 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010004012036108324, + "loss": 1.3978, + "theoretical_loss": 3.3533064319524017, + "tokens_seen": 2646867968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010002006018054163, + "loss": 1.1463, + "theoretical_loss": 3.3532933376736866, + "tokens_seen": 2646999040 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001, + "loss": 1.3013, + "theoretical_loss": 3.353280244224887, + "tokens_seen": 2647130112 + }, + { + "epoch": 0.8, + "learning_rate": 9.997993981945838e-05, + "loss": 1.1793, + "theoretical_loss": 3.353267151605909, + "tokens_seen": 2647261184 + }, + { + "epoch": 0.8, + "learning_rate": 9.995987963891676e-05, + "loss": 1.1761, + "theoretical_loss": 3.35325405981666, + "tokens_seen": 2647392256 + }, + { + "epoch": 0.8, + "learning_rate": 9.993981945837513e-05, + "loss": 1.3826, + "theoretical_loss": 3.353240968857045, + "tokens_seen": 2647523328 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.4488636255264282, + "objective/train/avg_token_score": 0.01175718940794468, + "objective/train/docs_used": 1489706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4132579565048218, + "objective/train/original_loss": 3.9285576343536377, + "objective/train/theoretical_loss": 3.3532278787269707, + "objective/train/tokens_used": 2647654400, + "objective/train/unlikelihood_loss": 0.11655532568693161, + "theoretical_loss": 3.3532278787269707, + "tokens_seen": 2647654400 + }, + { + "epoch": 0.8, + "learning_rate": 9.991975927783351e-05, + "loss": 1.2613, + "theoretical_loss": 3.3532278787269707, + "tokens_seen": 2647654400 + }, + { + "epoch": 0.8, + "learning_rate": 9.989969909729188e-05, + "loss": 1.2547, + "theoretical_loss": 3.353214789426344, + "tokens_seen": 2647785472 + }, + { + "epoch": 0.8, + "learning_rate": 9.987963891675026e-05, + "loss": 1.2982, + "theoretical_loss": 3.353201700955071, + "tokens_seen": 2647916544 + }, + { + "epoch": 0.8, + "learning_rate": 9.985957873620862e-05, + "loss": 1.3354, + "theoretical_loss": 3.3531886133130584, + "tokens_seen": 2648047616 + }, + { + "epoch": 0.8, + "learning_rate": 9.9839518555667e-05, + "loss": 1.2973, + "theoretical_loss": 3.353175526500212, + "tokens_seen": 2648178688 + }, + { + "epoch": 0.8, + "learning_rate": 9.981945837512537e-05, + "loss": 1.3268, + "theoretical_loss": 3.3531624405164386, + "tokens_seen": 2648309760 + }, + { + "epoch": 0.8, + "learning_rate": 9.979939819458375e-05, + "loss": 1.1815, + "theoretical_loss": 3.3531493553616447, + "tokens_seen": 2648440832 + }, + { + "epoch": 0.8, + "learning_rate": 9.977933801404212e-05, + "loss": 1.3832, + "theoretical_loss": 3.353136271035736, + "tokens_seen": 2648571904 + }, + { + "epoch": 0.8, + "learning_rate": 9.97592778335005e-05, + "loss": 1.4158, + "theoretical_loss": 3.3531231875386207, + "tokens_seen": 2648702976 + }, + { + "epoch": 0.8, + "learning_rate": 9.973921765295888e-05, + "loss": 1.3395, + "theoretical_loss": 3.3531101048702037, + "tokens_seen": 2648834048 + }, + { + "epoch": 0.8, + "learning_rate": 9.971915747241725e-05, + "loss": 1.361, + "theoretical_loss": 3.353097023030392, + "tokens_seen": 2648965120 + }, + { + "epoch": 0.8, + "learning_rate": 9.969909729187563e-05, + "loss": 1.3796, + "theoretical_loss": 3.3530839420190923, + "tokens_seen": 2649096192 + }, + { + "epoch": 0.8, + "learning_rate": 9.9679037111334e-05, + "loss": 1.2975, + "theoretical_loss": 3.353070861836211, + "tokens_seen": 2649227264 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.37738269567489624, + "objective/train/avg_token_score": 0.0074289580807089806, + "objective/train/docs_used": 1491205, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1632531881332397, + "objective/train/original_loss": 3.9999003410339355, + "objective/train/theoretical_loss": 3.3530643220553977, + "objective/train/tokens_used": 2649292800, + "objective/train/unlikelihood_loss": 0.1265093833208084, + "theoretical_loss": 3.3530643220553977, + "tokens_seen": 2649292800 + }, + { + "epoch": 0.8, + "learning_rate": 9.965897693079238e-05, + "loss": 1.1644, + "theoretical_loss": 3.3530577824816543, + "tokens_seen": 2649358336 + }, + { + "epoch": 0.8, + "learning_rate": 9.963891675025076e-05, + "loss": 1.1743, + "theoretical_loss": 3.3530447039553293, + "tokens_seen": 2649489408 + }, + { + "epoch": 0.8, + "learning_rate": 9.961885656970913e-05, + "loss": 1.2388, + "theoretical_loss": 3.3530316262571427, + "tokens_seen": 2649620480 + }, + { + "epoch": 0.8, + "learning_rate": 9.959879638916751e-05, + "loss": 1.2416, + "theoretical_loss": 3.3530185493870004, + "tokens_seen": 2649751552 + }, + { + "epoch": 0.8, + "learning_rate": 9.957873620862588e-05, + "loss": 1.185, + "theoretical_loss": 3.3530054733448096, + "tokens_seen": 2649882624 + }, + { + "epoch": 0.8, + "learning_rate": 9.955867602808426e-05, + "loss": 1.3292, + "theoretical_loss": 3.352992398130477, + "tokens_seen": 2650013696 + }, + { + "epoch": 0.8, + "learning_rate": 9.953861584754263e-05, + "loss": 1.2923, + "theoretical_loss": 3.3529793237439085, + "tokens_seen": 2650144768 + }, + { + "epoch": 0.8, + "learning_rate": 9.951855566700101e-05, + "loss": 1.305, + "theoretical_loss": 3.3529662501850117, + "tokens_seen": 2650275840 + }, + { + "epoch": 0.8, + "learning_rate": 9.949849548645939e-05, + "loss": 1.1814, + "theoretical_loss": 3.352953177453693, + "tokens_seen": 2650406912 + }, + { + "epoch": 0.8, + "learning_rate": 9.947843530591776e-05, + "loss": 1.236, + "theoretical_loss": 3.352940105549858, + "tokens_seen": 2650537984 + }, + { + "epoch": 0.8, + "learning_rate": 9.945837512537612e-05, + "loss": 1.2518, + "theoretical_loss": 3.3529270344734154, + "tokens_seen": 2650669056 + }, + { + "epoch": 0.8, + "learning_rate": 9.94383149448345e-05, + "loss": 1.2328, + "theoretical_loss": 3.3529139642242702, + "tokens_seen": 2650800128 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.3904569745063782, + "objective/train/avg_token_score": 0.020729629322886467, + "objective/train/docs_used": 1491801, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2473176717758179, + "objective/train/original_loss": 3.7924132347106934, + "objective/train/theoretical_loss": 3.35290089480233, + "objective/train/tokens_used": 2650931200, + "objective/train/unlikelihood_loss": 0.1717841625213623, + "theoretical_loss": 3.35290089480233, + "tokens_seen": 2650931200 + }, + { + "epoch": 0.8, + "learning_rate": 9.941825476429288e-05, + "loss": 1.1824, + "theoretical_loss": 3.35290089480233, + "tokens_seen": 2650931200 + }, + { + "epoch": 0.8, + "learning_rate": 9.939819458375125e-05, + "loss": 1.3919, + "theoretical_loss": 3.352887826207501, + "tokens_seen": 2651062272 + }, + { + "epoch": 0.8, + "learning_rate": 9.937813440320963e-05, + "loss": 1.1717, + "theoretical_loss": 3.3528747584396905, + "tokens_seen": 2651193344 + }, + { + "epoch": 0.8, + "learning_rate": 9.9358074222668e-05, + "loss": 1.3955, + "theoretical_loss": 3.352861691498805, + "tokens_seen": 2651324416 + }, + { + "epoch": 0.8, + "learning_rate": 9.933801404212638e-05, + "loss": 1.291, + "theoretical_loss": 3.352848625384752, + "tokens_seen": 2651455488 + }, + { + "epoch": 0.8, + "learning_rate": 9.931795386158475e-05, + "loss": 1.3656, + "theoretical_loss": 3.352835560097437, + "tokens_seen": 2651586560 + }, + { + "epoch": 0.8, + "learning_rate": 9.929789368104313e-05, + "loss": 1.3729, + "theoretical_loss": 3.352822495636768, + "tokens_seen": 2651717632 + }, + { + "epoch": 0.8, + "learning_rate": 9.927783350050152e-05, + "loss": 1.2382, + "theoretical_loss": 3.3528094320026507, + "tokens_seen": 2651848704 + }, + { + "epoch": 0.8, + "learning_rate": 9.925777331995988e-05, + "loss": 1.2162, + "theoretical_loss": 3.352796369194993, + "tokens_seen": 2651979776 + }, + { + "epoch": 0.8, + "learning_rate": 9.923771313941826e-05, + "loss": 1.1794, + "theoretical_loss": 3.352783307213701, + "tokens_seen": 2652110848 + }, + { + "epoch": 0.8, + "learning_rate": 9.921765295887663e-05, + "loss": 1.2463, + "theoretical_loss": 3.352770246058683, + "tokens_seen": 2652241920 + }, + { + "epoch": 0.8, + "learning_rate": 9.919759277833501e-05, + "loss": 1.3083, + "theoretical_loss": 3.352757185729844, + "tokens_seen": 2652372992 + }, + { + "epoch": 0.8, + "learning_rate": 9.917753259779339e-05, + "loss": 1.3244, + "theoretical_loss": 3.352744126227092, + "tokens_seen": 2652504064 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.39913246035575867, + "objective/train/avg_token_score": 0.006367010064423084, + "objective/train/docs_used": 1493158, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2216801643371582, + "objective/train/original_loss": 3.5998973846435547, + "objective/train/theoretical_loss": 3.3527375967854693, + "objective/train/tokens_used": 2652569600, + "objective/train/unlikelihood_loss": 0.2739179730415344, + "theoretical_loss": 3.3527375967854693, + "tokens_seen": 2652569600 + }, + { + "epoch": 0.8, + "learning_rate": 9.915747241725176e-05, + "loss": 1.3198, + "theoretical_loss": 3.3527310675503337, + "tokens_seen": 2652635136 + }, + { + "epoch": 0.8, + "learning_rate": 9.913741223671014e-05, + "loss": 1.2321, + "theoretical_loss": 3.352718009699476, + "tokens_seen": 2652766208 + }, + { + "epoch": 0.8, + "learning_rate": 9.911735205616851e-05, + "loss": 1.3215, + "theoretical_loss": 3.3527049526744257, + "tokens_seen": 2652897280 + }, + { + "epoch": 0.8, + "learning_rate": 9.909729187562689e-05, + "loss": 1.2419, + "theoretical_loss": 3.3526918964750907, + "tokens_seen": 2653028352 + }, + { + "epoch": 0.8, + "learning_rate": 9.907723169508525e-05, + "loss": 1.1832, + "theoretical_loss": 3.3526788411013766, + "tokens_seen": 2653159424 + }, + { + "epoch": 0.8, + "learning_rate": 9.905717151454363e-05, + "loss": 1.0819, + "theoretical_loss": 3.3526657865531915, + "tokens_seen": 2653290496 + }, + { + "epoch": 0.8, + "learning_rate": 9.9037111334002e-05, + "loss": 1.2174, + "theoretical_loss": 3.352652732830442, + "tokens_seen": 2653421568 + }, + { + "epoch": 0.8, + "learning_rate": 9.901705115346038e-05, + "loss": 1.2608, + "theoretical_loss": 3.352639679933035, + "tokens_seen": 2653552640 + }, + { + "epoch": 0.8, + "learning_rate": 9.899699097291875e-05, + "loss": 1.489, + "theoretical_loss": 3.352626627860878, + "tokens_seen": 2653683712 + }, + { + "epoch": 0.8, + "learning_rate": 9.897693079237713e-05, + "loss": 1.3697, + "theoretical_loss": 3.3526135766138774, + "tokens_seen": 2653814784 + }, + { + "epoch": 0.8, + "learning_rate": 9.89568706118355e-05, + "loss": 1.4526, + "theoretical_loss": 3.352600526191941, + "tokens_seen": 2653945856 + }, + { + "epoch": 0.8, + "learning_rate": 9.893681043129388e-05, + "loss": 1.3022, + "theoretical_loss": 3.3525874765949752, + "tokens_seen": 2654076928 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.4624877870082855, + "objective/train/avg_token_score": 0.01894267462193966, + "objective/train/docs_used": 1493841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4704859256744385, + "objective/train/original_loss": 3.85762619972229, + "objective/train/theoretical_loss": 3.352574427822888, + "objective/train/tokens_used": 2654208000, + "objective/train/unlikelihood_loss": 0.13125164806842804, + "theoretical_loss": 3.352574427822888, + "tokens_seen": 2654208000 + }, + { + "epoch": 0.8, + "learning_rate": 9.891675025075226e-05, + "loss": 1.3929, + "theoretical_loss": 3.352574427822888, + "tokens_seen": 2654208000 + }, + { + "epoch": 0.8, + "learning_rate": 9.889669007021065e-05, + "loss": 1.2252, + "theoretical_loss": 3.352561379875586, + "tokens_seen": 2654339072 + }, + { + "epoch": 0.8, + "learning_rate": 9.887662988966901e-05, + "loss": 1.2306, + "theoretical_loss": 3.3525483327529755, + "tokens_seen": 2654470144 + }, + { + "epoch": 0.8, + "learning_rate": 9.885656970912738e-05, + "loss": 1.1879, + "theoretical_loss": 3.352535286454965, + "tokens_seen": 2654601216 + }, + { + "epoch": 0.8, + "learning_rate": 9.883650952858576e-05, + "loss": 1.307, + "theoretical_loss": 3.3525222409814615, + "tokens_seen": 2654732288 + }, + { + "epoch": 0.8, + "learning_rate": 9.881644934804414e-05, + "loss": 1.3224, + "theoretical_loss": 3.3525091963323717, + "tokens_seen": 2654863360 + }, + { + "epoch": 0.8, + "learning_rate": 9.879638916750251e-05, + "loss": 1.248, + "theoretical_loss": 3.352496152507603, + "tokens_seen": 2654994432 + }, + { + "epoch": 0.8, + "learning_rate": 9.877632898696089e-05, + "loss": 1.4018, + "theoretical_loss": 3.3524831095070624, + "tokens_seen": 2655125504 + }, + { + "epoch": 0.8, + "learning_rate": 9.875626880641926e-05, + "loss": 1.2217, + "theoretical_loss": 3.3524700673306573, + "tokens_seen": 2655256576 + }, + { + "epoch": 0.8, + "learning_rate": 9.873620862587764e-05, + "loss": 1.19, + "theoretical_loss": 3.352457025978295, + "tokens_seen": 2655387648 + }, + { + "epoch": 0.8, + "learning_rate": 9.871614844533601e-05, + "loss": 1.2636, + "theoretical_loss": 3.3524439854498826, + "tokens_seen": 2655518720 + }, + { + "epoch": 0.8, + "learning_rate": 9.869608826479439e-05, + "loss": 1.1503, + "theoretical_loss": 3.352430945745328, + "tokens_seen": 2655649792 + }, + { + "epoch": 0.8, + "learning_rate": 9.867602808425275e-05, + "loss": 1.2282, + "theoretical_loss": 3.3524179068645377, + "tokens_seen": 2655780864 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.49095797538757324, + "objective/train/avg_token_score": 0.006420293357223272, + "objective/train/docs_used": 1494379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.452853798866272, + "objective/train/original_loss": 3.478565216064453, + "objective/train/theoretical_loss": 3.3524113877330253, + "objective/train/tokens_used": 2655846400, + "objective/train/unlikelihood_loss": 0.1664523184299469, + "theoretical_loss": 3.3524113877330253, + "tokens_seen": 2655846400 + }, + { + "epoch": 0.8, + "learning_rate": 9.865596790371113e-05, + "loss": 1.3069, + "theoretical_loss": 3.3524048688074193, + "tokens_seen": 2655911936 + }, + { + "epoch": 0.8, + "learning_rate": 9.86359077231695e-05, + "loss": 1.3338, + "theoretical_loss": 3.35239183157388, + "tokens_seen": 2656043008 + }, + { + "epoch": 0.8, + "learning_rate": 9.861584754262788e-05, + "loss": 1.2018, + "theoretical_loss": 3.3523787951638275, + "tokens_seen": 2656174080 + }, + { + "epoch": 0.8, + "learning_rate": 9.859578736208626e-05, + "loss": 1.2104, + "theoretical_loss": 3.352365759577169, + "tokens_seen": 2656305152 + }, + { + "epoch": 0.8, + "learning_rate": 9.857572718154463e-05, + "loss": 1.1317, + "theoretical_loss": 3.3523527248138114, + "tokens_seen": 2656436224 + }, + { + "epoch": 0.81, + "learning_rate": 9.855566700100301e-05, + "loss": 1.1492, + "theoretical_loss": 3.352339690873663, + "tokens_seen": 2656567296 + }, + { + "epoch": 0.81, + "learning_rate": 9.85356068204614e-05, + "loss": 1.1762, + "theoretical_loss": 3.35232665775663, + "tokens_seen": 2656698368 + }, + { + "epoch": 0.81, + "learning_rate": 9.851554663991977e-05, + "loss": 1.183, + "theoretical_loss": 3.352313625462621, + "tokens_seen": 2656829440 + }, + { + "epoch": 0.81, + "learning_rate": 9.849548645937815e-05, + "loss": 1.3456, + "theoretical_loss": 3.352300593991543, + "tokens_seen": 2656960512 + }, + { + "epoch": 0.81, + "learning_rate": 9.847542627883651e-05, + "loss": 1.2732, + "theoretical_loss": 3.352287563343303, + "tokens_seen": 2657091584 + }, + { + "epoch": 0.81, + "learning_rate": 9.845536609829489e-05, + "loss": 1.2459, + "theoretical_loss": 3.352274533517809, + "tokens_seen": 2657222656 + }, + { + "epoch": 0.81, + "learning_rate": 9.843530591775326e-05, + "loss": 1.1798, + "theoretical_loss": 3.3522615045149684, + "tokens_seen": 2657353728 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.2653347849845886, + "objective/train/avg_token_score": 0.027506673708558083, + "objective/train/docs_used": 1495638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8548570871353149, + "objective/train/original_loss": 5.163658142089844, + "objective/train/theoretical_loss": 3.3522484763346885, + "objective/train/tokens_used": 2657484800, + "objective/train/unlikelihood_loss": 0.0953448936343193, + "theoretical_loss": 3.3522484763346885, + "tokens_seen": 2657484800 + }, + { + "epoch": 0.81, + "learning_rate": 9.841524573721164e-05, + "loss": 1.1783, + "theoretical_loss": 3.3522484763346885, + "tokens_seen": 2657484800 + }, + { + "epoch": 0.81, + "learning_rate": 9.839518555667001e-05, + "loss": 1.3015, + "theoretical_loss": 3.352235448976877, + "tokens_seen": 2657615872 + }, + { + "epoch": 0.81, + "learning_rate": 9.837512537612839e-05, + "loss": 1.3113, + "theoretical_loss": 3.3522224224414408, + "tokens_seen": 2657746944 + }, + { + "epoch": 0.81, + "learning_rate": 9.835506519558677e-05, + "loss": 1.3858, + "theoretical_loss": 3.352209396728288, + "tokens_seen": 2657878016 + }, + { + "epoch": 0.81, + "learning_rate": 9.833500501504514e-05, + "loss": 1.1368, + "theoretical_loss": 3.3521963718373264, + "tokens_seen": 2658009088 + }, + { + "epoch": 0.81, + "learning_rate": 9.831494483450352e-05, + "loss": 1.3311, + "theoretical_loss": 3.3521833477684635, + "tokens_seen": 2658140160 + }, + { + "epoch": 0.81, + "learning_rate": 9.829488465396188e-05, + "loss": 1.3003, + "theoretical_loss": 3.352170324521606, + "tokens_seen": 2658271232 + }, + { + "epoch": 0.81, + "learning_rate": 9.827482447342026e-05, + "loss": 1.2277, + "theoretical_loss": 3.3521573020966624, + "tokens_seen": 2658402304 + }, + { + "epoch": 0.81, + "learning_rate": 9.825476429287863e-05, + "loss": 1.1593, + "theoretical_loss": 3.3521442804935404, + "tokens_seen": 2658533376 + }, + { + "epoch": 0.81, + "learning_rate": 9.823470411233701e-05, + "loss": 1.3097, + "theoretical_loss": 3.352131259712147, + "tokens_seen": 2658664448 + }, + { + "epoch": 0.81, + "learning_rate": 9.821464393179538e-05, + "loss": 1.216, + "theoretical_loss": 3.3521182397523894, + "tokens_seen": 2658795520 + }, + { + "epoch": 0.81, + "learning_rate": 9.819458375125376e-05, + "loss": 1.2245, + "theoretical_loss": 3.3521052206141766, + "tokens_seen": 2658926592 + }, + { + "epoch": 0.81, + "learning_rate": 9.817452357071213e-05, + "loss": 1.239, + "theoretical_loss": 3.3520922022974156, + "tokens_seen": 2659057664 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.4679863154888153, + "objective/train/avg_token_score": 0.031814493238925934, + "objective/train/docs_used": 1496378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.441231369972229, + "objective/train/original_loss": 3.803694009780884, + "objective/train/theoretical_loss": 3.3520856934470507, + "objective/train/tokens_used": 2659123200, + "objective/train/unlikelihood_loss": 0.133505180478096, + "theoretical_loss": 3.3520856934470507, + "tokens_seen": 2659123200 + }, + { + "epoch": 0.81, + "learning_rate": 9.815446339017052e-05, + "loss": 1.3641, + "theoretical_loss": 3.352079184802014, + "tokens_seen": 2659188736 + }, + { + "epoch": 0.81, + "learning_rate": 9.81344032096289e-05, + "loss": 1.1068, + "theoretical_loss": 3.3520661681278794, + "tokens_seen": 2659319808 + }, + { + "epoch": 0.81, + "learning_rate": 9.811434302908728e-05, + "loss": 1.261, + "theoretical_loss": 3.35205315227492, + "tokens_seen": 2659450880 + }, + { + "epoch": 0.81, + "learning_rate": 9.809428284854564e-05, + "loss": 1.2075, + "theoretical_loss": 3.352040137243043, + "tokens_seen": 2659581952 + }, + { + "epoch": 0.81, + "learning_rate": 9.807422266800401e-05, + "loss": 1.343, + "theoretical_loss": 3.3520271230321566, + "tokens_seen": 2659713024 + }, + { + "epoch": 0.81, + "learning_rate": 9.805416248746239e-05, + "loss": 1.2426, + "theoretical_loss": 3.352014109642168, + "tokens_seen": 2659844096 + }, + { + "epoch": 0.81, + "learning_rate": 9.803410230692076e-05, + "loss": 1.3185, + "theoretical_loss": 3.352001097072985, + "tokens_seen": 2659975168 + }, + { + "epoch": 0.81, + "learning_rate": 9.801404212637914e-05, + "loss": 1.2245, + "theoretical_loss": 3.3519880853245163, + "tokens_seen": 2660106240 + }, + { + "epoch": 0.81, + "learning_rate": 9.799398194583752e-05, + "loss": 1.2612, + "theoretical_loss": 3.3519750743966688, + "tokens_seen": 2660237312 + }, + { + "epoch": 0.81, + "learning_rate": 9.797392176529589e-05, + "loss": 1.3792, + "theoretical_loss": 3.3519620642893506, + "tokens_seen": 2660368384 + }, + { + "epoch": 0.81, + "learning_rate": 9.795386158475427e-05, + "loss": 1.3142, + "theoretical_loss": 3.3519490550024695, + "tokens_seen": 2660499456 + }, + { + "epoch": 0.81, + "learning_rate": 9.793380140421264e-05, + "loss": 1.2278, + "theoretical_loss": 3.3519360465359336, + "tokens_seen": 2660630528 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.3397482931613922, + "objective/train/avg_token_score": 0.01035211980342865, + "objective/train/docs_used": 1497834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1545310020446777, + "objective/train/original_loss": 3.890047311782837, + "objective/train/theoretical_loss": 3.3519230388896504, + "objective/train/tokens_used": 2660761600, + "objective/train/unlikelihood_loss": 0.2319631725549698, + "theoretical_loss": 3.3519230388896504, + "tokens_seen": 2660761600 + }, + { + "epoch": 0.81, + "learning_rate": 9.791374122367102e-05, + "loss": 1.2663, + "theoretical_loss": 3.3519230388896504, + "tokens_seen": 2660761600 + }, + { + "epoch": 0.81, + "learning_rate": 9.789368104312938e-05, + "loss": 1.2232, + "theoretical_loss": 3.351910032063528, + "tokens_seen": 2660892672 + }, + { + "epoch": 0.81, + "learning_rate": 9.787362086258776e-05, + "loss": 1.5238, + "theoretical_loss": 3.3518970260574736, + "tokens_seen": 2661023744 + }, + { + "epoch": 0.81, + "learning_rate": 9.785356068204613e-05, + "loss": 1.1802, + "theoretical_loss": 3.3518840208713967, + "tokens_seen": 2661154816 + }, + { + "epoch": 0.81, + "learning_rate": 9.783350050150451e-05, + "loss": 1.3932, + "theoretical_loss": 3.3518710165052035, + "tokens_seen": 2661285888 + }, + { + "epoch": 0.81, + "learning_rate": 9.781344032096288e-05, + "loss": 1.3411, + "theoretical_loss": 3.351858012958803, + "tokens_seen": 2661416960 + }, + { + "epoch": 0.81, + "learning_rate": 9.779338014042126e-05, + "loss": 1.2331, + "theoretical_loss": 3.3518450102321027, + "tokens_seen": 2661548032 + }, + { + "epoch": 0.81, + "learning_rate": 9.777331995987965e-05, + "loss": 1.4732, + "theoretical_loss": 3.3518320083250104, + "tokens_seen": 2661679104 + }, + { + "epoch": 0.81, + "learning_rate": 9.775325977933803e-05, + "loss": 1.2461, + "theoretical_loss": 3.3518190072374345, + "tokens_seen": 2661810176 + }, + { + "epoch": 0.81, + "learning_rate": 9.77331995987964e-05, + "loss": 1.4106, + "theoretical_loss": 3.351806006969283, + "tokens_seen": 2661941248 + }, + { + "epoch": 0.81, + "learning_rate": 9.771313941825476e-05, + "loss": 1.3709, + "theoretical_loss": 3.351793007520464, + "tokens_seen": 2662072320 + }, + { + "epoch": 0.81, + "learning_rate": 9.769307923771314e-05, + "loss": 1.3412, + "theoretical_loss": 3.351780008890885, + "tokens_seen": 2662203392 + }, + { + "epoch": 0.81, + "learning_rate": 9.767301905717152e-05, + "loss": 1.2367, + "theoretical_loss": 3.351767011080454, + "tokens_seen": 2662334464 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.48552054166793823, + "objective/train/avg_token_score": 0.006614937912672758, + "objective/train/docs_used": 1498376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.578578233718872, + "objective/train/original_loss": 3.6186280250549316, + "objective/train/theoretical_loss": 3.3517605124823904, + "objective/train/tokens_used": 2662400000, + "objective/train/unlikelihood_loss": 0.17294025421142578, + "theoretical_loss": 3.3517605124823904, + "tokens_seen": 2662400000 + }, + { + "epoch": 0.81, + "learning_rate": 9.765295887662989e-05, + "loss": 1.3075, + "theoretical_loss": 3.35175401408908, + "tokens_seen": 2662465536 + }, + { + "epoch": 0.81, + "learning_rate": 9.763289869608827e-05, + "loss": 1.2214, + "theoretical_loss": 3.35174101791667, + "tokens_seen": 2662596608 + }, + { + "epoch": 0.81, + "learning_rate": 9.761283851554664e-05, + "loss": 1.3492, + "theoretical_loss": 3.3517280225631327, + "tokens_seen": 2662727680 + }, + { + "epoch": 0.81, + "learning_rate": 9.759277833500502e-05, + "loss": 1.3305, + "theoretical_loss": 3.351715028028376, + "tokens_seen": 2662858752 + }, + { + "epoch": 0.81, + "learning_rate": 9.75727181544634e-05, + "loss": 1.0578, + "theoretical_loss": 3.351702034312308, + "tokens_seen": 2662989824 + }, + { + "epoch": 0.81, + "learning_rate": 9.755265797392177e-05, + "loss": 1.3662, + "theoretical_loss": 3.351689041414837, + "tokens_seen": 2663120896 + }, + { + "epoch": 0.81, + "learning_rate": 9.753259779338015e-05, + "loss": 1.3488, + "theoretical_loss": 3.351676049335871, + "tokens_seen": 2663251968 + }, + { + "epoch": 0.81, + "learning_rate": 9.751253761283851e-05, + "loss": 1.3649, + "theoretical_loss": 3.351663058075318, + "tokens_seen": 2663383040 + }, + { + "epoch": 0.81, + "learning_rate": 9.749247743229688e-05, + "loss": 1.2497, + "theoretical_loss": 3.3516500676330865, + "tokens_seen": 2663514112 + }, + { + "epoch": 0.81, + "learning_rate": 9.747241725175526e-05, + "loss": 1.2242, + "theoretical_loss": 3.3516370780090847, + "tokens_seen": 2663645184 + }, + { + "epoch": 0.81, + "learning_rate": 9.745235707121364e-05, + "loss": 1.1057, + "theoretical_loss": 3.35162408920322, + "tokens_seen": 2663776256 + }, + { + "epoch": 0.81, + "learning_rate": 9.743229689067201e-05, + "loss": 1.3802, + "theoretical_loss": 3.351611101215402, + "tokens_seen": 2663907328 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.4923020601272583, + "objective/train/avg_token_score": 0.0321308858692646, + "objective/train/docs_used": 1499572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5439066886901855, + "objective/train/original_loss": 3.718693971633911, + "objective/train/theoretical_loss": 3.3515981140455375, + "objective/train/tokens_used": 2664038400, + "objective/train/unlikelihood_loss": 0.09321712702512741, + "theoretical_loss": 3.3515981140455375, + "tokens_seen": 2664038400 + }, + { + "epoch": 0.81, + "learning_rate": 9.741223671013039e-05, + "loss": 1.2152, + "theoretical_loss": 3.3515981140455375, + "tokens_seen": 2664038400 + }, + { + "epoch": 0.81, + "learning_rate": 9.739217652958878e-05, + "loss": 1.314, + "theoretical_loss": 3.3515851276935362, + "tokens_seen": 2664169472 + }, + { + "epoch": 0.81, + "learning_rate": 9.737211634904715e-05, + "loss": 1.3247, + "theoretical_loss": 3.351572142159305, + "tokens_seen": 2664300544 + }, + { + "epoch": 0.81, + "learning_rate": 9.735205616850553e-05, + "loss": 1.478, + "theoretical_loss": 3.351559157442753, + "tokens_seen": 2664431616 + }, + { + "epoch": 0.81, + "learning_rate": 9.73319959879639e-05, + "loss": 1.288, + "theoretical_loss": 3.351546173543788, + "tokens_seen": 2664562688 + }, + { + "epoch": 0.81, + "learning_rate": 9.731193580742227e-05, + "loss": 1.3119, + "theoretical_loss": 3.351533190462319, + "tokens_seen": 2664693760 + }, + { + "epoch": 0.81, + "learning_rate": 9.729187562688064e-05, + "loss": 1.1484, + "theoretical_loss": 3.351520208198254, + "tokens_seen": 2664824832 + }, + { + "epoch": 0.81, + "learning_rate": 9.727181544633902e-05, + "loss": 1.2122, + "theoretical_loss": 3.351507226751501, + "tokens_seen": 2664955904 + }, + { + "epoch": 0.81, + "learning_rate": 9.72517552657974e-05, + "loss": 1.3512, + "theoretical_loss": 3.3514942461219683, + "tokens_seen": 2665086976 + }, + { + "epoch": 0.81, + "learning_rate": 9.723169508525577e-05, + "loss": 1.1535, + "theoretical_loss": 3.3514812663095648, + "tokens_seen": 2665218048 + }, + { + "epoch": 0.81, + "learning_rate": 9.721163490471415e-05, + "loss": 1.2987, + "theoretical_loss": 3.3514682873141988, + "tokens_seen": 2665349120 + }, + { + "epoch": 0.81, + "learning_rate": 9.719157472417252e-05, + "loss": 1.205, + "theoretical_loss": 3.351455309135778, + "tokens_seen": 2665480192 + }, + { + "epoch": 0.81, + "learning_rate": 9.71715145436309e-05, + "loss": 1.3348, + "theoretical_loss": 3.3514423317742117, + "tokens_seen": 2665611264 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.29203322529792786, + "objective/train/avg_token_score": 0.03380707651376724, + "objective/train/docs_used": 1500816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7567684650421143, + "objective/train/original_loss": 3.778568744659424, + "objective/train/theoretical_loss": 3.3514358433997202, + "objective/train/tokens_used": 2665676800, + "objective/train/unlikelihood_loss": 0.1839800328016281, + "theoretical_loss": 3.3514358433997202, + "tokens_seen": 2665676800 + }, + { + "epoch": 0.81, + "learning_rate": 9.715145436308927e-05, + "loss": 1.2025, + "theoretical_loss": 3.351429355229408, + "tokens_seen": 2665742336 + }, + { + "epoch": 0.81, + "learning_rate": 9.713139418254765e-05, + "loss": 1.1466, + "theoretical_loss": 3.3514163795012752, + "tokens_seen": 2665873408 + }, + { + "epoch": 0.81, + "learning_rate": 9.711133400200601e-05, + "loss": 1.2889, + "theoretical_loss": 3.3514034045897216, + "tokens_seen": 2666004480 + }, + { + "epoch": 0.81, + "learning_rate": 9.709127382146439e-05, + "loss": 1.2518, + "theoretical_loss": 3.3513904304946562, + "tokens_seen": 2666135552 + }, + { + "epoch": 0.81, + "learning_rate": 9.707121364092276e-05, + "loss": 1.3209, + "theoretical_loss": 3.351377457215987, + "tokens_seen": 2666266624 + }, + { + "epoch": 0.81, + "learning_rate": 9.705115346038114e-05, + "loss": 1.323, + "theoretical_loss": 3.3513644847536233, + "tokens_seen": 2666397696 + }, + { + "epoch": 0.81, + "learning_rate": 9.703109327983953e-05, + "loss": 1.2304, + "theoretical_loss": 3.3513515131074723, + "tokens_seen": 2666528768 + }, + { + "epoch": 0.81, + "learning_rate": 9.70110330992979e-05, + "loss": 1.2812, + "theoretical_loss": 3.3513385422774435, + "tokens_seen": 2666659840 + }, + { + "epoch": 0.81, + "learning_rate": 9.699097291875628e-05, + "loss": 1.0919, + "theoretical_loss": 3.3513255722634456, + "tokens_seen": 2666790912 + }, + { + "epoch": 0.81, + "learning_rate": 9.697091273821465e-05, + "loss": 1.1807, + "theoretical_loss": 3.351312603065386, + "tokens_seen": 2666921984 + }, + { + "epoch": 0.81, + "learning_rate": 9.695085255767303e-05, + "loss": 1.2553, + "theoretical_loss": 3.3512996346831745, + "tokens_seen": 2667053056 + }, + { + "epoch": 0.81, + "learning_rate": 9.693079237713139e-05, + "loss": 1.2469, + "theoretical_loss": 3.351286667116719, + "tokens_seen": 2667184128 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.30296921730041504, + "objective/train/avg_token_score": 0.02518758550286293, + "objective/train/docs_used": 1501537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0230324268341064, + "objective/train/original_loss": 4.1214070320129395, + "objective/train/theoretical_loss": 3.3512737003659288, + "objective/train/tokens_used": 2667315200, + "objective/train/unlikelihood_loss": 0.16172662377357483, + "theoretical_loss": 3.3512737003659288, + "tokens_seen": 2667315200 + }, + { + "epoch": 0.81, + "learning_rate": 9.691073219658977e-05, + "loss": 1.2092, + "theoretical_loss": 3.3512737003659288, + "tokens_seen": 2667315200 + }, + { + "epoch": 0.81, + "learning_rate": 9.689067201604814e-05, + "loss": 1.1864, + "theoretical_loss": 3.3512607344307113, + "tokens_seen": 2667446272 + }, + { + "epoch": 0.81, + "learning_rate": 9.687061183550652e-05, + "loss": 1.1825, + "theoretical_loss": 3.351247769310976, + "tokens_seen": 2667577344 + }, + { + "epoch": 0.81, + "learning_rate": 9.68505516549649e-05, + "loss": 1.2599, + "theoretical_loss": 3.351234805006632, + "tokens_seen": 2667708416 + }, + { + "epoch": 0.81, + "learning_rate": 9.683049147442327e-05, + "loss": 1.2783, + "theoretical_loss": 3.3512218415175865, + "tokens_seen": 2667839488 + }, + { + "epoch": 0.81, + "learning_rate": 9.681043129388165e-05, + "loss": 1.1777, + "theoretical_loss": 3.3512088788437495, + "tokens_seen": 2667970560 + }, + { + "epoch": 0.81, + "learning_rate": 9.679037111334002e-05, + "loss": 1.3165, + "theoretical_loss": 3.351195916985029, + "tokens_seen": 2668101632 + }, + { + "epoch": 0.81, + "learning_rate": 9.67703109327984e-05, + "loss": 1.1792, + "theoretical_loss": 3.351182955941334, + "tokens_seen": 2668232704 + }, + { + "epoch": 0.81, + "learning_rate": 9.675025075225677e-05, + "loss": 1.2774, + "theoretical_loss": 3.3511699957125733, + "tokens_seen": 2668363776 + }, + { + "epoch": 0.81, + "learning_rate": 9.673019057171514e-05, + "loss": 1.2327, + "theoretical_loss": 3.3511570362986554, + "tokens_seen": 2668494848 + }, + { + "epoch": 0.81, + "learning_rate": 9.671013039117351e-05, + "loss": 1.3106, + "theoretical_loss": 3.351144077699489, + "tokens_seen": 2668625920 + }, + { + "epoch": 0.81, + "learning_rate": 9.669007021063189e-05, + "loss": 1.2289, + "theoretical_loss": 3.3511311199149834, + "tokens_seen": 2668756992 + }, + { + "epoch": 0.81, + "learning_rate": 9.667001003009026e-05, + "loss": 1.2001, + "theoretical_loss": 3.3511181629450464, + "tokens_seen": 2668888064 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.4080522954463959, + "objective/train/avg_token_score": 0.044903259724378586, + "objective/train/docs_used": 1502272, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3515387773513794, + "objective/train/original_loss": 3.731774091720581, + "objective/train/theoretical_loss": 3.351111684765513, + "objective/train/tokens_used": 2668953600, + "objective/train/unlikelihood_loss": 0.17190276086330414, + "theoretical_loss": 3.351111684765513, + "tokens_seen": 2668953600 + }, + { + "epoch": 0.81, + "learning_rate": 9.664994984954865e-05, + "loss": 1.2201, + "theoretical_loss": 3.3511052067895877, + "tokens_seen": 2669019136 + }, + { + "epoch": 0.81, + "learning_rate": 9.662988966900703e-05, + "loss": 1.4584, + "theoretical_loss": 3.3510922514485157, + "tokens_seen": 2669150208 + }, + { + "epoch": 0.81, + "learning_rate": 9.66098294884654e-05, + "loss": 1.2804, + "theoretical_loss": 3.351079296921739, + "tokens_seen": 2669281280 + }, + { + "epoch": 0.81, + "learning_rate": 9.658976930792378e-05, + "loss": 1.3549, + "theoretical_loss": 3.351066343209167, + "tokens_seen": 2669412352 + }, + { + "epoch": 0.81, + "learning_rate": 9.656970912738216e-05, + "loss": 1.2105, + "theoretical_loss": 3.3510533903107085, + "tokens_seen": 2669543424 + }, + { + "epoch": 0.81, + "learning_rate": 9.654964894684053e-05, + "loss": 1.2586, + "theoretical_loss": 3.351040438226272, + "tokens_seen": 2669674496 + }, + { + "epoch": 0.81, + "learning_rate": 9.65295887662989e-05, + "loss": 1.2992, + "theoretical_loss": 3.3510274869557666, + "tokens_seen": 2669805568 + }, + { + "epoch": 0.81, + "learning_rate": 9.650952858575727e-05, + "loss": 1.3161, + "theoretical_loss": 3.351014536499101, + "tokens_seen": 2669936640 + }, + { + "epoch": 0.81, + "learning_rate": 9.648946840521565e-05, + "loss": 1.3025, + "theoretical_loss": 3.351001586856184, + "tokens_seen": 2670067712 + }, + { + "epoch": 0.81, + "learning_rate": 9.646940822467402e-05, + "loss": 1.4333, + "theoretical_loss": 3.3509886380269247, + "tokens_seen": 2670198784 + }, + { + "epoch": 0.81, + "learning_rate": 9.64493480441324e-05, + "loss": 1.2228, + "theoretical_loss": 3.3509756900112326, + "tokens_seen": 2670329856 + }, + { + "epoch": 0.81, + "learning_rate": 9.642928786359077e-05, + "loss": 1.2721, + "theoretical_loss": 3.350962742809016, + "tokens_seen": 2670460928 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.25836998224258423, + "objective/train/avg_token_score": 0.04649694636464119, + "objective/train/docs_used": 1503662, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.5875056982040405, + "objective/train/original_loss": 4.2908525466918945, + "objective/train/theoretical_loss": 3.350949796420184, + "objective/train/tokens_used": 2670592000, + "objective/train/unlikelihood_loss": 0.12709859013557434, + "theoretical_loss": 3.350949796420184, + "tokens_seen": 2670592000 + }, + { + "epoch": 0.81, + "learning_rate": 9.640922768304915e-05, + "loss": 1.2836, + "theoretical_loss": 3.350949796420184, + "tokens_seen": 2670592000 + }, + { + "epoch": 0.81, + "learning_rate": 9.638916750250753e-05, + "loss": 1.0667, + "theoretical_loss": 3.3509368508446453, + "tokens_seen": 2670723072 + }, + { + "epoch": 0.81, + "learning_rate": 9.63691073219659e-05, + "loss": 1.3202, + "theoretical_loss": 3.3509239060823095, + "tokens_seen": 2670854144 + }, + { + "epoch": 0.81, + "learning_rate": 9.634904714142426e-05, + "loss": 1.3155, + "theoretical_loss": 3.3509109621330855, + "tokens_seen": 2670985216 + }, + { + "epoch": 0.81, + "learning_rate": 9.632898696088264e-05, + "loss": 1.246, + "theoretical_loss": 3.3508980189968818, + "tokens_seen": 2671116288 + }, + { + "epoch": 0.81, + "learning_rate": 9.630892678034102e-05, + "loss": 1.3138, + "theoretical_loss": 3.350885076673608, + "tokens_seen": 2671247360 + }, + { + "epoch": 0.81, + "learning_rate": 9.628886659979939e-05, + "loss": 1.3806, + "theoretical_loss": 3.350872135163173, + "tokens_seen": 2671378432 + }, + { + "epoch": 0.81, + "learning_rate": 9.626880641925778e-05, + "loss": 1.3543, + "theoretical_loss": 3.3508591944654853, + "tokens_seen": 2671509504 + }, + { + "epoch": 0.81, + "learning_rate": 9.624874623871616e-05, + "loss": 1.3975, + "theoretical_loss": 3.350846254580455, + "tokens_seen": 2671640576 + }, + { + "epoch": 0.81, + "learning_rate": 9.622868605817453e-05, + "loss": 1.3639, + "theoretical_loss": 3.3508333155079906, + "tokens_seen": 2671771648 + }, + { + "epoch": 0.81, + "learning_rate": 9.620862587763291e-05, + "loss": 1.3444, + "theoretical_loss": 3.3508203772480014, + "tokens_seen": 2671902720 + }, + { + "epoch": 0.81, + "learning_rate": 9.618856569709128e-05, + "loss": 1.2843, + "theoretical_loss": 3.350807439800396, + "tokens_seen": 2672033792 + }, + { + "epoch": 0.81, + "learning_rate": 9.616850551654966e-05, + "loss": 1.4235, + "theoretical_loss": 3.3507945031650843, + "tokens_seen": 2672164864 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.6066715717315674, + "objective/train/avg_token_score": 0.0023586663883179426, + "objective/train/docs_used": 1504337, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.729880452156067, + "objective/train/original_loss": 3.1985061168670654, + "objective/train/theoretical_loss": 3.3507880351520103, + "objective/train/tokens_used": 2672230400, + "objective/train/unlikelihood_loss": 0.13412892818450928, + "theoretical_loss": 3.3507880351520103, + "tokens_seen": 2672230400 + }, + { + "epoch": 0.81, + "learning_rate": 9.614844533600802e-05, + "loss": 1.4617, + "theoretical_loss": 3.3507815673419756, + "tokens_seen": 2672295936 + }, + { + "epoch": 0.81, + "learning_rate": 9.61283851554664e-05, + "loss": 1.2773, + "theoretical_loss": 3.350768632330978, + "tokens_seen": 2672427008 + }, + { + "epoch": 0.81, + "learning_rate": 9.610832497492477e-05, + "loss": 1.2637, + "theoretical_loss": 3.3507556981320015, + "tokens_seen": 2672558080 + }, + { + "epoch": 0.81, + "learning_rate": 9.608826479438315e-05, + "loss": 1.1287, + "theoretical_loss": 3.3507427647449552, + "tokens_seen": 2672689152 + }, + { + "epoch": 0.81, + "learning_rate": 9.606820461384152e-05, + "loss": 1.3609, + "theoretical_loss": 3.350729832169748, + "tokens_seen": 2672820224 + }, + { + "epoch": 0.81, + "learning_rate": 9.60481444332999e-05, + "loss": 1.4659, + "theoretical_loss": 3.3507169004062893, + "tokens_seen": 2672951296 + }, + { + "epoch": 0.81, + "learning_rate": 9.602808425275828e-05, + "loss": 1.2362, + "theoretical_loss": 3.350703969454489, + "tokens_seen": 2673082368 + }, + { + "epoch": 0.81, + "learning_rate": 9.600802407221665e-05, + "loss": 1.245, + "theoretical_loss": 3.3506910393142553, + "tokens_seen": 2673213440 + }, + { + "epoch": 0.81, + "learning_rate": 9.598796389167503e-05, + "loss": 1.1795, + "theoretical_loss": 3.3506781099854983, + "tokens_seen": 2673344512 + }, + { + "epoch": 0.81, + "learning_rate": 9.59679037111334e-05, + "loss": 1.1465, + "theoretical_loss": 3.3506651814681265, + "tokens_seen": 2673475584 + }, + { + "epoch": 0.81, + "learning_rate": 9.594784353059177e-05, + "loss": 1.2806, + "theoretical_loss": 3.35065225376205, + "tokens_seen": 2673606656 + }, + { + "epoch": 0.81, + "learning_rate": 9.592778335005014e-05, + "loss": 1.4094, + "theoretical_loss": 3.3506393268671775, + "tokens_seen": 2673737728 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.47782257199287415, + "objective/train/avg_token_score": 0.003605975303798914, + "objective/train/docs_used": 1505818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.450667142868042, + "objective/train/original_loss": 3.332578182220459, + "objective/train/theoretical_loss": 3.3506264007834186, + "objective/train/tokens_used": 2673868800, + "objective/train/unlikelihood_loss": 0.15881864726543427, + "theoretical_loss": 3.3506264007834186, + "tokens_seen": 2673868800 + }, + { + "epoch": 0.81, + "learning_rate": 9.590772316950853e-05, + "loss": 1.2755, + "theoretical_loss": 3.3506264007834186, + "tokens_seen": 2673868800 + }, + { + "epoch": 0.81, + "learning_rate": 9.588766298896691e-05, + "loss": 1.2656, + "theoretical_loss": 3.350613475510683, + "tokens_seen": 2673999872 + }, + { + "epoch": 0.81, + "learning_rate": 9.586760280842528e-05, + "loss": 1.4452, + "theoretical_loss": 3.3506005510488794, + "tokens_seen": 2674130944 + }, + { + "epoch": 0.81, + "learning_rate": 9.584754262788366e-05, + "loss": 1.2831, + "theoretical_loss": 3.3505876273979176, + "tokens_seen": 2674262016 + }, + { + "epoch": 0.81, + "learning_rate": 9.582748244734203e-05, + "loss": 1.2823, + "theoretical_loss": 3.350574704557707, + "tokens_seen": 2674393088 + }, + { + "epoch": 0.81, + "learning_rate": 9.580742226680041e-05, + "loss": 1.1329, + "theoretical_loss": 3.3505617825281564, + "tokens_seen": 2674524160 + }, + { + "epoch": 0.81, + "learning_rate": 9.578736208625879e-05, + "loss": 1.2909, + "theoretical_loss": 3.350548861309176, + "tokens_seen": 2674655232 + }, + { + "epoch": 0.81, + "learning_rate": 9.576730190571716e-05, + "loss": 1.2028, + "theoretical_loss": 3.350535940900675, + "tokens_seen": 2674786304 + }, + { + "epoch": 0.81, + "learning_rate": 9.574724172517552e-05, + "loss": 1.261, + "theoretical_loss": 3.3505230213025627, + "tokens_seen": 2674917376 + }, + { + "epoch": 0.81, + "learning_rate": 9.57271815446339e-05, + "loss": 1.3396, + "theoretical_loss": 3.350510102514749, + "tokens_seen": 2675048448 + }, + { + "epoch": 0.81, + "learning_rate": 9.570712136409228e-05, + "loss": 1.3645, + "theoretical_loss": 3.3504971845371427, + "tokens_seen": 2675179520 + }, + { + "epoch": 0.81, + "learning_rate": 9.568706118355065e-05, + "loss": 1.4175, + "theoretical_loss": 3.3504842673696538, + "tokens_seen": 2675310592 + }, + { + "epoch": 0.81, + "learning_rate": 9.566700100300903e-05, + "loss": 1.3285, + "theoretical_loss": 3.3504713510121915, + "tokens_seen": 2675441664 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.35673263669013977, + "objective/train/avg_token_score": 0.023167684674263, + "objective/train/docs_used": 1507167, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.225681185722351, + "objective/train/original_loss": 3.8578453063964844, + "objective/train/theoretical_loss": 3.3504648931371923, + "objective/train/tokens_used": 2675507200, + "objective/train/unlikelihood_loss": 0.17090138792991638, + "theoretical_loss": 3.3504648931371923, + "tokens_seen": 2675507200 + }, + { + "epoch": 0.81, + "learning_rate": 9.56469408224674e-05, + "loss": 1.3548, + "theoretical_loss": 3.350458435464666, + "tokens_seen": 2675572736 + }, + { + "epoch": 0.81, + "learning_rate": 9.562688064192578e-05, + "loss": 1.4085, + "theoretical_loss": 3.3504455207269856, + "tokens_seen": 2675703808 + }, + { + "epoch": 0.81, + "learning_rate": 9.560682046138415e-05, + "loss": 1.3883, + "theoretical_loss": 3.350432606799061, + "tokens_seen": 2675834880 + }, + { + "epoch": 0.81, + "learning_rate": 9.558676028084253e-05, + "loss": 1.408, + "theoretical_loss": 3.350419693680801, + "tokens_seen": 2675965952 + }, + { + "epoch": 0.81, + "learning_rate": 9.556670010030089e-05, + "loss": 1.1492, + "theoretical_loss": 3.350406781372116, + "tokens_seen": 2676097024 + }, + { + "epoch": 0.81, + "learning_rate": 9.554663991975927e-05, + "loss": 1.2456, + "theoretical_loss": 3.3503938698729145, + "tokens_seen": 2676228096 + }, + { + "epoch": 0.81, + "learning_rate": 9.552657973921766e-05, + "loss": 1.1367, + "theoretical_loss": 3.350380959183107, + "tokens_seen": 2676359168 + }, + { + "epoch": 0.81, + "learning_rate": 9.550651955867603e-05, + "loss": 1.3905, + "theoretical_loss": 3.350368049302603, + "tokens_seen": 2676490240 + }, + { + "epoch": 0.81, + "learning_rate": 9.548645937813441e-05, + "loss": 1.2792, + "theoretical_loss": 3.350355140231312, + "tokens_seen": 2676621312 + }, + { + "epoch": 0.81, + "learning_rate": 9.546639919759278e-05, + "loss": 1.1788, + "theoretical_loss": 3.350342231969143, + "tokens_seen": 2676752384 + }, + { + "epoch": 0.81, + "learning_rate": 9.544633901705116e-05, + "loss": 1.3094, + "theoretical_loss": 3.350329324516007, + "tokens_seen": 2676883456 + }, + { + "epoch": 0.81, + "learning_rate": 9.542627883650954e-05, + "loss": 1.3294, + "theoretical_loss": 3.350316417871813, + "tokens_seen": 2677014528 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.43047410249710083, + "objective/train/avg_token_score": 0.013451692648231983, + "objective/train/docs_used": 1507786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.234620451927185, + "objective/train/original_loss": 3.6118714809417725, + "objective/train/theoretical_loss": 3.3503035120364704, + "objective/train/tokens_used": 2677145600, + "objective/train/unlikelihood_loss": 0.12924598157405853, + "theoretical_loss": 3.3503035120364704, + "tokens_seen": 2677145600 + }, + { + "epoch": 0.81, + "learning_rate": 9.540621865596791e-05, + "loss": 1.2553, + "theoretical_loss": 3.3503035120364704, + "tokens_seen": 2677145600 + }, + { + "epoch": 0.81, + "learning_rate": 9.538615847542629e-05, + "loss": 1.2428, + "theoretical_loss": 3.350290607009889, + "tokens_seen": 2677276672 + }, + { + "epoch": 0.81, + "learning_rate": 9.536609829488465e-05, + "loss": 1.2967, + "theoretical_loss": 3.350277702791979, + "tokens_seen": 2677407744 + }, + { + "epoch": 0.81, + "learning_rate": 9.534603811434303e-05, + "loss": 1.3342, + "theoretical_loss": 3.35026479938265, + "tokens_seen": 2677538816 + }, + { + "epoch": 0.81, + "learning_rate": 9.53259779338014e-05, + "loss": 1.2347, + "theoretical_loss": 3.3502518967818116, + "tokens_seen": 2677669888 + }, + { + "epoch": 0.81, + "learning_rate": 9.530591775325978e-05, + "loss": 1.1435, + "theoretical_loss": 3.3502389949893736, + "tokens_seen": 2677800960 + }, + { + "epoch": 0.81, + "learning_rate": 9.528585757271815e-05, + "loss": 1.1988, + "theoretical_loss": 3.3502260940052455, + "tokens_seen": 2677932032 + }, + { + "epoch": 0.81, + "learning_rate": 9.526579739217653e-05, + "loss": 1.2296, + "theoretical_loss": 3.3502131938293376, + "tokens_seen": 2678063104 + }, + { + "epoch": 0.81, + "learning_rate": 9.52457372116349e-05, + "loss": 1.2097, + "theoretical_loss": 3.350200294461559, + "tokens_seen": 2678194176 + }, + { + "epoch": 0.81, + "learning_rate": 9.522567703109328e-05, + "loss": 1.26, + "theoretical_loss": 3.3501873959018207, + "tokens_seen": 2678325248 + }, + { + "epoch": 0.81, + "learning_rate": 9.520561685055166e-05, + "loss": 1.2199, + "theoretical_loss": 3.350174498150032, + "tokens_seen": 2678456320 + }, + { + "epoch": 0.81, + "learning_rate": 9.518555667001003e-05, + "loss": 1.2767, + "theoretical_loss": 3.350161601206102, + "tokens_seen": 2678587392 + }, + { + "epoch": 0.81, + "learning_rate": 9.51654964894684e-05, + "loss": 1.282, + "theoretical_loss": 3.3501487050699414, + "tokens_seen": 2678718464 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.3144550323486328, + "objective/train/avg_token_score": 0.01753219962120056, + "objective/train/docs_used": 1508529, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0490449666976929, + "objective/train/original_loss": 4.2530083656311035, + "objective/train/theoretical_loss": 3.3501422573047464, + "objective/train/tokens_used": 2678784000, + "objective/train/unlikelihood_loss": 0.1255263090133667, + "theoretical_loss": 3.3501422573047464, + "tokens_seen": 2678784000 + }, + { + "epoch": 0.81, + "learning_rate": 9.514543630892678e-05, + "loss": 1.3471, + "theoretical_loss": 3.35013580974146, + "tokens_seen": 2678849536 + }, + { + "epoch": 0.81, + "learning_rate": 9.512537612838516e-05, + "loss": 1.5868, + "theoretical_loss": 3.3501229152205676, + "tokens_seen": 2678980608 + }, + { + "epoch": 0.81, + "learning_rate": 9.510531594784354e-05, + "loss": 1.4165, + "theoretical_loss": 3.350110021507174, + "tokens_seen": 2679111680 + }, + { + "epoch": 0.81, + "learning_rate": 9.508525576730191e-05, + "loss": 1.3623, + "theoretical_loss": 3.350097128601189, + "tokens_seen": 2679242752 + }, + { + "epoch": 0.81, + "learning_rate": 9.506519558676029e-05, + "loss": 1.3604, + "theoretical_loss": 3.350084236502523, + "tokens_seen": 2679373824 + }, + { + "epoch": 0.81, + "learning_rate": 9.504513540621866e-05, + "loss": 1.3482, + "theoretical_loss": 3.350071345211086, + "tokens_seen": 2679504896 + }, + { + "epoch": 0.81, + "learning_rate": 9.502507522567704e-05, + "loss": 1.3423, + "theoretical_loss": 3.3500584547267875, + "tokens_seen": 2679635968 + }, + { + "epoch": 0.81, + "learning_rate": 9.500501504513541e-05, + "loss": 1.2987, + "theoretical_loss": 3.3500455650495375, + "tokens_seen": 2679767040 + }, + { + "epoch": 0.81, + "learning_rate": 9.498495486459379e-05, + "loss": 1.5455, + "theoretical_loss": 3.3500326761792465, + "tokens_seen": 2679898112 + }, + { + "epoch": 0.81, + "learning_rate": 9.496489468405215e-05, + "loss": 1.2933, + "theoretical_loss": 3.3500197881158242, + "tokens_seen": 2680029184 + }, + { + "epoch": 0.81, + "learning_rate": 9.494483450351053e-05, + "loss": 1.2937, + "theoretical_loss": 3.35000690085918, + "tokens_seen": 2680160256 + }, + { + "epoch": 0.81, + "learning_rate": 9.49247743229689e-05, + "loss": 1.2741, + "theoretical_loss": 3.3499940144092255, + "tokens_seen": 2680291328 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.5135630369186401, + "objective/train/avg_token_score": 0.012380465865135193, + "objective/train/docs_used": 1509790, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5361977815628052, + "objective/train/original_loss": 3.784867763519287, + "objective/train/theoretical_loss": 3.349981128765869, + "objective/train/tokens_used": 2680422400, + "objective/train/unlikelihood_loss": 0.12286731600761414, + "theoretical_loss": 3.349981128765869, + "tokens_seen": 2680422400 + }, + { + "epoch": 0.81, + "learning_rate": 9.490471414242728e-05, + "loss": 1.3137, + "theoretical_loss": 3.349981128765869, + "tokens_seen": 2680422400 + }, + { + "epoch": 0.81, + "learning_rate": 9.488465396188566e-05, + "loss": 1.2425, + "theoretical_loss": 3.3499682439290224, + "tokens_seen": 2680553472 + }, + { + "epoch": 0.81, + "learning_rate": 9.486459378134403e-05, + "loss": 1.1937, + "theoretical_loss": 3.349955359898594, + "tokens_seen": 2680684544 + }, + { + "epoch": 0.81, + "learning_rate": 9.484453360080241e-05, + "loss": 1.3773, + "theoretical_loss": 3.349942476674495, + "tokens_seen": 2680815616 + }, + { + "epoch": 0.81, + "learning_rate": 9.482447342026078e-05, + "loss": 1.2548, + "theoretical_loss": 3.349929594256635, + "tokens_seen": 2680946688 + }, + { + "epoch": 0.81, + "learning_rate": 9.480441323971916e-05, + "loss": 1.3099, + "theoretical_loss": 3.3499167126449243, + "tokens_seen": 2681077760 + }, + { + "epoch": 0.81, + "learning_rate": 9.478435305917752e-05, + "loss": 1.509, + "theoretical_loss": 3.3499038318392733, + "tokens_seen": 2681208832 + }, + { + "epoch": 0.81, + "learning_rate": 9.476429287863591e-05, + "loss": 1.2796, + "theoretical_loss": 3.3498909518395923, + "tokens_seen": 2681339904 + }, + { + "epoch": 0.81, + "learning_rate": 9.474423269809429e-05, + "loss": 1.1136, + "theoretical_loss": 3.3498780726457906, + "tokens_seen": 2681470976 + }, + { + "epoch": 0.81, + "learning_rate": 9.472417251755266e-05, + "loss": 1.3854, + "theoretical_loss": 3.349865194257779, + "tokens_seen": 2681602048 + }, + { + "epoch": 0.81, + "learning_rate": 9.470411233701104e-05, + "loss": 1.4132, + "theoretical_loss": 3.349852316675468, + "tokens_seen": 2681733120 + }, + { + "epoch": 0.81, + "learning_rate": 9.468405215646941e-05, + "loss": 1.27, + "theoretical_loss": 3.3498394398987674, + "tokens_seen": 2681864192 + }, + { + "epoch": 0.81, + "learning_rate": 9.466399197592779e-05, + "loss": 1.2135, + "theoretical_loss": 3.349826563927587, + "tokens_seen": 2681995264 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.2815249264240265, + "objective/train/avg_token_score": 0.021093031391501427, + "objective/train/docs_used": 1510339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7807753086090088, + "objective/train/original_loss": 3.872734308242798, + "objective/train/theoretical_loss": 3.3498201262440395, + "objective/train/tokens_used": 2682060800, + "objective/train/unlikelihood_loss": 0.1991308331489563, + "theoretical_loss": 3.3498201262440395, + "tokens_seen": 2682060800 + }, + { + "epoch": 0.81, + "learning_rate": 9.464393179538617e-05, + "loss": 1.2448, + "theoretical_loss": 3.349813688761838, + "tokens_seen": 2682126336 + }, + { + "epoch": 0.81, + "learning_rate": 9.462387161484454e-05, + "loss": 1.3004, + "theoretical_loss": 3.34980081440143, + "tokens_seen": 2682257408 + }, + { + "epoch": 0.81, + "learning_rate": 9.460381143430292e-05, + "loss": 1.2407, + "theoretical_loss": 3.3497879408462734, + "tokens_seen": 2682388480 + }, + { + "epoch": 0.81, + "learning_rate": 9.458375125376128e-05, + "loss": 1.1149, + "theoretical_loss": 3.3497750680962786, + "tokens_seen": 2682519552 + }, + { + "epoch": 0.81, + "learning_rate": 9.456369107321966e-05, + "loss": 1.2026, + "theoretical_loss": 3.3497621961513557, + "tokens_seen": 2682650624 + }, + { + "epoch": 0.81, + "learning_rate": 9.454363089267803e-05, + "loss": 1.3796, + "theoretical_loss": 3.3497493250114156, + "tokens_seen": 2682781696 + }, + { + "epoch": 0.81, + "learning_rate": 9.45235707121364e-05, + "loss": 1.3228, + "theoretical_loss": 3.3497364546763677, + "tokens_seen": 2682912768 + }, + { + "epoch": 0.81, + "learning_rate": 9.450351053159478e-05, + "loss": 1.1336, + "theoretical_loss": 3.3497235851461236, + "tokens_seen": 2683043840 + }, + { + "epoch": 0.81, + "learning_rate": 9.448345035105316e-05, + "loss": 1.1422, + "theoretical_loss": 3.349710716420592, + "tokens_seen": 2683174912 + }, + { + "epoch": 0.81, + "learning_rate": 9.446339017051153e-05, + "loss": 1.2564, + "theoretical_loss": 3.3496978484996847, + "tokens_seen": 2683305984 + }, + { + "epoch": 0.81, + "learning_rate": 9.444332998996991e-05, + "loss": 1.2398, + "theoretical_loss": 3.349684981383312, + "tokens_seen": 2683437056 + }, + { + "epoch": 0.81, + "learning_rate": 9.442326980942829e-05, + "loss": 1.2775, + "theoretical_loss": 3.349672115071383, + "tokens_seen": 2683568128 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.5013440847396851, + "objective/train/avg_token_score": 0.00836414098739624, + "objective/train/docs_used": 1511447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5724949836730957, + "objective/train/original_loss": 3.927595853805542, + "objective/train/theoretical_loss": 3.3496592495638096, + "objective/train/tokens_used": 2683699200, + "objective/train/unlikelihood_loss": 0.1015087440609932, + "theoretical_loss": 3.3496592495638096, + "tokens_seen": 2683699200 + }, + { + "epoch": 0.81, + "learning_rate": 9.440320962888667e-05, + "loss": 1.367, + "theoretical_loss": 3.3496592495638096, + "tokens_seen": 2683699200 + }, + { + "epoch": 0.81, + "learning_rate": 9.438314944834504e-05, + "loss": 1.4385, + "theoretical_loss": 3.3496463848605016, + "tokens_seen": 2683830272 + }, + { + "epoch": 0.81, + "learning_rate": 9.436308926780341e-05, + "loss": 1.3234, + "theoretical_loss": 3.3496335209613695, + "tokens_seen": 2683961344 + }, + { + "epoch": 0.81, + "learning_rate": 9.434302908726179e-05, + "loss": 1.3106, + "theoretical_loss": 3.3496206578663235, + "tokens_seen": 2684092416 + }, + { + "epoch": 0.81, + "learning_rate": 9.432296890672016e-05, + "loss": 1.221, + "theoretical_loss": 3.3496077955752743, + "tokens_seen": 2684223488 + }, + { + "epoch": 0.81, + "learning_rate": 9.430290872617854e-05, + "loss": 1.1699, + "theoretical_loss": 3.349594934088133, + "tokens_seen": 2684354560 + }, + { + "epoch": 0.81, + "learning_rate": 9.428284854563692e-05, + "loss": 1.2357, + "theoretical_loss": 3.3495820734048087, + "tokens_seen": 2684485632 + }, + { + "epoch": 0.81, + "learning_rate": 9.426278836509529e-05, + "loss": 1.2187, + "theoretical_loss": 3.3495692135252133, + "tokens_seen": 2684616704 + }, + { + "epoch": 0.81, + "learning_rate": 9.424272818455367e-05, + "loss": 1.2956, + "theoretical_loss": 3.3495563544492564, + "tokens_seen": 2684747776 + }, + { + "epoch": 0.81, + "learning_rate": 9.422266800401204e-05, + "loss": 1.0452, + "theoretical_loss": 3.3495434961768495, + "tokens_seen": 2684878848 + }, + { + "epoch": 0.81, + "learning_rate": 9.420260782347042e-05, + "loss": 1.3346, + "theoretical_loss": 3.3495306387079022, + "tokens_seen": 2685009920 + }, + { + "epoch": 0.81, + "learning_rate": 9.418254764292878e-05, + "loss": 1.2412, + "theoretical_loss": 3.3495177820423256, + "tokens_seen": 2685140992 + }, + { + "epoch": 0.81, + "learning_rate": 9.416248746238716e-05, + "loss": 1.1921, + "theoretical_loss": 3.34950492618003, + "tokens_seen": 2685272064 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.4741568863391876, + "objective/train/avg_token_score": 0.004732546862214804, + "objective/train/docs_used": 1512148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4895588159561157, + "objective/train/original_loss": 3.7662782669067383, + "objective/train/theoretical_loss": 3.3494984985500844, + "objective/train/tokens_used": 2685337600, + "objective/train/unlikelihood_loss": 0.19918754696846008, + "theoretical_loss": 3.3494984985500844, + "tokens_seen": 2685337600 + }, + { + "epoch": 0.81, + "learning_rate": 9.414242728184553e-05, + "loss": 1.3307, + "theoretical_loss": 3.349492071120926, + "tokens_seen": 2685403136 + }, + { + "epoch": 0.81, + "learning_rate": 9.412236710130391e-05, + "loss": 1.2149, + "theoretical_loss": 3.349479216864925, + "tokens_seen": 2685534208 + }, + { + "epoch": 0.81, + "learning_rate": 9.410230692076228e-05, + "loss": 1.2081, + "theoretical_loss": 3.3494663634119366, + "tokens_seen": 2685665280 + }, + { + "epoch": 0.81, + "learning_rate": 9.408224674022066e-05, + "loss": 1.2588, + "theoretical_loss": 3.349453510761872, + "tokens_seen": 2685796352 + }, + { + "epoch": 0.81, + "learning_rate": 9.406218655967904e-05, + "loss": 1.1994, + "theoretical_loss": 3.3494406589146415, + "tokens_seen": 2685927424 + }, + { + "epoch": 0.81, + "learning_rate": 9.404212637913741e-05, + "loss": 1.3973, + "theoretical_loss": 3.349427807870156, + "tokens_seen": 2686058496 + }, + { + "epoch": 0.81, + "learning_rate": 9.40220661985958e-05, + "loss": 1.2847, + "theoretical_loss": 3.3494149576283263, + "tokens_seen": 2686189568 + }, + { + "epoch": 0.81, + "learning_rate": 9.400200601805416e-05, + "loss": 1.1821, + "theoretical_loss": 3.3494021081890635, + "tokens_seen": 2686320640 + }, + { + "epoch": 0.81, + "learning_rate": 9.398194583751254e-05, + "loss": 1.102, + "theoretical_loss": 3.349389259552277, + "tokens_seen": 2686451712 + }, + { + "epoch": 0.81, + "learning_rate": 9.396188565697092e-05, + "loss": 1.2426, + "theoretical_loss": 3.349376411717879, + "tokens_seen": 2686582784 + }, + { + "epoch": 0.81, + "learning_rate": 9.394182547642929e-05, + "loss": 1.3559, + "theoretical_loss": 3.3493635646857793, + "tokens_seen": 2686713856 + }, + { + "epoch": 0.81, + "learning_rate": 9.392176529588767e-05, + "loss": 1.2887, + "theoretical_loss": 3.349350718455889, + "tokens_seen": 2686844928 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.38776880502700806, + "objective/train/avg_token_score": 0.02399761974811554, + "objective/train/docs_used": 1513440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1675814390182495, + "objective/train/original_loss": 4.33937931060791, + "objective/train/theoretical_loss": 3.349337873028119, + "objective/train/tokens_used": 2686976000, + "objective/train/unlikelihood_loss": 0.14494581520557404, + "theoretical_loss": 3.349337873028119, + "tokens_seen": 2686976000 + }, + { + "epoch": 0.81, + "learning_rate": 9.390170511534604e-05, + "loss": 1.2462, + "theoretical_loss": 3.349337873028119, + "tokens_seen": 2686976000 + }, + { + "epoch": 0.81, + "learning_rate": 9.388164493480442e-05, + "loss": 1.2429, + "theoretical_loss": 3.3493250284023794, + "tokens_seen": 2687107072 + }, + { + "epoch": 0.81, + "learning_rate": 9.38615847542628e-05, + "loss": 1.3428, + "theoretical_loss": 3.349312184578582, + "tokens_seen": 2687238144 + }, + { + "epoch": 0.81, + "learning_rate": 9.384152457372117e-05, + "loss": 1.1991, + "theoretical_loss": 3.3492993415566374, + "tokens_seen": 2687369216 + }, + { + "epoch": 0.81, + "learning_rate": 9.382146439317955e-05, + "loss": 1.4393, + "theoretical_loss": 3.349286499336456, + "tokens_seen": 2687500288 + }, + { + "epoch": 0.81, + "learning_rate": 9.380140421263791e-05, + "loss": 1.2048, + "theoretical_loss": 3.3492736579179487, + "tokens_seen": 2687631360 + }, + { + "epoch": 0.81, + "learning_rate": 9.378134403209628e-05, + "loss": 1.3211, + "theoretical_loss": 3.3492608173010265, + "tokens_seen": 2687762432 + }, + { + "epoch": 0.81, + "learning_rate": 9.376128385155466e-05, + "loss": 1.4165, + "theoretical_loss": 3.3492479774856005, + "tokens_seen": 2687893504 + }, + { + "epoch": 0.81, + "learning_rate": 9.374122367101304e-05, + "loss": 1.3091, + "theoretical_loss": 3.349235138471581, + "tokens_seen": 2688024576 + }, + { + "epoch": 0.81, + "learning_rate": 9.372116349047141e-05, + "loss": 1.403, + "theoretical_loss": 3.3492223002588797, + "tokens_seen": 2688155648 + }, + { + "epoch": 0.81, + "learning_rate": 9.370110330992979e-05, + "loss": 1.2448, + "theoretical_loss": 3.3492094628474067, + "tokens_seen": 2688286720 + }, + { + "epoch": 0.81, + "learning_rate": 9.368104312938816e-05, + "loss": 1.3198, + "theoretical_loss": 3.349196626237074, + "tokens_seen": 2688417792 + }, + { + "epoch": 0.81, + "learning_rate": 9.366098294884654e-05, + "loss": 1.4835, + "theoretical_loss": 3.3491837904277912, + "tokens_seen": 2688548864 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.49938905239105225, + "objective/train/avg_token_score": 0.010914425365626812, + "objective/train/docs_used": 1514160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5798979997634888, + "objective/train/original_loss": 3.772454261779785, + "objective/train/theoretical_loss": 3.349177372823516, + "objective/train/tokens_used": 2688614400, + "objective/train/unlikelihood_loss": 0.13101530075073242, + "theoretical_loss": 3.349177372823516, + "tokens_seen": 2688614400 + }, + { + "epoch": 0.81, + "learning_rate": 9.364092276830493e-05, + "loss": 1.265, + "theoretical_loss": 3.34917095541947, + "tokens_seen": 2688679936 + }, + { + "epoch": 0.81, + "learning_rate": 9.36208625877633e-05, + "loss": 1.2591, + "theoretical_loss": 3.349158121212022, + "tokens_seen": 2688811008 + }, + { + "epoch": 0.81, + "learning_rate": 9.360080240722167e-05, + "loss": 1.2901, + "theoretical_loss": 3.349145287805357, + "tokens_seen": 2688942080 + }, + { + "epoch": 0.81, + "learning_rate": 9.358074222668004e-05, + "loss": 1.2606, + "theoretical_loss": 3.3491324551993866, + "tokens_seen": 2689073152 + }, + { + "epoch": 0.81, + "learning_rate": 9.356068204613842e-05, + "loss": 1.3179, + "theoretical_loss": 3.3491196233940217, + "tokens_seen": 2689204224 + }, + { + "epoch": 0.81, + "learning_rate": 9.35406218655968e-05, + "loss": 1.1241, + "theoretical_loss": 3.3491067923891733, + "tokens_seen": 2689335296 + }, + { + "epoch": 0.81, + "learning_rate": 9.352056168505517e-05, + "loss": 1.1644, + "theoretical_loss": 3.3490939621847526, + "tokens_seen": 2689466368 + }, + { + "epoch": 0.82, + "learning_rate": 9.350050150451354e-05, + "loss": 1.267, + "theoretical_loss": 3.3490811327806704, + "tokens_seen": 2689597440 + }, + { + "epoch": 0.82, + "learning_rate": 9.348044132397192e-05, + "loss": 1.2055, + "theoretical_loss": 3.3490683041768383, + "tokens_seen": 2689728512 + }, + { + "epoch": 0.82, + "learning_rate": 9.34603811434303e-05, + "loss": 1.221, + "theoretical_loss": 3.3490554763731666, + "tokens_seen": 2689859584 + }, + { + "epoch": 0.82, + "learning_rate": 9.344032096288867e-05, + "loss": 1.2765, + "theoretical_loss": 3.3490426493695673, + "tokens_seen": 2689990656 + }, + { + "epoch": 0.82, + "learning_rate": 9.342026078234703e-05, + "loss": 1.3613, + "theoretical_loss": 3.349029823165951, + "tokens_seen": 2690121728 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.43365103006362915, + "objective/train/avg_token_score": 0.04775285720825195, + "objective/train/docs_used": 1514886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.25999116897583, + "objective/train/original_loss": 3.776130437850952, + "objective/train/theoretical_loss": 3.349016997762229, + "objective/train/tokens_used": 2690252800, + "objective/train/unlikelihood_loss": 0.16561372578144073, + "theoretical_loss": 3.349016997762229, + "tokens_seen": 2690252800 + }, + { + "epoch": 0.82, + "learning_rate": 9.340020060180541e-05, + "loss": 1.3074, + "theoretical_loss": 3.349016997762229, + "tokens_seen": 2690252800 + }, + { + "epoch": 0.82, + "learning_rate": 9.338014042126379e-05, + "loss": 1.1719, + "theoretical_loss": 3.349004173158312, + "tokens_seen": 2690383872 + }, + { + "epoch": 0.82, + "learning_rate": 9.336008024072216e-05, + "loss": 1.2589, + "theoretical_loss": 3.3489913493541117, + "tokens_seen": 2690514944 + }, + { + "epoch": 0.82, + "learning_rate": 9.334002006018054e-05, + "loss": 1.3566, + "theoretical_loss": 3.3489785263495393, + "tokens_seen": 2690646016 + }, + { + "epoch": 0.82, + "learning_rate": 9.331995987963891e-05, + "loss": 1.2686, + "theoretical_loss": 3.3489657041445056, + "tokens_seen": 2690777088 + }, + { + "epoch": 0.82, + "learning_rate": 9.329989969909729e-05, + "loss": 1.1809, + "theoretical_loss": 3.3489528827389217, + "tokens_seen": 2690908160 + }, + { + "epoch": 0.82, + "learning_rate": 9.327983951855568e-05, + "loss": 1.3501, + "theoretical_loss": 3.3489400621326997, + "tokens_seen": 2691039232 + }, + { + "epoch": 0.82, + "learning_rate": 9.325977933801405e-05, + "loss": 1.1608, + "theoretical_loss": 3.3489272423257495, + "tokens_seen": 2691170304 + }, + { + "epoch": 0.82, + "learning_rate": 9.323971915747243e-05, + "loss": 1.2874, + "theoretical_loss": 3.3489144233179835, + "tokens_seen": 2691301376 + }, + { + "epoch": 0.82, + "learning_rate": 9.321965897693079e-05, + "loss": 1.1712, + "theoretical_loss": 3.3489016051093126, + "tokens_seen": 2691432448 + }, + { + "epoch": 0.82, + "learning_rate": 9.319959879638917e-05, + "loss": 1.2814, + "theoretical_loss": 3.3488887876996483, + "tokens_seen": 2691563520 + }, + { + "epoch": 0.82, + "learning_rate": 9.317953861584754e-05, + "loss": 1.125, + "theoretical_loss": 3.348875971088901, + "tokens_seen": 2691694592 + }, + { + "epoch": 0.82, + "learning_rate": 9.315947843530592e-05, + "loss": 1.1978, + "theoretical_loss": 3.348863155276983, + "tokens_seen": 2691825664 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.5318915247917175, + "objective/train/avg_token_score": 0.004946494475007057, + "objective/train/docs_used": 1515566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5836353302001953, + "objective/train/original_loss": 3.4328551292419434, + "objective/train/theoretical_loss": 3.348856747670557, + "objective/train/tokens_used": 2691891200, + "objective/train/unlikelihood_loss": 0.13799813389778137, + "theoretical_loss": 3.348856747670557, + "tokens_seen": 2691891200 + }, + { + "epoch": 0.82, + "learning_rate": 9.31394182547643e-05, + "loss": 1.2464, + "theoretical_loss": 3.3488503402638052, + "tokens_seen": 2691956736 + }, + { + "epoch": 0.82, + "learning_rate": 9.311935807422267e-05, + "loss": 1.266, + "theoretical_loss": 3.3488375260492784, + "tokens_seen": 2692087808 + }, + { + "epoch": 0.82, + "learning_rate": 9.309929789368105e-05, + "loss": 1.3598, + "theoretical_loss": 3.348824712633315, + "tokens_seen": 2692218880 + }, + { + "epoch": 0.82, + "learning_rate": 9.307923771313942e-05, + "loss": 1.3655, + "theoretical_loss": 3.348811900015826, + "tokens_seen": 2692349952 + }, + { + "epoch": 0.82, + "learning_rate": 9.30591775325978e-05, + "loss": 1.2405, + "theoretical_loss": 3.3487990881967225, + "tokens_seen": 2692481024 + }, + { + "epoch": 0.82, + "learning_rate": 9.303911735205617e-05, + "loss": 1.285, + "theoretical_loss": 3.348786277175916, + "tokens_seen": 2692612096 + }, + { + "epoch": 0.82, + "learning_rate": 9.301905717151454e-05, + "loss": 1.1838, + "theoretical_loss": 3.348773466953318, + "tokens_seen": 2692743168 + }, + { + "epoch": 0.82, + "learning_rate": 9.299899699097291e-05, + "loss": 1.2155, + "theoretical_loss": 3.34876065752884, + "tokens_seen": 2692874240 + }, + { + "epoch": 0.82, + "learning_rate": 9.297893681043129e-05, + "loss": 1.3632, + "theoretical_loss": 3.3487478489023927, + "tokens_seen": 2693005312 + }, + { + "epoch": 0.82, + "learning_rate": 9.295887662988966e-05, + "loss": 1.4094, + "theoretical_loss": 3.3487350410738883, + "tokens_seen": 2693136384 + }, + { + "epoch": 0.82, + "learning_rate": 9.293881644934804e-05, + "loss": 1.1847, + "theoretical_loss": 3.3487222340432385, + "tokens_seen": 2693267456 + }, + { + "epoch": 0.82, + "learning_rate": 9.291875626880642e-05, + "loss": 1.2647, + "theoretical_loss": 3.3487094278103537, + "tokens_seen": 2693398528 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.3852028250694275, + "objective/train/avg_token_score": 0.018061740323901176, + "objective/train/docs_used": 1516840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9547469615936279, + "objective/train/original_loss": 3.448294162750244, + "objective/train/theoretical_loss": 3.348696622375147, + "objective/train/tokens_used": 2693529600, + "objective/train/unlikelihood_loss": 0.1810632050037384, + "theoretical_loss": 3.348696622375147, + "tokens_seen": 2693529600 + }, + { + "epoch": 0.82, + "learning_rate": 9.28986960882648e-05, + "loss": 1.2805, + "theoretical_loss": 3.348696622375147, + "tokens_seen": 2693529600 + }, + { + "epoch": 0.82, + "learning_rate": 9.287863590772318e-05, + "loss": 1.2389, + "theoretical_loss": 3.348683817737528, + "tokens_seen": 2693660672 + }, + { + "epoch": 0.82, + "learning_rate": 9.285857572718156e-05, + "loss": 1.2306, + "theoretical_loss": 3.34867101389741, + "tokens_seen": 2693791744 + }, + { + "epoch": 0.82, + "learning_rate": 9.283851554663993e-05, + "loss": 1.1305, + "theoretical_loss": 3.348658210854703, + "tokens_seen": 2693922816 + }, + { + "epoch": 0.82, + "learning_rate": 9.28184553660983e-05, + "loss": 1.2403, + "theoretical_loss": 3.348645408609319, + "tokens_seen": 2694053888 + }, + { + "epoch": 0.82, + "learning_rate": 9.279839518555667e-05, + "loss": 1.3037, + "theoretical_loss": 3.3486326071611705, + "tokens_seen": 2694184960 + }, + { + "epoch": 0.82, + "learning_rate": 9.277833500501505e-05, + "loss": 1.1999, + "theoretical_loss": 3.348619806510168, + "tokens_seen": 2694316032 + }, + { + "epoch": 0.82, + "learning_rate": 9.275827482447342e-05, + "loss": 1.3277, + "theoretical_loss": 3.348607006656224, + "tokens_seen": 2694447104 + }, + { + "epoch": 0.82, + "learning_rate": 9.27382146439318e-05, + "loss": 1.3112, + "theoretical_loss": 3.3485942075992488, + "tokens_seen": 2694578176 + }, + { + "epoch": 0.82, + "learning_rate": 9.271815446339017e-05, + "loss": 1.0889, + "theoretical_loss": 3.3485814093391553, + "tokens_seen": 2694709248 + }, + { + "epoch": 0.82, + "learning_rate": 9.269809428284855e-05, + "loss": 1.4201, + "theoretical_loss": 3.348568611875854, + "tokens_seen": 2694840320 + }, + { + "epoch": 0.82, + "learning_rate": 9.267803410230693e-05, + "loss": 1.216, + "theoretical_loss": 3.3485558152092576, + "tokens_seen": 2694971392 + }, + { + "epoch": 0.82, + "learning_rate": 9.26579739217653e-05, + "loss": 1.2323, + "theoretical_loss": 3.348543019339277, + "tokens_seen": 2695102464 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.40267595648765564, + "objective/train/avg_token_score": 0.016301974654197693, + "objective/train/docs_used": 1517438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1428282260894775, + "objective/train/original_loss": 3.9693284034729004, + "objective/train/theoretical_loss": 3.3485366217029906, + "objective/train/tokens_used": 2695168000, + "objective/train/unlikelihood_loss": 0.13207590579986572, + "theoretical_loss": 3.3485366217029906, + "tokens_seen": 2695168000 + }, + { + "epoch": 0.82, + "learning_rate": 9.263791374122366e-05, + "loss": 1.1211, + "theoretical_loss": 3.348530224265825, + "tokens_seen": 2695233536 + }, + { + "epoch": 0.82, + "learning_rate": 9.261785356068204e-05, + "loss": 1.1756, + "theoretical_loss": 3.3485174299888114, + "tokens_seen": 2695364608 + }, + { + "epoch": 0.82, + "learning_rate": 9.259779338014041e-05, + "loss": 1.2743, + "theoretical_loss": 3.3485046365081494, + "tokens_seen": 2695495680 + }, + { + "epoch": 0.82, + "learning_rate": 9.257773319959879e-05, + "loss": 1.3325, + "theoretical_loss": 3.34849184382375, + "tokens_seen": 2695626752 + }, + { + "epoch": 0.82, + "learning_rate": 9.255767301905717e-05, + "loss": 1.2733, + "theoretical_loss": 3.3484790519355254, + "tokens_seen": 2695757824 + }, + { + "epoch": 0.82, + "learning_rate": 9.253761283851554e-05, + "loss": 1.3297, + "theoretical_loss": 3.348466260843387, + "tokens_seen": 2695888896 + }, + { + "epoch": 0.82, + "learning_rate": 9.251755265797393e-05, + "loss": 1.2583, + "theoretical_loss": 3.348453470547247, + "tokens_seen": 2696019968 + }, + { + "epoch": 0.82, + "learning_rate": 9.249749247743231e-05, + "loss": 1.2757, + "theoretical_loss": 3.3484406810470166, + "tokens_seen": 2696151040 + }, + { + "epoch": 0.82, + "learning_rate": 9.247743229689068e-05, + "loss": 1.3199, + "theoretical_loss": 3.348427892342608, + "tokens_seen": 2696282112 + }, + { + "epoch": 0.82, + "learning_rate": 9.245737211634906e-05, + "loss": 1.3099, + "theoretical_loss": 3.3484151044339328, + "tokens_seen": 2696413184 + }, + { + "epoch": 0.82, + "learning_rate": 9.243731193580742e-05, + "loss": 1.3225, + "theoretical_loss": 3.3484023173209025, + "tokens_seen": 2696544256 + }, + { + "epoch": 0.82, + "learning_rate": 9.24172517552658e-05, + "loss": 1.2622, + "theoretical_loss": 3.3483895310034297, + "tokens_seen": 2696675328 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.5185728073120117, + "objective/train/avg_token_score": 0.01496554259210825, + "objective/train/docs_used": 1518043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3652535676956177, + "objective/train/original_loss": 3.2484819889068604, + "objective/train/theoretical_loss": 3.3483767454814255, + "objective/train/tokens_used": 2696806400, + "objective/train/unlikelihood_loss": 0.16000565886497498, + "theoretical_loss": 3.3483767454814255, + "tokens_seen": 2696806400 + }, + { + "epoch": 0.82, + "learning_rate": 9.239719157472417e-05, + "loss": 1.1252, + "theoretical_loss": 3.3483767454814255, + "tokens_seen": 2696806400 + }, + { + "epoch": 0.82, + "learning_rate": 9.237713139418255e-05, + "loss": 1.3858, + "theoretical_loss": 3.348363960754802, + "tokens_seen": 2696937472 + }, + { + "epoch": 0.82, + "learning_rate": 9.235707121364092e-05, + "loss": 1.322, + "theoretical_loss": 3.3483511768234715, + "tokens_seen": 2697068544 + }, + { + "epoch": 0.82, + "learning_rate": 9.23370110330993e-05, + "loss": 1.1975, + "theoretical_loss": 3.348338393687345, + "tokens_seen": 2697199616 + }, + { + "epoch": 0.82, + "learning_rate": 9.231695085255768e-05, + "loss": 1.238, + "theoretical_loss": 3.3483256113463353, + "tokens_seen": 2697330688 + }, + { + "epoch": 0.82, + "learning_rate": 9.229689067201605e-05, + "loss": 1.2543, + "theoretical_loss": 3.3483128298003537, + "tokens_seen": 2697461760 + }, + { + "epoch": 0.82, + "learning_rate": 9.227683049147443e-05, + "loss": 1.2687, + "theoretical_loss": 3.3483000490493127, + "tokens_seen": 2697592832 + }, + { + "epoch": 0.82, + "learning_rate": 9.22567703109328e-05, + "loss": 1.1522, + "theoretical_loss": 3.348287269093124, + "tokens_seen": 2697723904 + }, + { + "epoch": 0.82, + "learning_rate": 9.223671013039117e-05, + "loss": 1.3664, + "theoretical_loss": 3.3482744899316987, + "tokens_seen": 2697854976 + }, + { + "epoch": 0.82, + "learning_rate": 9.221664994984954e-05, + "loss": 1.4639, + "theoretical_loss": 3.34826171156495, + "tokens_seen": 2697986048 + }, + { + "epoch": 0.82, + "learning_rate": 9.219658976930792e-05, + "loss": 1.2217, + "theoretical_loss": 3.348248933992789, + "tokens_seen": 2698117120 + }, + { + "epoch": 0.82, + "learning_rate": 9.217652958876629e-05, + "loss": 1.2865, + "theoretical_loss": 3.3482361572151285, + "tokens_seen": 2698248192 + }, + { + "epoch": 0.82, + "learning_rate": 9.215646940822467e-05, + "loss": 1.2726, + "theoretical_loss": 3.34822338123188, + "tokens_seen": 2698379264 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.3475073277950287, + "objective/train/avg_token_score": 0.017542244866490364, + "objective/train/docs_used": 1519063, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0185778141021729, + "objective/train/original_loss": 3.676762819290161, + "objective/train/theoretical_loss": 3.3482169935381325, + "objective/train/tokens_used": 2698444800, + "objective/train/unlikelihood_loss": 0.16054868698120117, + "theoretical_loss": 3.3482169935381325, + "tokens_seen": 2698444800 + }, + { + "epoch": 0.82, + "learning_rate": 9.213640922768306e-05, + "loss": 1.1182, + "theoretical_loss": 3.3482106060429553, + "tokens_seen": 2698510336 + }, + { + "epoch": 0.82, + "learning_rate": 9.211634904714143e-05, + "loss": 1.4697, + "theoretical_loss": 3.348197831648267, + "tokens_seen": 2698641408 + }, + { + "epoch": 0.82, + "learning_rate": 9.209628886659981e-05, + "loss": 1.2545, + "theoretical_loss": 3.3481850580477266, + "tokens_seen": 2698772480 + }, + { + "epoch": 0.82, + "learning_rate": 9.207622868605819e-05, + "loss": 1.5265, + "theoretical_loss": 3.3481722852412465, + "tokens_seen": 2698903552 + }, + { + "epoch": 0.82, + "learning_rate": 9.205616850551656e-05, + "loss": 1.3634, + "theoretical_loss": 3.348159513228739, + "tokens_seen": 2699034624 + }, + { + "epoch": 0.82, + "learning_rate": 9.203610832497492e-05, + "loss": 1.1676, + "theoretical_loss": 3.348146742010116, + "tokens_seen": 2699165696 + }, + { + "epoch": 0.82, + "learning_rate": 9.20160481444333e-05, + "loss": 1.1303, + "theoretical_loss": 3.348133971585289, + "tokens_seen": 2699296768 + }, + { + "epoch": 0.82, + "learning_rate": 9.199598796389168e-05, + "loss": 1.3624, + "theoretical_loss": 3.348121201954171, + "tokens_seen": 2699427840 + }, + { + "epoch": 0.82, + "learning_rate": 9.197592778335005e-05, + "loss": 1.3531, + "theoretical_loss": 3.3481084331166735, + "tokens_seen": 2699558912 + }, + { + "epoch": 0.82, + "learning_rate": 9.195586760280843e-05, + "loss": 1.2819, + "theoretical_loss": 3.3480956650727096, + "tokens_seen": 2699689984 + }, + { + "epoch": 0.82, + "learning_rate": 9.19358074222668e-05, + "loss": 1.2188, + "theoretical_loss": 3.34808289782219, + "tokens_seen": 2699821056 + }, + { + "epoch": 0.82, + "learning_rate": 9.191574724172518e-05, + "loss": 1.3953, + "theoretical_loss": 3.348070131365028, + "tokens_seen": 2699952128 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.3700513243675232, + "objective/train/avg_token_score": 0.00978762749582529, + "objective/train/docs_used": 1519691, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1999402046203613, + "objective/train/original_loss": 3.340207099914551, + "objective/train/theoretical_loss": 3.3480573657011354, + "objective/train/tokens_used": 2700083200, + "objective/train/unlikelihood_loss": 0.2166636735200882, + "theoretical_loss": 3.3480573657011354, + "tokens_seen": 2700083200 + }, + { + "epoch": 0.82, + "learning_rate": 9.189568706118355e-05, + "loss": 1.2437, + "theoretical_loss": 3.3480573657011354, + "tokens_seen": 2700083200 + }, + { + "epoch": 0.82, + "learning_rate": 9.187562688064193e-05, + "loss": 1.1689, + "theoretical_loss": 3.3480446008304248, + "tokens_seen": 2700214272 + }, + { + "epoch": 0.82, + "learning_rate": 9.185556670010029e-05, + "loss": 1.2955, + "theoretical_loss": 3.3480318367528077, + "tokens_seen": 2700345344 + }, + { + "epoch": 0.82, + "learning_rate": 9.183550651955867e-05, + "loss": 1.2754, + "theoretical_loss": 3.3480190734681967, + "tokens_seen": 2700476416 + }, + { + "epoch": 0.82, + "learning_rate": 9.181544633901704e-05, + "loss": 1.3027, + "theoretical_loss": 3.348006310976504, + "tokens_seen": 2700607488 + }, + { + "epoch": 0.82, + "learning_rate": 9.179538615847542e-05, + "loss": 1.3274, + "theoretical_loss": 3.3479935492776423, + "tokens_seen": 2700738560 + }, + { + "epoch": 0.82, + "learning_rate": 9.177532597793381e-05, + "loss": 1.3308, + "theoretical_loss": 3.3479807883715234, + "tokens_seen": 2700869632 + }, + { + "epoch": 0.82, + "learning_rate": 9.175526579739218e-05, + "loss": 1.231, + "theoretical_loss": 3.3479680282580593, + "tokens_seen": 2701000704 + }, + { + "epoch": 0.82, + "learning_rate": 9.175526579739218e-05, + "loss": 1.3915, + "theoretical_loss": 3.347955268937163, + "tokens_seen": 2701131776 + }, + { + "epoch": 0.82, + "learning_rate": 9.173520561685056e-05, + "loss": 1.2801, + "theoretical_loss": 3.347942510408746, + "tokens_seen": 2701262848 + }, + { + "epoch": 0.82, + "learning_rate": 9.171514543630894e-05, + "loss": 1.2675, + "theoretical_loss": 3.347929752672722, + "tokens_seen": 2701393920 + }, + { + "epoch": 0.82, + "learning_rate": 9.169508525576731e-05, + "loss": 1.0863, + "theoretical_loss": 3.347916995729002, + "tokens_seen": 2701524992 + }, + { + "epoch": 0.82, + "learning_rate": 9.167502507522569e-05, + "loss": 1.4396, + "theoretical_loss": 3.3479042395774985, + "tokens_seen": 2701656064 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.42204299569129944, + "objective/train/avg_token_score": 0.012690963223576546, + "objective/train/docs_used": 1520269, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3133646249771118, + "objective/train/original_loss": 3.5583252906799316, + "objective/train/theoretical_loss": 3.347897861798801, + "objective/train/tokens_used": 2701721600, + "objective/train/unlikelihood_loss": 0.2020862102508545, + "theoretical_loss": 3.347897861798801, + "tokens_seen": 2701721600 + }, + { + "epoch": 0.82, + "learning_rate": 9.165496489468405e-05, + "loss": 1.203, + "theoretical_loss": 3.347891484218125, + "tokens_seen": 2701787136 + }, + { + "epoch": 0.82, + "learning_rate": 9.163490471414243e-05, + "loss": 1.2197, + "theoretical_loss": 3.3478787296507924, + "tokens_seen": 2701918208 + }, + { + "epoch": 0.82, + "learning_rate": 9.16148445336008e-05, + "loss": 1.2338, + "theoretical_loss": 3.347865975875414, + "tokens_seen": 2702049280 + }, + { + "epoch": 0.82, + "learning_rate": 9.159478435305918e-05, + "loss": 1.3634, + "theoretical_loss": 3.3478532228919016, + "tokens_seen": 2702180352 + }, + { + "epoch": 0.82, + "learning_rate": 9.157472417251755e-05, + "loss": 1.1987, + "theoretical_loss": 3.3478404707001683, + "tokens_seen": 2702311424 + }, + { + "epoch": 0.82, + "learning_rate": 9.155466399197593e-05, + "loss": 1.3, + "theoretical_loss": 3.3478277193001267, + "tokens_seen": 2702442496 + }, + { + "epoch": 0.82, + "learning_rate": 9.15346038114343e-05, + "loss": 1.4177, + "theoretical_loss": 3.3478149686916883, + "tokens_seen": 2702573568 + }, + { + "epoch": 0.82, + "learning_rate": 9.151454363089268e-05, + "loss": 1.3156, + "theoretical_loss": 3.347802218874766, + "tokens_seen": 2702704640 + }, + { + "epoch": 0.82, + "learning_rate": 9.149448345035106e-05, + "loss": 1.1941, + "theoretical_loss": 3.347789469849273, + "tokens_seen": 2702835712 + }, + { + "epoch": 0.82, + "learning_rate": 9.147442326980943e-05, + "loss": 1.1661, + "theoretical_loss": 3.3477767216151206, + "tokens_seen": 2702966784 + }, + { + "epoch": 0.82, + "learning_rate": 9.14543630892678e-05, + "loss": 1.3253, + "theoretical_loss": 3.347763974172222, + "tokens_seen": 2703097856 + }, + { + "epoch": 0.82, + "learning_rate": 9.143430290872617e-05, + "loss": 1.263, + "theoretical_loss": 3.3477512275204897, + "tokens_seen": 2703228928 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.36271995306015015, + "objective/train/avg_token_score": 0.01180272176861763, + "objective/train/docs_used": 1521568, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0338417291641235, + "objective/train/original_loss": 3.5833494663238525, + "objective/train/theoretical_loss": 3.347738481659836, + "objective/train/tokens_used": 2703360000, + "objective/train/unlikelihood_loss": 0.16099487245082855, + "theoretical_loss": 3.347738481659836, + "tokens_seen": 2703360000 + }, + { + "epoch": 0.82, + "learning_rate": 9.141424272818455e-05, + "loss": 1.1938, + "theoretical_loss": 3.347738481659836, + "tokens_seen": 2703360000 + }, + { + "epoch": 0.82, + "learning_rate": 9.139418254764294e-05, + "loss": 1.2742, + "theoretical_loss": 3.347725736590174, + "tokens_seen": 2703491072 + }, + { + "epoch": 0.82, + "learning_rate": 9.137412236710131e-05, + "loss": 1.2563, + "theoretical_loss": 3.3477129923114153, + "tokens_seen": 2703622144 + }, + { + "epoch": 0.82, + "learning_rate": 9.135406218655969e-05, + "loss": 1.2887, + "theoretical_loss": 3.3477002488234735, + "tokens_seen": 2703753216 + }, + { + "epoch": 0.82, + "learning_rate": 9.133400200601806e-05, + "loss": 1.1658, + "theoretical_loss": 3.3476875061262605, + "tokens_seen": 2703884288 + }, + { + "epoch": 0.82, + "learning_rate": 9.131394182547644e-05, + "loss": 1.4015, + "theoretical_loss": 3.347674764219689, + "tokens_seen": 2704015360 + }, + { + "epoch": 0.82, + "learning_rate": 9.129388164493481e-05, + "loss": 1.243, + "theoretical_loss": 3.3476620231036716, + "tokens_seen": 2704146432 + }, + { + "epoch": 0.82, + "learning_rate": 9.127382146439319e-05, + "loss": 1.1407, + "theoretical_loss": 3.3476492827781215, + "tokens_seen": 2704277504 + }, + { + "epoch": 0.82, + "learning_rate": 9.125376128385155e-05, + "loss": 1.1301, + "theoretical_loss": 3.347636543242951, + "tokens_seen": 2704408576 + }, + { + "epoch": 0.82, + "learning_rate": 9.123370110330993e-05, + "loss": 1.3582, + "theoretical_loss": 3.3476238044980726, + "tokens_seen": 2704539648 + }, + { + "epoch": 0.82, + "learning_rate": 9.12136409227683e-05, + "loss": 1.2083, + "theoretical_loss": 3.347611066543399, + "tokens_seen": 2704670720 + }, + { + "epoch": 0.82, + "learning_rate": 9.119358074222668e-05, + "loss": 1.2227, + "theoretical_loss": 3.347598329378843, + "tokens_seen": 2704801792 + }, + { + "epoch": 0.82, + "learning_rate": 9.117352056168506e-05, + "loss": 1.2138, + "theoretical_loss": 3.3475855930043172, + "tokens_seen": 2704932864 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.45375123620033264, + "objective/train/avg_token_score": 0.011906135827302933, + "objective/train/docs_used": 1522374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5304033756256104, + "objective/train/original_loss": 3.503509521484375, + "objective/train/theoretical_loss": 3.3475792251132885, + "objective/train/tokens_used": 2704998400, + "objective/train/unlikelihood_loss": 0.1869519203901291, + "theoretical_loss": 3.3475792251132885, + "tokens_seen": 2704998400 + }, + { + "epoch": 0.82, + "learning_rate": 9.115346038114343e-05, + "loss": 1.3262, + "theoretical_loss": 3.3475728574197348, + "tokens_seen": 2705063936 + }, + { + "epoch": 0.82, + "learning_rate": 9.113340020060181e-05, + "loss": 1.156, + "theoretical_loss": 3.347560122625008, + "tokens_seen": 2705195008 + }, + { + "epoch": 0.82, + "learning_rate": 9.111334002006018e-05, + "loss": 1.3835, + "theoretical_loss": 3.3475473886200495, + "tokens_seen": 2705326080 + }, + { + "epoch": 0.82, + "learning_rate": 9.109327983951856e-05, + "loss": 1.1473, + "theoretical_loss": 3.3475346554047722, + "tokens_seen": 2705457152 + }, + { + "epoch": 0.82, + "learning_rate": 9.107321965897692e-05, + "loss": 1.186, + "theoretical_loss": 3.347521922979089, + "tokens_seen": 2705588224 + }, + { + "epoch": 0.82, + "learning_rate": 9.10531594784353e-05, + "loss": 1.3039, + "theoretical_loss": 3.3475091913429127, + "tokens_seen": 2705719296 + }, + { + "epoch": 0.82, + "learning_rate": 9.103309929789367e-05, + "loss": 1.4094, + "theoretical_loss": 3.3474964604961563, + "tokens_seen": 2705850368 + }, + { + "epoch": 0.82, + "learning_rate": 9.101303911735206e-05, + "loss": 1.1248, + "theoretical_loss": 3.3474837304387317, + "tokens_seen": 2705981440 + }, + { + "epoch": 0.82, + "learning_rate": 9.099297893681044e-05, + "loss": 1.3409, + "theoretical_loss": 3.3474710011705526, + "tokens_seen": 2706112512 + }, + { + "epoch": 0.82, + "learning_rate": 9.097291875626881e-05, + "loss": 1.2587, + "theoretical_loss": 3.347458272691532, + "tokens_seen": 2706243584 + }, + { + "epoch": 0.82, + "learning_rate": 9.095285857572719e-05, + "loss": 1.1998, + "theoretical_loss": 3.347445545001582, + "tokens_seen": 2706374656 + }, + { + "epoch": 0.82, + "learning_rate": 9.093279839518557e-05, + "loss": 1.2798, + "theoretical_loss": 3.3474328181006157, + "tokens_seen": 2706505728 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.473240464925766, + "objective/train/avg_token_score": 0.011922241188585758, + "objective/train/docs_used": 1523555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.284079670906067, + "objective/train/original_loss": 3.533848524093628, + "objective/train/theoretical_loss": 3.347420091988546, + "objective/train/tokens_used": 2706636800, + "objective/train/unlikelihood_loss": 0.15924687683582306, + "theoretical_loss": 3.347420091988546, + "tokens_seen": 2706636800 + }, + { + "epoch": 0.82, + "learning_rate": 9.091273821464394e-05, + "loss": 1.3605, + "theoretical_loss": 3.347420091988546, + "tokens_seen": 2706636800 + }, + { + "epoch": 0.82, + "learning_rate": 9.089267803410232e-05, + "loss": 1.1019, + "theoretical_loss": 3.3474073666652866, + "tokens_seen": 2706767872 + }, + { + "epoch": 0.82, + "learning_rate": 9.087261785356068e-05, + "loss": 1.1679, + "theoretical_loss": 3.347394642130749, + "tokens_seen": 2706898944 + }, + { + "epoch": 0.82, + "learning_rate": 9.085255767301905e-05, + "loss": 1.1912, + "theoretical_loss": 3.3473819183848477, + "tokens_seen": 2707030016 + }, + { + "epoch": 0.82, + "learning_rate": 9.083249749247743e-05, + "loss": 1.1375, + "theoretical_loss": 3.3473691954274942, + "tokens_seen": 2707161088 + }, + { + "epoch": 0.82, + "learning_rate": 9.08124373119358e-05, + "loss": 1.4196, + "theoretical_loss": 3.347356473258602, + "tokens_seen": 2707292160 + }, + { + "epoch": 0.82, + "learning_rate": 9.079237713139418e-05, + "loss": 1.3316, + "theoretical_loss": 3.347343751878084, + "tokens_seen": 2707423232 + }, + { + "epoch": 0.82, + "learning_rate": 9.077231695085256e-05, + "loss": 1.2661, + "theoretical_loss": 3.3473310312858535, + "tokens_seen": 2707554304 + }, + { + "epoch": 0.82, + "learning_rate": 9.075225677031093e-05, + "loss": 1.238, + "theoretical_loss": 3.347318311481824, + "tokens_seen": 2707685376 + }, + { + "epoch": 0.82, + "learning_rate": 9.073219658976931e-05, + "loss": 1.2467, + "theoretical_loss": 3.347305592465907, + "tokens_seen": 2707816448 + }, + { + "epoch": 0.82, + "learning_rate": 9.071213640922769e-05, + "loss": 1.2268, + "theoretical_loss": 3.347292874238016, + "tokens_seen": 2707947520 + }, + { + "epoch": 0.82, + "learning_rate": 9.069207622868606e-05, + "loss": 1.1381, + "theoretical_loss": 3.3472801567980652, + "tokens_seen": 2708078592 + }, + { + "epoch": 0.82, + "learning_rate": 9.067201604814442e-05, + "loss": 1.2746, + "theoretical_loss": 3.3472674401459663, + "tokens_seen": 2708209664 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.2875122129917145, + "objective/train/avg_token_score": 0.013411489315330982, + "objective/train/docs_used": 1524121, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8509514331817627, + "objective/train/original_loss": 3.872152090072632, + "objective/train/theoretical_loss": 3.3472610821153346, + "objective/train/tokens_used": 2708275200, + "objective/train/unlikelihood_loss": 0.23739905655384064, + "theoretical_loss": 3.3472610821153346, + "tokens_seen": 2708275200 + }, + { + "epoch": 0.82, + "learning_rate": 9.065195586760281e-05, + "loss": 1.2063, + "theoretical_loss": 3.3472547242816333, + "tokens_seen": 2708340736 + }, + { + "epoch": 0.82, + "learning_rate": 9.063189568706119e-05, + "loss": 1.2374, + "theoretical_loss": 3.3472420092049786, + "tokens_seen": 2708471808 + }, + { + "epoch": 0.82, + "learning_rate": 9.061183550651956e-05, + "loss": 1.2232, + "theoretical_loss": 3.3472292949159157, + "tokens_seen": 2708602880 + }, + { + "epoch": 0.82, + "learning_rate": 9.059177532597794e-05, + "loss": 1.2782, + "theoretical_loss": 3.3472165814143575, + "tokens_seen": 2708733952 + }, + { + "epoch": 0.82, + "learning_rate": 9.057171514543632e-05, + "loss": 1.3078, + "theoretical_loss": 3.347203868700217, + "tokens_seen": 2708865024 + }, + { + "epoch": 0.82, + "learning_rate": 9.055165496489469e-05, + "loss": 1.3128, + "theoretical_loss": 3.347191156773408, + "tokens_seen": 2708996096 + }, + { + "epoch": 0.82, + "learning_rate": 9.053159478435307e-05, + "loss": 1.4307, + "theoretical_loss": 3.3471784456338427, + "tokens_seen": 2709127168 + }, + { + "epoch": 0.82, + "learning_rate": 9.051153460381144e-05, + "loss": 1.3144, + "theoretical_loss": 3.3471657352814352, + "tokens_seen": 2709258240 + }, + { + "epoch": 0.82, + "learning_rate": 9.04914744232698e-05, + "loss": 1.1172, + "theoretical_loss": 3.3471530257160977, + "tokens_seen": 2709389312 + }, + { + "epoch": 0.82, + "learning_rate": 9.047141424272818e-05, + "loss": 1.1872, + "theoretical_loss": 3.347140316937744, + "tokens_seen": 2709520384 + }, + { + "epoch": 0.82, + "learning_rate": 9.045135406218656e-05, + "loss": 1.2494, + "theoretical_loss": 3.3471276089462876, + "tokens_seen": 2709651456 + }, + { + "epoch": 0.82, + "learning_rate": 9.043129388164493e-05, + "loss": 1.2454, + "theoretical_loss": 3.347114901741641, + "tokens_seen": 2709782528 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.47458454966545105, + "objective/train/avg_token_score": 0.0026432343292981386, + "objective/train/docs_used": 1524904, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4641034603118896, + "objective/train/original_loss": 3.360577344894409, + "objective/train/theoretical_loss": 3.3471021953237177, + "objective/train/tokens_used": 2709913600, + "objective/train/unlikelihood_loss": 0.21535059809684753, + "theoretical_loss": 3.3471021953237177, + "tokens_seen": 2709913600 + }, + { + "epoch": 0.82, + "learning_rate": 9.041123370110331e-05, + "loss": 1.2434, + "theoretical_loss": 3.3471021953237177, + "tokens_seen": 2709913600 + }, + { + "epoch": 0.82, + "learning_rate": 9.039117352056168e-05, + "loss": 1.0729, + "theoretical_loss": 3.347089489692431, + "tokens_seen": 2710044672 + }, + { + "epoch": 0.82, + "learning_rate": 9.037111334002006e-05, + "loss": 1.252, + "theoretical_loss": 3.347076784847694, + "tokens_seen": 2710175744 + }, + { + "epoch": 0.82, + "learning_rate": 9.035105315947844e-05, + "loss": 1.2833, + "theoretical_loss": 3.3470640807894205, + "tokens_seen": 2710306816 + }, + { + "epoch": 0.82, + "learning_rate": 9.033099297893681e-05, + "loss": 1.1458, + "theoretical_loss": 3.347051377517523, + "tokens_seen": 2710437888 + }, + { + "epoch": 0.82, + "learning_rate": 9.031093279839519e-05, + "loss": 1.1965, + "theoretical_loss": 3.3470386750319157, + "tokens_seen": 2710568960 + }, + { + "epoch": 0.82, + "learning_rate": 9.029087261785355e-05, + "loss": 1.1955, + "theoretical_loss": 3.347025973332511, + "tokens_seen": 2710700032 + }, + { + "epoch": 0.82, + "learning_rate": 9.027081243731194e-05, + "loss": 1.246, + "theoretical_loss": 3.3470132724192228, + "tokens_seen": 2710831104 + }, + { + "epoch": 0.82, + "learning_rate": 9.025075225677032e-05, + "loss": 1.3337, + "theoretical_loss": 3.3470005722919645, + "tokens_seen": 2710962176 + }, + { + "epoch": 0.82, + "learning_rate": 9.023069207622869e-05, + "loss": 1.2998, + "theoretical_loss": 3.346987872950649, + "tokens_seen": 2711093248 + }, + { + "epoch": 0.82, + "learning_rate": 9.021063189568707e-05, + "loss": 1.1285, + "theoretical_loss": 3.34697517439519, + "tokens_seen": 2711224320 + }, + { + "epoch": 0.82, + "learning_rate": 9.019057171514544e-05, + "loss": 1.2736, + "theoretical_loss": 3.3469624766255004, + "tokens_seen": 2711355392 + }, + { + "epoch": 0.82, + "learning_rate": 9.017051153460382e-05, + "loss": 1.2447, + "theoretical_loss": 3.3469497796414944, + "tokens_seen": 2711486464 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.3655303120613098, + "objective/train/avg_token_score": 0.011316794902086258, + "objective/train/docs_used": 1526439, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0749824047088623, + "objective/train/original_loss": 3.4696121215820312, + "objective/train/theoretical_loss": 3.3469434314440956, + "objective/train/tokens_used": 2711552000, + "objective/train/unlikelihood_loss": 0.2747214734554291, + "theoretical_loss": 3.3469434314440956, + "tokens_seen": 2711552000 + }, + { + "epoch": 0.82, + "learning_rate": 9.01504513540622e-05, + "loss": 1.3111, + "theoretical_loss": 3.346937083443085, + "tokens_seen": 2711617536 + }, + { + "epoch": 0.82, + "learning_rate": 9.013039117352057e-05, + "loss": 1.3223, + "theoretical_loss": 3.3469243880301853, + "tokens_seen": 2711748608 + }, + { + "epoch": 0.82, + "learning_rate": 9.011033099297895e-05, + "loss": 1.1923, + "theoretical_loss": 3.3469116934027094, + "tokens_seen": 2711879680 + }, + { + "epoch": 0.82, + "learning_rate": 9.009027081243731e-05, + "loss": 1.2205, + "theoretical_loss": 3.3468989995605702, + "tokens_seen": 2712010752 + }, + { + "epoch": 0.82, + "learning_rate": 9.007021063189568e-05, + "loss": 1.2602, + "theoretical_loss": 3.3468863065036816, + "tokens_seen": 2712141824 + }, + { + "epoch": 0.82, + "learning_rate": 9.005015045135406e-05, + "loss": 1.3156, + "theoretical_loss": 3.3468736142319564, + "tokens_seen": 2712272896 + }, + { + "epoch": 0.82, + "learning_rate": 9.003009027081244e-05, + "loss": 1.1802, + "theoretical_loss": 3.346860922745309, + "tokens_seen": 2712403968 + }, + { + "epoch": 0.82, + "learning_rate": 9.001003009027081e-05, + "loss": 1.1112, + "theoretical_loss": 3.3468482320436523, + "tokens_seen": 2712535040 + }, + { + "epoch": 0.82, + "learning_rate": 8.998996990972919e-05, + "loss": 1.347, + "theoretical_loss": 3.3468355421269, + "tokens_seen": 2712666112 + }, + { + "epoch": 0.82, + "learning_rate": 8.996990972918756e-05, + "loss": 1.2715, + "theoretical_loss": 3.3468228529949653, + "tokens_seen": 2712797184 + }, + { + "epoch": 0.82, + "learning_rate": 8.994984954864594e-05, + "loss": 1.2003, + "theoretical_loss": 3.3468101646477626, + "tokens_seen": 2712928256 + }, + { + "epoch": 0.82, + "learning_rate": 8.992978936810431e-05, + "loss": 1.1662, + "theoretical_loss": 3.3467974770852043, + "tokens_seen": 2713059328 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.2963709533214569, + "objective/train/avg_token_score": 0.011671148240566254, + "objective/train/docs_used": 1527123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7441717386245728, + "objective/train/original_loss": 3.7743561267852783, + "objective/train/theoretical_loss": 3.346784790307205, + "objective/train/tokens_used": 2713190400, + "objective/train/unlikelihood_loss": 0.17131181061267853, + "theoretical_loss": 3.346784790307205, + "tokens_seen": 2713190400 + }, + { + "epoch": 0.82, + "learning_rate": 8.990972918756269e-05, + "loss": 1.2238, + "theoretical_loss": 3.346784790307205, + "tokens_seen": 2713190400 + }, + { + "epoch": 0.82, + "learning_rate": 8.988966900702107e-05, + "loss": 1.3026, + "theoretical_loss": 3.3467721043136773, + "tokens_seen": 2713321472 + }, + { + "epoch": 0.82, + "learning_rate": 8.986960882647944e-05, + "loss": 1.3051, + "theoretical_loss": 3.346759419104536, + "tokens_seen": 2713452544 + }, + { + "epoch": 0.82, + "learning_rate": 8.984954864593782e-05, + "loss": 1.1277, + "theoretical_loss": 3.346746734679694, + "tokens_seen": 2713583616 + }, + { + "epoch": 0.82, + "learning_rate": 8.982948846539619e-05, + "loss": 1.3184, + "theoretical_loss": 3.3467340510390646, + "tokens_seen": 2713714688 + }, + { + "epoch": 0.82, + "learning_rate": 8.980942828485457e-05, + "loss": 1.2598, + "theoretical_loss": 3.3467213681825623, + "tokens_seen": 2713845760 + }, + { + "epoch": 0.82, + "learning_rate": 8.978936810431294e-05, + "loss": 1.3834, + "theoretical_loss": 3.3467086861101, + "tokens_seen": 2713976832 + }, + { + "epoch": 0.82, + "learning_rate": 8.976930792377132e-05, + "loss": 1.432, + "theoretical_loss": 3.3466960048215917, + "tokens_seen": 2714107904 + }, + { + "epoch": 0.82, + "learning_rate": 8.97492477432297e-05, + "loss": 1.1821, + "theoretical_loss": 3.3466833243169516, + "tokens_seen": 2714238976 + }, + { + "epoch": 0.82, + "learning_rate": 8.972918756268807e-05, + "loss": 1.1926, + "theoretical_loss": 3.346670644596092, + "tokens_seen": 2714370048 + }, + { + "epoch": 0.82, + "learning_rate": 8.970912738214643e-05, + "loss": 1.429, + "theoretical_loss": 3.346657965658928, + "tokens_seen": 2714501120 + }, + { + "epoch": 0.82, + "learning_rate": 8.968906720160481e-05, + "loss": 1.2784, + "theoretical_loss": 3.346645287505373, + "tokens_seen": 2714632192 + }, + { + "epoch": 0.82, + "learning_rate": 8.966900702106319e-05, + "loss": 1.421, + "theoretical_loss": 3.34663261013534, + "tokens_seen": 2714763264 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.4042644202709198, + "objective/train/avg_token_score": 0.01667145825922489, + "objective/train/docs_used": 1528322, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2479891777038574, + "objective/train/original_loss": 3.938692569732666, + "objective/train/theoretical_loss": 3.3466262717441175, + "objective/train/tokens_used": 2714828800, + "objective/train/unlikelihood_loss": 0.14671659469604492, + "theoretical_loss": 3.3466262717441175, + "tokens_seen": 2714828800 + }, + { + "epoch": 0.82, + "learning_rate": 8.964894684052156e-05, + "loss": 1.4155, + "theoretical_loss": 3.346619933548743, + "tokens_seen": 2714894336 + }, + { + "epoch": 0.82, + "learning_rate": 8.962888665997994e-05, + "loss": 1.4839, + "theoretical_loss": 3.3466072577454966, + "tokens_seen": 2715025408 + }, + { + "epoch": 0.82, + "learning_rate": 8.960882647943831e-05, + "loss": 1.3369, + "theoretical_loss": 3.346594582725514, + "tokens_seen": 2715156480 + }, + { + "epoch": 0.82, + "learning_rate": 8.958876629889669e-05, + "loss": 1.4229, + "theoretical_loss": 3.346581908488709, + "tokens_seen": 2715287552 + }, + { + "epoch": 0.82, + "learning_rate": 8.956870611835506e-05, + "loss": 1.2839, + "theoretical_loss": 3.3465692350349956, + "tokens_seen": 2715418624 + }, + { + "epoch": 0.82, + "learning_rate": 8.954864593781344e-05, + "loss": 1.38, + "theoretical_loss": 3.3465565623642872, + "tokens_seen": 2715549696 + }, + { + "epoch": 0.82, + "learning_rate": 8.952858575727182e-05, + "loss": 1.3525, + "theoretical_loss": 3.346543890476498, + "tokens_seen": 2715680768 + }, + { + "epoch": 0.82, + "learning_rate": 8.950852557673019e-05, + "loss": 1.324, + "theoretical_loss": 3.3465312193715415, + "tokens_seen": 2715811840 + }, + { + "epoch": 0.82, + "learning_rate": 8.948846539618857e-05, + "loss": 1.347, + "theoretical_loss": 3.346518549049332, + "tokens_seen": 2715942912 + }, + { + "epoch": 0.82, + "learning_rate": 8.946840521564694e-05, + "loss": 1.1807, + "theoretical_loss": 3.3465058795097833, + "tokens_seen": 2716073984 + }, + { + "epoch": 0.82, + "learning_rate": 8.944834503510532e-05, + "loss": 1.3023, + "theoretical_loss": 3.3464932107528087, + "tokens_seen": 2716205056 + }, + { + "epoch": 0.82, + "learning_rate": 8.94282848545637e-05, + "loss": 1.3555, + "theoretical_loss": 3.346480542778323, + "tokens_seen": 2716336128 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.41703322529792786, + "objective/train/avg_token_score": 0.014088377356529236, + "objective/train/docs_used": 1528969, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.211898684501648, + "objective/train/original_loss": 3.3479769229888916, + "objective/train/theoretical_loss": 3.346467875586239, + "objective/train/tokens_used": 2716467200, + "objective/train/unlikelihood_loss": 0.22551369667053223, + "theoretical_loss": 3.346467875586239, + "tokens_seen": 2716467200 + }, + { + "epoch": 0.82, + "learning_rate": 8.940822467402207e-05, + "loss": 1.3179, + "theoretical_loss": 3.346467875586239, + "tokens_seen": 2716467200 + }, + { + "epoch": 0.82, + "learning_rate": 8.938816449348045e-05, + "loss": 1.2064, + "theoretical_loss": 3.346455209176472, + "tokens_seen": 2716598272 + }, + { + "epoch": 0.82, + "learning_rate": 8.936810431293882e-05, + "loss": 1.2643, + "theoretical_loss": 3.346442543548935, + "tokens_seen": 2716729344 + }, + { + "epoch": 0.82, + "learning_rate": 8.93480441323972e-05, + "loss": 1.4909, + "theoretical_loss": 3.346429878703542, + "tokens_seen": 2716860416 + }, + { + "epoch": 0.82, + "learning_rate": 8.932798395185557e-05, + "loss": 1.4039, + "theoretical_loss": 3.3464172146402076, + "tokens_seen": 2716991488 + }, + { + "epoch": 0.82, + "learning_rate": 8.930792377131394e-05, + "loss": 1.2159, + "theoretical_loss": 3.346404551358845, + "tokens_seen": 2717122560 + }, + { + "epoch": 0.82, + "learning_rate": 8.928786359077231e-05, + "loss": 1.3335, + "theoretical_loss": 3.3463918888593684, + "tokens_seen": 2717253632 + }, + { + "epoch": 0.82, + "learning_rate": 8.926780341023069e-05, + "loss": 1.2168, + "theoretical_loss": 3.3463792271416923, + "tokens_seen": 2717384704 + }, + { + "epoch": 0.82, + "learning_rate": 8.924774322968906e-05, + "loss": 1.404, + "theoretical_loss": 3.34636656620573, + "tokens_seen": 2717515776 + }, + { + "epoch": 0.82, + "learning_rate": 8.922768304914744e-05, + "loss": 1.1649, + "theoretical_loss": 3.3463539060513963, + "tokens_seen": 2717646848 + }, + { + "epoch": 0.82, + "learning_rate": 8.920762286860582e-05, + "loss": 1.2592, + "theoretical_loss": 3.346341246678605, + "tokens_seen": 2717777920 + }, + { + "epoch": 0.82, + "learning_rate": 8.918756268806419e-05, + "loss": 1.292, + "theoretical_loss": 3.3463285880872697, + "tokens_seen": 2717908992 + }, + { + "epoch": 0.82, + "learning_rate": 8.916750250752257e-05, + "loss": 1.2563, + "theoretical_loss": 3.3463159302773047, + "tokens_seen": 2718040064 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.30364125967025757, + "objective/train/avg_token_score": 0.0360247828066349, + "objective/train/docs_used": 1530384, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9233498573303223, + "objective/train/original_loss": 4.559920310974121, + "objective/train/theoretical_loss": 3.3463096016653093, + "objective/train/tokens_used": 2718105600, + "objective/train/unlikelihood_loss": 0.09250689297914505, + "theoretical_loss": 3.3463096016653093, + "tokens_seen": 2718105600 + }, + { + "epoch": 0.82, + "learning_rate": 8.914744232698096e-05, + "loss": 1.3542, + "theoretical_loss": 3.3463032732486244, + "tokens_seen": 2718171136 + }, + { + "epoch": 0.82, + "learning_rate": 8.912738214643933e-05, + "loss": 1.3133, + "theoretical_loss": 3.346290617001143, + "tokens_seen": 2718302208 + }, + { + "epoch": 0.82, + "learning_rate": 8.91073219658977e-05, + "loss": 1.4469, + "theoretical_loss": 3.3462779615347737, + "tokens_seen": 2718433280 + }, + { + "epoch": 0.82, + "learning_rate": 8.908726178535607e-05, + "loss": 1.3264, + "theoretical_loss": 3.3462653068494315, + "tokens_seen": 2718564352 + }, + { + "epoch": 0.82, + "learning_rate": 8.906720160481445e-05, + "loss": 1.3084, + "theoretical_loss": 3.3462526529450303, + "tokens_seen": 2718695424 + }, + { + "epoch": 0.82, + "learning_rate": 8.904714142427282e-05, + "loss": 1.1968, + "theoretical_loss": 3.3462399998214845, + "tokens_seen": 2718826496 + }, + { + "epoch": 0.82, + "learning_rate": 8.90270812437312e-05, + "loss": 1.1399, + "theoretical_loss": 3.346227347478708, + "tokens_seen": 2718957568 + }, + { + "epoch": 0.82, + "learning_rate": 8.900702106318957e-05, + "loss": 1.2113, + "theoretical_loss": 3.346214695916615, + "tokens_seen": 2719088640 + }, + { + "epoch": 0.82, + "learning_rate": 8.898696088264795e-05, + "loss": 1.3968, + "theoretical_loss": 3.3462020451351195, + "tokens_seen": 2719219712 + }, + { + "epoch": 0.82, + "learning_rate": 8.896690070210633e-05, + "loss": 1.31, + "theoretical_loss": 3.346189395134136, + "tokens_seen": 2719350784 + }, + { + "epoch": 0.82, + "learning_rate": 8.89468405215647e-05, + "loss": 1.2687, + "theoretical_loss": 3.3461767459135787, + "tokens_seen": 2719481856 + }, + { + "epoch": 0.82, + "learning_rate": 8.892678034102306e-05, + "loss": 1.0824, + "theoretical_loss": 3.346164097473362, + "tokens_seen": 2719612928 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.4692693054676056, + "objective/train/avg_token_score": 0.025277819484472275, + "objective/train/docs_used": 1531036, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4607290029525757, + "objective/train/original_loss": 3.745020866394043, + "objective/train/theoretical_loss": 3.3461514498133997, + "objective/train/tokens_used": 2719744000, + "objective/train/unlikelihood_loss": 0.1545896977186203, + "theoretical_loss": 3.3461514498133997, + "tokens_seen": 2719744000 + }, + { + "epoch": 0.82, + "learning_rate": 8.890672016048144e-05, + "loss": 1.4738, + "theoretical_loss": 3.3461514498133997, + "tokens_seen": 2719744000 + }, + { + "epoch": 0.82, + "learning_rate": 8.888665997993981e-05, + "loss": 1.3575, + "theoretical_loss": 3.3461388029336065, + "tokens_seen": 2719875072 + }, + { + "epoch": 0.82, + "learning_rate": 8.886659979939819e-05, + "loss": 1.1766, + "theoretical_loss": 3.346126156833897, + "tokens_seen": 2720006144 + }, + { + "epoch": 0.82, + "learning_rate": 8.884653961885657e-05, + "loss": 1.3407, + "theoretical_loss": 3.3461135115141842, + "tokens_seen": 2720137216 + }, + { + "epoch": 0.82, + "learning_rate": 8.882647943831494e-05, + "loss": 1.2676, + "theoretical_loss": 3.3461008669743837, + "tokens_seen": 2720268288 + }, + { + "epoch": 0.82, + "learning_rate": 8.880641925777332e-05, + "loss": 1.2725, + "theoretical_loss": 3.346088223214409, + "tokens_seen": 2720399360 + }, + { + "epoch": 0.82, + "learning_rate": 8.87863590772317e-05, + "loss": 1.246, + "theoretical_loss": 3.3460755802341753, + "tokens_seen": 2720530432 + }, + { + "epoch": 0.82, + "learning_rate": 8.876629889669008e-05, + "loss": 1.3654, + "theoretical_loss": 3.346062938033596, + "tokens_seen": 2720661504 + }, + { + "epoch": 0.82, + "learning_rate": 8.874623871614846e-05, + "loss": 1.1452, + "theoretical_loss": 3.346050296612586, + "tokens_seen": 2720792576 + }, + { + "epoch": 0.82, + "learning_rate": 8.872617853560682e-05, + "loss": 1.2636, + "theoretical_loss": 3.3460376559710596, + "tokens_seen": 2720923648 + }, + { + "epoch": 0.82, + "learning_rate": 8.87061183550652e-05, + "loss": 1.4028, + "theoretical_loss": 3.346025016108931, + "tokens_seen": 2721054720 + }, + { + "epoch": 0.82, + "learning_rate": 8.868605817452357e-05, + "loss": 1.4017, + "theoretical_loss": 3.3460123770261148, + "tokens_seen": 2721185792 + }, + { + "epoch": 0.82, + "learning_rate": 8.866599799398195e-05, + "loss": 1.2415, + "theoretical_loss": 3.3459997387225258, + "tokens_seen": 2721316864 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.3342497646808624, + "objective/train/avg_token_score": 0.02312716282904148, + "objective/train/docs_used": 1532195, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0391418933868408, + "objective/train/original_loss": 3.931753396987915, + "objective/train/theoretical_loss": 3.345993419862914, + "objective/train/tokens_used": 2721382400, + "objective/train/unlikelihood_loss": 0.16659608483314514, + "theoretical_loss": 3.345993419862914, + "tokens_seen": 2721382400 + }, + { + "epoch": 0.82, + "learning_rate": 8.864593781344032e-05, + "loss": 1.2373, + "theoretical_loss": 3.3459871011980775, + "tokens_seen": 2721447936 + }, + { + "epoch": 0.82, + "learning_rate": 8.86258776328987e-05, + "loss": 1.2523, + "theoretical_loss": 3.345974464452685, + "tokens_seen": 2721579008 + }, + { + "epoch": 0.82, + "learning_rate": 8.860581745235708e-05, + "loss": 1.1349, + "theoretical_loss": 3.3459618284862627, + "tokens_seen": 2721710080 + }, + { + "epoch": 0.82, + "learning_rate": 8.858575727181545e-05, + "loss": 1.1338, + "theoretical_loss": 3.345949193298725, + "tokens_seen": 2721841152 + }, + { + "epoch": 0.82, + "learning_rate": 8.856569709127383e-05, + "loss": 1.2534, + "theoretical_loss": 3.3459365588899863, + "tokens_seen": 2721972224 + }, + { + "epoch": 0.82, + "learning_rate": 8.85456369107322e-05, + "loss": 1.2871, + "theoretical_loss": 3.3459239252599615, + "tokens_seen": 2722103296 + }, + { + "epoch": 0.82, + "learning_rate": 8.852557673019057e-05, + "loss": 1.3392, + "theoretical_loss": 3.3459112924085646, + "tokens_seen": 2722234368 + }, + { + "epoch": 0.82, + "learning_rate": 8.850551654964894e-05, + "loss": 1.1964, + "theoretical_loss": 3.34589866033571, + "tokens_seen": 2722365440 + }, + { + "epoch": 0.82, + "learning_rate": 8.848545636910732e-05, + "loss": 1.1246, + "theoretical_loss": 3.345886029041313, + "tokens_seen": 2722496512 + }, + { + "epoch": 0.83, + "learning_rate": 8.846539618856569e-05, + "loss": 1.2343, + "theoretical_loss": 3.3458733985252875, + "tokens_seen": 2722627584 + }, + { + "epoch": 0.83, + "learning_rate": 8.844533600802407e-05, + "loss": 1.3115, + "theoretical_loss": 3.3458607687875483, + "tokens_seen": 2722758656 + }, + { + "epoch": 0.83, + "learning_rate": 8.842527582748244e-05, + "loss": 1.2779, + "theoretical_loss": 3.3458481398280098, + "tokens_seen": 2722889728 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.5094085931777954, + "objective/train/avg_token_score": 0.0032778680324554443, + "objective/train/docs_used": 1532891, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4592459201812744, + "objective/train/original_loss": 3.324869394302368, + "objective/train/theoretical_loss": 3.345835511646587, + "objective/train/tokens_used": 2723020800, + "objective/train/unlikelihood_loss": 0.21928562223911285, + "theoretical_loss": 3.345835511646587, + "tokens_seen": 2723020800 + }, + { + "epoch": 0.83, + "learning_rate": 8.840521564694082e-05, + "loss": 1.2416, + "theoretical_loss": 3.345835511646587, + "tokens_seen": 2723020800 + }, + { + "epoch": 0.83, + "learning_rate": 8.838515546639921e-05, + "loss": 1.3268, + "theoretical_loss": 3.345822884243194, + "tokens_seen": 2723151872 + }, + { + "epoch": 0.83, + "learning_rate": 8.836509528585759e-05, + "loss": 1.2028, + "theoretical_loss": 3.345810257617746, + "tokens_seen": 2723282944 + }, + { + "epoch": 0.83, + "learning_rate": 8.834503510531596e-05, + "loss": 1.3334, + "theoretical_loss": 3.3457976317701568, + "tokens_seen": 2723414016 + }, + { + "epoch": 0.83, + "learning_rate": 8.832497492477432e-05, + "loss": 1.2092, + "theoretical_loss": 3.345785006700342, + "tokens_seen": 2723545088 + }, + { + "epoch": 0.83, + "learning_rate": 8.83049147442327e-05, + "loss": 1.2179, + "theoretical_loss": 3.3457723824082155, + "tokens_seen": 2723676160 + }, + { + "epoch": 0.83, + "learning_rate": 8.828485456369108e-05, + "loss": 1.2535, + "theoretical_loss": 3.3457597588936925, + "tokens_seen": 2723807232 + }, + { + "epoch": 0.83, + "learning_rate": 8.826479438314945e-05, + "loss": 1.2535, + "theoretical_loss": 3.3457471361566875, + "tokens_seen": 2723938304 + }, + { + "epoch": 0.83, + "learning_rate": 8.824473420260783e-05, + "loss": 1.0553, + "theoretical_loss": 3.345734514197115, + "tokens_seen": 2724069376 + }, + { + "epoch": 0.83, + "learning_rate": 8.82246740220662e-05, + "loss": 1.0886, + "theoretical_loss": 3.3457218930148898, + "tokens_seen": 2724200448 + }, + { + "epoch": 0.83, + "learning_rate": 8.820461384152458e-05, + "loss": 1.3274, + "theoretical_loss": 3.345709272609927, + "tokens_seen": 2724331520 + }, + { + "epoch": 0.83, + "learning_rate": 8.818455366098295e-05, + "loss": 1.3011, + "theoretical_loss": 3.345696652982141, + "tokens_seen": 2724462592 + }, + { + "epoch": 0.83, + "learning_rate": 8.816449348044133e-05, + "loss": 1.3876, + "theoretical_loss": 3.3456840341314464, + "tokens_seen": 2724593664 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.37939882278442383, + "objective/train/avg_token_score": 0.0035177096724510193, + "objective/train/docs_used": 1534156, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1749433279037476, + "objective/train/original_loss": 3.6425223350524902, + "objective/train/theoretical_loss": 3.345677724997482, + "objective/train/tokens_used": 2724659200, + "objective/train/unlikelihood_loss": 0.16193607449531555, + "theoretical_loss": 3.345677724997482, + "tokens_seen": 2724659200 + }, + { + "epoch": 0.83, + "learning_rate": 8.814443329989969e-05, + "loss": 1.1046, + "theoretical_loss": 3.3456714160577583, + "tokens_seen": 2724724736 + }, + { + "epoch": 0.83, + "learning_rate": 8.812437311935807e-05, + "loss": 1.2485, + "theoretical_loss": 3.3456587987609914, + "tokens_seen": 2724855808 + }, + { + "epoch": 0.83, + "learning_rate": 8.810431293881644e-05, + "loss": 1.3146, + "theoretical_loss": 3.3456461822410604, + "tokens_seen": 2724986880 + }, + { + "epoch": 0.83, + "learning_rate": 8.808425275827482e-05, + "loss": 1.2396, + "theoretical_loss": 3.3456335664978805, + "tokens_seen": 2725117952 + }, + { + "epoch": 0.83, + "learning_rate": 8.80641925777332e-05, + "loss": 1.2064, + "theoretical_loss": 3.345620951531366, + "tokens_seen": 2725249024 + }, + { + "epoch": 0.83, + "learning_rate": 8.804413239719157e-05, + "loss": 1.3302, + "theoretical_loss": 3.3456083373414316, + "tokens_seen": 2725380096 + }, + { + "epoch": 0.83, + "learning_rate": 8.802407221664996e-05, + "loss": 1.2804, + "theoretical_loss": 3.3455957239279925, + "tokens_seen": 2725511168 + }, + { + "epoch": 0.83, + "learning_rate": 8.800401203610834e-05, + "loss": 1.3451, + "theoretical_loss": 3.345583111290964, + "tokens_seen": 2725642240 + }, + { + "epoch": 0.83, + "learning_rate": 8.798395185556671e-05, + "loss": 1.2217, + "theoretical_loss": 3.34557049943026, + "tokens_seen": 2725773312 + }, + { + "epoch": 0.83, + "learning_rate": 8.796389167502509e-05, + "loss": 1.2018, + "theoretical_loss": 3.3455578883457964, + "tokens_seen": 2725904384 + }, + { + "epoch": 0.83, + "learning_rate": 8.794383149448345e-05, + "loss": 1.3879, + "theoretical_loss": 3.3455452780374872, + "tokens_seen": 2726035456 + }, + { + "epoch": 0.83, + "learning_rate": 8.792377131394183e-05, + "loss": 1.4013, + "theoretical_loss": 3.345532668505248, + "tokens_seen": 2726166528 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.2993035316467285, + "objective/train/avg_token_score": 0.054462555795907974, + "objective/train/docs_used": 1534846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.882710337638855, + "objective/train/original_loss": 3.8628334999084473, + "objective/train/theoretical_loss": 3.345520059748993, + "objective/train/tokens_used": 2726297600, + "objective/train/unlikelihood_loss": 0.1987127810716629, + "theoretical_loss": 3.345520059748993, + "tokens_seen": 2726297600 + }, + { + "epoch": 0.83, + "learning_rate": 8.79037111334002e-05, + "loss": 1.2427, + "theoretical_loss": 3.345520059748993, + "tokens_seen": 2726297600 + }, + { + "epoch": 0.83, + "learning_rate": 8.788365095285858e-05, + "loss": 1.3514, + "theoretical_loss": 3.345507451768638, + "tokens_seen": 2726428672 + }, + { + "epoch": 0.83, + "learning_rate": 8.786359077231695e-05, + "loss": 1.2514, + "theoretical_loss": 3.3454948445640973, + "tokens_seen": 2726559744 + }, + { + "epoch": 0.83, + "learning_rate": 8.784353059177533e-05, + "loss": 1.3435, + "theoretical_loss": 3.3454822381352862, + "tokens_seen": 2726690816 + }, + { + "epoch": 0.83, + "learning_rate": 8.78234704112337e-05, + "loss": 1.1805, + "theoretical_loss": 3.3454696324821196, + "tokens_seen": 2726821888 + }, + { + "epoch": 0.83, + "learning_rate": 8.780341023069208e-05, + "loss": 1.3086, + "theoretical_loss": 3.345457027604513, + "tokens_seen": 2726952960 + }, + { + "epoch": 0.83, + "learning_rate": 8.778335005015046e-05, + "loss": 1.2421, + "theoretical_loss": 3.3454444235023804, + "tokens_seen": 2727084032 + }, + { + "epoch": 0.83, + "learning_rate": 8.776328986960883e-05, + "loss": 1.2792, + "theoretical_loss": 3.3454318201756372, + "tokens_seen": 2727215104 + }, + { + "epoch": 0.83, + "learning_rate": 8.77432296890672e-05, + "loss": 1.1117, + "theoretical_loss": 3.345419217624199, + "tokens_seen": 2727346176 + }, + { + "epoch": 0.83, + "learning_rate": 8.772316950852557e-05, + "loss": 1.2058, + "theoretical_loss": 3.34540661584798, + "tokens_seen": 2727477248 + }, + { + "epoch": 0.83, + "learning_rate": 8.770310932798395e-05, + "loss": 1.2076, + "theoretical_loss": 3.3453940148468955, + "tokens_seen": 2727608320 + }, + { + "epoch": 0.83, + "learning_rate": 8.768304914744232e-05, + "loss": 1.1638, + "theoretical_loss": 3.345381414620861, + "tokens_seen": 2727739392 + }, + { + "epoch": 0.83, + "learning_rate": 8.76629889669007e-05, + "loss": 1.0564, + "theoretical_loss": 3.345368815169792, + "tokens_seen": 2727870464 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.2622189521789551, + "objective/train/avg_token_score": 0.012576471082866192, + "objective/train/docs_used": 1535777, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9074233174324036, + "objective/train/original_loss": 4.545993804931641, + "objective/train/theoretical_loss": 3.345362515734842, + "objective/train/tokens_used": 2727936000, + "objective/train/unlikelihood_loss": 0.11085735261440277, + "theoretical_loss": 3.345362515734842, + "tokens_seen": 2727936000 + }, + { + "epoch": 0.83, + "learning_rate": 8.764292878635909e-05, + "loss": 1.1842, + "theoretical_loss": 3.345356216493602, + "tokens_seen": 2728001536 + }, + { + "epoch": 0.83, + "learning_rate": 8.762286860581746e-05, + "loss": 1.3511, + "theoretical_loss": 3.3453436185922074, + "tokens_seen": 2728132608 + }, + { + "epoch": 0.83, + "learning_rate": 8.760280842527584e-05, + "loss": 1.2268, + "theoretical_loss": 3.345331021465523, + "tokens_seen": 2728263680 + }, + { + "epoch": 0.83, + "learning_rate": 8.758274824473421e-05, + "loss": 1.3259, + "theoretical_loss": 3.345318425113464, + "tokens_seen": 2728394752 + }, + { + "epoch": 0.83, + "learning_rate": 8.756268806419259e-05, + "loss": 1.2466, + "theoretical_loss": 3.3453058295359455, + "tokens_seen": 2728525824 + }, + { + "epoch": 0.83, + "learning_rate": 8.754262788365095e-05, + "loss": 1.319, + "theoretical_loss": 3.3452932347328823, + "tokens_seen": 2728656896 + }, + { + "epoch": 0.83, + "learning_rate": 8.752256770310933e-05, + "loss": 1.3152, + "theoretical_loss": 3.3452806407041904, + "tokens_seen": 2728787968 + }, + { + "epoch": 0.83, + "learning_rate": 8.75025075225677e-05, + "loss": 1.2826, + "theoretical_loss": 3.3452680474497845, + "tokens_seen": 2728919040 + }, + { + "epoch": 0.83, + "learning_rate": 8.748244734202608e-05, + "loss": 1.2831, + "theoretical_loss": 3.3452554549695797, + "tokens_seen": 2729050112 + }, + { + "epoch": 0.83, + "learning_rate": 8.746238716148446e-05, + "loss": 1.1731, + "theoretical_loss": 3.3452428632634916, + "tokens_seen": 2729181184 + }, + { + "epoch": 0.83, + "learning_rate": 8.744232698094283e-05, + "loss": 1.1495, + "theoretical_loss": 3.3452302723314347, + "tokens_seen": 2729312256 + }, + { + "epoch": 0.83, + "learning_rate": 8.742226680040121e-05, + "loss": 1.2075, + "theoretical_loss": 3.3452176821733253, + "tokens_seen": 2729443328 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.4729349911212921, + "objective/train/avg_token_score": 0.010250686667859554, + "objective/train/docs_used": 1536637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.400274634361267, + "objective/train/original_loss": 3.8939852714538574, + "objective/train/theoretical_loss": 3.3452050927890777, + "objective/train/tokens_used": 2729574400, + "objective/train/unlikelihood_loss": 0.14774316549301147, + "theoretical_loss": 3.3452050927890777, + "tokens_seen": 2729574400 + }, + { + "epoch": 0.83, + "learning_rate": 8.740220661985958e-05, + "loss": 1.3355, + "theoretical_loss": 3.3452050927890777, + "tokens_seen": 2729574400 + }, + { + "epoch": 0.83, + "learning_rate": 8.738214643931796e-05, + "loss": 1.3368, + "theoretical_loss": 3.345192504178608, + "tokens_seen": 2729705472 + }, + { + "epoch": 0.83, + "learning_rate": 8.736208625877632e-05, + "loss": 1.2107, + "theoretical_loss": 3.345179916341831, + "tokens_seen": 2729836544 + }, + { + "epoch": 0.83, + "learning_rate": 8.73420260782347e-05, + "loss": 1.1261, + "theoretical_loss": 3.345167329278662, + "tokens_seen": 2729967616 + }, + { + "epoch": 0.83, + "learning_rate": 8.732196589769307e-05, + "loss": 1.2091, + "theoretical_loss": 3.3451547429890165, + "tokens_seen": 2730098688 + }, + { + "epoch": 0.83, + "learning_rate": 8.730190571715145e-05, + "loss": 1.0476, + "theoretical_loss": 3.34514215747281, + "tokens_seen": 2730229760 + }, + { + "epoch": 0.83, + "learning_rate": 8.728184553660982e-05, + "loss": 1.251, + "theoretical_loss": 3.3451295727299573, + "tokens_seen": 2730360832 + }, + { + "epoch": 0.83, + "learning_rate": 8.726178535606821e-05, + "loss": 1.3443, + "theoretical_loss": 3.345116988760374, + "tokens_seen": 2730491904 + }, + { + "epoch": 0.83, + "learning_rate": 8.724172517552659e-05, + "loss": 1.4159, + "theoretical_loss": 3.3451044055639754, + "tokens_seen": 2730622976 + }, + { + "epoch": 0.83, + "learning_rate": 8.722166499498496e-05, + "loss": 1.2214, + "theoretical_loss": 3.3450918231406774, + "tokens_seen": 2730754048 + }, + { + "epoch": 0.83, + "learning_rate": 8.720160481444334e-05, + "loss": 1.2571, + "theoretical_loss": 3.345079241490395, + "tokens_seen": 2730885120 + }, + { + "epoch": 0.83, + "learning_rate": 8.718154463390172e-05, + "loss": 1.3131, + "theoretical_loss": 3.345066660613043, + "tokens_seen": 2731016192 + }, + { + "epoch": 0.83, + "learning_rate": 8.716148445336008e-05, + "loss": 1.3883, + "theoretical_loss": 3.345054080508538, + "tokens_seen": 2731147264 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.32233625650405884, + "objective/train/avg_token_score": 0.03610222041606903, + "objective/train/docs_used": 1538108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0409904718399048, + "objective/train/original_loss": 4.060774803161621, + "objective/train/theoretical_loss": 3.3450477907460767, + "objective/train/tokens_used": 2731212800, + "objective/train/unlikelihood_loss": 0.12987667322158813, + "theoretical_loss": 3.3450477907460767, + "tokens_seen": 2731212800 + }, + { + "epoch": 0.83, + "learning_rate": 8.714142427281845e-05, + "loss": 1.3476, + "theoretical_loss": 3.345041501176795, + "tokens_seen": 2731278336 + }, + { + "epoch": 0.83, + "learning_rate": 8.712136409227683e-05, + "loss": 1.1781, + "theoretical_loss": 3.345028922617729, + "tokens_seen": 2731409408 + }, + { + "epoch": 0.83, + "learning_rate": 8.71013039117352e-05, + "loss": 1.1867, + "theoretical_loss": 3.3450163448312558, + "tokens_seen": 2731540480 + }, + { + "epoch": 0.83, + "learning_rate": 8.708124373119358e-05, + "loss": 1.3142, + "theoretical_loss": 3.345003767817291, + "tokens_seen": 2731671552 + }, + { + "epoch": 0.83, + "learning_rate": 8.706118355065196e-05, + "loss": 1.2961, + "theoretical_loss": 3.3449911915757498, + "tokens_seen": 2731802624 + }, + { + "epoch": 0.83, + "learning_rate": 8.704112337011033e-05, + "loss": 1.3214, + "theoretical_loss": 3.344978616106548, + "tokens_seen": 2731933696 + }, + { + "epoch": 0.83, + "learning_rate": 8.702106318956871e-05, + "loss": 1.1531, + "theoretical_loss": 3.344966041409601, + "tokens_seen": 2732064768 + }, + { + "epoch": 0.83, + "learning_rate": 8.700100300902709e-05, + "loss": 1.1691, + "theoretical_loss": 3.344953467484824, + "tokens_seen": 2732195840 + }, + { + "epoch": 0.83, + "learning_rate": 8.698094282848546e-05, + "loss": 1.3242, + "theoretical_loss": 3.344940894332133, + "tokens_seen": 2732326912 + }, + { + "epoch": 0.83, + "learning_rate": 8.696088264794382e-05, + "loss": 1.2863, + "theoretical_loss": 3.3449283219514436, + "tokens_seen": 2732457984 + }, + { + "epoch": 0.83, + "learning_rate": 8.69408224674022e-05, + "loss": 1.1673, + "theoretical_loss": 3.344915750342671, + "tokens_seen": 2732589056 + }, + { + "epoch": 0.83, + "learning_rate": 8.692076228686057e-05, + "loss": 1.2983, + "theoretical_loss": 3.344903179505731, + "tokens_seen": 2732720128 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.4675586521625519, + "objective/train/avg_token_score": 0.010580961592495441, + "objective/train/docs_used": 1538743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3433258533477783, + "objective/train/original_loss": 3.747938632965088, + "objective/train/theoretical_loss": 3.3448906094405393, + "objective/train/tokens_used": 2732851200, + "objective/train/unlikelihood_loss": 0.12963837385177612, + "theoretical_loss": 3.3448906094405393, + "tokens_seen": 2732851200 + }, + { + "epoch": 0.83, + "learning_rate": 8.690070210631895e-05, + "loss": 1.2078, + "theoretical_loss": 3.3448906094405393, + "tokens_seen": 2732851200 + }, + { + "epoch": 0.83, + "learning_rate": 8.688064192577734e-05, + "loss": 1.3269, + "theoretical_loss": 3.3448780401470115, + "tokens_seen": 2732982272 + }, + { + "epoch": 0.83, + "learning_rate": 8.686058174523572e-05, + "loss": 1.2415, + "theoretical_loss": 3.344865471625063, + "tokens_seen": 2733113344 + }, + { + "epoch": 0.83, + "learning_rate": 8.684052156469409e-05, + "loss": 1.2667, + "theoretical_loss": 3.3448529038746093, + "tokens_seen": 2733244416 + }, + { + "epoch": 0.83, + "learning_rate": 8.682046138415247e-05, + "loss": 1.19, + "theoretical_loss": 3.344840336895566, + "tokens_seen": 2733375488 + }, + { + "epoch": 0.83, + "learning_rate": 8.680040120361084e-05, + "loss": 1.1346, + "theoretical_loss": 3.3448277706878495, + "tokens_seen": 2733506560 + }, + { + "epoch": 0.83, + "learning_rate": 8.67803410230692e-05, + "loss": 1.1427, + "theoretical_loss": 3.344815205251375, + "tokens_seen": 2733637632 + }, + { + "epoch": 0.83, + "learning_rate": 8.676028084252758e-05, + "loss": 1.1811, + "theoretical_loss": 3.344802640586058, + "tokens_seen": 2733768704 + }, + { + "epoch": 0.83, + "learning_rate": 8.674022066198596e-05, + "loss": 1.2836, + "theoretical_loss": 3.344790076691815, + "tokens_seen": 2733899776 + }, + { + "epoch": 0.83, + "learning_rate": 8.672016048144433e-05, + "loss": 1.2078, + "theoretical_loss": 3.3447775135685607, + "tokens_seen": 2734030848 + }, + { + "epoch": 0.83, + "learning_rate": 8.670010030090271e-05, + "loss": 1.2391, + "theoretical_loss": 3.3447649512162108, + "tokens_seen": 2734161920 + }, + { + "epoch": 0.83, + "learning_rate": 8.668004012036108e-05, + "loss": 1.1598, + "theoretical_loss": 3.344752389634682, + "tokens_seen": 2734292992 + }, + { + "epoch": 0.83, + "learning_rate": 8.665997993981946e-05, + "loss": 1.3652, + "theoretical_loss": 3.3447398288238896, + "tokens_seen": 2734424064 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.263868510723114, + "objective/train/avg_token_score": 0.012338168919086456, + "objective/train/docs_used": 1539437, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8755597472190857, + "objective/train/original_loss": 3.812645196914673, + "objective/train/theoretical_loss": 3.344733548707493, + "objective/train/tokens_used": 2734489600, + "objective/train/unlikelihood_loss": 0.20088450610637665, + "theoretical_loss": 3.344733548707493, + "tokens_seen": 2734489600 + }, + { + "epoch": 0.83, + "learning_rate": 8.663991975927784e-05, + "loss": 1.2032, + "theoretical_loss": 3.3447272687837493, + "tokens_seen": 2734555136 + }, + { + "epoch": 0.83, + "learning_rate": 8.661985957873621e-05, + "loss": 1.278, + "theoretical_loss": 3.344714709514177, + "tokens_seen": 2734686208 + }, + { + "epoch": 0.83, + "learning_rate": 8.659979939819459e-05, + "loss": 1.3385, + "theoretical_loss": 3.344702151015088, + "tokens_seen": 2734817280 + }, + { + "epoch": 0.83, + "learning_rate": 8.657973921765295e-05, + "loss": 1.2526, + "theoretical_loss": 3.3446895932863985, + "tokens_seen": 2734948352 + }, + { + "epoch": 0.83, + "learning_rate": 8.655967903711133e-05, + "loss": 1.4198, + "theoretical_loss": 3.3446770363280245, + "tokens_seen": 2735079424 + }, + { + "epoch": 0.83, + "learning_rate": 8.65396188565697e-05, + "loss": 1.2748, + "theoretical_loss": 3.344664480139882, + "tokens_seen": 2735210496 + }, + { + "epoch": 0.83, + "learning_rate": 8.651955867602809e-05, + "loss": 1.2761, + "theoretical_loss": 3.344651924721886, + "tokens_seen": 2735341568 + }, + { + "epoch": 0.83, + "learning_rate": 8.649949849548647e-05, + "loss": 1.426, + "theoretical_loss": 3.344639370073953, + "tokens_seen": 2735472640 + }, + { + "epoch": 0.83, + "learning_rate": 8.647943831494484e-05, + "loss": 1.4167, + "theoretical_loss": 3.344626816195999, + "tokens_seen": 2735603712 + }, + { + "epoch": 0.83, + "learning_rate": 8.645937813440322e-05, + "loss": 1.1993, + "theoretical_loss": 3.3446142630879394, + "tokens_seen": 2735734784 + }, + { + "epoch": 0.83, + "learning_rate": 8.64393179538616e-05, + "loss": 1.2051, + "theoretical_loss": 3.3446017107496906, + "tokens_seen": 2735865856 + }, + { + "epoch": 0.83, + "learning_rate": 8.641925777331997e-05, + "loss": 1.3244, + "theoretical_loss": 3.3445891591811683, + "tokens_seen": 2735996928 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.48619258403778076, + "objective/train/avg_token_score": 0.00662623718380928, + "objective/train/docs_used": 1540841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4345982074737549, + "objective/train/original_loss": 3.6939713954925537, + "objective/train/theoretical_loss": 3.344576608382288, + "objective/train/tokens_used": 2736128000, + "objective/train/unlikelihood_loss": 0.17649896442890167, + "theoretical_loss": 3.344576608382288, + "tokens_seen": 2736128000 + }, + { + "epoch": 0.83, + "learning_rate": 8.639919759277835e-05, + "loss": 1.2836, + "theoretical_loss": 3.344576608382288, + "tokens_seen": 2736128000 + }, + { + "epoch": 0.83, + "learning_rate": 8.637913741223671e-05, + "loss": 1.3237, + "theoretical_loss": 3.344564058352966, + "tokens_seen": 2736259072 + }, + { + "epoch": 0.83, + "learning_rate": 8.635907723169508e-05, + "loss": 1.2834, + "theoretical_loss": 3.344551509093119, + "tokens_seen": 2736390144 + }, + { + "epoch": 0.83, + "learning_rate": 8.633901705115346e-05, + "loss": 1.2998, + "theoretical_loss": 3.3445389606026614, + "tokens_seen": 2736521216 + }, + { + "epoch": 0.83, + "learning_rate": 8.631895687061183e-05, + "loss": 1.2897, + "theoretical_loss": 3.3445264128815104, + "tokens_seen": 2736652288 + }, + { + "epoch": 0.83, + "learning_rate": 8.629889669007021e-05, + "loss": 1.2448, + "theoretical_loss": 3.344513865929582, + "tokens_seen": 2736783360 + }, + { + "epoch": 0.83, + "learning_rate": 8.627883650952859e-05, + "loss": 1.156, + "theoretical_loss": 3.344501319746791, + "tokens_seen": 2736914432 + }, + { + "epoch": 0.83, + "learning_rate": 8.625877632898696e-05, + "loss": 1.2436, + "theoretical_loss": 3.3444887743330547, + "tokens_seen": 2737045504 + }, + { + "epoch": 0.83, + "learning_rate": 8.623871614844534e-05, + "loss": 1.2543, + "theoretical_loss": 3.344476229688289, + "tokens_seen": 2737176576 + }, + { + "epoch": 0.83, + "learning_rate": 8.621865596790371e-05, + "loss": 1.2386, + "theoretical_loss": 3.3444636858124093, + "tokens_seen": 2737307648 + }, + { + "epoch": 0.83, + "learning_rate": 8.619859578736209e-05, + "loss": 1.2663, + "theoretical_loss": 3.344451142705332, + "tokens_seen": 2737438720 + }, + { + "epoch": 0.83, + "learning_rate": 8.617853560682045e-05, + "loss": 1.317, + "theoretical_loss": 3.344438600366973, + "tokens_seen": 2737569792 + }, + { + "epoch": 0.83, + "learning_rate": 8.615847542627883e-05, + "loss": 1.1559, + "theoretical_loss": 3.3444260587972487, + "tokens_seen": 2737700864 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.3699902296066284, + "objective/train/avg_token_score": 0.023374298587441444, + "objective/train/docs_used": 1541498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.118401050567627, + "objective/train/original_loss": 4.135018348693848, + "objective/train/theoretical_loss": 3.3444197883005984, + "objective/train/tokens_used": 2737766400, + "objective/train/unlikelihood_loss": 0.15922151505947113, + "theoretical_loss": 3.3444197883005984, + "tokens_seen": 2737766400 + }, + { + "epoch": 0.83, + "learning_rate": 8.613841524573722e-05, + "loss": 1.2549, + "theoretical_loss": 3.344413517996075, + "tokens_seen": 2737831936 + }, + { + "epoch": 0.83, + "learning_rate": 8.611835506519559e-05, + "loss": 1.312, + "theoretical_loss": 3.3444009779633683, + "tokens_seen": 2737963008 + }, + { + "epoch": 0.83, + "learning_rate": 8.609829488465397e-05, + "loss": 1.2331, + "theoretical_loss": 3.344388438699044, + "tokens_seen": 2738094080 + }, + { + "epoch": 0.83, + "learning_rate": 8.607823470411234e-05, + "loss": 1.306, + "theoretical_loss": 3.344375900203019, + "tokens_seen": 2738225152 + }, + { + "epoch": 0.83, + "learning_rate": 8.605817452357072e-05, + "loss": 1.2453, + "theoretical_loss": 3.3443633624752094, + "tokens_seen": 2738356224 + }, + { + "epoch": 0.83, + "learning_rate": 8.60381143430291e-05, + "loss": 1.2249, + "theoretical_loss": 3.344350825515531, + "tokens_seen": 2738487296 + }, + { + "epoch": 0.83, + "learning_rate": 8.601805416248747e-05, + "loss": 1.2263, + "theoretical_loss": 3.3443382893238995, + "tokens_seen": 2738618368 + }, + { + "epoch": 0.83, + "learning_rate": 8.599799398194583e-05, + "loss": 1.2633, + "theoretical_loss": 3.3443257539002325, + "tokens_seen": 2738749440 + }, + { + "epoch": 0.83, + "learning_rate": 8.597793380140421e-05, + "loss": 1.2919, + "theoretical_loss": 3.3443132192444454, + "tokens_seen": 2738880512 + }, + { + "epoch": 0.83, + "learning_rate": 8.595787362086259e-05, + "loss": 1.2055, + "theoretical_loss": 3.344300685356454, + "tokens_seen": 2739011584 + }, + { + "epoch": 0.83, + "learning_rate": 8.593781344032096e-05, + "loss": 1.35, + "theoretical_loss": 3.3442881522361754, + "tokens_seen": 2739142656 + }, + { + "epoch": 0.83, + "learning_rate": 8.591775325977934e-05, + "loss": 1.3847, + "theoretical_loss": 3.344275619883525, + "tokens_seen": 2739273728 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.2631353735923767, + "objective/train/avg_token_score": 0.03710479289293289, + "objective/train/docs_used": 1542857, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7379708886146545, + "objective/train/original_loss": 4.422711372375488, + "objective/train/theoretical_loss": 3.34426308829842, + "objective/train/tokens_used": 2739404800, + "objective/train/unlikelihood_loss": 0.23243777453899384, + "theoretical_loss": 3.34426308829842, + "tokens_seen": 2739404800 + }, + { + "epoch": 0.83, + "learning_rate": 8.589769307923771e-05, + "loss": 1.1924, + "theoretical_loss": 3.34426308829842, + "tokens_seen": 2739404800 + }, + { + "epoch": 0.83, + "learning_rate": 8.587763289869609e-05, + "loss": 1.4378, + "theoretical_loss": 3.3442505574807755, + "tokens_seen": 2739535872 + }, + { + "epoch": 0.83, + "learning_rate": 8.585757271815446e-05, + "loss": 1.2172, + "theoretical_loss": 3.3442380274305084, + "tokens_seen": 2739666944 + }, + { + "epoch": 0.83, + "learning_rate": 8.583751253761284e-05, + "loss": 1.249, + "theoretical_loss": 3.3442254981475354, + "tokens_seen": 2739798016 + }, + { + "epoch": 0.83, + "learning_rate": 8.581745235707122e-05, + "loss": 1.22, + "theoretical_loss": 3.344212969631773, + "tokens_seen": 2739929088 + }, + { + "epoch": 0.83, + "learning_rate": 8.579739217652958e-05, + "loss": 1.3091, + "theoretical_loss": 3.3442004418831357, + "tokens_seen": 2740060160 + }, + { + "epoch": 0.83, + "learning_rate": 8.577733199598795e-05, + "loss": 1.3025, + "theoretical_loss": 3.344187914901542, + "tokens_seen": 2740191232 + }, + { + "epoch": 0.83, + "learning_rate": 8.575727181544634e-05, + "loss": 1.1892, + "theoretical_loss": 3.3441753886869066, + "tokens_seen": 2740322304 + }, + { + "epoch": 0.83, + "learning_rate": 8.573721163490472e-05, + "loss": 1.293, + "theoretical_loss": 3.3441628632391476, + "tokens_seen": 2740453376 + }, + { + "epoch": 0.83, + "learning_rate": 8.57171514543631e-05, + "loss": 1.2311, + "theoretical_loss": 3.3441503385581797, + "tokens_seen": 2740584448 + }, + { + "epoch": 0.83, + "learning_rate": 8.569709127382147e-05, + "loss": 1.1935, + "theoretical_loss": 3.34413781464392, + "tokens_seen": 2740715520 + }, + { + "epoch": 0.83, + "learning_rate": 8.567703109327985e-05, + "loss": 1.2054, + "theoretical_loss": 3.344125291496285, + "tokens_seen": 2740846592 + }, + { + "epoch": 0.83, + "learning_rate": 8.565697091273822e-05, + "loss": 1.1167, + "theoretical_loss": 3.3441127691151906, + "tokens_seen": 2740977664 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.31457722187042236, + "objective/train/avg_token_score": 0.02668732963502407, + "objective/train/docs_used": 1543628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9384981989860535, + "objective/train/original_loss": 3.969606637954712, + "objective/train/theoretical_loss": 3.3441065082120707, + "objective/train/tokens_used": 2741043200, + "objective/train/unlikelihood_loss": 0.16692137718200684, + "theoretical_loss": 3.3441065082120707, + "tokens_seen": 2741043200 + }, + { + "epoch": 0.83, + "learning_rate": 8.56369107321966e-05, + "loss": 1.168, + "theoretical_loss": 3.344100247500554, + "tokens_seen": 2741108736 + }, + { + "epoch": 0.83, + "learning_rate": 8.561685055165497e-05, + "loss": 1.2526, + "theoretical_loss": 3.3440877266522913, + "tokens_seen": 2741239808 + }, + { + "epoch": 0.83, + "learning_rate": 8.559679037111334e-05, + "loss": 1.311, + "theoretical_loss": 3.3440752065703183, + "tokens_seen": 2741370880 + }, + { + "epoch": 0.83, + "learning_rate": 8.557673019057171e-05, + "loss": 1.2974, + "theoretical_loss": 3.344062687254553, + "tokens_seen": 2741501952 + }, + { + "epoch": 0.83, + "learning_rate": 8.555667001003009e-05, + "loss": 1.2954, + "theoretical_loss": 3.3440501687049102, + "tokens_seen": 2741633024 + }, + { + "epoch": 0.83, + "learning_rate": 8.553660982948846e-05, + "loss": 1.2881, + "theoretical_loss": 3.3440376509213072, + "tokens_seen": 2741764096 + }, + { + "epoch": 0.83, + "learning_rate": 8.551654964894684e-05, + "loss": 1.3538, + "theoretical_loss": 3.344025133903661, + "tokens_seen": 2741895168 + }, + { + "epoch": 0.83, + "learning_rate": 8.549648946840522e-05, + "loss": 1.3542, + "theoretical_loss": 3.344012617651887, + "tokens_seen": 2742026240 + }, + { + "epoch": 0.83, + "learning_rate": 8.547642928786359e-05, + "loss": 1.1924, + "theoretical_loss": 3.3440001021659027, + "tokens_seen": 2742157312 + }, + { + "epoch": 0.83, + "learning_rate": 8.545636910732197e-05, + "loss": 1.2176, + "theoretical_loss": 3.343987587445624, + "tokens_seen": 2742288384 + }, + { + "epoch": 0.83, + "learning_rate": 8.543630892678034e-05, + "loss": 1.2715, + "theoretical_loss": 3.343975073490968, + "tokens_seen": 2742419456 + }, + { + "epoch": 0.83, + "learning_rate": 8.54162487462387e-05, + "loss": 1.4077, + "theoretical_loss": 3.3439625603018506, + "tokens_seen": 2742550528 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.4199657738208771, + "objective/train/avg_token_score": 0.01230496447533369, + "objective/train/docs_used": 1544002, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2996952533721924, + "objective/train/original_loss": 3.784885883331299, + "objective/train/theoretical_loss": 3.343950047878189, + "objective/train/tokens_used": 2742681600, + "objective/train/unlikelihood_loss": 0.19966673851013184, + "theoretical_loss": 3.343950047878189, + "tokens_seen": 2742681600 + }, + { + "epoch": 0.83, + "learning_rate": 8.53961885656971e-05, + "loss": 1.27, + "theoretical_loss": 3.343950047878189, + "tokens_seen": 2742681600 + }, + { + "epoch": 0.83, + "learning_rate": 8.537612838515547e-05, + "loss": 1.1867, + "theoretical_loss": 3.3439375362198995, + "tokens_seen": 2742812672 + }, + { + "epoch": 0.83, + "learning_rate": 8.535606820461385e-05, + "loss": 1.1819, + "theoretical_loss": 3.343925025326899, + "tokens_seen": 2742943744 + }, + { + "epoch": 0.83, + "learning_rate": 8.533600802407222e-05, + "loss": 1.2612, + "theoretical_loss": 3.3439125151991034, + "tokens_seen": 2743074816 + }, + { + "epoch": 0.83, + "learning_rate": 8.53159478435306e-05, + "loss": 1.2465, + "theoretical_loss": 3.34390000583643, + "tokens_seen": 2743205888 + }, + { + "epoch": 0.83, + "learning_rate": 8.529588766298897e-05, + "loss": 1.187, + "theoretical_loss": 3.3438874972387955, + "tokens_seen": 2743336960 + }, + { + "epoch": 0.83, + "learning_rate": 8.527582748244735e-05, + "loss": 1.1971, + "theoretical_loss": 3.343874989406116, + "tokens_seen": 2743468032 + }, + { + "epoch": 0.83, + "learning_rate": 8.525576730190572e-05, + "loss": 1.1658, + "theoretical_loss": 3.343862482338309, + "tokens_seen": 2743599104 + }, + { + "epoch": 0.83, + "learning_rate": 8.52357071213641e-05, + "loss": 1.2199, + "theoretical_loss": 3.3438499760352904, + "tokens_seen": 2743730176 + }, + { + "epoch": 0.83, + "learning_rate": 8.521564694082246e-05, + "loss": 1.4284, + "theoretical_loss": 3.343837470496977, + "tokens_seen": 2743861248 + }, + { + "epoch": 0.83, + "learning_rate": 8.519558676028084e-05, + "loss": 1.363, + "theoretical_loss": 3.343824965723286, + "tokens_seen": 2743992320 + }, + { + "epoch": 0.83, + "learning_rate": 8.517552657973921e-05, + "loss": 1.1571, + "theoretical_loss": 3.343812461714134, + "tokens_seen": 2744123392 + }, + { + "epoch": 0.83, + "learning_rate": 8.515546639919759e-05, + "loss": 1.1158, + "theoretical_loss": 3.3437999584694373, + "tokens_seen": 2744254464 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.39913246035575867, + "objective/train/avg_token_score": 0.0340750515460968, + "objective/train/docs_used": 1545179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0665594339370728, + "objective/train/original_loss": 4.13002872467041, + "objective/train/theoretical_loss": 3.343793707133734, + "objective/train/tokens_used": 2744320000, + "objective/train/unlikelihood_loss": 0.12560541927814484, + "theoretical_loss": 3.343793707133734, + "tokens_seen": 2744320000 + }, + { + "epoch": 0.83, + "learning_rate": 8.513540621865597e-05, + "loss": 1.3453, + "theoretical_loss": 3.343787455989113, + "tokens_seen": 2744385536 + }, + { + "epoch": 0.83, + "learning_rate": 8.511534603811434e-05, + "loss": 1.4422, + "theoretical_loss": 3.343774954273078, + "tokens_seen": 2744516608 + }, + { + "epoch": 0.83, + "learning_rate": 8.509528585757272e-05, + "loss": 1.1918, + "theoretical_loss": 3.343762453321249, + "tokens_seen": 2744647680 + }, + { + "epoch": 0.83, + "learning_rate": 8.50752256770311e-05, + "loss": 1.2534, + "theoretical_loss": 3.3437499531335426, + "tokens_seen": 2744778752 + }, + { + "epoch": 0.83, + "learning_rate": 8.505516549648947e-05, + "loss": 1.13, + "theoretical_loss": 3.343737453709876, + "tokens_seen": 2744909824 + }, + { + "epoch": 0.83, + "learning_rate": 8.503510531594785e-05, + "loss": 1.4074, + "theoretical_loss": 3.343724955050165, + "tokens_seen": 2745040896 + }, + { + "epoch": 0.83, + "learning_rate": 8.501504513540622e-05, + "loss": 1.1895, + "theoretical_loss": 3.3437124571543273, + "tokens_seen": 2745171968 + }, + { + "epoch": 0.83, + "learning_rate": 8.49949849548646e-05, + "loss": 1.2418, + "theoretical_loss": 3.34369996002228, + "tokens_seen": 2745303040 + }, + { + "epoch": 0.83, + "learning_rate": 8.497492477432297e-05, + "loss": 1.2595, + "theoretical_loss": 3.3436874636539393, + "tokens_seen": 2745434112 + }, + { + "epoch": 0.83, + "learning_rate": 8.495486459378135e-05, + "loss": 1.1886, + "theoretical_loss": 3.3436749680492226, + "tokens_seen": 2745565184 + }, + { + "epoch": 0.83, + "learning_rate": 8.493480441323972e-05, + "loss": 1.201, + "theoretical_loss": 3.343662473208046, + "tokens_seen": 2745696256 + }, + { + "epoch": 0.83, + "learning_rate": 8.49147442326981e-05, + "loss": 1.2985, + "theoretical_loss": 3.3436499791303276, + "tokens_seen": 2745827328 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.3685239553451538, + "objective/train/avg_token_score": 0.027301322668790817, + "objective/train/docs_used": 1545727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1802539825439453, + "objective/train/original_loss": 4.224027633666992, + "objective/train/theoretical_loss": 3.343637485815983, + "objective/train/tokens_used": 2745958400, + "objective/train/unlikelihood_loss": 0.11381108313798904, + "theoretical_loss": 3.343637485815983, + "tokens_seen": 2745958400 + }, + { + "epoch": 0.83, + "learning_rate": 8.489468405215648e-05, + "loss": 1.2465, + "theoretical_loss": 3.343637485815983, + "tokens_seen": 2745958400 + }, + { + "epoch": 0.83, + "learning_rate": 8.487462387161485e-05, + "loss": 1.2824, + "theoretical_loss": 3.34362499326493, + "tokens_seen": 2746089472 + }, + { + "epoch": 0.83, + "learning_rate": 8.485456369107323e-05, + "loss": 1.2336, + "theoretical_loss": 3.343612501477086, + "tokens_seen": 2746220544 + }, + { + "epoch": 0.83, + "learning_rate": 8.48345035105316e-05, + "loss": 1.2109, + "theoretical_loss": 3.3436000104523664, + "tokens_seen": 2746351616 + }, + { + "epoch": 0.83, + "learning_rate": 8.481444332998997e-05, + "loss": 1.2271, + "theoretical_loss": 3.343587520190689, + "tokens_seen": 2746482688 + }, + { + "epoch": 0.83, + "learning_rate": 8.479438314944834e-05, + "loss": 1.2678, + "theoretical_loss": 3.343575030691971, + "tokens_seen": 2746613760 + }, + { + "epoch": 0.83, + "learning_rate": 8.477432296890672e-05, + "loss": 1.1466, + "theoretical_loss": 3.3435625419561292, + "tokens_seen": 2746744832 + }, + { + "epoch": 0.83, + "learning_rate": 8.475426278836509e-05, + "loss": 1.3285, + "theoretical_loss": 3.3435500539830807, + "tokens_seen": 2746875904 + }, + { + "epoch": 0.83, + "learning_rate": 8.473420260782347e-05, + "loss": 1.2593, + "theoretical_loss": 3.343537566772742, + "tokens_seen": 2747006976 + }, + { + "epoch": 0.83, + "learning_rate": 8.471414242728184e-05, + "loss": 1.1694, + "theoretical_loss": 3.3435250803250307, + "tokens_seen": 2747138048 + }, + { + "epoch": 0.83, + "learning_rate": 8.469408224674022e-05, + "loss": 1.1145, + "theoretical_loss": 3.3435125946398636, + "tokens_seen": 2747269120 + }, + { + "epoch": 0.83, + "learning_rate": 8.46740220661986e-05, + "loss": 1.2546, + "theoretical_loss": 3.3435001097171577, + "tokens_seen": 2747400192 + }, + { + "epoch": 0.83, + "learning_rate": 8.465396188565697e-05, + "loss": 1.2857, + "theoretical_loss": 3.3434876255568304, + "tokens_seen": 2747531264 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.27761486172676086, + "objective/train/avg_token_score": 0.025462528690695763, + "objective/train/docs_used": 1547118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8825854063034058, + "objective/train/original_loss": 4.37655782699585, + "objective/train/theoretical_loss": 3.3434813837625326, + "objective/train/tokens_used": 2747596800, + "objective/train/unlikelihood_loss": 0.1215750202536583, + "theoretical_loss": 3.3434813837625326, + "tokens_seen": 2747596800 + }, + { + "epoch": 0.83, + "learning_rate": 8.463390170511536e-05, + "loss": 1.3648, + "theoretical_loss": 3.3434751421587987, + "tokens_seen": 2747662336 + }, + { + "epoch": 0.83, + "learning_rate": 8.461384152457372e-05, + "loss": 1.2696, + "theoretical_loss": 3.343462659522979, + "tokens_seen": 2747793408 + }, + { + "epoch": 0.83, + "learning_rate": 8.45937813440321e-05, + "loss": 1.3207, + "theoretical_loss": 3.3434501776492898, + "tokens_seen": 2747924480 + }, + { + "epoch": 0.83, + "learning_rate": 8.457372116349047e-05, + "loss": 1.1792, + "theoretical_loss": 3.343437696537647, + "tokens_seen": 2748055552 + }, + { + "epoch": 0.83, + "learning_rate": 8.455366098294885e-05, + "loss": 1.2596, + "theoretical_loss": 3.343425216187968, + "tokens_seen": 2748186624 + }, + { + "epoch": 0.83, + "learning_rate": 8.453360080240723e-05, + "loss": 1.1481, + "theoretical_loss": 3.3434127366001696, + "tokens_seen": 2748317696 + }, + { + "epoch": 0.83, + "learning_rate": 8.45135406218656e-05, + "loss": 1.1967, + "theoretical_loss": 3.34340025777417, + "tokens_seen": 2748448768 + }, + { + "epoch": 0.83, + "learning_rate": 8.449348044132398e-05, + "loss": 1.142, + "theoretical_loss": 3.343387779709886, + "tokens_seen": 2748579840 + }, + { + "epoch": 0.83, + "learning_rate": 8.447342026078235e-05, + "loss": 1.1484, + "theoretical_loss": 3.3433753024072335, + "tokens_seen": 2748710912 + }, + { + "epoch": 0.83, + "learning_rate": 8.445336008024073e-05, + "loss": 1.2334, + "theoretical_loss": 3.343362825866132, + "tokens_seen": 2748841984 + }, + { + "epoch": 0.83, + "learning_rate": 8.443329989969909e-05, + "loss": 1.3209, + "theoretical_loss": 3.3433503500864967, + "tokens_seen": 2748973056 + }, + { + "epoch": 0.83, + "learning_rate": 8.441323971915747e-05, + "loss": 1.3014, + "theoretical_loss": 3.343337875068246, + "tokens_seen": 2749104128 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.3883797526359558, + "objective/train/avg_token_score": 0.007824760861694813, + "objective/train/docs_used": 1547686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.138835072517395, + "objective/train/original_loss": 3.7230825424194336, + "objective/train/theoretical_loss": 3.3433254008112963, + "objective/train/tokens_used": 2749235200, + "objective/train/unlikelihood_loss": 0.17129123210906982, + "theoretical_loss": 3.3433254008112963, + "tokens_seen": 2749235200 + }, + { + "epoch": 0.83, + "learning_rate": 8.439317953861584e-05, + "loss": 1.1974, + "theoretical_loss": 3.3433254008112963, + "tokens_seen": 2749235200 + }, + { + "epoch": 0.83, + "learning_rate": 8.437311935807422e-05, + "loss": 1.3869, + "theoretical_loss": 3.3433129273155657, + "tokens_seen": 2749366272 + }, + { + "epoch": 0.83, + "learning_rate": 8.43530591775326e-05, + "loss": 1.2084, + "theoretical_loss": 3.343300454580971, + "tokens_seen": 2749497344 + }, + { + "epoch": 0.83, + "learning_rate": 8.433299899699097e-05, + "loss": 1.1882, + "theoretical_loss": 3.343287982607429, + "tokens_seen": 2749628416 + }, + { + "epoch": 0.83, + "learning_rate": 8.431293881644935e-05, + "loss": 1.0658, + "theoretical_loss": 3.343275511394858, + "tokens_seen": 2749759488 + }, + { + "epoch": 0.83, + "learning_rate": 8.429287863590772e-05, + "loss": 1.4299, + "theoretical_loss": 3.3432630409431745, + "tokens_seen": 2749890560 + }, + { + "epoch": 0.83, + "learning_rate": 8.42728184553661e-05, + "loss": 1.298, + "theoretical_loss": 3.3432505712522964, + "tokens_seen": 2750021632 + }, + { + "epoch": 0.83, + "learning_rate": 8.425275827482449e-05, + "loss": 1.4151, + "theoretical_loss": 3.3432381023221405, + "tokens_seen": 2750152704 + }, + { + "epoch": 0.83, + "learning_rate": 8.423269809428285e-05, + "loss": 1.1978, + "theoretical_loss": 3.3432256341526245, + "tokens_seen": 2750283776 + }, + { + "epoch": 0.83, + "learning_rate": 8.421263791374123e-05, + "loss": 1.2686, + "theoretical_loss": 3.3432131667436655, + "tokens_seen": 2750414848 + }, + { + "epoch": 0.83, + "learning_rate": 8.41925777331996e-05, + "loss": 1.189, + "theoretical_loss": 3.3432007000951813, + "tokens_seen": 2750545920 + }, + { + "epoch": 0.83, + "learning_rate": 8.417251755265798e-05, + "loss": 1.3373, + "theoretical_loss": 3.343188234207089, + "tokens_seen": 2750676992 + }, + { + "epoch": 0.83, + "learning_rate": 8.415245737211635e-05, + "loss": 1.1851, + "theoretical_loss": 3.3431757690793056, + "tokens_seen": 2750808064 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.32893452048301697, + "objective/train/avg_token_score": 0.01493498869240284, + "objective/train/docs_used": 1548689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9260087013244629, + "objective/train/original_loss": 4.0082902908325195, + "objective/train/theoretical_loss": 3.343169536800504, + "objective/train/tokens_used": 2750873600, + "objective/train/unlikelihood_loss": 0.15131215751171112, + "theoretical_loss": 3.343169536800504, + "tokens_seen": 2750873600 + }, + { + "epoch": 0.83, + "learning_rate": 8.413239719157473e-05, + "loss": 1.1232, + "theoretical_loss": 3.343163304711749, + "tokens_seen": 2750939136 + }, + { + "epoch": 0.83, + "learning_rate": 8.41123370110331e-05, + "loss": 1.4136, + "theoretical_loss": 3.3431508411043365, + "tokens_seen": 2751070208 + }, + { + "epoch": 0.83, + "learning_rate": 8.409227683049148e-05, + "loss": 1.324, + "theoretical_loss": 3.3431383782569855, + "tokens_seen": 2751201280 + }, + { + "epoch": 0.83, + "learning_rate": 8.407221664994986e-05, + "loss": 1.3384, + "theoretical_loss": 3.343125916169613, + "tokens_seen": 2751332352 + }, + { + "epoch": 0.83, + "learning_rate": 8.405215646940823e-05, + "loss": 1.2633, + "theoretical_loss": 3.3431134548421375, + "tokens_seen": 2751463424 + }, + { + "epoch": 0.83, + "learning_rate": 8.40320962888666e-05, + "loss": 1.3711, + "theoretical_loss": 3.343100994274476, + "tokens_seen": 2751594496 + }, + { + "epoch": 0.83, + "learning_rate": 8.401203610832497e-05, + "loss": 1.3065, + "theoretical_loss": 3.3430885344665455, + "tokens_seen": 2751725568 + }, + { + "epoch": 0.83, + "learning_rate": 8.399197592778335e-05, + "loss": 1.1979, + "theoretical_loss": 3.343076075418264, + "tokens_seen": 2751856640 + }, + { + "epoch": 0.83, + "learning_rate": 8.397191574724172e-05, + "loss": 1.1483, + "theoretical_loss": 3.3430636171295487, + "tokens_seen": 2751987712 + }, + { + "epoch": 0.83, + "learning_rate": 8.39518555667001e-05, + "loss": 1.2606, + "theoretical_loss": 3.3430511596003174, + "tokens_seen": 2752118784 + }, + { + "epoch": 0.83, + "learning_rate": 8.393179538615847e-05, + "loss": 1.2947, + "theoretical_loss": 3.3430387028304875, + "tokens_seen": 2752249856 + }, + { + "epoch": 0.83, + "learning_rate": 8.391173520561685e-05, + "loss": 1.1336, + "theoretical_loss": 3.3430262468199765, + "tokens_seen": 2752380928 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.5227272510528564, + "objective/train/avg_token_score": 0.013304021209478378, + "objective/train/docs_used": 1549388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7342450618743896, + "objective/train/original_loss": 3.953026533126831, + "objective/train/theoretical_loss": 3.343013791568702, + "objective/train/tokens_used": 2752512000, + "objective/train/unlikelihood_loss": 0.11011378467082977, + "theoretical_loss": 3.343013791568702, + "tokens_seen": 2752512000 + }, + { + "epoch": 0.83, + "learning_rate": 8.389167502507524e-05, + "loss": 1.3203, + "theoretical_loss": 3.343013791568702, + "tokens_seen": 2752512000 + }, + { + "epoch": 0.83, + "learning_rate": 8.387161484453361e-05, + "loss": 1.2156, + "theoretical_loss": 3.343001337076582, + "tokens_seen": 2752643072 + }, + { + "epoch": 0.83, + "learning_rate": 8.385155466399198e-05, + "loss": 1.2911, + "theoretical_loss": 3.342988883343533, + "tokens_seen": 2752774144 + }, + { + "epoch": 0.83, + "learning_rate": 8.383149448345035e-05, + "loss": 1.2124, + "theoretical_loss": 3.3429764303694736, + "tokens_seen": 2752905216 + }, + { + "epoch": 0.83, + "learning_rate": 8.381143430290873e-05, + "loss": 1.2371, + "theoretical_loss": 3.3429639781543212, + "tokens_seen": 2753036288 + }, + { + "epoch": 0.83, + "learning_rate": 8.37913741223671e-05, + "loss": 1.2724, + "theoretical_loss": 3.342951526697993, + "tokens_seen": 2753167360 + }, + { + "epoch": 0.83, + "learning_rate": 8.377131394182548e-05, + "loss": 1.3328, + "theoretical_loss": 3.342939076000407, + "tokens_seen": 2753298432 + }, + { + "epoch": 0.83, + "learning_rate": 8.375125376128386e-05, + "loss": 1.2543, + "theoretical_loss": 3.342926626061481, + "tokens_seen": 2753429504 + }, + { + "epoch": 0.83, + "learning_rate": 8.373119358074223e-05, + "loss": 1.2994, + "theoretical_loss": 3.3429141768811323, + "tokens_seen": 2753560576 + }, + { + "epoch": 0.83, + "learning_rate": 8.37111334002006e-05, + "loss": 1.2832, + "theoretical_loss": 3.3429017284592786, + "tokens_seen": 2753691648 + }, + { + "epoch": 0.83, + "learning_rate": 8.369107321965898e-05, + "loss": 1.4411, + "theoretical_loss": 3.3428892807958377, + "tokens_seen": 2753822720 + }, + { + "epoch": 0.83, + "learning_rate": 8.367101303911736e-05, + "loss": 1.3023, + "theoretical_loss": 3.3428768338907275, + "tokens_seen": 2753953792 + }, + { + "epoch": 0.83, + "learning_rate": 8.365095285857572e-05, + "loss": 1.1437, + "theoretical_loss": 3.3428643877438655, + "tokens_seen": 2754084864 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.4078079164028168, + "objective/train/avg_token_score": 0.007602006196975708, + "objective/train/docs_used": 1550558, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2472883462905884, + "objective/train/original_loss": 3.5971837043762207, + "objective/train/theoretical_loss": 3.3428581649547517, + "objective/train/tokens_used": 2754150400, + "objective/train/unlikelihood_loss": 0.1822628676891327, + "theoretical_loss": 3.3428581649547517, + "tokens_seen": 2754150400 + }, + { + "epoch": 0.83, + "learning_rate": 8.36308926780341e-05, + "loss": 1.2829, + "theoretical_loss": 3.3428519423551695, + "tokens_seen": 2754215936 + }, + { + "epoch": 0.83, + "learning_rate": 8.361083249749247e-05, + "loss": 1.154, + "theoretical_loss": 3.3428394977245572, + "tokens_seen": 2754347008 + }, + { + "epoch": 0.83, + "learning_rate": 8.359077231695085e-05, + "loss": 1.1693, + "theoretical_loss": 3.342827053851946, + "tokens_seen": 2754478080 + }, + { + "epoch": 0.83, + "learning_rate": 8.357071213640922e-05, + "loss": 1.2604, + "theoretical_loss": 3.3428146107372543, + "tokens_seen": 2754609152 + }, + { + "epoch": 0.83, + "learning_rate": 8.35506519558676e-05, + "loss": 1.244, + "theoretical_loss": 3.3428021683803992, + "tokens_seen": 2754740224 + }, + { + "epoch": 0.83, + "learning_rate": 8.353059177532598e-05, + "loss": 1.1399, + "theoretical_loss": 3.3427897267812994, + "tokens_seen": 2754871296 + }, + { + "epoch": 0.83, + "learning_rate": 8.351053159478436e-05, + "loss": 1.2288, + "theoretical_loss": 3.3427772859398717, + "tokens_seen": 2755002368 + }, + { + "epoch": 0.83, + "learning_rate": 8.349047141424274e-05, + "loss": 1.34, + "theoretical_loss": 3.342764845856035, + "tokens_seen": 2755133440 + }, + { + "epoch": 0.83, + "learning_rate": 8.347041123370112e-05, + "loss": 1.2764, + "theoretical_loss": 3.342752406529706, + "tokens_seen": 2755264512 + }, + { + "epoch": 0.83, + "learning_rate": 8.345035105315948e-05, + "loss": 1.2086, + "theoretical_loss": 3.342739967960803, + "tokens_seen": 2755395584 + }, + { + "epoch": 0.84, + "learning_rate": 8.343029087261785e-05, + "loss": 1.3865, + "theoretical_loss": 3.3427275301492436, + "tokens_seen": 2755526656 + }, + { + "epoch": 0.84, + "learning_rate": 8.341023069207623e-05, + "loss": 1.4317, + "theoretical_loss": 3.342715093094947, + "tokens_seen": 2755657728 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.42344820499420166, + "objective/train/avg_token_score": 0.008604937233030796, + "objective/train/docs_used": 1551539, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3243757486343384, + "objective/train/original_loss": 3.750819444656372, + "objective/train/theoretical_loss": 3.342702656797829, + "objective/train/tokens_used": 2755788800, + "objective/train/unlikelihood_loss": 0.132038876414299, + "theoretical_loss": 3.342702656797829, + "tokens_seen": 2755788800 + }, + { + "epoch": 0.84, + "learning_rate": 8.33901705115346e-05, + "loss": 1.3185, + "theoretical_loss": 3.342702656797829, + "tokens_seen": 2755788800 + }, + { + "epoch": 0.84, + "learning_rate": 8.337011033099298e-05, + "loss": 1.1418, + "theoretical_loss": 3.342690221257809, + "tokens_seen": 2755919872 + }, + { + "epoch": 0.84, + "learning_rate": 8.335005015045136e-05, + "loss": 1.1816, + "theoretical_loss": 3.3426777864748045, + "tokens_seen": 2756050944 + }, + { + "epoch": 0.84, + "learning_rate": 8.332998996990973e-05, + "loss": 1.2307, + "theoretical_loss": 3.3426653524487335, + "tokens_seen": 2756182016 + }, + { + "epoch": 0.84, + "learning_rate": 8.330992978936811e-05, + "loss": 1.3394, + "theoretical_loss": 3.342652919179513, + "tokens_seen": 2756313088 + }, + { + "epoch": 0.84, + "learning_rate": 8.328986960882648e-05, + "loss": 1.3183, + "theoretical_loss": 3.342640486667063, + "tokens_seen": 2756444160 + }, + { + "epoch": 0.84, + "learning_rate": 8.326980942828486e-05, + "loss": 1.2916, + "theoretical_loss": 3.342628054911299, + "tokens_seen": 2756575232 + }, + { + "epoch": 0.84, + "learning_rate": 8.324974924774322e-05, + "loss": 1.2882, + "theoretical_loss": 3.342615623912141, + "tokens_seen": 2756706304 + }, + { + "epoch": 0.84, + "learning_rate": 8.32296890672016e-05, + "loss": 1.29, + "theoretical_loss": 3.3426031936695058, + "tokens_seen": 2756837376 + }, + { + "epoch": 0.84, + "learning_rate": 8.320962888665997e-05, + "loss": 1.2778, + "theoretical_loss": 3.3425907641833117, + "tokens_seen": 2756968448 + }, + { + "epoch": 0.84, + "learning_rate": 8.318956870611835e-05, + "loss": 1.3118, + "theoretical_loss": 3.342578335453477, + "tokens_seen": 2757099520 + }, + { + "epoch": 0.84, + "learning_rate": 8.316950852557673e-05, + "loss": 1.2993, + "theoretical_loss": 3.342565907479919, + "tokens_seen": 2757230592 + }, + { + "epoch": 0.84, + "learning_rate": 8.31494483450351e-05, + "loss": 1.3041, + "theoretical_loss": 3.3425534802625565, + "tokens_seen": 2757361664 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.4055474102497101, + "objective/train/avg_token_score": 0.03308877348899841, + "objective/train/docs_used": 1552106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2325690984725952, + "objective/train/original_loss": 3.585541248321533, + "objective/train/theoretical_loss": 3.342547266937423, + "objective/train/tokens_used": 2757427200, + "objective/train/unlikelihood_loss": 0.2333201915025711, + "theoretical_loss": 3.342547266937423, + "tokens_seen": 2757427200 + }, + { + "epoch": 0.84, + "learning_rate": 8.312938816449349e-05, + "loss": 1.1875, + "theoretical_loss": 3.342541053801307, + "tokens_seen": 2757492736 + }, + { + "epoch": 0.84, + "learning_rate": 8.310932798395187e-05, + "loss": 1.2053, + "theoretical_loss": 3.3425286280960895, + "tokens_seen": 2757623808 + }, + { + "epoch": 0.84, + "learning_rate": 8.308926780341024e-05, + "loss": 1.3244, + "theoretical_loss": 3.3425162031468205, + "tokens_seen": 2757754880 + }, + { + "epoch": 0.84, + "learning_rate": 8.30692076228686e-05, + "loss": 1.1291, + "theoretical_loss": 3.3425037789534198, + "tokens_seen": 2757885952 + }, + { + "epoch": 0.84, + "learning_rate": 8.304914744232698e-05, + "loss": 1.2554, + "theoretical_loss": 3.342491355515804, + "tokens_seen": 2758017024 + }, + { + "epoch": 0.84, + "learning_rate": 8.302908726178536e-05, + "loss": 1.3863, + "theoretical_loss": 3.3424789328338917, + "tokens_seen": 2758148096 + }, + { + "epoch": 0.84, + "learning_rate": 8.300902708124373e-05, + "loss": 1.2097, + "theoretical_loss": 3.342466510907602, + "tokens_seen": 2758279168 + }, + { + "epoch": 0.84, + "learning_rate": 8.298896690070211e-05, + "loss": 1.2232, + "theoretical_loss": 3.3424540897368518, + "tokens_seen": 2758410240 + }, + { + "epoch": 0.84, + "learning_rate": 8.296890672016048e-05, + "loss": 1.3142, + "theoretical_loss": 3.3424416693215595, + "tokens_seen": 2758541312 + }, + { + "epoch": 0.84, + "learning_rate": 8.294884653961886e-05, + "loss": 1.1789, + "theoretical_loss": 3.3424292496616435, + "tokens_seen": 2758672384 + }, + { + "epoch": 0.84, + "learning_rate": 8.292878635907724e-05, + "loss": 1.1872, + "theoretical_loss": 3.342416830757022, + "tokens_seen": 2758803456 + }, + { + "epoch": 0.84, + "learning_rate": 8.290872617853561e-05, + "loss": 1.2381, + "theoretical_loss": 3.342404412607613, + "tokens_seen": 2758934528 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.4722018539905548, + "objective/train/avg_token_score": 0.01291768066585064, + "objective/train/docs_used": 1553410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3550536632537842, + "objective/train/original_loss": 3.856735944747925, + "objective/train/theoretical_loss": 3.342391995213335, + "objective/train/tokens_used": 2759065600, + "objective/train/unlikelihood_loss": 0.122179314494133, + "theoretical_loss": 3.342391995213335, + "tokens_seen": 2759065600 + }, + { + "epoch": 0.84, + "learning_rate": 8.288866599799399e-05, + "loss": 1.2407, + "theoretical_loss": 3.342391995213335, + "tokens_seen": 2759065600 + }, + { + "epoch": 0.84, + "learning_rate": 8.286860581745235e-05, + "loss": 1.2246, + "theoretical_loss": 3.3423795785741057, + "tokens_seen": 2759196672 + }, + { + "epoch": 0.84, + "learning_rate": 8.284854563691073e-05, + "loss": 1.4334, + "theoretical_loss": 3.3423671626898437, + "tokens_seen": 2759327744 + }, + { + "epoch": 0.84, + "learning_rate": 8.28284854563691e-05, + "loss": 1.3601, + "theoretical_loss": 3.342354747560467, + "tokens_seen": 2759458816 + }, + { + "epoch": 0.84, + "learning_rate": 8.280842527582748e-05, + "loss": 1.15, + "theoretical_loss": 3.3423423331858944, + "tokens_seen": 2759589888 + }, + { + "epoch": 0.84, + "learning_rate": 8.278836509528585e-05, + "loss": 1.1904, + "theoretical_loss": 3.342329919566044, + "tokens_seen": 2759720960 + }, + { + "epoch": 0.84, + "learning_rate": 8.276830491474424e-05, + "loss": 1.239, + "theoretical_loss": 3.3423175067008333, + "tokens_seen": 2759852032 + }, + { + "epoch": 0.84, + "learning_rate": 8.274824473420262e-05, + "loss": 1.2791, + "theoretical_loss": 3.3423050945901815, + "tokens_seen": 2759983104 + }, + { + "epoch": 0.84, + "learning_rate": 8.2728184553661e-05, + "loss": 1.3717, + "theoretical_loss": 3.3422926832340063, + "tokens_seen": 2760114176 + }, + { + "epoch": 0.84, + "learning_rate": 8.270812437311937e-05, + "loss": 1.2644, + "theoretical_loss": 3.3422802726322263, + "tokens_seen": 2760245248 + }, + { + "epoch": 0.84, + "learning_rate": 8.268806419257775e-05, + "loss": 1.1631, + "theoretical_loss": 3.3422678627847597, + "tokens_seen": 2760376320 + }, + { + "epoch": 0.84, + "learning_rate": 8.266800401203611e-05, + "loss": 1.2132, + "theoretical_loss": 3.342255453691525, + "tokens_seen": 2760507392 + }, + { + "epoch": 0.84, + "learning_rate": 8.264794383149448e-05, + "loss": 1.3527, + "theoretical_loss": 3.3422430453524403, + "tokens_seen": 2760638464 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.2939271628856659, + "objective/train/avg_token_score": 0.01782006025314331, + "objective/train/docs_used": 1554093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9320819973945618, + "objective/train/original_loss": 4.28510046005249, + "objective/train/theoretical_loss": 3.342236841465679, + "objective/train/tokens_used": 2760704000, + "objective/train/unlikelihood_loss": 0.11627472192049026, + "theoretical_loss": 3.342236841465679, + "tokens_seen": 2760704000 + }, + { + "epoch": 0.84, + "learning_rate": 8.262788365095286e-05, + "loss": 1.3271, + "theoretical_loss": 3.3422306377674245, + "tokens_seen": 2760769536 + }, + { + "epoch": 0.84, + "learning_rate": 8.260782347041123e-05, + "loss": 1.0273, + "theoretical_loss": 3.3422182309363953, + "tokens_seen": 2760900608 + }, + { + "epoch": 0.84, + "learning_rate": 8.258776328986961e-05, + "loss": 1.2708, + "theoretical_loss": 3.3422058248592714, + "tokens_seen": 2761031680 + }, + { + "epoch": 0.84, + "learning_rate": 8.256770310932799e-05, + "loss": 1.2184, + "theoretical_loss": 3.3421934195359713, + "tokens_seen": 2761162752 + }, + { + "epoch": 0.84, + "learning_rate": 8.254764292878636e-05, + "loss": 1.208, + "theoretical_loss": 3.3421810149664135, + "tokens_seen": 2761293824 + }, + { + "epoch": 0.84, + "learning_rate": 8.252758274824474e-05, + "loss": 1.2993, + "theoretical_loss": 3.342168611150516, + "tokens_seen": 2761424896 + }, + { + "epoch": 0.84, + "learning_rate": 8.250752256770311e-05, + "loss": 1.2936, + "theoretical_loss": 3.3421562080881975, + "tokens_seen": 2761555968 + }, + { + "epoch": 0.84, + "learning_rate": 8.248746238716148e-05, + "loss": 1.243, + "theoretical_loss": 3.3421438057793766, + "tokens_seen": 2761687040 + }, + { + "epoch": 0.84, + "learning_rate": 8.246740220661985e-05, + "loss": 1.4129, + "theoretical_loss": 3.3421314042239714, + "tokens_seen": 2761818112 + }, + { + "epoch": 0.84, + "learning_rate": 8.244734202607823e-05, + "loss": 1.1625, + "theoretical_loss": 3.3421190034219004, + "tokens_seen": 2761949184 + }, + { + "epoch": 0.84, + "learning_rate": 8.24272818455366e-05, + "loss": 1.3258, + "theoretical_loss": 3.342106603373083, + "tokens_seen": 2762080256 + }, + { + "epoch": 0.84, + "learning_rate": 8.240722166499498e-05, + "loss": 1.4115, + "theoretical_loss": 3.342094204077436, + "tokens_seen": 2762211328 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.5678763389587402, + "objective/train/avg_token_score": 0.0028113778680562973, + "objective/train/docs_used": 1555178, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6307514905929565, + "objective/train/original_loss": 2.73469614982605, + "objective/train/theoretical_loss": 3.34208180553488, + "objective/train/tokens_used": 2762342400, + "objective/train/unlikelihood_loss": 0.26427000761032104, + "theoretical_loss": 3.34208180553488, + "tokens_seen": 2762342400 + }, + { + "epoch": 0.84, + "learning_rate": 8.238716148445337e-05, + "loss": 1.2978, + "theoretical_loss": 3.34208180553488, + "tokens_seen": 2762342400 + }, + { + "epoch": 0.84, + "learning_rate": 8.236710130391174e-05, + "loss": 1.298, + "theoretical_loss": 3.3420694077453317, + "tokens_seen": 2762473472 + }, + { + "epoch": 0.84, + "learning_rate": 8.234704112337012e-05, + "loss": 1.1208, + "theoretical_loss": 3.3420570107087104, + "tokens_seen": 2762604544 + }, + { + "epoch": 0.84, + "learning_rate": 8.23269809428285e-05, + "loss": 1.3116, + "theoretical_loss": 3.3420446144249345, + "tokens_seen": 2762735616 + }, + { + "epoch": 0.84, + "learning_rate": 8.230692076228687e-05, + "loss": 1.3383, + "theoretical_loss": 3.342032218893923, + "tokens_seen": 2762866688 + }, + { + "epoch": 0.84, + "learning_rate": 8.228686058174523e-05, + "loss": 1.2978, + "theoretical_loss": 3.342019824115594, + "tokens_seen": 2762997760 + }, + { + "epoch": 0.84, + "learning_rate": 8.226680040120361e-05, + "loss": 1.1978, + "theoretical_loss": 3.3420074300898666, + "tokens_seen": 2763128832 + }, + { + "epoch": 0.84, + "learning_rate": 8.224674022066199e-05, + "loss": 1.2334, + "theoretical_loss": 3.341995036816659, + "tokens_seen": 2763259904 + }, + { + "epoch": 0.84, + "learning_rate": 8.222668004012036e-05, + "loss": 1.0612, + "theoretical_loss": 3.3419826442958893, + "tokens_seen": 2763390976 + }, + { + "epoch": 0.84, + "learning_rate": 8.220661985957874e-05, + "loss": 1.2323, + "theoretical_loss": 3.3419702525274775, + "tokens_seen": 2763522048 + }, + { + "epoch": 0.84, + "learning_rate": 8.218655967903711e-05, + "loss": 1.2807, + "theoretical_loss": 3.341957861511341, + "tokens_seen": 2763653120 + }, + { + "epoch": 0.84, + "learning_rate": 8.216649949849549e-05, + "loss": 1.2137, + "theoretical_loss": 3.341945471247399, + "tokens_seen": 2763784192 + }, + { + "epoch": 0.84, + "learning_rate": 8.214643931795386e-05, + "loss": 1.4525, + "theoretical_loss": 3.34193308173557, + "tokens_seen": 2763915264 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.3822091817855835, + "objective/train/avg_token_score": 0.022706829011440277, + "objective/train/docs_used": 1555836, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.129343032836914, + "objective/train/original_loss": 3.731940984725952, + "objective/train/theoretical_loss": 3.3419268872616725, + "objective/train/tokens_used": 2763980800, + "objective/train/unlikelihood_loss": 0.1798480898141861, + "theoretical_loss": 3.3419268872616725, + "tokens_seen": 2763980800 + }, + { + "epoch": 0.84, + "learning_rate": 8.212637913741224e-05, + "loss": 1.3303, + "theoretical_loss": 3.3419206929757728, + "tokens_seen": 2764046336 + }, + { + "epoch": 0.84, + "learning_rate": 8.210631895687062e-05, + "loss": 1.2667, + "theoretical_loss": 3.341908304967926, + "tokens_seen": 2764177408 + }, + { + "epoch": 0.84, + "learning_rate": 8.208625877632898e-05, + "loss": 1.3507, + "theoretical_loss": 3.3418959177119487, + "tokens_seen": 2764308480 + }, + { + "epoch": 0.84, + "learning_rate": 8.206619859578735e-05, + "loss": 1.3381, + "theoretical_loss": 3.341883531207759, + "tokens_seen": 2764439552 + }, + { + "epoch": 0.84, + "learning_rate": 8.204613841524573e-05, + "loss": 1.3531, + "theoretical_loss": 3.341871145455276, + "tokens_seen": 2764570624 + }, + { + "epoch": 0.84, + "learning_rate": 8.20260782347041e-05, + "loss": 1.1985, + "theoretical_loss": 3.3418587604544183, + "tokens_seen": 2764701696 + }, + { + "epoch": 0.84, + "learning_rate": 8.20060180541625e-05, + "loss": 1.2312, + "theoretical_loss": 3.3418463762051047, + "tokens_seen": 2764832768 + }, + { + "epoch": 0.84, + "learning_rate": 8.198595787362087e-05, + "loss": 1.3149, + "theoretical_loss": 3.3418339927072545, + "tokens_seen": 2764963840 + }, + { + "epoch": 0.84, + "learning_rate": 8.196589769307925e-05, + "loss": 1.3425, + "theoretical_loss": 3.3418216099607854, + "tokens_seen": 2765094912 + }, + { + "epoch": 0.84, + "learning_rate": 8.194583751253762e-05, + "loss": 1.2032, + "theoretical_loss": 3.3418092279656166, + "tokens_seen": 2765225984 + }, + { + "epoch": 0.84, + "learning_rate": 8.1925777331996e-05, + "loss": 1.197, + "theoretical_loss": 3.3417968467216674, + "tokens_seen": 2765357056 + }, + { + "epoch": 0.84, + "learning_rate": 8.190571715145437e-05, + "loss": 1.1478, + "theoretical_loss": 3.341784466228856, + "tokens_seen": 2765488128 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.5179007649421692, + "objective/train/avg_token_score": 0.009363217279314995, + "objective/train/docs_used": 1556560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6109825372695923, + "objective/train/original_loss": 3.9694690704345703, + "objective/train/theoretical_loss": 3.341772086487102, + "objective/train/tokens_used": 2765619200, + "objective/train/unlikelihood_loss": 0.12845991551876068, + "theoretical_loss": 3.341772086487102, + "tokens_seen": 2765619200 + }, + { + "epoch": 0.84, + "learning_rate": 8.188565697091274e-05, + "loss": 1.2298, + "theoretical_loss": 3.341772086487102, + "tokens_seen": 2765619200 + }, + { + "epoch": 0.84, + "learning_rate": 8.186559679037111e-05, + "loss": 1.1551, + "theoretical_loss": 3.3417597074963235, + "tokens_seen": 2765750272 + }, + { + "epoch": 0.84, + "learning_rate": 8.184553660982949e-05, + "loss": 1.2615, + "theoretical_loss": 3.3417473292564397, + "tokens_seen": 2765881344 + }, + { + "epoch": 0.84, + "learning_rate": 8.182547642928786e-05, + "loss": 1.2966, + "theoretical_loss": 3.3417349517673696, + "tokens_seen": 2766012416 + }, + { + "epoch": 0.84, + "learning_rate": 8.180541624874624e-05, + "loss": 1.1483, + "theoretical_loss": 3.341722575029032, + "tokens_seen": 2766143488 + }, + { + "epoch": 0.84, + "learning_rate": 8.178535606820462e-05, + "loss": 1.2293, + "theoretical_loss": 3.341710199041345, + "tokens_seen": 2766274560 + }, + { + "epoch": 0.84, + "learning_rate": 8.176529588766299e-05, + "loss": 1.0717, + "theoretical_loss": 3.3416978238042283, + "tokens_seen": 2766405632 + }, + { + "epoch": 0.84, + "learning_rate": 8.174523570712137e-05, + "loss": 1.1794, + "theoretical_loss": 3.341685449317601, + "tokens_seen": 2766536704 + }, + { + "epoch": 0.84, + "learning_rate": 8.172517552657974e-05, + "loss": 1.3374, + "theoretical_loss": 3.341673075581382, + "tokens_seen": 2766667776 + }, + { + "epoch": 0.84, + "learning_rate": 8.17051153460381e-05, + "loss": 1.3251, + "theoretical_loss": 3.3416607025954894, + "tokens_seen": 2766798848 + }, + { + "epoch": 0.84, + "learning_rate": 8.168505516549648e-05, + "loss": 1.2307, + "theoretical_loss": 3.341648330359843, + "tokens_seen": 2766929920 + }, + { + "epoch": 0.84, + "learning_rate": 8.166499498495486e-05, + "loss": 1.3235, + "theoretical_loss": 3.341635958874362, + "tokens_seen": 2767060992 + }, + { + "epoch": 0.84, + "learning_rate": 8.164493480441323e-05, + "loss": 1.3152, + "theoretical_loss": 3.3416235881389644, + "tokens_seen": 2767192064 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.41312316060066223, + "objective/train/avg_token_score": 0.01982910931110382, + "objective/train/docs_used": 1557767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2895864248275757, + "objective/train/original_loss": 4.062778472900391, + "objective/train/theoretical_loss": 3.341617403052522, + "objective/train/tokens_used": 2767257600, + "objective/train/unlikelihood_loss": 0.13824814558029175, + "theoretical_loss": 3.341617403052522, + "tokens_seen": 2767257600 + }, + { + "epoch": 0.84, + "learning_rate": 8.162487462387162e-05, + "loss": 1.1215, + "theoretical_loss": 3.3416112181535698, + "tokens_seen": 2767323136 + }, + { + "epoch": 0.84, + "learning_rate": 8.160481444333e-05, + "loss": 1.3127, + "theoretical_loss": 3.341598848918097, + "tokens_seen": 2767454208 + }, + { + "epoch": 0.84, + "learning_rate": 8.158475426278837e-05, + "loss": 1.2443, + "theoretical_loss": 3.3415864804324653, + "tokens_seen": 2767585280 + }, + { + "epoch": 0.84, + "learning_rate": 8.156469408224675e-05, + "loss": 1.2149, + "theoretical_loss": 3.3415741126965934, + "tokens_seen": 2767716352 + }, + { + "epoch": 0.84, + "learning_rate": 8.154463390170512e-05, + "loss": 1.3318, + "theoretical_loss": 3.3415617457104005, + "tokens_seen": 2767847424 + }, + { + "epoch": 0.84, + "learning_rate": 8.15245737211635e-05, + "loss": 1.2473, + "theoretical_loss": 3.341549379473806, + "tokens_seen": 2767978496 + }, + { + "epoch": 0.84, + "learning_rate": 8.150451354062186e-05, + "loss": 1.3705, + "theoretical_loss": 3.3415370139867284, + "tokens_seen": 2768109568 + }, + { + "epoch": 0.84, + "learning_rate": 8.148445336008024e-05, + "loss": 1.2634, + "theoretical_loss": 3.341524649249087, + "tokens_seen": 2768240640 + }, + { + "epoch": 0.84, + "learning_rate": 8.146439317953861e-05, + "loss": 1.165, + "theoretical_loss": 3.3415122852608015, + "tokens_seen": 2768371712 + }, + { + "epoch": 0.84, + "learning_rate": 8.144433299899699e-05, + "loss": 1.2783, + "theoretical_loss": 3.34149992202179, + "tokens_seen": 2768502784 + }, + { + "epoch": 0.84, + "learning_rate": 8.142427281845537e-05, + "loss": 1.494, + "theoretical_loss": 3.3414875595319717, + "tokens_seen": 2768633856 + }, + { + "epoch": 0.84, + "learning_rate": 8.140421263791374e-05, + "loss": 1.1813, + "theoretical_loss": 3.3414751977912664, + "tokens_seen": 2768764928 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.3814760446548462, + "objective/train/avg_token_score": 0.026460736989974976, + "objective/train/docs_used": 1558514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2470753192901611, + "objective/train/original_loss": 3.9920318126678467, + "objective/train/theoretical_loss": 3.341462836799593, + "objective/train/tokens_used": 2768896000, + "objective/train/unlikelihood_loss": 0.1718367487192154, + "theoretical_loss": 3.341462836799593, + "tokens_seen": 2768896000 + }, + { + "epoch": 0.84, + "learning_rate": 8.138415245737212e-05, + "loss": 1.3278, + "theoretical_loss": 3.341462836799593, + "tokens_seen": 2768896000 + }, + { + "epoch": 0.84, + "learning_rate": 8.13640922768305e-05, + "loss": 1.3047, + "theoretical_loss": 3.3414504765568704, + "tokens_seen": 2769027072 + }, + { + "epoch": 0.84, + "learning_rate": 8.134403209628887e-05, + "loss": 1.3784, + "theoretical_loss": 3.3414381170630185, + "tokens_seen": 2769158144 + }, + { + "epoch": 0.84, + "learning_rate": 8.132397191574724e-05, + "loss": 1.3088, + "theoretical_loss": 3.341425758317955, + "tokens_seen": 2769289216 + }, + { + "epoch": 0.84, + "learning_rate": 8.130391173520561e-05, + "loss": 1.3249, + "theoretical_loss": 3.341413400321601, + "tokens_seen": 2769420288 + }, + { + "epoch": 0.84, + "learning_rate": 8.128385155466398e-05, + "loss": 1.1991, + "theoretical_loss": 3.3414010430738745, + "tokens_seen": 2769551360 + }, + { + "epoch": 0.84, + "learning_rate": 8.126379137412237e-05, + "loss": 1.2143, + "theoretical_loss": 3.341388686574695, + "tokens_seen": 2769682432 + }, + { + "epoch": 0.84, + "learning_rate": 8.124373119358075e-05, + "loss": 1.2999, + "theoretical_loss": 3.341376330823982, + "tokens_seen": 2769813504 + }, + { + "epoch": 0.84, + "learning_rate": 8.122367101303912e-05, + "loss": 1.231, + "theoretical_loss": 3.341363975821654, + "tokens_seen": 2769944576 + }, + { + "epoch": 0.84, + "learning_rate": 8.12036108324975e-05, + "loss": 1.2327, + "theoretical_loss": 3.3413516215676307, + "tokens_seen": 2770075648 + }, + { + "epoch": 0.84, + "learning_rate": 8.118355065195588e-05, + "loss": 1.2571, + "theoretical_loss": 3.341339268061832, + "tokens_seen": 2770206720 + }, + { + "epoch": 0.84, + "learning_rate": 8.116349047141425e-05, + "loss": 1.3741, + "theoretical_loss": 3.341326915304176, + "tokens_seen": 2770337792 + }, + { + "epoch": 0.84, + "learning_rate": 8.114343029087263e-05, + "loss": 1.3044, + "theoretical_loss": 3.3413145632945827, + "tokens_seen": 2770468864 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.4084188640117645, + "objective/train/avg_token_score": 0.02833310142159462, + "objective/train/docs_used": 1559839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2233176231384277, + "objective/train/original_loss": 3.880275249481201, + "objective/train/theoretical_loss": 3.3413083875702845, + "objective/train/tokens_used": 2770534400, + "objective/train/unlikelihood_loss": 0.1216341108083725, + "theoretical_loss": 3.3413083875702845, + "tokens_seen": 2770534400 + }, + { + "epoch": 0.84, + "learning_rate": 8.1123370110331e-05, + "loss": 1.1953, + "theoretical_loss": 3.3413022120329714, + "tokens_seen": 2770599936 + }, + { + "epoch": 0.84, + "learning_rate": 8.110330992978937e-05, + "loss": 1.3077, + "theoretical_loss": 3.3412898615192614, + "tokens_seen": 2770731008 + }, + { + "epoch": 0.84, + "learning_rate": 8.108324974924774e-05, + "loss": 1.4107, + "theoretical_loss": 3.341277511753372, + "tokens_seen": 2770862080 + }, + { + "epoch": 0.84, + "learning_rate": 8.106318956870612e-05, + "loss": 1.2999, + "theoretical_loss": 3.3412651627352226, + "tokens_seen": 2770993152 + }, + { + "epoch": 0.84, + "learning_rate": 8.104312938816449e-05, + "loss": 1.3173, + "theoretical_loss": 3.341252814464732, + "tokens_seen": 2771124224 + }, + { + "epoch": 0.84, + "learning_rate": 8.102306920762287e-05, + "loss": 1.2464, + "theoretical_loss": 3.3412404669418203, + "tokens_seen": 2771255296 + }, + { + "epoch": 0.84, + "learning_rate": 8.100300902708124e-05, + "loss": 1.3868, + "theoretical_loss": 3.3412281201664067, + "tokens_seen": 2771386368 + }, + { + "epoch": 0.84, + "learning_rate": 8.098294884653962e-05, + "loss": 1.1723, + "theoretical_loss": 3.3412157741384108, + "tokens_seen": 2771517440 + }, + { + "epoch": 0.84, + "learning_rate": 8.0962888665998e-05, + "loss": 1.1837, + "theoretical_loss": 3.3412034288577512, + "tokens_seen": 2771648512 + }, + { + "epoch": 0.84, + "learning_rate": 8.094282848545637e-05, + "loss": 1.1909, + "theoretical_loss": 3.341191084324348, + "tokens_seen": 2771779584 + }, + { + "epoch": 0.84, + "learning_rate": 8.092276830491473e-05, + "loss": 1.2746, + "theoretical_loss": 3.341178740538121, + "tokens_seen": 2771910656 + }, + { + "epoch": 0.84, + "learning_rate": 8.090270812437311e-05, + "loss": 1.2689, + "theoretical_loss": 3.3411663974989887, + "tokens_seen": 2772041728 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.37658846378326416, + "objective/train/avg_token_score": 0.012215867638587952, + "objective/train/docs_used": 1560640, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0935765504837036, + "objective/train/original_loss": 3.5343821048736572, + "objective/train/theoretical_loss": 3.341154055206871, + "objective/train/tokens_used": 2772172800, + "objective/train/unlikelihood_loss": 0.14383217692375183, + "theoretical_loss": 3.341154055206871, + "tokens_seen": 2772172800 + }, + { + "epoch": 0.84, + "learning_rate": 8.08826479438315e-05, + "loss": 1.1927, + "theoretical_loss": 3.341154055206871, + "tokens_seen": 2772172800 + }, + { + "epoch": 0.84, + "learning_rate": 8.086258776328987e-05, + "loss": 1.1853, + "theoretical_loss": 3.3411417136616874, + "tokens_seen": 2772303872 + }, + { + "epoch": 0.84, + "learning_rate": 8.084252758274825e-05, + "loss": 1.1288, + "theoretical_loss": 3.3411293728633575, + "tokens_seen": 2772434944 + }, + { + "epoch": 0.84, + "learning_rate": 8.082246740220663e-05, + "loss": 1.3463, + "theoretical_loss": 3.341117032811801, + "tokens_seen": 2772566016 + }, + { + "epoch": 0.84, + "learning_rate": 8.0802407221665e-05, + "loss": 1.1885, + "theoretical_loss": 3.3411046935069364, + "tokens_seen": 2772697088 + }, + { + "epoch": 0.84, + "learning_rate": 8.078234704112338e-05, + "loss": 1.2717, + "theoretical_loss": 3.3410923549486844, + "tokens_seen": 2772828160 + }, + { + "epoch": 0.84, + "learning_rate": 8.076228686058175e-05, + "loss": 1.1797, + "theoretical_loss": 3.3410800171369637, + "tokens_seen": 2772959232 + }, + { + "epoch": 0.84, + "learning_rate": 8.074222668004013e-05, + "loss": 1.1593, + "theoretical_loss": 3.3410676800716943, + "tokens_seen": 2773090304 + }, + { + "epoch": 0.84, + "learning_rate": 8.072216649949849e-05, + "loss": 1.2579, + "theoretical_loss": 3.341055343752796, + "tokens_seen": 2773221376 + }, + { + "epoch": 0.84, + "learning_rate": 8.070210631895687e-05, + "loss": 1.3677, + "theoretical_loss": 3.341043008180187, + "tokens_seen": 2773352448 + }, + { + "epoch": 0.84, + "learning_rate": 8.068204613841524e-05, + "loss": 1.2951, + "theoretical_loss": 3.3410306733537887, + "tokens_seen": 2773483520 + }, + { + "epoch": 0.84, + "learning_rate": 8.066198595787362e-05, + "loss": 1.3868, + "theoretical_loss": 3.34101833927352, + "tokens_seen": 2773614592 + }, + { + "epoch": 0.84, + "learning_rate": 8.0641925777332e-05, + "loss": 1.3233, + "theoretical_loss": 3.3410060059393003, + "tokens_seen": 2773745664 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.36125367879867554, + "objective/train/avg_token_score": 0.016872845590114594, + "objective/train/docs_used": 1561209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1843006610870361, + "objective/train/original_loss": 3.9403271675109863, + "objective/train/theoretical_loss": 3.340999839551934, + "objective/train/tokens_used": 2773811200, + "objective/train/unlikelihood_loss": 0.17866834998130798, + "theoretical_loss": 3.340999839551934, + "tokens_seen": 2773811200 + }, + { + "epoch": 0.84, + "learning_rate": 8.062186559679037e-05, + "loss": 1.2129, + "theoretical_loss": 3.3409936733510492, + "tokens_seen": 2773876736 + }, + { + "epoch": 0.84, + "learning_rate": 8.060180541624875e-05, + "loss": 1.3813, + "theoretical_loss": 3.3409813415086864, + "tokens_seen": 2774007808 + }, + { + "epoch": 0.84, + "learning_rate": 8.058174523570712e-05, + "loss": 1.1757, + "theoretical_loss": 3.340969010412132, + "tokens_seen": 2774138880 + }, + { + "epoch": 0.84, + "learning_rate": 8.05616850551655e-05, + "loss": 1.2836, + "theoretical_loss": 3.340956680061305, + "tokens_seen": 2774269952 + }, + { + "epoch": 0.84, + "learning_rate": 8.054162487462387e-05, + "loss": 1.2056, + "theoretical_loss": 3.340944350456125, + "tokens_seen": 2774401024 + }, + { + "epoch": 0.84, + "learning_rate": 8.052156469408224e-05, + "loss": 1.2352, + "theoretical_loss": 3.3409320215965126, + "tokens_seen": 2774532096 + }, + { + "epoch": 0.84, + "learning_rate": 8.050150451354063e-05, + "loss": 1.1812, + "theoretical_loss": 3.3409196934823866, + "tokens_seen": 2774663168 + }, + { + "epoch": 0.84, + "learning_rate": 8.0481444332999e-05, + "loss": 1.2676, + "theoretical_loss": 3.340907366113667, + "tokens_seen": 2774794240 + }, + { + "epoch": 0.84, + "learning_rate": 8.046138415245738e-05, + "loss": 1.2394, + "theoretical_loss": 3.340895039490274, + "tokens_seen": 2774925312 + }, + { + "epoch": 0.84, + "learning_rate": 8.044132397191575e-05, + "loss": 1.315, + "theoretical_loss": 3.3408827136121264, + "tokens_seen": 2775056384 + }, + { + "epoch": 0.84, + "learning_rate": 8.042126379137413e-05, + "loss": 1.1734, + "theoretical_loss": 3.340870388479145, + "tokens_seen": 2775187456 + }, + { + "epoch": 0.84, + "learning_rate": 8.04012036108325e-05, + "loss": 1.3192, + "theoretical_loss": 3.3408580640912486, + "tokens_seen": 2775318528 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.4089687168598175, + "objective/train/avg_token_score": 0.004248063545674086, + "objective/train/docs_used": 1562750, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2832120656967163, + "objective/train/original_loss": 3.8246822357177734, + "objective/train/theoretical_loss": 3.3408457404483576, + "objective/train/tokens_used": 2775449600, + "objective/train/unlikelihood_loss": 0.14795228838920593, + "theoretical_loss": 3.3408457404483576, + "tokens_seen": 2775449600 + }, + { + "epoch": 0.84, + "learning_rate": 8.038114343029088e-05, + "loss": 1.2475, + "theoretical_loss": 3.3408457404483576, + "tokens_seen": 2775449600 + }, + { + "epoch": 0.84, + "learning_rate": 8.036108324974926e-05, + "loss": 1.223, + "theoretical_loss": 3.3408334175503915, + "tokens_seen": 2775580672 + }, + { + "epoch": 0.84, + "learning_rate": 8.034102306920763e-05, + "loss": 1.2509, + "theoretical_loss": 3.3408210953972697, + "tokens_seen": 2775711744 + }, + { + "epoch": 0.84, + "learning_rate": 8.0320962888666e-05, + "loss": 1.2352, + "theoretical_loss": 3.340808773988913, + "tokens_seen": 2775842816 + }, + { + "epoch": 0.84, + "learning_rate": 8.030090270812437e-05, + "loss": 1.1748, + "theoretical_loss": 3.3407964533252406, + "tokens_seen": 2775973888 + }, + { + "epoch": 0.84, + "learning_rate": 8.028084252758275e-05, + "loss": 1.2684, + "theoretical_loss": 3.3407841334061725, + "tokens_seen": 2776104960 + }, + { + "epoch": 0.84, + "learning_rate": 8.026078234704112e-05, + "loss": 1.4451, + "theoretical_loss": 3.3407718142316285, + "tokens_seen": 2776236032 + }, + { + "epoch": 0.84, + "learning_rate": 8.02407221664995e-05, + "loss": 1.1628, + "theoretical_loss": 3.340759495801528, + "tokens_seen": 2776367104 + }, + { + "epoch": 0.84, + "learning_rate": 8.022066198595787e-05, + "loss": 1.2129, + "theoretical_loss": 3.340747178115792, + "tokens_seen": 2776498176 + }, + { + "epoch": 0.84, + "learning_rate": 8.020060180541625e-05, + "loss": 1.2706, + "theoretical_loss": 3.3407348611743393, + "tokens_seen": 2776629248 + }, + { + "epoch": 0.84, + "learning_rate": 8.018054162487462e-05, + "loss": 1.2253, + "theoretical_loss": 3.34072254497709, + "tokens_seen": 2776760320 + }, + { + "epoch": 0.84, + "learning_rate": 8.0160481444333e-05, + "loss": 1.3983, + "theoretical_loss": 3.3407102295239643, + "tokens_seen": 2776891392 + }, + { + "epoch": 0.84, + "learning_rate": 8.014042126379138e-05, + "loss": 1.2878, + "theoretical_loss": 3.340697914814882, + "tokens_seen": 2777022464 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.28042522072792053, + "objective/train/avg_token_score": 0.012957296334207058, + "objective/train/docs_used": 1563516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8230996131896973, + "objective/train/original_loss": 3.622981548309326, + "objective/train/theoretical_loss": 3.3406917577393322, + "objective/train/tokens_used": 2777088000, + "objective/train/unlikelihood_loss": 0.23247753083705902, + "theoretical_loss": 3.3406917577393322, + "tokens_seen": 2777088000 + }, + { + "epoch": 0.84, + "learning_rate": 8.012036108324975e-05, + "loss": 1.3697, + "theoretical_loss": 3.3406856008497634, + "tokens_seen": 2777153536 + }, + { + "epoch": 0.84, + "learning_rate": 8.010030090270813e-05, + "loss": 1.2943, + "theoretical_loss": 3.340673287628528, + "tokens_seen": 2777284608 + }, + { + "epoch": 0.84, + "learning_rate": 8.00802407221665e-05, + "loss": 1.2181, + "theoretical_loss": 3.3406609751510956, + "tokens_seen": 2777415680 + }, + { + "epoch": 0.84, + "learning_rate": 8.006018054162488e-05, + "loss": 1.1569, + "theoretical_loss": 3.3406486634173866, + "tokens_seen": 2777546752 + }, + { + "epoch": 0.84, + "learning_rate": 8.004012036108325e-05, + "loss": 1.3049, + "theoretical_loss": 3.3406363524273206, + "tokens_seen": 2777677824 + }, + { + "epoch": 0.84, + "learning_rate": 8.002006018054163e-05, + "loss": 1.2592, + "theoretical_loss": 3.340624042180818, + "tokens_seen": 2777808896 + }, + { + "epoch": 0.84, + "learning_rate": 8e-05, + "loss": 1.3057, + "theoretical_loss": 3.3406117326777984, + "tokens_seen": 2777939968 + }, + { + "epoch": 0.84, + "learning_rate": 7.997993981945838e-05, + "loss": 1.4402, + "theoretical_loss": 3.3405994239181824, + "tokens_seen": 2778071040 + }, + { + "epoch": 0.84, + "learning_rate": 7.995987963891676e-05, + "loss": 1.1824, + "theoretical_loss": 3.3405871159018896, + "tokens_seen": 2778202112 + }, + { + "epoch": 0.84, + "learning_rate": 7.993981945837512e-05, + "loss": 1.2896, + "theoretical_loss": 3.3405748086288396, + "tokens_seen": 2778333184 + }, + { + "epoch": 0.84, + "learning_rate": 7.99197592778335e-05, + "loss": 1.1784, + "theoretical_loss": 3.340562502098954, + "tokens_seen": 2778464256 + }, + { + "epoch": 0.84, + "learning_rate": 7.989969909729187e-05, + "loss": 1.3047, + "theoretical_loss": 3.340550196312151, + "tokens_seen": 2778595328 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.33913734555244446, + "objective/train/avg_token_score": 0.00944016594439745, + "objective/train/docs_used": 1564998, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.053399920463562, + "objective/train/original_loss": 4.037463665008545, + "objective/train/theoretical_loss": 3.3405378912683514, + "objective/train/tokens_used": 2778726400, + "objective/train/unlikelihood_loss": 0.13899977505207062, + "theoretical_loss": 3.3405378912683514, + "tokens_seen": 2778726400 + }, + { + "epoch": 0.84, + "learning_rate": 7.987963891675025e-05, + "loss": 1.2108, + "theoretical_loss": 3.3405378912683514, + "tokens_seen": 2778726400 + }, + { + "epoch": 0.84, + "learning_rate": 7.985957873620862e-05, + "loss": 1.3485, + "theoretical_loss": 3.340525586967476, + "tokens_seen": 2778857472 + }, + { + "epoch": 0.84, + "learning_rate": 7.9839518555667e-05, + "loss": 1.1587, + "theoretical_loss": 3.340513283409444, + "tokens_seen": 2778988544 + }, + { + "epoch": 0.84, + "learning_rate": 7.981945837512538e-05, + "loss": 1.2382, + "theoretical_loss": 3.3405009805941757, + "tokens_seen": 2779119616 + }, + { + "epoch": 0.84, + "learning_rate": 7.979939819458375e-05, + "loss": 1.205, + "theoretical_loss": 3.3404886785215915, + "tokens_seen": 2779250688 + }, + { + "epoch": 0.84, + "learning_rate": 7.977933801404213e-05, + "loss": 1.3568, + "theoretical_loss": 3.3404763771916115, + "tokens_seen": 2779381760 + }, + { + "epoch": 0.84, + "learning_rate": 7.975927783350052e-05, + "loss": 1.1316, + "theoretical_loss": 3.340464076604156, + "tokens_seen": 2779512832 + }, + { + "epoch": 0.84, + "learning_rate": 7.973921765295888e-05, + "loss": 1.1915, + "theoretical_loss": 3.3404517767591444, + "tokens_seen": 2779643904 + }, + { + "epoch": 0.84, + "learning_rate": 7.971915747241725e-05, + "loss": 1.2927, + "theoretical_loss": 3.3404394776564974, + "tokens_seen": 2779774976 + }, + { + "epoch": 0.84, + "learning_rate": 7.969909729187563e-05, + "loss": 1.2681, + "theoretical_loss": 3.3404271792961358, + "tokens_seen": 2779906048 + }, + { + "epoch": 0.84, + "learning_rate": 7.9679037111334e-05, + "loss": 1.1229, + "theoretical_loss": 3.3404148816779786, + "tokens_seen": 2780037120 + }, + { + "epoch": 0.84, + "learning_rate": 7.965897693079238e-05, + "loss": 1.2304, + "theoretical_loss": 3.340402584801947, + "tokens_seen": 2780168192 + }, + { + "epoch": 0.84, + "learning_rate": 7.963891675025076e-05, + "loss": 1.2012, + "theoretical_loss": 3.340390288667961, + "tokens_seen": 2780299264 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.3978494703769684, + "objective/train/avg_token_score": 0.006315363571047783, + "objective/train/docs_used": 1565682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2045960426330566, + "objective/train/original_loss": 3.8014519214630127, + "objective/train/theoretical_loss": 3.34038414087921, + "objective/train/tokens_used": 2780364800, + "objective/train/unlikelihood_loss": 0.11603409796953201, + "theoretical_loss": 3.34038414087921, + "tokens_seen": 2780364800 + }, + { + "epoch": 0.84, + "learning_rate": 7.961885656970913e-05, + "loss": 1.1909, + "theoretical_loss": 3.3403779932759403, + "tokens_seen": 2780430336 + }, + { + "epoch": 0.84, + "learning_rate": 7.959879638916751e-05, + "loss": 1.344, + "theoretical_loss": 3.340365698625806, + "tokens_seen": 2780561408 + }, + { + "epoch": 0.84, + "learning_rate": 7.957873620862588e-05, + "loss": 1.1819, + "theoretical_loss": 3.3403534047174777, + "tokens_seen": 2780692480 + }, + { + "epoch": 0.84, + "learning_rate": 7.955867602808425e-05, + "loss": 1.2884, + "theoretical_loss": 3.3403411115508757, + "tokens_seen": 2780823552 + }, + { + "epoch": 0.84, + "learning_rate": 7.953861584754262e-05, + "loss": 1.1907, + "theoretical_loss": 3.340328819125921, + "tokens_seen": 2780954624 + }, + { + "epoch": 0.84, + "learning_rate": 7.9518555667001e-05, + "loss": 1.3567, + "theoretical_loss": 3.340316527442533, + "tokens_seen": 2781085696 + }, + { + "epoch": 0.84, + "learning_rate": 7.949849548645937e-05, + "loss": 1.3788, + "theoretical_loss": 3.3403042365006326, + "tokens_seen": 2781216768 + }, + { + "epoch": 0.84, + "learning_rate": 7.947843530591775e-05, + "loss": 1.3228, + "theoretical_loss": 3.34029194630014, + "tokens_seen": 2781347840 + }, + { + "epoch": 0.84, + "learning_rate": 7.945837512537613e-05, + "loss": 1.2583, + "theoretical_loss": 3.3402796568409756, + "tokens_seen": 2781478912 + }, + { + "epoch": 0.84, + "learning_rate": 7.94383149448345e-05, + "loss": 1.3992, + "theoretical_loss": 3.3402673681230595, + "tokens_seen": 2781609984 + }, + { + "epoch": 0.84, + "learning_rate": 7.941825476429288e-05, + "loss": 1.2003, + "theoretical_loss": 3.340255080146312, + "tokens_seen": 2781741056 + }, + { + "epoch": 0.84, + "learning_rate": 7.939819458375125e-05, + "loss": 1.1597, + "theoretical_loss": 3.340242792910654, + "tokens_seen": 2781872128 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.38178151845932007, + "objective/train/avg_token_score": 0.03355756029486656, + "objective/train/docs_used": 1566838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2482209205627441, + "objective/train/original_loss": 3.8355207443237305, + "objective/train/theoretical_loss": 3.340230506416005, + "objective/train/tokens_used": 2782003200, + "objective/train/unlikelihood_loss": 0.15949471294879913, + "theoretical_loss": 3.340230506416005, + "tokens_seen": 2782003200 + }, + { + "epoch": 0.84, + "learning_rate": 7.937813440320964e-05, + "loss": 1.2105, + "theoretical_loss": 3.340230506416005, + "tokens_seen": 2782003200 + }, + { + "epoch": 0.84, + "learning_rate": 7.9358074222668e-05, + "loss": 1.3102, + "theoretical_loss": 3.340218220662287, + "tokens_seen": 2782134272 + }, + { + "epoch": 0.84, + "learning_rate": 7.933801404212638e-05, + "loss": 1.2034, + "theoretical_loss": 3.3402059356494185, + "tokens_seen": 2782265344 + }, + { + "epoch": 0.84, + "learning_rate": 7.931795386158476e-05, + "loss": 1.1984, + "theoretical_loss": 3.340193651377321, + "tokens_seen": 2782396416 + }, + { + "epoch": 0.84, + "learning_rate": 7.929789368104313e-05, + "loss": 1.3223, + "theoretical_loss": 3.3401813678459154, + "tokens_seen": 2782527488 + }, + { + "epoch": 0.84, + "learning_rate": 7.927783350050151e-05, + "loss": 1.3483, + "theoretical_loss": 3.3401690850551207, + "tokens_seen": 2782658560 + }, + { + "epoch": 0.84, + "learning_rate": 7.925777331995988e-05, + "loss": 1.1487, + "theoretical_loss": 3.3401568030048585, + "tokens_seen": 2782789632 + }, + { + "epoch": 0.84, + "learning_rate": 7.923771313941826e-05, + "loss": 1.3376, + "theoretical_loss": 3.3401445216950494, + "tokens_seen": 2782920704 + }, + { + "epoch": 0.84, + "learning_rate": 7.921765295887664e-05, + "loss": 1.3728, + "theoretical_loss": 3.340132241125613, + "tokens_seen": 2783051776 + }, + { + "epoch": 0.84, + "learning_rate": 7.919759277833501e-05, + "loss": 1.2445, + "theoretical_loss": 3.3401199612964705, + "tokens_seen": 2783182848 + }, + { + "epoch": 0.84, + "learning_rate": 7.917753259779339e-05, + "loss": 1.2988, + "theoretical_loss": 3.3401076822075417, + "tokens_seen": 2783313920 + }, + { + "epoch": 0.84, + "learning_rate": 7.915747241725175e-05, + "loss": 1.1898, + "theoretical_loss": 3.340095403858748, + "tokens_seen": 2783444992 + }, + { + "epoch": 0.84, + "learning_rate": 7.913741223671012e-05, + "loss": 1.1491, + "theoretical_loss": 3.3400831262500095, + "tokens_seen": 2783576064 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.3977883756160736, + "objective/train/avg_token_score": 0.029079750180244446, + "objective/train/docs_used": 1567583, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0518020391464233, + "objective/train/original_loss": 4.401224136352539, + "objective/train/theoretical_loss": 3.340076987723136, + "objective/train/tokens_used": 2783641600, + "objective/train/unlikelihood_loss": 0.10878827422857285, + "theoretical_loss": 3.340076987723136, + "tokens_seen": 2783641600 + }, + { + "epoch": 0.84, + "learning_rate": 7.91173520561685e-05, + "loss": 1.1347, + "theoretical_loss": 3.3400708493812465, + "tokens_seen": 2783707136 + }, + { + "epoch": 0.84, + "learning_rate": 7.909729187562688e-05, + "loss": 1.3699, + "theoretical_loss": 3.3400585732523798, + "tokens_seen": 2783838208 + }, + { + "epoch": 0.84, + "learning_rate": 7.907723169508525e-05, + "loss": 1.4035, + "theoretical_loss": 3.3400462978633305, + "tokens_seen": 2783969280 + }, + { + "epoch": 0.84, + "learning_rate": 7.905717151454363e-05, + "loss": 1.1128, + "theoretical_loss": 3.3400340232140184, + "tokens_seen": 2784100352 + }, + { + "epoch": 0.84, + "learning_rate": 7.9037111334002e-05, + "loss": 1.2347, + "theoretical_loss": 3.3400217493043645, + "tokens_seen": 2784231424 + }, + { + "epoch": 0.84, + "learning_rate": 7.901705115346038e-05, + "loss": 1.2211, + "theoretical_loss": 3.340009476134289, + "tokens_seen": 2784362496 + }, + { + "epoch": 0.84, + "learning_rate": 7.899699097291877e-05, + "loss": 1.1299, + "theoretical_loss": 3.3399972037037133, + "tokens_seen": 2784493568 + }, + { + "epoch": 0.84, + "learning_rate": 7.897693079237714e-05, + "loss": 1.4618, + "theoretical_loss": 3.339984932012557, + "tokens_seen": 2784624640 + }, + { + "epoch": 0.84, + "learning_rate": 7.895687061183551e-05, + "loss": 1.3221, + "theoretical_loss": 3.339972661060742, + "tokens_seen": 2784755712 + }, + { + "epoch": 0.84, + "learning_rate": 7.893681043129388e-05, + "loss": 1.1252, + "theoretical_loss": 3.339960390848188, + "tokens_seen": 2784886784 + }, + { + "epoch": 0.84, + "learning_rate": 7.891675025075226e-05, + "loss": 1.2194, + "theoretical_loss": 3.3399481213748157, + "tokens_seen": 2785017856 + }, + { + "epoch": 0.84, + "learning_rate": 7.889669007021063e-05, + "loss": 1.0931, + "theoretical_loss": 3.3399358526405463, + "tokens_seen": 2785148928 + }, + { + "debugging/length": 556.5, + "debugging/num_segments": 4, + "epoch": 0.84, + "objective/train/aligned_token_count": 0.5740469098091125, + "objective/train/avg_token_score": 0.005750786978751421, + "objective/train/docs_used": 1568290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.641748309135437, + "objective/train/original_loss": 3.1441538333892822, + "objective/train/theoretical_loss": 3.3399235846453004, + "objective/train/tokens_used": 2785280000, + "objective/train/unlikelihood_loss": 0.2065691351890564, + "theoretical_loss": 3.3399235846453004, + "tokens_seen": 2785280000 + }, + { + "epoch": 0.84, + "learning_rate": 7.887662988966901e-05, + "loss": 1.1859, + "theoretical_loss": 3.3399235846453004, + "tokens_seen": 2785280000 + }, + { + "epoch": 0.84, + "learning_rate": 7.885656970912739e-05, + "loss": 1.3952, + "theoretical_loss": 3.3399113173889985, + "tokens_seen": 2785411072 + }, + { + "epoch": 0.84, + "learning_rate": 7.883650952858576e-05, + "loss": 1.271, + "theoretical_loss": 3.339899050871561, + "tokens_seen": 2785542144 + }, + { + "epoch": 0.84, + "learning_rate": 7.881644934804414e-05, + "loss": 1.351, + "theoretical_loss": 3.3398867850929093, + "tokens_seen": 2785673216 + }, + { + "epoch": 0.84, + "learning_rate": 7.879638916750251e-05, + "loss": 1.1769, + "theoretical_loss": 3.339874520052964, + "tokens_seen": 2785804288 + }, + { + "epoch": 0.84, + "learning_rate": 7.877632898696088e-05, + "loss": 1.1512, + "theoretical_loss": 3.3398622557516453, + "tokens_seen": 2785935360 + }, + { + "epoch": 0.84, + "learning_rate": 7.875626880641925e-05, + "loss": 1.0989, + "theoretical_loss": 3.3398499921888742, + "tokens_seen": 2786066432 + }, + { + "epoch": 0.84, + "learning_rate": 7.873620862587763e-05, + "loss": 1.2463, + "theoretical_loss": 3.339837729364572, + "tokens_seen": 2786197504 + }, + { + "epoch": 0.84, + "learning_rate": 7.8716148445336e-05, + "loss": 1.1156, + "theoretical_loss": 3.3398254672786596, + "tokens_seen": 2786328576 + }, + { + "epoch": 0.84, + "learning_rate": 7.869608826479438e-05, + "loss": 1.4296, + "theoretical_loss": 3.3398132059310566, + "tokens_seen": 2786459648 + }, + { + "epoch": 0.84, + "learning_rate": 7.867602808425275e-05, + "loss": 1.0673, + "theoretical_loss": 3.3398009453216853, + "tokens_seen": 2786590720 + }, + { + "epoch": 0.84, + "learning_rate": 7.865596790371113e-05, + "loss": 1.3032, + "theoretical_loss": 3.3397886854504653, + "tokens_seen": 2786721792 + }, + { + "epoch": 0.84, + "learning_rate": 7.863590772316952e-05, + "loss": 1.3277, + "theoretical_loss": 3.339776426317318, + "tokens_seen": 2786852864 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.4995112419128418, + "objective/train/avg_token_score": 0.016624638810753822, + "objective/train/docs_used": 1569381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4126964807510376, + "objective/train/original_loss": 3.5819883346557617, + "objective/train/theoretical_loss": 3.339770297027497, + "objective/train/tokens_used": 2786918400, + "objective/train/unlikelihood_loss": 0.1618114709854126, + "theoretical_loss": 3.339770297027497, + "tokens_seen": 2786918400 + }, + { + "epoch": 0.84, + "learning_rate": 7.86158475426279e-05, + "loss": 1.2967, + "theoretical_loss": 3.3397641679221644, + "tokens_seen": 2786983936 + }, + { + "epoch": 0.84, + "learning_rate": 7.859578736208627e-05, + "loss": 1.2359, + "theoretical_loss": 3.339751910264925, + "tokens_seen": 2787115008 + }, + { + "epoch": 0.84, + "learning_rate": 7.857572718154463e-05, + "loss": 1.2247, + "theoretical_loss": 3.339739653345521, + "tokens_seen": 2787246080 + }, + { + "epoch": 0.84, + "learning_rate": 7.855566700100301e-05, + "loss": 1.4465, + "theoretical_loss": 3.3397273971638732, + "tokens_seen": 2787377152 + }, + { + "epoch": 0.84, + "learning_rate": 7.853560682046139e-05, + "loss": 1.1986, + "theoretical_loss": 3.339715141719902, + "tokens_seen": 2787508224 + }, + { + "epoch": 0.84, + "learning_rate": 7.851554663991976e-05, + "loss": 1.1961, + "theoretical_loss": 3.3397028870135292, + "tokens_seen": 2787639296 + }, + { + "epoch": 0.84, + "learning_rate": 7.849548645937814e-05, + "loss": 1.2224, + "theoretical_loss": 3.3396906330446754, + "tokens_seen": 2787770368 + }, + { + "epoch": 0.84, + "learning_rate": 7.847542627883651e-05, + "loss": 1.3691, + "theoretical_loss": 3.339678379813261, + "tokens_seen": 2787901440 + }, + { + "epoch": 0.84, + "learning_rate": 7.845536609829489e-05, + "loss": 1.2656, + "theoretical_loss": 3.339666127319208, + "tokens_seen": 2788032512 + }, + { + "epoch": 0.84, + "learning_rate": 7.843530591775326e-05, + "loss": 1.3018, + "theoretical_loss": 3.3396538755624365, + "tokens_seen": 2788163584 + }, + { + "epoch": 0.84, + "learning_rate": 7.841524573721164e-05, + "loss": 1.3431, + "theoretical_loss": 3.3396416245428675, + "tokens_seen": 2788294656 + }, + { + "epoch": 0.84, + "learning_rate": 7.839518555667002e-05, + "loss": 1.3185, + "theoretical_loss": 3.3396293742604226, + "tokens_seen": 2788425728 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.3700513243675232, + "objective/train/avg_token_score": 0.019972683861851692, + "objective/train/docs_used": 1570209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1522916555404663, + "objective/train/original_loss": 4.168471813201904, + "objective/train/theoretical_loss": 3.3396171247150224, + "objective/train/tokens_used": 2788556800, + "objective/train/unlikelihood_loss": 0.15561096370220184, + "theoretical_loss": 3.3396171247150224, + "tokens_seen": 2788556800 + }, + { + "epoch": 0.85, + "learning_rate": 7.837512537612838e-05, + "loss": 1.1535, + "theoretical_loss": 3.3396171247150224, + "tokens_seen": 2788556800 + }, + { + "epoch": 0.85, + "learning_rate": 7.835506519558675e-05, + "loss": 1.2571, + "theoretical_loss": 3.3396048759065877, + "tokens_seen": 2788687872 + }, + { + "epoch": 0.85, + "learning_rate": 7.833500501504513e-05, + "loss": 1.4261, + "theoretical_loss": 3.33959262783504, + "tokens_seen": 2788818944 + }, + { + "epoch": 0.85, + "learning_rate": 7.83149448345035e-05, + "loss": 1.3417, + "theoretical_loss": 3.3395803805003004, + "tokens_seen": 2788950016 + }, + { + "epoch": 0.85, + "learning_rate": 7.829488465396188e-05, + "loss": 1.3417, + "theoretical_loss": 3.339568133902289, + "tokens_seen": 2789081088 + }, + { + "epoch": 0.85, + "learning_rate": 7.827482447342026e-05, + "loss": 1.2748, + "theoretical_loss": 3.339555888040928, + "tokens_seen": 2789212160 + }, + { + "epoch": 0.85, + "learning_rate": 7.825476429287865e-05, + "loss": 1.3169, + "theoretical_loss": 3.339543642916138, + "tokens_seen": 2789343232 + }, + { + "epoch": 0.85, + "learning_rate": 7.823470411233702e-05, + "loss": 1.0933, + "theoretical_loss": 3.33953139852784, + "tokens_seen": 2789474304 + }, + { + "epoch": 0.85, + "learning_rate": 7.82146439317954e-05, + "loss": 1.3801, + "theoretical_loss": 3.339519154875955, + "tokens_seen": 2789605376 + }, + { + "epoch": 0.85, + "learning_rate": 7.819458375125377e-05, + "loss": 1.2574, + "theoretical_loss": 3.3395069119604046, + "tokens_seen": 2789736448 + }, + { + "epoch": 0.85, + "learning_rate": 7.817452357071214e-05, + "loss": 1.3014, + "theoretical_loss": 3.3394946697811094, + "tokens_seen": 2789867520 + }, + { + "epoch": 0.85, + "learning_rate": 7.815446339017051e-05, + "loss": 1.3156, + "theoretical_loss": 3.339482428337991, + "tokens_seen": 2789998592 + }, + { + "epoch": 0.85, + "learning_rate": 7.813440320962889e-05, + "loss": 1.4696, + "theoretical_loss": 3.3394701876309703, + "tokens_seen": 2790129664 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.32948437333106995, + "objective/train/avg_token_score": 0.015189097262918949, + "objective/train/docs_used": 1571508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9645533561706543, + "objective/train/original_loss": 4.2598137855529785, + "objective/train/theoretical_loss": 3.3394640675534717, + "objective/train/tokens_used": 2790195200, + "objective/train/unlikelihood_loss": 0.09993351995944977, + "theoretical_loss": 3.3394640675534717, + "tokens_seen": 2790195200 + }, + { + "epoch": 0.85, + "learning_rate": 7.811434302908726e-05, + "loss": 1.3301, + "theoretical_loss": 3.3394579476599686, + "tokens_seen": 2790260736 + }, + { + "epoch": 0.85, + "learning_rate": 7.809428284854564e-05, + "loss": 1.2402, + "theoretical_loss": 3.3394457084249067, + "tokens_seen": 2790391808 + }, + { + "epoch": 0.85, + "learning_rate": 7.807422266800401e-05, + "loss": 1.399, + "theoretical_loss": 3.339433469925706, + "tokens_seen": 2790522880 + }, + { + "epoch": 0.85, + "learning_rate": 7.805416248746239e-05, + "loss": 1.2726, + "theoretical_loss": 3.339421232162288, + "tokens_seen": 2790653952 + }, + { + "epoch": 0.85, + "learning_rate": 7.803410230692077e-05, + "loss": 1.2106, + "theoretical_loss": 3.3394089951345736, + "tokens_seen": 2790785024 + }, + { + "epoch": 0.85, + "learning_rate": 7.801404212637914e-05, + "loss": 1.2256, + "theoretical_loss": 3.3393967588424838, + "tokens_seen": 2790916096 + }, + { + "epoch": 0.85, + "learning_rate": 7.79939819458375e-05, + "loss": 1.3554, + "theoretical_loss": 3.3393845232859403, + "tokens_seen": 2791047168 + }, + { + "epoch": 0.85, + "learning_rate": 7.797392176529588e-05, + "loss": 1.2339, + "theoretical_loss": 3.339372288464864, + "tokens_seen": 2791178240 + }, + { + "epoch": 0.85, + "learning_rate": 7.795386158475426e-05, + "loss": 1.2408, + "theoretical_loss": 3.3393600543791764, + "tokens_seen": 2791309312 + }, + { + "epoch": 0.85, + "learning_rate": 7.793380140421263e-05, + "loss": 1.258, + "theoretical_loss": 3.339347821028799, + "tokens_seen": 2791440384 + }, + { + "epoch": 0.85, + "learning_rate": 7.791374122367101e-05, + "loss": 1.273, + "theoretical_loss": 3.339335588413652, + "tokens_seen": 2791571456 + }, + { + "epoch": 0.85, + "learning_rate": 7.789368104312938e-05, + "loss": 1.2929, + "theoretical_loss": 3.339323356533658, + "tokens_seen": 2791702528 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.49560117721557617, + "objective/train/avg_token_score": 0.011096492409706116, + "objective/train/docs_used": 1572267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5659152269363403, + "objective/train/original_loss": 4.025465488433838, + "objective/train/theoretical_loss": 3.3393111253887375, + "objective/train/tokens_used": 2791833600, + "objective/train/unlikelihood_loss": 0.10172708332538605, + "theoretical_loss": 3.3393111253887375, + "tokens_seen": 2791833600 + }, + { + "epoch": 0.85, + "learning_rate": 7.787362086258777e-05, + "loss": 1.324, + "theoretical_loss": 3.3393111253887375, + "tokens_seen": 2791833600 + }, + { + "epoch": 0.85, + "learning_rate": 7.785356068204615e-05, + "loss": 1.369, + "theoretical_loss": 3.3392988949788123, + "tokens_seen": 2791964672 + }, + { + "epoch": 0.85, + "learning_rate": 7.783350050150452e-05, + "loss": 1.218, + "theoretical_loss": 3.3392866653038036, + "tokens_seen": 2792095744 + }, + { + "epoch": 0.85, + "learning_rate": 7.78134403209629e-05, + "loss": 1.3805, + "theoretical_loss": 3.3392744363636324, + "tokens_seen": 2792226816 + }, + { + "epoch": 0.85, + "learning_rate": 7.779338014042126e-05, + "loss": 1.324, + "theoretical_loss": 3.33926220815822, + "tokens_seen": 2792357888 + }, + { + "epoch": 0.85, + "learning_rate": 7.777331995987964e-05, + "loss": 1.1953, + "theoretical_loss": 3.339249980687489, + "tokens_seen": 2792488960 + }, + { + "epoch": 0.85, + "learning_rate": 7.775325977933801e-05, + "loss": 1.154, + "theoretical_loss": 3.339237753951359, + "tokens_seen": 2792620032 + }, + { + "epoch": 0.85, + "learning_rate": 7.773319959879639e-05, + "loss": 1.2345, + "theoretical_loss": 3.3392255279497527, + "tokens_seen": 2792751104 + }, + { + "epoch": 0.85, + "learning_rate": 7.771313941825477e-05, + "loss": 1.3558, + "theoretical_loss": 3.339213302682591, + "tokens_seen": 2792882176 + }, + { + "epoch": 0.85, + "learning_rate": 7.769307923771314e-05, + "loss": 1.2653, + "theoretical_loss": 3.339201078149795, + "tokens_seen": 2793013248 + }, + { + "epoch": 0.85, + "learning_rate": 7.767301905717152e-05, + "loss": 1.2985, + "theoretical_loss": 3.3391888543512867, + "tokens_seen": 2793144320 + }, + { + "epoch": 0.85, + "learning_rate": 7.765295887662989e-05, + "loss": 1.1542, + "theoretical_loss": 3.3391766312869873, + "tokens_seen": 2793275392 + }, + { + "epoch": 0.85, + "learning_rate": 7.763289869608827e-05, + "loss": 1.3256, + "theoretical_loss": 3.339164408956819, + "tokens_seen": 2793406464 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.4783724248409271, + "objective/train/avg_token_score": 0.006179228890687227, + "objective/train/docs_used": 1573597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4321849346160889, + "objective/train/original_loss": 3.508026123046875, + "objective/train/theoretical_loss": 3.3391582980670087, + "objective/train/tokens_used": 2793472000, + "objective/train/unlikelihood_loss": 0.16941435635089874, + "theoretical_loss": 3.3391582980670087, + "tokens_seen": 2793472000 + }, + { + "epoch": 0.85, + "learning_rate": 7.761283851554664e-05, + "loss": 1.2709, + "theoretical_loss": 3.3391521873607015, + "tokens_seen": 2793537536 + }, + { + "epoch": 0.85, + "learning_rate": 7.759277833500501e-05, + "loss": 1.3271, + "theoretical_loss": 3.3391399664985575, + "tokens_seen": 2793668608 + }, + { + "epoch": 0.85, + "learning_rate": 7.757271815446338e-05, + "loss": 1.2812, + "theoretical_loss": 3.3391277463703086, + "tokens_seen": 2793799680 + }, + { + "epoch": 0.85, + "learning_rate": 7.755265797392176e-05, + "loss": 1.3509, + "theoretical_loss": 3.339115526975876, + "tokens_seen": 2793930752 + }, + { + "epoch": 0.85, + "learning_rate": 7.753259779338013e-05, + "loss": 1.4998, + "theoretical_loss": 3.339103308315181, + "tokens_seen": 2794061824 + }, + { + "epoch": 0.85, + "learning_rate": 7.751253761283852e-05, + "loss": 1.4086, + "theoretical_loss": 3.3390910903881457, + "tokens_seen": 2794192896 + }, + { + "epoch": 0.85, + "learning_rate": 7.74924774322969e-05, + "loss": 1.1174, + "theoretical_loss": 3.339078873194691, + "tokens_seen": 2794323968 + }, + { + "epoch": 0.85, + "learning_rate": 7.747241725175528e-05, + "loss": 1.2606, + "theoretical_loss": 3.3390666567347385, + "tokens_seen": 2794455040 + }, + { + "epoch": 0.85, + "learning_rate": 7.745235707121365e-05, + "loss": 1.3283, + "theoretical_loss": 3.3390544410082104, + "tokens_seen": 2794586112 + }, + { + "epoch": 0.85, + "learning_rate": 7.743229689067203e-05, + "loss": 1.3103, + "theoretical_loss": 3.3390422260150276, + "tokens_seen": 2794717184 + }, + { + "epoch": 0.85, + "learning_rate": 7.74122367101304e-05, + "loss": 1.2653, + "theoretical_loss": 3.339030011755112, + "tokens_seen": 2794848256 + }, + { + "epoch": 0.85, + "learning_rate": 7.739217652958876e-05, + "loss": 1.4621, + "theoretical_loss": 3.3390177982283853, + "tokens_seen": 2794979328 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.3169599175453186, + "objective/train/avg_token_score": 0.012208246625959873, + "objective/train/docs_used": 1574190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.936874508857727, + "objective/train/original_loss": 4.44386100769043, + "objective/train/theoretical_loss": 3.339005585434769, + "objective/train/tokens_used": 2795110400, + "objective/train/unlikelihood_loss": 0.11070653796195984, + "theoretical_loss": 3.339005585434769, + "tokens_seen": 2795110400 + }, + { + "epoch": 0.85, + "learning_rate": 7.737211634904714e-05, + "loss": 1.2292, + "theoretical_loss": 3.339005585434769, + "tokens_seen": 2795110400 + }, + { + "epoch": 0.85, + "learning_rate": 7.735205616850552e-05, + "loss": 1.328, + "theoretical_loss": 3.3389933733741843, + "tokens_seen": 2795241472 + }, + { + "epoch": 0.85, + "learning_rate": 7.733199598796389e-05, + "loss": 1.1132, + "theoretical_loss": 3.3389811620465535, + "tokens_seen": 2795372544 + }, + { + "epoch": 0.85, + "learning_rate": 7.731193580742227e-05, + "loss": 1.3994, + "theoretical_loss": 3.3389689514517977, + "tokens_seen": 2795503616 + }, + { + "epoch": 0.85, + "learning_rate": 7.729187562688064e-05, + "loss": 1.3333, + "theoretical_loss": 3.338956741589839, + "tokens_seen": 2795634688 + }, + { + "epoch": 0.85, + "learning_rate": 7.727181544633902e-05, + "loss": 1.1802, + "theoretical_loss": 3.338944532460599, + "tokens_seen": 2795765760 + }, + { + "epoch": 0.85, + "learning_rate": 7.72517552657974e-05, + "loss": 1.2116, + "theoretical_loss": 3.338932324063999, + "tokens_seen": 2795896832 + }, + { + "epoch": 0.85, + "learning_rate": 7.723169508525577e-05, + "loss": 1.2326, + "theoretical_loss": 3.3389201163999607, + "tokens_seen": 2796027904 + }, + { + "epoch": 0.85, + "learning_rate": 7.721163490471413e-05, + "loss": 1.2646, + "theoretical_loss": 3.338907909468406, + "tokens_seen": 2796158976 + }, + { + "epoch": 0.85, + "learning_rate": 7.719157472417251e-05, + "loss": 1.2101, + "theoretical_loss": 3.3388957032692574, + "tokens_seen": 2796290048 + }, + { + "epoch": 0.85, + "learning_rate": 7.717151454363088e-05, + "loss": 1.2293, + "theoretical_loss": 3.3388834978024358, + "tokens_seen": 2796421120 + }, + { + "epoch": 0.85, + "learning_rate": 7.715145436308926e-05, + "loss": 1.2694, + "theoretical_loss": 3.3388712930678626, + "tokens_seen": 2796552192 + }, + { + "epoch": 0.85, + "learning_rate": 7.713139418254765e-05, + "loss": 1.2645, + "theoretical_loss": 3.33885908906546, + "tokens_seen": 2796683264 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.2875122129917145, + "objective/train/avg_token_score": 0.02959074079990387, + "objective/train/docs_used": 1575609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.899075448513031, + "objective/train/original_loss": 4.558325290679932, + "objective/train/theoretical_loss": 3.3388529873387984, + "objective/train/tokens_used": 2796748800, + "objective/train/unlikelihood_loss": 0.15404757857322693, + "theoretical_loss": 3.3388529873387984, + "tokens_seen": 2796748800 + }, + { + "epoch": 0.85, + "learning_rate": 7.711133400200603e-05, + "loss": 1.3694, + "theoretical_loss": 3.33884688579515, + "tokens_seen": 2796814336 + }, + { + "epoch": 0.85, + "learning_rate": 7.70912738214644e-05, + "loss": 1.3493, + "theoretical_loss": 3.338834683256854, + "tokens_seen": 2796945408 + }, + { + "epoch": 0.85, + "learning_rate": 7.707121364092278e-05, + "loss": 1.4204, + "theoretical_loss": 3.338822481450494, + "tokens_seen": 2797076480 + }, + { + "epoch": 0.85, + "learning_rate": 7.705115346038115e-05, + "loss": 1.2799, + "theoretical_loss": 3.3388102803759914, + "tokens_seen": 2797207552 + }, + { + "epoch": 0.85, + "learning_rate": 7.703109327983953e-05, + "loss": 1.1208, + "theoretical_loss": 3.3387980800332686, + "tokens_seen": 2797338624 + }, + { + "epoch": 0.85, + "learning_rate": 7.701103309929789e-05, + "loss": 1.2333, + "theoretical_loss": 3.338785880422247, + "tokens_seen": 2797469696 + }, + { + "epoch": 0.85, + "learning_rate": 7.699097291875627e-05, + "loss": 1.2205, + "theoretical_loss": 3.338773681542849, + "tokens_seen": 2797600768 + }, + { + "epoch": 0.85, + "learning_rate": 7.697091273821464e-05, + "loss": 1.1482, + "theoretical_loss": 3.338761483394996, + "tokens_seen": 2797731840 + }, + { + "epoch": 0.85, + "learning_rate": 7.695085255767302e-05, + "loss": 1.3854, + "theoretical_loss": 3.3387492859786096, + "tokens_seen": 2797862912 + }, + { + "epoch": 0.85, + "learning_rate": 7.69307923771314e-05, + "loss": 1.2767, + "theoretical_loss": 3.338737089293612, + "tokens_seen": 2797993984 + }, + { + "epoch": 0.85, + "learning_rate": 7.691073219658977e-05, + "loss": 1.3752, + "theoretical_loss": 3.338724893339925, + "tokens_seen": 2798125056 + }, + { + "epoch": 0.85, + "learning_rate": 7.689067201604815e-05, + "loss": 1.3083, + "theoretical_loss": 3.3387126981174706, + "tokens_seen": 2798256128 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.48106059432029724, + "objective/train/avg_token_score": 0.013374615460634232, + "objective/train/docs_used": 1576275, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3446063995361328, + "objective/train/original_loss": 3.4479525089263916, + "objective/train/theoretical_loss": 3.338700503626171, + "objective/train/tokens_used": 2798387200, + "objective/train/unlikelihood_loss": 0.13868092000484467, + "theoretical_loss": 3.338700503626171, + "tokens_seen": 2798387200 + }, + { + "epoch": 0.85, + "learning_rate": 7.687061183550652e-05, + "loss": 1.3477, + "theoretical_loss": 3.338700503626171, + "tokens_seen": 2798387200 + }, + { + "epoch": 0.85, + "learning_rate": 7.68505516549649e-05, + "loss": 1.2003, + "theoretical_loss": 3.3386883098659474, + "tokens_seen": 2798518272 + }, + { + "epoch": 0.85, + "learning_rate": 7.683049147442327e-05, + "loss": 1.1923, + "theoretical_loss": 3.3386761168367225, + "tokens_seen": 2798649344 + }, + { + "epoch": 0.85, + "learning_rate": 7.681043129388164e-05, + "loss": 1.4111, + "theoretical_loss": 3.3386639245384178, + "tokens_seen": 2798780416 + }, + { + "epoch": 0.85, + "learning_rate": 7.679037111334001e-05, + "loss": 1.3958, + "theoretical_loss": 3.338651732970955, + "tokens_seen": 2798911488 + }, + { + "epoch": 0.85, + "learning_rate": 7.677031093279839e-05, + "loss": 1.199, + "theoretical_loss": 3.3386395421342567, + "tokens_seen": 2799042560 + }, + { + "epoch": 0.85, + "learning_rate": 7.675025075225678e-05, + "loss": 1.3581, + "theoretical_loss": 3.3386273520282446, + "tokens_seen": 2799173632 + }, + { + "epoch": 0.85, + "learning_rate": 7.673019057171515e-05, + "loss": 1.2313, + "theoretical_loss": 3.3386151626528404, + "tokens_seen": 2799304704 + }, + { + "epoch": 0.85, + "learning_rate": 7.671013039117353e-05, + "loss": 1.3083, + "theoretical_loss": 3.3386029740079666, + "tokens_seen": 2799435776 + }, + { + "epoch": 0.85, + "learning_rate": 7.66900702106319e-05, + "loss": 1.157, + "theoretical_loss": 3.338590786093545, + "tokens_seen": 2799566848 + }, + { + "epoch": 0.85, + "learning_rate": 7.667001003009028e-05, + "loss": 1.2771, + "theoretical_loss": 3.338578598909497, + "tokens_seen": 2799697920 + }, + { + "epoch": 0.85, + "learning_rate": 7.664994984954866e-05, + "loss": 1.3491, + "theoretical_loss": 3.338566412455746, + "tokens_seen": 2799828992 + }, + { + "epoch": 0.85, + "learning_rate": 7.662988966900703e-05, + "loss": 1.1337, + "theoretical_loss": 3.3385542267322132, + "tokens_seen": 2799960064 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.4013318717479706, + "objective/train/avg_token_score": 0.01674633100628853, + "objective/train/docs_used": 1577506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2244120836257935, + "objective/train/original_loss": 3.6098687648773193, + "objective/train/theoretical_loss": 3.338548134144254, + "objective/train/tokens_used": 2800025600, + "objective/train/unlikelihood_loss": 0.18752144277095795, + "theoretical_loss": 3.338548134144254, + "tokens_seen": 2800025600 + }, + { + "epoch": 0.85, + "learning_rate": 7.66098294884654e-05, + "loss": 1.3175, + "theoretical_loss": 3.3385420417388207, + "tokens_seen": 2800091136 + }, + { + "epoch": 0.85, + "learning_rate": 7.658976930792377e-05, + "loss": 1.3669, + "theoretical_loss": 3.3385298574754905, + "tokens_seen": 2800222208 + }, + { + "epoch": 0.85, + "learning_rate": 7.656970912738215e-05, + "loss": 1.1248, + "theoretical_loss": 3.338517673942145, + "tokens_seen": 2800353280 + }, + { + "epoch": 0.85, + "learning_rate": 7.654964894684052e-05, + "loss": 1.2091, + "theoretical_loss": 3.338505491138706, + "tokens_seen": 2800484352 + }, + { + "epoch": 0.85, + "learning_rate": 7.65295887662989e-05, + "loss": 1.2826, + "theoretical_loss": 3.3384933090650963, + "tokens_seen": 2800615424 + }, + { + "epoch": 0.85, + "learning_rate": 7.650952858575727e-05, + "loss": 1.2073, + "theoretical_loss": 3.338481127721237, + "tokens_seen": 2800746496 + }, + { + "epoch": 0.85, + "learning_rate": 7.648946840521565e-05, + "loss": 1.2258, + "theoretical_loss": 3.3384689471070508, + "tokens_seen": 2800877568 + }, + { + "epoch": 0.85, + "learning_rate": 7.646940822467402e-05, + "loss": 1.371, + "theoretical_loss": 3.3384567672224597, + "tokens_seen": 2801008640 + }, + { + "epoch": 0.85, + "learning_rate": 7.64493480441324e-05, + "loss": 1.2079, + "theoretical_loss": 3.3384445880673863, + "tokens_seen": 2801139712 + }, + { + "epoch": 0.85, + "learning_rate": 7.642928786359076e-05, + "loss": 1.3853, + "theoretical_loss": 3.3384324096417517, + "tokens_seen": 2801270784 + }, + { + "epoch": 0.85, + "learning_rate": 7.640922768304914e-05, + "loss": 1.3608, + "theoretical_loss": 3.3384202319454794, + "tokens_seen": 2801401856 + }, + { + "epoch": 0.85, + "learning_rate": 7.638916750250751e-05, + "loss": 1.3748, + "theoretical_loss": 3.338408054978491, + "tokens_seen": 2801532928 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.43145161867141724, + "objective/train/avg_token_score": 0.002928493544459343, + "objective/train/docs_used": 1578159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3293853998184204, + "objective/train/original_loss": 3.6028993129730225, + "objective/train/theoretical_loss": 3.3383958787407084, + "objective/train/tokens_used": 2801664000, + "objective/train/unlikelihood_loss": 0.1675603687763214, + "theoretical_loss": 3.3383958787407084, + "tokens_seen": 2801664000 + }, + { + "epoch": 0.85, + "learning_rate": 7.63691073219659e-05, + "loss": 1.2675, + "theoretical_loss": 3.3383958787407084, + "tokens_seen": 2801664000 + }, + { + "epoch": 0.85, + "learning_rate": 7.634904714142428e-05, + "loss": 1.3036, + "theoretical_loss": 3.3383837032320542, + "tokens_seen": 2801795072 + }, + { + "epoch": 0.85, + "learning_rate": 7.632898696088265e-05, + "loss": 1.2289, + "theoretical_loss": 3.3383715284524507, + "tokens_seen": 2801926144 + }, + { + "epoch": 0.85, + "learning_rate": 7.630892678034103e-05, + "loss": 1.2575, + "theoretical_loss": 3.3383593544018195, + "tokens_seen": 2802057216 + }, + { + "epoch": 0.85, + "learning_rate": 7.62888665997994e-05, + "loss": 1.351, + "theoretical_loss": 3.338347181080084, + "tokens_seen": 2802188288 + }, + { + "epoch": 0.85, + "learning_rate": 7.626880641925778e-05, + "loss": 1.2534, + "theoretical_loss": 3.3383350084871655, + "tokens_seen": 2802319360 + }, + { + "epoch": 0.85, + "learning_rate": 7.624874623871616e-05, + "loss": 1.228, + "theoretical_loss": 3.3383228366229867, + "tokens_seen": 2802450432 + }, + { + "epoch": 0.85, + "learning_rate": 7.622868605817452e-05, + "loss": 1.2943, + "theoretical_loss": 3.3383106654874695, + "tokens_seen": 2802581504 + }, + { + "epoch": 0.85, + "learning_rate": 7.62086258776329e-05, + "loss": 1.2869, + "theoretical_loss": 3.3382984950805366, + "tokens_seen": 2802712576 + }, + { + "epoch": 0.85, + "learning_rate": 7.618856569709127e-05, + "loss": 1.1982, + "theoretical_loss": 3.3382863254021107, + "tokens_seen": 2802843648 + }, + { + "epoch": 0.85, + "learning_rate": 7.616850551654965e-05, + "loss": 1.2709, + "theoretical_loss": 3.338274156452113, + "tokens_seen": 2802974720 + }, + { + "epoch": 0.85, + "learning_rate": 7.614844533600802e-05, + "loss": 1.2658, + "theoretical_loss": 3.3382619882304665, + "tokens_seen": 2803105792 + }, + { + "epoch": 0.85, + "learning_rate": 7.61283851554664e-05, + "loss": 1.3248, + "theoretical_loss": 3.3382498207370936, + "tokens_seen": 2803236864 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.37512218952178955, + "objective/train/avg_token_score": 0.012656928040087223, + "objective/train/docs_used": 1578699, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.165199637413025, + "objective/train/original_loss": 3.5133962631225586, + "objective/train/theoretical_loss": 3.3382437372634857, + "objective/train/tokens_used": 2803302400, + "objective/train/unlikelihood_loss": 0.2455424666404724, + "theoretical_loss": 3.3382437372634857, + "tokens_seen": 2803302400 + }, + { + "epoch": 0.85, + "learning_rate": 7.610832497492477e-05, + "loss": 1.1612, + "theoretical_loss": 3.338237653971917, + "tokens_seen": 2803367936 + }, + { + "epoch": 0.85, + "learning_rate": 7.608826479438315e-05, + "loss": 1.1899, + "theoretical_loss": 3.3382254879348583, + "tokens_seen": 2803499008 + }, + { + "epoch": 0.85, + "learning_rate": 7.606820461384153e-05, + "loss": 1.2067, + "theoretical_loss": 3.3382133226258404, + "tokens_seen": 2803630080 + }, + { + "epoch": 0.85, + "learning_rate": 7.60481444332999e-05, + "loss": 1.3415, + "theoretical_loss": 3.3382011580447855, + "tokens_seen": 2803761152 + }, + { + "epoch": 0.85, + "learning_rate": 7.602808425275826e-05, + "loss": 1.2189, + "theoretical_loss": 3.338188994191616, + "tokens_seen": 2803892224 + }, + { + "epoch": 0.85, + "learning_rate": 7.600802407221665e-05, + "loss": 1.4436, + "theoretical_loss": 3.3381768310662543, + "tokens_seen": 2804023296 + }, + { + "epoch": 0.85, + "learning_rate": 7.598796389167503e-05, + "loss": 1.2537, + "theoretical_loss": 3.338164668668623, + "tokens_seen": 2804154368 + }, + { + "epoch": 0.85, + "learning_rate": 7.59679037111334e-05, + "loss": 1.1278, + "theoretical_loss": 3.338152506998645, + "tokens_seen": 2804285440 + }, + { + "epoch": 0.85, + "learning_rate": 7.594784353059178e-05, + "loss": 1.3127, + "theoretical_loss": 3.3381403460562415, + "tokens_seen": 2804416512 + }, + { + "epoch": 0.85, + "learning_rate": 7.592778335005016e-05, + "loss": 1.2716, + "theoretical_loss": 3.338128185841336, + "tokens_seen": 2804547584 + }, + { + "epoch": 0.85, + "learning_rate": 7.590772316950853e-05, + "loss": 1.2243, + "theoretical_loss": 3.3381160263538505, + "tokens_seen": 2804678656 + }, + { + "epoch": 0.85, + "learning_rate": 7.588766298896691e-05, + "loss": 1.3622, + "theoretical_loss": 3.338103867593708, + "tokens_seen": 2804809728 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.3668743968009949, + "objective/train/avg_token_score": 0.025385232642292976, + "objective/train/docs_used": 1579013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1552060842514038, + "objective/train/original_loss": 3.9527788162231445, + "objective/train/theoretical_loss": 3.33809170956083, + "objective/train/tokens_used": 2804940800, + "objective/train/unlikelihood_loss": 0.1534159928560257, + "theoretical_loss": 3.33809170956083, + "tokens_seen": 2804940800 + }, + { + "epoch": 0.85, + "learning_rate": 7.586760280842528e-05, + "loss": 1.2675, + "theoretical_loss": 3.33809170956083, + "tokens_seen": 2804940800 + }, + { + "epoch": 0.85, + "learning_rate": 7.584754262788365e-05, + "loss": 1.2691, + "theoretical_loss": 3.3380795522551403, + "tokens_seen": 2805071872 + }, + { + "epoch": 0.85, + "learning_rate": 7.582748244734202e-05, + "loss": 1.24, + "theoretical_loss": 3.3380673956765605, + "tokens_seen": 2805202944 + }, + { + "epoch": 0.85, + "learning_rate": 7.58074222668004e-05, + "loss": 1.2823, + "theoretical_loss": 3.338055239825014, + "tokens_seen": 2805334016 + }, + { + "epoch": 0.85, + "learning_rate": 7.578736208625877e-05, + "loss": 1.3367, + "theoretical_loss": 3.3380430847004225, + "tokens_seen": 2805465088 + }, + { + "epoch": 0.85, + "learning_rate": 7.576730190571715e-05, + "loss": 1.1179, + "theoretical_loss": 3.3380309303027085, + "tokens_seen": 2805596160 + }, + { + "epoch": 0.85, + "learning_rate": 7.574724172517553e-05, + "loss": 1.2287, + "theoretical_loss": 3.3380187766317957, + "tokens_seen": 2805727232 + }, + { + "epoch": 0.85, + "learning_rate": 7.57271815446339e-05, + "loss": 1.2012, + "theoretical_loss": 3.3380066236876056, + "tokens_seen": 2805858304 + }, + { + "epoch": 0.85, + "learning_rate": 7.570712136409228e-05, + "loss": 1.0033, + "theoretical_loss": 3.337994471470061, + "tokens_seen": 2805989376 + }, + { + "epoch": 0.85, + "learning_rate": 7.568706118355065e-05, + "loss": 1.1668, + "theoretical_loss": 3.337982319979085, + "tokens_seen": 2806120448 + }, + { + "epoch": 0.85, + "learning_rate": 7.566700100300903e-05, + "loss": 1.1708, + "theoretical_loss": 3.3379701692145995, + "tokens_seen": 2806251520 + }, + { + "epoch": 0.85, + "learning_rate": 7.564694082246739e-05, + "loss": 1.2118, + "theoretical_loss": 3.3379580191765275, + "tokens_seen": 2806382592 + }, + { + "epoch": 0.85, + "learning_rate": 7.562688064192578e-05, + "loss": 1.2476, + "theoretical_loss": 3.337945869864792, + "tokens_seen": 2806513664 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.3569159209728241, + "objective/train/avg_token_score": 0.026775576174259186, + "objective/train/docs_used": 1579588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.024167776107788, + "objective/train/original_loss": 3.7356154918670654, + "objective/train/theoretical_loss": 3.337939795481276, + "objective/train/tokens_used": 2806579200, + "objective/train/unlikelihood_loss": 0.14818251132965088, + "theoretical_loss": 3.337939795481276, + "tokens_seen": 2806579200 + }, + { + "epoch": 0.85, + "learning_rate": 7.560682046138416e-05, + "loss": 1.0123, + "theoretical_loss": 3.337933721279315, + "tokens_seen": 2806644736 + }, + { + "epoch": 0.85, + "learning_rate": 7.558676028084253e-05, + "loss": 1.1236, + "theoretical_loss": 3.33792157342002, + "tokens_seen": 2806775808 + }, + { + "epoch": 0.85, + "learning_rate": 7.556670010030091e-05, + "loss": 1.2986, + "theoretical_loss": 3.3379094262868287, + "tokens_seen": 2806906880 + }, + { + "epoch": 0.85, + "learning_rate": 7.554663991975928e-05, + "loss": 1.2168, + "theoretical_loss": 3.3378972798796647, + "tokens_seen": 2807037952 + }, + { + "epoch": 0.85, + "learning_rate": 7.552657973921766e-05, + "loss": 1.1439, + "theoretical_loss": 3.33788513419845, + "tokens_seen": 2807169024 + }, + { + "epoch": 0.85, + "learning_rate": 7.550651955867604e-05, + "loss": 1.1927, + "theoretical_loss": 3.337872989243108, + "tokens_seen": 2807300096 + }, + { + "epoch": 0.85, + "learning_rate": 7.548645937813441e-05, + "loss": 1.1861, + "theoretical_loss": 3.3378608450135605, + "tokens_seen": 2807431168 + }, + { + "epoch": 0.85, + "learning_rate": 7.546639919759279e-05, + "loss": 1.3433, + "theoretical_loss": 3.3378487015097313, + "tokens_seen": 2807562240 + }, + { + "epoch": 0.85, + "learning_rate": 7.544633901705115e-05, + "loss": 1.2192, + "theoretical_loss": 3.337836558731542, + "tokens_seen": 2807693312 + }, + { + "epoch": 0.85, + "learning_rate": 7.542627883650952e-05, + "loss": 1.1973, + "theoretical_loss": 3.3378244166789166, + "tokens_seen": 2807824384 + }, + { + "epoch": 0.85, + "learning_rate": 7.54062186559679e-05, + "loss": 1.2437, + "theoretical_loss": 3.3378122753517774, + "tokens_seen": 2807955456 + }, + { + "epoch": 0.85, + "learning_rate": 7.538615847542628e-05, + "loss": 1.289, + "theoretical_loss": 3.3378001347500468, + "tokens_seen": 2808086528 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.21682551503181458, + "objective/train/avg_token_score": 0.019244065508246422, + "objective/train/docs_used": 1580206, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7438857555389404, + "objective/train/original_loss": 4.072500228881836, + "objective/train/theoretical_loss": 3.337787994873648, + "objective/train/tokens_used": 2808217600, + "objective/train/unlikelihood_loss": 0.1915651112794876, + "theoretical_loss": 3.337787994873648, + "tokens_seen": 2808217600 + }, + { + "epoch": 0.85, + "learning_rate": 7.536609829488465e-05, + "loss": 1.2263, + "theoretical_loss": 3.337787994873648, + "tokens_seen": 2808217600 + }, + { + "epoch": 0.85, + "learning_rate": 7.534603811434303e-05, + "loss": 1.0744, + "theoretical_loss": 3.3377758557225032, + "tokens_seen": 2808348672 + }, + { + "epoch": 0.85, + "learning_rate": 7.53259779338014e-05, + "loss": 1.1075, + "theoretical_loss": 3.337763717296536, + "tokens_seen": 2808479744 + }, + { + "epoch": 0.85, + "learning_rate": 7.530591775325978e-05, + "loss": 1.2366, + "theoretical_loss": 3.337751579595669, + "tokens_seen": 2808610816 + }, + { + "epoch": 0.85, + "learning_rate": 7.528585757271816e-05, + "loss": 1.2474, + "theoretical_loss": 3.3377394426198252, + "tokens_seen": 2808741888 + }, + { + "epoch": 0.85, + "learning_rate": 7.526579739217652e-05, + "loss": 1.1242, + "theoretical_loss": 3.3377273063689277, + "tokens_seen": 2808872960 + }, + { + "epoch": 0.85, + "learning_rate": 7.524573721163491e-05, + "loss": 1.1595, + "theoretical_loss": 3.3377151708428983, + "tokens_seen": 2809004032 + }, + { + "epoch": 0.85, + "learning_rate": 7.522567703109328e-05, + "loss": 1.22, + "theoretical_loss": 3.33770303604166, + "tokens_seen": 2809135104 + }, + { + "epoch": 0.85, + "learning_rate": 7.520561685055166e-05, + "loss": 1.0777, + "theoretical_loss": 3.337690901965137, + "tokens_seen": 2809266176 + }, + { + "epoch": 0.85, + "learning_rate": 7.518555667001003e-05, + "loss": 1.1595, + "theoretical_loss": 3.3376787686132516, + "tokens_seen": 2809397248 + }, + { + "epoch": 0.85, + "learning_rate": 7.516549648946841e-05, + "loss": 1.3072, + "theoretical_loss": 3.337666635985926, + "tokens_seen": 2809528320 + }, + { + "epoch": 0.85, + "learning_rate": 7.514543630892679e-05, + "loss": 1.2684, + "theoretical_loss": 3.3376545040830843, + "tokens_seen": 2809659392 + }, + { + "epoch": 0.85, + "learning_rate": 7.512537612838516e-05, + "loss": 1.1767, + "theoretical_loss": 3.3376423729046483, + "tokens_seen": 2809790464 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.4002932608127594, + "objective/train/avg_token_score": 0.010252654552459717, + "objective/train/docs_used": 1581474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1945027112960815, + "objective/train/original_loss": 3.2994935512542725, + "objective/train/theoretical_loss": 3.3376363075870588, + "objective/train/tokens_used": 2809856000, + "objective/train/unlikelihood_loss": 0.2251259982585907, + "theoretical_loss": 3.3376363075870588, + "tokens_seen": 2809856000 + }, + { + "epoch": 0.85, + "learning_rate": 7.510531594784354e-05, + "loss": 1.1773, + "theoretical_loss": 3.3376302424505417, + "tokens_seen": 2809921536 + }, + { + "epoch": 0.85, + "learning_rate": 7.508525576730191e-05, + "loss": 1.2514, + "theoretical_loss": 3.337618112720687, + "tokens_seen": 2810052608 + }, + { + "epoch": 0.85, + "learning_rate": 7.506519558676028e-05, + "loss": 1.1973, + "theoretical_loss": 3.337605983715008, + "tokens_seen": 2810183680 + }, + { + "epoch": 0.85, + "learning_rate": 7.504513540621865e-05, + "loss": 1.1571, + "theoretical_loss": 3.337593855433427, + "tokens_seen": 2810314752 + }, + { + "epoch": 0.85, + "learning_rate": 7.502507522567703e-05, + "loss": 1.2268, + "theoretical_loss": 3.3375817278758664, + "tokens_seen": 2810445824 + }, + { + "epoch": 0.85, + "learning_rate": 7.50050150451354e-05, + "loss": 1.1722, + "theoretical_loss": 3.337569601042251, + "tokens_seen": 2810576896 + }, + { + "epoch": 0.85, + "learning_rate": 7.498495486459378e-05, + "loss": 1.2085, + "theoretical_loss": 3.3375574749325017, + "tokens_seen": 2810707968 + }, + { + "epoch": 0.85, + "learning_rate": 7.496489468405215e-05, + "loss": 1.259, + "theoretical_loss": 3.3375453495465432, + "tokens_seen": 2810839040 + }, + { + "epoch": 0.85, + "learning_rate": 7.494483450351053e-05, + "loss": 1.3317, + "theoretical_loss": 3.337533224884298, + "tokens_seen": 2810970112 + }, + { + "epoch": 0.85, + "learning_rate": 7.49247743229689e-05, + "loss": 1.1894, + "theoretical_loss": 3.337521100945689, + "tokens_seen": 2811101184 + }, + { + "epoch": 0.85, + "learning_rate": 7.490471414242728e-05, + "loss": 1.2915, + "theoretical_loss": 3.3375089777306397, + "tokens_seen": 2811232256 + }, + { + "epoch": 0.85, + "learning_rate": 7.488465396188567e-05, + "loss": 1.1668, + "theoretical_loss": 3.3374968552390722, + "tokens_seen": 2811363328 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.34310850501060486, + "objective/train/avg_token_score": 0.028889602050185204, + "objective/train/docs_used": 1582095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1121172904968262, + "objective/train/original_loss": 4.132122993469238, + "objective/train/theoretical_loss": 3.337484733470911, + "objective/train/tokens_used": 2811494400, + "objective/train/unlikelihood_loss": 0.16524726152420044, + "theoretical_loss": 3.337484733470911, + "tokens_seen": 2811494400 + }, + { + "epoch": 0.85, + "learning_rate": 7.486459378134403e-05, + "loss": 1.2458, + "theoretical_loss": 3.337484733470911, + "tokens_seen": 2811494400 + }, + { + "epoch": 0.85, + "learning_rate": 7.484453360080241e-05, + "loss": 1.1179, + "theoretical_loss": 3.337472612426078, + "tokens_seen": 2811625472 + }, + { + "epoch": 0.85, + "learning_rate": 7.482447342026079e-05, + "loss": 1.1274, + "theoretical_loss": 3.337460492104497, + "tokens_seen": 2811756544 + }, + { + "epoch": 0.85, + "learning_rate": 7.480441323971916e-05, + "loss": 1.3078, + "theoretical_loss": 3.337448372506091, + "tokens_seen": 2811887616 + }, + { + "epoch": 0.85, + "learning_rate": 7.478435305917754e-05, + "loss": 1.3548, + "theoretical_loss": 3.337436253630783, + "tokens_seen": 2812018688 + }, + { + "epoch": 0.85, + "learning_rate": 7.476429287863591e-05, + "loss": 1.3582, + "theoretical_loss": 3.3374241354784964, + "tokens_seen": 2812149760 + }, + { + "epoch": 0.85, + "learning_rate": 7.474423269809429e-05, + "loss": 1.309, + "theoretical_loss": 3.3374120180491538, + "tokens_seen": 2812280832 + }, + { + "epoch": 0.85, + "learning_rate": 7.472417251755266e-05, + "loss": 1.2267, + "theoretical_loss": 3.337399901342679, + "tokens_seen": 2812411904 + }, + { + "epoch": 0.85, + "learning_rate": 7.470411233701104e-05, + "loss": 1.2915, + "theoretical_loss": 3.3373877853589953, + "tokens_seen": 2812542976 + }, + { + "epoch": 0.85, + "learning_rate": 7.468405215646942e-05, + "loss": 1.1389, + "theoretical_loss": 3.337375670098025, + "tokens_seen": 2812674048 + }, + { + "epoch": 0.85, + "learning_rate": 7.466399197592778e-05, + "loss": 1.2138, + "theoretical_loss": 3.3373635555596923, + "tokens_seen": 2812805120 + }, + { + "epoch": 0.85, + "learning_rate": 7.464393179538615e-05, + "loss": 1.201, + "theoretical_loss": 3.3373514417439196, + "tokens_seen": 2812936192 + }, + { + "epoch": 0.85, + "learning_rate": 7.462387161484453e-05, + "loss": 1.2367, + "theoretical_loss": 3.337339328650631, + "tokens_seen": 2813067264 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.38703566789627075, + "objective/train/avg_token_score": 0.027719927951693535, + "objective/train/docs_used": 1583341, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2912611961364746, + "objective/train/original_loss": 3.7349629402160645, + "objective/train/theoretical_loss": 3.3373332723748943, + "objective/train/tokens_used": 2813132800, + "objective/train/unlikelihood_loss": 0.17921416461467743, + "theoretical_loss": 3.3373332723748943, + "tokens_seen": 2813132800 + }, + { + "epoch": 0.85, + "learning_rate": 7.46038114343029e-05, + "loss": 1.2741, + "theoretical_loss": 3.3373272162797494, + "tokens_seen": 2813198336 + }, + { + "epoch": 0.85, + "learning_rate": 7.458375125376128e-05, + "loss": 1.2043, + "theoretical_loss": 3.3373151046311977, + "tokens_seen": 2813329408 + }, + { + "epoch": 0.85, + "learning_rate": 7.456369107321966e-05, + "loss": 1.1663, + "theoretical_loss": 3.3373029937048995, + "tokens_seen": 2813460480 + }, + { + "epoch": 0.85, + "learning_rate": 7.454363089267803e-05, + "loss": 1.1561, + "theoretical_loss": 3.3372908835007777, + "tokens_seen": 2813591552 + }, + { + "epoch": 0.85, + "learning_rate": 7.452357071213641e-05, + "loss": 1.1433, + "theoretical_loss": 3.337278774018756, + "tokens_seen": 2813722624 + }, + { + "epoch": 0.85, + "learning_rate": 7.45035105315948e-05, + "loss": 1.3301, + "theoretical_loss": 3.3372666652587584, + "tokens_seen": 2813853696 + }, + { + "epoch": 0.85, + "learning_rate": 7.448345035105317e-05, + "loss": 1.3673, + "theoretical_loss": 3.337254557220707, + "tokens_seen": 2813984768 + }, + { + "epoch": 0.85, + "learning_rate": 7.446339017051154e-05, + "loss": 1.0857, + "theoretical_loss": 3.3372424499045255, + "tokens_seen": 2814115840 + }, + { + "epoch": 0.85, + "learning_rate": 7.444332998996991e-05, + "loss": 1.3313, + "theoretical_loss": 3.337230343310137, + "tokens_seen": 2814246912 + }, + { + "epoch": 0.85, + "learning_rate": 7.442326980942829e-05, + "loss": 1.1842, + "theoretical_loss": 3.3372182374374657, + "tokens_seen": 2814377984 + }, + { + "epoch": 0.85, + "learning_rate": 7.440320962888666e-05, + "loss": 1.2849, + "theoretical_loss": 3.3372061322864344, + "tokens_seen": 2814509056 + }, + { + "epoch": 0.85, + "learning_rate": 7.438314944834504e-05, + "loss": 1.2315, + "theoretical_loss": 3.3371940278569663, + "tokens_seen": 2814640128 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.5045210123062134, + "objective/train/avg_token_score": 0.01266464777290821, + "objective/train/docs_used": 1583792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5312663316726685, + "objective/train/original_loss": 3.347247362136841, + "objective/train/theoretical_loss": 3.337181924148985, + "objective/train/tokens_used": 2814771200, + "objective/train/unlikelihood_loss": 0.152060404419899, + "theoretical_loss": 3.337181924148985, + "tokens_seen": 2814771200 + }, + { + "epoch": 0.85, + "learning_rate": 7.436308926780341e-05, + "loss": 1.2481, + "theoretical_loss": 3.337181924148985, + "tokens_seen": 2814771200 + }, + { + "epoch": 0.85, + "learning_rate": 7.434302908726179e-05, + "loss": 1.2304, + "theoretical_loss": 3.3371698211624143, + "tokens_seen": 2814902272 + }, + { + "epoch": 0.85, + "learning_rate": 7.432296890672017e-05, + "loss": 1.213, + "theoretical_loss": 3.337157718897177, + "tokens_seen": 2815033344 + }, + { + "epoch": 0.85, + "learning_rate": 7.430290872617854e-05, + "loss": 1.3442, + "theoretical_loss": 3.3371456173531966, + "tokens_seen": 2815164416 + }, + { + "epoch": 0.85, + "learning_rate": 7.42828485456369e-05, + "loss": 1.3318, + "theoretical_loss": 3.337133516530397, + "tokens_seen": 2815295488 + }, + { + "epoch": 0.85, + "learning_rate": 7.426278836509528e-05, + "loss": 1.2181, + "theoretical_loss": 3.337121416428701, + "tokens_seen": 2815426560 + }, + { + "epoch": 0.85, + "learning_rate": 7.424272818455366e-05, + "loss": 1.2042, + "theoretical_loss": 3.3371093170480326, + "tokens_seen": 2815557632 + }, + { + "epoch": 0.85, + "learning_rate": 7.422266800401203e-05, + "loss": 1.1281, + "theoretical_loss": 3.337097218388315, + "tokens_seen": 2815688704 + }, + { + "epoch": 0.85, + "learning_rate": 7.420260782347041e-05, + "loss": 1.1908, + "theoretical_loss": 3.337085120449472, + "tokens_seen": 2815819776 + }, + { + "epoch": 0.85, + "learning_rate": 7.418254764292878e-05, + "loss": 1.1901, + "theoretical_loss": 3.3370730232314267, + "tokens_seen": 2815950848 + }, + { + "epoch": 0.85, + "learning_rate": 7.416248746238716e-05, + "loss": 1.2865, + "theoretical_loss": 3.3370609267341025, + "tokens_seen": 2816081920 + }, + { + "epoch": 0.85, + "learning_rate": 7.414242728184553e-05, + "loss": 1.2895, + "theoretical_loss": 3.3370488309574236, + "tokens_seen": 2816212992 + }, + { + "epoch": 0.85, + "learning_rate": 7.412236710130392e-05, + "loss": 1.3377, + "theoretical_loss": 3.337036735901313, + "tokens_seen": 2816344064 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.32080888748168945, + "objective/train/avg_token_score": 0.027940059080719948, + "objective/train/docs_used": 1585248, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0242851972579956, + "objective/train/original_loss": 4.537588119506836, + "objective/train/theoretical_loss": 3.337030688643447, + "objective/train/tokens_used": 2816409600, + "objective/train/unlikelihood_loss": 0.1206049770116806, + "theoretical_loss": 3.337030688643447, + "tokens_seen": 2816409600 + }, + { + "epoch": 0.85, + "learning_rate": 7.41023069207623e-05, + "loss": 1.1872, + "theoretical_loss": 3.3370246415656943, + "tokens_seen": 2816475136 + }, + { + "epoch": 0.85, + "learning_rate": 7.408224674022066e-05, + "loss": 1.257, + "theoretical_loss": 3.3370125479504913, + "tokens_seen": 2816606208 + }, + { + "epoch": 0.85, + "learning_rate": 7.406218655967904e-05, + "loss": 1.2577, + "theoretical_loss": 3.337000455055627, + "tokens_seen": 2816737280 + }, + { + "epoch": 0.85, + "learning_rate": 7.404212637913741e-05, + "loss": 1.2217, + "theoretical_loss": 3.3369883628810255, + "tokens_seen": 2816868352 + }, + { + "epoch": 0.85, + "learning_rate": 7.402206619859579e-05, + "loss": 1.1748, + "theoretical_loss": 3.3369762714266105, + "tokens_seen": 2816999424 + }, + { + "epoch": 0.85, + "learning_rate": 7.400200601805417e-05, + "loss": 1.1237, + "theoretical_loss": 3.336964180692305, + "tokens_seen": 2817130496 + }, + { + "epoch": 0.85, + "learning_rate": 7.398194583751254e-05, + "loss": 1.2082, + "theoretical_loss": 3.3369520906780332, + "tokens_seen": 2817261568 + }, + { + "epoch": 0.85, + "learning_rate": 7.396188565697092e-05, + "loss": 1.1031, + "theoretical_loss": 3.3369400013837183, + "tokens_seen": 2817392640 + }, + { + "epoch": 0.85, + "learning_rate": 7.394182547642929e-05, + "loss": 1.2762, + "theoretical_loss": 3.3369279128092844, + "tokens_seen": 2817523712 + }, + { + "epoch": 0.85, + "learning_rate": 7.392176529588767e-05, + "loss": 1.3051, + "theoretical_loss": 3.3369158249546547, + "tokens_seen": 2817654784 + }, + { + "epoch": 0.85, + "learning_rate": 7.390170511534604e-05, + "loss": 1.0803, + "theoretical_loss": 3.336903737819753, + "tokens_seen": 2817785856 + }, + { + "epoch": 0.85, + "learning_rate": 7.38816449348044e-05, + "loss": 1.3281, + "theoretical_loss": 3.336891651404503, + "tokens_seen": 2817916928 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.4583333432674408, + "objective/train/avg_token_score": 0.01124577596783638, + "objective/train/docs_used": 1585920, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4360170364379883, + "objective/train/original_loss": 3.8255817890167236, + "objective/train/theoretical_loss": 3.3368795657088284, + "objective/train/tokens_used": 2818048000, + "objective/train/unlikelihood_loss": 0.19564400613307953, + "theoretical_loss": 3.3368795657088284, + "tokens_seen": 2818048000 + }, + { + "epoch": 0.85, + "learning_rate": 7.386158475426278e-05, + "loss": 1.2195, + "theoretical_loss": 3.3368795657088284, + "tokens_seen": 2818048000 + }, + { + "epoch": 0.85, + "learning_rate": 7.384152457372116e-05, + "loss": 1.2508, + "theoretical_loss": 3.3368674807326526, + "tokens_seen": 2818179072 + }, + { + "epoch": 0.85, + "learning_rate": 7.382146439317953e-05, + "loss": 1.0631, + "theoretical_loss": 3.3368553964759, + "tokens_seen": 2818310144 + }, + { + "epoch": 0.85, + "learning_rate": 7.380140421263791e-05, + "loss": 1.2348, + "theoretical_loss": 3.336843312938494, + "tokens_seen": 2818441216 + }, + { + "epoch": 0.85, + "learning_rate": 7.378134403209629e-05, + "loss": 1.234, + "theoretical_loss": 3.3368312301203575, + "tokens_seen": 2818572288 + }, + { + "epoch": 0.85, + "learning_rate": 7.376128385155466e-05, + "loss": 1.1385, + "theoretical_loss": 3.3368191480214153, + "tokens_seen": 2818703360 + }, + { + "epoch": 0.85, + "learning_rate": 7.374122367101305e-05, + "loss": 1.1782, + "theoretical_loss": 3.336807066641591, + "tokens_seen": 2818834432 + }, + { + "epoch": 0.85, + "learning_rate": 7.372116349047143e-05, + "loss": 1.2343, + "theoretical_loss": 3.3367949859808084, + "tokens_seen": 2818965504 + }, + { + "epoch": 0.85, + "learning_rate": 7.37011033099298e-05, + "loss": 1.3202, + "theoretical_loss": 3.3367829060389904, + "tokens_seen": 2819096576 + }, + { + "epoch": 0.85, + "learning_rate": 7.368104312938816e-05, + "loss": 1.338, + "theoretical_loss": 3.336770826816062, + "tokens_seen": 2819227648 + }, + { + "epoch": 0.85, + "learning_rate": 7.366098294884654e-05, + "loss": 1.2503, + "theoretical_loss": 3.3367587483119463, + "tokens_seen": 2819358720 + }, + { + "epoch": 0.85, + "learning_rate": 7.364092276830492e-05, + "loss": 1.3074, + "theoretical_loss": 3.336746670526567, + "tokens_seen": 2819489792 + }, + { + "epoch": 0.85, + "learning_rate": 7.362086258776329e-05, + "loss": 1.4311, + "theoretical_loss": 3.3367345934598482, + "tokens_seen": 2819620864 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.37805473804473877, + "objective/train/avg_token_score": 0.009625033475458622, + "objective/train/docs_used": 1586975, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1623295545578003, + "objective/train/original_loss": 3.799468517303467, + "objective/train/theoretical_loss": 3.336728555195963, + "objective/train/tokens_used": 2819686400, + "objective/train/unlikelihood_loss": 0.172540545463562, + "theoretical_loss": 3.336728555195963, + "tokens_seen": 2819686400 + }, + { + "epoch": 0.85, + "learning_rate": 7.360080240722167e-05, + "loss": 1.177, + "theoretical_loss": 3.336722517111714, + "tokens_seen": 2819751936 + }, + { + "epoch": 0.85, + "learning_rate": 7.358074222668004e-05, + "loss": 1.2277, + "theoretical_loss": 3.3367104414820874, + "tokens_seen": 2819883008 + }, + { + "epoch": 0.85, + "learning_rate": 7.356068204613842e-05, + "loss": 1.2344, + "theoretical_loss": 3.3366983665708934, + "tokens_seen": 2820014080 + }, + { + "epoch": 0.85, + "learning_rate": 7.35406218655968e-05, + "loss": 1.2826, + "theoretical_loss": 3.336686292378055, + "tokens_seen": 2820145152 + }, + { + "epoch": 0.85, + "learning_rate": 7.352056168505517e-05, + "loss": 1.1677, + "theoretical_loss": 3.3366742189034966, + "tokens_seen": 2820276224 + }, + { + "epoch": 0.85, + "learning_rate": 7.350050150451353e-05, + "loss": 1.2858, + "theoretical_loss": 3.3366621461471415, + "tokens_seen": 2820407296 + }, + { + "epoch": 0.85, + "learning_rate": 7.348044132397191e-05, + "loss": 1.1143, + "theoretical_loss": 3.3366500741089142, + "tokens_seen": 2820538368 + }, + { + "epoch": 0.85, + "learning_rate": 7.346038114343028e-05, + "loss": 1.3722, + "theoretical_loss": 3.336638002788738, + "tokens_seen": 2820669440 + }, + { + "epoch": 0.85, + "learning_rate": 7.344032096288866e-05, + "loss": 1.2643, + "theoretical_loss": 3.3366259321865375, + "tokens_seen": 2820800512 + }, + { + "epoch": 0.85, + "learning_rate": 7.342026078234704e-05, + "loss": 1.2742, + "theoretical_loss": 3.336613862302236, + "tokens_seen": 2820931584 + }, + { + "epoch": 0.85, + "learning_rate": 7.340020060180541e-05, + "loss": 1.2475, + "theoretical_loss": 3.3366017931357583, + "tokens_seen": 2821062656 + }, + { + "epoch": 0.85, + "learning_rate": 7.33801404212638e-05, + "loss": 1.2777, + "theoretical_loss": 3.3365897246870277, + "tokens_seen": 2821193728 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.3772605061531067, + "objective/train/avg_token_score": 0.010240985080599785, + "objective/train/docs_used": 1587530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2139612436294556, + "objective/train/original_loss": 3.578423023223877, + "objective/train/theoretical_loss": 3.336577656955968, + "objective/train/tokens_used": 2821324800, + "objective/train/unlikelihood_loss": 0.21624819934368134, + "theoretical_loss": 3.336577656955968, + "tokens_seen": 2821324800 + }, + { + "epoch": 0.85, + "learning_rate": 7.336008024072218e-05, + "loss": 1.2233, + "theoretical_loss": 3.336577656955968, + "tokens_seen": 2821324800 + }, + { + "epoch": 0.85, + "learning_rate": 7.334002006018055e-05, + "loss": 1.0934, + "theoretical_loss": 3.336565589942504, + "tokens_seen": 2821455872 + }, + { + "epoch": 0.86, + "learning_rate": 7.331995987963893e-05, + "loss": 1.2607, + "theoretical_loss": 3.3365535236465584, + "tokens_seen": 2821586944 + }, + { + "epoch": 0.86, + "learning_rate": 7.329989969909729e-05, + "loss": 1.1654, + "theoretical_loss": 3.336541458068057, + "tokens_seen": 2821718016 + }, + { + "epoch": 0.86, + "learning_rate": 7.327983951855567e-05, + "loss": 1.313, + "theoretical_loss": 3.3365293932069218, + "tokens_seen": 2821849088 + }, + { + "epoch": 0.86, + "learning_rate": 7.325977933801404e-05, + "loss": 1.1697, + "theoretical_loss": 3.3365173290630783, + "tokens_seen": 2821980160 + }, + { + "epoch": 0.86, + "learning_rate": 7.323971915747242e-05, + "loss": 1.3975, + "theoretical_loss": 3.3365052656364504, + "tokens_seen": 2822111232 + }, + { + "epoch": 0.86, + "learning_rate": 7.32196589769308e-05, + "loss": 1.317, + "theoretical_loss": 3.3364932029269614, + "tokens_seen": 2822242304 + }, + { + "epoch": 0.86, + "learning_rate": 7.319959879638917e-05, + "loss": 1.322, + "theoretical_loss": 3.336481140934536, + "tokens_seen": 2822373376 + }, + { + "epoch": 0.86, + "learning_rate": 7.317953861584755e-05, + "loss": 1.2132, + "theoretical_loss": 3.336469079659098, + "tokens_seen": 2822504448 + }, + { + "epoch": 0.86, + "learning_rate": 7.315947843530592e-05, + "loss": 1.2984, + "theoretical_loss": 3.3364570191005716, + "tokens_seen": 2822635520 + }, + { + "epoch": 0.86, + "learning_rate": 7.31394182547643e-05, + "loss": 1.2425, + "theoretical_loss": 3.3364449592588805, + "tokens_seen": 2822766592 + }, + { + "epoch": 0.86, + "learning_rate": 7.311935807422267e-05, + "loss": 1.32, + "theoretical_loss": 3.33643290013395, + "tokens_seen": 2822897664 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.4258309006690979, + "objective/train/avg_token_score": 0.018132364377379417, + "objective/train/docs_used": 1588829, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3766655921936035, + "objective/train/original_loss": 4.019596099853516, + "objective/train/theoretical_loss": 3.3364268708402456, + "objective/train/tokens_used": 2822963200, + "objective/train/unlikelihood_loss": 0.14318612217903137, + "theoretical_loss": 3.3364268708402456, + "tokens_seen": 2822963200 + }, + { + "epoch": 0.86, + "learning_rate": 7.309929789368104e-05, + "loss": 1.3068, + "theoretical_loss": 3.336420841725703, + "tokens_seen": 2823028736 + }, + { + "epoch": 0.86, + "learning_rate": 7.307923771313941e-05, + "loss": 1.2818, + "theoretical_loss": 3.336408784034064, + "tokens_seen": 2823159808 + }, + { + "epoch": 0.86, + "learning_rate": 7.305917753259779e-05, + "loss": 1.3614, + "theoretical_loss": 3.3363967270589567, + "tokens_seen": 2823290880 + }, + { + "epoch": 0.86, + "learning_rate": 7.303911735205616e-05, + "loss": 1.2895, + "theoretical_loss": 3.336384670800306, + "tokens_seen": 2823421952 + }, + { + "epoch": 0.86, + "learning_rate": 7.301905717151454e-05, + "loss": 1.2343, + "theoretical_loss": 3.3363726152580364, + "tokens_seen": 2823553024 + }, + { + "epoch": 0.86, + "learning_rate": 7.299899699097293e-05, + "loss": 1.2866, + "theoretical_loss": 3.336360560432071, + "tokens_seen": 2823684096 + }, + { + "epoch": 0.86, + "learning_rate": 7.29789368104313e-05, + "loss": 1.1608, + "theoretical_loss": 3.336348506322335, + "tokens_seen": 2823815168 + }, + { + "epoch": 0.86, + "learning_rate": 7.295887662988968e-05, + "loss": 1.2966, + "theoretical_loss": 3.3363364529287516, + "tokens_seen": 2823946240 + }, + { + "epoch": 0.86, + "learning_rate": 7.293881644934806e-05, + "loss": 1.269, + "theoretical_loss": 3.3363244002512453, + "tokens_seen": 2824077312 + }, + { + "epoch": 0.86, + "learning_rate": 7.291875626880642e-05, + "loss": 1.2995, + "theoretical_loss": 3.3363123482897405, + "tokens_seen": 2824208384 + }, + { + "epoch": 0.86, + "learning_rate": 7.28986960882648e-05, + "loss": 1.3093, + "theoretical_loss": 3.336300297044162, + "tokens_seen": 2824339456 + }, + { + "epoch": 0.86, + "learning_rate": 7.287863590772317e-05, + "loss": 1.2086, + "theoretical_loss": 3.3362882465144335, + "tokens_seen": 2824470528 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.44183772802352905, + "objective/train/avg_token_score": 0.012311633676290512, + "objective/train/docs_used": 1589488, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4225103855133057, + "objective/train/original_loss": 3.5990383625030518, + "objective/train/theoretical_loss": 3.336276196700479, + "objective/train/tokens_used": 2824601600, + "objective/train/unlikelihood_loss": 0.1816691756248474, + "theoretical_loss": 3.336276196700479, + "tokens_seen": 2824601600 + }, + { + "epoch": 0.86, + "learning_rate": 7.285857572718154e-05, + "loss": 1.107, + "theoretical_loss": 3.336276196700479, + "tokens_seen": 2824601600 + }, + { + "epoch": 0.86, + "learning_rate": 7.283851554663992e-05, + "loss": 1.119, + "theoretical_loss": 3.3362641476022232, + "tokens_seen": 2824732672 + }, + { + "epoch": 0.86, + "learning_rate": 7.28184553660983e-05, + "loss": 1.2026, + "theoretical_loss": 3.33625209921959, + "tokens_seen": 2824863744 + }, + { + "epoch": 0.86, + "learning_rate": 7.279839518555667e-05, + "loss": 1.2609, + "theoretical_loss": 3.336240051552504, + "tokens_seen": 2824994816 + }, + { + "epoch": 0.86, + "learning_rate": 7.277833500501505e-05, + "loss": 1.1801, + "theoretical_loss": 3.3362280046008896, + "tokens_seen": 2825125888 + }, + { + "epoch": 0.86, + "learning_rate": 7.275827482447342e-05, + "loss": 1.2518, + "theoretical_loss": 3.3362159583646713, + "tokens_seen": 2825256960 + }, + { + "epoch": 0.86, + "learning_rate": 7.27382146439318e-05, + "loss": 1.3634, + "theoretical_loss": 3.3362039128437724, + "tokens_seen": 2825388032 + }, + { + "epoch": 0.86, + "learning_rate": 7.271815446339016e-05, + "loss": 1.2422, + "theoretical_loss": 3.336191868038118, + "tokens_seen": 2825519104 + }, + { + "epoch": 0.86, + "learning_rate": 7.269809428284854e-05, + "loss": 1.2513, + "theoretical_loss": 3.336179823947633, + "tokens_seen": 2825650176 + }, + { + "epoch": 0.86, + "learning_rate": 7.267803410230691e-05, + "loss": 1.1977, + "theoretical_loss": 3.3361677805722407, + "tokens_seen": 2825781248 + }, + { + "epoch": 0.86, + "learning_rate": 7.265797392176529e-05, + "loss": 1.1191, + "theoretical_loss": 3.336155737911866, + "tokens_seen": 2825912320 + }, + { + "epoch": 0.86, + "learning_rate": 7.263791374122367e-05, + "loss": 1.2142, + "theoretical_loss": 3.3361436959664332, + "tokens_seen": 2826043392 + }, + { + "epoch": 0.86, + "learning_rate": 7.261785356068205e-05, + "loss": 1.2811, + "theoretical_loss": 3.336131654735867, + "tokens_seen": 2826174464 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.43713343143463135, + "objective/train/avg_token_score": 0.012129022739827633, + "objective/train/docs_used": 1590914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.292755365371704, + "objective/train/original_loss": 3.778115749359131, + "objective/train/theoretical_loss": 3.336125634388635, + "objective/train/tokens_used": 2826240000, + "objective/train/unlikelihood_loss": 0.1617189198732376, + "theoretical_loss": 3.336125634388635, + "tokens_seen": 2826240000 + }, + { + "epoch": 0.86, + "learning_rate": 7.259779338014043e-05, + "loss": 1.2679, + "theoretical_loss": 3.336119614220091, + "tokens_seen": 2826305536 + }, + { + "epoch": 0.86, + "learning_rate": 7.25777331995988e-05, + "loss": 1.263, + "theoretical_loss": 3.3361075744190307, + "tokens_seen": 2826436608 + }, + { + "epoch": 0.86, + "learning_rate": 7.255767301905718e-05, + "loss": 1.0862, + "theoretical_loss": 3.3360955353326096, + "tokens_seen": 2826567680 + }, + { + "epoch": 0.86, + "learning_rate": 7.253761283851556e-05, + "loss": 1.122, + "theoretical_loss": 3.3360834969607525, + "tokens_seen": 2826698752 + }, + { + "epoch": 0.86, + "learning_rate": 7.251755265797392e-05, + "loss": 1.2989, + "theoretical_loss": 3.3360714593033842, + "tokens_seen": 2826829824 + }, + { + "epoch": 0.86, + "learning_rate": 7.24974924774323e-05, + "loss": 1.3151, + "theoretical_loss": 3.3360594223604285, + "tokens_seen": 2826960896 + }, + { + "epoch": 0.86, + "learning_rate": 7.247743229689067e-05, + "loss": 1.0819, + "theoretical_loss": 3.3360473861318103, + "tokens_seen": 2827091968 + }, + { + "epoch": 0.86, + "learning_rate": 7.245737211634905e-05, + "loss": 1.1523, + "theoretical_loss": 3.336035350617454, + "tokens_seen": 2827223040 + }, + { + "epoch": 0.86, + "learning_rate": 7.243731193580742e-05, + "loss": 1.2226, + "theoretical_loss": 3.3360233158172843, + "tokens_seen": 2827354112 + }, + { + "epoch": 0.86, + "learning_rate": 7.24172517552658e-05, + "loss": 1.3342, + "theoretical_loss": 3.3360112817312255, + "tokens_seen": 2827485184 + }, + { + "epoch": 0.86, + "learning_rate": 7.239719157472417e-05, + "loss": 1.2471, + "theoretical_loss": 3.3359992483592023, + "tokens_seen": 2827616256 + }, + { + "epoch": 0.86, + "learning_rate": 7.237713139418255e-05, + "loss": 1.0536, + "theoretical_loss": 3.335987215701139, + "tokens_seen": 2827747328 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.41373410820961, + "objective/train/avg_token_score": 0.021237870678305626, + "objective/train/docs_used": 1591402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3565528392791748, + "objective/train/original_loss": 4.168304920196533, + "objective/train/theoretical_loss": 3.3359751837569602, + "objective/train/tokens_used": 2827878400, + "objective/train/unlikelihood_loss": 0.1408301442861557, + "theoretical_loss": 3.3359751837569602, + "tokens_seen": 2827878400 + }, + { + "epoch": 0.86, + "learning_rate": 7.235707121364093e-05, + "loss": 1.3602, + "theoretical_loss": 3.3359751837569602, + "tokens_seen": 2827878400 + }, + { + "epoch": 0.86, + "learning_rate": 7.23370110330993e-05, + "loss": 1.1903, + "theoretical_loss": 3.3359631525265905, + "tokens_seen": 2828009472 + }, + { + "epoch": 0.86, + "learning_rate": 7.231695085255766e-05, + "loss": 1.2483, + "theoretical_loss": 3.3359511220099547, + "tokens_seen": 2828140544 + }, + { + "epoch": 0.86, + "learning_rate": 7.229689067201604e-05, + "loss": 1.1963, + "theoretical_loss": 3.335939092206977, + "tokens_seen": 2828271616 + }, + { + "epoch": 0.86, + "learning_rate": 7.227683049147442e-05, + "loss": 1.3664, + "theoretical_loss": 3.335927063117582, + "tokens_seen": 2828402688 + }, + { + "epoch": 0.86, + "learning_rate": 7.22567703109328e-05, + "loss": 1.2324, + "theoretical_loss": 3.335915034741695, + "tokens_seen": 2828533760 + }, + { + "epoch": 0.86, + "learning_rate": 7.223671013039118e-05, + "loss": 1.2285, + "theoretical_loss": 3.3359030070792395, + "tokens_seen": 2828664832 + }, + { + "epoch": 0.86, + "learning_rate": 7.221664994984956e-05, + "loss": 1.3077, + "theoretical_loss": 3.335890980130141, + "tokens_seen": 2828795904 + }, + { + "epoch": 0.86, + "learning_rate": 7.219658976930793e-05, + "loss": 1.29, + "theoretical_loss": 3.3358789538943237, + "tokens_seen": 2828926976 + }, + { + "epoch": 0.86, + "learning_rate": 7.217652958876631e-05, + "loss": 1.1668, + "theoretical_loss": 3.3358669283717126, + "tokens_seen": 2829058048 + }, + { + "epoch": 0.86, + "learning_rate": 7.215646940822468e-05, + "loss": 1.1884, + "theoretical_loss": 3.3358549035622325, + "tokens_seen": 2829189120 + }, + { + "epoch": 0.86, + "learning_rate": 7.213640922768305e-05, + "loss": 1.1885, + "theoretical_loss": 3.3358428794658073, + "tokens_seen": 2829320192 + }, + { + "epoch": 0.86, + "learning_rate": 7.211634904714142e-05, + "loss": 1.2916, + "theoretical_loss": 3.335830856082362, + "tokens_seen": 2829451264 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.3159213066101074, + "objective/train/avg_token_score": 0.009660233743488789, + "objective/train/docs_used": 1592635, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0792804956436157, + "objective/train/original_loss": 3.8665072917938232, + "objective/train/theoretical_loss": 3.3358248446579837, + "objective/train/tokens_used": 2829516800, + "objective/train/unlikelihood_loss": 0.18616065382957458, + "theoretical_loss": 3.3358248446579837, + "tokens_seen": 2829516800 + }, + { + "epoch": 0.86, + "learning_rate": 7.20962888665998e-05, + "loss": 1.224, + "theoretical_loss": 3.335818833411822, + "tokens_seen": 2829582336 + }, + { + "epoch": 0.86, + "learning_rate": 7.207622868605817e-05, + "loss": 1.2224, + "theoretical_loss": 3.335806811454111, + "tokens_seen": 2829713408 + }, + { + "epoch": 0.86, + "learning_rate": 7.205616850551655e-05, + "loss": 1.2821, + "theoretical_loss": 3.335794790209154, + "tokens_seen": 2829844480 + }, + { + "epoch": 0.86, + "learning_rate": 7.203610832497493e-05, + "loss": 1.4111, + "theoretical_loss": 3.335782769676876, + "tokens_seen": 2829975552 + }, + { + "epoch": 0.86, + "learning_rate": 7.20160481444333e-05, + "loss": 1.2835, + "theoretical_loss": 3.3357707498572022, + "tokens_seen": 2830106624 + }, + { + "epoch": 0.86, + "learning_rate": 7.199598796389168e-05, + "loss": 1.1554, + "theoretical_loss": 3.3357587307500562, + "tokens_seen": 2830237696 + }, + { + "epoch": 0.86, + "learning_rate": 7.197592778335005e-05, + "loss": 1.2359, + "theoretical_loss": 3.3357467123553635, + "tokens_seen": 2830368768 + }, + { + "epoch": 0.86, + "learning_rate": 7.195586760280843e-05, + "loss": 1.2098, + "theoretical_loss": 3.335734694673049, + "tokens_seen": 2830499840 + }, + { + "epoch": 0.86, + "learning_rate": 7.193580742226679e-05, + "loss": 1.3525, + "theoretical_loss": 3.335722677703037, + "tokens_seen": 2830630912 + }, + { + "epoch": 0.86, + "learning_rate": 7.191574724172517e-05, + "loss": 1.1515, + "theoretical_loss": 3.3357106614452525, + "tokens_seen": 2830761984 + }, + { + "epoch": 0.86, + "learning_rate": 7.189568706118354e-05, + "loss": 1.2817, + "theoretical_loss": 3.3356986458996207, + "tokens_seen": 2830893056 + }, + { + "epoch": 0.86, + "learning_rate": 7.187562688064193e-05, + "loss": 1.3025, + "theoretical_loss": 3.3356866310660656, + "tokens_seen": 2831024128 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.3095674514770508, + "objective/train/avg_token_score": 0.033163972198963165, + "objective/train/docs_used": 1593167, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8527742624282837, + "objective/train/original_loss": 4.027623176574707, + "objective/train/theoretical_loss": 3.335674616944513, + "objective/train/tokens_used": 2831155200, + "objective/train/unlikelihood_loss": 0.17942959070205688, + "theoretical_loss": 3.335674616944513, + "tokens_seen": 2831155200 + }, + { + "epoch": 0.86, + "learning_rate": 7.185556670010031e-05, + "loss": 1.2585, + "theoretical_loss": 3.335674616944513, + "tokens_seen": 2831155200 + }, + { + "epoch": 0.86, + "learning_rate": 7.183550651955868e-05, + "loss": 1.2223, + "theoretical_loss": 3.3356626035348866, + "tokens_seen": 2831286272 + }, + { + "epoch": 0.86, + "learning_rate": 7.181544633901706e-05, + "loss": 1.3632, + "theoretical_loss": 3.3356505908371124, + "tokens_seen": 2831417344 + }, + { + "epoch": 0.86, + "learning_rate": 7.179538615847543e-05, + "loss": 1.3008, + "theoretical_loss": 3.3356385788511145, + "tokens_seen": 2831548416 + }, + { + "epoch": 0.86, + "learning_rate": 7.177532597793381e-05, + "loss": 1.317, + "theoretical_loss": 3.335626567576818, + "tokens_seen": 2831679488 + }, + { + "epoch": 0.86, + "learning_rate": 7.175526579739219e-05, + "loss": 1.3097, + "theoretical_loss": 3.335614557014148, + "tokens_seen": 2831810560 + }, + { + "epoch": 0.86, + "learning_rate": 7.173520561685055e-05, + "loss": 1.3604, + "theoretical_loss": 3.3356025471630297, + "tokens_seen": 2831941632 + }, + { + "epoch": 0.86, + "learning_rate": 7.171514543630892e-05, + "loss": 1.1707, + "theoretical_loss": 3.335590538023387, + "tokens_seen": 2832072704 + }, + { + "epoch": 0.86, + "learning_rate": 7.16950852557673e-05, + "loss": 1.1256, + "theoretical_loss": 3.3355785295951454, + "tokens_seen": 2832203776 + }, + { + "epoch": 0.86, + "learning_rate": 7.167502507522568e-05, + "loss": 1.4685, + "theoretical_loss": 3.3355665218782304, + "tokens_seen": 2832334848 + }, + { + "epoch": 0.86, + "learning_rate": 7.165496489468405e-05, + "loss": 1.3855, + "theoretical_loss": 3.335554514872566, + "tokens_seen": 2832465920 + }, + { + "epoch": 0.86, + "learning_rate": 7.163490471414243e-05, + "loss": 1.1837, + "theoretical_loss": 3.3355425085780777, + "tokens_seen": 2832596992 + }, + { + "epoch": 0.86, + "learning_rate": 7.16148445336008e-05, + "loss": 1.2261, + "theoretical_loss": 3.33553050299469, + "tokens_seen": 2832728064 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.3640640377998352, + "objective/train/avg_token_score": 0.009606300853192806, + "objective/train/docs_used": 1594447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0494271516799927, + "objective/train/original_loss": 3.7360522747039795, + "objective/train/theoretical_loss": 3.3355245004696354, + "objective/train/tokens_used": 2832793600, + "objective/train/unlikelihood_loss": 0.1711009442806244, + "theoretical_loss": 3.3355245004696354, + "tokens_seen": 2832793600 + }, + { + "epoch": 0.86, + "learning_rate": 7.159478435305918e-05, + "loss": 1.308, + "theoretical_loss": 3.3355184981223283, + "tokens_seen": 2832859136 + }, + { + "epoch": 0.86, + "learning_rate": 7.157472417251756e-05, + "loss": 1.2534, + "theoretical_loss": 3.335506493960917, + "tokens_seen": 2832990208 + }, + { + "epoch": 0.86, + "learning_rate": 7.155466399197592e-05, + "loss": 1.383, + "theoretical_loss": 3.3354944905103823, + "tokens_seen": 2833121280 + }, + { + "epoch": 0.86, + "learning_rate": 7.153460381143429e-05, + "loss": 1.2552, + "theoretical_loss": 3.3354824877706486, + "tokens_seen": 2833252352 + }, + { + "epoch": 0.86, + "learning_rate": 7.151454363089267e-05, + "loss": 1.3019, + "theoretical_loss": 3.3354704857416406, + "tokens_seen": 2833383424 + }, + { + "epoch": 0.86, + "learning_rate": 7.149448345035106e-05, + "loss": 1.1107, + "theoretical_loss": 3.3354584844232833, + "tokens_seen": 2833514496 + }, + { + "epoch": 0.86, + "learning_rate": 7.147442326980943e-05, + "loss": 1.3534, + "theoretical_loss": 3.3354464838155025, + "tokens_seen": 2833645568 + }, + { + "epoch": 0.86, + "learning_rate": 7.145436308926781e-05, + "loss": 1.3663, + "theoretical_loss": 3.3354344839182226, + "tokens_seen": 2833776640 + }, + { + "epoch": 0.86, + "learning_rate": 7.143430290872619e-05, + "loss": 1.4675, + "theoretical_loss": 3.3354224847313687, + "tokens_seen": 2833907712 + }, + { + "epoch": 0.86, + "learning_rate": 7.141424272818456e-05, + "loss": 1.1707, + "theoretical_loss": 3.335410486254866, + "tokens_seen": 2834038784 + }, + { + "epoch": 0.86, + "learning_rate": 7.139418254764294e-05, + "loss": 1.2624, + "theoretical_loss": 3.3353984884886403, + "tokens_seen": 2834169856 + }, + { + "epoch": 0.86, + "learning_rate": 7.137412236710131e-05, + "loss": 1.223, + "theoretical_loss": 3.3353864914326152, + "tokens_seen": 2834300928 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.43890517950057983, + "objective/train/avg_token_score": 0.008698572404682636, + "objective/train/docs_used": 1595013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1514116525650024, + "objective/train/original_loss": 3.3446478843688965, + "objective/train/theoretical_loss": 3.3353744950867172, + "objective/train/tokens_used": 2834432000, + "objective/train/unlikelihood_loss": 0.19200152158737183, + "theoretical_loss": 3.3353744950867172, + "tokens_seen": 2834432000 + }, + { + "epoch": 0.86, + "learning_rate": 7.135406218655968e-05, + "loss": 1.2343, + "theoretical_loss": 3.3353744950867172, + "tokens_seen": 2834432000 + }, + { + "epoch": 0.86, + "learning_rate": 7.133400200601805e-05, + "loss": 1.3305, + "theoretical_loss": 3.335362499450871, + "tokens_seen": 2834563072 + }, + { + "epoch": 0.86, + "learning_rate": 7.131394182547643e-05, + "loss": 1.2601, + "theoretical_loss": 3.3353505045250014, + "tokens_seen": 2834694144 + }, + { + "epoch": 0.86, + "learning_rate": 7.12938816449348e-05, + "loss": 1.2864, + "theoretical_loss": 3.3353385103090343, + "tokens_seen": 2834825216 + }, + { + "epoch": 0.86, + "learning_rate": 7.127382146439318e-05, + "loss": 1.4629, + "theoretical_loss": 3.3353265168028936, + "tokens_seen": 2834956288 + }, + { + "epoch": 0.86, + "learning_rate": 7.125376128385155e-05, + "loss": 1.2266, + "theoretical_loss": 3.3353145240065056, + "tokens_seen": 2835087360 + }, + { + "epoch": 0.86, + "learning_rate": 7.123370110330993e-05, + "loss": 1.1666, + "theoretical_loss": 3.3353025319197953, + "tokens_seen": 2835218432 + }, + { + "epoch": 0.86, + "learning_rate": 7.12136409227683e-05, + "loss": 1.3662, + "theoretical_loss": 3.3352905405426876, + "tokens_seen": 2835349504 + }, + { + "epoch": 0.86, + "learning_rate": 7.119358074222668e-05, + "loss": 1.1621, + "theoretical_loss": 3.335278549875108, + "tokens_seen": 2835480576 + }, + { + "epoch": 0.86, + "learning_rate": 7.117352056168506e-05, + "loss": 1.2985, + "theoretical_loss": 3.335266559916981, + "tokens_seen": 2835611648 + }, + { + "epoch": 0.86, + "learning_rate": 7.115346038114342e-05, + "loss": 1.2836, + "theoretical_loss": 3.335254570668233, + "tokens_seen": 2835742720 + }, + { + "epoch": 0.86, + "learning_rate": 7.113340020060181e-05, + "loss": 1.239, + "theoretical_loss": 3.3352425821287888, + "tokens_seen": 2835873792 + }, + { + "epoch": 0.86, + "learning_rate": 7.111334002006018e-05, + "loss": 1.3976, + "theoretical_loss": 3.335230594298573, + "tokens_seen": 2836004864 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.41923263669013977, + "objective/train/avg_token_score": 0.036697641015052795, + "objective/train/docs_used": 1596277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3408255577087402, + "objective/train/original_loss": 3.7716434001922607, + "objective/train/theoretical_loss": 3.3352246006494024, + "objective/train/tokens_used": 2836070400, + "objective/train/unlikelihood_loss": 0.1568891555070877, + "theoretical_loss": 3.3352246006494024, + "tokens_seen": 2836070400 + }, + { + "epoch": 0.86, + "learning_rate": 7.109327983951856e-05, + "loss": 1.2906, + "theoretical_loss": 3.3352186071775116, + "tokens_seen": 2836135936 + }, + { + "epoch": 0.86, + "learning_rate": 7.107321965897694e-05, + "loss": 1.371, + "theoretical_loss": 3.3352066207655295, + "tokens_seen": 2836267008 + }, + { + "epoch": 0.86, + "learning_rate": 7.105315947843531e-05, + "loss": 1.3456, + "theoretical_loss": 3.3351946350625523, + "tokens_seen": 2836398080 + }, + { + "epoch": 0.86, + "learning_rate": 7.103309929789369e-05, + "loss": 1.2414, + "theoretical_loss": 3.3351826500685053, + "tokens_seen": 2836529152 + }, + { + "epoch": 0.86, + "learning_rate": 7.101303911735206e-05, + "loss": 1.2952, + "theoretical_loss": 3.335170665783313, + "tokens_seen": 2836660224 + }, + { + "epoch": 0.86, + "learning_rate": 7.099297893681044e-05, + "loss": 1.425, + "theoretical_loss": 3.335158682206902, + "tokens_seen": 2836791296 + }, + { + "epoch": 0.86, + "learning_rate": 7.097291875626882e-05, + "loss": 1.2877, + "theoretical_loss": 3.335146699339197, + "tokens_seen": 2836922368 + }, + { + "epoch": 0.86, + "learning_rate": 7.095285857572718e-05, + "loss": 1.3824, + "theoretical_loss": 3.3351347171801233, + "tokens_seen": 2837053440 + }, + { + "epoch": 0.86, + "learning_rate": 7.093279839518555e-05, + "loss": 1.1606, + "theoretical_loss": 3.3351227357296063, + "tokens_seen": 2837184512 + }, + { + "epoch": 0.86, + "learning_rate": 7.091273821464393e-05, + "loss": 1.1417, + "theoretical_loss": 3.335110754987571, + "tokens_seen": 2837315584 + }, + { + "epoch": 0.86, + "learning_rate": 7.08926780341023e-05, + "loss": 1.2309, + "theoretical_loss": 3.3350987749539436, + "tokens_seen": 2837446656 + }, + { + "epoch": 0.86, + "learning_rate": 7.087261785356068e-05, + "loss": 1.284, + "theoretical_loss": 3.3350867956286487, + "tokens_seen": 2837577728 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.37041789293289185, + "objective/train/avg_token_score": 0.015085759572684765, + "objective/train/docs_used": 1596937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0254428386688232, + "objective/train/original_loss": 4.238986015319824, + "objective/train/theoretical_loss": 3.3350748170116127, + "objective/train/tokens_used": 2837708800, + "objective/train/unlikelihood_loss": 0.14830394089221954, + "theoretical_loss": 3.3350748170116127, + "tokens_seen": 2837708800 + }, + { + "epoch": 0.86, + "learning_rate": 7.085255767301906e-05, + "loss": 1.3152, + "theoretical_loss": 3.3350748170116127, + "tokens_seen": 2837708800 + }, + { + "epoch": 0.86, + "learning_rate": 7.083249749247743e-05, + "loss": 1.1637, + "theoretical_loss": 3.3350628391027595, + "tokens_seen": 2837839872 + }, + { + "epoch": 0.86, + "learning_rate": 7.081243731193581e-05, + "loss": 1.3541, + "theoretical_loss": 3.3350508619020163, + "tokens_seen": 2837970944 + }, + { + "epoch": 0.86, + "learning_rate": 7.079237713139418e-05, + "loss": 1.2198, + "theoretical_loss": 3.3350388854093067, + "tokens_seen": 2838102016 + }, + { + "epoch": 0.86, + "learning_rate": 7.077231695085255e-05, + "loss": 1.2553, + "theoretical_loss": 3.335026909624558, + "tokens_seen": 2838233088 + }, + { + "epoch": 0.86, + "learning_rate": 7.075225677031094e-05, + "loss": 1.2525, + "theoretical_loss": 3.335014934547694, + "tokens_seen": 2838364160 + }, + { + "epoch": 0.86, + "learning_rate": 7.073219658976931e-05, + "loss": 1.2491, + "theoretical_loss": 3.3350029601786417, + "tokens_seen": 2838495232 + }, + { + "epoch": 0.86, + "learning_rate": 7.071213640922769e-05, + "loss": 1.2011, + "theoretical_loss": 3.334990986517325, + "tokens_seen": 2838626304 + }, + { + "epoch": 0.86, + "learning_rate": 7.069207622868606e-05, + "loss": 1.3687, + "theoretical_loss": 3.3349790135636708, + "tokens_seen": 2838757376 + }, + { + "epoch": 0.86, + "learning_rate": 7.067201604814444e-05, + "loss": 1.2484, + "theoretical_loss": 3.3349670413176034, + "tokens_seen": 2838888448 + }, + { + "epoch": 0.86, + "learning_rate": 7.065195586760281e-05, + "loss": 1.2848, + "theoretical_loss": 3.3349550697790495, + "tokens_seen": 2839019520 + }, + { + "epoch": 0.86, + "learning_rate": 7.063189568706119e-05, + "loss": 1.447, + "theoretical_loss": 3.334943098947934, + "tokens_seen": 2839150592 + }, + { + "epoch": 0.86, + "learning_rate": 7.061183550651957e-05, + "loss": 1.2926, + "theoretical_loss": 3.334931128824182, + "tokens_seen": 2839281664 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.45674487948417664, + "objective/train/avg_token_score": 0.013437516987323761, + "objective/train/docs_used": 1597942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2807369232177734, + "objective/train/original_loss": 3.046680450439453, + "objective/train/theoretical_loss": 3.3349251440275443, + "objective/train/tokens_used": 2839347200, + "objective/train/unlikelihood_loss": 0.2506890892982483, + "theoretical_loss": 3.3349251440275443, + "tokens_seen": 2839347200 + }, + { + "epoch": 0.86, + "learning_rate": 7.059177532597794e-05, + "loss": 1.474, + "theoretical_loss": 3.3349191594077197, + "tokens_seen": 2839412736 + }, + { + "epoch": 0.86, + "learning_rate": 7.05717151454363e-05, + "loss": 1.2534, + "theoretical_loss": 3.3349071906984724, + "tokens_seen": 2839543808 + }, + { + "epoch": 0.86, + "learning_rate": 7.055165496489468e-05, + "loss": 1.2875, + "theoretical_loss": 3.334895222696366, + "tokens_seen": 2839674880 + }, + { + "epoch": 0.86, + "learning_rate": 7.053159478435306e-05, + "loss": 1.3591, + "theoretical_loss": 3.334883255401326, + "tokens_seen": 2839805952 + }, + { + "epoch": 0.86, + "learning_rate": 7.051153460381143e-05, + "loss": 1.2198, + "theoretical_loss": 3.3348712888132774, + "tokens_seen": 2839937024 + }, + { + "epoch": 0.86, + "learning_rate": 7.049147442326981e-05, + "loss": 1.4253, + "theoretical_loss": 3.3348593229321466, + "tokens_seen": 2840068096 + }, + { + "epoch": 0.86, + "learning_rate": 7.047141424272818e-05, + "loss": 1.2129, + "theoretical_loss": 3.334847357757859, + "tokens_seen": 2840199168 + }, + { + "epoch": 0.86, + "learning_rate": 7.045135406218656e-05, + "loss": 1.285, + "theoretical_loss": 3.33483539329034, + "tokens_seen": 2840330240 + }, + { + "epoch": 0.86, + "learning_rate": 7.043129388164493e-05, + "loss": 1.2067, + "theoretical_loss": 3.3348234295295147, + "tokens_seen": 2840461312 + }, + { + "epoch": 0.86, + "learning_rate": 7.041123370110331e-05, + "loss": 1.3585, + "theoretical_loss": 3.33481146647531, + "tokens_seen": 2840592384 + }, + { + "epoch": 0.86, + "learning_rate": 7.039117352056169e-05, + "loss": 1.2428, + "theoretical_loss": 3.3347995041276506, + "tokens_seen": 2840723456 + }, + { + "epoch": 0.86, + "learning_rate": 7.037111334002006e-05, + "loss": 1.298, + "theoretical_loss": 3.334787542486463, + "tokens_seen": 2840854528 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.37383919954299927, + "objective/train/avg_token_score": 0.013131293468177319, + "objective/train/docs_used": 1598724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0707870721817017, + "objective/train/original_loss": 4.044744491577148, + "objective/train/theoretical_loss": 3.3347755815516718, + "objective/train/tokens_used": 2840985600, + "objective/train/unlikelihood_loss": 0.1608046144247055, + "theoretical_loss": 3.3347755815516718, + "tokens_seen": 2840985600 + }, + { + "epoch": 0.86, + "learning_rate": 7.035105315947844e-05, + "loss": 1.1826, + "theoretical_loss": 3.3347755815516718, + "tokens_seen": 2840985600 + }, + { + "epoch": 0.86, + "learning_rate": 7.033099297893681e-05, + "loss": 1.3628, + "theoretical_loss": 3.3347636213232037, + "tokens_seen": 2841116672 + }, + { + "epoch": 0.86, + "learning_rate": 7.031093279839519e-05, + "loss": 1.3395, + "theoretical_loss": 3.334751661800984, + "tokens_seen": 2841247744 + }, + { + "epoch": 0.86, + "learning_rate": 7.029087261785357e-05, + "loss": 1.3354, + "theoretical_loss": 3.334739702984938, + "tokens_seen": 2841378816 + }, + { + "epoch": 0.86, + "learning_rate": 7.027081243731194e-05, + "loss": 1.2247, + "theoretical_loss": 3.3347277448749924, + "tokens_seen": 2841509888 + }, + { + "epoch": 0.86, + "learning_rate": 7.025075225677032e-05, + "loss": 1.2976, + "theoretical_loss": 3.334715787471072, + "tokens_seen": 2841640960 + }, + { + "epoch": 0.86, + "learning_rate": 7.023069207622869e-05, + "loss": 1.4139, + "theoretical_loss": 3.3347038307731034, + "tokens_seen": 2841772032 + }, + { + "epoch": 0.86, + "learning_rate": 7.021063189568707e-05, + "loss": 1.2641, + "theoretical_loss": 3.3346918747810115, + "tokens_seen": 2841903104 + }, + { + "epoch": 0.86, + "learning_rate": 7.019057171514544e-05, + "loss": 1.3523, + "theoretical_loss": 3.3346799194947225, + "tokens_seen": 2842034176 + }, + { + "epoch": 0.86, + "learning_rate": 7.01705115346038e-05, + "loss": 1.249, + "theoretical_loss": 3.3346679649141624, + "tokens_seen": 2842165248 + }, + { + "epoch": 0.86, + "learning_rate": 7.015045135406218e-05, + "loss": 1.366, + "theoretical_loss": 3.3346560110392565, + "tokens_seen": 2842296320 + }, + { + "epoch": 0.86, + "learning_rate": 7.013039117352056e-05, + "loss": 1.2711, + "theoretical_loss": 3.334644057869931, + "tokens_seen": 2842427392 + }, + { + "epoch": 0.86, + "learning_rate": 7.011033099297893e-05, + "loss": 1.2389, + "theoretical_loss": 3.334632105406112, + "tokens_seen": 2842558464 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.4332233667373657, + "objective/train/avg_token_score": 0.016299284994602203, + "objective/train/docs_used": 1599978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3885562419891357, + "objective/train/original_loss": 3.8281116485595703, + "objective/train/theoretical_loss": 3.334626129438744, + "objective/train/tokens_used": 2842624000, + "objective/train/unlikelihood_loss": 0.15794087946414948, + "theoretical_loss": 3.334626129438744, + "tokens_seen": 2842624000 + }, + { + "epoch": 0.86, + "learning_rate": 7.009027081243731e-05, + "loss": 1.333, + "theoretical_loss": 3.3346201536477245, + "tokens_seen": 2842689536 + }, + { + "epoch": 0.86, + "learning_rate": 7.007021063189569e-05, + "loss": 1.288, + "theoretical_loss": 3.334608202594695, + "tokens_seen": 2842820608 + }, + { + "epoch": 0.86, + "learning_rate": 7.005015045135406e-05, + "loss": 1.3699, + "theoretical_loss": 3.334596252246949, + "tokens_seen": 2842951680 + }, + { + "epoch": 0.86, + "learning_rate": 7.003009027081244e-05, + "loss": 1.3209, + "theoretical_loss": 3.334584302604412, + "tokens_seen": 2843082752 + }, + { + "epoch": 0.86, + "learning_rate": 7.001003009027081e-05, + "loss": 1.4059, + "theoretical_loss": 3.334572353667011, + "tokens_seen": 2843213824 + }, + { + "epoch": 0.86, + "learning_rate": 6.998996990972919e-05, + "loss": 1.4069, + "theoretical_loss": 3.334560405434671, + "tokens_seen": 2843344896 + }, + { + "epoch": 0.86, + "learning_rate": 6.996990972918756e-05, + "loss": 1.3755, + "theoretical_loss": 3.334548457907318, + "tokens_seen": 2843475968 + }, + { + "epoch": 0.86, + "learning_rate": 6.994984954864594e-05, + "loss": 1.31, + "theoretical_loss": 3.3345365110848784, + "tokens_seen": 2843607040 + }, + { + "epoch": 0.86, + "learning_rate": 6.992978936810432e-05, + "loss": 1.2479, + "theoretical_loss": 3.334524564967278, + "tokens_seen": 2843738112 + }, + { + "epoch": 0.86, + "learning_rate": 6.990972918756269e-05, + "loss": 1.1901, + "theoretical_loss": 3.3345126195544417, + "tokens_seen": 2843869184 + }, + { + "epoch": 0.86, + "learning_rate": 6.988966900702107e-05, + "loss": 1.4135, + "theoretical_loss": 3.334500674846297, + "tokens_seen": 2844000256 + }, + { + "epoch": 0.86, + "learning_rate": 6.986960882647944e-05, + "loss": 1.3842, + "theoretical_loss": 3.334488730842769, + "tokens_seen": 2844131328 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.35483869910240173, + "objective/train/avg_token_score": 0.005291230045258999, + "objective/train/docs_used": 1601259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.097688913345337, + "objective/train/original_loss": 4.374083042144775, + "objective/train/theoretical_loss": 3.3344767875437835, + "objective/train/tokens_used": 2844262400, + "objective/train/unlikelihood_loss": 0.1223866194486618, + "theoretical_loss": 3.3344767875437835, + "tokens_seen": 2844262400 + }, + { + "epoch": 0.86, + "learning_rate": 6.984954864593782e-05, + "loss": 1.3653, + "theoretical_loss": 3.3344767875437835, + "tokens_seen": 2844262400 + }, + { + "epoch": 0.86, + "learning_rate": 6.98294884653962e-05, + "loss": 1.3167, + "theoretical_loss": 3.3344648449492666, + "tokens_seen": 2844393472 + }, + { + "epoch": 0.86, + "learning_rate": 6.980942828485457e-05, + "loss": 1.3037, + "theoretical_loss": 3.3344529030591445, + "tokens_seen": 2844524544 + }, + { + "epoch": 0.86, + "learning_rate": 6.978936810431293e-05, + "loss": 1.1993, + "theoretical_loss": 3.3344409618733435, + "tokens_seen": 2844655616 + }, + { + "epoch": 0.86, + "learning_rate": 6.976930792377131e-05, + "loss": 1.3076, + "theoretical_loss": 3.334429021391789, + "tokens_seen": 2844786688 + }, + { + "epoch": 0.86, + "learning_rate": 6.974924774322968e-05, + "loss": 1.3285, + "theoretical_loss": 3.3344170816144074, + "tokens_seen": 2844917760 + }, + { + "epoch": 0.86, + "learning_rate": 6.972918756268806e-05, + "loss": 1.3565, + "theoretical_loss": 3.3344051425411245, + "tokens_seen": 2845048832 + }, + { + "epoch": 0.86, + "learning_rate": 6.970912738214644e-05, + "loss": 1.3143, + "theoretical_loss": 3.3343932041718665, + "tokens_seen": 2845179904 + }, + { + "epoch": 0.86, + "learning_rate": 6.968906720160481e-05, + "loss": 1.2424, + "theoretical_loss": 3.3343812665065595, + "tokens_seen": 2845310976 + }, + { + "epoch": 0.86, + "learning_rate": 6.966900702106319e-05, + "loss": 1.2777, + "theoretical_loss": 3.334369329545129, + "tokens_seen": 2845442048 + }, + { + "epoch": 0.86, + "learning_rate": 6.964894684052156e-05, + "loss": 1.2581, + "theoretical_loss": 3.334357393287502, + "tokens_seen": 2845573120 + }, + { + "epoch": 0.86, + "learning_rate": 6.962888665997995e-05, + "loss": 1.2125, + "theoretical_loss": 3.334345457733604, + "tokens_seen": 2845704192 + }, + { + "epoch": 0.86, + "learning_rate": 6.960882647943833e-05, + "loss": 1.1494, + "theoretical_loss": 3.334333522883361, + "tokens_seen": 2845835264 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.47281280159950256, + "objective/train/avg_token_score": 0.00908656045794487, + "objective/train/docs_used": 1601933, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2652736902236938, + "objective/train/original_loss": 3.225541591644287, + "objective/train/theoretical_loss": 3.3343275557220875, + "objective/train/tokens_used": 2845900800, + "objective/train/unlikelihood_loss": 0.21953557431697845, + "theoretical_loss": 3.3343275557220875, + "tokens_seen": 2845900800 + }, + { + "epoch": 0.86, + "learning_rate": 6.958876629889669e-05, + "loss": 1.3327, + "theoretical_loss": 3.3343215887366995, + "tokens_seen": 2845966336 + }, + { + "epoch": 0.86, + "learning_rate": 6.956870611835507e-05, + "loss": 1.1956, + "theoretical_loss": 3.334309655293546, + "tokens_seen": 2846097408 + }, + { + "epoch": 0.86, + "learning_rate": 6.954864593781344e-05, + "loss": 1.2525, + "theoretical_loss": 3.334297722553825, + "tokens_seen": 2846228480 + }, + { + "epoch": 0.86, + "learning_rate": 6.952858575727182e-05, + "loss": 1.3334, + "theoretical_loss": 3.3342857905174648, + "tokens_seen": 2846359552 + }, + { + "epoch": 0.86, + "learning_rate": 6.95085255767302e-05, + "loss": 1.3977, + "theoretical_loss": 3.33427385918439, + "tokens_seen": 2846490624 + }, + { + "epoch": 0.86, + "learning_rate": 6.948846539618857e-05, + "loss": 1.2335, + "theoretical_loss": 3.334261928554527, + "tokens_seen": 2846621696 + }, + { + "epoch": 0.86, + "learning_rate": 6.946840521564695e-05, + "loss": 1.3028, + "theoretical_loss": 3.3342499986278025, + "tokens_seen": 2846752768 + }, + { + "epoch": 0.86, + "learning_rate": 6.944834503510532e-05, + "loss": 1.3404, + "theoretical_loss": 3.334238069404143, + "tokens_seen": 2846883840 + }, + { + "epoch": 0.86, + "learning_rate": 6.94282848545637e-05, + "loss": 1.2696, + "theoretical_loss": 3.334226140883473, + "tokens_seen": 2847014912 + }, + { + "epoch": 0.86, + "learning_rate": 6.940822467402207e-05, + "loss": 1.3446, + "theoretical_loss": 3.334214213065721, + "tokens_seen": 2847145984 + }, + { + "epoch": 0.86, + "learning_rate": 6.938816449348044e-05, + "loss": 1.2983, + "theoretical_loss": 3.334202285950811, + "tokens_seen": 2847277056 + }, + { + "epoch": 0.86, + "learning_rate": 6.936810431293881e-05, + "loss": 1.3288, + "theoretical_loss": 3.334190359538671, + "tokens_seen": 2847408128 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.4161168038845062, + "objective/train/avg_token_score": 0.014336055144667625, + "objective/train/docs_used": 1603273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2674328088760376, + "objective/train/original_loss": 3.6697781085968018, + "objective/train/theoretical_loss": 3.3341784338292264, + "objective/train/tokens_used": 2847539200, + "objective/train/unlikelihood_loss": 0.21973960101604462, + "theoretical_loss": 3.3341784338292264, + "tokens_seen": 2847539200 + }, + { + "epoch": 0.86, + "learning_rate": 6.934804413239719e-05, + "loss": 1.2565, + "theoretical_loss": 3.3341784338292264, + "tokens_seen": 2847539200 + }, + { + "epoch": 0.86, + "learning_rate": 6.932798395185556e-05, + "loss": 1.2621, + "theoretical_loss": 3.3341665088224035, + "tokens_seen": 2847670272 + }, + { + "epoch": 0.86, + "learning_rate": 6.930792377131394e-05, + "loss": 1.4322, + "theoretical_loss": 3.3341545845181284, + "tokens_seen": 2847801344 + }, + { + "epoch": 0.86, + "learning_rate": 6.928786359077231e-05, + "loss": 1.2509, + "theoretical_loss": 3.334142660916328, + "tokens_seen": 2847932416 + }, + { + "epoch": 0.86, + "learning_rate": 6.926780341023069e-05, + "loss": 1.2574, + "theoretical_loss": 3.334130738016928, + "tokens_seen": 2848063488 + }, + { + "epoch": 0.86, + "learning_rate": 6.924774322968908e-05, + "loss": 1.3734, + "theoretical_loss": 3.334118815819855, + "tokens_seen": 2848194560 + }, + { + "epoch": 0.86, + "learning_rate": 6.922768304914746e-05, + "loss": 1.2458, + "theoretical_loss": 3.3341068943250356, + "tokens_seen": 2848325632 + }, + { + "epoch": 0.86, + "learning_rate": 6.920762286860582e-05, + "loss": 1.3614, + "theoretical_loss": 3.334094973532395, + "tokens_seen": 2848456704 + }, + { + "epoch": 0.86, + "learning_rate": 6.918756268806419e-05, + "loss": 1.3249, + "theoretical_loss": 3.334083053441861, + "tokens_seen": 2848587776 + }, + { + "epoch": 0.86, + "learning_rate": 6.916750250752257e-05, + "loss": 1.3429, + "theoretical_loss": 3.334071134053359, + "tokens_seen": 2848718848 + }, + { + "epoch": 0.86, + "learning_rate": 6.914744232698094e-05, + "loss": 1.3776, + "theoretical_loss": 3.3340592153668154, + "tokens_seen": 2848849920 + }, + { + "epoch": 0.86, + "learning_rate": 6.912738214643932e-05, + "loss": 1.2168, + "theoretical_loss": 3.334047297382157, + "tokens_seen": 2848980992 + }, + { + "epoch": 0.86, + "learning_rate": 6.91073219658977e-05, + "loss": 1.2721, + "theoretical_loss": 3.33403538009931, + "tokens_seen": 2849112064 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.47531768679618835, + "objective/train/avg_token_score": 0.012980761006474495, + "objective/train/docs_used": 1603897, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5306082963943481, + "objective/train/original_loss": 3.4500648975372314, + "objective/train/theoretical_loss": 3.334029421721042, + "objective/train/tokens_used": 2849177600, + "objective/train/unlikelihood_loss": 0.2010214775800705, + "theoretical_loss": 3.334029421721042, + "tokens_seen": 2849177600 + }, + { + "epoch": 0.86, + "learning_rate": 6.908726178535607e-05, + "loss": 1.3498, + "theoretical_loss": 3.3340234635182004, + "tokens_seen": 2849243136 + }, + { + "epoch": 0.86, + "learning_rate": 6.906720160481445e-05, + "loss": 1.3368, + "theoretical_loss": 3.334011547638755, + "tokens_seen": 2849374208 + }, + { + "epoch": 0.86, + "learning_rate": 6.904714142427282e-05, + "loss": 1.3004, + "theoretical_loss": 3.3339996324609, + "tokens_seen": 2849505280 + }, + { + "epoch": 0.86, + "learning_rate": 6.90270812437312e-05, + "loss": 1.4437, + "theoretical_loss": 3.3339877179845625, + "tokens_seen": 2849636352 + }, + { + "epoch": 0.86, + "learning_rate": 6.900702106318956e-05, + "loss": 1.4589, + "theoretical_loss": 3.333975804209668, + "tokens_seen": 2849767424 + }, + { + "epoch": 0.86, + "learning_rate": 6.898696088264794e-05, + "loss": 1.2807, + "theoretical_loss": 3.3339638911361433, + "tokens_seen": 2849898496 + }, + { + "epoch": 0.86, + "learning_rate": 6.896690070210631e-05, + "loss": 1.4405, + "theoretical_loss": 3.333951978763915, + "tokens_seen": 2850029568 + }, + { + "epoch": 0.86, + "learning_rate": 6.894684052156469e-05, + "loss": 1.3319, + "theoretical_loss": 3.3339400670929096, + "tokens_seen": 2850160640 + }, + { + "epoch": 0.86, + "learning_rate": 6.892678034102306e-05, + "loss": 1.3089, + "theoretical_loss": 3.3339281561230534, + "tokens_seen": 2850291712 + }, + { + "epoch": 0.86, + "learning_rate": 6.890672016048144e-05, + "loss": 1.3431, + "theoretical_loss": 3.333916245854273, + "tokens_seen": 2850422784 + }, + { + "epoch": 0.86, + "learning_rate": 6.888665997993982e-05, + "loss": 1.2356, + "theoretical_loss": 3.333904336286494, + "tokens_seen": 2850553856 + }, + { + "epoch": 0.86, + "learning_rate": 6.88665997993982e-05, + "loss": 1.2856, + "theoretical_loss": 3.3338924274196446, + "tokens_seen": 2850684928 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.377443790435791, + "objective/train/avg_token_score": 0.016033044084906578, + "objective/train/docs_used": 1604602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1996384859085083, + "objective/train/original_loss": 3.828817844390869, + "objective/train/theoretical_loss": 3.3338805192536505, + "objective/train/tokens_used": 2850816000, + "objective/train/unlikelihood_loss": 0.20030707120895386, + "theoretical_loss": 3.3338805192536505, + "tokens_seen": 2850816000 + }, + { + "epoch": 0.86, + "learning_rate": 6.884653961885658e-05, + "loss": 1.3276, + "theoretical_loss": 3.3338805192536505, + "tokens_seen": 2850816000 + }, + { + "epoch": 0.86, + "learning_rate": 6.882647943831496e-05, + "loss": 1.2924, + "theoretical_loss": 3.333868611788438, + "tokens_seen": 2850947072 + }, + { + "epoch": 0.86, + "learning_rate": 6.880641925777332e-05, + "loss": 1.3186, + "theoretical_loss": 3.333856705023934, + "tokens_seen": 2851078144 + }, + { + "epoch": 0.86, + "learning_rate": 6.87863590772317e-05, + "loss": 1.3247, + "theoretical_loss": 3.3338447989600644, + "tokens_seen": 2851209216 + }, + { + "epoch": 0.86, + "learning_rate": 6.876629889669007e-05, + "loss": 1.3304, + "theoretical_loss": 3.3338328935967567, + "tokens_seen": 2851340288 + }, + { + "epoch": 0.86, + "learning_rate": 6.874623871614845e-05, + "loss": 1.3365, + "theoretical_loss": 3.3338209889339367, + "tokens_seen": 2851471360 + }, + { + "epoch": 0.86, + "learning_rate": 6.872617853560682e-05, + "loss": 1.4347, + "theoretical_loss": 3.3338090849715316, + "tokens_seen": 2851602432 + }, + { + "epoch": 0.86, + "learning_rate": 6.87061183550652e-05, + "loss": 1.2434, + "theoretical_loss": 3.3337971817094676, + "tokens_seen": 2851733504 + }, + { + "epoch": 0.86, + "learning_rate": 6.868605817452357e-05, + "loss": 1.5719, + "theoretical_loss": 3.3337852791476714, + "tokens_seen": 2851864576 + }, + { + "epoch": 0.86, + "learning_rate": 6.866599799398195e-05, + "loss": 1.1704, + "theoretical_loss": 3.33377337728607, + "tokens_seen": 2851995648 + }, + { + "epoch": 0.86, + "learning_rate": 6.864593781344033e-05, + "loss": 1.4004, + "theoretical_loss": 3.333761476124589, + "tokens_seen": 2852126720 + }, + { + "epoch": 0.86, + "learning_rate": 6.862587763289869e-05, + "loss": 1.2106, + "theoretical_loss": 3.3337495756631563, + "tokens_seen": 2852257792 + }, + { + "epoch": 0.86, + "learning_rate": 6.860581745235706e-05, + "loss": 1.376, + "theoretical_loss": 3.3337376759016975, + "tokens_seen": 2852388864 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.3690738081932068, + "objective/train/avg_token_score": 0.008985217660665512, + "objective/train/docs_used": 1605428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2495214939117432, + "objective/train/original_loss": 3.891439437866211, + "objective/train/theoretical_loss": 3.333731726283436, + "objective/train/tokens_used": 2852454400, + "objective/train/unlikelihood_loss": 0.25465765595436096, + "theoretical_loss": 3.333731726283436, + "tokens_seen": 2852454400 + }, + { + "epoch": 0.86, + "learning_rate": 6.858575727181544e-05, + "loss": 1.2594, + "theoretical_loss": 3.33372577684014, + "tokens_seen": 2852519936 + }, + { + "epoch": 0.86, + "learning_rate": 6.856569709127382e-05, + "loss": 1.4581, + "theoretical_loss": 3.3337138784784104, + "tokens_seen": 2852651008 + }, + { + "epoch": 0.86, + "learning_rate": 6.854563691073219e-05, + "loss": 1.2913, + "theoretical_loss": 3.333701980816435, + "tokens_seen": 2852782080 + }, + { + "epoch": 0.86, + "learning_rate": 6.852557673019057e-05, + "loss": 1.3899, + "theoretical_loss": 3.3336900838541412, + "tokens_seen": 2852913152 + }, + { + "epoch": 0.86, + "learning_rate": 6.850551654964896e-05, + "loss": 1.3267, + "theoretical_loss": 3.3336781875914547, + "tokens_seen": 2853044224 + }, + { + "epoch": 0.86, + "learning_rate": 6.848545636910733e-05, + "loss": 1.2263, + "theoretical_loss": 3.333666292028303, + "tokens_seen": 2853175296 + }, + { + "epoch": 0.86, + "learning_rate": 6.846539618856571e-05, + "loss": 1.3629, + "theoretical_loss": 3.3336543971646124, + "tokens_seen": 2853306368 + }, + { + "epoch": 0.86, + "learning_rate": 6.844533600802408e-05, + "loss": 1.3188, + "theoretical_loss": 3.33364250300031, + "tokens_seen": 2853437440 + }, + { + "epoch": 0.86, + "learning_rate": 6.842527582748245e-05, + "loss": 1.2258, + "theoretical_loss": 3.333630609535322, + "tokens_seen": 2853568512 + }, + { + "epoch": 0.86, + "learning_rate": 6.840521564694082e-05, + "loss": 1.17, + "theoretical_loss": 3.333618716769576, + "tokens_seen": 2853699584 + }, + { + "epoch": 0.86, + "learning_rate": 6.83851554663992e-05, + "loss": 1.3307, + "theoretical_loss": 3.3336068247029984, + "tokens_seen": 2853830656 + }, + { + "epoch": 0.86, + "learning_rate": 6.836509528585757e-05, + "loss": 1.2792, + "theoretical_loss": 3.3335949333355157, + "tokens_seen": 2853961728 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.37762707471847534, + "objective/train/avg_token_score": 0.0110054025426507, + "objective/train/docs_used": 1606174, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0913350582122803, + "objective/train/original_loss": 3.5523927211761475, + "objective/train/theoretical_loss": 3.3335830426670547, + "objective/train/tokens_used": 2854092800, + "objective/train/unlikelihood_loss": 0.20045088231563568, + "theoretical_loss": 3.3335830426670547, + "tokens_seen": 2854092800 + }, + { + "epoch": 0.86, + "learning_rate": 6.834503510531595e-05, + "loss": 1.2234, + "theoretical_loss": 3.3335830426670547, + "tokens_seen": 2854092800 + }, + { + "epoch": 0.86, + "learning_rate": 6.832497492477433e-05, + "loss": 1.2267, + "theoretical_loss": 3.3335711526975427, + "tokens_seen": 2854223872 + }, + { + "epoch": 0.86, + "learning_rate": 6.83049147442327e-05, + "loss": 1.2741, + "theoretical_loss": 3.3335592634269062, + "tokens_seen": 2854354944 + }, + { + "epoch": 0.86, + "learning_rate": 6.828485456369108e-05, + "loss": 1.4825, + "theoretical_loss": 3.333547374855072, + "tokens_seen": 2854486016 + }, + { + "epoch": 0.87, + "learning_rate": 6.826479438314945e-05, + "loss": 1.2447, + "theoretical_loss": 3.333535486981967, + "tokens_seen": 2854617088 + }, + { + "epoch": 0.87, + "learning_rate": 6.824473420260783e-05, + "loss": 1.3396, + "theoretical_loss": 3.333523599807518, + "tokens_seen": 2854748160 + }, + { + "epoch": 0.87, + "learning_rate": 6.822467402206619e-05, + "loss": 1.3259, + "theoretical_loss": 3.333511713331652, + "tokens_seen": 2854879232 + }, + { + "epoch": 0.87, + "learning_rate": 6.820461384152457e-05, + "loss": 1.2725, + "theoretical_loss": 3.333499827554296, + "tokens_seen": 2855010304 + }, + { + "epoch": 0.87, + "learning_rate": 6.818455366098294e-05, + "loss": 1.4294, + "theoretical_loss": 3.3334879424753767, + "tokens_seen": 2855141376 + }, + { + "epoch": 0.87, + "learning_rate": 6.816449348044132e-05, + "loss": 1.2992, + "theoretical_loss": 3.3334760580948206, + "tokens_seen": 2855272448 + }, + { + "epoch": 0.87, + "learning_rate": 6.81444332998997e-05, + "loss": 1.1888, + "theoretical_loss": 3.3334641744125553, + "tokens_seen": 2855403520 + }, + { + "epoch": 0.87, + "learning_rate": 6.812437311935808e-05, + "loss": 1.385, + "theoretical_loss": 3.3334522914285074, + "tokens_seen": 2855534592 + }, + { + "epoch": 0.87, + "learning_rate": 6.810431293881646e-05, + "loss": 1.3277, + "theoretical_loss": 3.3334404091426038, + "tokens_seen": 2855665664 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.31506597995758057, + "objective/train/avg_token_score": 0.052908092737197876, + "objective/train/docs_used": 1607592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8574541211128235, + "objective/train/original_loss": 4.105648517608643, + "objective/train/theoretical_loss": 3.333434468261433, + "objective/train/tokens_used": 2855731200, + "objective/train/unlikelihood_loss": 0.19063830375671387, + "theoretical_loss": 3.333434468261433, + "tokens_seen": 2855731200 + }, + { + "epoch": 0.87, + "learning_rate": 6.808425275827483e-05, + "loss": 1.2695, + "theoretical_loss": 3.333428527554771, + "tokens_seen": 2855796736 + }, + { + "epoch": 0.87, + "learning_rate": 6.806419257773321e-05, + "loss": 1.2253, + "theoretical_loss": 3.3334166466649373, + "tokens_seen": 2855927808 + }, + { + "epoch": 0.87, + "learning_rate": 6.804413239719159e-05, + "loss": 1.4241, + "theoretical_loss": 3.3334047664730284, + "tokens_seen": 2856058880 + }, + { + "epoch": 0.87, + "learning_rate": 6.802407221664995e-05, + "loss": 1.3855, + "theoretical_loss": 3.3333928869789715, + "tokens_seen": 2856189952 + }, + { + "epoch": 0.87, + "learning_rate": 6.800401203610832e-05, + "loss": 1.297, + "theoretical_loss": 3.333381008182694, + "tokens_seen": 2856321024 + }, + { + "epoch": 0.87, + "learning_rate": 6.79839518555667e-05, + "loss": 1.2119, + "theoretical_loss": 3.3333691300841224, + "tokens_seen": 2856452096 + }, + { + "epoch": 0.87, + "learning_rate": 6.796389167502508e-05, + "loss": 1.3601, + "theoretical_loss": 3.333357252683184, + "tokens_seen": 2856583168 + }, + { + "epoch": 0.87, + "learning_rate": 6.794383149448345e-05, + "loss": 1.284, + "theoretical_loss": 3.3333453759798064, + "tokens_seen": 2856714240 + }, + { + "epoch": 0.87, + "learning_rate": 6.792377131394183e-05, + "loss": 1.3012, + "theoretical_loss": 3.3333334999739157, + "tokens_seen": 2856845312 + }, + { + "epoch": 0.87, + "learning_rate": 6.79037111334002e-05, + "loss": 1.395, + "theoretical_loss": 3.333321624665439, + "tokens_seen": 2856976384 + }, + { + "epoch": 0.87, + "learning_rate": 6.788365095285858e-05, + "loss": 1.1199, + "theoretical_loss": 3.333309750054304, + "tokens_seen": 2857107456 + }, + { + "epoch": 0.87, + "learning_rate": 6.786359077231695e-05, + "loss": 1.3052, + "theoretical_loss": 3.333297876140437, + "tokens_seen": 2857238528 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.4627932608127594, + "objective/train/avg_token_score": 0.006728402338922024, + "objective/train/docs_used": 1608324, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3982802629470825, + "objective/train/original_loss": 3.4615628719329834, + "objective/train/theoretical_loss": 3.3332860029237654, + "objective/train/tokens_used": 2857369600, + "objective/train/unlikelihood_loss": 0.24126167595386505, + "theoretical_loss": 3.3332860029237654, + "tokens_seen": 2857369600 + }, + { + "epoch": 0.87, + "learning_rate": 6.784353059177532e-05, + "loss": 1.318, + "theoretical_loss": 3.3332860029237654, + "tokens_seen": 2857369600 + }, + { + "epoch": 0.87, + "learning_rate": 6.782347041123369e-05, + "loss": 1.268, + "theoretical_loss": 3.3332741304042166, + "tokens_seen": 2857500672 + }, + { + "epoch": 0.87, + "learning_rate": 6.780341023069207e-05, + "loss": 1.309, + "theoretical_loss": 3.3332622585817173, + "tokens_seen": 2857631744 + }, + { + "epoch": 0.87, + "learning_rate": 6.778335005015044e-05, + "loss": 1.1872, + "theoretical_loss": 3.3332503874561947, + "tokens_seen": 2857762816 + }, + { + "epoch": 0.87, + "learning_rate": 6.776328986960882e-05, + "loss": 1.3684, + "theoretical_loss": 3.333238517027576, + "tokens_seen": 2857893888 + }, + { + "epoch": 0.87, + "learning_rate": 6.774322968906721e-05, + "loss": 1.2115, + "theoretical_loss": 3.3332266472957883, + "tokens_seen": 2858024960 + }, + { + "epoch": 0.87, + "learning_rate": 6.772316950852559e-05, + "loss": 1.3487, + "theoretical_loss": 3.333214778260759, + "tokens_seen": 2858156032 + }, + { + "epoch": 0.87, + "learning_rate": 6.770310932798396e-05, + "loss": 1.4327, + "theoretical_loss": 3.3332029099224147, + "tokens_seen": 2858287104 + }, + { + "epoch": 0.87, + "learning_rate": 6.768304914744234e-05, + "loss": 1.3748, + "theoretical_loss": 3.3331910422806827, + "tokens_seen": 2858418176 + }, + { + "epoch": 0.87, + "learning_rate": 6.766298896690071e-05, + "loss": 1.3617, + "theoretical_loss": 3.3331791753354905, + "tokens_seen": 2858549248 + }, + { + "epoch": 0.87, + "learning_rate": 6.764292878635908e-05, + "loss": 1.4061, + "theoretical_loss": 3.333167309086765, + "tokens_seen": 2858680320 + }, + { + "epoch": 0.87, + "learning_rate": 6.762286860581745e-05, + "loss": 1.3127, + "theoretical_loss": 3.3331554435344333, + "tokens_seen": 2858811392 + }, + { + "epoch": 0.87, + "learning_rate": 6.760280842527583e-05, + "loss": 1.3195, + "theoretical_loss": 3.333143578678423, + "tokens_seen": 2858942464 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.4979838728904724, + "objective/train/avg_token_score": 0.005906440783292055, + "objective/train/docs_used": 1609642, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4297292232513428, + "objective/train/original_loss": 3.183546781539917, + "objective/train/theoretical_loss": 3.3331376465115157, + "objective/train/tokens_used": 2859008000, + "objective/train/unlikelihood_loss": 0.19771678745746613, + "theoretical_loss": 3.3331376465115157, + "tokens_seen": 2859008000 + }, + { + "epoch": 0.87, + "learning_rate": 6.75827482447342e-05, + "loss": 1.2916, + "theoretical_loss": 3.333131714518661, + "tokens_seen": 2859073536 + }, + { + "epoch": 0.87, + "learning_rate": 6.756268806419258e-05, + "loss": 1.3553, + "theoretical_loss": 3.333119851055075, + "tokens_seen": 2859204608 + }, + { + "epoch": 0.87, + "learning_rate": 6.754262788365095e-05, + "loss": 1.4637, + "theoretical_loss": 3.3331079882875914, + "tokens_seen": 2859335680 + }, + { + "epoch": 0.87, + "learning_rate": 6.752256770310933e-05, + "loss": 1.3219, + "theoretical_loss": 3.3330961262161383, + "tokens_seen": 2859466752 + }, + { + "epoch": 0.87, + "learning_rate": 6.75025075225677e-05, + "loss": 1.3772, + "theoretical_loss": 3.3330842648406422, + "tokens_seen": 2859597824 + }, + { + "epoch": 0.87, + "learning_rate": 6.748244734202608e-05, + "loss": 1.2946, + "theoretical_loss": 3.333072404161031, + "tokens_seen": 2859728896 + }, + { + "epoch": 0.87, + "learning_rate": 6.746238716148446e-05, + "loss": 1.2835, + "theoretical_loss": 3.3330605441772314, + "tokens_seen": 2859859968 + }, + { + "epoch": 0.87, + "learning_rate": 6.744232698094282e-05, + "loss": 1.281, + "theoretical_loss": 3.3330486848891714, + "tokens_seen": 2859991040 + }, + { + "epoch": 0.87, + "learning_rate": 6.74222668004012e-05, + "loss": 1.4224, + "theoretical_loss": 3.3330368262967776, + "tokens_seen": 2860122112 + }, + { + "epoch": 0.87, + "learning_rate": 6.740220661985957e-05, + "loss": 1.3619, + "theoretical_loss": 3.333024968399978, + "tokens_seen": 2860253184 + }, + { + "epoch": 0.87, + "learning_rate": 6.738214643931795e-05, + "loss": 1.2781, + "theoretical_loss": 3.3330131111986994, + "tokens_seen": 2860384256 + }, + { + "epoch": 0.87, + "learning_rate": 6.736208625877634e-05, + "loss": 1.3396, + "theoretical_loss": 3.333001254692869, + "tokens_seen": 2860515328 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.5284090638160706, + "objective/train/avg_token_score": 0.011530854739248753, + "objective/train/docs_used": 1610230, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.534618616104126, + "objective/train/original_loss": 3.891272783279419, + "objective/train/theoretical_loss": 3.3329893988824146, + "objective/train/tokens_used": 2860646400, + "objective/train/unlikelihood_loss": 0.10185238718986511, + "theoretical_loss": 3.3329893988824146, + "tokens_seen": 2860646400 + }, + { + "epoch": 0.87, + "learning_rate": 6.734202607823471e-05, + "loss": 1.3161, + "theoretical_loss": 3.3329893988824146, + "tokens_seen": 2860646400 + }, + { + "epoch": 0.87, + "learning_rate": 6.732196589769309e-05, + "loss": 1.1945, + "theoretical_loss": 3.3329775437672637, + "tokens_seen": 2860777472 + }, + { + "epoch": 0.87, + "learning_rate": 6.730190571715146e-05, + "loss": 1.4955, + "theoretical_loss": 3.3329656893473434, + "tokens_seen": 2860908544 + }, + { + "epoch": 0.87, + "learning_rate": 6.728184553660984e-05, + "loss": 1.2858, + "theoretical_loss": 3.3329538356225807, + "tokens_seen": 2861039616 + }, + { + "epoch": 0.87, + "learning_rate": 6.726178535606822e-05, + "loss": 1.3463, + "theoretical_loss": 3.3329419825929034, + "tokens_seen": 2861170688 + }, + { + "epoch": 0.87, + "learning_rate": 6.724172517552658e-05, + "loss": 1.2278, + "theoretical_loss": 3.332930130258239, + "tokens_seen": 2861301760 + }, + { + "epoch": 0.87, + "learning_rate": 6.722166499498495e-05, + "loss": 1.4468, + "theoretical_loss": 3.3329182786185143, + "tokens_seen": 2861432832 + }, + { + "epoch": 0.87, + "learning_rate": 6.720160481444333e-05, + "loss": 1.4236, + "theoretical_loss": 3.3329064276736577, + "tokens_seen": 2861563904 + }, + { + "epoch": 0.87, + "learning_rate": 6.71815446339017e-05, + "loss": 1.1277, + "theoretical_loss": 3.332894577423596, + "tokens_seen": 2861694976 + }, + { + "epoch": 0.87, + "learning_rate": 6.716148445336008e-05, + "loss": 1.2379, + "theoretical_loss": 3.332882727868257, + "tokens_seen": 2861826048 + }, + { + "epoch": 0.87, + "learning_rate": 6.714142427281846e-05, + "loss": 1.3642, + "theoretical_loss": 3.3328708790075674, + "tokens_seen": 2861957120 + }, + { + "epoch": 0.87, + "learning_rate": 6.712136409227683e-05, + "loss": 1.3474, + "theoretical_loss": 3.332859030841455, + "tokens_seen": 2862088192 + }, + { + "epoch": 0.87, + "learning_rate": 6.710130391173521e-05, + "loss": 1.1327, + "theoretical_loss": 3.3328471833698483, + "tokens_seen": 2862219264 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.44831377267837524, + "objective/train/avg_token_score": 0.004848118405789137, + "objective/train/docs_used": 1611371, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3105586767196655, + "objective/train/original_loss": 3.430337429046631, + "objective/train/theoretical_loss": 3.3328412598944612, + "objective/train/tokens_used": 2862284800, + "objective/train/unlikelihood_loss": 0.15741032361984253, + "theoretical_loss": 3.3328412598944612, + "tokens_seen": 2862284800 + }, + { + "epoch": 0.87, + "learning_rate": 6.708124373119358e-05, + "loss": 1.3081, + "theoretical_loss": 3.3328353365926735, + "tokens_seen": 2862350336 + }, + { + "epoch": 0.87, + "learning_rate": 6.706118355065195e-05, + "loss": 1.526, + "theoretical_loss": 3.3328234905098584, + "tokens_seen": 2862481408 + }, + { + "epoch": 0.87, + "learning_rate": 6.704112337011032e-05, + "loss": 1.1944, + "theoretical_loss": 3.332811645121331, + "tokens_seen": 2862612480 + }, + { + "epoch": 0.87, + "learning_rate": 6.70210631895687e-05, + "loss": 1.1616, + "theoretical_loss": 3.3327998004270185, + "tokens_seen": 2862743552 + }, + { + "epoch": 0.87, + "learning_rate": 6.700100300902709e-05, + "loss": 1.2635, + "theoretical_loss": 3.332787956426848, + "tokens_seen": 2862874624 + }, + { + "epoch": 0.87, + "learning_rate": 6.698094282848546e-05, + "loss": 1.445, + "theoretical_loss": 3.332776113120748, + "tokens_seen": 2863005696 + }, + { + "epoch": 0.87, + "learning_rate": 6.696088264794384e-05, + "loss": 1.4494, + "theoretical_loss": 3.332764270508645, + "tokens_seen": 2863136768 + }, + { + "epoch": 0.87, + "learning_rate": 6.694082246740221e-05, + "loss": 1.2344, + "theoretical_loss": 3.3327524285904673, + "tokens_seen": 2863267840 + }, + { + "epoch": 0.87, + "learning_rate": 6.692076228686059e-05, + "loss": 1.3001, + "theoretical_loss": 3.332740587366142, + "tokens_seen": 2863398912 + }, + { + "epoch": 0.87, + "learning_rate": 6.690070210631897e-05, + "loss": 1.3164, + "theoretical_loss": 3.3327287468355973, + "tokens_seen": 2863529984 + }, + { + "epoch": 0.87, + "learning_rate": 6.688064192577734e-05, + "loss": 1.2534, + "theoretical_loss": 3.33271690699876, + "tokens_seen": 2863661056 + }, + { + "epoch": 0.87, + "learning_rate": 6.68605817452357e-05, + "loss": 1.4071, + "theoretical_loss": 3.3327050678555588, + "tokens_seen": 2863792128 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.5003665685653687, + "objective/train/avg_token_score": 0.002088776556774974, + "objective/train/docs_used": 1612003, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6052948236465454, + "objective/train/original_loss": 3.4606614112854004, + "objective/train/theoretical_loss": 3.3326932294059204, + "objective/train/tokens_used": 2863923200, + "objective/train/unlikelihood_loss": 0.1993006318807602, + "theoretical_loss": 3.3326932294059204, + "tokens_seen": 2863923200 + }, + { + "epoch": 0.87, + "learning_rate": 6.684052156469408e-05, + "loss": 1.3383, + "theoretical_loss": 3.3326932294059204, + "tokens_seen": 2863923200 + }, + { + "epoch": 0.87, + "learning_rate": 6.682046138415246e-05, + "loss": 1.4262, + "theoretical_loss": 3.3326813916497726, + "tokens_seen": 2864054272 + }, + { + "epoch": 0.87, + "learning_rate": 6.680040120361083e-05, + "loss": 1.3974, + "theoretical_loss": 3.332669554587043, + "tokens_seen": 2864185344 + }, + { + "epoch": 0.87, + "learning_rate": 6.678034102306921e-05, + "loss": 1.376, + "theoretical_loss": 3.3326577182176598, + "tokens_seen": 2864316416 + }, + { + "epoch": 0.87, + "learning_rate": 6.676028084252758e-05, + "loss": 1.2557, + "theoretical_loss": 3.33264588254155, + "tokens_seen": 2864447488 + }, + { + "epoch": 0.87, + "learning_rate": 6.674022066198596e-05, + "loss": 1.3393, + "theoretical_loss": 3.332634047558641, + "tokens_seen": 2864578560 + }, + { + "epoch": 0.87, + "learning_rate": 6.672016048144433e-05, + "loss": 1.3275, + "theoretical_loss": 3.332622213268862, + "tokens_seen": 2864709632 + }, + { + "epoch": 0.87, + "learning_rate": 6.670010030090271e-05, + "loss": 1.457, + "theoretical_loss": 3.332610379672139, + "tokens_seen": 2864840704 + }, + { + "epoch": 0.87, + "learning_rate": 6.668004012036109e-05, + "loss": 1.3394, + "theoretical_loss": 3.3325985467684007, + "tokens_seen": 2864971776 + }, + { + "epoch": 0.87, + "learning_rate": 6.665997993981945e-05, + "loss": 1.247, + "theoretical_loss": 3.3325867145575745, + "tokens_seen": 2865102848 + }, + { + "epoch": 0.87, + "learning_rate": 6.663991975927782e-05, + "loss": 1.3859, + "theoretical_loss": 3.3325748830395883, + "tokens_seen": 2865233920 + }, + { + "epoch": 0.87, + "learning_rate": 6.661985957873621e-05, + "loss": 1.282, + "theoretical_loss": 3.3325630522143697, + "tokens_seen": 2865364992 + }, + { + "epoch": 0.87, + "learning_rate": 6.659979939819459e-05, + "loss": 1.4043, + "theoretical_loss": 3.332551222081846, + "tokens_seen": 2865496064 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.4406769275665283, + "objective/train/avg_token_score": 0.017183687537908554, + "objective/train/docs_used": 1613208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.391896367073059, + "objective/train/original_loss": 3.6886258125305176, + "objective/train/theoretical_loss": 3.3325453072753226, + "objective/train/tokens_used": 2865561600, + "objective/train/unlikelihood_loss": 0.1588216871023178, + "theoretical_loss": 3.3325453072753226, + "tokens_seen": 2865561600 + }, + { + "epoch": 0.87, + "learning_rate": 6.657973921765296e-05, + "loss": 1.3865, + "theoretical_loss": 3.3325393926419458, + "tokens_seen": 2865627136 + }, + { + "epoch": 0.87, + "learning_rate": 6.655967903711134e-05, + "loss": 1.3983, + "theoretical_loss": 3.332527563894596, + "tokens_seen": 2865758208 + }, + { + "epoch": 0.87, + "learning_rate": 6.653961885656972e-05, + "loss": 1.2911, + "theoretical_loss": 3.3325157358397255, + "tokens_seen": 2865889280 + }, + { + "epoch": 0.87, + "learning_rate": 6.651955867602809e-05, + "loss": 1.1726, + "theoretical_loss": 3.3325039084772614, + "tokens_seen": 2866020352 + }, + { + "epoch": 0.87, + "learning_rate": 6.649949849548647e-05, + "loss": 1.174, + "theoretical_loss": 3.332492081807131, + "tokens_seen": 2866151424 + }, + { + "epoch": 0.87, + "learning_rate": 6.647943831494484e-05, + "loss": 1.2979, + "theoretical_loss": 3.332480255829263, + "tokens_seen": 2866282496 + }, + { + "epoch": 0.87, + "learning_rate": 6.64593781344032e-05, + "loss": 1.2457, + "theoretical_loss": 3.332468430543585, + "tokens_seen": 2866413568 + }, + { + "epoch": 0.87, + "learning_rate": 6.643931795386158e-05, + "loss": 1.3409, + "theoretical_loss": 3.332456605950025, + "tokens_seen": 2866544640 + }, + { + "epoch": 0.87, + "learning_rate": 6.641925777331996e-05, + "loss": 1.2835, + "theoretical_loss": 3.33244478204851, + "tokens_seen": 2866675712 + }, + { + "epoch": 0.87, + "learning_rate": 6.639919759277833e-05, + "loss": 1.3953, + "theoretical_loss": 3.3324329588389685, + "tokens_seen": 2866806784 + }, + { + "epoch": 0.87, + "learning_rate": 6.637913741223671e-05, + "loss": 1.3204, + "theoretical_loss": 3.3324211363213285, + "tokens_seen": 2866937856 + }, + { + "epoch": 0.87, + "learning_rate": 6.635907723169509e-05, + "loss": 1.464, + "theoretical_loss": 3.332409314495518, + "tokens_seen": 2867068928 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.41361191868782043, + "objective/train/avg_token_score": 0.018469050526618958, + "objective/train/docs_used": 1613631, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2846004962921143, + "objective/train/original_loss": 3.72257399559021, + "objective/train/theoretical_loss": 3.332397493361464, + "objective/train/tokens_used": 2867200000, + "objective/train/unlikelihood_loss": 0.13972750306129456, + "theoretical_loss": 3.332397493361464, + "tokens_seen": 2867200000 + }, + { + "epoch": 0.87, + "learning_rate": 6.633901705115346e-05, + "loss": 1.2034, + "theoretical_loss": 3.332397493361464, + "tokens_seen": 2867200000 + }, + { + "epoch": 0.87, + "learning_rate": 6.631895687061184e-05, + "loss": 1.4544, + "theoretical_loss": 3.332385672919095, + "tokens_seen": 2867331072 + }, + { + "epoch": 0.87, + "learning_rate": 6.629889669007021e-05, + "loss": 1.3936, + "theoretical_loss": 3.332373853168339, + "tokens_seen": 2867462144 + }, + { + "epoch": 0.87, + "learning_rate": 6.627883650952857e-05, + "loss": 1.4206, + "theoretical_loss": 3.3323620341091242, + "tokens_seen": 2867593216 + }, + { + "epoch": 0.87, + "learning_rate": 6.625877632898695e-05, + "loss": 1.3104, + "theoretical_loss": 3.3323502157413776, + "tokens_seen": 2867724288 + }, + { + "epoch": 0.87, + "learning_rate": 6.623871614844534e-05, + "loss": 1.2581, + "theoretical_loss": 3.332338398065028, + "tokens_seen": 2867855360 + }, + { + "epoch": 0.87, + "learning_rate": 6.621865596790372e-05, + "loss": 1.3496, + "theoretical_loss": 3.332326581080003, + "tokens_seen": 2867986432 + }, + { + "epoch": 0.87, + "learning_rate": 6.619859578736209e-05, + "loss": 1.3144, + "theoretical_loss": 3.332314764786231, + "tokens_seen": 2868117504 + }, + { + "epoch": 0.87, + "learning_rate": 6.617853560682047e-05, + "loss": 1.4908, + "theoretical_loss": 3.332302949183639, + "tokens_seen": 2868248576 + }, + { + "epoch": 0.87, + "learning_rate": 6.615847542627884e-05, + "loss": 1.3571, + "theoretical_loss": 3.3322911342721557, + "tokens_seen": 2868379648 + }, + { + "epoch": 0.87, + "learning_rate": 6.613841524573722e-05, + "loss": 1.2527, + "theoretical_loss": 3.3322793200517093, + "tokens_seen": 2868510720 + }, + { + "epoch": 0.87, + "learning_rate": 6.61183550651956e-05, + "loss": 1.3744, + "theoretical_loss": 3.332267506522227, + "tokens_seen": 2868641792 + }, + { + "epoch": 0.87, + "learning_rate": 6.609829488465397e-05, + "loss": 1.4862, + "theoretical_loss": 3.3322556936836376, + "tokens_seen": 2868772864 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.4593108594417572, + "objective/train/avg_token_score": 0.024252774193882942, + "objective/train/docs_used": 1614970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2984389066696167, + "objective/train/original_loss": 3.913759231567383, + "objective/train/theoretical_loss": 3.332249787523405, + "objective/train/tokens_used": 2868838400, + "objective/train/unlikelihood_loss": 0.11159937083721161, + "theoretical_loss": 3.332249787523405, + "tokens_seen": 2868838400 + }, + { + "epoch": 0.87, + "learning_rate": 6.607823470411233e-05, + "loss": 1.3242, + "theoretical_loss": 3.332243881535869, + "tokens_seen": 2868903936 + }, + { + "epoch": 0.87, + "learning_rate": 6.605817452357071e-05, + "loss": 1.3559, + "theoretical_loss": 3.332232070078849, + "tokens_seen": 2869035008 + }, + { + "epoch": 0.87, + "learning_rate": 6.603811434302908e-05, + "loss": 1.397, + "theoretical_loss": 3.332220259312505, + "tokens_seen": 2869166080 + }, + { + "epoch": 0.87, + "learning_rate": 6.601805416248746e-05, + "loss": 1.3636, + "theoretical_loss": 3.3322084492367665, + "tokens_seen": 2869297152 + }, + { + "epoch": 0.87, + "learning_rate": 6.599799398194584e-05, + "loss": 1.3648, + "theoretical_loss": 3.332196639851561, + "tokens_seen": 2869428224 + }, + { + "epoch": 0.87, + "learning_rate": 6.597793380140421e-05, + "loss": 1.2633, + "theoretical_loss": 3.332184831156816, + "tokens_seen": 2869559296 + }, + { + "epoch": 0.87, + "learning_rate": 6.595787362086259e-05, + "loss": 1.2673, + "theoretical_loss": 3.3321730231524604, + "tokens_seen": 2869690368 + }, + { + "epoch": 0.87, + "learning_rate": 6.593781344032096e-05, + "loss": 1.3465, + "theoretical_loss": 3.332161215838422, + "tokens_seen": 2869821440 + }, + { + "epoch": 0.87, + "learning_rate": 6.591775325977934e-05, + "loss": 1.301, + "theoretical_loss": 3.3321494092146287, + "tokens_seen": 2869952512 + }, + { + "epoch": 0.87, + "learning_rate": 6.589769307923771e-05, + "loss": 1.2425, + "theoretical_loss": 3.3321376032810086, + "tokens_seen": 2870083584 + }, + { + "epoch": 0.87, + "learning_rate": 6.587763289869609e-05, + "loss": 1.3284, + "theoretical_loss": 3.3321257980374903, + "tokens_seen": 2870214656 + }, + { + "epoch": 0.87, + "learning_rate": 6.585757271815447e-05, + "loss": 1.4037, + "theoretical_loss": 3.3321139934840014, + "tokens_seen": 2870345728 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.43982160091400146, + "objective/train/avg_token_score": 0.004152040462940931, + "objective/train/docs_used": 1615721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3432600498199463, + "objective/train/original_loss": 3.3957910537719727, + "objective/train/theoretical_loss": 3.3321021896204708, + "objective/train/tokens_used": 2870476800, + "objective/train/unlikelihood_loss": 0.20257171988487244, + "theoretical_loss": 3.3321021896204708, + "tokens_seen": 2870476800 + }, + { + "epoch": 0.87, + "learning_rate": 6.583751253761284e-05, + "loss": 1.416, + "theoretical_loss": 3.3321021896204708, + "tokens_seen": 2870476800 + }, + { + "epoch": 0.87, + "learning_rate": 6.581745235707122e-05, + "loss": 1.3003, + "theoretical_loss": 3.3320903864468256, + "tokens_seen": 2870607872 + }, + { + "epoch": 0.87, + "learning_rate": 6.57973921765296e-05, + "loss": 1.2179, + "theoretical_loss": 3.3320785839629954, + "tokens_seen": 2870738944 + }, + { + "epoch": 0.87, + "learning_rate": 6.577733199598797e-05, + "loss": 1.3438, + "theoretical_loss": 3.3320667821689067, + "tokens_seen": 2870870016 + }, + { + "epoch": 0.87, + "learning_rate": 6.575727181544635e-05, + "loss": 1.4544, + "theoretical_loss": 3.3320549810644895, + "tokens_seen": 2871001088 + }, + { + "epoch": 0.87, + "learning_rate": 6.573721163490472e-05, + "loss": 1.3183, + "theoretical_loss": 3.3320431806496704, + "tokens_seen": 2871132160 + }, + { + "epoch": 0.87, + "learning_rate": 6.57171514543631e-05, + "loss": 1.2482, + "theoretical_loss": 3.332031380924379, + "tokens_seen": 2871263232 + }, + { + "epoch": 0.87, + "learning_rate": 6.569709127382147e-05, + "loss": 1.3709, + "theoretical_loss": 3.332019581888542, + "tokens_seen": 2871394304 + }, + { + "epoch": 0.87, + "learning_rate": 6.567703109327984e-05, + "loss": 1.2322, + "theoretical_loss": 3.332007783542089, + "tokens_seen": 2871525376 + }, + { + "epoch": 0.87, + "learning_rate": 6.565697091273821e-05, + "loss": 1.4849, + "theoretical_loss": 3.331995985884948, + "tokens_seen": 2871656448 + }, + { + "epoch": 0.87, + "learning_rate": 6.563691073219659e-05, + "loss": 1.2936, + "theoretical_loss": 3.3319841889170467, + "tokens_seen": 2871787520 + }, + { + "epoch": 0.87, + "learning_rate": 6.561685055165496e-05, + "loss": 1.3325, + "theoretical_loss": 3.331972392638314, + "tokens_seen": 2871918592 + }, + { + "epoch": 0.87, + "learning_rate": 6.559679037111334e-05, + "loss": 1.4311, + "theoretical_loss": 3.3319605970486776, + "tokens_seen": 2872049664 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.5185728073120117, + "objective/train/avg_token_score": 0.02663896232843399, + "objective/train/docs_used": 1616749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5208244323730469, + "objective/train/original_loss": 3.5708069801330566, + "objective/train/theoretical_loss": 3.331954699512248, + "objective/train/tokens_used": 2872115200, + "objective/train/unlikelihood_loss": 0.15981435775756836, + "theoretical_loss": 3.331954699512248, + "tokens_seen": 2872115200 + }, + { + "epoch": 0.87, + "learning_rate": 6.557673019057171e-05, + "loss": 1.2764, + "theoretical_loss": 3.3319488021480663, + "tokens_seen": 2872180736 + }, + { + "epoch": 0.87, + "learning_rate": 6.555667001003009e-05, + "loss": 1.3624, + "theoretical_loss": 3.331937007936408, + "tokens_seen": 2872311808 + }, + { + "epoch": 0.87, + "learning_rate": 6.553660982948847e-05, + "loss": 1.3946, + "theoretical_loss": 3.3319252144136318, + "tokens_seen": 2872442880 + }, + { + "epoch": 0.87, + "learning_rate": 6.551654964894684e-05, + "loss": 1.3436, + "theoretical_loss": 3.3319134215796646, + "tokens_seen": 2872573952 + }, + { + "epoch": 0.87, + "learning_rate": 6.549648946840522e-05, + "loss": 1.3765, + "theoretical_loss": 3.3319016294344364, + "tokens_seen": 2872705024 + }, + { + "epoch": 0.87, + "learning_rate": 6.547642928786359e-05, + "loss": 1.4517, + "theoretical_loss": 3.331889837977874, + "tokens_seen": 2872836096 + }, + { + "epoch": 0.87, + "learning_rate": 6.545636910732197e-05, + "loss": 1.2604, + "theoretical_loss": 3.331878047209907, + "tokens_seen": 2872967168 + }, + { + "epoch": 0.87, + "learning_rate": 6.543630892678034e-05, + "loss": 1.4177, + "theoretical_loss": 3.3318662571304634, + "tokens_seen": 2873098240 + }, + { + "epoch": 0.87, + "learning_rate": 6.541624874623872e-05, + "loss": 1.2393, + "theoretical_loss": 3.3318544677394715, + "tokens_seen": 2873229312 + }, + { + "epoch": 0.87, + "learning_rate": 6.53961885656971e-05, + "loss": 1.4401, + "theoretical_loss": 3.3318426790368596, + "tokens_seen": 2873360384 + }, + { + "epoch": 0.87, + "learning_rate": 6.537612838515547e-05, + "loss": 1.3579, + "theoretical_loss": 3.3318308910225563, + "tokens_seen": 2873491456 + }, + { + "epoch": 0.87, + "learning_rate": 6.535606820461385e-05, + "loss": 1.3766, + "theoretical_loss": 3.3318191036964895, + "tokens_seen": 2873622528 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.4447091817855835, + "objective/train/avg_token_score": 0.039378996938467026, + "objective/train/docs_used": 1617455, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4377071857452393, + "objective/train/original_loss": 3.944552183151245, + "objective/train/theoretical_loss": 3.3318073170585882, + "objective/train/tokens_used": 2873753600, + "objective/train/unlikelihood_loss": 0.14096328616142273, + "theoretical_loss": 3.3318073170585882, + "tokens_seen": 2873753600 + }, + { + "epoch": 0.87, + "learning_rate": 6.533600802407222e-05, + "loss": 1.4897, + "theoretical_loss": 3.3318073170585882, + "tokens_seen": 2873753600 + }, + { + "epoch": 0.87, + "learning_rate": 6.53159478435306e-05, + "loss": 1.2382, + "theoretical_loss": 3.331795531108781, + "tokens_seen": 2873884672 + }, + { + "epoch": 0.87, + "learning_rate": 6.529588766298896e-05, + "loss": 1.3278, + "theoretical_loss": 3.331783745846996, + "tokens_seen": 2874015744 + }, + { + "epoch": 0.87, + "learning_rate": 6.527582748244734e-05, + "loss": 1.3788, + "theoretical_loss": 3.3317719612731613, + "tokens_seen": 2874146816 + }, + { + "epoch": 0.87, + "learning_rate": 6.525576730190571e-05, + "loss": 1.3148, + "theoretical_loss": 3.331760177387206, + "tokens_seen": 2874277888 + }, + { + "epoch": 0.87, + "learning_rate": 6.523570712136409e-05, + "loss": 1.3892, + "theoretical_loss": 3.3317483941890584, + "tokens_seen": 2874408960 + }, + { + "epoch": 0.87, + "learning_rate": 6.521564694082246e-05, + "loss": 1.4119, + "theoretical_loss": 3.3317366116786467, + "tokens_seen": 2874540032 + }, + { + "epoch": 0.87, + "learning_rate": 6.519558676028084e-05, + "loss": 1.3592, + "theoretical_loss": 3.3317248298559, + "tokens_seen": 2874671104 + }, + { + "epoch": 0.87, + "learning_rate": 6.517552657973922e-05, + "loss": 1.3651, + "theoretical_loss": 3.3317130487207463, + "tokens_seen": 2874802176 + }, + { + "epoch": 0.87, + "learning_rate": 6.515546639919759e-05, + "loss": 1.4356, + "theoretical_loss": 3.331701268273114, + "tokens_seen": 2874933248 + }, + { + "epoch": 0.87, + "learning_rate": 6.513540621865597e-05, + "loss": 1.3672, + "theoretical_loss": 3.3316894885129322, + "tokens_seen": 2875064320 + }, + { + "epoch": 0.87, + "learning_rate": 6.511534603811436e-05, + "loss": 1.3009, + "theoretical_loss": 3.331677709440129, + "tokens_seen": 2875195392 + }, + { + "epoch": 0.87, + "learning_rate": 6.509528585757272e-05, + "loss": 1.3681, + "theoretical_loss": 3.3316659310546335, + "tokens_seen": 2875326464 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.5684261918067932, + "objective/train/avg_token_score": 0.01399162970483303, + "objective/train/docs_used": 1618853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6989622116088867, + "objective/train/original_loss": 3.4395627975463867, + "objective/train/theoretical_loss": 3.331660042119603, + "objective/train/tokens_used": 2875392000, + "objective/train/unlikelihood_loss": 0.1300835907459259, + "theoretical_loss": 3.331660042119603, + "tokens_seen": 2875392000 + }, + { + "epoch": 0.87, + "learning_rate": 6.50752256770311e-05, + "loss": 1.2895, + "theoretical_loss": 3.3316541533563733, + "tokens_seen": 2875457536 + }, + { + "epoch": 0.87, + "learning_rate": 6.505516549648947e-05, + "loss": 1.3017, + "theoretical_loss": 3.331642376345278, + "tokens_seen": 2875588608 + }, + { + "epoch": 0.87, + "learning_rate": 6.503510531594785e-05, + "loss": 1.3593, + "theoretical_loss": 3.331630600021276, + "tokens_seen": 2875719680 + }, + { + "epoch": 0.87, + "learning_rate": 6.501504513540622e-05, + "loss": 1.3346, + "theoretical_loss": 3.331618824384295, + "tokens_seen": 2875850752 + }, + { + "epoch": 0.87, + "learning_rate": 6.49949849548646e-05, + "loss": 1.3057, + "theoretical_loss": 3.3316070494342647, + "tokens_seen": 2875981824 + }, + { + "epoch": 0.87, + "learning_rate": 6.497492477432297e-05, + "loss": 1.4479, + "theoretical_loss": 3.3315952751711126, + "tokens_seen": 2876112896 + }, + { + "epoch": 0.87, + "learning_rate": 6.495486459378135e-05, + "loss": 1.3436, + "theoretical_loss": 3.3315835015947686, + "tokens_seen": 2876243968 + }, + { + "epoch": 0.87, + "learning_rate": 6.493480441323973e-05, + "loss": 1.214, + "theoretical_loss": 3.331571728705161, + "tokens_seen": 2876375040 + }, + { + "epoch": 0.87, + "learning_rate": 6.491474423269809e-05, + "loss": 1.2283, + "theoretical_loss": 3.3315599565022174, + "tokens_seen": 2876506112 + }, + { + "epoch": 0.87, + "learning_rate": 6.489468405215646e-05, + "loss": 1.2869, + "theoretical_loss": 3.331548184985868, + "tokens_seen": 2876637184 + }, + { + "epoch": 0.87, + "learning_rate": 6.487462387161484e-05, + "loss": 1.3593, + "theoretical_loss": 3.331536414156041, + "tokens_seen": 2876768256 + }, + { + "epoch": 0.87, + "learning_rate": 6.485456369107322e-05, + "loss": 1.226, + "theoretical_loss": 3.331524644012664, + "tokens_seen": 2876899328 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.4385386109352112, + "objective/train/avg_token_score": 0.011424999684095383, + "objective/train/docs_used": 1619565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.326205849647522, + "objective/train/original_loss": 3.6918387413024902, + "objective/train/theoretical_loss": 3.331512874555667, + "objective/train/tokens_used": 2877030400, + "objective/train/unlikelihood_loss": 0.1877777874469757, + "theoretical_loss": 3.331512874555667, + "tokens_seen": 2877030400 + }, + { + "epoch": 0.87, + "learning_rate": 6.483450351053159e-05, + "loss": 1.2353, + "theoretical_loss": 3.331512874555667, + "tokens_seen": 2877030400 + }, + { + "epoch": 0.87, + "learning_rate": 6.481444332998997e-05, + "loss": 1.4071, + "theoretical_loss": 3.331501105784978, + "tokens_seen": 2877161472 + }, + { + "epoch": 0.87, + "learning_rate": 6.479438314944834e-05, + "loss": 1.3261, + "theoretical_loss": 3.3314893377005257, + "tokens_seen": 2877292544 + }, + { + "epoch": 0.87, + "learning_rate": 6.477432296890672e-05, + "loss": 1.3699, + "theoretical_loss": 3.3314775703022397, + "tokens_seen": 2877423616 + }, + { + "epoch": 0.87, + "learning_rate": 6.47542627883651e-05, + "loss": 1.3214, + "theoretical_loss": 3.331465803590048, + "tokens_seen": 2877554688 + }, + { + "epoch": 0.87, + "learning_rate": 6.473420260782348e-05, + "loss": 1.2761, + "theoretical_loss": 3.331454037563879, + "tokens_seen": 2877685760 + }, + { + "epoch": 0.87, + "learning_rate": 6.471414242728185e-05, + "loss": 1.3949, + "theoretical_loss": 3.3314422722236623, + "tokens_seen": 2877816832 + }, + { + "epoch": 0.87, + "learning_rate": 6.469408224674022e-05, + "loss": 1.4182, + "theoretical_loss": 3.3314305075693262, + "tokens_seen": 2877947904 + }, + { + "epoch": 0.87, + "learning_rate": 6.46740220661986e-05, + "loss": 1.3736, + "theoretical_loss": 3.3314187436008, + "tokens_seen": 2878078976 + }, + { + "epoch": 0.87, + "learning_rate": 6.465396188565697e-05, + "loss": 1.3066, + "theoretical_loss": 3.3314069803180115, + "tokens_seen": 2878210048 + }, + { + "epoch": 0.87, + "learning_rate": 6.463390170511535e-05, + "loss": 1.3555, + "theoretical_loss": 3.33139521772089, + "tokens_seen": 2878341120 + }, + { + "epoch": 0.87, + "learning_rate": 6.461384152457372e-05, + "loss": 1.3576, + "theoretical_loss": 3.3313834558093647, + "tokens_seen": 2878472192 + }, + { + "epoch": 0.87, + "learning_rate": 6.45937813440321e-05, + "loss": 1.2766, + "theoretical_loss": 3.331371694583364, + "tokens_seen": 2878603264 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.5211387872695923, + "objective/train/avg_token_score": 0.012294248677790165, + "objective/train/docs_used": 1620263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4304755926132202, + "objective/train/original_loss": 3.224083662033081, + "objective/train/theoretical_loss": 3.3313658142274134, + "objective/train/tokens_used": 2878668800, + "objective/train/unlikelihood_loss": 0.192518413066864, + "theoretical_loss": 3.3313658142274134, + "tokens_seen": 2878668800 + }, + { + "epoch": 0.87, + "learning_rate": 6.457372116349048e-05, + "loss": 1.3398, + "theoretical_loss": 3.331359934042817, + "tokens_seen": 2878734336 + }, + { + "epoch": 0.87, + "learning_rate": 6.455366098294885e-05, + "loss": 1.3189, + "theoretical_loss": 3.331348174187652, + "tokens_seen": 2878865408 + }, + { + "epoch": 0.87, + "learning_rate": 6.453360080240723e-05, + "loss": 1.357, + "theoretical_loss": 3.3313364150177986, + "tokens_seen": 2878996480 + }, + { + "epoch": 0.87, + "learning_rate": 6.451354062186559e-05, + "loss": 1.3094, + "theoretical_loss": 3.3313246565331855, + "tokens_seen": 2879127552 + }, + { + "epoch": 0.87, + "learning_rate": 6.449348044132397e-05, + "loss": 1.2951, + "theoretical_loss": 3.331312898733741, + "tokens_seen": 2879258624 + }, + { + "epoch": 0.87, + "learning_rate": 6.447342026078234e-05, + "loss": 1.2274, + "theoretical_loss": 3.3313011416193943, + "tokens_seen": 2879389696 + }, + { + "epoch": 0.87, + "learning_rate": 6.445336008024072e-05, + "loss": 1.1486, + "theoretical_loss": 3.3312893851900744, + "tokens_seen": 2879520768 + }, + { + "epoch": 0.87, + "learning_rate": 6.44332998996991e-05, + "loss": 1.5108, + "theoretical_loss": 3.3312776294457103, + "tokens_seen": 2879651840 + }, + { + "epoch": 0.87, + "learning_rate": 6.441323971915747e-05, + "loss": 1.4204, + "theoretical_loss": 3.3312658743862307, + "tokens_seen": 2879782912 + }, + { + "epoch": 0.87, + "learning_rate": 6.439317953861585e-05, + "loss": 1.3157, + "theoretical_loss": 3.3312541200115646, + "tokens_seen": 2879913984 + }, + { + "epoch": 0.87, + "learning_rate": 6.437311935807423e-05, + "loss": 1.2528, + "theoretical_loss": 3.331242366321641, + "tokens_seen": 2880045056 + }, + { + "epoch": 0.87, + "learning_rate": 6.435305917753261e-05, + "loss": 1.2619, + "theoretical_loss": 3.3312306133163894, + "tokens_seen": 2880176128 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.3613147735595703, + "objective/train/avg_token_score": 0.00828995555639267, + "objective/train/docs_used": 1621719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1412471532821655, + "objective/train/original_loss": 3.9663939476013184, + "objective/train/theoretical_loss": 3.3312188609957376, + "objective/train/tokens_used": 2880307200, + "objective/train/unlikelihood_loss": 0.15132713317871094, + "theoretical_loss": 3.3312188609957376, + "tokens_seen": 2880307200 + }, + { + "epoch": 0.87, + "learning_rate": 6.433299899699099e-05, + "loss": 1.3049, + "theoretical_loss": 3.3312188609957376, + "tokens_seen": 2880307200 + }, + { + "epoch": 0.87, + "learning_rate": 6.431293881644935e-05, + "loss": 1.3149, + "theoretical_loss": 3.3312071093596147, + "tokens_seen": 2880438272 + }, + { + "epoch": 0.87, + "learning_rate": 6.429287863590772e-05, + "loss": 1.2731, + "theoretical_loss": 3.3311953584079506, + "tokens_seen": 2880569344 + }, + { + "epoch": 0.87, + "learning_rate": 6.42728184553661e-05, + "loss": 1.3631, + "theoretical_loss": 3.331183608140674, + "tokens_seen": 2880700416 + }, + { + "epoch": 0.87, + "learning_rate": 6.425275827482448e-05, + "loss": 1.3448, + "theoretical_loss": 3.3311718585577137, + "tokens_seen": 2880831488 + }, + { + "epoch": 0.87, + "learning_rate": 6.423269809428285e-05, + "loss": 1.409, + "theoretical_loss": 3.3311601096589984, + "tokens_seen": 2880962560 + }, + { + "epoch": 0.87, + "learning_rate": 6.421263791374123e-05, + "loss": 1.3704, + "theoretical_loss": 3.3311483614444577, + "tokens_seen": 2881093632 + }, + { + "epoch": 0.87, + "learning_rate": 6.41925777331996e-05, + "loss": 1.3016, + "theoretical_loss": 3.3311366139140204, + "tokens_seen": 2881224704 + }, + { + "epoch": 0.87, + "learning_rate": 6.417251755265798e-05, + "loss": 1.3057, + "theoretical_loss": 3.3311248670676155, + "tokens_seen": 2881355776 + }, + { + "epoch": 0.87, + "learning_rate": 6.415245737211635e-05, + "loss": 1.2378, + "theoretical_loss": 3.331113120905172, + "tokens_seen": 2881486848 + }, + { + "epoch": 0.87, + "learning_rate": 6.413239719157472e-05, + "loss": 1.3458, + "theoretical_loss": 3.331101375426619, + "tokens_seen": 2881617920 + }, + { + "epoch": 0.87, + "learning_rate": 6.411233701103309e-05, + "loss": 1.2651, + "theoretical_loss": 3.331089630631886, + "tokens_seen": 2881748992 + }, + { + "epoch": 0.87, + "learning_rate": 6.409227683049147e-05, + "loss": 1.3386, + "theoretical_loss": 3.3310778865209016, + "tokens_seen": 2881880064 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.4593108594417572, + "objective/train/avg_token_score": 0.013208732940256596, + "objective/train/docs_used": 1622138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4282798767089844, + "objective/train/original_loss": 3.9295661449432373, + "objective/train/theoretical_loss": 3.331072014721793, + "objective/train/tokens_used": 2881945600, + "objective/train/unlikelihood_loss": 0.13330362737178802, + "theoretical_loss": 3.331072014721793, + "tokens_seen": 2881945600 + }, + { + "epoch": 0.87, + "learning_rate": 6.407221664994984e-05, + "loss": 1.343, + "theoretical_loss": 3.3310661430935946, + "tokens_seen": 2882011136 + }, + { + "epoch": 0.87, + "learning_rate": 6.405215646940822e-05, + "loss": 1.2025, + "theoretical_loss": 3.331054400349895, + "tokens_seen": 2882142208 + }, + { + "epoch": 0.87, + "learning_rate": 6.40320962888666e-05, + "loss": 1.3283, + "theoretical_loss": 3.3310426582897312, + "tokens_seen": 2882273280 + }, + { + "epoch": 0.87, + "learning_rate": 6.401203610832497e-05, + "loss": 1.3771, + "theoretical_loss": 3.3310309169130328, + "tokens_seen": 2882404352 + }, + { + "epoch": 0.87, + "learning_rate": 6.399197592778336e-05, + "loss": 1.2835, + "theoretical_loss": 3.331019176219729, + "tokens_seen": 2882535424 + }, + { + "epoch": 0.87, + "learning_rate": 6.397191574724174e-05, + "loss": 1.3721, + "theoretical_loss": 3.3310074362097484, + "tokens_seen": 2882666496 + }, + { + "epoch": 0.87, + "learning_rate": 6.395185556670011e-05, + "loss": 1.0752, + "theoretical_loss": 3.3309956968830203, + "tokens_seen": 2882797568 + }, + { + "epoch": 0.87, + "learning_rate": 6.393179538615847e-05, + "loss": 1.405, + "theoretical_loss": 3.3309839582394742, + "tokens_seen": 2882928640 + }, + { + "epoch": 0.87, + "learning_rate": 6.391173520561685e-05, + "loss": 1.4592, + "theoretical_loss": 3.330972220279039, + "tokens_seen": 2883059712 + }, + { + "epoch": 0.87, + "learning_rate": 6.389167502507523e-05, + "loss": 1.2402, + "theoretical_loss": 3.330960483001644, + "tokens_seen": 2883190784 + }, + { + "epoch": 0.87, + "learning_rate": 6.38716148445336e-05, + "loss": 1.3666, + "theoretical_loss": 3.3309487464072185, + "tokens_seen": 2883321856 + }, + { + "epoch": 0.87, + "learning_rate": 6.385155466399198e-05, + "loss": 1.3954, + "theoretical_loss": 3.330937010495692, + "tokens_seen": 2883452928 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.38556939363479614, + "objective/train/avg_token_score": 0.013066642917692661, + "objective/train/docs_used": 1623211, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2261489629745483, + "objective/train/original_loss": 3.976255178451538, + "objective/train/theoretical_loss": 3.3309252752669924, + "objective/train/tokens_used": 2883584000, + "objective/train/unlikelihood_loss": 0.14143656194210052, + "theoretical_loss": 3.3309252752669924, + "tokens_seen": 2883584000 + }, + { + "epoch": 0.87, + "learning_rate": 6.383149448345035e-05, + "loss": 1.1216, + "theoretical_loss": 3.3309252752669924, + "tokens_seen": 2883584000 + }, + { + "epoch": 0.87, + "learning_rate": 6.381143430290873e-05, + "loss": 1.2771, + "theoretical_loss": 3.3309135407210504, + "tokens_seen": 2883715072 + }, + { + "epoch": 0.87, + "learning_rate": 6.37913741223671e-05, + "loss": 1.369, + "theoretical_loss": 3.3309018068577947, + "tokens_seen": 2883846144 + }, + { + "epoch": 0.87, + "learning_rate": 6.377131394182548e-05, + "loss": 1.4563, + "theoretical_loss": 3.3308900736771547, + "tokens_seen": 2883977216 + }, + { + "epoch": 0.87, + "learning_rate": 6.375125376128386e-05, + "loss": 1.4261, + "theoretical_loss": 3.3308783411790595, + "tokens_seen": 2884108288 + }, + { + "epoch": 0.87, + "learning_rate": 6.373119358074222e-05, + "loss": 1.3609, + "theoretical_loss": 3.3308666093634383, + "tokens_seen": 2884239360 + }, + { + "epoch": 0.87, + "learning_rate": 6.37111334002006e-05, + "loss": 1.3345, + "theoretical_loss": 3.3308548782302205, + "tokens_seen": 2884370432 + }, + { + "epoch": 0.87, + "learning_rate": 6.369107321965897e-05, + "loss": 1.2839, + "theoretical_loss": 3.3308431477793357, + "tokens_seen": 2884501504 + }, + { + "epoch": 0.87, + "learning_rate": 6.367101303911735e-05, + "loss": 1.271, + "theoretical_loss": 3.330831418010713, + "tokens_seen": 2884632576 + }, + { + "epoch": 0.87, + "learning_rate": 6.365095285857572e-05, + "loss": 1.4147, + "theoretical_loss": 3.330819688924281, + "tokens_seen": 2884763648 + }, + { + "epoch": 0.87, + "learning_rate": 6.36308926780341e-05, + "loss": 1.2696, + "theoretical_loss": 3.3308079605199703, + "tokens_seen": 2884894720 + }, + { + "epoch": 0.87, + "learning_rate": 6.361083249749249e-05, + "loss": 1.3404, + "theoretical_loss": 3.3307962327977094, + "tokens_seen": 2885025792 + }, + { + "epoch": 0.87, + "learning_rate": 6.359077231695086e-05, + "loss": 1.3969, + "theoretical_loss": 3.330784505757428, + "tokens_seen": 2885156864 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.3494012653827667, + "objective/train/avg_token_score": 0.008406944572925568, + "objective/train/docs_used": 1623892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0990692377090454, + "objective/train/original_loss": 3.7343385219573975, + "objective/train/theoretical_loss": 3.3307786424930073, + "objective/train/tokens_used": 2885222400, + "objective/train/unlikelihood_loss": 0.1601683348417282, + "theoretical_loss": 3.3307786424930073, + "tokens_seen": 2885222400 + }, + { + "epoch": 0.87, + "learning_rate": 6.357071213640924e-05, + "loss": 1.1778, + "theoretical_loss": 3.330772779399055, + "tokens_seen": 2885287936 + }, + { + "epoch": 0.87, + "learning_rate": 6.355065195586761e-05, + "loss": 1.1863, + "theoretical_loss": 3.3307610537225205, + "tokens_seen": 2885419008 + }, + { + "epoch": 0.87, + "learning_rate": 6.353059177532598e-05, + "loss": 1.3244, + "theoretical_loss": 3.3307493287277534, + "tokens_seen": 2885550080 + }, + { + "epoch": 0.87, + "learning_rate": 6.351053159478435e-05, + "loss": 1.3045, + "theoretical_loss": 3.3307376044146833, + "tokens_seen": 2885681152 + }, + { + "epoch": 0.87, + "learning_rate": 6.349047141424273e-05, + "loss": 1.2556, + "theoretical_loss": 3.3307258807832394, + "tokens_seen": 2885812224 + }, + { + "epoch": 0.87, + "learning_rate": 6.34704112337011e-05, + "loss": 1.302, + "theoretical_loss": 3.330714157833351, + "tokens_seen": 2885943296 + }, + { + "epoch": 0.87, + "learning_rate": 6.345035105315948e-05, + "loss": 1.2016, + "theoretical_loss": 3.3307024355649477, + "tokens_seen": 2886074368 + }, + { + "epoch": 0.87, + "learning_rate": 6.343029087261786e-05, + "loss": 1.4146, + "theoretical_loss": 3.330690713977959, + "tokens_seen": 2886205440 + }, + { + "epoch": 0.87, + "learning_rate": 6.341023069207623e-05, + "loss": 1.4541, + "theoretical_loss": 3.330678993072315, + "tokens_seen": 2886336512 + }, + { + "epoch": 0.87, + "learning_rate": 6.339017051153461e-05, + "loss": 1.4002, + "theoretical_loss": 3.3306672728479434, + "tokens_seen": 2886467584 + }, + { + "epoch": 0.87, + "learning_rate": 6.337011033099298e-05, + "loss": 1.2749, + "theoretical_loss": 3.3306555533047755, + "tokens_seen": 2886598656 + }, + { + "epoch": 0.87, + "learning_rate": 6.335005015045135e-05, + "loss": 1.3344, + "theoretical_loss": 3.3306438344427396, + "tokens_seen": 2886729728 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.45668378472328186, + "objective/train/avg_token_score": 0.009321307763457298, + "objective/train/docs_used": 1624832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.296159267425537, + "objective/train/original_loss": 3.630028009414673, + "objective/train/theoretical_loss": 3.330632116261766, + "objective/train/tokens_used": 2886860800, + "objective/train/unlikelihood_loss": 0.16138891875743866, + "theoretical_loss": 3.330632116261766, + "tokens_seen": 2886860800 + }, + { + "epoch": 0.87, + "learning_rate": 6.332998996990972e-05, + "loss": 1.3048, + "theoretical_loss": 3.330632116261766, + "tokens_seen": 2886860800 + }, + { + "epoch": 0.87, + "learning_rate": 6.33099297893681e-05, + "loss": 1.2116, + "theoretical_loss": 3.3306203987617833, + "tokens_seen": 2886991872 + }, + { + "epoch": 0.87, + "learning_rate": 6.328986960882647e-05, + "loss": 1.1998, + "theoretical_loss": 3.330608681942722, + "tokens_seen": 2887122944 + }, + { + "epoch": 0.87, + "learning_rate": 6.326980942828485e-05, + "loss": 1.453, + "theoretical_loss": 3.330596965804511, + "tokens_seen": 2887254016 + }, + { + "epoch": 0.87, + "learning_rate": 6.324974924774324e-05, + "loss": 1.1217, + "theoretical_loss": 3.33058525034708, + "tokens_seen": 2887385088 + }, + { + "epoch": 0.88, + "learning_rate": 6.322968906720161e-05, + "loss": 1.3154, + "theoretical_loss": 3.3305735355703585, + "tokens_seen": 2887516160 + }, + { + "epoch": 0.88, + "learning_rate": 6.320962888665999e-05, + "loss": 1.3012, + "theoretical_loss": 3.330561821474276, + "tokens_seen": 2887647232 + }, + { + "epoch": 0.88, + "learning_rate": 6.318956870611837e-05, + "loss": 1.3897, + "theoretical_loss": 3.3305501080587625, + "tokens_seen": 2887778304 + }, + { + "epoch": 0.88, + "learning_rate": 6.316950852557674e-05, + "loss": 1.3173, + "theoretical_loss": 3.330538395323747, + "tokens_seen": 2887909376 + }, + { + "epoch": 0.88, + "learning_rate": 6.31494483450351e-05, + "loss": 1.2462, + "theoretical_loss": 3.3305266832691585, + "tokens_seen": 2888040448 + }, + { + "epoch": 0.88, + "learning_rate": 6.312938816449348e-05, + "loss": 1.3055, + "theoretical_loss": 3.330514971894928, + "tokens_seen": 2888171520 + }, + { + "epoch": 0.88, + "learning_rate": 6.310932798395186e-05, + "loss": 1.4154, + "theoretical_loss": 3.3305032612009846, + "tokens_seen": 2888302592 + }, + { + "epoch": 0.88, + "learning_rate": 6.308926780341023e-05, + "loss": 1.1442, + "theoretical_loss": 3.330491551187258, + "tokens_seen": 2888433664 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.3909457325935364, + "objective/train/avg_token_score": 0.020535394549369812, + "objective/train/docs_used": 1625440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.279106855392456, + "objective/train/original_loss": 3.865607738494873, + "objective/train/theoretical_loss": 3.3304856964354537, + "objective/train/tokens_used": 2888499200, + "objective/train/unlikelihood_loss": 0.16304628551006317, + "theoretical_loss": 3.3304856964354537, + "tokens_seen": 2888499200 + }, + { + "epoch": 0.88, + "learning_rate": 6.306920762286861e-05, + "loss": 1.3297, + "theoretical_loss": 3.330479841853677, + "tokens_seen": 2888564736 + }, + { + "epoch": 0.88, + "learning_rate": 6.304914744232698e-05, + "loss": 1.2444, + "theoretical_loss": 3.3304681332001724, + "tokens_seen": 2888695808 + }, + { + "epoch": 0.88, + "learning_rate": 6.302908726178536e-05, + "loss": 1.3934, + "theoretical_loss": 3.3304564252266733, + "tokens_seen": 2888826880 + }, + { + "epoch": 0.88, + "learning_rate": 6.300902708124373e-05, + "loss": 1.4634, + "theoretical_loss": 3.330444717933109, + "tokens_seen": 2888957952 + }, + { + "epoch": 0.88, + "learning_rate": 6.298896690070211e-05, + "loss": 1.3767, + "theoretical_loss": 3.3304330113194096, + "tokens_seen": 2889089024 + }, + { + "epoch": 0.88, + "learning_rate": 6.296890672016049e-05, + "loss": 1.1796, + "theoretical_loss": 3.3304213053855047, + "tokens_seen": 2889220096 + }, + { + "epoch": 0.88, + "learning_rate": 6.294884653961885e-05, + "loss": 1.2047, + "theoretical_loss": 3.330409600131324, + "tokens_seen": 2889351168 + }, + { + "epoch": 0.88, + "learning_rate": 6.292878635907722e-05, + "loss": 1.3022, + "theoretical_loss": 3.330397895556797, + "tokens_seen": 2889482240 + }, + { + "epoch": 0.88, + "learning_rate": 6.29087261785356e-05, + "loss": 1.2558, + "theoretical_loss": 3.3303861916618542, + "tokens_seen": 2889613312 + }, + { + "epoch": 0.88, + "learning_rate": 6.288866599799398e-05, + "loss": 1.2082, + "theoretical_loss": 3.3303744884464246, + "tokens_seen": 2889744384 + }, + { + "epoch": 0.88, + "learning_rate": 6.286860581745236e-05, + "loss": 1.0932, + "theoretical_loss": 3.3303627859104377, + "tokens_seen": 2889875456 + }, + { + "epoch": 0.88, + "learning_rate": 6.284854563691074e-05, + "loss": 1.4798, + "theoretical_loss": 3.3303510840538237, + "tokens_seen": 2890006528 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.38178151845932007, + "objective/train/avg_token_score": 0.014190832152962685, + "objective/train/docs_used": 1626814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2385454177856445, + "objective/train/original_loss": 4.187222480773926, + "objective/train/theoretical_loss": 3.330339382876512, + "objective/train/tokens_used": 2890137600, + "objective/train/unlikelihood_loss": 0.12847599387168884, + "theoretical_loss": 3.330339382876512, + "tokens_seen": 2890137600 + }, + { + "epoch": 0.88, + "learning_rate": 6.282848545636912e-05, + "loss": 1.3261, + "theoretical_loss": 3.330339382876512, + "tokens_seen": 2890137600 + }, + { + "epoch": 0.88, + "learning_rate": 6.280842527582749e-05, + "loss": 1.2799, + "theoretical_loss": 3.330327682378433, + "tokens_seen": 2890268672 + }, + { + "epoch": 0.88, + "learning_rate": 6.278836509528587e-05, + "loss": 1.3002, + "theoretical_loss": 3.330315982559516, + "tokens_seen": 2890399744 + }, + { + "epoch": 0.88, + "learning_rate": 6.276830491474424e-05, + "loss": 1.3483, + "theoretical_loss": 3.3303042834196908, + "tokens_seen": 2890530816 + }, + { + "epoch": 0.88, + "learning_rate": 6.27482447342026e-05, + "loss": 1.2543, + "theoretical_loss": 3.330292584958887, + "tokens_seen": 2890661888 + }, + { + "epoch": 0.88, + "learning_rate": 6.272818455366098e-05, + "loss": 1.3675, + "theoretical_loss": 3.330280887177035, + "tokens_seen": 2890792960 + }, + { + "epoch": 0.88, + "learning_rate": 6.270812437311936e-05, + "loss": 1.3172, + "theoretical_loss": 3.3302691900740644, + "tokens_seen": 2890924032 + }, + { + "epoch": 0.88, + "learning_rate": 6.268806419257773e-05, + "loss": 1.2413, + "theoretical_loss": 3.3302574936499045, + "tokens_seen": 2891055104 + }, + { + "epoch": 0.88, + "learning_rate": 6.266800401203611e-05, + "loss": 1.1565, + "theoretical_loss": 3.3302457979044857, + "tokens_seen": 2891186176 + }, + { + "epoch": 0.88, + "learning_rate": 6.264794383149448e-05, + "loss": 1.1093, + "theoretical_loss": 3.3302341028377374, + "tokens_seen": 2891317248 + }, + { + "epoch": 0.88, + "learning_rate": 6.262788365095286e-05, + "loss": 1.3474, + "theoretical_loss": 3.33022240844959, + "tokens_seen": 2891448320 + }, + { + "epoch": 0.88, + "learning_rate": 6.260782347041124e-05, + "loss": 1.3966, + "theoretical_loss": 3.330210714739973, + "tokens_seen": 2891579392 + }, + { + "epoch": 0.88, + "learning_rate": 6.258776328986961e-05, + "loss": 1.3403, + "theoretical_loss": 3.3301990217088164, + "tokens_seen": 2891710464 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.3145161271095276, + "objective/train/avg_token_score": 0.0159786157310009, + "objective/train/docs_used": 1627476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9842202663421631, + "objective/train/original_loss": 4.0572662353515625, + "objective/train/theoretical_loss": 3.330193175447639, + "objective/train/tokens_used": 2891776000, + "objective/train/unlikelihood_loss": 0.1932132989168167, + "theoretical_loss": 3.330193175447639, + "tokens_seen": 2891776000 + }, + { + "epoch": 0.88, + "learning_rate": 6.256770310932797e-05, + "loss": 1.0973, + "theoretical_loss": 3.3301873293560504, + "tokens_seen": 2891841536 + }, + { + "epoch": 0.88, + "learning_rate": 6.254764292878635e-05, + "loss": 1.3262, + "theoretical_loss": 3.330175637681604, + "tokens_seen": 2891972608 + }, + { + "epoch": 0.88, + "learning_rate": 6.252758274824473e-05, + "loss": 1.2991, + "theoretical_loss": 3.330163946685408, + "tokens_seen": 2892103680 + }, + { + "epoch": 0.88, + "learning_rate": 6.25075225677031e-05, + "loss": 1.345, + "theoretical_loss": 3.3301522563673918, + "tokens_seen": 2892234752 + }, + { + "epoch": 0.88, + "learning_rate": 6.248746238716149e-05, + "loss": 1.1634, + "theoretical_loss": 3.3301405667274855, + "tokens_seen": 2892365824 + }, + { + "epoch": 0.88, + "learning_rate": 6.246740220661985e-05, + "loss": 1.187, + "theoretical_loss": 3.330128877765619, + "tokens_seen": 2892496896 + }, + { + "epoch": 0.88, + "learning_rate": 6.244734202607823e-05, + "loss": 1.4551, + "theoretical_loss": 3.3301171894817223, + "tokens_seen": 2892627968 + }, + { + "epoch": 0.88, + "learning_rate": 6.24272818455366e-05, + "loss": 1.1833, + "theoretical_loss": 3.3301055018757255, + "tokens_seen": 2892759040 + }, + { + "epoch": 0.88, + "learning_rate": 6.240722166499498e-05, + "loss": 1.3033, + "theoretical_loss": 3.330093814947558, + "tokens_seen": 2892890112 + }, + { + "epoch": 0.88, + "learning_rate": 6.238716148445337e-05, + "loss": 1.2089, + "theoretical_loss": 3.3300821286971507, + "tokens_seen": 2893021184 + }, + { + "epoch": 0.88, + "learning_rate": 6.236710130391173e-05, + "loss": 1.19, + "theoretical_loss": 3.3300704431244332, + "tokens_seen": 2893152256 + }, + { + "epoch": 0.88, + "learning_rate": 6.234704112337011e-05, + "loss": 1.361, + "theoretical_loss": 3.330058758229335, + "tokens_seen": 2893283328 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.29154446721076965, + "objective/train/avg_token_score": 0.010086962021887302, + "objective/train/docs_used": 1628768, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8337754607200623, + "objective/train/original_loss": 3.7066619396209717, + "objective/train/theoretical_loss": 3.3300470740117865, + "objective/train/tokens_used": 2893414400, + "objective/train/unlikelihood_loss": 0.19545117020606995, + "theoretical_loss": 3.3300470740117865, + "tokens_seen": 2893414400 + }, + { + "epoch": 0.88, + "learning_rate": 6.232698094282848e-05, + "loss": 1.1654, + "theoretical_loss": 3.3300470740117865, + "tokens_seen": 2893414400 + }, + { + "epoch": 0.88, + "learning_rate": 6.230692076228686e-05, + "loss": 1.4406, + "theoretical_loss": 3.3300353904717177, + "tokens_seen": 2893545472 + }, + { + "epoch": 0.88, + "learning_rate": 6.228686058174524e-05, + "loss": 1.445, + "theoretical_loss": 3.330023707609059, + "tokens_seen": 2893676544 + }, + { + "epoch": 0.88, + "learning_rate": 6.226680040120361e-05, + "loss": 1.2415, + "theoretical_loss": 3.33001202542374, + "tokens_seen": 2893807616 + }, + { + "epoch": 0.88, + "learning_rate": 6.224674022066199e-05, + "loss": 1.1878, + "theoretical_loss": 3.3300003439156907, + "tokens_seen": 2893938688 + }, + { + "epoch": 0.88, + "learning_rate": 6.222668004012036e-05, + "loss": 1.3002, + "theoretical_loss": 3.3299886630848414, + "tokens_seen": 2894069760 + }, + { + "epoch": 0.88, + "learning_rate": 6.220661985957874e-05, + "loss": 1.2524, + "theoretical_loss": 3.3299769829311217, + "tokens_seen": 2894200832 + }, + { + "epoch": 0.88, + "learning_rate": 6.218655967903711e-05, + "loss": 1.1899, + "theoretical_loss": 3.3299653034544625, + "tokens_seen": 2894331904 + }, + { + "epoch": 0.88, + "learning_rate": 6.216649949849549e-05, + "loss": 1.2334, + "theoretical_loss": 3.3299536246547934, + "tokens_seen": 2894462976 + }, + { + "epoch": 0.88, + "learning_rate": 6.214643931795387e-05, + "loss": 1.2031, + "theoretical_loss": 3.3299419465320446, + "tokens_seen": 2894594048 + }, + { + "epoch": 0.88, + "learning_rate": 6.212637913741224e-05, + "loss": 1.2864, + "theoretical_loss": 3.3299302690861463, + "tokens_seen": 2894725120 + }, + { + "epoch": 0.88, + "learning_rate": 6.210631895687062e-05, + "loss": 1.3617, + "theoretical_loss": 3.329918592317028, + "tokens_seen": 2894856192 + }, + { + "epoch": 0.88, + "learning_rate": 6.2086258776329e-05, + "loss": 1.3241, + "theoretical_loss": 3.329906916224621, + "tokens_seen": 2894987264 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.35728248953819275, + "objective/train/avg_token_score": 0.06757847964763641, + "objective/train/docs_used": 1629434, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1836307048797607, + "objective/train/original_loss": 4.506058692932129, + "objective/train/theoretical_loss": 3.3299010784321617, + "objective/train/tokens_used": 2895052800, + "objective/train/unlikelihood_loss": 0.1311463564634323, + "theoretical_loss": 3.3299010784321617, + "tokens_seen": 2895052800 + }, + { + "epoch": 0.88, + "learning_rate": 6.206619859578736e-05, + "loss": 1.3259, + "theoretical_loss": 3.3298952408088542, + "tokens_seen": 2895118336 + }, + { + "epoch": 0.88, + "learning_rate": 6.204613841524573e-05, + "loss": 1.3217, + "theoretical_loss": 3.329883566069659, + "tokens_seen": 2895249408 + }, + { + "epoch": 0.88, + "learning_rate": 6.202607823470412e-05, + "loss": 1.2918, + "theoretical_loss": 3.329871892006964, + "tokens_seen": 2895380480 + }, + { + "epoch": 0.88, + "learning_rate": 6.20060180541625e-05, + "loss": 1.3366, + "theoretical_loss": 3.329860218620701, + "tokens_seen": 2895511552 + }, + { + "epoch": 0.88, + "learning_rate": 6.198595787362086e-05, + "loss": 1.319, + "theoretical_loss": 3.3298485459107994, + "tokens_seen": 2895642624 + }, + { + "epoch": 0.88, + "learning_rate": 6.196589769307923e-05, + "loss": 1.2432, + "theoretical_loss": 3.3298368738771895, + "tokens_seen": 2895773696 + }, + { + "epoch": 0.88, + "learning_rate": 6.194583751253761e-05, + "loss": 1.3615, + "theoretical_loss": 3.3298252025198014, + "tokens_seen": 2895904768 + }, + { + "epoch": 0.88, + "learning_rate": 6.192577733199599e-05, + "loss": 1.0734, + "theoretical_loss": 3.3298135318385653, + "tokens_seen": 2896035840 + }, + { + "epoch": 0.88, + "learning_rate": 6.190571715145436e-05, + "loss": 1.1371, + "theoretical_loss": 3.3298018618334115, + "tokens_seen": 2896166912 + }, + { + "epoch": 0.88, + "learning_rate": 6.188565697091274e-05, + "loss": 1.3437, + "theoretical_loss": 3.3297901925042708, + "tokens_seen": 2896297984 + }, + { + "epoch": 0.88, + "learning_rate": 6.186559679037111e-05, + "loss": 1.1417, + "theoretical_loss": 3.3297785238510724, + "tokens_seen": 2896429056 + }, + { + "epoch": 0.88, + "learning_rate": 6.184553660982949e-05, + "loss": 1.4393, + "theoretical_loss": 3.329766855873747, + "tokens_seen": 2896560128 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.4984726309776306, + "objective/train/avg_token_score": 0.01084060501307249, + "objective/train/docs_used": 1630757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5055627822875977, + "objective/train/original_loss": 3.4355175495147705, + "objective/train/theoretical_loss": 3.329755188572226, + "objective/train/tokens_used": 2896691200, + "objective/train/unlikelihood_loss": 0.11755337566137314, + "theoretical_loss": 3.329755188572226, + "tokens_seen": 2896691200 + }, + { + "epoch": 0.88, + "learning_rate": 6.182547642928787e-05, + "loss": 1.2636, + "theoretical_loss": 3.329755188572226, + "tokens_seen": 2896691200 + }, + { + "epoch": 0.88, + "learning_rate": 6.180541624874624e-05, + "loss": 1.2162, + "theoretical_loss": 3.3297435219464377, + "tokens_seen": 2896822272 + }, + { + "epoch": 0.88, + "learning_rate": 6.178535606820462e-05, + "loss": 1.3506, + "theoretical_loss": 3.329731855996313, + "tokens_seen": 2896953344 + }, + { + "epoch": 0.88, + "learning_rate": 6.176529588766299e-05, + "loss": 1.3426, + "theoretical_loss": 3.3297201907217833, + "tokens_seen": 2897084416 + }, + { + "epoch": 0.88, + "learning_rate": 6.174523570712137e-05, + "loss": 1.4663, + "theoretical_loss": 3.329708526122778, + "tokens_seen": 2897215488 + }, + { + "epoch": 0.88, + "learning_rate": 6.172517552657974e-05, + "loss": 1.1972, + "theoretical_loss": 3.3296968621992273, + "tokens_seen": 2897346560 + }, + { + "epoch": 0.88, + "learning_rate": 6.170511534603812e-05, + "loss": 1.3129, + "theoretical_loss": 3.329685198951062, + "tokens_seen": 2897477632 + }, + { + "epoch": 0.88, + "learning_rate": 6.168505516549648e-05, + "loss": 1.3152, + "theoretical_loss": 3.329673536378212, + "tokens_seen": 2897608704 + }, + { + "epoch": 0.88, + "learning_rate": 6.166499498495486e-05, + "loss": 1.2268, + "theoretical_loss": 3.329661874480608, + "tokens_seen": 2897739776 + }, + { + "epoch": 0.88, + "learning_rate": 6.164493480441325e-05, + "loss": 1.292, + "theoretical_loss": 3.3296502132581804, + "tokens_seen": 2897870848 + }, + { + "epoch": 0.88, + "learning_rate": 6.162487462387162e-05, + "loss": 1.3144, + "theoretical_loss": 3.3296385527108594, + "tokens_seen": 2898001920 + }, + { + "epoch": 0.88, + "learning_rate": 6.160481444333e-05, + "loss": 1.1778, + "theoretical_loss": 3.3296268928385757, + "tokens_seen": 2898132992 + }, + { + "epoch": 0.88, + "learning_rate": 6.158475426278836e-05, + "loss": 1.3296, + "theoretical_loss": 3.3296152336412588, + "tokens_seen": 2898264064 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.41831621527671814, + "objective/train/avg_token_score": 0.018961580470204353, + "objective/train/docs_used": 1631491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1156725883483887, + "objective/train/original_loss": 3.676473617553711, + "objective/train/theoretical_loss": 3.3296094042956916, + "objective/train/tokens_used": 2898329600, + "objective/train/unlikelihood_loss": 0.19601225852966309, + "theoretical_loss": 3.3296094042956916, + "tokens_seen": 2898329600 + }, + { + "epoch": 0.88, + "learning_rate": 6.156469408224674e-05, + "loss": 1.3641, + "theoretical_loss": 3.32960357511884, + "tokens_seen": 2898395136 + }, + { + "epoch": 0.88, + "learning_rate": 6.154463390170511e-05, + "loss": 1.42, + "theoretical_loss": 3.32959191727125, + "tokens_seen": 2898526208 + }, + { + "epoch": 0.88, + "learning_rate": 6.152457372116349e-05, + "loss": 1.4301, + "theoretical_loss": 3.329580260098418, + "tokens_seen": 2898657280 + }, + { + "epoch": 0.88, + "learning_rate": 6.150451354062186e-05, + "loss": 1.3579, + "theoretical_loss": 3.329568603600275, + "tokens_seen": 2898788352 + }, + { + "epoch": 0.88, + "learning_rate": 6.148445336008024e-05, + "loss": 1.2883, + "theoretical_loss": 3.329556947776752, + "tokens_seen": 2898919424 + }, + { + "epoch": 0.88, + "learning_rate": 6.146439317953862e-05, + "loss": 1.3316, + "theoretical_loss": 3.329545292627779, + "tokens_seen": 2899050496 + }, + { + "epoch": 0.88, + "learning_rate": 6.144433299899699e-05, + "loss": 1.2897, + "theoretical_loss": 3.329533638153286, + "tokens_seen": 2899181568 + }, + { + "epoch": 0.88, + "learning_rate": 6.142427281845537e-05, + "loss": 1.2616, + "theoretical_loss": 3.329521984353204, + "tokens_seen": 2899312640 + }, + { + "epoch": 0.88, + "learning_rate": 6.140421263791374e-05, + "loss": 1.2184, + "theoretical_loss": 3.329510331227464, + "tokens_seen": 2899443712 + }, + { + "epoch": 0.88, + "learning_rate": 6.138415245737212e-05, + "loss": 1.2457, + "theoretical_loss": 3.3294986787759955, + "tokens_seen": 2899574784 + }, + { + "epoch": 0.88, + "learning_rate": 6.13640922768305e-05, + "loss": 1.3078, + "theoretical_loss": 3.3294870269987293, + "tokens_seen": 2899705856 + }, + { + "epoch": 0.88, + "learning_rate": 6.134403209628887e-05, + "loss": 1.1539, + "theoretical_loss": 3.3294753758955964, + "tokens_seen": 2899836928 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.4546065628528595, + "objective/train/avg_token_score": 0.009885288774967194, + "objective/train/docs_used": 1632851, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1820050477981567, + "objective/train/original_loss": 3.6019256114959717, + "objective/train/theoretical_loss": 3.329463725466527, + "objective/train/tokens_used": 2899968000, + "objective/train/unlikelihood_loss": 0.11705703288316727, + "theoretical_loss": 3.329463725466527, + "tokens_seen": 2899968000 + }, + { + "epoch": 0.88, + "learning_rate": 6.132397191574725e-05, + "loss": 1.2843, + "theoretical_loss": 3.329463725466527, + "tokens_seen": 2899968000 + }, + { + "epoch": 0.88, + "learning_rate": 6.130391173520561e-05, + "loss": 1.3548, + "theoretical_loss": 3.329452075711451, + "tokens_seen": 2900099072 + }, + { + "epoch": 0.88, + "learning_rate": 6.128385155466398e-05, + "loss": 1.2963, + "theoretical_loss": 3.3294404266303004, + "tokens_seen": 2900230144 + }, + { + "epoch": 0.88, + "learning_rate": 6.126379137412237e-05, + "loss": 1.4015, + "theoretical_loss": 3.3294287782230048, + "tokens_seen": 2900361216 + }, + { + "epoch": 0.88, + "learning_rate": 6.124373119358075e-05, + "loss": 1.1699, + "theoretical_loss": 3.3294171304894946, + "tokens_seen": 2900492288 + }, + { + "epoch": 0.88, + "learning_rate": 6.122367101303913e-05, + "loss": 1.0715, + "theoretical_loss": 3.3294054834297007, + "tokens_seen": 2900623360 + }, + { + "epoch": 0.88, + "learning_rate": 6.120361083249749e-05, + "loss": 1.24, + "theoretical_loss": 3.329393837043554, + "tokens_seen": 2900754432 + }, + { + "epoch": 0.88, + "learning_rate": 6.118355065195586e-05, + "loss": 1.0013, + "theoretical_loss": 3.3293821913309847, + "tokens_seen": 2900885504 + }, + { + "epoch": 0.88, + "learning_rate": 6.116349047141424e-05, + "loss": 1.3061, + "theoretical_loss": 3.329370546291923, + "tokens_seen": 2901016576 + }, + { + "epoch": 0.88, + "learning_rate": 6.114343029087262e-05, + "loss": 1.2007, + "theoretical_loss": 3.3293589019263004, + "tokens_seen": 2901147648 + }, + { + "epoch": 0.88, + "learning_rate": 6.112337011033099e-05, + "loss": 1.1936, + "theoretical_loss": 3.3293472582340473, + "tokens_seen": 2901278720 + }, + { + "epoch": 0.88, + "learning_rate": 6.110330992978937e-05, + "loss": 1.209, + "theoretical_loss": 3.329335615215094, + "tokens_seen": 2901409792 + }, + { + "epoch": 0.88, + "learning_rate": 6.108324974924774e-05, + "loss": 1.2914, + "theoretical_loss": 3.3293239728693713, + "tokens_seen": 2901540864 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.40065982937812805, + "objective/train/avg_token_score": 0.01995098404586315, + "objective/train/docs_used": 1633283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2250099182128906, + "objective/train/original_loss": 4.16509485244751, + "objective/train/theoretical_loss": 3.3293181519489496, + "objective/train/tokens_used": 2901606400, + "objective/train/unlikelihood_loss": 0.1360817700624466, + "theoretical_loss": 3.3293181519489496, + "tokens_seen": 2901606400 + }, + { + "epoch": 0.88, + "learning_rate": 6.106318956870612e-05, + "loss": 1.3229, + "theoretical_loss": 3.32931233119681, + "tokens_seen": 2901671936 + }, + { + "epoch": 0.88, + "learning_rate": 6.10431293881645e-05, + "loss": 1.292, + "theoretical_loss": 3.3293006901973405, + "tokens_seen": 2901803008 + }, + { + "epoch": 0.88, + "learning_rate": 6.102306920762287e-05, + "loss": 1.3224, + "theoretical_loss": 3.329289049870894, + "tokens_seen": 2901934080 + }, + { + "epoch": 0.88, + "learning_rate": 6.1003009027081246e-05, + "loss": 1.393, + "theoretical_loss": 3.3292774102174003, + "tokens_seen": 2902065152 + }, + { + "epoch": 0.88, + "learning_rate": 6.098294884653962e-05, + "loss": 1.3522, + "theoretical_loss": 3.3292657712367912, + "tokens_seen": 2902196224 + }, + { + "epoch": 0.88, + "learning_rate": 6.096288866599799e-05, + "loss": 1.4016, + "theoretical_loss": 3.3292541329289964, + "tokens_seen": 2902327296 + }, + { + "epoch": 0.88, + "learning_rate": 6.0942828485456366e-05, + "loss": 1.2971, + "theoretical_loss": 3.3292424952939474, + "tokens_seen": 2902458368 + }, + { + "epoch": 0.88, + "learning_rate": 6.092276830491474e-05, + "loss": 1.1478, + "theoretical_loss": 3.3292308583315746, + "tokens_seen": 2902589440 + }, + { + "epoch": 0.88, + "learning_rate": 6.0902708124373125e-05, + "loss": 1.273, + "theoretical_loss": 3.3292192220418086, + "tokens_seen": 2902720512 + }, + { + "epoch": 0.88, + "learning_rate": 6.08826479438315e-05, + "loss": 1.2736, + "theoretical_loss": 3.3292075864245803, + "tokens_seen": 2902851584 + }, + { + "epoch": 0.88, + "learning_rate": 6.086258776328987e-05, + "loss": 1.2753, + "theoretical_loss": 3.3291959514798206, + "tokens_seen": 2902982656 + }, + { + "epoch": 0.88, + "learning_rate": 6.0842527582748245e-05, + "loss": 1.3206, + "theoretical_loss": 3.3291843172074604, + "tokens_seen": 2903113728 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.4444648027420044, + "objective/train/avg_token_score": 0.014306573197245598, + "objective/train/docs_used": 1634619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3575096130371094, + "objective/train/original_loss": 3.6034510135650635, + "objective/train/theoretical_loss": 3.32917268360743, + "objective/train/tokens_used": 2903244800, + "objective/train/unlikelihood_loss": 0.12422193586826324, + "theoretical_loss": 3.32917268360743, + "tokens_seen": 2903244800 + }, + { + "epoch": 0.88, + "learning_rate": 6.082246740220662e-05, + "loss": 1.353, + "theoretical_loss": 3.32917268360743, + "tokens_seen": 2903244800 + }, + { + "epoch": 0.88, + "learning_rate": 6.0802407221665e-05, + "loss": 1.3965, + "theoretical_loss": 3.3291610506796605, + "tokens_seen": 2903375872 + }, + { + "epoch": 0.88, + "learning_rate": 6.078234704112337e-05, + "loss": 1.3672, + "theoretical_loss": 3.3291494184240826, + "tokens_seen": 2903506944 + }, + { + "epoch": 0.88, + "learning_rate": 6.076228686058174e-05, + "loss": 1.3366, + "theoretical_loss": 3.3291377868406276, + "tokens_seen": 2903638016 + }, + { + "epoch": 0.88, + "learning_rate": 6.074222668004012e-05, + "loss": 1.26, + "theoretical_loss": 3.329126155929225, + "tokens_seen": 2903769088 + }, + { + "epoch": 0.88, + "learning_rate": 6.072216649949849e-05, + "loss": 1.4088, + "theoretical_loss": 3.3291145256898074, + "tokens_seen": 2903900160 + }, + { + "epoch": 0.88, + "learning_rate": 6.0702106318956876e-05, + "loss": 1.3412, + "theoretical_loss": 3.329102896122304, + "tokens_seen": 2904031232 + }, + { + "epoch": 0.88, + "learning_rate": 6.068204613841525e-05, + "loss": 1.2337, + "theoretical_loss": 3.329091267226647, + "tokens_seen": 2904162304 + }, + { + "epoch": 0.88, + "learning_rate": 6.066198595787362e-05, + "loss": 1.2577, + "theoretical_loss": 3.329079639002767, + "tokens_seen": 2904293376 + }, + { + "epoch": 0.88, + "learning_rate": 6.0641925777331996e-05, + "loss": 1.2642, + "theoretical_loss": 3.3290680114505937, + "tokens_seen": 2904424448 + }, + { + "epoch": 0.88, + "learning_rate": 6.062186559679037e-05, + "loss": 1.3092, + "theoretical_loss": 3.32905638457006, + "tokens_seen": 2904555520 + }, + { + "epoch": 0.88, + "learning_rate": 6.060180541624875e-05, + "loss": 1.2492, + "theoretical_loss": 3.3290447583610945, + "tokens_seen": 2904686592 + }, + { + "epoch": 0.88, + "learning_rate": 6.0581745235707124e-05, + "loss": 1.3051, + "theoretical_loss": 3.3290331328236302, + "tokens_seen": 2904817664 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.38587486743927, + "objective/train/avg_token_score": 0.021833473816514015, + "objective/train/docs_used": 1635246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1521291732788086, + "objective/train/original_loss": 4.005815505981445, + "objective/train/theoretical_loss": 3.3290273203066887, + "objective/train/tokens_used": 2904883200, + "objective/train/unlikelihood_loss": 0.1741725355386734, + "theoretical_loss": 3.3290273203066887, + "tokens_seen": 2904883200 + }, + { + "epoch": 0.88, + "learning_rate": 6.056168505516549e-05, + "loss": 1.3131, + "theoretical_loss": 3.3290215079575964, + "tokens_seen": 2904948736 + }, + { + "epoch": 0.88, + "learning_rate": 6.054162487462387e-05, + "loss": 1.2851, + "theoretical_loss": 3.329009883762925, + "tokens_seen": 2905079808 + }, + { + "epoch": 0.88, + "learning_rate": 6.052156469408225e-05, + "loss": 1.1686, + "theoretical_loss": 3.3289982602395467, + "tokens_seen": 2905210880 + }, + { + "epoch": 0.88, + "learning_rate": 6.050150451354063e-05, + "loss": 1.2832, + "theoretical_loss": 3.3289866373873926, + "tokens_seen": 2905341952 + }, + { + "epoch": 0.88, + "learning_rate": 6.0481444332999e-05, + "loss": 1.3419, + "theoretical_loss": 3.3289750152063933, + "tokens_seen": 2905473024 + }, + { + "epoch": 0.88, + "learning_rate": 6.046138415245737e-05, + "loss": 1.3963, + "theoretical_loss": 3.3289633936964798, + "tokens_seen": 2905604096 + }, + { + "epoch": 0.88, + "learning_rate": 6.044132397191575e-05, + "loss": 1.3644, + "theoretical_loss": 3.3289517728575833, + "tokens_seen": 2905735168 + }, + { + "epoch": 0.88, + "learning_rate": 6.0421263791374123e-05, + "loss": 1.1562, + "theoretical_loss": 3.328940152689635, + "tokens_seen": 2905866240 + }, + { + "epoch": 0.88, + "learning_rate": 6.04012036108325e-05, + "loss": 1.1692, + "theoretical_loss": 3.3289285331925655, + "tokens_seen": 2905997312 + }, + { + "epoch": 0.88, + "learning_rate": 6.0381143430290875e-05, + "loss": 1.4131, + "theoretical_loss": 3.3289169143663058, + "tokens_seen": 2906128384 + }, + { + "epoch": 0.88, + "learning_rate": 6.0361083249749244e-05, + "loss": 1.3351, + "theoretical_loss": 3.3289052962107872, + "tokens_seen": 2906259456 + }, + { + "epoch": 0.88, + "learning_rate": 6.0341023069207627e-05, + "loss": 1.3242, + "theoretical_loss": 3.32889367872594, + "tokens_seen": 2906390528 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.5773460268974304, + "objective/train/avg_token_score": 0.0020925223361700773, + "objective/train/docs_used": 1636404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5995553731918335, + "objective/train/original_loss": 3.1140286922454834, + "objective/train/theoretical_loss": 3.3288820619116963, + "objective/train/tokens_used": 2906521600, + "objective/train/unlikelihood_loss": 0.18722966313362122, + "theoretical_loss": 3.3288820619116963, + "tokens_seen": 2906521600 + }, + { + "epoch": 0.88, + "learning_rate": 6.0320962888666e-05, + "loss": 1.3185, + "theoretical_loss": 3.3288820619116963, + "tokens_seen": 2906521600 + }, + { + "epoch": 0.88, + "learning_rate": 6.030090270812438e-05, + "loss": 1.1465, + "theoretical_loss": 3.328870445767987, + "tokens_seen": 2906652672 + }, + { + "epoch": 0.88, + "learning_rate": 6.0280842527582754e-05, + "loss": 1.1329, + "theoretical_loss": 3.3288588302947426, + "tokens_seen": 2906783744 + }, + { + "epoch": 0.88, + "learning_rate": 6.026078234704112e-05, + "loss": 1.3099, + "theoretical_loss": 3.328847215491894, + "tokens_seen": 2906914816 + }, + { + "epoch": 0.88, + "learning_rate": 6.02407221664995e-05, + "loss": 1.2213, + "theoretical_loss": 3.328835601359373, + "tokens_seen": 2907045888 + }, + { + "epoch": 0.88, + "learning_rate": 6.0220661985957875e-05, + "loss": 1.2633, + "theoretical_loss": 3.3288239878971106, + "tokens_seen": 2907176960 + }, + { + "epoch": 0.88, + "learning_rate": 6.020060180541625e-05, + "loss": 1.182, + "theoretical_loss": 3.3288123751050374, + "tokens_seen": 2907308032 + }, + { + "epoch": 0.88, + "learning_rate": 6.018054162487462e-05, + "loss": 1.2274, + "theoretical_loss": 3.328800762983085, + "tokens_seen": 2907439104 + }, + { + "epoch": 0.88, + "learning_rate": 6.0160481444332995e-05, + "loss": 1.3319, + "theoretical_loss": 3.328789151531184, + "tokens_seen": 2907570176 + }, + { + "epoch": 0.88, + "learning_rate": 6.014042126379138e-05, + "loss": 1.3854, + "theoretical_loss": 3.328777540749266, + "tokens_seen": 2907701248 + }, + { + "epoch": 0.88, + "learning_rate": 6.0120361083249754e-05, + "loss": 1.3254, + "theoretical_loss": 3.328765930637262, + "tokens_seen": 2907832320 + }, + { + "epoch": 0.88, + "learning_rate": 6.010030090270813e-05, + "loss": 1.2986, + "theoretical_loss": 3.3287543211951034, + "tokens_seen": 2907963392 + }, + { + "epoch": 0.88, + "learning_rate": 6.00802407221665e-05, + "loss": 1.2245, + "theoretical_loss": 3.328742712422721, + "tokens_seen": 2908094464 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.5661045908927917, + "objective/train/avg_token_score": 0.010200891643762589, + "objective/train/docs_used": 1636998, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.700148344039917, + "objective/train/original_loss": 3.3453757762908936, + "objective/train/theoretical_loss": 3.328736908287674, + "objective/train/tokens_used": 2908160000, + "objective/train/unlikelihood_loss": 0.18479131162166595, + "theoretical_loss": 3.328736908287674, + "tokens_seen": 2908160000 + }, + { + "epoch": 0.88, + "learning_rate": 6.0060180541624874e-05, + "loss": 1.4252, + "theoretical_loss": 3.328731104320046, + "tokens_seen": 2908225536 + }, + { + "epoch": 0.88, + "learning_rate": 6.004012036108325e-05, + "loss": 1.3302, + "theoretical_loss": 3.3287194968870093, + "tokens_seen": 2908356608 + }, + { + "epoch": 0.88, + "learning_rate": 6.0020060180541626e-05, + "loss": 1.223, + "theoretical_loss": 3.328707890123543, + "tokens_seen": 2908487680 + }, + { + "epoch": 0.88, + "learning_rate": 6e-05, + "loss": 1.3808, + "theoretical_loss": 3.3286962840295775, + "tokens_seen": 2908618752 + }, + { + "epoch": 0.88, + "learning_rate": 5.997993981945837e-05, + "loss": 1.2118, + "theoretical_loss": 3.328684678605044, + "tokens_seen": 2908749824 + }, + { + "epoch": 0.88, + "learning_rate": 5.995987963891675e-05, + "loss": 1.3051, + "theoretical_loss": 3.328673073849875, + "tokens_seen": 2908880896 + }, + { + "epoch": 0.88, + "learning_rate": 5.993981945837513e-05, + "loss": 1.2518, + "theoretical_loss": 3.328661469764, + "tokens_seen": 2909011968 + }, + { + "epoch": 0.88, + "learning_rate": 5.9919759277833505e-05, + "loss": 1.3945, + "theoretical_loss": 3.328649866347351, + "tokens_seen": 2909143040 + }, + { + "epoch": 0.88, + "learning_rate": 5.989969909729188e-05, + "loss": 1.4096, + "theoretical_loss": 3.3286382635998595, + "tokens_seen": 2909274112 + }, + { + "epoch": 0.88, + "learning_rate": 5.987963891675025e-05, + "loss": 1.2859, + "theoretical_loss": 3.328626661521456, + "tokens_seen": 2909405184 + }, + { + "epoch": 0.88, + "learning_rate": 5.9859578736208625e-05, + "loss": 1.2083, + "theoretical_loss": 3.3286150601120728, + "tokens_seen": 2909536256 + }, + { + "epoch": 0.88, + "learning_rate": 5.9839518555667e-05, + "loss": 1.1982, + "theoretical_loss": 3.3286034593716405, + "tokens_seen": 2909667328 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.4330400824546814, + "objective/train/avg_token_score": 0.005203070119023323, + "objective/train/docs_used": 1637612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2916932106018066, + "objective/train/original_loss": 3.4269025325775146, + "objective/train/theoretical_loss": 3.328591859300091, + "objective/train/tokens_used": 2909798400, + "objective/train/unlikelihood_loss": 0.2291727364063263, + "theoretical_loss": 3.328591859300091, + "tokens_seen": 2909798400 + }, + { + "epoch": 0.88, + "learning_rate": 5.981945837512538e-05, + "loss": 1.2816, + "theoretical_loss": 3.328591859300091, + "tokens_seen": 2909798400 + }, + { + "epoch": 0.88, + "learning_rate": 5.979939819458375e-05, + "loss": 1.2076, + "theoretical_loss": 3.328580259897355, + "tokens_seen": 2909929472 + }, + { + "epoch": 0.88, + "learning_rate": 5.977933801404212e-05, + "loss": 1.2542, + "theoretical_loss": 3.3285686611633634, + "tokens_seen": 2910060544 + }, + { + "epoch": 0.88, + "learning_rate": 5.9759277833500504e-05, + "loss": 1.1137, + "theoretical_loss": 3.3285570630980486, + "tokens_seen": 2910191616 + }, + { + "epoch": 0.88, + "learning_rate": 5.973921765295888e-05, + "loss": 1.3128, + "theoretical_loss": 3.328545465701341, + "tokens_seen": 2910322688 + }, + { + "epoch": 0.88, + "learning_rate": 5.9719157472417256e-05, + "loss": 1.267, + "theoretical_loss": 3.3285338689731727, + "tokens_seen": 2910453760 + }, + { + "epoch": 0.88, + "learning_rate": 5.969909729187563e-05, + "loss": 1.3929, + "theoretical_loss": 3.3285222729134754, + "tokens_seen": 2910584832 + }, + { + "epoch": 0.88, + "learning_rate": 5.9679037111334e-05, + "loss": 1.3245, + "theoretical_loss": 3.328510677522179, + "tokens_seen": 2910715904 + }, + { + "epoch": 0.88, + "learning_rate": 5.9658976930792376e-05, + "loss": 1.2684, + "theoretical_loss": 3.328499082799216, + "tokens_seen": 2910846976 + }, + { + "epoch": 0.88, + "learning_rate": 5.963891675025075e-05, + "loss": 1.3032, + "theoretical_loss": 3.3284874887445173, + "tokens_seen": 2910978048 + }, + { + "epoch": 0.88, + "learning_rate": 5.961885656970913e-05, + "loss": 1.2757, + "theoretical_loss": 3.3284758953580145, + "tokens_seen": 2911109120 + }, + { + "epoch": 0.88, + "learning_rate": 5.9598796389167504e-05, + "loss": 1.2685, + "theoretical_loss": 3.328464302639639, + "tokens_seen": 2911240192 + }, + { + "epoch": 0.88, + "learning_rate": 5.957873620862588e-05, + "loss": 1.3731, + "theoretical_loss": 3.328452710589322, + "tokens_seen": 2911371264 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.5169232487678528, + "objective/train/avg_token_score": 0.017247794196009636, + "objective/train/docs_used": 1639054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6239279508590698, + "objective/train/original_loss": 3.545232057571411, + "objective/train/theoretical_loss": 3.3284469148146645, + "objective/train/tokens_used": 2911436800, + "objective/train/unlikelihood_loss": 0.15329045057296753, + "theoretical_loss": 3.3284469148146645, + "tokens_seen": 2911436800 + }, + { + "epoch": 0.88, + "learning_rate": 5.9558676028084255e-05, + "loss": 1.1049, + "theoretical_loss": 3.3284411192069956, + "tokens_seen": 2911502336 + }, + { + "epoch": 0.88, + "learning_rate": 5.953861584754263e-05, + "loss": 1.1307, + "theoretical_loss": 3.328429528492591, + "tokens_seen": 2911633408 + }, + { + "epoch": 0.88, + "learning_rate": 5.951855566700101e-05, + "loss": 1.2602, + "theoretical_loss": 3.3284179384460386, + "tokens_seen": 2911764480 + }, + { + "epoch": 0.88, + "learning_rate": 5.949849548645938e-05, + "loss": 1.2401, + "theoretical_loss": 3.3284063490672713, + "tokens_seen": 2911895552 + }, + { + "epoch": 0.88, + "learning_rate": 5.947843530591775e-05, + "loss": 1.452, + "theoretical_loss": 3.3283947603562196, + "tokens_seen": 2912026624 + }, + { + "epoch": 0.88, + "learning_rate": 5.945837512537613e-05, + "loss": 1.3752, + "theoretical_loss": 3.3283831723128157, + "tokens_seen": 2912157696 + }, + { + "epoch": 0.88, + "learning_rate": 5.9438314944834503e-05, + "loss": 1.4251, + "theoretical_loss": 3.32837158493699, + "tokens_seen": 2912288768 + }, + { + "epoch": 0.88, + "learning_rate": 5.941825476429288e-05, + "loss": 1.2261, + "theoretical_loss": 3.3283599982286756, + "tokens_seen": 2912419840 + }, + { + "epoch": 0.88, + "learning_rate": 5.9398194583751255e-05, + "loss": 1.2534, + "theoretical_loss": 3.3283484121878026, + "tokens_seen": 2912550912 + }, + { + "epoch": 0.88, + "learning_rate": 5.937813440320963e-05, + "loss": 1.3056, + "theoretical_loss": 3.328336826814303, + "tokens_seen": 2912681984 + }, + { + "epoch": 0.88, + "learning_rate": 5.9358074222668007e-05, + "loss": 1.2346, + "theoretical_loss": 3.3283252421081087, + "tokens_seen": 2912813056 + }, + { + "epoch": 0.88, + "learning_rate": 5.933801404212638e-05, + "loss": 1.3051, + "theoretical_loss": 3.328313658069151, + "tokens_seen": 2912944128 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.4637707769870758, + "objective/train/avg_token_score": 0.00774266105145216, + "objective/train/docs_used": 1639882, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.305120825767517, + "objective/train/original_loss": 3.7098798751831055, + "objective/train/theoretical_loss": 3.3283020746973606, + "objective/train/tokens_used": 2913075200, + "objective/train/unlikelihood_loss": 0.16465739905834198, + "theoretical_loss": 3.3283020746973606, + "tokens_seen": 2913075200 + }, + { + "epoch": 0.88, + "learning_rate": 5.931795386158476e-05, + "loss": 1.2774, + "theoretical_loss": 3.3283020746973606, + "tokens_seen": 2913075200 + }, + { + "epoch": 0.88, + "learning_rate": 5.929789368104313e-05, + "loss": 1.289, + "theoretical_loss": 3.3282904919926706, + "tokens_seen": 2913206272 + }, + { + "epoch": 0.88, + "learning_rate": 5.92778335005015e-05, + "loss": 1.1061, + "theoretical_loss": 3.3282789099550114, + "tokens_seen": 2913337344 + }, + { + "epoch": 0.88, + "learning_rate": 5.925777331995988e-05, + "loss": 1.2907, + "theoretical_loss": 3.328267328584315, + "tokens_seen": 2913468416 + }, + { + "epoch": 0.88, + "learning_rate": 5.9237713139418255e-05, + "loss": 1.3687, + "theoretical_loss": 3.328255747880513, + "tokens_seen": 2913599488 + }, + { + "epoch": 0.88, + "learning_rate": 5.921765295887663e-05, + "loss": 1.3681, + "theoretical_loss": 3.328244167843537, + "tokens_seen": 2913730560 + }, + { + "epoch": 0.88, + "learning_rate": 5.9197592778335006e-05, + "loss": 1.2927, + "theoretical_loss": 3.3282325884733184, + "tokens_seen": 2913861632 + }, + { + "epoch": 0.88, + "learning_rate": 5.917753259779338e-05, + "loss": 1.318, + "theoretical_loss": 3.328221009769789, + "tokens_seen": 2913992704 + }, + { + "epoch": 0.88, + "learning_rate": 5.915747241725176e-05, + "loss": 1.4338, + "theoretical_loss": 3.3282094317328808, + "tokens_seen": 2914123776 + }, + { + "epoch": 0.88, + "learning_rate": 5.9137412236710134e-05, + "loss": 1.3881, + "theoretical_loss": 3.3281978543625246, + "tokens_seen": 2914254848 + }, + { + "epoch": 0.88, + "learning_rate": 5.911735205616851e-05, + "loss": 1.1471, + "theoretical_loss": 3.3281862776586526, + "tokens_seen": 2914385920 + }, + { + "epoch": 0.88, + "learning_rate": 5.909729187562688e-05, + "loss": 1.2485, + "theoretical_loss": 3.3281747016211964, + "tokens_seen": 2914516992 + }, + { + "epoch": 0.88, + "learning_rate": 5.9077231695085254e-05, + "loss": 1.3619, + "theoretical_loss": 3.3281631262500877, + "tokens_seen": 2914648064 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.4761730134487152, + "objective/train/avg_token_score": 0.0159001462161541, + "objective/train/docs_used": 1641100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.40587317943573, + "objective/train/original_loss": 3.7493536472320557, + "objective/train/theoretical_loss": 3.3281573388143926, + "objective/train/tokens_used": 2914713600, + "objective/train/unlikelihood_loss": 0.1389016956090927, + "theoretical_loss": 3.3281573388143926, + "tokens_seen": 2914713600 + }, + { + "epoch": 0.88, + "learning_rate": 5.905717151454363e-05, + "loss": 1.3916, + "theoretical_loss": 3.328151551545258, + "tokens_seen": 2914779136 + }, + { + "epoch": 0.88, + "learning_rate": 5.9037111334002006e-05, + "loss": 1.1823, + "theoretical_loss": 3.3281399775066394, + "tokens_seen": 2914910208 + }, + { + "epoch": 0.88, + "learning_rate": 5.901705115346039e-05, + "loss": 1.2093, + "theoretical_loss": 3.328128404134163, + "tokens_seen": 2915041280 + }, + { + "epoch": 0.88, + "learning_rate": 5.899699097291876e-05, + "loss": 1.326, + "theoretical_loss": 3.328116831427761, + "tokens_seen": 2915172352 + }, + { + "epoch": 0.88, + "learning_rate": 5.897693079237713e-05, + "loss": 1.2463, + "theoretical_loss": 3.328105259387365, + "tokens_seen": 2915303424 + }, + { + "epoch": 0.88, + "learning_rate": 5.895687061183551e-05, + "loss": 1.1295, + "theoretical_loss": 3.328093688012906, + "tokens_seen": 2915434496 + }, + { + "epoch": 0.88, + "learning_rate": 5.8936810431293885e-05, + "loss": 1.3311, + "theoretical_loss": 3.328082117304317, + "tokens_seen": 2915565568 + }, + { + "epoch": 0.88, + "learning_rate": 5.891675025075226e-05, + "loss": 1.2873, + "theoretical_loss": 3.328070547261529, + "tokens_seen": 2915696640 + }, + { + "epoch": 0.88, + "learning_rate": 5.889669007021063e-05, + "loss": 1.3564, + "theoretical_loss": 3.3280589778844742, + "tokens_seen": 2915827712 + }, + { + "epoch": 0.88, + "learning_rate": 5.8876629889669005e-05, + "loss": 1.2404, + "theoretical_loss": 3.3280474091730836, + "tokens_seen": 2915958784 + }, + { + "epoch": 0.88, + "learning_rate": 5.885656970912738e-05, + "loss": 1.1543, + "theoretical_loss": 3.32803584112729, + "tokens_seen": 2916089856 + }, + { + "epoch": 0.88, + "learning_rate": 5.8836509528585764e-05, + "loss": 1.3726, + "theoretical_loss": 3.3280242737470243, + "tokens_seen": 2916220928 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.3861803412437439, + "objective/train/avg_token_score": 0.01588043011724949, + "objective/train/docs_used": 1641732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0960444211959839, + "objective/train/original_loss": 3.6837756633758545, + "objective/train/theoretical_loss": 3.3280127070322187, + "objective/train/tokens_used": 2916352000, + "objective/train/unlikelihood_loss": 0.23204754292964935, + "theoretical_loss": 3.3280127070322187, + "tokens_seen": 2916352000 + }, + { + "epoch": 0.88, + "learning_rate": 5.881644934804414e-05, + "loss": 1.2536, + "theoretical_loss": 3.3280127070322187, + "tokens_seen": 2916352000 + }, + { + "epoch": 0.88, + "learning_rate": 5.879638916750251e-05, + "loss": 1.3978, + "theoretical_loss": 3.328001140982805, + "tokens_seen": 2916483072 + }, + { + "epoch": 0.88, + "learning_rate": 5.8776328986960884e-05, + "loss": 1.2918, + "theoretical_loss": 3.327989575598715, + "tokens_seen": 2916614144 + }, + { + "epoch": 0.88, + "learning_rate": 5.875626880641926e-05, + "loss": 1.3649, + "theoretical_loss": 3.3279780108798804, + "tokens_seen": 2916745216 + }, + { + "epoch": 0.88, + "learning_rate": 5.8736208625877636e-05, + "loss": 1.365, + "theoretical_loss": 3.3279664468262333, + "tokens_seen": 2916876288 + }, + { + "epoch": 0.88, + "learning_rate": 5.8716148445336005e-05, + "loss": 1.2577, + "theoretical_loss": 3.3279548834377053, + "tokens_seen": 2917007360 + }, + { + "epoch": 0.88, + "learning_rate": 5.869608826479438e-05, + "loss": 1.226, + "theoretical_loss": 3.3279433207142284, + "tokens_seen": 2917138432 + }, + { + "epoch": 0.88, + "learning_rate": 5.8676028084252756e-05, + "loss": 1.1522, + "theoretical_loss": 3.3279317586557347, + "tokens_seen": 2917269504 + }, + { + "epoch": 0.88, + "learning_rate": 5.865596790371113e-05, + "loss": 1.2833, + "theoretical_loss": 3.3279201972621557, + "tokens_seen": 2917400576 + }, + { + "epoch": 0.88, + "learning_rate": 5.8635907723169515e-05, + "loss": 1.3272, + "theoretical_loss": 3.327908636533423, + "tokens_seen": 2917531648 + }, + { + "epoch": 0.88, + "learning_rate": 5.8615847542627884e-05, + "loss": 1.243, + "theoretical_loss": 3.3278970764694695, + "tokens_seen": 2917662720 + }, + { + "epoch": 0.88, + "learning_rate": 5.859578736208626e-05, + "loss": 1.2615, + "theoretical_loss": 3.3278855170702264, + "tokens_seen": 2917793792 + }, + { + "epoch": 0.88, + "learning_rate": 5.8575727181544635e-05, + "loss": 1.3121, + "theoretical_loss": 3.3278739583356254, + "tokens_seen": 2917924864 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.3415200412273407, + "objective/train/avg_token_score": 0.012168263085186481, + "objective/train/docs_used": 1643189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1077247858047485, + "objective/train/original_loss": 3.587278127670288, + "objective/train/theoretical_loss": 3.327868179217545, + "objective/train/tokens_used": 2917990400, + "objective/train/unlikelihood_loss": 0.23655030131340027, + "theoretical_loss": 3.327868179217545, + "tokens_seen": 2917990400 + }, + { + "epoch": 0.88, + "learning_rate": 5.855566700100301e-05, + "loss": 1.1956, + "theoretical_loss": 3.327862400265599, + "tokens_seen": 2918055936 + }, + { + "epoch": 0.88, + "learning_rate": 5.853560682046139e-05, + "loss": 1.1944, + "theoretical_loss": 3.3278508428600793, + "tokens_seen": 2918187008 + }, + { + "epoch": 0.88, + "learning_rate": 5.8515546639919756e-05, + "loss": 1.2438, + "theoretical_loss": 3.3278392861189974, + "tokens_seen": 2918318080 + }, + { + "epoch": 0.88, + "learning_rate": 5.849548645937813e-05, + "loss": 1.2397, + "theoretical_loss": 3.327827730042286, + "tokens_seen": 2918449152 + }, + { + "epoch": 0.88, + "learning_rate": 5.847542627883651e-05, + "loss": 1.2525, + "theoretical_loss": 3.3278161746298762, + "tokens_seen": 2918580224 + }, + { + "epoch": 0.88, + "learning_rate": 5.845536609829489e-05, + "loss": 1.2402, + "theoretical_loss": 3.3278046198817015, + "tokens_seen": 2918711296 + }, + { + "epoch": 0.88, + "learning_rate": 5.8435305917753266e-05, + "loss": 1.0207, + "theoretical_loss": 3.3277930657976924, + "tokens_seen": 2918842368 + }, + { + "epoch": 0.88, + "learning_rate": 5.8415245737211635e-05, + "loss": 1.3663, + "theoretical_loss": 3.327781512377782, + "tokens_seen": 2918973440 + }, + { + "epoch": 0.88, + "learning_rate": 5.839518555667001e-05, + "loss": 1.2659, + "theoretical_loss": 3.327769959621901, + "tokens_seen": 2919104512 + }, + { + "epoch": 0.88, + "learning_rate": 5.8375125376128387e-05, + "loss": 1.2267, + "theoretical_loss": 3.327758407529983, + "tokens_seen": 2919235584 + }, + { + "epoch": 0.88, + "learning_rate": 5.835506519558676e-05, + "loss": 1.216, + "theoretical_loss": 3.327746856101959, + "tokens_seen": 2919366656 + }, + { + "epoch": 0.88, + "learning_rate": 5.833500501504514e-05, + "loss": 1.1971, + "theoretical_loss": 3.327735305337761, + "tokens_seen": 2919497728 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.306512713432312, + "objective/train/avg_token_score": 0.016246486455202103, + "objective/train/docs_used": 1643758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9362947344779968, + "objective/train/original_loss": 4.464385986328125, + "objective/train/theoretical_loss": 3.327723755237322, + "objective/train/tokens_used": 2919628800, + "objective/train/unlikelihood_loss": 0.15055665373802185, + "theoretical_loss": 3.327723755237322, + "tokens_seen": 2919628800 + }, + { + "epoch": 0.88, + "learning_rate": 5.831494483450351e-05, + "loss": 1.1867, + "theoretical_loss": 3.327723755237322, + "tokens_seen": 2919628800 + }, + { + "epoch": 0.88, + "learning_rate": 5.829488465396188e-05, + "loss": 1.319, + "theoretical_loss": 3.3277122058005726, + "tokens_seen": 2919759872 + }, + { + "epoch": 0.88, + "learning_rate": 5.8274824473420266e-05, + "loss": 1.2217, + "theoretical_loss": 3.327700657027446, + "tokens_seen": 2919890944 + }, + { + "epoch": 0.88, + "learning_rate": 5.825476429287864e-05, + "loss": 1.4195, + "theoretical_loss": 3.327689108917874, + "tokens_seen": 2920022016 + }, + { + "epoch": 0.88, + "learning_rate": 5.823470411233702e-05, + "loss": 1.3003, + "theoretical_loss": 3.3276775614717886, + "tokens_seen": 2920153088 + }, + { + "epoch": 0.88, + "learning_rate": 5.8214643931795386e-05, + "loss": 1.2333, + "theoretical_loss": 3.327666014689122, + "tokens_seen": 2920284160 + }, + { + "epoch": 0.88, + "learning_rate": 5.819458375125376e-05, + "loss": 1.166, + "theoretical_loss": 3.3276544685698064, + "tokens_seen": 2920415232 + }, + { + "epoch": 0.89, + "learning_rate": 5.817452357071214e-05, + "loss": 1.3409, + "theoretical_loss": 3.327642923113774, + "tokens_seen": 2920546304 + }, + { + "epoch": 0.89, + "learning_rate": 5.8154463390170514e-05, + "loss": 1.1496, + "theoretical_loss": 3.327631378320956, + "tokens_seen": 2920677376 + }, + { + "epoch": 0.89, + "learning_rate": 5.813440320962889e-05, + "loss": 1.1386, + "theoretical_loss": 3.3276198341912857, + "tokens_seen": 2920808448 + }, + { + "epoch": 0.89, + "learning_rate": 5.811434302908726e-05, + "loss": 1.2982, + "theoretical_loss": 3.3276082907246947, + "tokens_seen": 2920939520 + }, + { + "epoch": 0.89, + "learning_rate": 5.8094282848545634e-05, + "loss": 1.1752, + "theoretical_loss": 3.3275967479211155, + "tokens_seen": 2921070592 + }, + { + "epoch": 0.89, + "learning_rate": 5.807422266800402e-05, + "loss": 1.3007, + "theoretical_loss": 3.32758520578048, + "tokens_seen": 2921201664 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.2697947323322296, + "objective/train/avg_token_score": 0.01509921345859766, + "objective/train/docs_used": 1645135, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7534770965576172, + "objective/train/original_loss": 4.256246566772461, + "objective/train/theoretical_loss": 3.3275794349587446, + "objective/train/tokens_used": 2921267200, + "objective/train/unlikelihood_loss": 0.21065622568130493, + "theoretical_loss": 3.3275794349587446, + "tokens_seen": 2921267200 + }, + { + "epoch": 0.89, + "learning_rate": 5.805416248746239e-05, + "loss": 1.1246, + "theoretical_loss": 3.3275736643027205, + "tokens_seen": 2921332736 + }, + { + "epoch": 0.89, + "learning_rate": 5.803410230692077e-05, + "loss": 1.3136, + "theoretical_loss": 3.327562123487769, + "tokens_seen": 2921463808 + }, + { + "epoch": 0.89, + "learning_rate": 5.801404212637914e-05, + "loss": 1.2098, + "theoretical_loss": 3.327550583335557, + "tokens_seen": 2921594880 + }, + { + "epoch": 0.89, + "learning_rate": 5.799398194583751e-05, + "loss": 1.3354, + "theoretical_loss": 3.3275390438460186, + "tokens_seen": 2921725952 + }, + { + "epoch": 0.89, + "learning_rate": 5.797392176529589e-05, + "loss": 1.243, + "theoretical_loss": 3.3275275050190847, + "tokens_seen": 2921857024 + }, + { + "epoch": 0.89, + "learning_rate": 5.7953861584754265e-05, + "loss": 1.1725, + "theoretical_loss": 3.327515966854688, + "tokens_seen": 2921988096 + }, + { + "epoch": 0.89, + "learning_rate": 5.7933801404212634e-05, + "loss": 1.1892, + "theoretical_loss": 3.32750442935276, + "tokens_seen": 2922119168 + }, + { + "epoch": 0.89, + "learning_rate": 5.791374122367101e-05, + "loss": 1.3278, + "theoretical_loss": 3.327492892513234, + "tokens_seen": 2922250240 + }, + { + "epoch": 0.89, + "learning_rate": 5.789368104312939e-05, + "loss": 1.2958, + "theoretical_loss": 3.3274813563360413, + "tokens_seen": 2922381312 + }, + { + "epoch": 0.89, + "learning_rate": 5.787362086258777e-05, + "loss": 1.0494, + "theoretical_loss": 3.3274698208211144, + "tokens_seen": 2922512384 + }, + { + "epoch": 0.89, + "learning_rate": 5.7853560682046144e-05, + "loss": 1.1514, + "theoretical_loss": 3.3274582859683863, + "tokens_seen": 2922643456 + }, + { + "epoch": 0.89, + "learning_rate": 5.783350050150451e-05, + "loss": 1.1731, + "theoretical_loss": 3.3274467517777886, + "tokens_seen": 2922774528 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.24499022960662842, + "objective/train/avg_token_score": 0.021615615114569664, + "objective/train/docs_used": 1645886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8127470016479492, + "objective/train/original_loss": 4.519876956939697, + "objective/train/theoretical_loss": 3.3274352182492537, + "objective/train/tokens_used": 2922905600, + "objective/train/unlikelihood_loss": 0.15470807254314423, + "theoretical_loss": 3.3274352182492537, + "tokens_seen": 2922905600 + }, + { + "epoch": 0.89, + "learning_rate": 5.781344032096289e-05, + "loss": 1.0607, + "theoretical_loss": 3.3274352182492537, + "tokens_seen": 2922905600 + }, + { + "epoch": 0.89, + "learning_rate": 5.7793380140421264e-05, + "loss": 1.2805, + "theoretical_loss": 3.3274236853827146, + "tokens_seen": 2923036672 + }, + { + "epoch": 0.89, + "learning_rate": 5.777331995987964e-05, + "loss": 1.1231, + "theoretical_loss": 3.327412153178102, + "tokens_seen": 2923167744 + }, + { + "epoch": 0.89, + "learning_rate": 5.7753259779338016e-05, + "loss": 1.3901, + "theoretical_loss": 3.32740062163535, + "tokens_seen": 2923298816 + }, + { + "epoch": 0.89, + "learning_rate": 5.7733199598796385e-05, + "loss": 1.1226, + "theoretical_loss": 3.32738909075439, + "tokens_seen": 2923429888 + }, + { + "epoch": 0.89, + "learning_rate": 5.771313941825477e-05, + "loss": 1.2883, + "theoretical_loss": 3.3273775605351545, + "tokens_seen": 2923560960 + }, + { + "epoch": 0.89, + "learning_rate": 5.769307923771314e-05, + "loss": 1.1703, + "theoretical_loss": 3.3273660309775757, + "tokens_seen": 2923692032 + }, + { + "epoch": 0.89, + "learning_rate": 5.767301905717152e-05, + "loss": 1.1315, + "theoretical_loss": 3.3273545020815867, + "tokens_seen": 2923823104 + }, + { + "epoch": 0.89, + "learning_rate": 5.7652958876629895e-05, + "loss": 1.1824, + "theoretical_loss": 3.327342973847119, + "tokens_seen": 2923954176 + }, + { + "epoch": 0.89, + "learning_rate": 5.7632898696088264e-05, + "loss": 1.1436, + "theoretical_loss": 3.3273314462741053, + "tokens_seen": 2924085248 + }, + { + "epoch": 0.89, + "learning_rate": 5.761283851554664e-05, + "loss": 1.1583, + "theoretical_loss": 3.3273199193624783, + "tokens_seen": 2924216320 + }, + { + "epoch": 0.89, + "learning_rate": 5.7592778335005015e-05, + "loss": 1.1762, + "theoretical_loss": 3.32730839311217, + "tokens_seen": 2924347392 + }, + { + "epoch": 0.89, + "learning_rate": 5.757271815446339e-05, + "loss": 1.3637, + "theoretical_loss": 3.327296867523113, + "tokens_seen": 2924478464 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.2566593289375305, + "objective/train/avg_token_score": 0.021657725796103477, + "objective/train/docs_used": 1646533, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7204339504241943, + "objective/train/original_loss": 4.355596542358398, + "objective/train/theoretical_loss": 3.327291104976532, + "objective/train/tokens_used": 2924544000, + "objective/train/unlikelihood_loss": 0.14263185858726501, + "theoretical_loss": 3.327291104976532, + "tokens_seen": 2924544000 + }, + { + "epoch": 0.89, + "learning_rate": 5.755265797392177e-05, + "loss": 1.1321, + "theoretical_loss": 3.3272853425952396, + "tokens_seen": 2924609536 + }, + { + "epoch": 0.89, + "learning_rate": 5.7532597793380136e-05, + "loss": 1.0994, + "theoretical_loss": 3.3272738183284822, + "tokens_seen": 2924740608 + }, + { + "epoch": 0.89, + "learning_rate": 5.751253761283852e-05, + "loss": 1.3151, + "theoretical_loss": 3.3272622947227735, + "tokens_seen": 2924871680 + }, + { + "epoch": 0.89, + "learning_rate": 5.7492477432296894e-05, + "loss": 1.0826, + "theoretical_loss": 3.327250771778046, + "tokens_seen": 2925002752 + }, + { + "epoch": 0.89, + "learning_rate": 5.747241725175527e-05, + "loss": 1.2982, + "theoretical_loss": 3.327239249494232, + "tokens_seen": 2925133824 + }, + { + "epoch": 0.89, + "learning_rate": 5.7452357071213646e-05, + "loss": 1.2361, + "theoretical_loss": 3.3272277278712643, + "tokens_seen": 2925264896 + }, + { + "epoch": 0.89, + "learning_rate": 5.7432296890672015e-05, + "loss": 1.1174, + "theoretical_loss": 3.3272162069090747, + "tokens_seen": 2925395968 + }, + { + "epoch": 0.89, + "learning_rate": 5.741223671013039e-05, + "loss": 1.1555, + "theoretical_loss": 3.327204686607596, + "tokens_seen": 2925527040 + }, + { + "epoch": 0.89, + "learning_rate": 5.7392176529588767e-05, + "loss": 1.2544, + "theoretical_loss": 3.327193166966761, + "tokens_seen": 2925658112 + }, + { + "epoch": 0.89, + "learning_rate": 5.737211634904714e-05, + "loss": 1.3775, + "theoretical_loss": 3.327181647986502, + "tokens_seen": 2925789184 + }, + { + "epoch": 0.89, + "learning_rate": 5.735205616850552e-05, + "loss": 1.2525, + "theoretical_loss": 3.3271701296667517, + "tokens_seen": 2925920256 + }, + { + "epoch": 0.89, + "learning_rate": 5.7331995987963894e-05, + "loss": 1.2441, + "theoretical_loss": 3.3271586120074423, + "tokens_seen": 2926051328 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.38110947608947754, + "objective/train/avg_token_score": 0.022239677608013153, + "objective/train/docs_used": 1647827, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2302402257919312, + "objective/train/original_loss": 4.154566764831543, + "objective/train/theoretical_loss": 3.327147095008507, + "objective/train/tokens_used": 2926182400, + "objective/train/unlikelihood_loss": 0.14194540679454803, + "theoretical_loss": 3.327147095008507, + "tokens_seen": 2926182400 + }, + { + "epoch": 0.89, + "learning_rate": 5.731193580742227e-05, + "loss": 1.3266, + "theoretical_loss": 3.327147095008507, + "tokens_seen": 2926182400 + }, + { + "epoch": 0.89, + "learning_rate": 5.7291875626880646e-05, + "loss": 1.3731, + "theoretical_loss": 3.327135578669877, + "tokens_seen": 2926313472 + }, + { + "epoch": 0.89, + "learning_rate": 5.727181544633902e-05, + "loss": 1.2344, + "theoretical_loss": 3.3271240629914867, + "tokens_seen": 2926444544 + }, + { + "epoch": 0.89, + "learning_rate": 5.725175526579739e-05, + "loss": 1.3051, + "theoretical_loss": 3.3271125479732673, + "tokens_seen": 2926575616 + }, + { + "epoch": 0.89, + "learning_rate": 5.7231695085255766e-05, + "loss": 1.1676, + "theoretical_loss": 3.327101033615152, + "tokens_seen": 2926706688 + }, + { + "epoch": 0.89, + "learning_rate": 5.721163490471414e-05, + "loss": 1.2245, + "theoretical_loss": 3.327089519917073, + "tokens_seen": 2926837760 + }, + { + "epoch": 0.89, + "learning_rate": 5.719157472417252e-05, + "loss": 1.1757, + "theoretical_loss": 3.3270780068789634, + "tokens_seen": 2926968832 + }, + { + "epoch": 0.89, + "learning_rate": 5.7171514543630893e-05, + "loss": 1.2412, + "theoretical_loss": 3.3270664945007553, + "tokens_seen": 2927099904 + }, + { + "epoch": 0.89, + "learning_rate": 5.715145436308926e-05, + "loss": 1.1445, + "theoretical_loss": 3.3270549827823817, + "tokens_seen": 2927230976 + }, + { + "epoch": 0.89, + "learning_rate": 5.7131394182547645e-05, + "loss": 1.1708, + "theoretical_loss": 3.3270434717237753, + "tokens_seen": 2927362048 + }, + { + "epoch": 0.89, + "learning_rate": 5.711133400200602e-05, + "loss": 1.2382, + "theoretical_loss": 3.3270319613248684, + "tokens_seen": 2927493120 + }, + { + "epoch": 0.89, + "learning_rate": 5.70912738214644e-05, + "loss": 1.1015, + "theoretical_loss": 3.327020451585594, + "tokens_seen": 2927624192 + }, + { + "epoch": 0.89, + "learning_rate": 5.707121364092277e-05, + "loss": 1.3012, + "theoretical_loss": 3.3270089425058846, + "tokens_seen": 2927755264 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.3965664803981781, + "objective/train/avg_token_score": 0.010380005463957787, + "objective/train/docs_used": 1648383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.168731927871704, + "objective/train/original_loss": 3.9091763496398926, + "objective/train/theoretical_loss": 3.3270031882133457, + "objective/train/tokens_used": 2927820800, + "objective/train/unlikelihood_loss": 0.14760564267635345, + "theoretical_loss": 3.3270031882133457, + "tokens_seen": 2927820800 + }, + { + "epoch": 0.89, + "learning_rate": 5.705115346038114e-05, + "loss": 1.2487, + "theoretical_loss": 3.3269974340856727, + "tokens_seen": 2927886336 + }, + { + "epoch": 0.89, + "learning_rate": 5.703109327983952e-05, + "loss": 1.226, + "theoretical_loss": 3.3269859263248915, + "tokens_seen": 2928017408 + }, + { + "epoch": 0.89, + "learning_rate": 5.701103309929789e-05, + "loss": 1.0914, + "theoretical_loss": 3.326974419223473, + "tokens_seen": 2928148480 + }, + { + "epoch": 0.89, + "learning_rate": 5.699097291875627e-05, + "loss": 1.296, + "theoretical_loss": 3.326962912781351, + "tokens_seen": 2928279552 + }, + { + "epoch": 0.89, + "learning_rate": 5.6970912738214645e-05, + "loss": 1.281, + "theoretical_loss": 3.3269514069984565, + "tokens_seen": 2928410624 + }, + { + "epoch": 0.89, + "learning_rate": 5.695085255767302e-05, + "loss": 1.2428, + "theoretical_loss": 3.3269399018747237, + "tokens_seen": 2928541696 + }, + { + "epoch": 0.89, + "learning_rate": 5.6930792377131396e-05, + "loss": 1.167, + "theoretical_loss": 3.326928397410085, + "tokens_seen": 2928672768 + }, + { + "epoch": 0.89, + "learning_rate": 5.691073219658977e-05, + "loss": 1.1669, + "theoretical_loss": 3.326916893604473, + "tokens_seen": 2928803840 + }, + { + "epoch": 0.89, + "learning_rate": 5.689067201604815e-05, + "loss": 1.252, + "theoretical_loss": 3.3269053904578203, + "tokens_seen": 2928934912 + }, + { + "epoch": 0.89, + "learning_rate": 5.6870611835506524e-05, + "loss": 1.1615, + "theoretical_loss": 3.3268938879700602, + "tokens_seen": 2929065984 + }, + { + "epoch": 0.89, + "learning_rate": 5.685055165496489e-05, + "loss": 1.1563, + "theoretical_loss": 3.3268823861411247, + "tokens_seen": 2929197056 + }, + { + "epoch": 0.89, + "learning_rate": 5.683049147442327e-05, + "loss": 1.3117, + "theoretical_loss": 3.326870884970947, + "tokens_seen": 2929328128 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.37939882278442383, + "objective/train/avg_token_score": 0.019319526851177216, + "objective/train/docs_used": 1649763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1360234022140503, + "objective/train/original_loss": 3.9247238636016846, + "objective/train/theoretical_loss": 3.32685938445946, + "objective/train/tokens_used": 2929459200, + "objective/train/unlikelihood_loss": 0.13622048497200012, + "theoretical_loss": 3.32685938445946, + "tokens_seen": 2929459200 + }, + { + "epoch": 0.89, + "learning_rate": 5.6810431293881644e-05, + "loss": 1.2061, + "theoretical_loss": 3.32685938445946, + "tokens_seen": 2929459200 + }, + { + "epoch": 0.89, + "learning_rate": 5.679037111334002e-05, + "loss": 1.239, + "theoretical_loss": 3.3268478846065968, + "tokens_seen": 2929590272 + }, + { + "epoch": 0.89, + "learning_rate": 5.67703109327984e-05, + "loss": 1.2699, + "theoretical_loss": 3.3268363854122893, + "tokens_seen": 2929721344 + }, + { + "epoch": 0.89, + "learning_rate": 5.675025075225677e-05, + "loss": 1.2171, + "theoretical_loss": 3.326824886876471, + "tokens_seen": 2929852416 + }, + { + "epoch": 0.89, + "learning_rate": 5.673019057171515e-05, + "loss": 1.0057, + "theoretical_loss": 3.3268133889990747, + "tokens_seen": 2929983488 + }, + { + "epoch": 0.89, + "learning_rate": 5.671013039117352e-05, + "loss": 1.3325, + "theoretical_loss": 3.326801891780033, + "tokens_seen": 2930114560 + }, + { + "epoch": 0.89, + "learning_rate": 5.66900702106319e-05, + "loss": 1.3205, + "theoretical_loss": 3.326790395219279, + "tokens_seen": 2930245632 + }, + { + "epoch": 0.89, + "learning_rate": 5.6670010030090275e-05, + "loss": 1.3306, + "theoretical_loss": 3.3267788993167455, + "tokens_seen": 2930376704 + }, + { + "epoch": 0.89, + "learning_rate": 5.6649949849548644e-05, + "loss": 1.1655, + "theoretical_loss": 3.326767404072365, + "tokens_seen": 2930507776 + }, + { + "epoch": 0.89, + "learning_rate": 5.662988966900702e-05, + "loss": 1.2686, + "theoretical_loss": 3.3267559094860712, + "tokens_seen": 2930638848 + }, + { + "epoch": 0.89, + "learning_rate": 5.6609829488465395e-05, + "loss": 1.2712, + "theoretical_loss": 3.3267444155577963, + "tokens_seen": 2930769920 + }, + { + "epoch": 0.89, + "learning_rate": 5.658976930792377e-05, + "loss": 1.3936, + "theoretical_loss": 3.3267329222874733, + "tokens_seen": 2930900992 + }, + { + "epoch": 0.89, + "learning_rate": 5.6569709127382154e-05, + "loss": 1.0946, + "theoretical_loss": 3.326721429675035, + "tokens_seen": 2931032064 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.47061339020729065, + "objective/train/avg_token_score": 0.005348629783838987, + "objective/train/docs_used": 1650742, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4176630973815918, + "objective/train/original_loss": 3.5441367626190186, + "objective/train/theoretical_loss": 3.326715683615502, + "objective/train/tokens_used": 2931097600, + "objective/train/unlikelihood_loss": 0.1363418996334076, + "theoretical_loss": 3.326715683615502, + "tokens_seen": 2931097600 + }, + { + "epoch": 0.89, + "learning_rate": 5.654964894684052e-05, + "loss": 1.2535, + "theoretical_loss": 3.3267099377204152, + "tokens_seen": 2931163136 + }, + { + "epoch": 0.89, + "learning_rate": 5.65295887662989e-05, + "loss": 1.3347, + "theoretical_loss": 3.3266984464235456, + "tokens_seen": 2931294208 + }, + { + "epoch": 0.89, + "learning_rate": 5.6509528585757274e-05, + "loss": 1.2375, + "theoretical_loss": 3.32668695578436, + "tokens_seen": 2931425280 + }, + { + "epoch": 0.89, + "learning_rate": 5.648946840521565e-05, + "loss": 1.2879, + "theoretical_loss": 3.3266754658027913, + "tokens_seen": 2931556352 + }, + { + "epoch": 0.89, + "learning_rate": 5.646940822467402e-05, + "loss": 1.24, + "theoretical_loss": 3.326663976478772, + "tokens_seen": 2931687424 + }, + { + "epoch": 0.89, + "learning_rate": 5.6449348044132395e-05, + "loss": 1.364, + "theoretical_loss": 3.326652487812235, + "tokens_seen": 2931818496 + }, + { + "epoch": 0.89, + "learning_rate": 5.642928786359077e-05, + "loss": 1.1673, + "theoretical_loss": 3.3266409998031143, + "tokens_seen": 2931949568 + }, + { + "epoch": 0.89, + "learning_rate": 5.6409227683049147e-05, + "loss": 1.2204, + "theoretical_loss": 3.3266295124513423, + "tokens_seen": 2932080640 + }, + { + "epoch": 0.89, + "learning_rate": 5.638916750250753e-05, + "loss": 1.2866, + "theoretical_loss": 3.326618025756851, + "tokens_seen": 2932211712 + }, + { + "epoch": 0.89, + "learning_rate": 5.63691073219659e-05, + "loss": 1.1281, + "theoretical_loss": 3.326606539719575, + "tokens_seen": 2932342784 + }, + { + "epoch": 0.89, + "learning_rate": 5.6349047141424274e-05, + "loss": 1.2666, + "theoretical_loss": 3.3265950543394465, + "tokens_seen": 2932473856 + }, + { + "epoch": 0.89, + "learning_rate": 5.632898696088265e-05, + "loss": 1.2563, + "theoretical_loss": 3.326583569616399, + "tokens_seen": 2932604928 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.3084677457809448, + "objective/train/avg_token_score": 0.028737593442201614, + "objective/train/docs_used": 1651324, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9479824900627136, + "objective/train/original_loss": 4.095191478729248, + "objective/train/theoretical_loss": 3.3265720855503647, + "objective/train/tokens_used": 2932736000, + "objective/train/unlikelihood_loss": 0.16108188033103943, + "theoretical_loss": 3.3265720855503647, + "tokens_seen": 2932736000 + }, + { + "epoch": 0.89, + "learning_rate": 5.6308926780341025e-05, + "loss": 1.09, + "theoretical_loss": 3.3265720855503647, + "tokens_seen": 2932736000 + }, + { + "epoch": 0.89, + "learning_rate": 5.62888665997994e-05, + "loss": 1.2265, + "theoretical_loss": 3.326560602141277, + "tokens_seen": 2932867072 + }, + { + "epoch": 0.89, + "learning_rate": 5.626880641925777e-05, + "loss": 1.2808, + "theoretical_loss": 3.3265491193890697, + "tokens_seen": 2932998144 + }, + { + "epoch": 0.89, + "learning_rate": 5.6248746238716146e-05, + "loss": 1.4312, + "theoretical_loss": 3.3265376372936752, + "tokens_seen": 2933129216 + }, + { + "epoch": 0.89, + "learning_rate": 5.622868605817452e-05, + "loss": 1.1274, + "theoretical_loss": 3.3265261558550265, + "tokens_seen": 2933260288 + }, + { + "epoch": 0.89, + "learning_rate": 5.6208625877632904e-05, + "loss": 1.2733, + "theoretical_loss": 3.326514675073057, + "tokens_seen": 2933391360 + }, + { + "epoch": 0.89, + "learning_rate": 5.618856569709128e-05, + "loss": 1.134, + "theoretical_loss": 3.3265031949476995, + "tokens_seen": 2933522432 + }, + { + "epoch": 0.89, + "learning_rate": 5.616850551654965e-05, + "loss": 1.1203, + "theoretical_loss": 3.3264917154788876, + "tokens_seen": 2933653504 + }, + { + "epoch": 0.89, + "learning_rate": 5.6148445336008025e-05, + "loss": 1.2049, + "theoretical_loss": 3.326480236666554, + "tokens_seen": 2933784576 + }, + { + "epoch": 0.89, + "learning_rate": 5.61283851554664e-05, + "loss": 1.3873, + "theoretical_loss": 3.326468758510632, + "tokens_seen": 2933915648 + }, + { + "epoch": 0.89, + "learning_rate": 5.610832497492478e-05, + "loss": 1.1543, + "theoretical_loss": 3.3264572810110544, + "tokens_seen": 2934046720 + }, + { + "epoch": 0.89, + "learning_rate": 5.608826479438315e-05, + "loss": 1.3281, + "theoretical_loss": 3.326445804167755, + "tokens_seen": 2934177792 + }, + { + "epoch": 0.89, + "learning_rate": 5.606820461384152e-05, + "loss": 1.3037, + "theoretical_loss": 3.3264343279806665, + "tokens_seen": 2934308864 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.4626099765300751, + "objective/train/avg_token_score": 0.013979252427816391, + "objective/train/docs_used": 1652566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.355569839477539, + "objective/train/original_loss": 3.638237953186035, + "objective/train/theoretical_loss": 3.3264285901331805, + "objective/train/tokens_used": 2934374400, + "objective/train/unlikelihood_loss": 0.1562100499868393, + "theoretical_loss": 3.3264285901331805, + "tokens_seen": 2934374400 + }, + { + "epoch": 0.89, + "learning_rate": 5.60481444332999e-05, + "loss": 1.3098, + "theoretical_loss": 3.3264228524497224, + "tokens_seen": 2934439936 + }, + { + "epoch": 0.89, + "learning_rate": 5.602808425275827e-05, + "loss": 1.2, + "theoretical_loss": 3.326411377574855, + "tokens_seen": 2934571008 + }, + { + "epoch": 0.89, + "learning_rate": 5.6008024072216656e-05, + "loss": 1.3961, + "theoretical_loss": 3.3263999033559983, + "tokens_seen": 2934702080 + }, + { + "epoch": 0.89, + "learning_rate": 5.598796389167503e-05, + "loss": 1.1667, + "theoretical_loss": 3.3263884297930857, + "tokens_seen": 2934833152 + }, + { + "epoch": 0.89, + "learning_rate": 5.59679037111334e-05, + "loss": 1.2075, + "theoretical_loss": 3.32637695688605, + "tokens_seen": 2934964224 + }, + { + "epoch": 0.89, + "learning_rate": 5.5947843530591776e-05, + "loss": 1.2857, + "theoretical_loss": 3.3263654846348243, + "tokens_seen": 2935095296 + }, + { + "epoch": 0.89, + "learning_rate": 5.592778335005015e-05, + "loss": 1.0976, + "theoretical_loss": 3.3263540130393423, + "tokens_seen": 2935226368 + }, + { + "epoch": 0.89, + "learning_rate": 5.590772316950853e-05, + "loss": 1.2585, + "theoretical_loss": 3.3263425420995367, + "tokens_seen": 2935357440 + }, + { + "epoch": 0.89, + "learning_rate": 5.5887662988966904e-05, + "loss": 1.3461, + "theoretical_loss": 3.326331071815341, + "tokens_seen": 2935488512 + }, + { + "epoch": 0.89, + "learning_rate": 5.586760280842527e-05, + "loss": 1.3737, + "theoretical_loss": 3.3263196021866883, + "tokens_seen": 2935619584 + }, + { + "epoch": 0.89, + "learning_rate": 5.584754262788365e-05, + "loss": 1.43, + "theoretical_loss": 3.3263081332135123, + "tokens_seen": 2935750656 + }, + { + "epoch": 0.89, + "learning_rate": 5.582748244734203e-05, + "loss": 1.4614, + "theoretical_loss": 3.3262966648957457, + "tokens_seen": 2935881728 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.36271995306015015, + "objective/train/avg_token_score": 0.032035600394010544, + "objective/train/docs_used": 1653144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0375267267227173, + "objective/train/original_loss": 4.282741069793701, + "objective/train/theoretical_loss": 3.326285197233322, + "objective/train/tokens_used": 2936012800, + "objective/train/unlikelihood_loss": 0.08842463046312332, + "theoretical_loss": 3.326285197233322, + "tokens_seen": 2936012800 + }, + { + "epoch": 0.89, + "learning_rate": 5.580742226680041e-05, + "loss": 1.3826, + "theoretical_loss": 3.326285197233322, + "tokens_seen": 2936012800 + }, + { + "epoch": 0.89, + "learning_rate": 5.5787362086258776e-05, + "loss": 1.2662, + "theoretical_loss": 3.326273730226175, + "tokens_seen": 2936143872 + }, + { + "epoch": 0.89, + "learning_rate": 5.576730190571715e-05, + "loss": 1.1654, + "theoretical_loss": 3.3262622638742374, + "tokens_seen": 2936274944 + }, + { + "epoch": 0.89, + "learning_rate": 5.574724172517553e-05, + "loss": 1.3484, + "theoretical_loss": 3.3262507981774423, + "tokens_seen": 2936406016 + }, + { + "epoch": 0.89, + "learning_rate": 5.57271815446339e-05, + "loss": 1.2771, + "theoretical_loss": 3.326239333135724, + "tokens_seen": 2936537088 + }, + { + "epoch": 0.89, + "learning_rate": 5.570712136409228e-05, + "loss": 1.2055, + "theoretical_loss": 3.326227868749015, + "tokens_seen": 2936668160 + }, + { + "epoch": 0.89, + "learning_rate": 5.568706118355065e-05, + "loss": 1.1779, + "theoretical_loss": 3.3262164050172487, + "tokens_seen": 2936799232 + }, + { + "epoch": 0.89, + "learning_rate": 5.5667001003009024e-05, + "loss": 1.1636, + "theoretical_loss": 3.326204941940359, + "tokens_seen": 2936930304 + }, + { + "epoch": 0.89, + "learning_rate": 5.5646940822467406e-05, + "loss": 1.3261, + "theoretical_loss": 3.3261934795182793, + "tokens_seen": 2937061376 + }, + { + "epoch": 0.89, + "learning_rate": 5.562688064192578e-05, + "loss": 1.1925, + "theoretical_loss": 3.326182017750942, + "tokens_seen": 2937192448 + }, + { + "epoch": 0.89, + "learning_rate": 5.560682046138416e-05, + "loss": 1.1994, + "theoretical_loss": 3.326170556638281, + "tokens_seen": 2937323520 + }, + { + "epoch": 0.89, + "learning_rate": 5.558676028084253e-05, + "loss": 1.1549, + "theoretical_loss": 3.32615909618023, + "tokens_seen": 2937454592 + }, + { + "epoch": 0.89, + "learning_rate": 5.55667001003009e-05, + "loss": 1.2837, + "theoretical_loss": 3.3261476363767226, + "tokens_seen": 2937585664 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.2945992052555084, + "objective/train/avg_token_score": 0.019253844395279884, + "objective/train/docs_used": 1654272, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8883702754974365, + "objective/train/original_loss": 3.977719306945801, + "objective/train/theoretical_loss": 3.3261419067204017, + "objective/train/tokens_used": 2937651200, + "objective/train/unlikelihood_loss": 0.24530082941055298, + "theoretical_loss": 3.3261419067204017, + "tokens_seen": 2937651200 + }, + { + "epoch": 0.89, + "learning_rate": 5.554663991975928e-05, + "loss": 1.1288, + "theoretical_loss": 3.326136177227691, + "tokens_seen": 2937716736 + }, + { + "epoch": 0.89, + "learning_rate": 5.5526579739217654e-05, + "loss": 1.2411, + "theoretical_loss": 3.32612471873307, + "tokens_seen": 2937847808 + }, + { + "epoch": 0.89, + "learning_rate": 5.550651955867603e-05, + "loss": 1.2479, + "theoretical_loss": 3.326113260892792, + "tokens_seen": 2937978880 + }, + { + "epoch": 0.89, + "learning_rate": 5.54864593781344e-05, + "loss": 1.2355, + "theoretical_loss": 3.3261018037067913, + "tokens_seen": 2938109952 + }, + { + "epoch": 0.89, + "learning_rate": 5.5466399197592775e-05, + "loss": 1.2028, + "theoretical_loss": 3.326090347175001, + "tokens_seen": 2938241024 + }, + { + "epoch": 0.89, + "learning_rate": 5.544633901705116e-05, + "loss": 1.3526, + "theoretical_loss": 3.3260788912973545, + "tokens_seen": 2938372096 + }, + { + "epoch": 0.89, + "learning_rate": 5.542627883650953e-05, + "loss": 1.2162, + "theoretical_loss": 3.326067436073785, + "tokens_seen": 2938503168 + }, + { + "epoch": 0.89, + "learning_rate": 5.540621865596791e-05, + "loss": 1.1378, + "theoretical_loss": 3.3260559815042265, + "tokens_seen": 2938634240 + }, + { + "epoch": 0.89, + "learning_rate": 5.538615847542628e-05, + "loss": 1.2589, + "theoretical_loss": 3.3260445275886124, + "tokens_seen": 2938765312 + }, + { + "epoch": 0.89, + "learning_rate": 5.5366098294884654e-05, + "loss": 1.1726, + "theoretical_loss": 3.326033074326876, + "tokens_seen": 2938896384 + }, + { + "epoch": 0.89, + "learning_rate": 5.534603811434303e-05, + "loss": 1.3587, + "theoretical_loss": 3.326021621718951, + "tokens_seen": 2939027456 + }, + { + "epoch": 0.89, + "learning_rate": 5.5325977933801405e-05, + "loss": 1.2422, + "theoretical_loss": 3.3260101697647704, + "tokens_seen": 2939158528 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.3164711594581604, + "objective/train/avg_token_score": 0.03815637156367302, + "objective/train/docs_used": 1655060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8832775354385376, + "objective/train/original_loss": 3.799997568130493, + "objective/train/theoretical_loss": 3.325998718464268, + "objective/train/tokens_used": 2939289600, + "objective/train/unlikelihood_loss": 0.2495737224817276, + "theoretical_loss": 3.325998718464268, + "tokens_seen": 2939289600 + }, + { + "epoch": 0.89, + "learning_rate": 5.530591775325978e-05, + "loss": 1.0654, + "theoretical_loss": 3.325998718464268, + "tokens_seen": 2939289600 + }, + { + "epoch": 0.89, + "learning_rate": 5.528585757271815e-05, + "loss": 1.0899, + "theoretical_loss": 3.3259872678173776, + "tokens_seen": 2939420672 + }, + { + "epoch": 0.89, + "learning_rate": 5.526579739217653e-05, + "loss": 1.2964, + "theoretical_loss": 3.325975817824033, + "tokens_seen": 2939551744 + }, + { + "epoch": 0.89, + "learning_rate": 5.524573721163491e-05, + "loss": 1.2213, + "theoretical_loss": 3.325964368484167, + "tokens_seen": 2939682816 + }, + { + "epoch": 0.89, + "learning_rate": 5.5225677031093284e-05, + "loss": 1.3142, + "theoretical_loss": 3.325952919797714, + "tokens_seen": 2939813888 + }, + { + "epoch": 0.89, + "learning_rate": 5.520561685055166e-05, + "loss": 1.3146, + "theoretical_loss": 3.325941471764607, + "tokens_seen": 2939944960 + }, + { + "epoch": 0.89, + "learning_rate": 5.518555667001003e-05, + "loss": 1.3115, + "theoretical_loss": 3.3259300243847796, + "tokens_seen": 2940076032 + }, + { + "epoch": 0.89, + "learning_rate": 5.5165496489468405e-05, + "loss": 1.4168, + "theoretical_loss": 3.3259185776581655, + "tokens_seen": 2940207104 + }, + { + "epoch": 0.89, + "learning_rate": 5.514543630892678e-05, + "loss": 1.3711, + "theoretical_loss": 3.3259071315846986, + "tokens_seen": 2940338176 + }, + { + "epoch": 0.89, + "learning_rate": 5.512537612838516e-05, + "loss": 1.318, + "theoretical_loss": 3.325895686164312, + "tokens_seen": 2940469248 + }, + { + "epoch": 0.89, + "learning_rate": 5.5105315947843526e-05, + "loss": 1.2948, + "theoretical_loss": 3.3258842413969396, + "tokens_seen": 2940600320 + }, + { + "epoch": 0.89, + "learning_rate": 5.508525576730191e-05, + "loss": 1.2716, + "theoretical_loss": 3.325872797282515, + "tokens_seen": 2940731392 + }, + { + "epoch": 0.89, + "learning_rate": 5.5065195586760284e-05, + "loss": 1.2372, + "theoretical_loss": 3.325861353820972, + "tokens_seen": 2940862464 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.4786168038845062, + "objective/train/avg_token_score": 0.02846800535917282, + "objective/train/docs_used": 1656552, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5028764009475708, + "objective/train/original_loss": 3.7987568378448486, + "objective/train/theoretical_loss": 3.32585563233501, + "objective/train/tokens_used": 2940928000, + "objective/train/unlikelihood_loss": 0.13460196554660797, + "theoretical_loss": 3.32585563233501, + "tokens_seen": 2940928000 + }, + { + "epoch": 0.89, + "learning_rate": 5.504513540621866e-05, + "loss": 1.3813, + "theoretical_loss": 3.325849911012244, + "tokens_seen": 2940993536 + }, + { + "epoch": 0.89, + "learning_rate": 5.5025075225677036e-05, + "loss": 1.2089, + "theoretical_loss": 3.325838468856265, + "tokens_seen": 2941124608 + }, + { + "epoch": 0.89, + "learning_rate": 5.5005015045135405e-05, + "loss": 1.4202, + "theoretical_loss": 3.325827027352968, + "tokens_seen": 2941255680 + }, + { + "epoch": 0.89, + "learning_rate": 5.498495486459378e-05, + "loss": 1.243, + "theoretical_loss": 3.3258155865022876, + "tokens_seen": 2941386752 + }, + { + "epoch": 0.89, + "learning_rate": 5.4964894684052156e-05, + "loss": 1.2146, + "theoretical_loss": 3.325804146304157, + "tokens_seen": 2941517824 + }, + { + "epoch": 0.89, + "learning_rate": 5.494483450351053e-05, + "loss": 1.3179, + "theoretical_loss": 3.32579270675851, + "tokens_seen": 2941648896 + }, + { + "epoch": 0.89, + "learning_rate": 5.492477432296891e-05, + "loss": 1.1463, + "theoretical_loss": 3.32578126786528, + "tokens_seen": 2941779968 + }, + { + "epoch": 0.89, + "learning_rate": 5.490471414242728e-05, + "loss": 1.2639, + "theoretical_loss": 3.325769829624401, + "tokens_seen": 2941911040 + }, + { + "epoch": 0.89, + "learning_rate": 5.488465396188566e-05, + "loss": 1.2262, + "theoretical_loss": 3.325758392035807, + "tokens_seen": 2942042112 + }, + { + "epoch": 0.89, + "learning_rate": 5.4864593781344035e-05, + "loss": 1.2088, + "theoretical_loss": 3.325746955099431, + "tokens_seen": 2942173184 + }, + { + "epoch": 0.89, + "learning_rate": 5.484453360080241e-05, + "loss": 1.2498, + "theoretical_loss": 3.325735518815208, + "tokens_seen": 2942304256 + }, + { + "epoch": 0.89, + "learning_rate": 5.482447342026079e-05, + "loss": 1.0412, + "theoretical_loss": 3.3257240831830703, + "tokens_seen": 2942435328 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.2995479106903076, + "objective/train/avg_token_score": 0.005987007170915604, + "objective/train/docs_used": 1657166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9604674577713013, + "objective/train/original_loss": 3.6863038539886475, + "objective/train/theoretical_loss": 3.325712648202953, + "objective/train/tokens_used": 2942566400, + "objective/train/unlikelihood_loss": 0.2351844608783722, + "theoretical_loss": 3.325712648202953, + "tokens_seen": 2942566400 + }, + { + "epoch": 0.89, + "learning_rate": 5.4804413239719156e-05, + "loss": 1.2403, + "theoretical_loss": 3.325712648202953, + "tokens_seen": 2942566400 + }, + { + "epoch": 0.89, + "learning_rate": 5.478435305917753e-05, + "loss": 1.2431, + "theoretical_loss": 3.3257012138747886, + "tokens_seen": 2942697472 + }, + { + "epoch": 0.89, + "learning_rate": 5.476429287863591e-05, + "loss": 1.2921, + "theoretical_loss": 3.325689780198512, + "tokens_seen": 2942828544 + }, + { + "epoch": 0.89, + "learning_rate": 5.474423269809428e-05, + "loss": 1.4553, + "theoretical_loss": 3.3256783471740565, + "tokens_seen": 2942959616 + }, + { + "epoch": 0.89, + "learning_rate": 5.472417251755266e-05, + "loss": 1.3058, + "theoretical_loss": 3.325666914801356, + "tokens_seen": 2943090688 + }, + { + "epoch": 0.89, + "learning_rate": 5.4704112337011035e-05, + "loss": 1.197, + "theoretical_loss": 3.325655483080344, + "tokens_seen": 2943221760 + }, + { + "epoch": 0.89, + "learning_rate": 5.468405215646941e-05, + "loss": 1.1584, + "theoretical_loss": 3.3256440520109547, + "tokens_seen": 2943352832 + }, + { + "epoch": 0.89, + "learning_rate": 5.4663991975927786e-05, + "loss": 1.2246, + "theoretical_loss": 3.325632621593122, + "tokens_seen": 2943483904 + }, + { + "epoch": 0.89, + "learning_rate": 5.464393179538616e-05, + "loss": 1.2667, + "theoretical_loss": 3.3256211918267793, + "tokens_seen": 2943614976 + }, + { + "epoch": 0.89, + "learning_rate": 5.462387161484454e-05, + "loss": 1.211, + "theoretical_loss": 3.3256097627118613, + "tokens_seen": 2943746048 + }, + { + "epoch": 0.89, + "learning_rate": 5.460381143430291e-05, + "loss": 1.203, + "theoretical_loss": 3.3255983342483013, + "tokens_seen": 2943877120 + }, + { + "epoch": 0.89, + "learning_rate": 5.458375125376128e-05, + "loss": 1.2227, + "theoretical_loss": 3.325586906436033, + "tokens_seen": 2944008192 + }, + { + "epoch": 0.89, + "learning_rate": 5.456369107321966e-05, + "loss": 1.335, + "theoretical_loss": 3.32557547927499, + "tokens_seen": 2944139264 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.3869134783744812, + "objective/train/avg_token_score": 0.013260968960821629, + "objective/train/docs_used": 1658433, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1815499067306519, + "objective/train/original_loss": 3.772541046142578, + "objective/train/theoretical_loss": 3.325569765938658, + "objective/train/tokens_used": 2944204800, + "objective/train/unlikelihood_loss": 0.15618467330932617, + "theoretical_loss": 3.325569765938658, + "tokens_seen": 2944204800 + }, + { + "epoch": 0.89, + "learning_rate": 5.4543630892678034e-05, + "loss": 1.2605, + "theoretical_loss": 3.3255640527651074, + "tokens_seen": 2944270336 + }, + { + "epoch": 0.89, + "learning_rate": 5.452357071213641e-05, + "loss": 1.1072, + "theoretical_loss": 3.325552626906318, + "tokens_seen": 2944401408 + }, + { + "epoch": 0.89, + "learning_rate": 5.4503510531594786e-05, + "loss": 1.0823, + "theoretical_loss": 3.325541201698557, + "tokens_seen": 2944532480 + }, + { + "epoch": 0.89, + "learning_rate": 5.448345035105316e-05, + "loss": 1.1349, + "theoretical_loss": 3.3255297771417562, + "tokens_seen": 2944663552 + }, + { + "epoch": 0.89, + "learning_rate": 5.446339017051154e-05, + "loss": 1.2987, + "theoretical_loss": 3.325518353235852, + "tokens_seen": 2944794624 + }, + { + "epoch": 0.89, + "learning_rate": 5.444332998996991e-05, + "loss": 1.409, + "theoretical_loss": 3.3255069299807762, + "tokens_seen": 2944925696 + }, + { + "epoch": 0.89, + "learning_rate": 5.442326980942829e-05, + "loss": 1.206, + "theoretical_loss": 3.325495507376464, + "tokens_seen": 2945056768 + }, + { + "epoch": 0.89, + "learning_rate": 5.440320962888666e-05, + "loss": 1.4298, + "theoretical_loss": 3.3254840854228496, + "tokens_seen": 2945187840 + }, + { + "epoch": 0.89, + "learning_rate": 5.4383149448345034e-05, + "loss": 1.0155, + "theoretical_loss": 3.3254726641198658, + "tokens_seen": 2945318912 + }, + { + "epoch": 0.89, + "learning_rate": 5.436308926780341e-05, + "loss": 1.1771, + "theoretical_loss": 3.3254612434674473, + "tokens_seen": 2945449984 + }, + { + "epoch": 0.89, + "learning_rate": 5.4343029087261785e-05, + "loss": 1.3705, + "theoretical_loss": 3.325449823465528, + "tokens_seen": 2945581056 + }, + { + "epoch": 0.89, + "learning_rate": 5.432296890672016e-05, + "loss": 1.2699, + "theoretical_loss": 3.3254384041140423, + "tokens_seen": 2945712128 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.48154935240745544, + "objective/train/avg_token_score": 0.008832979947328568, + "objective/train/docs_used": 1659139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5175586938858032, + "objective/train/original_loss": 3.252930164337158, + "objective/train/theoretical_loss": 3.325426985412924, + "objective/train/tokens_used": 2945843200, + "objective/train/unlikelihood_loss": 0.23667791485786438, + "theoretical_loss": 3.325426985412924, + "tokens_seen": 2945843200 + }, + { + "epoch": 0.89, + "learning_rate": 5.430290872617854e-05, + "loss": 1.2992, + "theoretical_loss": 3.325426985412924, + "tokens_seen": 2945843200 + }, + { + "epoch": 0.89, + "learning_rate": 5.428284854563691e-05, + "loss": 1.3085, + "theoretical_loss": 3.325415567362106, + "tokens_seen": 2945974272 + }, + { + "epoch": 0.89, + "learning_rate": 5.426278836509529e-05, + "loss": 1.1481, + "theoretical_loss": 3.3254041499615243, + "tokens_seen": 2946105344 + }, + { + "epoch": 0.89, + "learning_rate": 5.4242728184553664e-05, + "loss": 1.2692, + "theoretical_loss": 3.3253927332111117, + "tokens_seen": 2946236416 + }, + { + "epoch": 0.89, + "learning_rate": 5.4222668004012033e-05, + "loss": 1.2342, + "theoretical_loss": 3.3253813171108026, + "tokens_seen": 2946367488 + }, + { + "epoch": 0.89, + "learning_rate": 5.420260782347041e-05, + "loss": 1.314, + "theoretical_loss": 3.3253699016605305, + "tokens_seen": 2946498560 + }, + { + "epoch": 0.89, + "learning_rate": 5.4182547642928785e-05, + "loss": 1.182, + "theoretical_loss": 3.3253584868602304, + "tokens_seen": 2946629632 + }, + { + "epoch": 0.89, + "learning_rate": 5.416248746238716e-05, + "loss": 1.237, + "theoretical_loss": 3.3253470727098358, + "tokens_seen": 2946760704 + }, + { + "epoch": 0.89, + "learning_rate": 5.4142427281845543e-05, + "loss": 1.3176, + "theoretical_loss": 3.325335659209281, + "tokens_seen": 2946891776 + }, + { + "epoch": 0.89, + "learning_rate": 5.412236710130391e-05, + "loss": 1.3466, + "theoretical_loss": 3.3253242463585, + "tokens_seen": 2947022848 + }, + { + "epoch": 0.89, + "learning_rate": 5.410230692076229e-05, + "loss": 1.1699, + "theoretical_loss": 3.325312834157427, + "tokens_seen": 2947153920 + }, + { + "epoch": 0.89, + "learning_rate": 5.4082246740220664e-05, + "loss": 1.2439, + "theoretical_loss": 3.3253014226059956, + "tokens_seen": 2947284992 + }, + { + "epoch": 0.89, + "learning_rate": 5.406218655967904e-05, + "loss": 1.3684, + "theoretical_loss": 3.3252900117041406, + "tokens_seen": 2947416064 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.37139540910720825, + "objective/train/avg_token_score": 0.015401455573737621, + "objective/train/docs_used": 1659674, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.178943157196045, + "objective/train/original_loss": 3.7728030681610107, + "objective/train/theoretical_loss": 3.325284306496784, + "objective/train/tokens_used": 2947481600, + "objective/train/unlikelihood_loss": 0.15705488622188568, + "theoretical_loss": 3.325284306496784, + "tokens_seen": 2947481600 + }, + { + "epoch": 0.89, + "learning_rate": 5.4042126379137416e-05, + "loss": 1.1463, + "theoretical_loss": 3.325278601451796, + "tokens_seen": 2947547136 + }, + { + "epoch": 0.89, + "learning_rate": 5.4022066198595785e-05, + "loss": 1.0422, + "theoretical_loss": 3.325267191848896, + "tokens_seen": 2947678208 + }, + { + "epoch": 0.89, + "learning_rate": 5.400200601805416e-05, + "loss": 1.1995, + "theoretical_loss": 3.3252557828953746, + "tokens_seen": 2947809280 + }, + { + "epoch": 0.89, + "learning_rate": 5.3981945837512536e-05, + "loss": 1.1773, + "theoretical_loss": 3.3252443745911657, + "tokens_seen": 2947940352 + }, + { + "epoch": 0.89, + "learning_rate": 5.396188565697091e-05, + "loss": 1.3882, + "theoretical_loss": 3.325232966936204, + "tokens_seen": 2948071424 + }, + { + "epoch": 0.89, + "learning_rate": 5.3941825476429295e-05, + "loss": 1.2373, + "theoretical_loss": 3.325221559930424, + "tokens_seen": 2948202496 + }, + { + "epoch": 0.89, + "learning_rate": 5.3921765295887664e-05, + "loss": 1.179, + "theoretical_loss": 3.3252101535737584, + "tokens_seen": 2948333568 + }, + { + "epoch": 0.89, + "learning_rate": 5.390170511534604e-05, + "loss": 1.3306, + "theoretical_loss": 3.325198747866143, + "tokens_seen": 2948464640 + }, + { + "epoch": 0.89, + "learning_rate": 5.3881644934804415e-05, + "loss": 1.1833, + "theoretical_loss": 3.3251873428075114, + "tokens_seen": 2948595712 + }, + { + "epoch": 0.89, + "learning_rate": 5.386158475426279e-05, + "loss": 1.2678, + "theoretical_loss": 3.3251759383977975, + "tokens_seen": 2948726784 + }, + { + "epoch": 0.89, + "learning_rate": 5.384152457372117e-05, + "loss": 1.3563, + "theoretical_loss": 3.325164534636936, + "tokens_seen": 2948857856 + }, + { + "epoch": 0.89, + "learning_rate": 5.3821464393179536e-05, + "loss": 1.2847, + "theoretical_loss": 3.325153131524861, + "tokens_seen": 2948988928 + }, + { + "debugging/length": 458.375, + "debugging/num_segments": 8, + "epoch": 0.89, + "objective/train/aligned_token_count": 0.3677908182144165, + "objective/train/avg_token_score": 0.013196587562561035, + "objective/train/docs_used": 1661100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1688182353973389, + "objective/train/original_loss": 3.8849282264709473, + "objective/train/theoretical_loss": 3.3251417290615066, + "objective/train/tokens_used": 2949120000, + "objective/train/unlikelihood_loss": 0.16007323563098907, + "theoretical_loss": 3.3251417290615066, + "tokens_seen": 2949120000 + }, + { + "epoch": 0.89, + "learning_rate": 5.380140421263791e-05, + "loss": 1.1188, + "theoretical_loss": 3.3251417290615066, + "tokens_seen": 2949120000 + }, + { + "epoch": 0.89, + "learning_rate": 5.378134403209629e-05, + "loss": 1.3147, + "theoretical_loss": 3.3251303272468076, + "tokens_seen": 2949251072 + }, + { + "epoch": 0.89, + "learning_rate": 5.376128385155467e-05, + "loss": 1.2012, + "theoretical_loss": 3.3251189260806977, + "tokens_seen": 2949382144 + }, + { + "epoch": 0.89, + "learning_rate": 5.3741223671013046e-05, + "loss": 1.4034, + "theoretical_loss": 3.3251075255631113, + "tokens_seen": 2949513216 + }, + { + "epoch": 0.89, + "learning_rate": 5.3721163490471415e-05, + "loss": 1.2466, + "theoretical_loss": 3.3250961256939826, + "tokens_seen": 2949644288 + }, + { + "epoch": 0.89, + "learning_rate": 5.370110330992979e-05, + "loss": 1.1902, + "theoretical_loss": 3.3250847264732464, + "tokens_seen": 2949775360 + }, + { + "epoch": 0.89, + "learning_rate": 5.3681043129388166e-05, + "loss": 1.3208, + "theoretical_loss": 3.325073327900837, + "tokens_seen": 2949906432 + }, + { + "epoch": 0.89, + "learning_rate": 5.366098294884654e-05, + "loss": 1.3024, + "theoretical_loss": 3.325061929976688, + "tokens_seen": 2950037504 + }, + { + "epoch": 0.89, + "learning_rate": 5.364092276830491e-05, + "loss": 1.3344, + "theoretical_loss": 3.325050532700734, + "tokens_seen": 2950168576 + }, + { + "epoch": 0.89, + "learning_rate": 5.362086258776329e-05, + "loss": 1.2327, + "theoretical_loss": 3.3250391360729097, + "tokens_seen": 2950299648 + }, + { + "epoch": 0.89, + "learning_rate": 5.360080240722166e-05, + "loss": 1.3081, + "theoretical_loss": 3.325027740093149, + "tokens_seen": 2950430720 + }, + { + "epoch": 0.89, + "learning_rate": 5.3580742226680045e-05, + "loss": 1.2035, + "theoretical_loss": 3.3250163447613867, + "tokens_seen": 2950561792 + }, + { + "epoch": 0.89, + "learning_rate": 5.356068204613842e-05, + "loss": 1.2259, + "theoretical_loss": 3.325004950077557, + "tokens_seen": 2950692864 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.4657258093357086, + "objective/train/avg_token_score": 0.007186875678598881, + "objective/train/docs_used": 1661802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3467381000518799, + "objective/train/original_loss": 3.416149854660034, + "objective/train/theoretical_loss": 3.3249992529785963, + "objective/train/tokens_used": 2950758400, + "objective/train/unlikelihood_loss": 0.17353259027004242, + "theoretical_loss": 3.3249992529785963, + "tokens_seen": 2950758400 + }, + { + "epoch": 0.89, + "learning_rate": 5.354062186559679e-05, + "loss": 1.2929, + "theoretical_loss": 3.324993556041594, + "tokens_seen": 2950823936 + }, + { + "epoch": 0.89, + "learning_rate": 5.3520561685055166e-05, + "loss": 1.1889, + "theoretical_loss": 3.3249821626534324, + "tokens_seen": 2950955008 + }, + { + "epoch": 0.89, + "learning_rate": 5.350050150451354e-05, + "loss": 1.3388, + "theoretical_loss": 3.3249707699130067, + "tokens_seen": 2951086080 + }, + { + "epoch": 0.89, + "learning_rate": 5.348044132397192e-05, + "loss": 1.3242, + "theoretical_loss": 3.3249593778202513, + "tokens_seen": 2951217152 + }, + { + "epoch": 0.89, + "learning_rate": 5.346038114343029e-05, + "loss": 1.2167, + "theoretical_loss": 3.3249479863751, + "tokens_seen": 2951348224 + }, + { + "epoch": 0.89, + "learning_rate": 5.344032096288866e-05, + "loss": 1.3579, + "theoretical_loss": 3.324936595577488, + "tokens_seen": 2951479296 + }, + { + "epoch": 0.89, + "learning_rate": 5.342026078234704e-05, + "loss": 1.2261, + "theoretical_loss": 3.3249252054273493, + "tokens_seen": 2951610368 + }, + { + "epoch": 0.89, + "learning_rate": 5.3400200601805414e-05, + "loss": 1.2713, + "theoretical_loss": 3.3249138159246185, + "tokens_seen": 2951741440 + }, + { + "epoch": 0.89, + "learning_rate": 5.3380140421263796e-05, + "loss": 1.2422, + "theoretical_loss": 3.3249024270692304, + "tokens_seen": 2951872512 + }, + { + "epoch": 0.89, + "learning_rate": 5.336008024072217e-05, + "loss": 1.2223, + "theoretical_loss": 3.3248910388611184, + "tokens_seen": 2952003584 + }, + { + "epoch": 0.89, + "learning_rate": 5.334002006018054e-05, + "loss": 1.3929, + "theoretical_loss": 3.3248796513002183, + "tokens_seen": 2952134656 + }, + { + "epoch": 0.89, + "learning_rate": 5.331995987963892e-05, + "loss": 1.3227, + "theoretical_loss": 3.3248682643864633, + "tokens_seen": 2952265728 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.37390029430389404, + "objective/train/avg_token_score": 0.030581364408135414, + "objective/train/docs_used": 1662642, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1310961246490479, + "objective/train/original_loss": 3.828850746154785, + "objective/train/theoretical_loss": 3.324856878119789, + "objective/train/tokens_used": 2952396800, + "objective/train/unlikelihood_loss": 0.18363754451274872, + "theoretical_loss": 3.324856878119789, + "tokens_seen": 2952396800 + }, + { + "epoch": 0.89, + "learning_rate": 5.329989969909729e-05, + "loss": 1.2576, + "theoretical_loss": 3.324856878119789, + "tokens_seen": 2952396800 + }, + { + "epoch": 0.89, + "learning_rate": 5.327983951855567e-05, + "loss": 1.1012, + "theoretical_loss": 3.3248454925001294, + "tokens_seen": 2952527872 + }, + { + "epoch": 0.89, + "learning_rate": 5.3259779338014044e-05, + "loss": 1.3559, + "theoretical_loss": 3.324834107527419, + "tokens_seen": 2952658944 + }, + { + "epoch": 0.89, + "learning_rate": 5.3239719157472413e-05, + "loss": 1.1903, + "theoretical_loss": 3.3248227232015926, + "tokens_seen": 2952790016 + }, + { + "epoch": 0.89, + "learning_rate": 5.321965897693079e-05, + "loss": 1.2671, + "theoretical_loss": 3.3248113395225842, + "tokens_seen": 2952921088 + }, + { + "epoch": 0.89, + "learning_rate": 5.319959879638917e-05, + "loss": 1.3011, + "theoretical_loss": 3.324799956490329, + "tokens_seen": 2953052160 + }, + { + "epoch": 0.89, + "learning_rate": 5.317953861584755e-05, + "loss": 1.3432, + "theoretical_loss": 3.324788574104761, + "tokens_seen": 2953183232 + }, + { + "epoch": 0.89, + "learning_rate": 5.315947843530592e-05, + "loss": 1.274, + "theoretical_loss": 3.324777192365815, + "tokens_seen": 2953314304 + }, + { + "epoch": 0.89, + "learning_rate": 5.313941825476429e-05, + "loss": 1.2175, + "theoretical_loss": 3.3247658112734255, + "tokens_seen": 2953445376 + }, + { + "epoch": 0.9, + "learning_rate": 5.311935807422267e-05, + "loss": 1.2939, + "theoretical_loss": 3.324754430827527, + "tokens_seen": 2953576448 + }, + { + "epoch": 0.9, + "learning_rate": 5.3099297893681044e-05, + "loss": 1.3207, + "theoretical_loss": 3.3247430510280545, + "tokens_seen": 2953707520 + }, + { + "epoch": 0.9, + "learning_rate": 5.307923771313942e-05, + "loss": 1.3414, + "theoretical_loss": 3.324731671874942, + "tokens_seen": 2953838592 + }, + { + "epoch": 0.9, + "learning_rate": 5.3059177532597796e-05, + "loss": 1.1611, + "theoretical_loss": 3.3247202933681246, + "tokens_seen": 2953969664 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.4425097703933716, + "objective/train/avg_token_score": 0.01323393639177084, + "objective/train/docs_used": 1663271, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1810972690582275, + "objective/train/original_loss": 3.4988863468170166, + "objective/train/theoretical_loss": 3.324714604357056, + "objective/train/tokens_used": 2954035200, + "objective/train/unlikelihood_loss": 0.16461294889450073, + "theoretical_loss": 3.324714604357056, + "tokens_seen": 2954035200 + }, + { + "epoch": 0.9, + "learning_rate": 5.3039117352056165e-05, + "loss": 1.3618, + "theoretical_loss": 3.3247089155075367, + "tokens_seen": 2954100736 + }, + { + "epoch": 0.9, + "learning_rate": 5.301905717151455e-05, + "loss": 1.2497, + "theoretical_loss": 3.3246975382931128, + "tokens_seen": 2954231808 + }, + { + "epoch": 0.9, + "learning_rate": 5.299899699097292e-05, + "loss": 1.2967, + "theoretical_loss": 3.324686161724788, + "tokens_seen": 2954362880 + }, + { + "epoch": 0.9, + "learning_rate": 5.29789368104313e-05, + "loss": 1.1733, + "theoretical_loss": 3.3246747858024968, + "tokens_seen": 2954493952 + }, + { + "epoch": 0.9, + "learning_rate": 5.2958876629889675e-05, + "loss": 1.4152, + "theoretical_loss": 3.324663410526173, + "tokens_seen": 2954625024 + }, + { + "epoch": 0.9, + "learning_rate": 5.2938816449348044e-05, + "loss": 1.312, + "theoretical_loss": 3.3246520358957525, + "tokens_seen": 2954756096 + }, + { + "epoch": 0.9, + "learning_rate": 5.291875626880642e-05, + "loss": 1.2323, + "theoretical_loss": 3.32464066191117, + "tokens_seen": 2954887168 + }, + { + "epoch": 0.9, + "learning_rate": 5.2898696088264795e-05, + "loss": 1.185, + "theoretical_loss": 3.3246292885723587, + "tokens_seen": 2955018240 + }, + { + "epoch": 0.9, + "learning_rate": 5.287863590772317e-05, + "loss": 1.3072, + "theoretical_loss": 3.3246179158792546, + "tokens_seen": 2955149312 + }, + { + "epoch": 0.9, + "learning_rate": 5.285857572718154e-05, + "loss": 1.2826, + "theoretical_loss": 3.3246065438317918, + "tokens_seen": 2955280384 + }, + { + "epoch": 0.9, + "learning_rate": 5.2838515546639916e-05, + "loss": 1.3314, + "theoretical_loss": 3.3245951724299054, + "tokens_seen": 2955411456 + }, + { + "epoch": 0.9, + "learning_rate": 5.28184553660983e-05, + "loss": 1.1305, + "theoretical_loss": 3.3245838016735303, + "tokens_seen": 2955542528 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.5393450856208801, + "objective/train/avg_token_score": 0.015763092786073685, + "objective/train/docs_used": 1664408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4905290603637695, + "objective/train/original_loss": 3.1369502544403076, + "objective/train/theoretical_loss": 3.3245724315626006, + "objective/train/tokens_used": 2955673600, + "objective/train/unlikelihood_loss": 0.1919698715209961, + "theoretical_loss": 3.3245724315626006, + "tokens_seen": 2955673600 + }, + { + "epoch": 0.9, + "learning_rate": 5.2798395185556674e-05, + "loss": 1.5169, + "theoretical_loss": 3.3245724315626006, + "tokens_seen": 2955673600 + }, + { + "epoch": 0.9, + "learning_rate": 5.277833500501505e-05, + "loss": 1.1148, + "theoretical_loss": 3.324561062097051, + "tokens_seen": 2955804672 + }, + { + "epoch": 0.9, + "learning_rate": 5.275827482447342e-05, + "loss": 1.3626, + "theoretical_loss": 3.324549693276817, + "tokens_seen": 2955935744 + }, + { + "epoch": 0.9, + "learning_rate": 5.2738214643931795e-05, + "loss": 1.2507, + "theoretical_loss": 3.324538325101833, + "tokens_seen": 2956066816 + }, + { + "epoch": 0.9, + "learning_rate": 5.271815446339017e-05, + "loss": 1.3711, + "theoretical_loss": 3.3245269575720338, + "tokens_seen": 2956197888 + }, + { + "epoch": 0.9, + "learning_rate": 5.2698094282848546e-05, + "loss": 1.49, + "theoretical_loss": 3.3245155906873536, + "tokens_seen": 2956328960 + }, + { + "epoch": 0.9, + "learning_rate": 5.267803410230692e-05, + "loss": 1.2607, + "theoretical_loss": 3.324504224447728, + "tokens_seen": 2956460032 + }, + { + "epoch": 0.9, + "learning_rate": 5.265797392176529e-05, + "loss": 1.342, + "theoretical_loss": 3.3244928588530915, + "tokens_seen": 2956591104 + }, + { + "epoch": 0.9, + "learning_rate": 5.2637913741223674e-05, + "loss": 1.2891, + "theoretical_loss": 3.324481493903379, + "tokens_seen": 2956722176 + }, + { + "epoch": 0.9, + "learning_rate": 5.261785356068205e-05, + "loss": 1.2988, + "theoretical_loss": 3.324470129598525, + "tokens_seen": 2956853248 + }, + { + "epoch": 0.9, + "learning_rate": 5.2597793380140425e-05, + "loss": 1.4232, + "theoretical_loss": 3.3244587659384646, + "tokens_seen": 2956984320 + }, + { + "epoch": 0.9, + "learning_rate": 5.25777331995988e-05, + "loss": 1.5116, + "theoretical_loss": 3.324447402923133, + "tokens_seen": 2957115392 + }, + { + "epoch": 0.9, + "learning_rate": 5.255767301905717e-05, + "loss": 1.27, + "theoretical_loss": 3.324436040552464, + "tokens_seen": 2957246464 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.4804496467113495, + "objective/train/avg_token_score": 0.010781029239296913, + "objective/train/docs_used": 1665006, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4167838096618652, + "objective/train/original_loss": 3.6592793464660645, + "objective/train/theoretical_loss": 3.324430359608858, + "objective/train/tokens_used": 2957312000, + "objective/train/unlikelihood_loss": 0.15632973611354828, + "theoretical_loss": 3.324430359608858, + "tokens_seen": 2957312000 + }, + { + "epoch": 0.9, + "learning_rate": 5.2537612838515546e-05, + "loss": 1.3752, + "theoretical_loss": 3.3244246788263934, + "tokens_seen": 2957377536 + }, + { + "epoch": 0.9, + "learning_rate": 5.251755265797392e-05, + "loss": 1.3876, + "theoretical_loss": 3.3244133177448556, + "tokens_seen": 2957508608 + }, + { + "epoch": 0.9, + "learning_rate": 5.24974924774323e-05, + "loss": 1.3199, + "theoretical_loss": 3.324401957307786, + "tokens_seen": 2957639680 + }, + { + "epoch": 0.9, + "learning_rate": 5.247743229689067e-05, + "loss": 1.3796, + "theoretical_loss": 3.3243905975151185, + "tokens_seen": 2957770752 + }, + { + "epoch": 0.9, + "learning_rate": 5.245737211634905e-05, + "loss": 1.2409, + "theoretical_loss": 3.324379238366789, + "tokens_seen": 2957901824 + }, + { + "epoch": 0.9, + "learning_rate": 5.2437311935807425e-05, + "loss": 1.3393, + "theoretical_loss": 3.3243678798627316, + "tokens_seen": 2958032896 + }, + { + "epoch": 0.9, + "learning_rate": 5.24172517552658e-05, + "loss": 1.1312, + "theoretical_loss": 3.324356522002882, + "tokens_seen": 2958163968 + }, + { + "epoch": 0.9, + "learning_rate": 5.2397191574724176e-05, + "loss": 1.4588, + "theoretical_loss": 3.3243451647871747, + "tokens_seen": 2958295040 + }, + { + "epoch": 0.9, + "learning_rate": 5.237713139418255e-05, + "loss": 1.4322, + "theoretical_loss": 3.3243338082155445, + "tokens_seen": 2958426112 + }, + { + "epoch": 0.9, + "learning_rate": 5.235707121364092e-05, + "loss": 1.2941, + "theoretical_loss": 3.3243224522879267, + "tokens_seen": 2958557184 + }, + { + "epoch": 0.9, + "learning_rate": 5.23370110330993e-05, + "loss": 1.2459, + "theoretical_loss": 3.324311097004256, + "tokens_seen": 2958688256 + }, + { + "epoch": 0.9, + "learning_rate": 5.231695085255767e-05, + "loss": 1.4284, + "theoretical_loss": 3.3242997423644676, + "tokens_seen": 2958819328 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.4499633312225342, + "objective/train/avg_token_score": 0.03426126018166542, + "objective/train/docs_used": 1665825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3506557941436768, + "objective/train/original_loss": 3.5653719902038574, + "objective/train/theoretical_loss": 3.3242883883684957, + "objective/train/tokens_used": 2958950400, + "objective/train/unlikelihood_loss": 0.3338664174079895, + "theoretical_loss": 3.3242883883684957, + "tokens_seen": 2958950400 + }, + { + "epoch": 0.9, + "learning_rate": 5.229689067201605e-05, + "loss": 1.2923, + "theoretical_loss": 3.3242883883684957, + "tokens_seen": 2958950400 + }, + { + "epoch": 0.9, + "learning_rate": 5.2276830491474424e-05, + "loss": 1.17, + "theoretical_loss": 3.3242770350162765, + "tokens_seen": 2959081472 + }, + { + "epoch": 0.9, + "learning_rate": 5.22567703109328e-05, + "loss": 1.3153, + "theoretical_loss": 3.324265682307744, + "tokens_seen": 2959212544 + }, + { + "epoch": 0.9, + "learning_rate": 5.2236710130391176e-05, + "loss": 1.2321, + "theoretical_loss": 3.324254330242834, + "tokens_seen": 2959343616 + }, + { + "epoch": 0.9, + "learning_rate": 5.221664994984955e-05, + "loss": 1.3485, + "theoretical_loss": 3.3242429788214807, + "tokens_seen": 2959474688 + }, + { + "epoch": 0.9, + "learning_rate": 5.219658976930793e-05, + "loss": 1.2643, + "theoretical_loss": 3.324231628043619, + "tokens_seen": 2959605760 + }, + { + "epoch": 0.9, + "learning_rate": 5.2176529588766297e-05, + "loss": 1.1952, + "theoretical_loss": 3.324220277909185, + "tokens_seen": 2959736832 + }, + { + "epoch": 0.9, + "learning_rate": 5.215646940822467e-05, + "loss": 1.2361, + "theoretical_loss": 3.324208928418113, + "tokens_seen": 2959867904 + }, + { + "epoch": 0.9, + "learning_rate": 5.213640922768305e-05, + "loss": 1.3058, + "theoretical_loss": 3.3241975795703382, + "tokens_seen": 2959998976 + }, + { + "epoch": 0.9, + "learning_rate": 5.2116349047141424e-05, + "loss": 1.2983, + "theoretical_loss": 3.324186231365796, + "tokens_seen": 2960130048 + }, + { + "epoch": 0.9, + "learning_rate": 5.20962888665998e-05, + "loss": 1.3351, + "theoretical_loss": 3.3241748838044205, + "tokens_seen": 2960261120 + }, + { + "epoch": 0.9, + "learning_rate": 5.2076228686058176e-05, + "loss": 1.2999, + "theoretical_loss": 3.3241635368861475, + "tokens_seen": 2960392192 + }, + { + "epoch": 0.9, + "learning_rate": 5.205616850551655e-05, + "loss": 1.2835, + "theoretical_loss": 3.324152190610912, + "tokens_seen": 2960523264 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.49169111251831055, + "objective/train/avg_token_score": 0.008369497954845428, + "objective/train/docs_used": 1666526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3928860425949097, + "objective/train/original_loss": 3.8278863430023193, + "objective/train/theoretical_loss": 3.3241465177144134, + "objective/train/tokens_used": 2960588800, + "objective/train/unlikelihood_loss": 0.14744646847248077, + "theoretical_loss": 3.3241465177144134, + "tokens_seen": 2960588800 + }, + { + "epoch": 0.9, + "learning_rate": 5.203610832497493e-05, + "loss": 1.3727, + "theoretical_loss": 3.324140844978649, + "tokens_seen": 2960654336 + }, + { + "epoch": 0.9, + "learning_rate": 5.20160481444333e-05, + "loss": 1.2696, + "theoretical_loss": 3.3241294999892936, + "tokens_seen": 2960785408 + }, + { + "epoch": 0.9, + "learning_rate": 5.199598796389168e-05, + "loss": 1.3658, + "theoretical_loss": 3.3241181556427812, + "tokens_seen": 2960916480 + }, + { + "epoch": 0.9, + "learning_rate": 5.197592778335005e-05, + "loss": 1.2754, + "theoretical_loss": 3.3241068119390467, + "tokens_seen": 2961047552 + }, + { + "epoch": 0.9, + "learning_rate": 5.1955867602808424e-05, + "loss": 1.3876, + "theoretical_loss": 3.324095468878025, + "tokens_seen": 2961178624 + }, + { + "epoch": 0.9, + "learning_rate": 5.19358074222668e-05, + "loss": 1.2343, + "theoretical_loss": 3.3240841264596517, + "tokens_seen": 2961309696 + }, + { + "epoch": 0.9, + "learning_rate": 5.1915747241725175e-05, + "loss": 1.5017, + "theoretical_loss": 3.324072784683861, + "tokens_seen": 2961440768 + }, + { + "epoch": 0.9, + "learning_rate": 5.189568706118355e-05, + "loss": 1.1203, + "theoretical_loss": 3.3240614435505895, + "tokens_seen": 2961571840 + }, + { + "epoch": 0.9, + "learning_rate": 5.187562688064193e-05, + "loss": 1.234, + "theoretical_loss": 3.3240501030597716, + "tokens_seen": 2961702912 + }, + { + "epoch": 0.9, + "learning_rate": 5.18555667001003e-05, + "loss": 1.1757, + "theoretical_loss": 3.324038763211342, + "tokens_seen": 2961833984 + }, + { + "epoch": 0.9, + "learning_rate": 5.183550651955868e-05, + "loss": 1.1195, + "theoretical_loss": 3.3240274240052368, + "tokens_seen": 2961965056 + }, + { + "epoch": 0.9, + "learning_rate": 5.1815446339017054e-05, + "loss": 1.2313, + "theoretical_loss": 3.3240160854413903, + "tokens_seen": 2962096128 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.2983871102333069, + "objective/train/avg_token_score": 0.026690945029258728, + "objective/train/docs_used": 1667691, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8403846621513367, + "objective/train/original_loss": 4.218449592590332, + "objective/train/theoretical_loss": 3.3240047475197385, + "objective/train/tokens_used": 2962227200, + "objective/train/unlikelihood_loss": 0.11826365441083908, + "theoretical_loss": 3.3240047475197385, + "tokens_seen": 2962227200 + }, + { + "epoch": 0.9, + "learning_rate": 5.179538615847543e-05, + "loss": 1.3243, + "theoretical_loss": 3.3240047475197385, + "tokens_seen": 2962227200 + }, + { + "epoch": 0.9, + "learning_rate": 5.17753259779338e-05, + "loss": 1.3408, + "theoretical_loss": 3.323993410240216, + "tokens_seen": 2962358272 + }, + { + "epoch": 0.9, + "learning_rate": 5.1755265797392175e-05, + "loss": 1.2242, + "theoretical_loss": 3.323982073602759, + "tokens_seen": 2962489344 + }, + { + "epoch": 0.9, + "learning_rate": 5.173520561685055e-05, + "loss": 1.1379, + "theoretical_loss": 3.3239707376073016, + "tokens_seen": 2962620416 + }, + { + "epoch": 0.9, + "learning_rate": 5.1715145436308926e-05, + "loss": 1.3492, + "theoretical_loss": 3.3239594022537795, + "tokens_seen": 2962751488 + }, + { + "epoch": 0.9, + "learning_rate": 5.169508525576731e-05, + "loss": 1.2602, + "theoretical_loss": 3.323948067542128, + "tokens_seen": 2962882560 + }, + { + "epoch": 0.9, + "learning_rate": 5.167502507522568e-05, + "loss": 1.2479, + "theoretical_loss": 3.3239367334722822, + "tokens_seen": 2963013632 + }, + { + "epoch": 0.9, + "learning_rate": 5.1654964894684054e-05, + "loss": 1.2093, + "theoretical_loss": 3.3239254000441774, + "tokens_seen": 2963144704 + }, + { + "epoch": 0.9, + "learning_rate": 5.163490471414243e-05, + "loss": 1.2479, + "theoretical_loss": 3.323914067257749, + "tokens_seen": 2963275776 + }, + { + "epoch": 0.9, + "learning_rate": 5.1614844533600805e-05, + "loss": 1.4341, + "theoretical_loss": 3.3239027351129327, + "tokens_seen": 2963406848 + }, + { + "epoch": 0.9, + "learning_rate": 5.159478435305918e-05, + "loss": 1.275, + "theoretical_loss": 3.3238914036096627, + "tokens_seen": 2963537920 + }, + { + "epoch": 0.9, + "learning_rate": 5.157472417251755e-05, + "loss": 1.2905, + "theoretical_loss": 3.3238800727478752, + "tokens_seen": 2963668992 + }, + { + "epoch": 0.9, + "learning_rate": 5.1554663991975926e-05, + "loss": 1.1855, + "theoretical_loss": 3.3238687425275053, + "tokens_seen": 2963800064 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.344696968793869, + "objective/train/avg_token_score": 0.01349207479506731, + "objective/train/docs_used": 1668204, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0966697931289673, + "objective/train/original_loss": 3.517697811126709, + "objective/train/theoretical_loss": 3.3238630776578315, + "objective/train/tokens_used": 2963865600, + "objective/train/unlikelihood_loss": 0.19886082410812378, + "theoretical_loss": 3.3238630776578315, + "tokens_seen": 2963865600 + }, + { + "epoch": 0.9, + "learning_rate": 5.15346038114343e-05, + "loss": 1.3018, + "theoretical_loss": 3.323857412948488, + "tokens_seen": 2963931136 + }, + { + "epoch": 0.9, + "learning_rate": 5.1514543630892684e-05, + "loss": 1.2711, + "theoretical_loss": 3.3238460840107593, + "tokens_seen": 2964062208 + }, + { + "epoch": 0.9, + "learning_rate": 5.149448345035106e-05, + "loss": 1.3609, + "theoretical_loss": 3.3238347557142545, + "tokens_seen": 2964193280 + }, + { + "epoch": 0.9, + "learning_rate": 5.147442326980943e-05, + "loss": 1.4194, + "theoretical_loss": 3.323823428058908, + "tokens_seen": 2964324352 + }, + { + "epoch": 0.9, + "learning_rate": 5.1454363089267805e-05, + "loss": 1.2786, + "theoretical_loss": 3.323812101044656, + "tokens_seen": 2964455424 + }, + { + "epoch": 0.9, + "learning_rate": 5.143430290872618e-05, + "loss": 1.2477, + "theoretical_loss": 3.3238007746714335, + "tokens_seen": 2964586496 + }, + { + "epoch": 0.9, + "learning_rate": 5.1414242728184556e-05, + "loss": 1.3271, + "theoretical_loss": 3.323789448939176, + "tokens_seen": 2964717568 + }, + { + "epoch": 0.9, + "learning_rate": 5.1394182547642925e-05, + "loss": 1.1217, + "theoretical_loss": 3.3237781238478195, + "tokens_seen": 2964848640 + }, + { + "epoch": 0.9, + "learning_rate": 5.13741223671013e-05, + "loss": 1.3499, + "theoretical_loss": 3.323766799397298, + "tokens_seen": 2964979712 + }, + { + "epoch": 0.9, + "learning_rate": 5.135406218655968e-05, + "loss": 1.3201, + "theoretical_loss": 3.3237554755875487, + "tokens_seen": 2965110784 + }, + { + "epoch": 0.9, + "learning_rate": 5.133400200601805e-05, + "loss": 1.2833, + "theoretical_loss": 3.3237441524185054, + "tokens_seen": 2965241856 + }, + { + "epoch": 0.9, + "learning_rate": 5.1313941825476435e-05, + "loss": 1.2562, + "theoretical_loss": 3.3237328298901048, + "tokens_seen": 2965372928 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.5851050615310669, + "objective/train/avg_token_score": 0.0165394376963377, + "objective/train/docs_used": 1669270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8566163778305054, + "objective/train/original_loss": 3.7737677097320557, + "objective/train/theoretical_loss": 3.323721508002281, + "objective/train/tokens_used": 2965504000, + "objective/train/unlikelihood_loss": 0.1251777708530426, + "theoretical_loss": 3.323721508002281, + "tokens_seen": 2965504000 + }, + { + "epoch": 0.9, + "learning_rate": 5.1293881644934804e-05, + "loss": 1.3864, + "theoretical_loss": 3.323721508002281, + "tokens_seen": 2965504000 + }, + { + "epoch": 0.9, + "learning_rate": 5.127382146439318e-05, + "loss": 1.4049, + "theoretical_loss": 3.3237101867549708, + "tokens_seen": 2965635072 + }, + { + "epoch": 0.9, + "learning_rate": 5.1253761283851556e-05, + "loss": 1.3428, + "theoretical_loss": 3.3236988661481086, + "tokens_seen": 2965766144 + }, + { + "epoch": 0.9, + "learning_rate": 5.123370110330993e-05, + "loss": 1.3324, + "theoretical_loss": 3.3236875461816306, + "tokens_seen": 2965897216 + }, + { + "epoch": 0.9, + "learning_rate": 5.121364092276831e-05, + "loss": 1.2306, + "theoretical_loss": 3.323676226855472, + "tokens_seen": 2966028288 + }, + { + "epoch": 0.9, + "learning_rate": 5.1193580742226677e-05, + "loss": 1.4052, + "theoretical_loss": 3.323664908169568, + "tokens_seen": 2966159360 + }, + { + "epoch": 0.9, + "learning_rate": 5.117352056168505e-05, + "loss": 1.1847, + "theoretical_loss": 3.323653590123855, + "tokens_seen": 2966290432 + }, + { + "epoch": 0.9, + "learning_rate": 5.115346038114343e-05, + "loss": 1.2103, + "theoretical_loss": 3.3236422727182675, + "tokens_seen": 2966421504 + }, + { + "epoch": 0.9, + "learning_rate": 5.113340020060181e-05, + "loss": 1.255, + "theoretical_loss": 3.323630955952741, + "tokens_seen": 2966552576 + }, + { + "epoch": 0.9, + "learning_rate": 5.1113340020060187e-05, + "loss": 1.1608, + "theoretical_loss": 3.323619639827212, + "tokens_seen": 2966683648 + }, + { + "epoch": 0.9, + "learning_rate": 5.1093279839518556e-05, + "loss": 1.3202, + "theoretical_loss": 3.323608324341615, + "tokens_seen": 2966814720 + }, + { + "epoch": 0.9, + "learning_rate": 5.107321965897693e-05, + "loss": 1.2839, + "theoretical_loss": 3.323597009495886, + "tokens_seen": 2966945792 + }, + { + "epoch": 0.9, + "learning_rate": 5.105315947843531e-05, + "loss": 1.3041, + "theoretical_loss": 3.323585695289961, + "tokens_seen": 2967076864 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.5216275453567505, + "objective/train/avg_token_score": 0.018266940489411354, + "objective/train/docs_used": 1669763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4626961946487427, + "objective/train/original_loss": 3.1907851696014404, + "objective/train/theoretical_loss": 3.323580038426905, + "objective/train/tokens_used": 2967142400, + "objective/train/unlikelihood_loss": 0.3073917627334595, + "theoretical_loss": 3.323580038426905, + "tokens_seen": 2967142400 + }, + { + "epoch": 0.9, + "learning_rate": 5.103309929789368e-05, + "loss": 1.306, + "theoretical_loss": 3.323574381723775, + "tokens_seen": 2967207936 + }, + { + "epoch": 0.9, + "learning_rate": 5.101303911735206e-05, + "loss": 1.3095, + "theoretical_loss": 3.323563068797264, + "tokens_seen": 2967339008 + }, + { + "epoch": 0.9, + "learning_rate": 5.099297893681043e-05, + "loss": 1.2501, + "theoretical_loss": 3.3235517565103625, + "tokens_seen": 2967470080 + }, + { + "epoch": 0.9, + "learning_rate": 5.0972918756268804e-05, + "loss": 1.2431, + "theoretical_loss": 3.3235404448630073, + "tokens_seen": 2967601152 + }, + { + "epoch": 0.9, + "learning_rate": 5.0952858575727186e-05, + "loss": 1.2564, + "theoretical_loss": 3.3235291338551334, + "tokens_seen": 2967732224 + }, + { + "epoch": 0.9, + "learning_rate": 5.093279839518556e-05, + "loss": 1.1663, + "theoretical_loss": 3.3235178234866765, + "tokens_seen": 2967863296 + }, + { + "epoch": 0.9, + "learning_rate": 5.091273821464394e-05, + "loss": 1.3316, + "theoretical_loss": 3.3235065137575726, + "tokens_seen": 2967994368 + }, + { + "epoch": 0.9, + "learning_rate": 5.089267803410231e-05, + "loss": 1.2232, + "theoretical_loss": 3.3234952046677564, + "tokens_seen": 2968125440 + }, + { + "epoch": 0.9, + "learning_rate": 5.087261785356068e-05, + "loss": 1.3051, + "theoretical_loss": 3.3234838962171644, + "tokens_seen": 2968256512 + }, + { + "epoch": 0.9, + "learning_rate": 5.085255767301906e-05, + "loss": 1.3556, + "theoretical_loss": 3.323472588405732, + "tokens_seen": 2968387584 + }, + { + "epoch": 0.9, + "learning_rate": 5.0832497492477434e-05, + "loss": 1.2749, + "theoretical_loss": 3.323461281233395, + "tokens_seen": 2968518656 + }, + { + "epoch": 0.9, + "learning_rate": 5.081243731193581e-05, + "loss": 1.3054, + "theoretical_loss": 3.3234499747000887, + "tokens_seen": 2968649728 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.41410067677497864, + "objective/train/avg_token_score": 0.005858480930328369, + "objective/train/docs_used": 1671048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.255754828453064, + "objective/train/original_loss": 3.632331132888794, + "objective/train/theoretical_loss": 3.323438668805749, + "objective/train/tokens_used": 2968780800, + "objective/train/unlikelihood_loss": 0.14213116466999054, + "theoretical_loss": 3.323438668805749, + "tokens_seen": 2968780800 + }, + { + "epoch": 0.9, + "learning_rate": 5.079237713139418e-05, + "loss": 1.2914, + "theoretical_loss": 3.323438668805749, + "tokens_seen": 2968780800 + }, + { + "epoch": 0.9, + "learning_rate": 5.0772316950852555e-05, + "loss": 1.2862, + "theoretical_loss": 3.3234273635503118, + "tokens_seen": 2968911872 + }, + { + "epoch": 0.9, + "learning_rate": 5.075225677031094e-05, + "loss": 1.3387, + "theoretical_loss": 3.323416058933712, + "tokens_seen": 2969042944 + }, + { + "epoch": 0.9, + "learning_rate": 5.073219658976931e-05, + "loss": 1.4577, + "theoretical_loss": 3.3234047549558863, + "tokens_seen": 2969174016 + }, + { + "epoch": 0.9, + "learning_rate": 5.071213640922768e-05, + "loss": 1.2843, + "theoretical_loss": 3.3233934516167696, + "tokens_seen": 2969305088 + }, + { + "epoch": 0.9, + "learning_rate": 5.069207622868606e-05, + "loss": 1.4789, + "theoretical_loss": 3.3233821489162985, + "tokens_seen": 2969436160 + }, + { + "epoch": 0.9, + "learning_rate": 5.0672016048144434e-05, + "loss": 1.2094, + "theoretical_loss": 3.3233708468544076, + "tokens_seen": 2969567232 + }, + { + "epoch": 0.9, + "learning_rate": 5.065195586760281e-05, + "loss": 1.2988, + "theoretical_loss": 3.3233595454310336, + "tokens_seen": 2969698304 + }, + { + "epoch": 0.9, + "learning_rate": 5.0631895687061185e-05, + "loss": 1.2893, + "theoretical_loss": 3.3233482446461116, + "tokens_seen": 2969829376 + }, + { + "epoch": 0.9, + "learning_rate": 5.0611835506519554e-05, + "loss": 1.2855, + "theoretical_loss": 3.323336944499578, + "tokens_seen": 2969960448 + }, + { + "epoch": 0.9, + "learning_rate": 5.059177532597793e-05, + "loss": 1.3504, + "theoretical_loss": 3.3233256449913675, + "tokens_seen": 2970091520 + }, + { + "epoch": 0.9, + "learning_rate": 5.057171514543631e-05, + "loss": 1.3277, + "theoretical_loss": 3.323314346121417, + "tokens_seen": 2970222592 + }, + { + "epoch": 0.9, + "learning_rate": 5.055165496489469e-05, + "loss": 1.0667, + "theoretical_loss": 3.323303047889662, + "tokens_seen": 2970353664 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.410740464925766, + "objective/train/avg_token_score": 0.01007784903049469, + "objective/train/docs_used": 1671649, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.162253499031067, + "objective/train/original_loss": 3.481692314147949, + "objective/train/theoretical_loss": 3.323297399013088, + "objective/train/tokens_used": 2970419200, + "objective/train/unlikelihood_loss": 0.16967543959617615, + "theoretical_loss": 3.323297399013088, + "tokens_seen": 2970419200 + }, + { + "epoch": 0.9, + "learning_rate": 5.0531594784353064e-05, + "loss": 1.2528, + "theoretical_loss": 3.3232917502960384, + "tokens_seen": 2970484736 + }, + { + "epoch": 0.9, + "learning_rate": 5.051153460381143e-05, + "loss": 1.3706, + "theoretical_loss": 3.323280453340481, + "tokens_seen": 2970615808 + }, + { + "epoch": 0.9, + "learning_rate": 5.049147442326981e-05, + "loss": 1.2807, + "theoretical_loss": 3.3232691570229274, + "tokens_seen": 2970746880 + }, + { + "epoch": 0.9, + "learning_rate": 5.0471414242728185e-05, + "loss": 1.2719, + "theoretical_loss": 3.3232578613433117, + "tokens_seen": 2970877952 + }, + { + "epoch": 0.9, + "learning_rate": 5.045135406218656e-05, + "loss": 1.1782, + "theoretical_loss": 3.3232465663015702, + "tokens_seen": 2971009024 + }, + { + "epoch": 0.9, + "learning_rate": 5.0431293881644936e-05, + "loss": 1.3543, + "theoretical_loss": 3.3232352718976395, + "tokens_seen": 2971140096 + }, + { + "epoch": 0.9, + "learning_rate": 5.0411233701103305e-05, + "loss": 1.3282, + "theoretical_loss": 3.3232239781314545, + "tokens_seen": 2971271168 + }, + { + "epoch": 0.9, + "learning_rate": 5.039117352056169e-05, + "loss": 1.2179, + "theoretical_loss": 3.323212685002952, + "tokens_seen": 2971402240 + }, + { + "epoch": 0.9, + "learning_rate": 5.0371113340020064e-05, + "loss": 1.3174, + "theoretical_loss": 3.323201392512067, + "tokens_seen": 2971533312 + }, + { + "epoch": 0.9, + "learning_rate": 5.035105315947844e-05, + "loss": 1.2979, + "theoretical_loss": 3.323190100658736, + "tokens_seen": 2971664384 + }, + { + "epoch": 0.9, + "learning_rate": 5.0330992978936815e-05, + "loss": 1.2982, + "theoretical_loss": 3.323178809442894, + "tokens_seen": 2971795456 + }, + { + "epoch": 0.9, + "learning_rate": 5.0310932798395184e-05, + "loss": 1.25, + "theoretical_loss": 3.323167518864478, + "tokens_seen": 2971926528 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.36198681592941284, + "objective/train/avg_token_score": 0.01861748844385147, + "objective/train/docs_used": 1672806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1584265232086182, + "objective/train/original_loss": 3.66042160987854, + "objective/train/theoretical_loss": 3.3231562289234233, + "objective/train/tokens_used": 2972057600, + "objective/train/unlikelihood_loss": 0.17715582251548767, + "theoretical_loss": 3.3231562289234233, + "tokens_seen": 2972057600 + }, + { + "epoch": 0.9, + "learning_rate": 5.029087261785356e-05, + "loss": 1.4752, + "theoretical_loss": 3.3231562289234233, + "tokens_seen": 2972057600 + }, + { + "epoch": 0.9, + "learning_rate": 5.0270812437311936e-05, + "loss": 1.2646, + "theoretical_loss": 3.3231449396196657, + "tokens_seen": 2972188672 + }, + { + "epoch": 0.9, + "learning_rate": 5.025075225677031e-05, + "loss": 1.4365, + "theoretical_loss": 3.3231336509531415, + "tokens_seen": 2972319744 + }, + { + "epoch": 0.9, + "learning_rate": 5.023069207622869e-05, + "loss": 1.2931, + "theoretical_loss": 3.3231223629237863, + "tokens_seen": 2972450816 + }, + { + "epoch": 0.9, + "learning_rate": 5.0210631895687057e-05, + "loss": 1.2273, + "theoretical_loss": 3.3231110755315365, + "tokens_seen": 2972581888 + }, + { + "epoch": 0.9, + "learning_rate": 5.019057171514544e-05, + "loss": 1.3031, + "theoretical_loss": 3.323099788776328, + "tokens_seen": 2972712960 + }, + { + "epoch": 0.9, + "learning_rate": 5.0170511534603815e-05, + "loss": 1.3511, + "theoretical_loss": 3.323088502658096, + "tokens_seen": 2972844032 + }, + { + "epoch": 0.9, + "learning_rate": 5.015045135406219e-05, + "loss": 1.2104, + "theoretical_loss": 3.323077217176777, + "tokens_seen": 2972975104 + }, + { + "epoch": 0.9, + "learning_rate": 5.0130391173520567e-05, + "loss": 1.192, + "theoretical_loss": 3.323065932332307, + "tokens_seen": 2973106176 + }, + { + "epoch": 0.9, + "learning_rate": 5.0110330992978936e-05, + "loss": 1.3191, + "theoretical_loss": 3.3230546481246224, + "tokens_seen": 2973237248 + }, + { + "epoch": 0.9, + "learning_rate": 5.009027081243731e-05, + "loss": 1.3631, + "theoretical_loss": 3.3230433645536586, + "tokens_seen": 2973368320 + }, + { + "epoch": 0.9, + "learning_rate": 5.007021063189569e-05, + "loss": 1.2625, + "theoretical_loss": 3.3230320816193517, + "tokens_seen": 2973499392 + }, + { + "epoch": 0.9, + "learning_rate": 5.005015045135406e-05, + "loss": 1.3319, + "theoretical_loss": 3.3230207993216374, + "tokens_seen": 2973630464 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.44776391983032227, + "objective/train/avg_token_score": 0.018690435215830803, + "objective/train/docs_used": 1673458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4716355800628662, + "objective/train/original_loss": 3.500473737716675, + "objective/train/theoretical_loss": 3.323015158411483, + "objective/train/tokens_used": 2973696000, + "objective/train/unlikelihood_loss": 0.18493488430976868, + "theoretical_loss": 3.323015158411483, + "tokens_seen": 2973696000 + }, + { + "epoch": 0.9, + "learning_rate": 5.003009027081243e-05, + "loss": 1.276, + "theoretical_loss": 3.3230095176604526, + "tokens_seen": 2973761536 + }, + { + "epoch": 0.9, + "learning_rate": 5.0010030090270814e-05, + "loss": 1.1556, + "theoretical_loss": 3.3229982366357325, + "tokens_seen": 2973892608 + }, + { + "epoch": 0.9, + "learning_rate": 4.998996990972919e-05, + "loss": 1.157, + "theoretical_loss": 3.322986956247414, + "tokens_seen": 2974023680 + }, + { + "epoch": 0.9, + "learning_rate": 4.9969909729187566e-05, + "loss": 1.4352, + "theoretical_loss": 3.322975676495432, + "tokens_seen": 2974154752 + }, + { + "epoch": 0.9, + "learning_rate": 4.994984954864594e-05, + "loss": 1.2373, + "theoretical_loss": 3.3229643973797236, + "tokens_seen": 2974285824 + }, + { + "epoch": 0.9, + "learning_rate": 4.992978936810431e-05, + "loss": 1.3049, + "theoretical_loss": 3.3229531189002244, + "tokens_seen": 2974416896 + }, + { + "epoch": 0.9, + "learning_rate": 4.990972918756269e-05, + "loss": 1.2431, + "theoretical_loss": 3.3229418410568705, + "tokens_seen": 2974547968 + }, + { + "epoch": 0.9, + "learning_rate": 4.988966900702106e-05, + "loss": 1.406, + "theoretical_loss": 3.3229305638495985, + "tokens_seen": 2974679040 + }, + { + "epoch": 0.9, + "learning_rate": 4.986960882647944e-05, + "loss": 1.3001, + "theoretical_loss": 3.3229192872783435, + "tokens_seen": 2974810112 + }, + { + "epoch": 0.9, + "learning_rate": 4.9849548645937814e-05, + "loss": 1.3257, + "theoretical_loss": 3.3229080113430425, + "tokens_seen": 2974941184 + }, + { + "epoch": 0.9, + "learning_rate": 4.982948846539619e-05, + "loss": 1.3267, + "theoretical_loss": 3.322896736043631, + "tokens_seen": 2975072256 + }, + { + "epoch": 0.9, + "learning_rate": 4.9809428284854566e-05, + "loss": 1.4026, + "theoretical_loss": 3.3228854613800456, + "tokens_seen": 2975203328 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.4116568863391876, + "objective/train/avg_token_score": 0.0072713508270680904, + "objective/train/docs_used": 1674243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2388291358947754, + "objective/train/original_loss": 3.3247992992401123, + "objective/train/theoretical_loss": 3.3228741873522223, + "objective/train/tokens_used": 2975334400, + "objective/train/unlikelihood_loss": 0.18993134796619415, + "theoretical_loss": 3.3228741873522223, + "tokens_seen": 2975334400 + }, + { + "epoch": 0.9, + "learning_rate": 4.978936810431294e-05, + "loss": 1.3084, + "theoretical_loss": 3.3228741873522223, + "tokens_seen": 2975334400 + }, + { + "epoch": 0.9, + "learning_rate": 4.976930792377132e-05, + "loss": 1.3181, + "theoretical_loss": 3.3228629139600967, + "tokens_seen": 2975465472 + }, + { + "epoch": 0.9, + "learning_rate": 4.974924774322969e-05, + "loss": 1.2609, + "theoretical_loss": 3.3228516412036058, + "tokens_seen": 2975596544 + }, + { + "epoch": 0.9, + "learning_rate": 4.972918756268806e-05, + "loss": 1.2611, + "theoretical_loss": 3.3228403690826855, + "tokens_seen": 2975727616 + }, + { + "epoch": 0.9, + "learning_rate": 4.970912738214644e-05, + "loss": 1.248, + "theoretical_loss": 3.3228290975972716, + "tokens_seen": 2975858688 + }, + { + "epoch": 0.9, + "learning_rate": 4.9689067201604814e-05, + "loss": 1.2631, + "theoretical_loss": 3.322817826747301, + "tokens_seen": 2975989760 + }, + { + "epoch": 0.9, + "learning_rate": 4.966900702106319e-05, + "loss": 1.4655, + "theoretical_loss": 3.3228065565327087, + "tokens_seen": 2976120832 + }, + { + "epoch": 0.9, + "learning_rate": 4.9648946840521565e-05, + "loss": 1.2943, + "theoretical_loss": 3.3227952869534323, + "tokens_seen": 2976251904 + }, + { + "epoch": 0.9, + "learning_rate": 4.962888665997994e-05, + "loss": 1.2202, + "theoretical_loss": 3.3227840180094073, + "tokens_seen": 2976382976 + }, + { + "epoch": 0.9, + "learning_rate": 4.960882647943832e-05, + "loss": 1.205, + "theoretical_loss": 3.3227727497005697, + "tokens_seen": 2976514048 + }, + { + "epoch": 0.9, + "learning_rate": 4.958876629889669e-05, + "loss": 1.2429, + "theoretical_loss": 3.3227614820268565, + "tokens_seen": 2976645120 + }, + { + "epoch": 0.9, + "learning_rate": 4.956870611835507e-05, + "loss": 1.2996, + "theoretical_loss": 3.322750214988203, + "tokens_seen": 2976776192 + }, + { + "epoch": 0.9, + "learning_rate": 4.9548645937813444e-05, + "loss": 1.359, + "theoretical_loss": 3.3227389485845458, + "tokens_seen": 2976907264 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.537206768989563, + "objective/train/avg_token_score": 0.005709453951567411, + "objective/train/docs_used": 1674993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5095945596694946, + "objective/train/original_loss": 3.1852078437805176, + "objective/train/theoretical_loss": 3.3227333156208214, + "objective/train/tokens_used": 2976972800, + "objective/train/unlikelihood_loss": 0.16800042986869812, + "theoretical_loss": 3.3227333156208214, + "tokens_seen": 2976972800 + }, + { + "epoch": 0.9, + "learning_rate": 4.952858575727181e-05, + "loss": 1.2846, + "theoretical_loss": 3.3227276828158216, + "tokens_seen": 2977038336 + }, + { + "epoch": 0.9, + "learning_rate": 4.950852557673019e-05, + "loss": 1.189, + "theoretical_loss": 3.322716417681966, + "tokens_seen": 2977169408 + }, + { + "epoch": 0.9, + "learning_rate": 4.9488465396188565e-05, + "loss": 1.3132, + "theoretical_loss": 3.3227051531829157, + "tokens_seen": 2977300480 + }, + { + "epoch": 0.9, + "learning_rate": 4.946840521564694e-05, + "loss": 1.2687, + "theoretical_loss": 3.322693889318607, + "tokens_seen": 2977431552 + }, + { + "epoch": 0.9, + "learning_rate": 4.944834503510532e-05, + "loss": 1.1651, + "theoretical_loss": 3.322682626088976, + "tokens_seen": 2977562624 + }, + { + "epoch": 0.9, + "learning_rate": 4.942828485456369e-05, + "loss": 1.277, + "theoretical_loss": 3.322671363493959, + "tokens_seen": 2977693696 + }, + { + "epoch": 0.9, + "learning_rate": 4.940822467402207e-05, + "loss": 1.3611, + "theoretical_loss": 3.3226601015334927, + "tokens_seen": 2977824768 + }, + { + "epoch": 0.9, + "learning_rate": 4.9388164493480444e-05, + "loss": 1.2939, + "theoretical_loss": 3.3226488402075125, + "tokens_seen": 2977955840 + }, + { + "epoch": 0.9, + "learning_rate": 4.936810431293882e-05, + "loss": 1.297, + "theoretical_loss": 3.322637579515956, + "tokens_seen": 2978086912 + }, + { + "epoch": 0.9, + "learning_rate": 4.9348044132397195e-05, + "loss": 1.2793, + "theoretical_loss": 3.3226263194587586, + "tokens_seen": 2978217984 + }, + { + "epoch": 0.9, + "learning_rate": 4.9327983951855564e-05, + "loss": 1.2976, + "theoretical_loss": 3.3226150600358566, + "tokens_seen": 2978349056 + }, + { + "epoch": 0.9, + "learning_rate": 4.930792377131394e-05, + "loss": 1.2626, + "theoretical_loss": 3.322603801247187, + "tokens_seen": 2978480128 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.4144061505794525, + "objective/train/avg_token_score": 0.02594846859574318, + "objective/train/docs_used": 1676283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.150779128074646, + "objective/train/original_loss": 3.456439733505249, + "objective/train/theoretical_loss": 3.3225925430926857, + "objective/train/tokens_used": 2978611200, + "objective/train/unlikelihood_loss": 0.2848641872406006, + "theoretical_loss": 3.3225925430926857, + "tokens_seen": 2978611200 + }, + { + "epoch": 0.9, + "learning_rate": 4.9287863590772316e-05, + "loss": 1.2518, + "theoretical_loss": 3.3225925430926857, + "tokens_seen": 2978611200 + }, + { + "epoch": 0.9, + "learning_rate": 4.92678034102307e-05, + "loss": 1.2536, + "theoretical_loss": 3.3225812855722894, + "tokens_seen": 2978742272 + }, + { + "epoch": 0.9, + "learning_rate": 4.9247743229689074e-05, + "loss": 1.1559, + "theoretical_loss": 3.3225700286859343, + "tokens_seen": 2978873344 + }, + { + "epoch": 0.9, + "learning_rate": 4.922768304914744e-05, + "loss": 1.2713, + "theoretical_loss": 3.3225587724335566, + "tokens_seen": 2979004416 + }, + { + "epoch": 0.9, + "learning_rate": 4.920762286860582e-05, + "loss": 1.3217, + "theoretical_loss": 3.322547516815093, + "tokens_seen": 2979135488 + }, + { + "epoch": 0.9, + "learning_rate": 4.9187562688064195e-05, + "loss": 1.2189, + "theoretical_loss": 3.3225362618304795, + "tokens_seen": 2979266560 + }, + { + "epoch": 0.9, + "learning_rate": 4.916750250752257e-05, + "loss": 1.3107, + "theoretical_loss": 3.322525007479653, + "tokens_seen": 2979397632 + }, + { + "epoch": 0.9, + "learning_rate": 4.914744232698094e-05, + "loss": 1.2327, + "theoretical_loss": 3.32251375376255, + "tokens_seen": 2979528704 + }, + { + "epoch": 0.9, + "learning_rate": 4.9127382146439316e-05, + "loss": 1.2976, + "theoretical_loss": 3.3225025006791067, + "tokens_seen": 2979659776 + }, + { + "epoch": 0.9, + "learning_rate": 4.910732196589769e-05, + "loss": 1.2229, + "theoretical_loss": 3.3224912482292597, + "tokens_seen": 2979790848 + }, + { + "epoch": 0.9, + "learning_rate": 4.908726178535607e-05, + "loss": 1.3756, + "theoretical_loss": 3.3224799964129446, + "tokens_seen": 2979921920 + }, + { + "epoch": 0.9, + "learning_rate": 4.906720160481445e-05, + "loss": 1.2239, + "theoretical_loss": 3.322468745230099, + "tokens_seen": 2980052992 + }, + { + "epoch": 0.9, + "learning_rate": 4.904714142427282e-05, + "loss": 1.4394, + "theoretical_loss": 3.322457494680659, + "tokens_seen": 2980184064 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.29349949955940247, + "objective/train/avg_token_score": 0.01918860524892807, + "objective/train/docs_used": 1676817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.974004864692688, + "objective/train/original_loss": 4.001476287841797, + "objective/train/theoretical_loss": 3.322451869643446, + "objective/train/tokens_used": 2980249600, + "objective/train/unlikelihood_loss": 0.1437520831823349, + "theoretical_loss": 3.322451869643446, + "tokens_seen": 2980249600 + }, + { + "epoch": 0.9, + "learning_rate": 4.9027081243731194e-05, + "loss": 1.3598, + "theoretical_loss": 3.3224462447645613, + "tokens_seen": 2980315136 + }, + { + "epoch": 0.9, + "learning_rate": 4.900702106318957e-05, + "loss": 1.3615, + "theoretical_loss": 3.322434995481742, + "tokens_seen": 2980446208 + }, + { + "epoch": 0.9, + "learning_rate": 4.8986960882647946e-05, + "loss": 1.3164, + "theoretical_loss": 3.322423746832137, + "tokens_seen": 2980577280 + }, + { + "epoch": 0.9, + "learning_rate": 4.896690070210632e-05, + "loss": 1.173, + "theoretical_loss": 3.3224124988156847, + "tokens_seen": 2980708352 + }, + { + "epoch": 0.9, + "learning_rate": 4.894684052156469e-05, + "loss": 1.3911, + "theoretical_loss": 3.3224012514323196, + "tokens_seen": 2980839424 + }, + { + "epoch": 0.9, + "learning_rate": 4.892678034102307e-05, + "loss": 1.3105, + "theoretical_loss": 3.3223900046819796, + "tokens_seen": 2980970496 + }, + { + "epoch": 0.9, + "learning_rate": 4.890672016048144e-05, + "loss": 1.2763, + "theoretical_loss": 3.3223787585646005, + "tokens_seen": 2981101568 + }, + { + "epoch": 0.9, + "learning_rate": 4.8886659979939825e-05, + "loss": 1.4578, + "theoretical_loss": 3.322367513080119, + "tokens_seen": 2981232640 + }, + { + "epoch": 0.9, + "learning_rate": 4.88665997993982e-05, + "loss": 1.1895, + "theoretical_loss": 3.322356268228472, + "tokens_seen": 2981363712 + }, + { + "epoch": 0.9, + "learning_rate": 4.884653961885657e-05, + "loss": 1.1844, + "theoretical_loss": 3.3223450240095955, + "tokens_seen": 2981494784 + }, + { + "epoch": 0.9, + "learning_rate": 4.8826479438314946e-05, + "loss": 1.2107, + "theoretical_loss": 3.3223337804234268, + "tokens_seen": 2981625856 + }, + { + "epoch": 0.9, + "learning_rate": 4.880641925777332e-05, + "loss": 1.3681, + "theoretical_loss": 3.322322537469902, + "tokens_seen": 2981756928 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.4495967626571655, + "objective/train/avg_token_score": 0.011645152233541012, + "objective/train/docs_used": 1677869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3791418075561523, + "objective/train/original_loss": 3.805734634399414, + "objective/train/theoretical_loss": 3.3223112951489577, + "objective/train/tokens_used": 2981888000, + "objective/train/unlikelihood_loss": 0.14310675859451294, + "theoretical_loss": 3.3223112951489577, + "tokens_seen": 2981888000 + }, + { + "epoch": 0.9, + "learning_rate": 4.87863590772317e-05, + "loss": 1.1517, + "theoretical_loss": 3.3223112951489577, + "tokens_seen": 2981888000 + }, + { + "epoch": 0.9, + "learning_rate": 4.876629889669007e-05, + "loss": 1.3213, + "theoretical_loss": 3.3223000534605305, + "tokens_seen": 2982019072 + }, + { + "epoch": 0.9, + "learning_rate": 4.874623871614844e-05, + "loss": 1.1709, + "theoretical_loss": 3.3222888124045573, + "tokens_seen": 2982150144 + }, + { + "epoch": 0.9, + "learning_rate": 4.872617853560682e-05, + "loss": 1.3307, + "theoretical_loss": 3.3222775719809743, + "tokens_seen": 2982281216 + }, + { + "epoch": 0.9, + "learning_rate": 4.8706118355065194e-05, + "loss": 1.3638, + "theoretical_loss": 3.3222663321897183, + "tokens_seen": 2982412288 + }, + { + "epoch": 0.9, + "learning_rate": 4.8686058174523576e-05, + "loss": 1.4666, + "theoretical_loss": 3.3222550930307264, + "tokens_seen": 2982543360 + }, + { + "epoch": 0.9, + "learning_rate": 4.866599799398195e-05, + "loss": 1.365, + "theoretical_loss": 3.3222438545039346, + "tokens_seen": 2982674432 + }, + { + "epoch": 0.9, + "learning_rate": 4.864593781344032e-05, + "loss": 1.2962, + "theoretical_loss": 3.3222326166092797, + "tokens_seen": 2982805504 + }, + { + "epoch": 0.9, + "learning_rate": 4.86258776328987e-05, + "loss": 1.2121, + "theoretical_loss": 3.322221379346699, + "tokens_seen": 2982936576 + }, + { + "epoch": 0.9, + "learning_rate": 4.860581745235707e-05, + "loss": 1.0939, + "theoretical_loss": 3.322210142716128, + "tokens_seen": 2983067648 + }, + { + "epoch": 0.9, + "learning_rate": 4.858575727181545e-05, + "loss": 1.1969, + "theoretical_loss": 3.3221989067175044, + "tokens_seen": 2983198720 + }, + { + "epoch": 0.9, + "learning_rate": 4.8565697091273824e-05, + "loss": 1.2313, + "theoretical_loss": 3.3221876713507643, + "tokens_seen": 2983329792 + }, + { + "epoch": 0.9, + "learning_rate": 4.854563691073219e-05, + "loss": 1.2404, + "theoretical_loss": 3.3221764366158446, + "tokens_seen": 2983460864 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.4998167157173157, + "objective/train/avg_token_score": 0.027897357940673828, + "objective/train/docs_used": 1678369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5467498302459717, + "objective/train/original_loss": 3.6268138885498047, + "objective/train/theoretical_loss": 3.322170819485298, + "objective/train/tokens_used": 2983526400, + "objective/train/unlikelihood_loss": 0.1697709858417511, + "theoretical_loss": 3.322170819485298, + "tokens_seen": 2983526400 + }, + { + "epoch": 0.9, + "learning_rate": 4.852557673019057e-05, + "loss": 1.3497, + "theoretical_loss": 3.322165202512682, + "tokens_seen": 2983591936 + }, + { + "epoch": 0.9, + "learning_rate": 4.850551654964895e-05, + "loss": 1.3077, + "theoretical_loss": 3.322153969041214, + "tokens_seen": 2983723008 + }, + { + "epoch": 0.9, + "learning_rate": 4.848545636910733e-05, + "loss": 1.341, + "theoretical_loss": 3.322142736201376, + "tokens_seen": 2983854080 + }, + { + "epoch": 0.9, + "learning_rate": 4.8465396188565696e-05, + "loss": 1.1776, + "theoretical_loss": 3.3221315039931056, + "tokens_seen": 2983985152 + }, + { + "epoch": 0.9, + "learning_rate": 4.844533600802407e-05, + "loss": 1.2326, + "theoretical_loss": 3.3221202724163392, + "tokens_seen": 2984116224 + }, + { + "epoch": 0.9, + "learning_rate": 4.842527582748245e-05, + "loss": 1.129, + "theoretical_loss": 3.3221090414710135, + "tokens_seen": 2984247296 + }, + { + "epoch": 0.9, + "learning_rate": 4.8405215646940824e-05, + "loss": 1.2295, + "theoretical_loss": 3.3220978111570654, + "tokens_seen": 2984378368 + }, + { + "epoch": 0.9, + "learning_rate": 4.83851554663992e-05, + "loss": 1.3013, + "theoretical_loss": 3.3220865814744323, + "tokens_seen": 2984509440 + }, + { + "epoch": 0.9, + "learning_rate": 4.836509528585757e-05, + "loss": 1.3939, + "theoretical_loss": 3.3220753524230497, + "tokens_seen": 2984640512 + }, + { + "epoch": 0.9, + "learning_rate": 4.8345035105315944e-05, + "loss": 1.2972, + "theoretical_loss": 3.3220641240028552, + "tokens_seen": 2984771584 + }, + { + "epoch": 0.9, + "learning_rate": 4.832497492477433e-05, + "loss": 1.3243, + "theoretical_loss": 3.322052896213786, + "tokens_seen": 2984902656 + }, + { + "epoch": 0.9, + "learning_rate": 4.83049147442327e-05, + "loss": 1.3592, + "theoretical_loss": 3.322041669055778, + "tokens_seen": 2985033728 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.3608260154724121, + "objective/train/avg_token_score": 0.023034678772091866, + "objective/train/docs_used": 1679141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1042897701263428, + "objective/train/original_loss": 4.110742092132568, + "objective/train/theoretical_loss": 3.3220304425287686, + "objective/train/tokens_used": 2985164800, + "objective/train/unlikelihood_loss": 0.12287315726280212, + "theoretical_loss": 3.3220304425287686, + "tokens_seen": 2985164800 + }, + { + "epoch": 0.9, + "learning_rate": 4.828485456369108e-05, + "loss": 1.3024, + "theoretical_loss": 3.3220304425287686, + "tokens_seen": 2985164800 + }, + { + "epoch": 0.9, + "learning_rate": 4.826479438314945e-05, + "loss": 1.2669, + "theoretical_loss": 3.3220192166326945, + "tokens_seen": 2985295872 + }, + { + "epoch": 0.9, + "learning_rate": 4.824473420260782e-05, + "loss": 1.0268, + "theoretical_loss": 3.3220079913674923, + "tokens_seen": 2985426944 + }, + { + "epoch": 0.9, + "learning_rate": 4.82246740220662e-05, + "loss": 1.275, + "theoretical_loss": 3.321996766733099, + "tokens_seen": 2985558016 + }, + { + "epoch": 0.9, + "learning_rate": 4.8204613841524575e-05, + "loss": 1.3346, + "theoretical_loss": 3.321985542729452, + "tokens_seen": 2985689088 + }, + { + "epoch": 0.9, + "learning_rate": 4.818455366098295e-05, + "loss": 1.2045, + "theoretical_loss": 3.321974319356487, + "tokens_seen": 2985820160 + }, + { + "epoch": 0.9, + "learning_rate": 4.816449348044132e-05, + "loss": 1.1568, + "theoretical_loss": 3.3219630966141422, + "tokens_seen": 2985951232 + }, + { + "epoch": 0.9, + "learning_rate": 4.8144433299899696e-05, + "loss": 1.1901, + "theoretical_loss": 3.3219518745023535, + "tokens_seen": 2986082304 + }, + { + "epoch": 0.9, + "learning_rate": 4.812437311935808e-05, + "loss": 1.3221, + "theoretical_loss": 3.3219406530210582, + "tokens_seen": 2986213376 + }, + { + "epoch": 0.9, + "learning_rate": 4.8104312938816454e-05, + "loss": 1.2083, + "theoretical_loss": 3.3219294321701933, + "tokens_seen": 2986344448 + }, + { + "epoch": 0.9, + "learning_rate": 4.808425275827483e-05, + "loss": 1.3117, + "theoretical_loss": 3.3219182119496953, + "tokens_seen": 2986475520 + }, + { + "epoch": 0.91, + "learning_rate": 4.80641925777332e-05, + "loss": 1.3051, + "theoretical_loss": 3.321906992359502, + "tokens_seen": 2986606592 + }, + { + "epoch": 0.91, + "learning_rate": 4.8044132397191574e-05, + "loss": 1.3271, + "theoretical_loss": 3.3218957733995493, + "tokens_seen": 2986737664 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.5298142433166504, + "objective/train/avg_token_score": 0.013697115704417229, + "objective/train/docs_used": 1680304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5507311820983887, + "objective/train/original_loss": 3.592539072036743, + "objective/train/theoretical_loss": 3.3218901641558936, + "objective/train/tokens_used": 2986803200, + "objective/train/unlikelihood_loss": 0.1529041975736618, + "theoretical_loss": 3.3218901641558936, + "tokens_seen": 2986803200 + }, + { + "epoch": 0.91, + "learning_rate": 4.802407221664995e-05, + "loss": 1.1987, + "theoretical_loss": 3.3218845550697744, + "tokens_seen": 2986868736 + }, + { + "epoch": 0.91, + "learning_rate": 4.8004012036108326e-05, + "loss": 1.2671, + "theoretical_loss": 3.3218733373701146, + "tokens_seen": 2986999808 + }, + { + "epoch": 0.91, + "learning_rate": 4.79839518555667e-05, + "loss": 1.2983, + "theoretical_loss": 3.3218621203005068, + "tokens_seen": 2987130880 + }, + { + "epoch": 0.91, + "learning_rate": 4.796389167502507e-05, + "loss": 1.3667, + "theoretical_loss": 3.321850903860888, + "tokens_seen": 2987261952 + }, + { + "epoch": 0.91, + "learning_rate": 4.7943831494483453e-05, + "loss": 1.1972, + "theoretical_loss": 3.3218396880511945, + "tokens_seen": 2987393024 + }, + { + "epoch": 0.91, + "learning_rate": 4.792377131394183e-05, + "loss": 1.2072, + "theoretical_loss": 3.321828472871364, + "tokens_seen": 2987524096 + }, + { + "epoch": 0.91, + "learning_rate": 4.7903711133400205e-05, + "loss": 1.2092, + "theoretical_loss": 3.3218172583213335, + "tokens_seen": 2987655168 + }, + { + "epoch": 0.91, + "learning_rate": 4.788365095285858e-05, + "loss": 1.2686, + "theoretical_loss": 3.3218060444010398, + "tokens_seen": 2987786240 + }, + { + "epoch": 0.91, + "learning_rate": 4.786359077231695e-05, + "loss": 1.2478, + "theoretical_loss": 3.3217948311104197, + "tokens_seen": 2987917312 + }, + { + "epoch": 0.91, + "learning_rate": 4.7843530591775326e-05, + "loss": 1.2415, + "theoretical_loss": 3.3217836184494107, + "tokens_seen": 2988048384 + }, + { + "epoch": 0.91, + "learning_rate": 4.78234704112337e-05, + "loss": 1.2083, + "theoretical_loss": 3.3217724064179492, + "tokens_seen": 2988179456 + }, + { + "epoch": 0.91, + "learning_rate": 4.780341023069208e-05, + "loss": 1.3688, + "theoretical_loss": 3.321761195015973, + "tokens_seen": 2988310528 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.4469085931777954, + "objective/train/avg_token_score": 0.05555637925863266, + "objective/train/docs_used": 1680987, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4558566808700562, + "objective/train/original_loss": 4.535808563232422, + "objective/train/theoretical_loss": 3.321749984243419, + "objective/train/tokens_used": 2988441600, + "objective/train/unlikelihood_loss": 0.09221773594617844, + "theoretical_loss": 3.321749984243419, + "tokens_seen": 2988441600 + }, + { + "epoch": 0.91, + "learning_rate": 4.7783350050150446e-05, + "loss": 1.2019, + "theoretical_loss": 3.321749984243419, + "tokens_seen": 2988441600 + }, + { + "epoch": 0.91, + "learning_rate": 4.776328986960883e-05, + "loss": 1.2041, + "theoretical_loss": 3.321738774100224, + "tokens_seen": 2988572672 + }, + { + "epoch": 0.91, + "learning_rate": 4.7743229689067205e-05, + "loss": 1.1229, + "theoretical_loss": 3.3217275645863245, + "tokens_seen": 2988703744 + }, + { + "epoch": 0.91, + "learning_rate": 4.772316950852558e-05, + "loss": 1.2345, + "theoretical_loss": 3.3217163557016582, + "tokens_seen": 2988834816 + }, + { + "epoch": 0.91, + "learning_rate": 4.7703109327983956e-05, + "loss": 1.134, + "theoretical_loss": 3.3217051474461625, + "tokens_seen": 2988965888 + }, + { + "epoch": 0.91, + "learning_rate": 4.7683049147442325e-05, + "loss": 1.2941, + "theoretical_loss": 3.3216939398197747, + "tokens_seen": 2989096960 + }, + { + "epoch": 0.91, + "learning_rate": 4.76629889669007e-05, + "loss": 1.2666, + "theoretical_loss": 3.3216827328224303, + "tokens_seen": 2989228032 + }, + { + "epoch": 0.91, + "learning_rate": 4.764292878635908e-05, + "loss": 1.2133, + "theoretical_loss": 3.321671526454068, + "tokens_seen": 2989359104 + }, + { + "epoch": 0.91, + "learning_rate": 4.762286860581745e-05, + "loss": 1.1908, + "theoretical_loss": 3.3216603207146247, + "tokens_seen": 2989490176 + }, + { + "epoch": 0.91, + "learning_rate": 4.760280842527583e-05, + "loss": 1.2851, + "theoretical_loss": 3.321649115604037, + "tokens_seen": 2989621248 + }, + { + "epoch": 0.91, + "learning_rate": 4.75827482447342e-05, + "loss": 1.4664, + "theoretical_loss": 3.321637911122242, + "tokens_seen": 2989752320 + }, + { + "epoch": 0.91, + "learning_rate": 4.756268806419258e-05, + "loss": 1.2248, + "theoretical_loss": 3.3216267072691776, + "tokens_seen": 2989883392 + }, + { + "epoch": 0.91, + "learning_rate": 4.7542627883650956e-05, + "loss": 1.3545, + "theoretical_loss": 3.3216155040447806, + "tokens_seen": 2990014464 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.5094085931777954, + "objective/train/avg_token_score": 0.010366490110754967, + "objective/train/docs_used": 1682396, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5647196769714355, + "objective/train/original_loss": 3.571889877319336, + "objective/train/theoretical_loss": 3.321609902668312, + "objective/train/tokens_used": 2990080000, + "objective/train/unlikelihood_loss": 0.1459992676973343, + "theoretical_loss": 3.321609902668312, + "tokens_seen": 2990080000 + }, + { + "epoch": 0.91, + "learning_rate": 4.752256770310933e-05, + "loss": 1.3572, + "theoretical_loss": 3.3216043014489873, + "tokens_seen": 2990145536 + }, + { + "epoch": 0.91, + "learning_rate": 4.750250752256771e-05, + "loss": 1.3324, + "theoretical_loss": 3.321593099481736, + "tokens_seen": 2990276608 + }, + { + "epoch": 0.91, + "learning_rate": 4.7482447342026076e-05, + "loss": 1.3912, + "theoretical_loss": 3.3215818981429637, + "tokens_seen": 2990407680 + }, + { + "epoch": 0.91, + "learning_rate": 4.746238716148445e-05, + "loss": 1.2837, + "theoretical_loss": 3.3215706974326076, + "tokens_seen": 2990538752 + }, + { + "epoch": 0.91, + "learning_rate": 4.744232698094283e-05, + "loss": 1.3051, + "theoretical_loss": 3.3215594973506044, + "tokens_seen": 2990669824 + }, + { + "epoch": 0.91, + "learning_rate": 4.7422266800401204e-05, + "loss": 1.0687, + "theoretical_loss": 3.3215482978968915, + "tokens_seen": 2990800896 + }, + { + "epoch": 0.91, + "learning_rate": 4.740220661985958e-05, + "loss": 1.2576, + "theoretical_loss": 3.3215370990714064, + "tokens_seen": 2990931968 + }, + { + "epoch": 0.91, + "learning_rate": 4.7382146439317955e-05, + "loss": 1.144, + "theoretical_loss": 3.3215259008740863, + "tokens_seen": 2991063040 + }, + { + "epoch": 0.91, + "learning_rate": 4.736208625877633e-05, + "loss": 1.2639, + "theoretical_loss": 3.321514703304868, + "tokens_seen": 2991194112 + }, + { + "epoch": 0.91, + "learning_rate": 4.734202607823471e-05, + "loss": 1.3199, + "theoretical_loss": 3.3215035063636895, + "tokens_seen": 2991325184 + }, + { + "epoch": 0.91, + "learning_rate": 4.732196589769308e-05, + "loss": 1.31, + "theoretical_loss": 3.3214923100504876, + "tokens_seen": 2991456256 + }, + { + "epoch": 0.91, + "learning_rate": 4.730190571715146e-05, + "loss": 1.0679, + "theoretical_loss": 3.3214811143651994, + "tokens_seen": 2991587328 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.37805473804473877, + "objective/train/avg_token_score": 0.018841581419110298, + "objective/train/docs_used": 1682734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1007354259490967, + "objective/train/original_loss": 4.1634955406188965, + "objective/train/theoretical_loss": 3.321469919307763, + "objective/train/tokens_used": 2991718400, + "objective/train/unlikelihood_loss": 0.14509688317775726, + "theoretical_loss": 3.321469919307763, + "tokens_seen": 2991718400 + }, + { + "epoch": 0.91, + "learning_rate": 4.728184553660983e-05, + "loss": 1.296, + "theoretical_loss": 3.321469919307763, + "tokens_seen": 2991718400 + }, + { + "epoch": 0.91, + "learning_rate": 4.72617853560682e-05, + "loss": 1.2212, + "theoretical_loss": 3.321458724878114, + "tokens_seen": 2991849472 + }, + { + "epoch": 0.91, + "learning_rate": 4.724172517552658e-05, + "loss": 1.215, + "theoretical_loss": 3.321447531076191, + "tokens_seen": 2991980544 + }, + { + "epoch": 0.91, + "learning_rate": 4.7221664994984955e-05, + "loss": 1.4859, + "theoretical_loss": 3.3214363379019316, + "tokens_seen": 2992111616 + }, + { + "epoch": 0.91, + "learning_rate": 4.720160481444334e-05, + "loss": 1.3871, + "theoretical_loss": 3.3214251453552723, + "tokens_seen": 2992242688 + }, + { + "epoch": 0.91, + "learning_rate": 4.7181544633901706e-05, + "loss": 1.3352, + "theoretical_loss": 3.321413953436151, + "tokens_seen": 2992373760 + }, + { + "epoch": 0.91, + "learning_rate": 4.716148445336008e-05, + "loss": 1.312, + "theoretical_loss": 3.3214027621445044, + "tokens_seen": 2992504832 + }, + { + "epoch": 0.91, + "learning_rate": 4.714142427281846e-05, + "loss": 1.371, + "theoretical_loss": 3.32139157148027, + "tokens_seen": 2992635904 + }, + { + "epoch": 0.91, + "learning_rate": 4.7121364092276834e-05, + "loss": 1.2778, + "theoretical_loss": 3.3213803814433858, + "tokens_seen": 2992766976 + }, + { + "epoch": 0.91, + "learning_rate": 4.710130391173521e-05, + "loss": 1.1907, + "theoretical_loss": 3.321369192033788, + "tokens_seen": 2992898048 + }, + { + "epoch": 0.91, + "learning_rate": 4.708124373119358e-05, + "loss": 1.1939, + "theoretical_loss": 3.321358003251415, + "tokens_seen": 2993029120 + }, + { + "epoch": 0.91, + "learning_rate": 4.7061183550651954e-05, + "loss": 1.1905, + "theoretical_loss": 3.321346815096204, + "tokens_seen": 2993160192 + }, + { + "epoch": 0.91, + "learning_rate": 4.704112337011033e-05, + "loss": 1.3722, + "theoretical_loss": 3.321335627568092, + "tokens_seen": 2993291264 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.3290567100048065, + "objective/train/avg_token_score": 0.01258845254778862, + "objective/train/docs_used": 1684023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0588655471801758, + "objective/train/original_loss": 4.040921211242676, + "objective/train/theoretical_loss": 3.321330034039179, + "objective/train/tokens_used": 2993356800, + "objective/train/unlikelihood_loss": 0.19029521942138672, + "theoretical_loss": 3.321330034039179, + "tokens_seen": 2993356800 + }, + { + "epoch": 0.91, + "learning_rate": 4.7021063189568706e-05, + "loss": 1.2947, + "theoretical_loss": 3.321324440667017, + "tokens_seen": 2993422336 + }, + { + "epoch": 0.91, + "learning_rate": 4.700100300902708e-05, + "loss": 1.3177, + "theoretical_loss": 3.3213132543929156, + "tokens_seen": 2993553408 + }, + { + "epoch": 0.91, + "learning_rate": 4.698094282848546e-05, + "loss": 1.394, + "theoretical_loss": 3.3213020687457258, + "tokens_seen": 2993684480 + }, + { + "epoch": 0.91, + "learning_rate": 4.6960882647943833e-05, + "loss": 1.3716, + "theoretical_loss": 3.3212908837253847, + "tokens_seen": 2993815552 + }, + { + "epoch": 0.91, + "learning_rate": 4.694082246740221e-05, + "loss": 1.3033, + "theoretical_loss": 3.3212796993318294, + "tokens_seen": 2993946624 + }, + { + "epoch": 0.91, + "learning_rate": 4.6920762286860585e-05, + "loss": 1.1658, + "theoretical_loss": 3.3212685155649986, + "tokens_seen": 2994077696 + }, + { + "epoch": 0.91, + "learning_rate": 4.6900702106318954e-05, + "loss": 1.2596, + "theoretical_loss": 3.3212573324248282, + "tokens_seen": 2994208768 + }, + { + "epoch": 0.91, + "learning_rate": 4.688064192577733e-05, + "loss": 1.3547, + "theoretical_loss": 3.321246149911257, + "tokens_seen": 2994339840 + }, + { + "epoch": 0.91, + "learning_rate": 4.6860581745235706e-05, + "loss": 1.2977, + "theoretical_loss": 3.3212349680242217, + "tokens_seen": 2994470912 + }, + { + "epoch": 0.91, + "learning_rate": 4.684052156469408e-05, + "loss": 1.3066, + "theoretical_loss": 3.32122378676366, + "tokens_seen": 2994601984 + }, + { + "epoch": 0.91, + "learning_rate": 4.6820461384152464e-05, + "loss": 1.2875, + "theoretical_loss": 3.321212606129509, + "tokens_seen": 2994733056 + }, + { + "epoch": 0.91, + "learning_rate": 4.680040120361083e-05, + "loss": 1.2464, + "theoretical_loss": 3.3212014261217067, + "tokens_seen": 2994864128 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.40292033553123474, + "objective/train/avg_token_score": 0.01786266826093197, + "objective/train/docs_used": 1684335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3193838596343994, + "objective/train/original_loss": 4.224428653717041, + "objective/train/theoretical_loss": 3.3211902467401906, + "objective/train/tokens_used": 2994995200, + "objective/train/unlikelihood_loss": 0.10640022903680801, + "theoretical_loss": 3.3211902467401906, + "tokens_seen": 2994995200 + }, + { + "epoch": 0.91, + "learning_rate": 4.678034102306921e-05, + "loss": 1.3039, + "theoretical_loss": 3.3211902467401906, + "tokens_seen": 2994995200 + }, + { + "epoch": 0.91, + "learning_rate": 4.6760280842527585e-05, + "loss": 1.3296, + "theoretical_loss": 3.3211790679848976, + "tokens_seen": 2995126272 + }, + { + "epoch": 0.91, + "learning_rate": 4.674022066198596e-05, + "loss": 1.191, + "theoretical_loss": 3.321167889855766, + "tokens_seen": 2995257344 + }, + { + "epoch": 0.91, + "learning_rate": 4.6720160481444336e-05, + "loss": 1.1915, + "theoretical_loss": 3.321156712352733, + "tokens_seen": 2995388416 + }, + { + "epoch": 0.91, + "learning_rate": 4.6700100300902705e-05, + "loss": 1.386, + "theoretical_loss": 3.3211455354757358, + "tokens_seen": 2995519488 + }, + { + "epoch": 0.91, + "learning_rate": 4.668004012036108e-05, + "loss": 1.287, + "theoretical_loss": 3.3211343592247125, + "tokens_seen": 2995650560 + }, + { + "epoch": 0.91, + "learning_rate": 4.665997993981946e-05, + "loss": 1.1567, + "theoretical_loss": 3.3211231835996005, + "tokens_seen": 2995781632 + }, + { + "epoch": 0.91, + "learning_rate": 4.663991975927784e-05, + "loss": 1.3672, + "theoretical_loss": 3.321112008600337, + "tokens_seen": 2995912704 + }, + { + "epoch": 0.91, + "learning_rate": 4.6619859578736215e-05, + "loss": 1.2572, + "theoretical_loss": 3.32110083422686, + "tokens_seen": 2996043776 + }, + { + "epoch": 0.91, + "learning_rate": 4.6599799398194584e-05, + "loss": 1.2996, + "theoretical_loss": 3.3210896604791067, + "tokens_seen": 2996174848 + }, + { + "epoch": 0.91, + "learning_rate": 4.657973921765296e-05, + "loss": 1.0706, + "theoretical_loss": 3.321078487357015, + "tokens_seen": 2996305920 + }, + { + "epoch": 0.91, + "learning_rate": 4.6559679037111336e-05, + "loss": 1.379, + "theoretical_loss": 3.3210673148605228, + "tokens_seen": 2996436992 + }, + { + "epoch": 0.91, + "learning_rate": 4.653961885656971e-05, + "loss": 1.296, + "theoretical_loss": 3.3210561429895673, + "tokens_seen": 2996568064 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.345796674489975, + "objective/train/avg_token_score": 0.010724522173404694, + "objective/train/docs_used": 1685661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9737365245819092, + "objective/train/original_loss": 3.9215855598449707, + "objective/train/theoretical_loss": 3.321050557288646, + "objective/train/tokens_used": 2996633600, + "objective/train/unlikelihood_loss": 0.13715173304080963, + "theoretical_loss": 3.321050557288646, + "tokens_seen": 2996633600 + }, + { + "epoch": 0.91, + "learning_rate": 4.651955867602809e-05, + "loss": 1.3261, + "theoretical_loss": 3.3210449717440857, + "tokens_seen": 2996699136 + }, + { + "epoch": 0.91, + "learning_rate": 4.6499498495486456e-05, + "loss": 1.243, + "theoretical_loss": 3.3210338011240164, + "tokens_seen": 2996830208 + }, + { + "epoch": 0.91, + "learning_rate": 4.647943831494483e-05, + "loss": 1.2079, + "theoretical_loss": 3.3210226311292965, + "tokens_seen": 2996961280 + }, + { + "epoch": 0.91, + "learning_rate": 4.645937813440321e-05, + "loss": 1.394, + "theoretical_loss": 3.321011461759864, + "tokens_seen": 2997092352 + }, + { + "epoch": 0.91, + "learning_rate": 4.643931795386159e-05, + "loss": 1.3738, + "theoretical_loss": 3.3210002930156564, + "tokens_seen": 2997223424 + }, + { + "epoch": 0.91, + "learning_rate": 4.6419257773319966e-05, + "loss": 1.1706, + "theoretical_loss": 3.3209891248966112, + "tokens_seen": 2997354496 + }, + { + "epoch": 0.91, + "learning_rate": 4.6399197592778335e-05, + "loss": 1.4895, + "theoretical_loss": 3.3209779574026665, + "tokens_seen": 2997485568 + }, + { + "epoch": 0.91, + "learning_rate": 4.637913741223671e-05, + "loss": 1.2635, + "theoretical_loss": 3.3209667905337596, + "tokens_seen": 2997616640 + }, + { + "epoch": 0.91, + "learning_rate": 4.635907723169509e-05, + "loss": 1.2752, + "theoretical_loss": 3.3209556242898284, + "tokens_seen": 2997747712 + }, + { + "epoch": 0.91, + "learning_rate": 4.633901705115346e-05, + "loss": 1.2413, + "theoretical_loss": 3.3209444586708106, + "tokens_seen": 2997878784 + }, + { + "epoch": 0.91, + "learning_rate": 4.631895687061183e-05, + "loss": 1.3749, + "theoretical_loss": 3.3209332936766436, + "tokens_seen": 2998009856 + }, + { + "epoch": 0.91, + "learning_rate": 4.629889669007021e-05, + "loss": 1.4043, + "theoretical_loss": 3.3209221293072653, + "tokens_seen": 2998140928 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.533663272857666, + "objective/train/avg_token_score": 0.017829090356826782, + "objective/train/docs_used": 1686525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5390069484710693, + "objective/train/original_loss": 3.824795722961426, + "objective/train/theoretical_loss": 3.3209109655626134, + "objective/train/tokens_used": 2998272000, + "objective/train/unlikelihood_loss": 0.14395175874233246, + "theoretical_loss": 3.3209109655626134, + "tokens_seen": 2998272000 + }, + { + "epoch": 0.91, + "learning_rate": 4.627883650952858e-05, + "loss": 1.208, + "theoretical_loss": 3.3209109655626134, + "tokens_seen": 2998272000 + }, + { + "epoch": 0.91, + "learning_rate": 4.6258776328986966e-05, + "loss": 1.2411, + "theoretical_loss": 3.320899802442626, + "tokens_seen": 2998403072 + }, + { + "epoch": 0.91, + "learning_rate": 4.623871614844534e-05, + "loss": 1.3967, + "theoretical_loss": 3.32088863994724, + "tokens_seen": 2998534144 + }, + { + "epoch": 0.91, + "learning_rate": 4.621865596790371e-05, + "loss": 1.3095, + "theoretical_loss": 3.3208774780763943, + "tokens_seen": 2998665216 + }, + { + "epoch": 0.91, + "learning_rate": 4.6198595787362086e-05, + "loss": 1.3811, + "theoretical_loss": 3.320866316830026, + "tokens_seen": 2998796288 + }, + { + "epoch": 0.91, + "learning_rate": 4.617853560682046e-05, + "loss": 1.2445, + "theoretical_loss": 3.320855156208072, + "tokens_seen": 2998927360 + }, + { + "epoch": 0.91, + "learning_rate": 4.615847542627884e-05, + "loss": 1.1651, + "theoretical_loss": 3.3208439962104723, + "tokens_seen": 2999058432 + }, + { + "epoch": 0.91, + "learning_rate": 4.6138415245737214e-05, + "loss": 1.3492, + "theoretical_loss": 3.3208328368371625, + "tokens_seen": 2999189504 + }, + { + "epoch": 0.91, + "learning_rate": 4.611835506519558e-05, + "loss": 1.2061, + "theoretical_loss": 3.3208216780880817, + "tokens_seen": 2999320576 + }, + { + "epoch": 0.91, + "learning_rate": 4.609829488465396e-05, + "loss": 1.1912, + "theoretical_loss": 3.320810519963167, + "tokens_seen": 2999451648 + }, + { + "epoch": 0.91, + "learning_rate": 4.6078234704112334e-05, + "loss": 1.3964, + "theoretical_loss": 3.3207993624623566, + "tokens_seen": 2999582720 + }, + { + "epoch": 0.91, + "learning_rate": 4.605817452357072e-05, + "loss": 1.2227, + "theoretical_loss": 3.3207882055855884, + "tokens_seen": 2999713792 + }, + { + "epoch": 0.91, + "learning_rate": 4.603811434302909e-05, + "loss": 1.3724, + "theoretical_loss": 3.3207770493327997, + "tokens_seen": 2999844864 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.40591397881507874, + "objective/train/avg_token_score": 0.009874888695776463, + "objective/train/docs_used": 1688067, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1594709157943726, + "objective/train/original_loss": 3.710409164428711, + "objective/train/theoretical_loss": 3.320771471440379, + "objective/train/tokens_used": 2999910400, + "objective/train/unlikelihood_loss": 0.1433904469013214, + "theoretical_loss": 3.320771471440379, + "tokens_seen": 2999910400 + }, + { + "epoch": 0.91, + "learning_rate": 4.601805416248746e-05, + "loss": 1.1463, + "theoretical_loss": 3.3207658937039293, + "tokens_seen": 2999975936 + }, + { + "epoch": 0.91, + "learning_rate": 4.599799398194584e-05, + "loss": 1.4594, + "theoretical_loss": 3.320754738698914, + "tokens_seen": 3000107008 + }, + { + "epoch": 0.91, + "learning_rate": 4.5977933801404213e-05, + "loss": 1.2401, + "theoretical_loss": 3.3207435843176922, + "tokens_seen": 3000238080 + }, + { + "epoch": 0.91, + "learning_rate": 4.595787362086259e-05, + "loss": 1.4304, + "theoretical_loss": 3.3207324305602013, + "tokens_seen": 3000369152 + }, + { + "epoch": 0.91, + "learning_rate": 4.5937813440320965e-05, + "loss": 1.275, + "theoretical_loss": 3.32072127742638, + "tokens_seen": 3000500224 + }, + { + "epoch": 0.91, + "learning_rate": 4.5917753259779334e-05, + "loss": 1.312, + "theoretical_loss": 3.3207101249161655, + "tokens_seen": 3000631296 + }, + { + "epoch": 0.91, + "learning_rate": 4.589769307923771e-05, + "loss": 1.2177, + "theoretical_loss": 3.3206989730294962, + "tokens_seen": 3000762368 + }, + { + "epoch": 0.91, + "learning_rate": 4.587763289869609e-05, + "loss": 1.2647, + "theoretical_loss": 3.3206878217663096, + "tokens_seen": 3000893440 + }, + { + "epoch": 0.91, + "learning_rate": 4.585757271815447e-05, + "loss": 1.4539, + "theoretical_loss": 3.3206766711265434, + "tokens_seen": 3001024512 + }, + { + "epoch": 0.91, + "learning_rate": 4.5837512537612844e-05, + "loss": 1.334, + "theoretical_loss": 3.3206655211101364, + "tokens_seen": 3001155584 + }, + { + "epoch": 0.91, + "learning_rate": 4.581745235707121e-05, + "loss": 1.3133, + "theoretical_loss": 3.320654371717026, + "tokens_seen": 3001286656 + }, + { + "epoch": 0.91, + "learning_rate": 4.579739217652959e-05, + "loss": 1.428, + "theoretical_loss": 3.3206432229471496, + "tokens_seen": 3001417728 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.4345674514770508, + "objective/train/avg_token_score": 0.004851819481700659, + "objective/train/docs_used": 1688535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2773637771606445, + "objective/train/original_loss": 3.9708919525146484, + "objective/train/theoretical_loss": 3.320632074800446, + "objective/train/tokens_used": 3001548800, + "objective/train/unlikelihood_loss": 0.16752760112285614, + "theoretical_loss": 3.320632074800446, + "tokens_seen": 3001548800 + }, + { + "epoch": 0.91, + "learning_rate": 4.5777331995987965e-05, + "loss": 1.3383, + "theoretical_loss": 3.320632074800446, + "tokens_seen": 3001548800 + }, + { + "epoch": 0.91, + "learning_rate": 4.575727181544634e-05, + "loss": 1.217, + "theoretical_loss": 3.320620927276853, + "tokens_seen": 3001679872 + }, + { + "epoch": 0.91, + "learning_rate": 4.5737211634904716e-05, + "loss": 1.2607, + "theoretical_loss": 3.320609780376308, + "tokens_seen": 3001810944 + }, + { + "epoch": 0.91, + "learning_rate": 4.5717151454363085e-05, + "loss": 1.3899, + "theoretical_loss": 3.3205986340987494, + "tokens_seen": 3001942016 + }, + { + "epoch": 0.91, + "learning_rate": 4.569709127382147e-05, + "loss": 1.2878, + "theoretical_loss": 3.3205874884441156, + "tokens_seen": 3002073088 + }, + { + "epoch": 0.91, + "learning_rate": 4.5677031093279844e-05, + "loss": 1.431, + "theoretical_loss": 3.320576343412344, + "tokens_seen": 3002204160 + }, + { + "epoch": 0.91, + "learning_rate": 4.565697091273822e-05, + "loss": 1.3211, + "theoretical_loss": 3.3205651990033727, + "tokens_seen": 3002335232 + }, + { + "epoch": 0.91, + "learning_rate": 4.5636910732196595e-05, + "loss": 1.3443, + "theoretical_loss": 3.3205540552171398, + "tokens_seen": 3002466304 + }, + { + "epoch": 0.91, + "learning_rate": 4.5616850551654964e-05, + "loss": 1.4616, + "theoretical_loss": 3.320542912053583, + "tokens_seen": 3002597376 + }, + { + "epoch": 0.91, + "learning_rate": 4.559679037111334e-05, + "loss": 1.2822, + "theoretical_loss": 3.3205317695126406, + "tokens_seen": 3002728448 + }, + { + "epoch": 0.91, + "learning_rate": 4.5576730190571716e-05, + "loss": 1.1819, + "theoretical_loss": 3.320520627594251, + "tokens_seen": 3002859520 + }, + { + "epoch": 0.91, + "learning_rate": 4.555667001003009e-05, + "loss": 1.2081, + "theoretical_loss": 3.3205094862983517, + "tokens_seen": 3002990592 + }, + { + "epoch": 0.91, + "learning_rate": 4.553660982948846e-05, + "loss": 1.3838, + "theoretical_loss": 3.320498345624881, + "tokens_seen": 3003121664 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.4362170100212097, + "objective/train/avg_token_score": 0.024111630395054817, + "objective/train/docs_used": 1689746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3588215112686157, + "objective/train/original_loss": 4.238311767578125, + "objective/train/theoretical_loss": 3.320492775521537, + "objective/train/tokens_used": 3003187200, + "objective/train/unlikelihood_loss": 0.12540104985237122, + "theoretical_loss": 3.320492775521537, + "tokens_seen": 3003187200 + }, + { + "epoch": 0.91, + "learning_rate": 4.5516549648946836e-05, + "loss": 1.3398, + "theoretical_loss": 3.320487205573777, + "tokens_seen": 3003252736 + }, + { + "epoch": 0.91, + "learning_rate": 4.549648946840522e-05, + "loss": 1.2824, + "theoretical_loss": 3.3204760661449777, + "tokens_seen": 3003383808 + }, + { + "epoch": 0.91, + "learning_rate": 4.5476429287863595e-05, + "loss": 1.2328, + "theoretical_loss": 3.3204649273384206, + "tokens_seen": 3003514880 + }, + { + "epoch": 0.91, + "learning_rate": 4.545636910732197e-05, + "loss": 1.3459, + "theoretical_loss": 3.320453789154045, + "tokens_seen": 3003645952 + }, + { + "epoch": 0.91, + "learning_rate": 4.543630892678034e-05, + "loss": 1.2898, + "theoretical_loss": 3.320442651591788, + "tokens_seen": 3003777024 + }, + { + "epoch": 0.91, + "learning_rate": 4.5416248746238715e-05, + "loss": 1.1518, + "theoretical_loss": 3.320431514651588, + "tokens_seen": 3003908096 + }, + { + "epoch": 0.91, + "learning_rate": 4.539618856569709e-05, + "loss": 1.2797, + "theoretical_loss": 3.3204203783333837, + "tokens_seen": 3004039168 + }, + { + "epoch": 0.91, + "learning_rate": 4.537612838515547e-05, + "loss": 1.3609, + "theoretical_loss": 3.3204092426371123, + "tokens_seen": 3004170240 + }, + { + "epoch": 0.91, + "learning_rate": 4.535606820461384e-05, + "loss": 1.3155, + "theoretical_loss": 3.3203981075627125, + "tokens_seen": 3004301312 + }, + { + "epoch": 0.91, + "learning_rate": 4.533600802407221e-05, + "loss": 1.3856, + "theoretical_loss": 3.3203869731101223, + "tokens_seen": 3004432384 + }, + { + "epoch": 0.91, + "learning_rate": 4.5315947843530594e-05, + "loss": 1.3424, + "theoretical_loss": 3.3203758392792797, + "tokens_seen": 3004563456 + }, + { + "epoch": 0.91, + "learning_rate": 4.529588766298897e-05, + "loss": 1.3725, + "theoretical_loss": 3.320364706070123, + "tokens_seen": 3004694528 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.33693793416023254, + "objective/train/avg_token_score": 0.044478919357061386, + "objective/train/docs_used": 1690381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0937467813491821, + "objective/train/original_loss": 4.479576587677002, + "objective/train/theoretical_loss": 3.32035357348259, + "objective/train/tokens_used": 3004825600, + "objective/train/unlikelihood_loss": 0.14575006067752838, + "theoretical_loss": 3.32035357348259, + "tokens_seen": 3004825600 + }, + { + "epoch": 0.91, + "learning_rate": 4.5275827482447346e-05, + "loss": 1.1394, + "theoretical_loss": 3.32035357348259, + "tokens_seen": 3004825600 + }, + { + "epoch": 0.91, + "learning_rate": 4.525576730190572e-05, + "loss": 1.3826, + "theoretical_loss": 3.32034244151662, + "tokens_seen": 3004956672 + }, + { + "epoch": 0.91, + "learning_rate": 4.523570712136409e-05, + "loss": 1.257, + "theoretical_loss": 3.3203313101721497, + "tokens_seen": 3005087744 + }, + { + "epoch": 0.91, + "learning_rate": 4.5215646940822466e-05, + "loss": 1.1659, + "theoretical_loss": 3.3203201794491184, + "tokens_seen": 3005218816 + }, + { + "epoch": 0.91, + "learning_rate": 4.519558676028084e-05, + "loss": 1.1411, + "theoretical_loss": 3.320309049347464, + "tokens_seen": 3005349888 + }, + { + "epoch": 0.91, + "learning_rate": 4.517552657973922e-05, + "loss": 1.3589, + "theoretical_loss": 3.3202979198671243, + "tokens_seen": 3005480960 + }, + { + "epoch": 0.91, + "learning_rate": 4.5155466399197594e-05, + "loss": 1.1364, + "theoretical_loss": 3.320286791008038, + "tokens_seen": 3005612032 + }, + { + "epoch": 0.91, + "learning_rate": 4.513540621865597e-05, + "loss": 1.242, + "theoretical_loss": 3.3202756627701433, + "tokens_seen": 3005743104 + }, + { + "epoch": 0.91, + "learning_rate": 4.5115346038114345e-05, + "loss": 1.3262, + "theoretical_loss": 3.3202645351533784, + "tokens_seen": 3005874176 + }, + { + "epoch": 0.91, + "learning_rate": 4.509528585757272e-05, + "loss": 1.2572, + "theoretical_loss": 3.320253408157681, + "tokens_seen": 3006005248 + }, + { + "epoch": 0.91, + "learning_rate": 4.50752256770311e-05, + "loss": 1.3114, + "theoretical_loss": 3.32024228178299, + "tokens_seen": 3006136320 + }, + { + "epoch": 0.91, + "learning_rate": 4.50752256770311e-05, + "loss": 1.272, + "theoretical_loss": 3.320231156029244, + "tokens_seen": 3006267392 + }, + { + "epoch": 0.91, + "learning_rate": 4.505516549648947e-05, + "loss": 1.2339, + "theoretical_loss": 3.3202200308963805, + "tokens_seen": 3006398464 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.5010386109352112, + "objective/train/avg_token_score": 0.014564288780093193, + "objective/train/docs_used": 1691682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6323567628860474, + "objective/train/original_loss": 3.8835341930389404, + "objective/train/theoretical_loss": 3.3202144685627606, + "objective/train/tokens_used": 3006464000, + "objective/train/unlikelihood_loss": 0.12981125712394714, + "theoretical_loss": 3.3202144685627606, + "tokens_seen": 3006464000 + }, + { + "epoch": 0.91, + "learning_rate": 4.503510531594784e-05, + "loss": 1.378, + "theoretical_loss": 3.3202089063843383, + "tokens_seen": 3006529536 + }, + { + "epoch": 0.91, + "learning_rate": 4.501504513540622e-05, + "loss": 1.3428, + "theoretical_loss": 3.320197782493055, + "tokens_seen": 3006660608 + }, + { + "epoch": 0.91, + "learning_rate": 4.4994984954864593e-05, + "loss": 1.3757, + "theoretical_loss": 3.3201866592224696, + "tokens_seen": 3006791680 + }, + { + "epoch": 0.91, + "learning_rate": 4.497492477432297e-05, + "loss": 1.3646, + "theoretical_loss": 3.32017553657252, + "tokens_seen": 3006922752 + }, + { + "epoch": 0.91, + "learning_rate": 4.4954864593781345e-05, + "loss": 1.3457, + "theoretical_loss": 3.320164414543145, + "tokens_seen": 3007053824 + }, + { + "epoch": 0.91, + "learning_rate": 4.493480441323972e-05, + "loss": 1.3104, + "theoretical_loss": 3.320153293134282, + "tokens_seen": 3007184896 + }, + { + "epoch": 0.91, + "learning_rate": 4.4914744232698097e-05, + "loss": 1.2445, + "theoretical_loss": 3.3201421723458706, + "tokens_seen": 3007315968 + }, + { + "epoch": 0.91, + "learning_rate": 4.489468405215647e-05, + "loss": 1.3243, + "theoretical_loss": 3.320131052177848, + "tokens_seen": 3007447040 + }, + { + "epoch": 0.91, + "learning_rate": 4.487462387161485e-05, + "loss": 1.2563, + "theoretical_loss": 3.3201199326301536, + "tokens_seen": 3007578112 + }, + { + "epoch": 0.91, + "learning_rate": 4.485456369107322e-05, + "loss": 1.341, + "theoretical_loss": 3.320108813702725, + "tokens_seen": 3007709184 + }, + { + "epoch": 0.91, + "learning_rate": 4.483450351053159e-05, + "loss": 1.349, + "theoretical_loss": 3.320097695395501, + "tokens_seen": 3007840256 + }, + { + "epoch": 0.91, + "learning_rate": 4.481444332998997e-05, + "loss": 1.389, + "theoretical_loss": 3.320086577708419, + "tokens_seen": 3007971328 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.35404446721076965, + "objective/train/avg_token_score": 0.01661643013358116, + "objective/train/docs_used": 1692449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1186039447784424, + "objective/train/original_loss": 4.065703868865967, + "objective/train/theoretical_loss": 3.320075460641419, + "objective/train/tokens_used": 3008102400, + "objective/train/unlikelihood_loss": 0.14603877067565918, + "theoretical_loss": 3.320075460641419, + "tokens_seen": 3008102400 + }, + { + "epoch": 0.91, + "learning_rate": 4.4794383149448345e-05, + "loss": 1.3728, + "theoretical_loss": 3.320075460641419, + "tokens_seen": 3008102400 + }, + { + "epoch": 0.91, + "learning_rate": 4.477432296890672e-05, + "loss": 1.193, + "theoretical_loss": 3.320064344194438, + "tokens_seen": 3008233472 + }, + { + "epoch": 0.91, + "learning_rate": 4.4754262788365096e-05, + "loss": 1.2557, + "theoretical_loss": 3.3200532283674153, + "tokens_seen": 3008364544 + }, + { + "epoch": 0.91, + "learning_rate": 4.473420260782347e-05, + "loss": 1.244, + "theoretical_loss": 3.320042113160289, + "tokens_seen": 3008495616 + }, + { + "epoch": 0.91, + "learning_rate": 4.471414242728185e-05, + "loss": 1.2098, + "theoretical_loss": 3.320030998572997, + "tokens_seen": 3008626688 + }, + { + "epoch": 0.91, + "learning_rate": 4.4694082246740224e-05, + "loss": 1.2481, + "theoretical_loss": 3.320019884605479, + "tokens_seen": 3008757760 + }, + { + "epoch": 0.91, + "learning_rate": 4.46740220661986e-05, + "loss": 1.1889, + "theoretical_loss": 3.3200087712576725, + "tokens_seen": 3008888832 + }, + { + "epoch": 0.91, + "learning_rate": 4.465396188565697e-05, + "loss": 1.4814, + "theoretical_loss": 3.319997658529516, + "tokens_seen": 3009019904 + }, + { + "epoch": 0.91, + "learning_rate": 4.4633901705115344e-05, + "loss": 1.2966, + "theoretical_loss": 3.3199865464209477, + "tokens_seen": 3009150976 + }, + { + "epoch": 0.91, + "learning_rate": 4.461384152457372e-05, + "loss": 1.2715, + "theoretical_loss": 3.319975434931907, + "tokens_seen": 3009282048 + }, + { + "epoch": 0.91, + "learning_rate": 4.4593781344032096e-05, + "loss": 1.2703, + "theoretical_loss": 3.3199643240623318, + "tokens_seen": 3009413120 + }, + { + "epoch": 0.91, + "learning_rate": 4.457372116349048e-05, + "loss": 1.3549, + "theoretical_loss": 3.3199532138121604, + "tokens_seen": 3009544192 + }, + { + "epoch": 0.91, + "learning_rate": 4.455366098294885e-05, + "loss": 1.2954, + "theoretical_loss": 3.3199421041813317, + "tokens_seen": 3009675264 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.5082477927207947, + "objective/train/avg_token_score": 0.016872771084308624, + "objective/train/docs_used": 1693425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4540519714355469, + "objective/train/original_loss": 3.6348342895507812, + "objective/train/theoretical_loss": 3.3199365495981517, + "objective/train/tokens_used": 3009740800, + "objective/train/unlikelihood_loss": 0.22106121480464935, + "theoretical_loss": 3.3199365495981517, + "tokens_seen": 3009740800 + }, + { + "epoch": 0.91, + "learning_rate": 4.453360080240722e-05, + "loss": 1.2493, + "theoretical_loss": 3.319930995169784, + "tokens_seen": 3009806336 + }, + { + "epoch": 0.91, + "learning_rate": 4.45135406218656e-05, + "loss": 1.2119, + "theoretical_loss": 3.319919886777456, + "tokens_seen": 3009937408 + }, + { + "epoch": 0.91, + "learning_rate": 4.4493480441323975e-05, + "loss": 1.3654, + "theoretical_loss": 3.319908779004286, + "tokens_seen": 3010068480 + }, + { + "epoch": 0.91, + "learning_rate": 4.447342026078235e-05, + "loss": 1.3938, + "theoretical_loss": 3.3198976718502125, + "tokens_seen": 3010199552 + }, + { + "epoch": 0.91, + "learning_rate": 4.445336008024072e-05, + "loss": 1.4727, + "theoretical_loss": 3.319886565315174, + "tokens_seen": 3010330624 + }, + { + "epoch": 0.91, + "learning_rate": 4.4433299899699095e-05, + "loss": 1.4114, + "theoretical_loss": 3.3198754593991096, + "tokens_seen": 3010461696 + }, + { + "epoch": 0.91, + "learning_rate": 4.441323971915747e-05, + "loss": 1.2932, + "theoretical_loss": 3.319864354101957, + "tokens_seen": 3010592768 + }, + { + "epoch": 0.91, + "learning_rate": 4.439317953861585e-05, + "loss": 1.3818, + "theoretical_loss": 3.3198532494236552, + "tokens_seen": 3010723840 + }, + { + "epoch": 0.91, + "learning_rate": 4.437311935807423e-05, + "loss": 1.3116, + "theoretical_loss": 3.3198421453641433, + "tokens_seen": 3010854912 + }, + { + "epoch": 0.91, + "learning_rate": 4.43530591775326e-05, + "loss": 1.2413, + "theoretical_loss": 3.3198310419233588, + "tokens_seen": 3010985984 + }, + { + "epoch": 0.91, + "learning_rate": 4.4332998996990974e-05, + "loss": 1.2754, + "theoretical_loss": 3.319819939101241, + "tokens_seen": 3011117056 + }, + { + "epoch": 0.91, + "learning_rate": 4.431293881644935e-05, + "loss": 1.3646, + "theoretical_loss": 3.319808836897729, + "tokens_seen": 3011248128 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.45277369022369385, + "objective/train/avg_token_score": 0.005208101123571396, + "objective/train/docs_used": 1694138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.380017638206482, + "objective/train/original_loss": 3.4633755683898926, + "objective/train/theoretical_loss": 3.31979773531276, + "objective/train/tokens_used": 3011379200, + "objective/train/unlikelihood_loss": 0.19366368651390076, + "theoretical_loss": 3.31979773531276, + "tokens_seen": 3011379200 + }, + { + "epoch": 0.91, + "learning_rate": 4.4292878635907726e-05, + "loss": 1.0467, + "theoretical_loss": 3.31979773531276, + "tokens_seen": 3011379200 + }, + { + "epoch": 0.91, + "learning_rate": 4.42728184553661e-05, + "loss": 1.2869, + "theoretical_loss": 3.319786634346274, + "tokens_seen": 3011510272 + }, + { + "epoch": 0.91, + "learning_rate": 4.425275827482447e-05, + "loss": 1.2477, + "theoretical_loss": 3.3197755339982082, + "tokens_seen": 3011641344 + }, + { + "epoch": 0.91, + "learning_rate": 4.4232698094282846e-05, + "loss": 1.3212, + "theoretical_loss": 3.3197644342685027, + "tokens_seen": 3011772416 + }, + { + "epoch": 0.91, + "learning_rate": 4.421263791374122e-05, + "loss": 1.3163, + "theoretical_loss": 3.3197533351570954, + "tokens_seen": 3011903488 + }, + { + "epoch": 0.91, + "learning_rate": 4.4192577733199605e-05, + "loss": 1.4822, + "theoretical_loss": 3.3197422366639247, + "tokens_seen": 3012034560 + }, + { + "epoch": 0.91, + "learning_rate": 4.417251755265798e-05, + "loss": 1.4753, + "theoretical_loss": 3.31973113878893, + "tokens_seen": 3012165632 + }, + { + "epoch": 0.91, + "learning_rate": 4.415245737211635e-05, + "loss": 1.3358, + "theoretical_loss": 3.3197200415320496, + "tokens_seen": 3012296704 + }, + { + "epoch": 0.91, + "learning_rate": 4.4132397191574725e-05, + "loss": 1.4355, + "theoretical_loss": 3.319708944893222, + "tokens_seen": 3012427776 + }, + { + "epoch": 0.91, + "learning_rate": 4.41123370110331e-05, + "loss": 1.3893, + "theoretical_loss": 3.3196978488723863, + "tokens_seen": 3012558848 + }, + { + "epoch": 0.91, + "learning_rate": 4.409227683049148e-05, + "loss": 1.2609, + "theoretical_loss": 3.3196867534694805, + "tokens_seen": 3012689920 + }, + { + "epoch": 0.91, + "learning_rate": 4.4072216649949846e-05, + "loss": 1.3234, + "theoretical_loss": 3.319675658684444, + "tokens_seen": 3012820992 + }, + { + "epoch": 0.91, + "learning_rate": 4.405215646940822e-05, + "loss": 1.258, + "theoretical_loss": 3.3196645645172156, + "tokens_seen": 3012952064 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.34640762209892273, + "objective/train/avg_token_score": 0.01617373153567314, + "objective/train/docs_used": 1695115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0799931287765503, + "objective/train/original_loss": 3.5459847450256348, + "objective/train/theoretical_loss": 3.3196590176652596, + "objective/train/tokens_used": 3013017600, + "objective/train/unlikelihood_loss": 0.26968008279800415, + "theoretical_loss": 3.3196590176652596, + "tokens_seen": 3013017600 + }, + { + "epoch": 0.91, + "learning_rate": 4.40320962888666e-05, + "loss": 1.3144, + "theoretical_loss": 3.319653470967733, + "tokens_seen": 3013083136 + }, + { + "epoch": 0.91, + "learning_rate": 4.401203610832498e-05, + "loss": 1.3654, + "theoretical_loss": 3.319642378035936, + "tokens_seen": 3013214208 + }, + { + "epoch": 0.91, + "learning_rate": 4.3991975927783356e-05, + "loss": 1.2116, + "theoretical_loss": 3.319631285721763, + "tokens_seen": 3013345280 + }, + { + "epoch": 0.91, + "learning_rate": 4.3971915747241725e-05, + "loss": 1.1573, + "theoretical_loss": 3.319620194025153, + "tokens_seen": 3013476352 + }, + { + "epoch": 0.91, + "learning_rate": 4.39518555667001e-05, + "loss": 1.2772, + "theoretical_loss": 3.319609102946044, + "tokens_seen": 3013607424 + }, + { + "epoch": 0.91, + "learning_rate": 4.3931795386158477e-05, + "loss": 1.2749, + "theoretical_loss": 3.3195980124843754, + "tokens_seen": 3013738496 + }, + { + "epoch": 0.91, + "learning_rate": 4.391173520561685e-05, + "loss": 1.3005, + "theoretical_loss": 3.319586922640086, + "tokens_seen": 3013869568 + }, + { + "epoch": 0.91, + "learning_rate": 4.389167502507523e-05, + "loss": 1.2417, + "theoretical_loss": 3.319575833413114, + "tokens_seen": 3014000640 + }, + { + "epoch": 0.91, + "learning_rate": 4.38716148445336e-05, + "loss": 1.2781, + "theoretical_loss": 3.319564744803399, + "tokens_seen": 3014131712 + }, + { + "epoch": 0.91, + "learning_rate": 4.385155466399197e-05, + "loss": 1.2011, + "theoretical_loss": 3.319553656810879, + "tokens_seen": 3014262784 + }, + { + "epoch": 0.91, + "learning_rate": 4.383149448345035e-05, + "loss": 1.338, + "theoretical_loss": 3.3195425694354936, + "tokens_seen": 3014393856 + }, + { + "epoch": 0.91, + "learning_rate": 4.381143430290873e-05, + "loss": 1.2644, + "theoretical_loss": 3.319531482677181, + "tokens_seen": 3014524928 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.5208944082260132, + "objective/train/avg_token_score": 0.007224384695291519, + "objective/train/docs_used": 1695676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.64421808719635, + "objective/train/original_loss": 3.711834669113159, + "objective/train/theoretical_loss": 3.3195203965358804, + "objective/train/tokens_used": 3014656000, + "objective/train/unlikelihood_loss": 0.14898541569709778, + "theoretical_loss": 3.3195203965358804, + "tokens_seen": 3014656000 + }, + { + "epoch": 0.91, + "learning_rate": 4.379137412236711e-05, + "loss": 1.308, + "theoretical_loss": 3.3195203965358804, + "tokens_seen": 3014656000 + }, + { + "epoch": 0.91, + "learning_rate": 4.3771313941825476e-05, + "loss": 1.2414, + "theoretical_loss": 3.3195093110115304, + "tokens_seen": 3014787072 + }, + { + "epoch": 0.91, + "learning_rate": 4.375125376128385e-05, + "loss": 1.2969, + "theoretical_loss": 3.31949822610407, + "tokens_seen": 3014918144 + }, + { + "epoch": 0.91, + "learning_rate": 4.373119358074223e-05, + "loss": 1.3693, + "theoretical_loss": 3.319487141813438, + "tokens_seen": 3015049216 + }, + { + "epoch": 0.91, + "learning_rate": 4.3711133400200604e-05, + "loss": 1.2078, + "theoretical_loss": 3.319476058139573, + "tokens_seen": 3015180288 + }, + { + "epoch": 0.91, + "learning_rate": 4.369107321965898e-05, + "loss": 1.1564, + "theoretical_loss": 3.319464975082415, + "tokens_seen": 3015311360 + }, + { + "epoch": 0.91, + "learning_rate": 4.367101303911735e-05, + "loss": 1.2788, + "theoretical_loss": 3.319453892641901, + "tokens_seen": 3015442432 + }, + { + "epoch": 0.91, + "learning_rate": 4.3650952858575724e-05, + "loss": 1.2031, + "theoretical_loss": 3.3194428108179714, + "tokens_seen": 3015573504 + }, + { + "epoch": 0.91, + "learning_rate": 4.363089267803411e-05, + "loss": 1.2386, + "theoretical_loss": 3.319431729610564, + "tokens_seen": 3015704576 + }, + { + "epoch": 0.91, + "learning_rate": 4.361083249749248e-05, + "loss": 1.1432, + "theoretical_loss": 3.319420649019619, + "tokens_seen": 3015835648 + }, + { + "epoch": 0.91, + "learning_rate": 4.359077231695086e-05, + "loss": 1.2746, + "theoretical_loss": 3.3194095690450744, + "tokens_seen": 3015966720 + }, + { + "epoch": 0.91, + "learning_rate": 4.357071213640923e-05, + "loss": 1.2189, + "theoretical_loss": 3.3193984896868693, + "tokens_seen": 3016097792 + }, + { + "epoch": 0.91, + "learning_rate": 4.35506519558676e-05, + "loss": 1.2009, + "theoretical_loss": 3.319387410944943, + "tokens_seen": 3016228864 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.578262448310852, + "objective/train/avg_token_score": 0.007034613285213709, + "objective/train/docs_used": 1696229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.835858941078186, + "objective/train/original_loss": 3.713496208190918, + "objective/train/theoretical_loss": 3.319381871805065, + "objective/train/tokens_used": 3016294400, + "objective/train/unlikelihood_loss": 0.09843910485506058, + "theoretical_loss": 3.319381871805065, + "tokens_seen": 3016294400 + }, + { + "epoch": 0.91, + "learning_rate": 4.353059177532598e-05, + "loss": 1.5123, + "theoretical_loss": 3.3193763328192336, + "tokens_seen": 3016359936 + }, + { + "epoch": 0.91, + "learning_rate": 4.3510531594784355e-05, + "loss": 1.2937, + "theoretical_loss": 3.319365255309681, + "tokens_seen": 3016491008 + }, + { + "epoch": 0.91, + "learning_rate": 4.349047141424273e-05, + "loss": 1.1043, + "theoretical_loss": 3.3193541784162237, + "tokens_seen": 3016622080 + }, + { + "epoch": 0.91, + "learning_rate": 4.34704112337011e-05, + "loss": 1.4574, + "theoretical_loss": 3.3193431021388005, + "tokens_seen": 3016753152 + }, + { + "epoch": 0.91, + "learning_rate": 4.3450351053159475e-05, + "loss": 1.2603, + "theoretical_loss": 3.3193320264773507, + "tokens_seen": 3016884224 + }, + { + "epoch": 0.91, + "learning_rate": 4.343029087261786e-05, + "loss": 1.2509, + "theoretical_loss": 3.3193209514318127, + "tokens_seen": 3017015296 + }, + { + "epoch": 0.91, + "learning_rate": 4.3410230692076234e-05, + "loss": 1.0934, + "theoretical_loss": 3.3193098770021265, + "tokens_seen": 3017146368 + }, + { + "epoch": 0.91, + "learning_rate": 4.33901705115346e-05, + "loss": 1.2556, + "theoretical_loss": 3.3192988031882305, + "tokens_seen": 3017277440 + }, + { + "epoch": 0.91, + "learning_rate": 4.337011033099298e-05, + "loss": 1.3069, + "theoretical_loss": 3.3192877299900636, + "tokens_seen": 3017408512 + }, + { + "epoch": 0.91, + "learning_rate": 4.3350050150451354e-05, + "loss": 1.109, + "theoretical_loss": 3.319276657407565, + "tokens_seen": 3017539584 + }, + { + "epoch": 0.91, + "learning_rate": 4.332998996990973e-05, + "loss": 1.2384, + "theoretical_loss": 3.319265585440674, + "tokens_seen": 3017670656 + }, + { + "epoch": 0.91, + "learning_rate": 4.3309929789368106e-05, + "loss": 1.3143, + "theoretical_loss": 3.3192545140893293, + "tokens_seen": 3017801728 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.4495967626571655, + "objective/train/avg_token_score": 0.025327319279313087, + "objective/train/docs_used": 1697654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2178560495376587, + "objective/train/original_loss": 3.5776970386505127, + "objective/train/theoretical_loss": 3.3192434433534697, + "objective/train/tokens_used": 3017932800, + "objective/train/unlikelihood_loss": 0.12565822899341583, + "theoretical_loss": 3.3192434433534697, + "tokens_seen": 3017932800 + }, + { + "epoch": 0.91, + "learning_rate": 4.3289869608826475e-05, + "loss": 1.4869, + "theoretical_loss": 3.3192434433534697, + "tokens_seen": 3017932800 + }, + { + "epoch": 0.91, + "learning_rate": 4.326980942828485e-05, + "loss": 1.2316, + "theoretical_loss": 3.3192323732330347, + "tokens_seen": 3018063872 + }, + { + "epoch": 0.91, + "learning_rate": 4.324974924774323e-05, + "loss": 1.4426, + "theoretical_loss": 3.319221303727963, + "tokens_seen": 3018194944 + }, + { + "epoch": 0.91, + "learning_rate": 4.322968906720161e-05, + "loss": 1.1345, + "theoretical_loss": 3.319210234838194, + "tokens_seen": 3018326016 + }, + { + "epoch": 0.91, + "learning_rate": 4.3209628886659985e-05, + "loss": 1.2046, + "theoretical_loss": 3.3191991665636666, + "tokens_seen": 3018457088 + }, + { + "epoch": 0.91, + "learning_rate": 4.3189568706118354e-05, + "loss": 1.2003, + "theoretical_loss": 3.3191880989043203, + "tokens_seen": 3018588160 + }, + { + "epoch": 0.91, + "learning_rate": 4.316950852557673e-05, + "loss": 1.1589, + "theoretical_loss": 3.319177031860094, + "tokens_seen": 3018719232 + }, + { + "epoch": 0.91, + "learning_rate": 4.3149448345035105e-05, + "loss": 1.1924, + "theoretical_loss": 3.3191659654309262, + "tokens_seen": 3018850304 + }, + { + "epoch": 0.91, + "learning_rate": 4.312938816449348e-05, + "loss": 1.2621, + "theoretical_loss": 3.3191548996167564, + "tokens_seen": 3018981376 + }, + { + "epoch": 0.91, + "learning_rate": 4.310932798395186e-05, + "loss": 1.1947, + "theoretical_loss": 3.319143834417524, + "tokens_seen": 3019112448 + }, + { + "epoch": 0.91, + "learning_rate": 4.3089267803410226e-05, + "loss": 1.2815, + "theoretical_loss": 3.319132769833168, + "tokens_seen": 3019243520 + }, + { + "epoch": 0.91, + "learning_rate": 4.306920762286861e-05, + "loss": 1.2956, + "theoretical_loss": 3.319121705863627, + "tokens_seen": 3019374592 + }, + { + "epoch": 0.92, + "learning_rate": 4.3049147442326984e-05, + "loss": 1.2208, + "theoretical_loss": 3.3191106425088415, + "tokens_seen": 3019505664 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.578262448310852, + "objective/train/avg_token_score": 0.004687321372330189, + "objective/train/docs_used": 1698322, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8135920763015747, + "objective/train/original_loss": 3.4685981273651123, + "objective/train/theoretical_loss": 3.3191051110619627, + "objective/train/tokens_used": 3019571200, + "objective/train/unlikelihood_loss": 0.16682766377925873, + "theoretical_loss": 3.3191051110619627, + "tokens_seen": 3019571200 + }, + { + "epoch": 0.92, + "learning_rate": 4.302908726178536e-05, + "loss": 1.3569, + "theoretical_loss": 3.319099579768749, + "tokens_seen": 3019636736 + }, + { + "epoch": 0.92, + "learning_rate": 4.3009027081243736e-05, + "loss": 1.3122, + "theoretical_loss": 3.31908851764329, + "tokens_seen": 3019767808 + }, + { + "epoch": 0.92, + "learning_rate": 4.2988966900702105e-05, + "loss": 1.2981, + "theoretical_loss": 3.319077456132403, + "tokens_seen": 3019898880 + }, + { + "epoch": 0.92, + "learning_rate": 4.296890672016048e-05, + "loss": 1.1636, + "theoretical_loss": 3.3190663952360273, + "tokens_seen": 3020029952 + }, + { + "epoch": 0.92, + "learning_rate": 4.2948846539618857e-05, + "loss": 1.2522, + "theoretical_loss": 3.319055334954102, + "tokens_seen": 3020161024 + }, + { + "epoch": 0.92, + "learning_rate": 4.292878635907723e-05, + "loss": 1.2371, + "theoretical_loss": 3.3190442752865663, + "tokens_seen": 3020292096 + }, + { + "epoch": 0.92, + "learning_rate": 4.290872617853561e-05, + "loss": 1.2204, + "theoretical_loss": 3.31903321623336, + "tokens_seen": 3020423168 + }, + { + "epoch": 0.92, + "learning_rate": 4.288866599799398e-05, + "loss": 1.3054, + "theoretical_loss": 3.319022157794421, + "tokens_seen": 3020554240 + }, + { + "epoch": 0.92, + "learning_rate": 4.286860581745236e-05, + "loss": 1.2044, + "theoretical_loss": 3.31901109996969, + "tokens_seen": 3020685312 + }, + { + "epoch": 0.92, + "learning_rate": 4.2848545636910736e-05, + "loss": 1.2825, + "theoretical_loss": 3.3190000427591055, + "tokens_seen": 3020816384 + }, + { + "epoch": 0.92, + "learning_rate": 4.282848545636911e-05, + "loss": 1.4688, + "theoretical_loss": 3.3189889861626067, + "tokens_seen": 3020947456 + }, + { + "epoch": 0.92, + "learning_rate": 4.280842527582749e-05, + "loss": 1.2005, + "theoretical_loss": 3.318977930180133, + "tokens_seen": 3021078528 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.48429861664772034, + "objective/train/avg_token_score": 0.01579097844660282, + "objective/train/docs_used": 1699637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3114441633224487, + "objective/train/original_loss": 3.2685775756835938, + "objective/train/theoretical_loss": 3.318966874811624, + "objective/train/tokens_used": 3021209600, + "objective/train/unlikelihood_loss": 0.17797166109085083, + "theoretical_loss": 3.318966874811624, + "tokens_seen": 3021209600 + }, + { + "epoch": 0.92, + "learning_rate": 4.2788365095285856e-05, + "loss": 1.2795, + "theoretical_loss": 3.318966874811624, + "tokens_seen": 3021209600 + }, + { + "epoch": 0.92, + "learning_rate": 4.276830491474423e-05, + "loss": 1.2869, + "theoretical_loss": 3.3189558200570186, + "tokens_seen": 3021340672 + }, + { + "epoch": 0.92, + "learning_rate": 4.274824473420261e-05, + "loss": 1.3611, + "theoretical_loss": 3.318944765916256, + "tokens_seen": 3021471744 + }, + { + "epoch": 0.92, + "learning_rate": 4.2728184553660983e-05, + "loss": 1.1737, + "theoretical_loss": 3.318933712389275, + "tokens_seen": 3021602816 + }, + { + "epoch": 0.92, + "learning_rate": 4.270812437311935e-05, + "loss": 1.1176, + "theoretical_loss": 3.318922659476016, + "tokens_seen": 3021733888 + }, + { + "epoch": 0.92, + "learning_rate": 4.2688064192577735e-05, + "loss": 1.3038, + "theoretical_loss": 3.3189116071764184, + "tokens_seen": 3021864960 + }, + { + "epoch": 0.92, + "learning_rate": 4.266800401203611e-05, + "loss": 1.2627, + "theoretical_loss": 3.31890055549042, + "tokens_seen": 3021996032 + }, + { + "epoch": 0.92, + "learning_rate": 4.264794383149449e-05, + "loss": 1.2324, + "theoretical_loss": 3.3188895044179616, + "tokens_seen": 3022127104 + }, + { + "epoch": 0.92, + "learning_rate": 4.262788365095286e-05, + "loss": 1.1403, + "theoretical_loss": 3.3188784539589817, + "tokens_seen": 3022258176 + }, + { + "epoch": 0.92, + "learning_rate": 4.260782347041123e-05, + "loss": 1.2849, + "theoretical_loss": 3.31886740411342, + "tokens_seen": 3022389248 + }, + { + "epoch": 0.92, + "learning_rate": 4.258776328986961e-05, + "loss": 1.3636, + "theoretical_loss": 3.3188563548812158, + "tokens_seen": 3022520320 + }, + { + "epoch": 0.92, + "learning_rate": 4.256770310932798e-05, + "loss": 1.356, + "theoretical_loss": 3.3188453062623084, + "tokens_seen": 3022651392 + }, + { + "epoch": 0.92, + "learning_rate": 4.254764292878636e-05, + "loss": 1.1181, + "theoretical_loss": 3.3188342582566372, + "tokens_seen": 3022782464 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.31836509704589844, + "objective/train/avg_token_score": 0.028091847896575928, + "objective/train/docs_used": 1700711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9431114196777344, + "objective/train/original_loss": 4.2416300773620605, + "objective/train/theoretical_loss": 3.318828734483746, + "objective/train/tokens_used": 3022848000, + "objective/train/unlikelihood_loss": 0.1561449021100998, + "theoretical_loss": 3.318828734483746, + "tokens_seen": 3022848000 + }, + { + "epoch": 0.92, + "learning_rate": 4.2527582748244735e-05, + "loss": 1.2413, + "theoretical_loss": 3.3188232108641413, + "tokens_seen": 3022913536 + }, + { + "epoch": 0.92, + "learning_rate": 4.250752256770311e-05, + "loss": 1.3727, + "theoretical_loss": 3.3188121640847608, + "tokens_seen": 3023044608 + }, + { + "epoch": 0.92, + "learning_rate": 4.2487462387161486e-05, + "loss": 1.1811, + "theoretical_loss": 3.3188011179184342, + "tokens_seen": 3023175680 + }, + { + "epoch": 0.92, + "learning_rate": 4.246740220661986e-05, + "loss": 1.2913, + "theoretical_loss": 3.3187900723651014, + "tokens_seen": 3023306752 + }, + { + "epoch": 0.92, + "learning_rate": 4.244734202607824e-05, + "loss": 1.1839, + "theoretical_loss": 3.318779027424702, + "tokens_seen": 3023437824 + }, + { + "epoch": 0.92, + "learning_rate": 4.2427281845536614e-05, + "loss": 1.3362, + "theoretical_loss": 3.318767983097175, + "tokens_seen": 3023568896 + }, + { + "epoch": 0.92, + "learning_rate": 4.240722166499498e-05, + "loss": 1.3762, + "theoretical_loss": 3.3187569393824603, + "tokens_seen": 3023699968 + }, + { + "epoch": 0.92, + "learning_rate": 4.238716148445336e-05, + "loss": 1.212, + "theoretical_loss": 3.3187458962804968, + "tokens_seen": 3023831040 + }, + { + "epoch": 0.92, + "learning_rate": 4.2367101303911734e-05, + "loss": 1.3003, + "theoretical_loss": 3.318734853791224, + "tokens_seen": 3023962112 + }, + { + "epoch": 0.92, + "learning_rate": 4.234704112337011e-05, + "loss": 1.3408, + "theoretical_loss": 3.318723811914582, + "tokens_seen": 3024093184 + }, + { + "epoch": 0.92, + "learning_rate": 4.2326980942828486e-05, + "loss": 1.2727, + "theoretical_loss": 3.3187127706505093, + "tokens_seen": 3024224256 + }, + { + "epoch": 0.92, + "learning_rate": 4.230692076228686e-05, + "loss": 1.298, + "theoretical_loss": 3.318701729998946, + "tokens_seen": 3024355328 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.24883919954299927, + "objective/train/avg_token_score": 0.010823563672602177, + "objective/train/docs_used": 1701440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7527838349342346, + "objective/train/original_loss": 3.688366413116455, + "objective/train/theoretical_loss": 3.3186906899598316, + "objective/train/tokens_used": 3024486400, + "objective/train/unlikelihood_loss": 0.1851574182510376, + "theoretical_loss": 3.3186906899598316, + "tokens_seen": 3024486400 + }, + { + "epoch": 0.92, + "learning_rate": 4.228686058174524e-05, + "loss": 1.1982, + "theoretical_loss": 3.3186906899598316, + "tokens_seen": 3024486400 + }, + { + "epoch": 0.92, + "learning_rate": 4.226680040120361e-05, + "loss": 1.337, + "theoretical_loss": 3.318679650533105, + "tokens_seen": 3024617472 + }, + { + "epoch": 0.92, + "learning_rate": 4.224674022066199e-05, + "loss": 1.2621, + "theoretical_loss": 3.3186686117187065, + "tokens_seen": 3024748544 + }, + { + "epoch": 0.92, + "learning_rate": 4.2226680040120365e-05, + "loss": 1.5247, + "theoretical_loss": 3.318657573516575, + "tokens_seen": 3024879616 + }, + { + "epoch": 0.92, + "learning_rate": 4.2206619859578734e-05, + "loss": 1.2228, + "theoretical_loss": 3.3186465359266504, + "tokens_seen": 3025010688 + }, + { + "epoch": 0.92, + "learning_rate": 4.218655967903711e-05, + "loss": 1.3244, + "theoretical_loss": 3.318635498948872, + "tokens_seen": 3025141760 + }, + { + "epoch": 0.92, + "learning_rate": 4.2166499498495485e-05, + "loss": 1.253, + "theoretical_loss": 3.3186244625831796, + "tokens_seen": 3025272832 + }, + { + "epoch": 0.92, + "learning_rate": 4.214643931795386e-05, + "loss": 1.167, + "theoretical_loss": 3.318613426829512, + "tokens_seen": 3025403904 + }, + { + "epoch": 0.92, + "learning_rate": 4.2126379137412244e-05, + "loss": 1.2832, + "theoretical_loss": 3.3186023916878096, + "tokens_seen": 3025534976 + }, + { + "epoch": 0.92, + "learning_rate": 4.210631895687061e-05, + "loss": 1.2701, + "theoretical_loss": 3.3185913571580117, + "tokens_seen": 3025666048 + }, + { + "epoch": 0.92, + "learning_rate": 4.208625877632899e-05, + "loss": 1.1682, + "theoretical_loss": 3.3185803232400577, + "tokens_seen": 3025797120 + }, + { + "epoch": 0.92, + "learning_rate": 4.2066198595787364e-05, + "loss": 1.3218, + "theoretical_loss": 3.318569289933887, + "tokens_seen": 3025928192 + }, + { + "epoch": 0.92, + "learning_rate": 4.204613841524574e-05, + "loss": 1.1995, + "theoretical_loss": 3.31855825723944, + "tokens_seen": 3026059264 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.4176441729068756, + "objective/train/avg_token_score": 0.02727578580379486, + "objective/train/docs_used": 1702127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3533005714416504, + "objective/train/original_loss": 4.183465957641602, + "objective/train/theoretical_loss": 3.3185527411215934, + "objective/train/tokens_used": 3026124800, + "objective/train/unlikelihood_loss": 0.11821340024471283, + "theoretical_loss": 3.3185527411215934, + "tokens_seen": 3026124800 + }, + { + "epoch": 0.92, + "learning_rate": 4.2026078234704116e-05, + "loss": 1.2969, + "theoretical_loss": 3.3185472251566552, + "tokens_seen": 3026190336 + }, + { + "epoch": 0.92, + "learning_rate": 4.2006018054162485e-05, + "loss": 1.2674, + "theoretical_loss": 3.318536193685473, + "tokens_seen": 3026321408 + }, + { + "epoch": 0.92, + "learning_rate": 4.198595787362086e-05, + "loss": 1.0905, + "theoretical_loss": 3.3185251628258325, + "tokens_seen": 3026452480 + }, + { + "epoch": 0.92, + "learning_rate": 4.1965897693079237e-05, + "loss": 1.1408, + "theoretical_loss": 3.3185141325776737, + "tokens_seen": 3026583552 + }, + { + "epoch": 0.92, + "learning_rate": 4.194583751253762e-05, + "loss": 1.1326, + "theoretical_loss": 3.318503102940936, + "tokens_seen": 3026714624 + }, + { + "epoch": 0.92, + "learning_rate": 4.192577733199599e-05, + "loss": 1.199, + "theoretical_loss": 3.318492073915559, + "tokens_seen": 3026845696 + }, + { + "epoch": 0.92, + "learning_rate": 4.1905717151454364e-05, + "loss": 1.2108, + "theoretical_loss": 3.318481045501483, + "tokens_seen": 3026976768 + }, + { + "epoch": 0.92, + "learning_rate": 4.188565697091274e-05, + "loss": 1.2305, + "theoretical_loss": 3.3184700176986466, + "tokens_seen": 3027107840 + }, + { + "epoch": 0.92, + "learning_rate": 4.1865596790371115e-05, + "loss": 1.186, + "theoretical_loss": 3.31845899050699, + "tokens_seen": 3027238912 + }, + { + "epoch": 0.92, + "learning_rate": 4.184553660982949e-05, + "loss": 1.3593, + "theoretical_loss": 3.318447963926453, + "tokens_seen": 3027369984 + }, + { + "epoch": 0.92, + "learning_rate": 4.182547642928786e-05, + "loss": 1.2166, + "theoretical_loss": 3.318436937956975, + "tokens_seen": 3027501056 + }, + { + "epoch": 0.92, + "learning_rate": 4.1805416248746236e-05, + "loss": 1.2017, + "theoretical_loss": 3.318425912598496, + "tokens_seen": 3027632128 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.5563294291496277, + "objective/train/avg_token_score": 0.02134942077100277, + "objective/train/docs_used": 1703090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5810927152633667, + "objective/train/original_loss": 3.4971516132354736, + "objective/train/theoretical_loss": 3.318414887850955, + "objective/train/tokens_used": 3027763200, + "objective/train/unlikelihood_loss": 0.17099981009960175, + "theoretical_loss": 3.318414887850955, + "tokens_seen": 3027763200 + }, + { + "epoch": 0.92, + "learning_rate": 4.178535606820461e-05, + "loss": 1.2436, + "theoretical_loss": 3.318414887850955, + "tokens_seen": 3027763200 + }, + { + "epoch": 0.92, + "learning_rate": 4.176529588766299e-05, + "loss": 1.2274, + "theoretical_loss": 3.318403863714292, + "tokens_seen": 3027894272 + }, + { + "epoch": 0.92, + "learning_rate": 4.174523570712137e-05, + "loss": 1.3524, + "theoretical_loss": 3.318392840188448, + "tokens_seen": 3028025344 + }, + { + "epoch": 0.92, + "learning_rate": 4.172517552657974e-05, + "loss": 1.3063, + "theoretical_loss": 3.3183818172733606, + "tokens_seen": 3028156416 + }, + { + "epoch": 0.92, + "learning_rate": 4.1705115346038115e-05, + "loss": 1.1563, + "theoretical_loss": 3.318370794968971, + "tokens_seen": 3028287488 + }, + { + "epoch": 0.92, + "learning_rate": 4.168505516549649e-05, + "loss": 1.3194, + "theoretical_loss": 3.318359773275218, + "tokens_seen": 3028418560 + }, + { + "epoch": 0.92, + "learning_rate": 4.166499498495487e-05, + "loss": 1.2088, + "theoretical_loss": 3.3183487521920423, + "tokens_seen": 3028549632 + }, + { + "epoch": 0.92, + "learning_rate": 4.164493480441324e-05, + "loss": 1.3817, + "theoretical_loss": 3.318337731719383, + "tokens_seen": 3028680704 + }, + { + "epoch": 0.92, + "learning_rate": 4.162487462387161e-05, + "loss": 1.1937, + "theoretical_loss": 3.31832671185718, + "tokens_seen": 3028811776 + }, + { + "epoch": 0.92, + "learning_rate": 4.160481444332999e-05, + "loss": 1.3079, + "theoretical_loss": 3.318315692605373, + "tokens_seen": 3028942848 + }, + { + "epoch": 0.92, + "learning_rate": 4.158475426278836e-05, + "loss": 1.2667, + "theoretical_loss": 3.3183046739639024, + "tokens_seen": 3029073920 + }, + { + "epoch": 0.92, + "learning_rate": 4.1564694082246746e-05, + "loss": 1.1591, + "theoretical_loss": 3.318293655932707, + "tokens_seen": 3029204992 + }, + { + "epoch": 0.92, + "learning_rate": 4.154463390170512e-05, + "loss": 1.3443, + "theoretical_loss": 3.318282638511727, + "tokens_seen": 3029336064 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.43468964099884033, + "objective/train/avg_token_score": 0.013460177928209305, + "objective/train/docs_used": 1703839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3280104398727417, + "objective/train/original_loss": 3.638216018676758, + "objective/train/theoretical_loss": 3.318277130030049, + "objective/train/tokens_used": 3029401600, + "objective/train/unlikelihood_loss": 0.1503523737192154, + "theoretical_loss": 3.318277130030049, + "tokens_seen": 3029401600 + }, + { + "epoch": 0.92, + "learning_rate": 4.152457372116349e-05, + "loss": 1.2574, + "theoretical_loss": 3.3182716217009025, + "tokens_seen": 3029467136 + }, + { + "epoch": 0.92, + "learning_rate": 4.1504513540621866e-05, + "loss": 1.4426, + "theoretical_loss": 3.318260605500173, + "tokens_seen": 3029598208 + }, + { + "epoch": 0.92, + "learning_rate": 4.148445336008024e-05, + "loss": 1.1073, + "theoretical_loss": 3.3182495899094784, + "tokens_seen": 3029729280 + }, + { + "epoch": 0.92, + "learning_rate": 4.146439317953862e-05, + "loss": 1.2889, + "theoretical_loss": 3.3182385749287584, + "tokens_seen": 3029860352 + }, + { + "epoch": 0.92, + "learning_rate": 4.1444332998996994e-05, + "loss": 1.4063, + "theoretical_loss": 3.318227560557953, + "tokens_seen": 3029991424 + }, + { + "epoch": 0.92, + "learning_rate": 4.142427281845536e-05, + "loss": 1.3177, + "theoretical_loss": 3.318216546797002, + "tokens_seen": 3030122496 + }, + { + "epoch": 0.92, + "learning_rate": 4.140421263791374e-05, + "loss": 1.1995, + "theoretical_loss": 3.318205533645845, + "tokens_seen": 3030253568 + }, + { + "epoch": 0.92, + "learning_rate": 4.138415245737212e-05, + "loss": 1.2413, + "theoretical_loss": 3.318194521104423, + "tokens_seen": 3030384640 + }, + { + "epoch": 0.92, + "learning_rate": 4.13640922768305e-05, + "loss": 1.181, + "theoretical_loss": 3.318183509172674, + "tokens_seen": 3030515712 + }, + { + "epoch": 0.92, + "learning_rate": 4.134403209628887e-05, + "loss": 1.2852, + "theoretical_loss": 3.318172497850539, + "tokens_seen": 3030646784 + }, + { + "epoch": 0.92, + "learning_rate": 4.132397191574724e-05, + "loss": 1.2642, + "theoretical_loss": 3.3181614871379583, + "tokens_seen": 3030777856 + }, + { + "epoch": 0.92, + "learning_rate": 4.130391173520562e-05, + "loss": 1.2127, + "theoretical_loss": 3.3181504770348713, + "tokens_seen": 3030908928 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.38831865787506104, + "objective/train/avg_token_score": 0.007643938064575195, + "objective/train/docs_used": 1705064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.092982530593872, + "objective/train/original_loss": 3.721627712249756, + "objective/train/theoretical_loss": 3.3181394675412177, + "objective/train/tokens_used": 3031040000, + "objective/train/unlikelihood_loss": 0.1595873385667801, + "theoretical_loss": 3.3181394675412177, + "tokens_seen": 3031040000 + }, + { + "epoch": 0.92, + "learning_rate": 4.128385155466399e-05, + "loss": 1.2505, + "theoretical_loss": 3.3181394675412177, + "tokens_seen": 3031040000 + }, + { + "epoch": 0.92, + "learning_rate": 4.126379137412237e-05, + "loss": 1.0361, + "theoretical_loss": 3.3181284586569375, + "tokens_seen": 3031171072 + }, + { + "epoch": 0.92, + "learning_rate": 4.124373119358074e-05, + "loss": 1.375, + "theoretical_loss": 3.3181174503819704, + "tokens_seen": 3031302144 + }, + { + "epoch": 0.92, + "learning_rate": 4.1223671013039114e-05, + "loss": 1.0893, + "theoretical_loss": 3.3181064427162568, + "tokens_seen": 3031433216 + }, + { + "epoch": 0.92, + "learning_rate": 4.120361083249749e-05, + "loss": 1.247, + "theoretical_loss": 3.318095435659737, + "tokens_seen": 3031564288 + }, + { + "epoch": 0.92, + "learning_rate": 4.118355065195587e-05, + "loss": 1.2226, + "theoretical_loss": 3.31808442921235, + "tokens_seen": 3031695360 + }, + { + "epoch": 0.92, + "learning_rate": 4.116349047141425e-05, + "loss": 1.238, + "theoretical_loss": 3.3180734233740363, + "tokens_seen": 3031826432 + }, + { + "epoch": 0.92, + "learning_rate": 4.114343029087262e-05, + "loss": 1.1732, + "theoretical_loss": 3.3180624181447356, + "tokens_seen": 3031957504 + }, + { + "epoch": 0.92, + "learning_rate": 4.112337011033099e-05, + "loss": 1.2621, + "theoretical_loss": 3.318051413524388, + "tokens_seen": 3032088576 + }, + { + "epoch": 0.92, + "learning_rate": 4.110330992978937e-05, + "loss": 1.3336, + "theoretical_loss": 3.3180404095129337, + "tokens_seen": 3032219648 + }, + { + "epoch": 0.92, + "learning_rate": 4.1083249749247744e-05, + "loss": 1.3707, + "theoretical_loss": 3.3180294061103126, + "tokens_seen": 3032350720 + }, + { + "epoch": 0.92, + "learning_rate": 4.106318956870612e-05, + "loss": 1.342, + "theoretical_loss": 3.318018403316464, + "tokens_seen": 3032481792 + }, + { + "epoch": 0.92, + "learning_rate": 4.104312938816449e-05, + "loss": 1.3323, + "theoretical_loss": 3.318007401131329, + "tokens_seen": 3032612864 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.4298020601272583, + "objective/train/avg_token_score": 0.009627298451960087, + "objective/train/docs_used": 1706237, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3461424112319946, + "objective/train/original_loss": 3.9273924827575684, + "objective/train/theoretical_loss": 3.3180019002670105, + "objective/train/tokens_used": 3032678400, + "objective/train/unlikelihood_loss": 0.15977182984352112, + "theoretical_loss": 3.3180019002670105, + "tokens_seen": 3032678400 + }, + { + "epoch": 0.92, + "learning_rate": 4.1023069207622865e-05, + "loss": 1.1744, + "theoretical_loss": 3.3179963995548474, + "tokens_seen": 3032743936 + }, + { + "epoch": 0.92, + "learning_rate": 4.100300902708125e-05, + "loss": 1.3974, + "theoretical_loss": 3.3179853985869587, + "tokens_seen": 3032875008 + }, + { + "epoch": 0.92, + "learning_rate": 4.098294884653962e-05, + "loss": 1.3351, + "theoretical_loss": 3.3179743982276033, + "tokens_seen": 3033006080 + }, + { + "epoch": 0.92, + "learning_rate": 4.0962888665998e-05, + "loss": 1.3034, + "theoretical_loss": 3.3179633984767207, + "tokens_seen": 3033137152 + }, + { + "epoch": 0.92, + "learning_rate": 4.094282848545637e-05, + "loss": 1.3054, + "theoretical_loss": 3.3179523993342515, + "tokens_seen": 3033268224 + }, + { + "epoch": 0.92, + "learning_rate": 4.0922768304914744e-05, + "loss": 1.1289, + "theoretical_loss": 3.317941400800136, + "tokens_seen": 3033399296 + }, + { + "epoch": 0.92, + "learning_rate": 4.090270812437312e-05, + "loss": 1.1713, + "theoretical_loss": 3.3179304028743135, + "tokens_seen": 3033530368 + }, + { + "epoch": 0.92, + "learning_rate": 4.0882647943831495e-05, + "loss": 1.2664, + "theoretical_loss": 3.317919405556725, + "tokens_seen": 3033661440 + }, + { + "epoch": 0.92, + "learning_rate": 4.086258776328987e-05, + "loss": 1.3385, + "theoretical_loss": 3.31790840884731, + "tokens_seen": 3033792512 + }, + { + "epoch": 0.92, + "learning_rate": 4.084252758274824e-05, + "loss": 1.3916, + "theoretical_loss": 3.317897412746008, + "tokens_seen": 3033923584 + }, + { + "epoch": 0.92, + "learning_rate": 4.0822467402206616e-05, + "loss": 1.1921, + "theoretical_loss": 3.31788641725276, + "tokens_seen": 3034054656 + }, + { + "epoch": 0.92, + "learning_rate": 4.0802407221665e-05, + "loss": 1.496, + "theoretical_loss": 3.317875422367506, + "tokens_seen": 3034185728 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.41012951731681824, + "objective/train/avg_token_score": 0.016227079555392265, + "objective/train/docs_used": 1706768, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.237316370010376, + "objective/train/original_loss": 3.816946268081665, + "objective/train/theoretical_loss": 3.3178644280901866, + "objective/train/tokens_used": 3034316800, + "objective/train/unlikelihood_loss": 0.1671862155199051, + "theoretical_loss": 3.3178644280901866, + "tokens_seen": 3034316800 + }, + { + "epoch": 0.92, + "learning_rate": 4.0782347041123374e-05, + "loss": 1.3894, + "theoretical_loss": 3.3178644280901866, + "tokens_seen": 3034316800 + }, + { + "epoch": 0.92, + "learning_rate": 4.076228686058175e-05, + "loss": 1.3333, + "theoretical_loss": 3.3178534344207407, + "tokens_seen": 3034447872 + }, + { + "epoch": 0.92, + "learning_rate": 4.074222668004012e-05, + "loss": 1.2803, + "theoretical_loss": 3.317842441359109, + "tokens_seen": 3034578944 + }, + { + "epoch": 0.92, + "learning_rate": 4.0722166499498495e-05, + "loss": 1.3153, + "theoretical_loss": 3.3178314489052316, + "tokens_seen": 3034710016 + }, + { + "epoch": 0.92, + "learning_rate": 4.070210631895687e-05, + "loss": 1.2112, + "theoretical_loss": 3.317820457059049, + "tokens_seen": 3034841088 + }, + { + "epoch": 0.92, + "learning_rate": 4.068204613841525e-05, + "loss": 1.2957, + "theoretical_loss": 3.3178094658205013, + "tokens_seen": 3034972160 + }, + { + "epoch": 0.92, + "learning_rate": 4.066198595787362e-05, + "loss": 1.2782, + "theoretical_loss": 3.3177984751895284, + "tokens_seen": 3035103232 + }, + { + "epoch": 0.92, + "learning_rate": 4.064192577733199e-05, + "loss": 1.1834, + "theoretical_loss": 3.31778748516607, + "tokens_seen": 3035234304 + }, + { + "epoch": 0.92, + "learning_rate": 4.0621865596790374e-05, + "loss": 1.5669, + "theoretical_loss": 3.3177764957500675, + "tokens_seen": 3035365376 + }, + { + "epoch": 0.92, + "learning_rate": 4.060180541624875e-05, + "loss": 1.3019, + "theoretical_loss": 3.31776550694146, + "tokens_seen": 3035496448 + }, + { + "epoch": 0.92, + "learning_rate": 4.0581745235707126e-05, + "loss": 1.4185, + "theoretical_loss": 3.3177545187401885, + "tokens_seen": 3035627520 + }, + { + "epoch": 0.92, + "learning_rate": 4.05616850551655e-05, + "loss": 1.2334, + "theoretical_loss": 3.317743531146193, + "tokens_seen": 3035758592 + }, + { + "epoch": 0.92, + "learning_rate": 4.054162487462387e-05, + "loss": 1.4266, + "theoretical_loss": 3.3177325441594134, + "tokens_seen": 3035889664 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.4626099765300751, + "objective/train/avg_token_score": 0.014427712187170982, + "objective/train/docs_used": 1707993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5025385618209839, + "objective/train/original_loss": 3.8050949573516846, + "objective/train/theoretical_loss": 3.317727050893711, + "objective/train/tokens_used": 3035955200, + "objective/train/unlikelihood_loss": 0.1589708924293518, + "theoretical_loss": 3.317727050893711, + "tokens_seen": 3035955200 + }, + { + "epoch": 0.92, + "learning_rate": 4.0521564694082246e-05, + "loss": 1.3035, + "theoretical_loss": 3.31772155777979, + "tokens_seen": 3036020736 + }, + { + "epoch": 0.92, + "learning_rate": 4.050150451354062e-05, + "loss": 1.2485, + "theoretical_loss": 3.3177105720072633, + "tokens_seen": 3036151808 + }, + { + "epoch": 0.92, + "learning_rate": 4.0481444332999e-05, + "loss": 1.2258, + "theoretical_loss": 3.317699586841773, + "tokens_seen": 3036282880 + }, + { + "epoch": 0.92, + "learning_rate": 4.046138415245737e-05, + "loss": 1.2663, + "theoretical_loss": 3.31768860228326, + "tokens_seen": 3036413952 + }, + { + "epoch": 0.92, + "learning_rate": 4.044132397191575e-05, + "loss": 1.3978, + "theoretical_loss": 3.3176776183316647, + "tokens_seen": 3036545024 + }, + { + "epoch": 0.92, + "learning_rate": 4.0421263791374125e-05, + "loss": 1.3055, + "theoretical_loss": 3.3176666349869266, + "tokens_seen": 3036676096 + }, + { + "epoch": 0.92, + "learning_rate": 4.04012036108325e-05, + "loss": 1.2778, + "theoretical_loss": 3.317655652248986, + "tokens_seen": 3036807168 + }, + { + "epoch": 0.92, + "learning_rate": 4.038114343029088e-05, + "loss": 1.3429, + "theoretical_loss": 3.3176446701177844, + "tokens_seen": 3036938240 + }, + { + "epoch": 0.92, + "learning_rate": 4.0361083249749246e-05, + "loss": 1.2174, + "theoretical_loss": 3.3176336885932605, + "tokens_seen": 3037069312 + }, + { + "epoch": 0.92, + "learning_rate": 4.034102306920762e-05, + "loss": 1.3535, + "theoretical_loss": 3.3176227076753557, + "tokens_seen": 3037200384 + }, + { + "epoch": 0.92, + "learning_rate": 4.0320962888666e-05, + "loss": 1.2478, + "theoretical_loss": 3.31761172736401, + "tokens_seen": 3037331456 + }, + { + "epoch": 0.92, + "learning_rate": 4.030090270812437e-05, + "loss": 1.0511, + "theoretical_loss": 3.3176007476591636, + "tokens_seen": 3037462528 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.4258919954299927, + "objective/train/avg_token_score": 0.013159292750060558, + "objective/train/docs_used": 1708712, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2779871225357056, + "objective/train/original_loss": 3.592541456222534, + "objective/train/theoretical_loss": 3.317589768560757, + "objective/train/tokens_used": 3037593600, + "objective/train/unlikelihood_loss": 0.20064784586429596, + "theoretical_loss": 3.317589768560757, + "tokens_seen": 3037593600 + }, + { + "epoch": 0.92, + "learning_rate": 4.028084252758275e-05, + "loss": 1.3244, + "theoretical_loss": 3.317589768560757, + "tokens_seen": 3037593600 + }, + { + "epoch": 0.92, + "learning_rate": 4.026078234704112e-05, + "loss": 1.3066, + "theoretical_loss": 3.3175787900687306, + "tokens_seen": 3037724672 + }, + { + "epoch": 0.92, + "learning_rate": 4.02407221664995e-05, + "loss": 1.1735, + "theoretical_loss": 3.3175678121830243, + "tokens_seen": 3037855744 + }, + { + "epoch": 0.92, + "learning_rate": 4.0220661985957876e-05, + "loss": 1.2769, + "theoretical_loss": 3.317556834903579, + "tokens_seen": 3037986816 + }, + { + "epoch": 0.92, + "learning_rate": 4.020060180541625e-05, + "loss": 1.4049, + "theoretical_loss": 3.3175458582303348, + "tokens_seen": 3038117888 + }, + { + "epoch": 0.92, + "learning_rate": 4.018054162487463e-05, + "loss": 1.2248, + "theoretical_loss": 3.317534882163232, + "tokens_seen": 3038248960 + }, + { + "epoch": 0.92, + "learning_rate": 4.0160481444333e-05, + "loss": 1.2355, + "theoretical_loss": 3.317523906702211, + "tokens_seen": 3038380032 + }, + { + "epoch": 0.92, + "learning_rate": 4.014042126379137e-05, + "loss": 1.2248, + "theoretical_loss": 3.3175129318472125, + "tokens_seen": 3038511104 + }, + { + "epoch": 0.92, + "learning_rate": 4.012036108324975e-05, + "loss": 1.2759, + "theoretical_loss": 3.3175019575981763, + "tokens_seen": 3038642176 + }, + { + "epoch": 0.92, + "learning_rate": 4.0100300902708124e-05, + "loss": 1.3751, + "theoretical_loss": 3.3174909839550435, + "tokens_seen": 3038773248 + }, + { + "epoch": 0.92, + "learning_rate": 4.00802407221665e-05, + "loss": 1.2228, + "theoretical_loss": 3.317480010917754, + "tokens_seen": 3038904320 + }, + { + "epoch": 0.92, + "learning_rate": 4.0060180541624876e-05, + "loss": 1.2524, + "theoretical_loss": 3.3174690384862484, + "tokens_seen": 3039035392 + }, + { + "epoch": 0.92, + "learning_rate": 4.004012036108325e-05, + "loss": 1.3156, + "theoretical_loss": 3.3174580666604676, + "tokens_seen": 3039166464 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.3866690993309021, + "objective/train/avg_token_score": 0.016038043424487114, + "objective/train/docs_used": 1709345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1991397142410278, + "objective/train/original_loss": 3.834657669067383, + "objective/train/theoretical_loss": 3.317452580974705, + "objective/train/tokens_used": 3039232000, + "objective/train/unlikelihood_loss": 0.22355617582798004, + "theoretical_loss": 3.317452580974705, + "tokens_seen": 3039232000 + }, + { + "epoch": 0.92, + "learning_rate": 4.002006018054163e-05, + "loss": 1.2319, + "theoretical_loss": 3.317447095440351, + "tokens_seen": 3039297536 + }, + { + "epoch": 0.92, + "learning_rate": 4e-05, + "loss": 1.2254, + "theoretical_loss": 3.31743612482584, + "tokens_seen": 3039428608 + }, + { + "epoch": 0.92, + "learning_rate": 3.997993981945838e-05, + "loss": 1.2048, + "theoretical_loss": 3.3174251548168745, + "tokens_seen": 3039559680 + }, + { + "epoch": 0.92, + "learning_rate": 3.995987963891675e-05, + "loss": 1.1731, + "theoretical_loss": 3.317414185413395, + "tokens_seen": 3039690752 + }, + { + "epoch": 0.92, + "learning_rate": 3.9939819458375124e-05, + "loss": 1.3327, + "theoretical_loss": 3.3174032166153427, + "tokens_seen": 3039821824 + }, + { + "epoch": 0.92, + "learning_rate": 3.99197592778335e-05, + "loss": 1.1398, + "theoretical_loss": 3.317392248422657, + "tokens_seen": 3039952896 + }, + { + "epoch": 0.92, + "learning_rate": 3.9899699097291875e-05, + "loss": 1.1565, + "theoretical_loss": 3.317381280835279, + "tokens_seen": 3040083968 + }, + { + "epoch": 0.92, + "learning_rate": 3.987963891675026e-05, + "loss": 1.6108, + "theoretical_loss": 3.317370313853149, + "tokens_seen": 3040215040 + }, + { + "epoch": 0.92, + "learning_rate": 3.985957873620863e-05, + "loss": 1.2896, + "theoretical_loss": 3.317359347476208, + "tokens_seen": 3040346112 + }, + { + "epoch": 0.92, + "learning_rate": 3.9839518555667e-05, + "loss": 1.351, + "theoretical_loss": 3.317348381704396, + "tokens_seen": 3040477184 + }, + { + "epoch": 0.92, + "learning_rate": 3.981945837512538e-05, + "loss": 1.3522, + "theoretical_loss": 3.3173374165376535, + "tokens_seen": 3040608256 + }, + { + "epoch": 0.92, + "learning_rate": 3.9799398194583754e-05, + "loss": 1.1682, + "theoretical_loss": 3.317326451975921, + "tokens_seen": 3040739328 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.4092741906642914, + "objective/train/avg_token_score": 0.030014660209417343, + "objective/train/docs_used": 1710687, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2014095783233643, + "objective/train/original_loss": 3.6972904205322266, + "objective/train/theoretical_loss": 3.3173154880191396, + "objective/train/tokens_used": 3040870400, + "objective/train/unlikelihood_loss": 0.1610511690378189, + "theoretical_loss": 3.3173154880191396, + "tokens_seen": 3040870400 + }, + { + "epoch": 0.92, + "learning_rate": 3.9779338014042123e-05, + "loss": 1.2014, + "theoretical_loss": 3.3173154880191396, + "tokens_seen": 3040870400 + }, + { + "epoch": 0.92, + "learning_rate": 3.97592778335005e-05, + "loss": 1.1769, + "theoretical_loss": 3.3173045246672492, + "tokens_seen": 3041001472 + }, + { + "epoch": 0.92, + "learning_rate": 3.9739217652958875e-05, + "loss": 1.2867, + "theoretical_loss": 3.317293561920191, + "tokens_seen": 3041132544 + }, + { + "epoch": 0.92, + "learning_rate": 3.971915747241725e-05, + "loss": 1.1372, + "theoretical_loss": 3.317282599777905, + "tokens_seen": 3041263616 + }, + { + "epoch": 0.92, + "learning_rate": 3.969909729187563e-05, + "loss": 1.2695, + "theoretical_loss": 3.317271638240332, + "tokens_seen": 3041394688 + }, + { + "epoch": 0.92, + "learning_rate": 3.9679037111334e-05, + "loss": 1.4549, + "theoretical_loss": 3.3172606773074125, + "tokens_seen": 3041525760 + }, + { + "epoch": 0.92, + "learning_rate": 3.965897693079238e-05, + "loss": 1.2218, + "theoretical_loss": 3.317249716979087, + "tokens_seen": 3041656832 + }, + { + "epoch": 0.92, + "learning_rate": 3.9638916750250754e-05, + "loss": 1.1839, + "theoretical_loss": 3.3172387572552964, + "tokens_seen": 3041787904 + }, + { + "epoch": 0.92, + "learning_rate": 3.961885656970913e-05, + "loss": 1.2676, + "theoretical_loss": 3.3172277981359812, + "tokens_seen": 3041918976 + }, + { + "epoch": 0.92, + "learning_rate": 3.9598796389167506e-05, + "loss": 1.2667, + "theoretical_loss": 3.317216839621082, + "tokens_seen": 3042050048 + }, + { + "epoch": 0.92, + "learning_rate": 3.9578736208625875e-05, + "loss": 1.3004, + "theoretical_loss": 3.317205881710539, + "tokens_seen": 3042181120 + }, + { + "epoch": 0.92, + "learning_rate": 3.955867602808425e-05, + "loss": 1.2733, + "theoretical_loss": 3.3171949244042938, + "tokens_seen": 3042312192 + }, + { + "epoch": 0.92, + "learning_rate": 3.9538615847542626e-05, + "loss": 1.3594, + "theoretical_loss": 3.3171839677022863, + "tokens_seen": 3042443264 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.4689638316631317, + "objective/train/avg_token_score": 0.015868349000811577, + "objective/train/docs_used": 1711510, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4553229808807373, + "objective/train/original_loss": 3.845700979232788, + "objective/train/theoretical_loss": 3.317178489577853, + "objective/train/tokens_used": 3042508800, + "objective/train/unlikelihood_loss": 0.1509564369916916, + "theoretical_loss": 3.317178489577853, + "tokens_seen": 3042508800 + }, + { + "epoch": 0.92, + "learning_rate": 3.9518555667001e-05, + "loss": 1.2005, + "theoretical_loss": 3.3171730116044573, + "tokens_seen": 3042574336 + }, + { + "epoch": 0.92, + "learning_rate": 3.9498495486459385e-05, + "loss": 1.2722, + "theoretical_loss": 3.3171620561107473, + "tokens_seen": 3042705408 + }, + { + "epoch": 0.92, + "learning_rate": 3.9478435305917754e-05, + "loss": 1.2631, + "theoretical_loss": 3.3171511012210972, + "tokens_seen": 3042836480 + }, + { + "epoch": 0.92, + "learning_rate": 3.945837512537613e-05, + "loss": 1.1037, + "theoretical_loss": 3.317140146935448, + "tokens_seen": 3042967552 + }, + { + "epoch": 0.92, + "learning_rate": 3.9438314944834505e-05, + "loss": 1.2541, + "theoretical_loss": 3.3171291932537397, + "tokens_seen": 3043098624 + }, + { + "epoch": 0.92, + "learning_rate": 3.941825476429288e-05, + "loss": 1.1808, + "theoretical_loss": 3.317118240175913, + "tokens_seen": 3043229696 + }, + { + "epoch": 0.92, + "learning_rate": 3.939819458375126e-05, + "loss": 1.0313, + "theoretical_loss": 3.3171072877019094, + "tokens_seen": 3043360768 + }, + { + "epoch": 0.92, + "learning_rate": 3.9378134403209626e-05, + "loss": 1.1504, + "theoretical_loss": 3.317096335831669, + "tokens_seen": 3043491840 + }, + { + "epoch": 0.92, + "learning_rate": 3.9358074222668e-05, + "loss": 1.2131, + "theoretical_loss": 3.3170853845651322, + "tokens_seen": 3043622912 + }, + { + "epoch": 0.92, + "learning_rate": 3.933801404212638e-05, + "loss": 1.2575, + "theoretical_loss": 3.3170744339022407, + "tokens_seen": 3043753984 + }, + { + "epoch": 0.92, + "learning_rate": 3.931795386158476e-05, + "loss": 1.1929, + "theoretical_loss": 3.3170634838429347, + "tokens_seen": 3043885056 + }, + { + "epoch": 0.92, + "learning_rate": 3.9297893681043136e-05, + "loss": 1.1228, + "theoretical_loss": 3.3170525343871544, + "tokens_seen": 3044016128 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.3033357858657837, + "objective/train/avg_token_score": 0.02656998299062252, + "objective/train/docs_used": 1712674, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9631859660148621, + "objective/train/original_loss": 3.366086483001709, + "objective/train/theoretical_loss": 3.3170415855348416, + "objective/train/tokens_used": 3044147200, + "objective/train/unlikelihood_loss": 0.24514815211296082, + "theoretical_loss": 3.3170415855348416, + "tokens_seen": 3044147200 + }, + { + "epoch": 0.92, + "learning_rate": 3.9277833500501505e-05, + "loss": 1.3571, + "theoretical_loss": 3.3170415855348416, + "tokens_seen": 3044147200 + }, + { + "epoch": 0.92, + "learning_rate": 3.925777331995988e-05, + "loss": 1.239, + "theoretical_loss": 3.3170306372859364, + "tokens_seen": 3044278272 + }, + { + "epoch": 0.92, + "learning_rate": 3.9237713139418256e-05, + "loss": 1.3619, + "theoretical_loss": 3.3170196896403796, + "tokens_seen": 3044409344 + }, + { + "epoch": 0.92, + "learning_rate": 3.921765295887663e-05, + "loss": 1.2781, + "theoretical_loss": 3.3170087425981123, + "tokens_seen": 3044540416 + }, + { + "epoch": 0.92, + "learning_rate": 3.919759277833501e-05, + "loss": 1.327, + "theoretical_loss": 3.316997796159075, + "tokens_seen": 3044671488 + }, + { + "epoch": 0.92, + "learning_rate": 3.917753259779338e-05, + "loss": 1.3195, + "theoretical_loss": 3.3169868503232083, + "tokens_seen": 3044802560 + }, + { + "epoch": 0.92, + "learning_rate": 3.915747241725175e-05, + "loss": 1.1627, + "theoretical_loss": 3.3169759050904535, + "tokens_seen": 3044933632 + }, + { + "epoch": 0.92, + "learning_rate": 3.913741223671013e-05, + "loss": 1.3116, + "theoretical_loss": 3.316964960460751, + "tokens_seen": 3045064704 + }, + { + "epoch": 0.92, + "learning_rate": 3.911735205616851e-05, + "loss": 1.2346, + "theoretical_loss": 3.3169540164340416, + "tokens_seen": 3045195776 + }, + { + "epoch": 0.92, + "learning_rate": 3.909729187562689e-05, + "loss": 1.237, + "theoretical_loss": 3.316943073010267, + "tokens_seen": 3045326848 + }, + { + "epoch": 0.92, + "learning_rate": 3.9077231695085256e-05, + "loss": 1.2712, + "theoretical_loss": 3.316932130189367, + "tokens_seen": 3045457920 + }, + { + "epoch": 0.92, + "learning_rate": 3.905717151454363e-05, + "loss": 1.2396, + "theoretical_loss": 3.3169211879712828, + "tokens_seen": 3045588992 + }, + { + "epoch": 0.92, + "learning_rate": 3.903711133400201e-05, + "loss": 1.1652, + "theoretical_loss": 3.316910246355955, + "tokens_seen": 3045720064 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.4389662742614746, + "objective/train/avg_token_score": 0.017658747732639313, + "objective/train/docs_used": 1713238, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3946064710617065, + "objective/train/original_loss": 3.853856325149536, + "objective/train/theoretical_loss": 3.316904775774306, + "objective/train/tokens_used": 3045785600, + "objective/train/unlikelihood_loss": 0.16613703966140747, + "theoretical_loss": 3.316904775774306, + "tokens_seen": 3045785600 + }, + { + "epoch": 0.92, + "learning_rate": 3.901705115346038e-05, + "loss": 1.3271, + "theoretical_loss": 3.316899305343325, + "tokens_seen": 3045851136 + }, + { + "epoch": 0.92, + "learning_rate": 3.899699097291875e-05, + "loss": 1.2253, + "theoretical_loss": 3.316888364933333, + "tokens_seen": 3045982208 + }, + { + "epoch": 0.92, + "learning_rate": 3.897693079237713e-05, + "loss": 1.1993, + "theoretical_loss": 3.316877425125921, + "tokens_seen": 3046113280 + }, + { + "epoch": 0.92, + "learning_rate": 3.8956870611835504e-05, + "loss": 1.3049, + "theoretical_loss": 3.3168664859210284, + "tokens_seen": 3046244352 + }, + { + "epoch": 0.92, + "learning_rate": 3.8936810431293886e-05, + "loss": 1.3387, + "theoretical_loss": 3.316855547318597, + "tokens_seen": 3046375424 + }, + { + "epoch": 0.92, + "learning_rate": 3.891675025075226e-05, + "loss": 1.2294, + "theoretical_loss": 3.3168446093185677, + "tokens_seen": 3046506496 + }, + { + "epoch": 0.92, + "learning_rate": 3.889669007021063e-05, + "loss": 1.395, + "theoretical_loss": 3.316833671920881, + "tokens_seen": 3046637568 + }, + { + "epoch": 0.92, + "learning_rate": 3.887662988966901e-05, + "loss": 1.3809, + "theoretical_loss": 3.3168227351254784, + "tokens_seen": 3046768640 + }, + { + "epoch": 0.92, + "learning_rate": 3.885656970912738e-05, + "loss": 1.3358, + "theoretical_loss": 3.3168117989323003, + "tokens_seen": 3046899712 + }, + { + "epoch": 0.92, + "learning_rate": 3.883650952858576e-05, + "loss": 1.3561, + "theoretical_loss": 3.316800863341288, + "tokens_seen": 3047030784 + }, + { + "epoch": 0.92, + "learning_rate": 3.8816449348044134e-05, + "loss": 1.4192, + "theoretical_loss": 3.3167899283523816, + "tokens_seen": 3047161856 + }, + { + "epoch": 0.92, + "learning_rate": 3.8796389167502503e-05, + "loss": 1.1433, + "theoretical_loss": 3.3167789939655234, + "tokens_seen": 3047292928 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.42326492071151733, + "objective/train/avg_token_score": 0.013150791637599468, + "objective/train/docs_used": 1714459, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.214699387550354, + "objective/train/original_loss": 3.308070659637451, + "objective/train/theoretical_loss": 3.3167680601806535, + "objective/train/tokens_used": 3047424000, + "objective/train/unlikelihood_loss": 0.21268604695796967, + "theoretical_loss": 3.3167680601806535, + "tokens_seen": 3047424000 + }, + { + "epoch": 0.92, + "learning_rate": 3.877632898696088e-05, + "loss": 1.3747, + "theoretical_loss": 3.3167680601806535, + "tokens_seen": 3047424000 + }, + { + "epoch": 0.92, + "learning_rate": 3.875626880641926e-05, + "loss": 1.4273, + "theoretical_loss": 3.3167571269977127, + "tokens_seen": 3047555072 + }, + { + "epoch": 0.92, + "learning_rate": 3.873620862587764e-05, + "loss": 1.2888, + "theoretical_loss": 3.316746194416643, + "tokens_seen": 3047686144 + }, + { + "epoch": 0.92, + "learning_rate": 3.871614844533601e-05, + "loss": 1.1603, + "theoretical_loss": 3.3167352624373843, + "tokens_seen": 3047817216 + }, + { + "epoch": 0.92, + "learning_rate": 3.869608826479438e-05, + "loss": 1.3795, + "theoretical_loss": 3.316724331059878, + "tokens_seen": 3047948288 + }, + { + "epoch": 0.92, + "learning_rate": 3.867602808425276e-05, + "loss": 1.3415, + "theoretical_loss": 3.3167134002840646, + "tokens_seen": 3048079360 + }, + { + "epoch": 0.92, + "learning_rate": 3.8655967903711134e-05, + "loss": 1.2248, + "theoretical_loss": 3.3167024701098864, + "tokens_seen": 3048210432 + }, + { + "epoch": 0.92, + "learning_rate": 3.863590772316951e-05, + "loss": 1.3228, + "theoretical_loss": 3.316691540537283, + "tokens_seen": 3048341504 + }, + { + "epoch": 0.92, + "learning_rate": 3.8615847542627886e-05, + "loss": 1.2837, + "theoretical_loss": 3.3166806115661966, + "tokens_seen": 3048472576 + }, + { + "epoch": 0.92, + "learning_rate": 3.8595787362086255e-05, + "loss": 1.4403, + "theoretical_loss": 3.3166696831965674, + "tokens_seen": 3048603648 + }, + { + "epoch": 0.92, + "learning_rate": 3.857572718154463e-05, + "loss": 1.287, + "theoretical_loss": 3.3166587554283367, + "tokens_seen": 3048734720 + }, + { + "epoch": 0.92, + "learning_rate": 3.855566700100301e-05, + "loss": 1.3654, + "theoretical_loss": 3.3166478282614458, + "tokens_seen": 3048865792 + }, + { + "epoch": 0.92, + "learning_rate": 3.853560682046139e-05, + "loss": 1.1578, + "theoretical_loss": 3.3166369016958352, + "tokens_seen": 3048996864 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.33962610363960266, + "objective/train/avg_token_score": 0.02284344844520092, + "objective/train/docs_used": 1714997, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0637476444244385, + "objective/train/original_loss": 3.714752674102783, + "objective/train/theoretical_loss": 3.316631438638492, + "objective/train/tokens_used": 3049062400, + "objective/train/unlikelihood_loss": 0.2239442765712738, + "theoretical_loss": 3.316631438638492, + "tokens_seen": 3049062400 + }, + { + "epoch": 0.92, + "learning_rate": 3.8515546639919765e-05, + "loss": 1.1747, + "theoretical_loss": 3.316625975731446, + "tokens_seen": 3049127936 + }, + { + "epoch": 0.92, + "learning_rate": 3.8495486459378134e-05, + "loss": 1.168, + "theoretical_loss": 3.3166150503682204, + "tokens_seen": 3049259008 + }, + { + "epoch": 0.92, + "learning_rate": 3.847542627883651e-05, + "loss": 1.2806, + "theoretical_loss": 3.3166041256060983, + "tokens_seen": 3049390080 + }, + { + "epoch": 0.92, + "learning_rate": 3.8455366098294885e-05, + "loss": 1.2422, + "theoretical_loss": 3.316593201445021, + "tokens_seen": 3049521152 + }, + { + "epoch": 0.92, + "learning_rate": 3.843530591775326e-05, + "loss": 1.4531, + "theoretical_loss": 3.3165822778849297, + "tokens_seen": 3049652224 + }, + { + "epoch": 0.92, + "learning_rate": 3.841524573721164e-05, + "loss": 1.2312, + "theoretical_loss": 3.316571354925766, + "tokens_seen": 3049783296 + }, + { + "epoch": 0.92, + "learning_rate": 3.8395185556670006e-05, + "loss": 1.3732, + "theoretical_loss": 3.3165604325674702, + "tokens_seen": 3049914368 + }, + { + "epoch": 0.92, + "learning_rate": 3.837512537612839e-05, + "loss": 1.2112, + "theoretical_loss": 3.316549510809984, + "tokens_seen": 3050045440 + }, + { + "epoch": 0.92, + "learning_rate": 3.8355065195586764e-05, + "loss": 1.2814, + "theoretical_loss": 3.3165385896532484, + "tokens_seen": 3050176512 + }, + { + "epoch": 0.92, + "learning_rate": 3.833500501504514e-05, + "loss": 1.3013, + "theoretical_loss": 3.316527669097204, + "tokens_seen": 3050307584 + }, + { + "epoch": 0.92, + "learning_rate": 3.8314944834503516e-05, + "loss": 1.2382, + "theoretical_loss": 3.316516749141793, + "tokens_seen": 3050438656 + }, + { + "epoch": 0.92, + "learning_rate": 3.8294884653961885e-05, + "loss": 1.3233, + "theoretical_loss": 3.316505829786956, + "tokens_seen": 3050569728 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.38831865787506104, + "objective/train/avg_token_score": 0.015651792287826538, + "objective/train/docs_used": 1716070, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2458277940750122, + "objective/train/original_loss": 3.847818374633789, + "objective/train/theoretical_loss": 3.316494911032634, + "objective/train/tokens_used": 3050700800, + "objective/train/unlikelihood_loss": 0.16639383137226105, + "theoretical_loss": 3.316494911032634, + "tokens_seen": 3050700800 + }, + { + "epoch": 0.92, + "learning_rate": 3.827482447342026e-05, + "loss": 1.2413, + "theoretical_loss": 3.316494911032634, + "tokens_seen": 3050700800 + }, + { + "epoch": 0.92, + "learning_rate": 3.8254764292878636e-05, + "loss": 1.3839, + "theoretical_loss": 3.316483992878768, + "tokens_seen": 3050831872 + }, + { + "epoch": 0.92, + "learning_rate": 3.823470411233701e-05, + "loss": 1.3912, + "theoretical_loss": 3.3164730753253, + "tokens_seen": 3050962944 + }, + { + "epoch": 0.92, + "learning_rate": 3.821464393179538e-05, + "loss": 1.196, + "theoretical_loss": 3.3164621583721705, + "tokens_seen": 3051094016 + }, + { + "epoch": 0.92, + "learning_rate": 3.819458375125376e-05, + "loss": 1.4393, + "theoretical_loss": 3.316451242019321, + "tokens_seen": 3051225088 + }, + { + "epoch": 0.92, + "learning_rate": 3.817452357071214e-05, + "loss": 1.2314, + "theoretical_loss": 3.3164403262666924, + "tokens_seen": 3051356160 + }, + { + "epoch": 0.92, + "learning_rate": 3.8154463390170515e-05, + "loss": 1.2996, + "theoretical_loss": 3.3164294111142265, + "tokens_seen": 3051487232 + }, + { + "epoch": 0.92, + "learning_rate": 3.813440320962889e-05, + "loss": 1.3271, + "theoretical_loss": 3.3164184965618637, + "tokens_seen": 3051618304 + }, + { + "epoch": 0.92, + "learning_rate": 3.811434302908726e-05, + "loss": 1.3326, + "theoretical_loss": 3.3164075826095463, + "tokens_seen": 3051749376 + }, + { + "epoch": 0.92, + "learning_rate": 3.8094282848545636e-05, + "loss": 1.3104, + "theoretical_loss": 3.316396669257214, + "tokens_seen": 3051880448 + }, + { + "epoch": 0.92, + "learning_rate": 3.807422266800401e-05, + "loss": 1.3321, + "theoretical_loss": 3.31638575650481, + "tokens_seen": 3052011520 + }, + { + "epoch": 0.92, + "learning_rate": 3.805416248746239e-05, + "loss": 1.1435, + "theoretical_loss": 3.316374844352274, + "tokens_seen": 3052142592 + }, + { + "epoch": 0.92, + "learning_rate": 3.803410230692076e-05, + "loss": 1.302, + "theoretical_loss": 3.316363932799548, + "tokens_seen": 3052273664 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.5131964683532715, + "objective/train/avg_token_score": 0.018213069066405296, + "objective/train/docs_used": 1716876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4249821901321411, + "objective/train/original_loss": 3.3467037677764893, + "objective/train/theoretical_loss": 3.3163584772480954, + "objective/train/tokens_used": 3052339200, + "objective/train/unlikelihood_loss": 0.14977829158306122, + "theoretical_loss": 3.3163584772480954, + "tokens_seen": 3052339200 + }, + { + "epoch": 0.92, + "learning_rate": 3.801404212637913e-05, + "loss": 1.3458, + "theoretical_loss": 3.3163530218465733, + "tokens_seen": 3052404736 + }, + { + "epoch": 0.93, + "learning_rate": 3.7993981945837515e-05, + "loss": 1.3678, + "theoretical_loss": 3.3163421114932907, + "tokens_seen": 3052535808 + }, + { + "epoch": 0.93, + "learning_rate": 3.797392176529589e-05, + "loss": 1.1792, + "theoretical_loss": 3.316331201739642, + "tokens_seen": 3052666880 + }, + { + "epoch": 0.93, + "learning_rate": 3.7953861584754266e-05, + "loss": 1.1458, + "theoretical_loss": 3.3163202925855675, + "tokens_seen": 3052797952 + }, + { + "epoch": 0.93, + "learning_rate": 3.793380140421264e-05, + "loss": 1.3801, + "theoretical_loss": 3.31630938403101, + "tokens_seen": 3052929024 + }, + { + "epoch": 0.93, + "learning_rate": 3.791374122367101e-05, + "loss": 1.2758, + "theoretical_loss": 3.31629847607591, + "tokens_seen": 3053060096 + }, + { + "epoch": 0.93, + "learning_rate": 3.789368104312939e-05, + "loss": 1.222, + "theoretical_loss": 3.3162875687202087, + "tokens_seen": 3053191168 + }, + { + "epoch": 0.93, + "learning_rate": 3.787362086258776e-05, + "loss": 1.22, + "theoretical_loss": 3.316276661963848, + "tokens_seen": 3053322240 + }, + { + "epoch": 0.93, + "learning_rate": 3.785356068204614e-05, + "loss": 1.2025, + "theoretical_loss": 3.3162657558067687, + "tokens_seen": 3053453312 + }, + { + "epoch": 0.93, + "learning_rate": 3.7833500501504514e-05, + "loss": 1.3453, + "theoretical_loss": 3.316254850248912, + "tokens_seen": 3053584384 + }, + { + "epoch": 0.93, + "learning_rate": 3.781344032096289e-05, + "loss": 1.3599, + "theoretical_loss": 3.3162439452902204, + "tokens_seen": 3053715456 + }, + { + "epoch": 0.93, + "learning_rate": 3.7793380140421266e-05, + "loss": 1.3464, + "theoretical_loss": 3.3162330409306335, + "tokens_seen": 3053846528 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.4224095940589905, + "objective/train/avg_token_score": 0.009513984434306622, + "objective/train/docs_used": 1718284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1984667778015137, + "objective/train/original_loss": 3.573622941970825, + "objective/train/theoretical_loss": 3.316222137170094, + "objective/train/tokens_used": 3053977600, + "objective/train/unlikelihood_loss": 0.18266238272190094, + "theoretical_loss": 3.316222137170094, + "tokens_seen": 3053977600 + }, + { + "epoch": 0.93, + "learning_rate": 3.777331995987964e-05, + "loss": 1.2893, + "theoretical_loss": 3.316222137170094, + "tokens_seen": 3053977600 + }, + { + "epoch": 0.93, + "learning_rate": 3.775325977933802e-05, + "loss": 1.1292, + "theoretical_loss": 3.3162112340085432, + "tokens_seen": 3054108672 + }, + { + "epoch": 0.93, + "learning_rate": 3.773319959879639e-05, + "loss": 1.2143, + "theoretical_loss": 3.316200331445922, + "tokens_seen": 3054239744 + }, + { + "epoch": 0.93, + "learning_rate": 3.771313941825476e-05, + "loss": 1.3596, + "theoretical_loss": 3.316189429482172, + "tokens_seen": 3054370816 + }, + { + "epoch": 0.93, + "learning_rate": 3.769307923771314e-05, + "loss": 1.4332, + "theoretical_loss": 3.3161785281172347, + "tokens_seen": 3054501888 + }, + { + "epoch": 0.93, + "learning_rate": 3.7673019057171514e-05, + "loss": 1.4367, + "theoretical_loss": 3.316167627351051, + "tokens_seen": 3054632960 + }, + { + "epoch": 0.93, + "learning_rate": 3.765295887662989e-05, + "loss": 1.3618, + "theoretical_loss": 3.3161567271835635, + "tokens_seen": 3054764032 + }, + { + "epoch": 0.93, + "learning_rate": 3.763289869608826e-05, + "loss": 1.3034, + "theoretical_loss": 3.3161458276147124, + "tokens_seen": 3054895104 + }, + { + "epoch": 0.93, + "learning_rate": 3.761283851554664e-05, + "loss": 1.123, + "theoretical_loss": 3.31613492864444, + "tokens_seen": 3055026176 + }, + { + "epoch": 0.93, + "learning_rate": 3.759277833500502e-05, + "loss": 1.3413, + "theoretical_loss": 3.316124030272687, + "tokens_seen": 3055157248 + }, + { + "epoch": 0.93, + "learning_rate": 3.757271815446339e-05, + "loss": 1.2371, + "theoretical_loss": 3.3161131324993955, + "tokens_seen": 3055288320 + }, + { + "epoch": 0.93, + "learning_rate": 3.755265797392177e-05, + "loss": 1.4327, + "theoretical_loss": 3.3161022353245064, + "tokens_seen": 3055419392 + }, + { + "epoch": 0.93, + "learning_rate": 3.753259779338014e-05, + "loss": 1.2959, + "theoretical_loss": 3.316091338747962, + "tokens_seen": 3055550464 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.4534457325935364, + "objective/train/avg_token_score": 0.02144760824739933, + "objective/train/docs_used": 1719055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3571922779083252, + "objective/train/original_loss": 4.095467567443848, + "objective/train/theoretical_loss": 3.3160858906840502, + "objective/train/tokens_used": 3055616000, + "objective/train/unlikelihood_loss": 0.08389967679977417, + "theoretical_loss": 3.3160858906840502, + "tokens_seen": 3055616000 + }, + { + "epoch": 0.93, + "learning_rate": 3.7512537612838514e-05, + "loss": 1.3159, + "theoretical_loss": 3.3160804427697026, + "tokens_seen": 3055681536 + }, + { + "epoch": 0.93, + "learning_rate": 3.749247743229689e-05, + "loss": 1.3267, + "theoretical_loss": 3.3160695473896706, + "tokens_seen": 3055812608 + }, + { + "epoch": 0.93, + "learning_rate": 3.7472417251755265e-05, + "loss": 1.2705, + "theoretical_loss": 3.3160586526078073, + "tokens_seen": 3055943680 + }, + { + "epoch": 0.93, + "learning_rate": 3.745235707121364e-05, + "loss": 1.1968, + "theoretical_loss": 3.316047758424054, + "tokens_seen": 3056074752 + }, + { + "epoch": 0.93, + "learning_rate": 3.743229689067202e-05, + "loss": 1.3122, + "theoretical_loss": 3.3160368648383525, + "tokens_seen": 3056205824 + }, + { + "epoch": 0.93, + "learning_rate": 3.741223671013039e-05, + "loss": 1.2024, + "theoretical_loss": 3.316025971850644, + "tokens_seen": 3056336896 + }, + { + "epoch": 0.93, + "learning_rate": 3.739217652958877e-05, + "loss": 1.3607, + "theoretical_loss": 3.3160150794608705, + "tokens_seen": 3056467968 + }, + { + "epoch": 0.93, + "learning_rate": 3.7372116349047144e-05, + "loss": 1.3724, + "theoretical_loss": 3.3160041876689728, + "tokens_seen": 3056599040 + }, + { + "epoch": 0.93, + "learning_rate": 3.735205616850552e-05, + "loss": 1.2382, + "theoretical_loss": 3.3159932964748933, + "tokens_seen": 3056730112 + }, + { + "epoch": 0.93, + "learning_rate": 3.733199598796389e-05, + "loss": 1.3184, + "theoretical_loss": 3.3159824058785734, + "tokens_seen": 3056861184 + }, + { + "epoch": 0.93, + "learning_rate": 3.7311935807422265e-05, + "loss": 1.1867, + "theoretical_loss": 3.3159715158799536, + "tokens_seen": 3056992256 + }, + { + "epoch": 0.93, + "learning_rate": 3.729187562688064e-05, + "loss": 1.3107, + "theoretical_loss": 3.3159606264789767, + "tokens_seen": 3057123328 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.3048020601272583, + "objective/train/avg_token_score": 0.007878215983510017, + "objective/train/docs_used": 1720294, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8455373644828796, + "objective/train/original_loss": 3.510383367538452, + "objective/train/theoretical_loss": 3.315949737675584, + "objective/train/tokens_used": 3057254400, + "objective/train/unlikelihood_loss": 0.21112555265426636, + "theoretical_loss": 3.315949737675584, + "tokens_seen": 3057254400 + }, + { + "epoch": 0.93, + "learning_rate": 3.7271815446339016e-05, + "loss": 1.2041, + "theoretical_loss": 3.315949737675584, + "tokens_seen": 3057254400 + }, + { + "epoch": 0.93, + "learning_rate": 3.72517552657974e-05, + "loss": 1.2519, + "theoretical_loss": 3.315938849469717, + "tokens_seen": 3057385472 + }, + { + "epoch": 0.93, + "learning_rate": 3.723169508525577e-05, + "loss": 1.2857, + "theoretical_loss": 3.315927961861317, + "tokens_seen": 3057516544 + }, + { + "epoch": 0.93, + "learning_rate": 3.7211634904714144e-05, + "loss": 1.2013, + "theoretical_loss": 3.315917074850326, + "tokens_seen": 3057647616 + }, + { + "epoch": 0.93, + "learning_rate": 3.719157472417252e-05, + "loss": 1.3299, + "theoretical_loss": 3.3159061884366854, + "tokens_seen": 3057778688 + }, + { + "epoch": 0.93, + "learning_rate": 3.7171514543630895e-05, + "loss": 1.3332, + "theoretical_loss": 3.315895302620337, + "tokens_seen": 3057909760 + }, + { + "epoch": 0.93, + "learning_rate": 3.715145436308927e-05, + "loss": 1.3943, + "theoretical_loss": 3.3158844174012225, + "tokens_seen": 3058040832 + }, + { + "epoch": 0.93, + "learning_rate": 3.713139418254764e-05, + "loss": 1.1996, + "theoretical_loss": 3.3158735327792828, + "tokens_seen": 3058171904 + }, + { + "epoch": 0.93, + "learning_rate": 3.7111334002006016e-05, + "loss": 1.3927, + "theoretical_loss": 3.3158626487544605, + "tokens_seen": 3058302976 + }, + { + "epoch": 0.93, + "learning_rate": 3.709127382146439e-05, + "loss": 1.3455, + "theoretical_loss": 3.315851765326697, + "tokens_seen": 3058434048 + }, + { + "epoch": 0.93, + "learning_rate": 3.707121364092277e-05, + "loss": 1.1177, + "theoretical_loss": 3.3158408824959333, + "tokens_seen": 3058565120 + }, + { + "epoch": 0.93, + "learning_rate": 3.705115346038115e-05, + "loss": 1.3079, + "theoretical_loss": 3.315830000262112, + "tokens_seen": 3058696192 + }, + { + "epoch": 0.93, + "learning_rate": 3.703109327983952e-05, + "loss": 1.3287, + "theoretical_loss": 3.315819118625174, + "tokens_seen": 3058827264 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.47385141253471375, + "objective/train/avg_token_score": 0.013703512959182262, + "objective/train/docs_used": 1720703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5184903144836426, + "objective/train/original_loss": 3.988107442855835, + "objective/train/theoretical_loss": 3.3158136780305183, + "objective/train/tokens_used": 3058892800, + "objective/train/unlikelihood_loss": 0.1477121114730835, + "theoretical_loss": 3.3158136780305183, + "tokens_seen": 3058892800 + }, + { + "epoch": 0.93, + "learning_rate": 3.7011033099297895e-05, + "loss": 1.3411, + "theoretical_loss": 3.315808237585062, + "tokens_seen": 3058958336 + }, + { + "epoch": 0.93, + "learning_rate": 3.699097291875627e-05, + "loss": 1.4064, + "theoretical_loss": 3.3157973571417165, + "tokens_seen": 3059089408 + }, + { + "epoch": 0.93, + "learning_rate": 3.6970912738214646e-05, + "loss": 1.3657, + "theoretical_loss": 3.31578647729508, + "tokens_seen": 3059220480 + }, + { + "epoch": 0.93, + "learning_rate": 3.695085255767302e-05, + "loss": 1.1577, + "theoretical_loss": 3.315775598045094, + "tokens_seen": 3059351552 + }, + { + "epoch": 0.93, + "learning_rate": 3.693079237713139e-05, + "loss": 1.3266, + "theoretical_loss": 3.3157647193916997, + "tokens_seen": 3059482624 + }, + { + "epoch": 0.93, + "learning_rate": 3.691073219658977e-05, + "loss": 1.2392, + "theoretical_loss": 3.31575384133484, + "tokens_seen": 3059613696 + }, + { + "epoch": 0.93, + "learning_rate": 3.689067201604814e-05, + "loss": 1.4605, + "theoretical_loss": 3.3157429638744556, + "tokens_seen": 3059744768 + }, + { + "epoch": 0.93, + "learning_rate": 3.6870611835506525e-05, + "loss": 1.37, + "theoretical_loss": 3.3157320870104883, + "tokens_seen": 3059875840 + }, + { + "epoch": 0.93, + "learning_rate": 3.68505516549649e-05, + "loss": 1.3898, + "theoretical_loss": 3.3157212107428804, + "tokens_seen": 3060006912 + }, + { + "epoch": 0.93, + "learning_rate": 3.683049147442327e-05, + "loss": 1.3616, + "theoretical_loss": 3.3157103350715738, + "tokens_seen": 3060137984 + }, + { + "epoch": 0.93, + "learning_rate": 3.6810431293881646e-05, + "loss": 1.316, + "theoretical_loss": 3.3156994599965093, + "tokens_seen": 3060269056 + }, + { + "epoch": 0.93, + "learning_rate": 3.679037111334002e-05, + "loss": 1.3829, + "theoretical_loss": 3.3156885855176297, + "tokens_seen": 3060400128 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.41660556197166443, + "objective/train/avg_token_score": 0.006159162148833275, + "objective/train/docs_used": 1721958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2425976991653442, + "objective/train/original_loss": 3.208489179611206, + "objective/train/theoretical_loss": 3.315677711634876, + "objective/train/tokens_used": 3060531200, + "objective/train/unlikelihood_loss": 0.23009684681892395, + "theoretical_loss": 3.315677711634876, + "tokens_seen": 3060531200 + }, + { + "epoch": 0.93, + "learning_rate": 3.67703109327984e-05, + "loss": 1.2511, + "theoretical_loss": 3.315677711634876, + "tokens_seen": 3060531200 + }, + { + "epoch": 0.93, + "learning_rate": 3.6750250752256767e-05, + "loss": 1.409, + "theoretical_loss": 3.31566683834819, + "tokens_seen": 3060662272 + }, + { + "epoch": 0.93, + "learning_rate": 3.673019057171514e-05, + "loss": 1.4431, + "theoretical_loss": 3.315655965657515, + "tokens_seen": 3060793344 + }, + { + "epoch": 0.93, + "learning_rate": 3.671013039117352e-05, + "loss": 1.3881, + "theoretical_loss": 3.315645093562791, + "tokens_seen": 3060924416 + }, + { + "epoch": 0.93, + "learning_rate": 3.66900702106319e-05, + "loss": 1.2533, + "theoretical_loss": 3.31563422206396, + "tokens_seen": 3061055488 + }, + { + "epoch": 0.93, + "learning_rate": 3.6670010030090277e-05, + "loss": 1.4656, + "theoretical_loss": 3.3156233511609647, + "tokens_seen": 3061186560 + }, + { + "epoch": 0.93, + "learning_rate": 3.6649949849548646e-05, + "loss": 1.2659, + "theoretical_loss": 3.3156124808537464, + "tokens_seen": 3061317632 + }, + { + "epoch": 0.93, + "learning_rate": 3.662988966900702e-05, + "loss": 1.3762, + "theoretical_loss": 3.315601611142247, + "tokens_seen": 3061448704 + }, + { + "epoch": 0.93, + "learning_rate": 3.66098294884654e-05, + "loss": 1.3237, + "theoretical_loss": 3.3155907420264086, + "tokens_seen": 3061579776 + }, + { + "epoch": 0.93, + "learning_rate": 3.658976930792377e-05, + "loss": 1.2969, + "theoretical_loss": 3.315579873506173, + "tokens_seen": 3061710848 + }, + { + "epoch": 0.93, + "learning_rate": 3.656970912738215e-05, + "loss": 1.3937, + "theoretical_loss": 3.3155690055814815, + "tokens_seen": 3061841920 + }, + { + "epoch": 0.93, + "learning_rate": 3.654964894684052e-05, + "loss": 1.5094, + "theoretical_loss": 3.3155581382522765, + "tokens_seen": 3061972992 + }, + { + "epoch": 0.93, + "learning_rate": 3.6529588766298894e-05, + "loss": 1.4733, + "theoretical_loss": 3.3155472715185, + "tokens_seen": 3062104064 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.31341642141342163, + "objective/train/avg_token_score": 0.054876964539289474, + "objective/train/docs_used": 1722530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9878774285316467, + "objective/train/original_loss": 4.689523696899414, + "objective/train/theoretical_loss": 3.3155418383748794, + "objective/train/tokens_used": 3062169600, + "objective/train/unlikelihood_loss": 0.09227162599563599, + "theoretical_loss": 3.3155418383748794, + "tokens_seen": 3062169600 + }, + { + "epoch": 0.93, + "learning_rate": 3.650952858575727e-05, + "loss": 1.3889, + "theoretical_loss": 3.3155364053800938, + "tokens_seen": 3062235136 + }, + { + "epoch": 0.93, + "learning_rate": 3.648946840521565e-05, + "loss": 1.437, + "theoretical_loss": 3.3155255398369996, + "tokens_seen": 3062366208 + }, + { + "epoch": 0.93, + "learning_rate": 3.646940822467403e-05, + "loss": 1.2704, + "theoretical_loss": 3.315514674889159, + "tokens_seen": 3062497280 + }, + { + "epoch": 0.93, + "learning_rate": 3.64493480441324e-05, + "loss": 1.3166, + "theoretical_loss": 3.315503810536515, + "tokens_seen": 3062628352 + }, + { + "epoch": 0.93, + "learning_rate": 3.642928786359077e-05, + "loss": 1.4804, + "theoretical_loss": 3.315492946779008, + "tokens_seen": 3062759424 + }, + { + "epoch": 0.93, + "learning_rate": 3.640922768304915e-05, + "loss": 1.4551, + "theoretical_loss": 3.3154820836165815, + "tokens_seen": 3062890496 + }, + { + "epoch": 0.93, + "learning_rate": 3.6389167502507524e-05, + "loss": 1.3716, + "theoretical_loss": 3.3154712210491764, + "tokens_seen": 3063021568 + }, + { + "epoch": 0.93, + "learning_rate": 3.63691073219659e-05, + "loss": 1.3446, + "theoretical_loss": 3.315460359076735, + "tokens_seen": 3063152640 + }, + { + "epoch": 0.93, + "learning_rate": 3.634904714142427e-05, + "loss": 1.3147, + "theoretical_loss": 3.315449497699199, + "tokens_seen": 3063283712 + }, + { + "epoch": 0.93, + "learning_rate": 3.6328986960882645e-05, + "loss": 1.383, + "theoretical_loss": 3.3154386369165105, + "tokens_seen": 3063414784 + }, + { + "epoch": 0.93, + "learning_rate": 3.630892678034103e-05, + "loss": 1.4227, + "theoretical_loss": 3.3154277767286118, + "tokens_seen": 3063545856 + }, + { + "epoch": 0.93, + "learning_rate": 3.62888665997994e-05, + "loss": 1.2375, + "theoretical_loss": 3.3154169171354444, + "tokens_seen": 3063676928 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.5514418482780457, + "objective/train/avg_token_score": 0.003539995988830924, + "objective/train/docs_used": 1723856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8581793308258057, + "objective/train/original_loss": 3.5110318660736084, + "objective/train/theoretical_loss": 3.3154060581369507, + "objective/train/tokens_used": 3063808000, + "objective/train/unlikelihood_loss": 0.14474062621593475, + "theoretical_loss": 3.3154060581369507, + "tokens_seen": 3063808000 + }, + { + "epoch": 0.93, + "learning_rate": 3.626880641925778e-05, + "loss": 1.4227, + "theoretical_loss": 3.3154060581369507, + "tokens_seen": 3063808000 + }, + { + "epoch": 0.93, + "learning_rate": 3.624874623871615e-05, + "loss": 1.4485, + "theoretical_loss": 3.315395199733073, + "tokens_seen": 3063939072 + }, + { + "epoch": 0.93, + "learning_rate": 3.6228686058174524e-05, + "loss": 1.4314, + "theoretical_loss": 3.315384341923752, + "tokens_seen": 3064070144 + }, + { + "epoch": 0.93, + "learning_rate": 3.62086258776329e-05, + "loss": 1.3532, + "theoretical_loss": 3.315373484708931, + "tokens_seen": 3064201216 + }, + { + "epoch": 0.93, + "learning_rate": 3.6188565697091275e-05, + "loss": 1.285, + "theoretical_loss": 3.3153626280885513, + "tokens_seen": 3064332288 + }, + { + "epoch": 0.93, + "learning_rate": 3.616850551654965e-05, + "loss": 1.2974, + "theoretical_loss": 3.315351772062555, + "tokens_seen": 3064463360 + }, + { + "epoch": 0.93, + "learning_rate": 3.614844533600802e-05, + "loss": 1.3269, + "theoretical_loss": 3.315340916630885, + "tokens_seen": 3064594432 + }, + { + "epoch": 0.93, + "learning_rate": 3.61283851554664e-05, + "loss": 1.3593, + "theoretical_loss": 3.3153300617934818, + "tokens_seen": 3064725504 + }, + { + "epoch": 0.93, + "learning_rate": 3.610832497492478e-05, + "loss": 1.2213, + "theoretical_loss": 3.3153192075502886, + "tokens_seen": 3064856576 + }, + { + "epoch": 0.93, + "learning_rate": 3.6088264794383154e-05, + "loss": 1.3969, + "theoretical_loss": 3.315308353901248, + "tokens_seen": 3064987648 + }, + { + "epoch": 0.93, + "learning_rate": 3.606820461384152e-05, + "loss": 1.2728, + "theoretical_loss": 3.3152975008463, + "tokens_seen": 3065118720 + }, + { + "epoch": 0.93, + "learning_rate": 3.60481444332999e-05, + "loss": 1.3936, + "theoretical_loss": 3.3152866483853884, + "tokens_seen": 3065249792 + }, + { + "epoch": 0.93, + "learning_rate": 3.6028084252758275e-05, + "loss": 1.4319, + "theoretical_loss": 3.3152757965184554, + "tokens_seen": 3065380864 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.2834799587726593, + "objective/train/avg_token_score": 0.03510686755180359, + "objective/train/docs_used": 1724430, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7266472578048706, + "objective/train/original_loss": 4.430474281311035, + "objective/train/theoretical_loss": 3.315270370807712, + "objective/train/tokens_used": 3065446400, + "objective/train/unlikelihood_loss": 0.1519739329814911, + "theoretical_loss": 3.315270370807712, + "tokens_seen": 3065446400 + }, + { + "epoch": 0.93, + "learning_rate": 3.600802407221665e-05, + "loss": 1.2888, + "theoretical_loss": 3.315264945245442, + "tokens_seen": 3065511936 + }, + { + "epoch": 0.93, + "learning_rate": 3.5987963891675026e-05, + "loss": 1.3021, + "theoretical_loss": 3.3152540945662907, + "tokens_seen": 3065643008 + }, + { + "epoch": 0.93, + "learning_rate": 3.5967903711133395e-05, + "loss": 1.3736, + "theoretical_loss": 3.3152432444809437, + "tokens_seen": 3065774080 + }, + { + "epoch": 0.93, + "learning_rate": 3.594784353059177e-05, + "loss": 1.3479, + "theoretical_loss": 3.3152323949893434, + "tokens_seen": 3065905152 + }, + { + "epoch": 0.93, + "learning_rate": 3.5927783350050154e-05, + "loss": 1.2979, + "theoretical_loss": 3.315221546091432, + "tokens_seen": 3066036224 + }, + { + "epoch": 0.93, + "learning_rate": 3.590772316950853e-05, + "loss": 1.3004, + "theoretical_loss": 3.3152106977871503, + "tokens_seen": 3066167296 + }, + { + "epoch": 0.93, + "learning_rate": 3.5887662988966905e-05, + "loss": 1.3614, + "theoretical_loss": 3.3151998500764424, + "tokens_seen": 3066298368 + }, + { + "epoch": 0.93, + "learning_rate": 3.5867602808425274e-05, + "loss": 1.2541, + "theoretical_loss": 3.315189002959249, + "tokens_seen": 3066429440 + }, + { + "epoch": 0.93, + "learning_rate": 3.584754262788365e-05, + "loss": 1.4078, + "theoretical_loss": 3.315178156435513, + "tokens_seen": 3066560512 + }, + { + "epoch": 0.93, + "learning_rate": 3.5827482447342026e-05, + "loss": 1.4511, + "theoretical_loss": 3.315167310505176, + "tokens_seen": 3066691584 + }, + { + "epoch": 0.93, + "learning_rate": 3.58074222668004e-05, + "loss": 1.3803, + "theoretical_loss": 3.315156465168181, + "tokens_seen": 3066822656 + }, + { + "epoch": 0.93, + "learning_rate": 3.578736208625878e-05, + "loss": 1.248, + "theoretical_loss": 3.3151456204244694, + "tokens_seen": 3066953728 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.48179373145103455, + "objective/train/avg_token_score": 0.02168223261833191, + "objective/train/docs_used": 1725702, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3882675170898438, + "objective/train/original_loss": 3.6002023220062256, + "objective/train/theoretical_loss": 3.3151347762739833, + "objective/train/tokens_used": 3067084800, + "objective/train/unlikelihood_loss": 0.14459151029586792, + "theoretical_loss": 3.3151347762739833, + "tokens_seen": 3067084800 + }, + { + "epoch": 0.93, + "learning_rate": 3.5767301905717147e-05, + "loss": 1.3976, + "theoretical_loss": 3.3151347762739833, + "tokens_seen": 3067084800 + }, + { + "epoch": 0.93, + "learning_rate": 3.574724172517553e-05, + "loss": 1.4218, + "theoretical_loss": 3.3151239327166655, + "tokens_seen": 3067215872 + }, + { + "epoch": 0.93, + "learning_rate": 3.5727181544633905e-05, + "loss": 1.1925, + "theoretical_loss": 3.315113089752458, + "tokens_seen": 3067346944 + }, + { + "epoch": 0.93, + "learning_rate": 3.570712136409228e-05, + "loss": 1.3917, + "theoretical_loss": 3.3151022473813034, + "tokens_seen": 3067478016 + }, + { + "epoch": 0.93, + "learning_rate": 3.5687061183550657e-05, + "loss": 1.2417, + "theoretical_loss": 3.3150914056031433, + "tokens_seen": 3067609088 + }, + { + "epoch": 0.93, + "learning_rate": 3.5667001003009026e-05, + "loss": 1.3334, + "theoretical_loss": 3.31508056441792, + "tokens_seen": 3067740160 + }, + { + "epoch": 0.93, + "learning_rate": 3.56469408224674e-05, + "loss": 1.2787, + "theoretical_loss": 3.315069723825576, + "tokens_seen": 3067871232 + }, + { + "epoch": 0.93, + "learning_rate": 3.562688064192578e-05, + "loss": 1.2086, + "theoretical_loss": 3.3150588838260537, + "tokens_seen": 3068002304 + }, + { + "epoch": 0.93, + "learning_rate": 3.560682046138415e-05, + "loss": 1.2592, + "theoretical_loss": 3.315048044419295, + "tokens_seen": 3068133376 + }, + { + "epoch": 0.93, + "learning_rate": 3.558676028084253e-05, + "loss": 1.2125, + "theoretical_loss": 3.315037205605242, + "tokens_seen": 3068264448 + }, + { + "epoch": 0.93, + "learning_rate": 3.5566700100300904e-05, + "loss": 1.316, + "theoretical_loss": 3.315026367383837, + "tokens_seen": 3068395520 + }, + { + "epoch": 0.93, + "learning_rate": 3.554663991975928e-05, + "loss": 1.3312, + "theoretical_loss": 3.315015529755023, + "tokens_seen": 3068526592 + }, + { + "epoch": 0.93, + "learning_rate": 3.5526579739217656e-05, + "loss": 1.2289, + "theoretical_loss": 3.315004692718742, + "tokens_seen": 3068657664 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.43285679817199707, + "objective/train/avg_token_score": 0.01323665864765644, + "objective/train/docs_used": 1726520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2593059539794922, + "objective/train/original_loss": 3.566624164581299, + "objective/train/theoretical_loss": 3.314999274422783, + "objective/train/tokens_used": 3068723200, + "objective/train/unlikelihood_loss": 0.16225460171699524, + "theoretical_loss": 3.314999274422783, + "tokens_seen": 3068723200 + }, + { + "epoch": 0.93, + "learning_rate": 3.550651955867603e-05, + "loss": 1.3251, + "theoretical_loss": 3.3149938562749357, + "tokens_seen": 3068788736 + }, + { + "epoch": 0.93, + "learning_rate": 3.548645937813441e-05, + "loss": 1.2575, + "theoretical_loss": 3.3149830204235466, + "tokens_seen": 3068919808 + }, + { + "epoch": 0.93, + "learning_rate": 3.546639919759278e-05, + "loss": 1.3417, + "theoretical_loss": 3.314972185164518, + "tokens_seen": 3069050880 + }, + { + "epoch": 0.93, + "learning_rate": 3.544633901705115e-05, + "loss": 1.2971, + "theoretical_loss": 3.314961350497791, + "tokens_seen": 3069181952 + }, + { + "epoch": 0.93, + "learning_rate": 3.542627883650953e-05, + "loss": 1.4245, + "theoretical_loss": 3.314950516423308, + "tokens_seen": 3069313024 + }, + { + "epoch": 0.93, + "learning_rate": 3.5406218655967904e-05, + "loss": 1.2307, + "theoretical_loss": 3.314939682941012, + "tokens_seen": 3069444096 + }, + { + "epoch": 0.93, + "learning_rate": 3.538615847542627e-05, + "loss": 1.1993, + "theoretical_loss": 3.3149288500508454, + "tokens_seen": 3069575168 + }, + { + "epoch": 0.93, + "learning_rate": 3.5366098294884656e-05, + "loss": 1.4158, + "theoretical_loss": 3.31491801775275, + "tokens_seen": 3069706240 + }, + { + "epoch": 0.93, + "learning_rate": 3.534603811434303e-05, + "loss": 1.4388, + "theoretical_loss": 3.3149071860466686, + "tokens_seen": 3069837312 + }, + { + "epoch": 0.93, + "learning_rate": 3.532597793380141e-05, + "loss": 1.2993, + "theoretical_loss": 3.314896354932543, + "tokens_seen": 3069968384 + }, + { + "epoch": 0.93, + "learning_rate": 3.530591775325978e-05, + "loss": 1.3013, + "theoretical_loss": 3.314885524410316, + "tokens_seen": 3070099456 + }, + { + "epoch": 0.93, + "learning_rate": 3.528585757271815e-05, + "loss": 1.3653, + "theoretical_loss": 3.31487469447993, + "tokens_seen": 3070230528 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.4199046790599823, + "objective/train/avg_token_score": 0.010247910395264626, + "objective/train/docs_used": 1727712, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1506016254425049, + "objective/train/original_loss": 3.8393826484680176, + "objective/train/theoretical_loss": 3.3148638651413274, + "objective/train/tokens_used": 3070361600, + "objective/train/unlikelihood_loss": 0.19275201857089996, + "theoretical_loss": 3.3148638651413274, + "tokens_seen": 3070361600 + }, + { + "epoch": 0.93, + "learning_rate": 3.526579739217653e-05, + "loss": 1.3245, + "theoretical_loss": 3.3148638651413274, + "tokens_seen": 3070361600 + }, + { + "epoch": 0.93, + "learning_rate": 3.5245737211634904e-05, + "loss": 1.2512, + "theoretical_loss": 3.31485303639445, + "tokens_seen": 3070492672 + }, + { + "epoch": 0.93, + "learning_rate": 3.522567703109328e-05, + "loss": 1.3836, + "theoretical_loss": 3.314842208239241, + "tokens_seen": 3070623744 + }, + { + "epoch": 0.93, + "learning_rate": 3.5205616850551655e-05, + "loss": 1.5179, + "theoretical_loss": 3.314831380675643, + "tokens_seen": 3070754816 + }, + { + "epoch": 0.93, + "learning_rate": 3.518555667001003e-05, + "loss": 1.425, + "theoretical_loss": 3.3148205537035973, + "tokens_seen": 3070885888 + }, + { + "epoch": 0.93, + "learning_rate": 3.516549648946841e-05, + "loss": 1.2921, + "theoretical_loss": 3.3148097273230475, + "tokens_seen": 3071016960 + }, + { + "epoch": 0.93, + "learning_rate": 3.514543630892678e-05, + "loss": 1.3606, + "theoretical_loss": 3.314798901533935, + "tokens_seen": 3071148032 + }, + { + "epoch": 0.93, + "learning_rate": 3.512537612838516e-05, + "loss": 1.2768, + "theoretical_loss": 3.314788076336203, + "tokens_seen": 3071279104 + }, + { + "epoch": 0.93, + "learning_rate": 3.5105315947843534e-05, + "loss": 1.1973, + "theoretical_loss": 3.314777251729794, + "tokens_seen": 3071410176 + }, + { + "epoch": 0.93, + "learning_rate": 3.50852557673019e-05, + "loss": 1.319, + "theoretical_loss": 3.31476642771465, + "tokens_seen": 3071541248 + }, + { + "epoch": 0.93, + "learning_rate": 3.506519558676028e-05, + "loss": 1.158, + "theoretical_loss": 3.314755604290714, + "tokens_seen": 3071672320 + }, + { + "epoch": 0.93, + "learning_rate": 3.5045135406218655e-05, + "loss": 1.2383, + "theoretical_loss": 3.314744781457928, + "tokens_seen": 3071803392 + }, + { + "epoch": 0.93, + "learning_rate": 3.502507522567703e-05, + "loss": 1.4116, + "theoretical_loss": 3.3147339592162344, + "tokens_seen": 3071934464 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.46664223074913025, + "objective/train/avg_token_score": 0.004469544161111116, + "objective/train/docs_used": 1728325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5256932973861694, + "objective/train/original_loss": 3.4253673553466797, + "objective/train/theoretical_loss": 3.3147285483170297, + "objective/train/tokens_used": 3072000000, + "objective/train/unlikelihood_loss": 0.165263369679451, + "theoretical_loss": 3.3147285483170297, + "tokens_seen": 3072000000 + }, + { + "epoch": 0.93, + "learning_rate": 3.5005015045135406e-05, + "loss": 1.2408, + "theoretical_loss": 3.314723137565576, + "tokens_seen": 3072065536 + }, + { + "epoch": 0.93, + "learning_rate": 3.498495486459378e-05, + "loss": 1.34, + "theoretical_loss": 3.3147123165058954, + "tokens_seen": 3072196608 + }, + { + "epoch": 0.93, + "learning_rate": 3.496489468405216e-05, + "loss": 1.378, + "theoretical_loss": 3.3147014960371353, + "tokens_seen": 3072327680 + }, + { + "epoch": 0.93, + "learning_rate": 3.4944834503510534e-05, + "loss": 1.4127, + "theoretical_loss": 3.3146906761592376, + "tokens_seen": 3072458752 + }, + { + "epoch": 0.93, + "learning_rate": 3.492477432296891e-05, + "loss": 1.2956, + "theoretical_loss": 3.314679856872145, + "tokens_seen": 3072589824 + }, + { + "epoch": 0.93, + "learning_rate": 3.4904714142427285e-05, + "loss": 1.2555, + "theoretical_loss": 3.3146690381758006, + "tokens_seen": 3072720896 + }, + { + "epoch": 0.93, + "learning_rate": 3.4884653961885654e-05, + "loss": 1.3113, + "theoretical_loss": 3.314658220070146, + "tokens_seen": 3072851968 + }, + { + "epoch": 0.93, + "learning_rate": 3.486459378134403e-05, + "loss": 1.2192, + "theoretical_loss": 3.3146474025551247, + "tokens_seen": 3072983040 + }, + { + "epoch": 0.93, + "learning_rate": 3.4844533600802406e-05, + "loss": 1.3415, + "theoretical_loss": 3.3146365856306783, + "tokens_seen": 3073114112 + }, + { + "epoch": 0.93, + "learning_rate": 3.482447342026078e-05, + "loss": 1.2247, + "theoretical_loss": 3.3146257692967507, + "tokens_seen": 3073245184 + }, + { + "epoch": 0.93, + "learning_rate": 3.4804413239719164e-05, + "loss": 1.3867, + "theoretical_loss": 3.314614953553283, + "tokens_seen": 3073376256 + }, + { + "epoch": 0.93, + "learning_rate": 3.478435305917753e-05, + "loss": 1.3114, + "theoretical_loss": 3.314604138400219, + "tokens_seen": 3073507328 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.3853861093521118, + "objective/train/avg_token_score": 0.017814919352531433, + "objective/train/docs_used": 1729584, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1569480895996094, + "objective/train/original_loss": 3.783881187438965, + "objective/train/theoretical_loss": 3.3145933238375003, + "objective/train/tokens_used": 3073638400, + "objective/train/unlikelihood_loss": 0.15111027657985687, + "theoretical_loss": 3.3145933238375003, + "tokens_seen": 3073638400 + }, + { + "epoch": 0.93, + "learning_rate": 3.476429287863591e-05, + "loss": 1.1976, + "theoretical_loss": 3.3145933238375003, + "tokens_seen": 3073638400 + }, + { + "epoch": 0.93, + "learning_rate": 3.4744232698094285e-05, + "loss": 1.2186, + "theoretical_loss": 3.3145825098650703, + "tokens_seen": 3073769472 + }, + { + "epoch": 0.93, + "learning_rate": 3.472417251755266e-05, + "loss": 1.249, + "theoretical_loss": 3.3145716964828713, + "tokens_seen": 3073900544 + }, + { + "epoch": 0.93, + "learning_rate": 3.4704112337011036e-05, + "loss": 1.2781, + "theoretical_loss": 3.314560883690846, + "tokens_seen": 3074031616 + }, + { + "epoch": 0.93, + "learning_rate": 3.4684052156469406e-05, + "loss": 1.2756, + "theoretical_loss": 3.3145500714889367, + "tokens_seen": 3074162688 + }, + { + "epoch": 0.93, + "learning_rate": 3.466399197592778e-05, + "loss": 1.1936, + "theoretical_loss": 3.3145392598770864, + "tokens_seen": 3074293760 + }, + { + "epoch": 0.93, + "learning_rate": 3.464393179538616e-05, + "loss": 1.1676, + "theoretical_loss": 3.3145284488552376, + "tokens_seen": 3074424832 + }, + { + "epoch": 0.93, + "learning_rate": 3.462387161484454e-05, + "loss": 1.3238, + "theoretical_loss": 3.314517638423333, + "tokens_seen": 3074555904 + }, + { + "epoch": 0.93, + "learning_rate": 3.460381143430291e-05, + "loss": 1.1917, + "theoretical_loss": 3.314506828581315, + "tokens_seen": 3074686976 + }, + { + "epoch": 0.93, + "learning_rate": 3.4583751253761284e-05, + "loss": 1.1972, + "theoretical_loss": 3.3144960193291264, + "tokens_seen": 3074818048 + }, + { + "epoch": 0.93, + "learning_rate": 3.456369107321966e-05, + "loss": 1.2877, + "theoretical_loss": 3.31448521066671, + "tokens_seen": 3074949120 + }, + { + "epoch": 0.93, + "learning_rate": 3.4543630892678036e-05, + "loss": 1.2605, + "theoretical_loss": 3.314474402594009, + "tokens_seen": 3075080192 + }, + { + "epoch": 0.93, + "learning_rate": 3.452357071213641e-05, + "loss": 1.337, + "theoretical_loss": 3.3144635951109653, + "tokens_seen": 3075211264 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.429618775844574, + "objective/train/avg_token_score": 0.0026913543697446585, + "objective/train/docs_used": 1730199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1834334135055542, + "objective/train/original_loss": 3.3242790699005127, + "objective/train/theoretical_loss": 3.314458191590547, + "objective/train/tokens_used": 3075276800, + "objective/train/unlikelihood_loss": 0.17779649794101715, + "theoretical_loss": 3.314458191590547, + "tokens_seen": 3075276800 + }, + { + "epoch": 0.93, + "learning_rate": 3.450351053159478e-05, + "loss": 1.3661, + "theoretical_loss": 3.3144527882175217, + "tokens_seen": 3075342336 + }, + { + "epoch": 0.93, + "learning_rate": 3.448345035105316e-05, + "loss": 1.3031, + "theoretical_loss": 3.314441981913621, + "tokens_seen": 3075473408 + }, + { + "epoch": 0.93, + "learning_rate": 3.446339017051153e-05, + "loss": 1.2194, + "theoretical_loss": 3.3144311761992062, + "tokens_seen": 3075604480 + }, + { + "epoch": 0.93, + "learning_rate": 3.444332998996991e-05, + "loss": 1.2711, + "theoretical_loss": 3.3144203710742195, + "tokens_seen": 3075735552 + }, + { + "epoch": 0.93, + "learning_rate": 3.442326980942829e-05, + "loss": 1.2526, + "theoretical_loss": 3.3144095665386044, + "tokens_seen": 3075866624 + }, + { + "epoch": 0.93, + "learning_rate": 3.440320962888666e-05, + "loss": 1.3761, + "theoretical_loss": 3.314398762592303, + "tokens_seen": 3075997696 + }, + { + "epoch": 0.93, + "learning_rate": 3.4383149448345036e-05, + "loss": 1.2568, + "theoretical_loss": 3.314387959235258, + "tokens_seen": 3076128768 + }, + { + "epoch": 0.93, + "learning_rate": 3.436308926780341e-05, + "loss": 1.3504, + "theoretical_loss": 3.3143771564674123, + "tokens_seen": 3076259840 + }, + { + "epoch": 0.93, + "learning_rate": 3.434302908726179e-05, + "loss": 1.1876, + "theoretical_loss": 3.3143663542887087, + "tokens_seen": 3076390912 + }, + { + "epoch": 0.93, + "learning_rate": 3.432296890672016e-05, + "loss": 1.1687, + "theoretical_loss": 3.31435555269909, + "tokens_seen": 3076521984 + }, + { + "epoch": 0.93, + "learning_rate": 3.430290872617853e-05, + "loss": 1.2827, + "theoretical_loss": 3.314344751698499, + "tokens_seen": 3076653056 + }, + { + "epoch": 0.93, + "learning_rate": 3.428284854563691e-05, + "loss": 1.3501, + "theoretical_loss": 3.314333951286879, + "tokens_seen": 3076784128 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.46761974692344666, + "objective/train/avg_token_score": 0.0030554733239114285, + "objective/train/docs_used": 1730859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2483396530151367, + "objective/train/original_loss": 3.234466552734375, + "objective/train/theoretical_loss": 3.314323151464172, + "objective/train/tokens_used": 3076915200, + "objective/train/unlikelihood_loss": 0.22166410088539124, + "theoretical_loss": 3.314323151464172, + "tokens_seen": 3076915200 + }, + { + "epoch": 0.93, + "learning_rate": 3.4262788365095284e-05, + "loss": 1.192, + "theoretical_loss": 3.314323151464172, + "tokens_seen": 3076915200 + }, + { + "epoch": 0.93, + "learning_rate": 3.4242728184553666e-05, + "loss": 1.2914, + "theoretical_loss": 3.3143123522303206, + "tokens_seen": 3077046272 + }, + { + "epoch": 0.93, + "learning_rate": 3.422266800401204e-05, + "loss": 1.0806, + "theoretical_loss": 3.3143015535852687, + "tokens_seen": 3077177344 + }, + { + "epoch": 0.93, + "learning_rate": 3.420260782347041e-05, + "loss": 1.2389, + "theoretical_loss": 3.3142907555289582, + "tokens_seen": 3077308416 + }, + { + "epoch": 0.93, + "learning_rate": 3.418254764292879e-05, + "loss": 1.3445, + "theoretical_loss": 3.3142799580613325, + "tokens_seen": 3077439488 + }, + { + "epoch": 0.93, + "learning_rate": 3.416248746238716e-05, + "loss": 1.1788, + "theoretical_loss": 3.3142691611823336, + "tokens_seen": 3077570560 + }, + { + "epoch": 0.93, + "learning_rate": 3.414242728184554e-05, + "loss": 1.3905, + "theoretical_loss": 3.3142583648919053, + "tokens_seen": 3077701632 + }, + { + "epoch": 0.93, + "learning_rate": 3.4122367101303914e-05, + "loss": 1.2385, + "theoretical_loss": 3.31424756918999, + "tokens_seen": 3077832704 + }, + { + "epoch": 0.93, + "learning_rate": 3.410230692076228e-05, + "loss": 1.3322, + "theoretical_loss": 3.3142367740765306, + "tokens_seen": 3077963776 + }, + { + "epoch": 0.93, + "learning_rate": 3.408224674022066e-05, + "loss": 1.3027, + "theoretical_loss": 3.31422597955147, + "tokens_seen": 3078094848 + }, + { + "epoch": 0.93, + "learning_rate": 3.406218655967904e-05, + "loss": 1.4196, + "theoretical_loss": 3.314215185614751, + "tokens_seen": 3078225920 + }, + { + "epoch": 0.93, + "learning_rate": 3.404212637913742e-05, + "loss": 1.1341, + "theoretical_loss": 3.3142043922663165, + "tokens_seen": 3078356992 + }, + { + "epoch": 0.93, + "learning_rate": 3.402206619859579e-05, + "loss": 1.4818, + "theoretical_loss": 3.3141935995061096, + "tokens_seen": 3078488064 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.4278470277786255, + "objective/train/avg_token_score": 0.006062714383006096, + "objective/train/docs_used": 1732124, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.285976529121399, + "objective/train/original_loss": 3.8018903732299805, + "objective/train/theoretical_loss": 3.314188203346574, + "objective/train/tokens_used": 3078553600, + "objective/train/unlikelihood_loss": 0.11319137364625931, + "theoretical_loss": 3.314188203346574, + "tokens_seen": 3078553600 + }, + { + "epoch": 0.93, + "learning_rate": 3.400200601805416e-05, + "loss": 1.2997, + "theoretical_loss": 3.314182807334073, + "tokens_seen": 3078619136 + }, + { + "epoch": 0.93, + "learning_rate": 3.398194583751254e-05, + "loss": 1.2689, + "theoretical_loss": 3.3141720157501497, + "tokens_seen": 3078750208 + }, + { + "epoch": 0.93, + "learning_rate": 3.3961885656970914e-05, + "loss": 1.3668, + "theoretical_loss": 3.314161224754282, + "tokens_seen": 3078881280 + }, + { + "epoch": 0.93, + "learning_rate": 3.394182547642929e-05, + "loss": 1.3251, + "theoretical_loss": 3.314150434346414, + "tokens_seen": 3079012352 + }, + { + "epoch": 0.93, + "learning_rate": 3.392176529588766e-05, + "loss": 1.258, + "theoretical_loss": 3.314139644526488, + "tokens_seen": 3079143424 + }, + { + "epoch": 0.93, + "learning_rate": 3.3901705115346034e-05, + "loss": 1.288, + "theoretical_loss": 3.314128855294447, + "tokens_seen": 3079274496 + }, + { + "epoch": 0.93, + "learning_rate": 3.388164493480441e-05, + "loss": 1.4194, + "theoretical_loss": 3.3141180666502335, + "tokens_seen": 3079405568 + }, + { + "epoch": 0.93, + "learning_rate": 3.386158475426279e-05, + "loss": 1.2244, + "theoretical_loss": 3.314107278593791, + "tokens_seen": 3079536640 + }, + { + "epoch": 0.93, + "learning_rate": 3.384152457372117e-05, + "loss": 1.198, + "theoretical_loss": 3.3140964911250625, + "tokens_seen": 3079667712 + }, + { + "epoch": 0.93, + "learning_rate": 3.382146439317954e-05, + "loss": 1.2078, + "theoretical_loss": 3.3140857042439906, + "tokens_seen": 3079798784 + }, + { + "epoch": 0.93, + "learning_rate": 3.380140421263791e-05, + "loss": 1.245, + "theoretical_loss": 3.314074917950519, + "tokens_seen": 3079929856 + }, + { + "epoch": 0.93, + "learning_rate": 3.378134403209629e-05, + "loss": 1.4205, + "theoretical_loss": 3.314064132244589, + "tokens_seen": 3080060928 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.39393940567970276, + "objective/train/avg_token_score": 0.01627458818256855, + "objective/train/docs_used": 1732703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.113985300064087, + "objective/train/original_loss": 3.851499080657959, + "objective/train/theoretical_loss": 3.314053347126146, + "objective/train/tokens_used": 3080192000, + "objective/train/unlikelihood_loss": 0.15091292560100555, + "theoretical_loss": 3.314053347126146, + "tokens_seen": 3080192000 + }, + { + "epoch": 0.93, + "learning_rate": 3.3761283851554665e-05, + "loss": 1.325, + "theoretical_loss": 3.314053347126146, + "tokens_seen": 3080192000 + }, + { + "epoch": 0.93, + "learning_rate": 3.374122367101304e-05, + "loss": 1.3427, + "theoretical_loss": 3.314042562595131, + "tokens_seen": 3080323072 + }, + { + "epoch": 0.93, + "learning_rate": 3.372116349047141e-05, + "loss": 1.2047, + "theoretical_loss": 3.314031778651488, + "tokens_seen": 3080454144 + }, + { + "epoch": 0.93, + "learning_rate": 3.3701103309929785e-05, + "loss": 1.2691, + "theoretical_loss": 3.3140209952951594, + "tokens_seen": 3080585216 + }, + { + "epoch": 0.93, + "learning_rate": 3.368104312938817e-05, + "loss": 1.2887, + "theoretical_loss": 3.314010212526089, + "tokens_seen": 3080716288 + }, + { + "epoch": 0.93, + "learning_rate": 3.3660982948846544e-05, + "loss": 1.2514, + "theoretical_loss": 3.3139994303442197, + "tokens_seen": 3080847360 + }, + { + "epoch": 0.93, + "learning_rate": 3.364092276830492e-05, + "loss": 1.3558, + "theoretical_loss": 3.313988648749494, + "tokens_seen": 3080978432 + }, + { + "epoch": 0.93, + "learning_rate": 3.362086258776329e-05, + "loss": 1.4302, + "theoretical_loss": 3.3139778677418548, + "tokens_seen": 3081109504 + }, + { + "epoch": 0.93, + "learning_rate": 3.3600802407221664e-05, + "loss": 1.2171, + "theoretical_loss": 3.313967087321246, + "tokens_seen": 3081240576 + }, + { + "epoch": 0.93, + "learning_rate": 3.358074222668004e-05, + "loss": 1.2407, + "theoretical_loss": 3.3139563074876106, + "tokens_seen": 3081371648 + }, + { + "epoch": 0.93, + "learning_rate": 3.3560682046138416e-05, + "loss": 1.2601, + "theoretical_loss": 3.3139455282408905, + "tokens_seen": 3081502720 + }, + { + "epoch": 0.93, + "learning_rate": 3.354062186559679e-05, + "loss": 1.4098, + "theoretical_loss": 3.3139347495810303, + "tokens_seen": 3081633792 + }, + { + "epoch": 0.93, + "learning_rate": 3.352056168505516e-05, + "loss": 1.3013, + "theoretical_loss": 3.3139239715079722, + "tokens_seen": 3081764864 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.24792277812957764, + "objective/train/avg_token_score": 0.01787692680954933, + "objective/train/docs_used": 1733946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7905077934265137, + "objective/train/original_loss": 4.442480564117432, + "objective/train/theoretical_loss": 3.313918582691476, + "objective/train/tokens_used": 3081830400, + "objective/train/unlikelihood_loss": 0.17213557660579681, + "theoretical_loss": 3.313918582691476, + "tokens_seen": 3081830400 + }, + { + "epoch": 0.93, + "learning_rate": 3.3500501504513543e-05, + "loss": 1.3375, + "theoretical_loss": 3.3139131940216595, + "tokens_seen": 3081895936 + }, + { + "epoch": 0.93, + "learning_rate": 3.348044132397192e-05, + "loss": 1.4004, + "theoretical_loss": 3.3139024171220353, + "tokens_seen": 3082027008 + }, + { + "epoch": 0.93, + "learning_rate": 3.3460381143430295e-05, + "loss": 1.2053, + "theoretical_loss": 3.3138916408090426, + "tokens_seen": 3082158080 + }, + { + "epoch": 0.93, + "learning_rate": 3.344032096288867e-05, + "loss": 1.3218, + "theoretical_loss": 3.3138808650826252, + "tokens_seen": 3082289152 + }, + { + "epoch": 0.93, + "learning_rate": 3.342026078234704e-05, + "loss": 1.3093, + "theoretical_loss": 3.313870089942725, + "tokens_seen": 3082420224 + }, + { + "epoch": 0.93, + "learning_rate": 3.3400200601805416e-05, + "loss": 1.2502, + "theoretical_loss": 3.313859315389286, + "tokens_seen": 3082551296 + }, + { + "epoch": 0.93, + "learning_rate": 3.338014042126379e-05, + "loss": 1.3237, + "theoretical_loss": 3.313848541422251, + "tokens_seen": 3082682368 + }, + { + "epoch": 0.93, + "learning_rate": 3.336008024072217e-05, + "loss": 1.2508, + "theoretical_loss": 3.313837768041564, + "tokens_seen": 3082813440 + }, + { + "epoch": 0.93, + "learning_rate": 3.334002006018054e-05, + "loss": 1.3537, + "theoretical_loss": 3.313826995247167, + "tokens_seen": 3082944512 + }, + { + "epoch": 0.93, + "learning_rate": 3.331995987963891e-05, + "loss": 1.0622, + "theoretical_loss": 3.313816223039004, + "tokens_seen": 3083075584 + }, + { + "epoch": 0.93, + "learning_rate": 3.3299899699097295e-05, + "loss": 1.298, + "theoretical_loss": 3.3138054514170174, + "tokens_seen": 3083206656 + }, + { + "epoch": 0.93, + "learning_rate": 3.327983951855567e-05, + "loss": 1.2427, + "theoretical_loss": 3.313794680381151, + "tokens_seen": 3083337728 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.33895406126976013, + "objective/train/avg_token_score": 0.014017453417181969, + "objective/train/docs_used": 1734728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8954251408576965, + "objective/train/original_loss": 3.9212803840637207, + "objective/train/theoretical_loss": 3.3137839099313475, + "objective/train/tokens_used": 3083468800, + "objective/train/unlikelihood_loss": 0.11457373946905136, + "theoretical_loss": 3.3137839099313475, + "tokens_seen": 3083468800 + }, + { + "epoch": 0.93, + "learning_rate": 3.3259779338014046e-05, + "loss": 1.118, + "theoretical_loss": 3.3137839099313475, + "tokens_seen": 3083468800 + }, + { + "epoch": 0.93, + "learning_rate": 3.323971915747242e-05, + "loss": 1.2672, + "theoretical_loss": 3.3137731400675507, + "tokens_seen": 3083599872 + }, + { + "epoch": 0.93, + "learning_rate": 3.321965897693079e-05, + "loss": 1.0032, + "theoretical_loss": 3.3137623707897035, + "tokens_seen": 3083730944 + }, + { + "epoch": 0.93, + "learning_rate": 3.319959879638917e-05, + "loss": 1.2763, + "theoretical_loss": 3.3137516020977493, + "tokens_seen": 3083862016 + }, + { + "epoch": 0.93, + "learning_rate": 3.317953861584754e-05, + "loss": 1.3552, + "theoretical_loss": 3.3137408339916306, + "tokens_seen": 3083993088 + }, + { + "epoch": 0.93, + "learning_rate": 3.315947843530592e-05, + "loss": 1.2959, + "theoretical_loss": 3.3137300664712916, + "tokens_seen": 3084124160 + }, + { + "epoch": 0.93, + "learning_rate": 3.313941825476429e-05, + "loss": 1.3279, + "theoretical_loss": 3.3137192995366753, + "tokens_seen": 3084255232 + }, + { + "epoch": 0.93, + "learning_rate": 3.311935807422267e-05, + "loss": 1.3103, + "theoretical_loss": 3.3137085331877243, + "tokens_seen": 3084386304 + }, + { + "epoch": 0.93, + "learning_rate": 3.3099297893681046e-05, + "loss": 1.3104, + "theoretical_loss": 3.313697767424383, + "tokens_seen": 3084517376 + }, + { + "epoch": 0.93, + "learning_rate": 3.307923771313942e-05, + "loss": 1.2194, + "theoretical_loss": 3.3136870022465934, + "tokens_seen": 3084648448 + }, + { + "epoch": 0.93, + "learning_rate": 3.30591775325978e-05, + "loss": 1.3144, + "theoretical_loss": 3.3136762376543, + "tokens_seen": 3084779520 + }, + { + "epoch": 0.93, + "learning_rate": 3.3039117352056166e-05, + "loss": 1.1596, + "theoretical_loss": 3.3136654736474447, + "tokens_seen": 3084910592 + }, + { + "epoch": 0.93, + "learning_rate": 3.301905717151454e-05, + "loss": 1.1591, + "theoretical_loss": 3.313654710225972, + "tokens_seen": 3085041664 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.548081636428833, + "objective/train/avg_token_score": 0.0021550876554101706, + "objective/train/docs_used": 1736039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6204084157943726, + "objective/train/original_loss": 3.2046093940734863, + "objective/train/theoretical_loss": 3.313649328734736, + "objective/train/tokens_used": 3085107200, + "objective/train/unlikelihood_loss": 0.1517690122127533, + "theoretical_loss": 3.313649328734736, + "tokens_seen": 3085107200 + }, + { + "epoch": 0.93, + "learning_rate": 3.299899699097292e-05, + "loss": 1.3097, + "theoretical_loss": 3.3136439473898243, + "tokens_seen": 3085172736 + }, + { + "epoch": 0.93, + "learning_rate": 3.2978936810431294e-05, + "loss": 1.3207, + "theoretical_loss": 3.313633185138946, + "tokens_seen": 3085303808 + }, + { + "epoch": 0.93, + "learning_rate": 3.295887662988967e-05, + "loss": 1.3623, + "theoretical_loss": 3.313622423473279, + "tokens_seen": 3085434880 + }, + { + "epoch": 0.94, + "learning_rate": 3.2938816449348045e-05, + "loss": 1.346, + "theoretical_loss": 3.313611662392768, + "tokens_seen": 3085565952 + }, + { + "epoch": 0.94, + "learning_rate": 3.291875626880642e-05, + "loss": 1.3595, + "theoretical_loss": 3.3136009018973556, + "tokens_seen": 3085697024 + }, + { + "epoch": 0.94, + "learning_rate": 3.28986960882648e-05, + "loss": 1.2987, + "theoretical_loss": 3.313590141986985, + "tokens_seen": 3085828096 + }, + { + "epoch": 0.94, + "learning_rate": 3.287863590772317e-05, + "loss": 1.3155, + "theoretical_loss": 3.3135793826616, + "tokens_seen": 3085959168 + }, + { + "epoch": 0.94, + "learning_rate": 3.285857572718155e-05, + "loss": 1.3426, + "theoretical_loss": 3.3135686239211437, + "tokens_seen": 3086090240 + }, + { + "epoch": 0.94, + "learning_rate": 3.283851554663992e-05, + "loss": 1.2394, + "theoretical_loss": 3.3135578657655596, + "tokens_seen": 3086221312 + }, + { + "epoch": 0.94, + "learning_rate": 3.281845536609829e-05, + "loss": 1.3268, + "theoretical_loss": 3.3135471081947907, + "tokens_seen": 3086352384 + }, + { + "epoch": 0.94, + "learning_rate": 3.279839518555667e-05, + "loss": 1.2134, + "theoretical_loss": 3.3135363512087808, + "tokens_seen": 3086483456 + }, + { + "epoch": 0.94, + "learning_rate": 3.2778335005015045e-05, + "loss": 1.2863, + "theoretical_loss": 3.313525594807473, + "tokens_seen": 3086614528 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.3734726309776306, + "objective/train/avg_token_score": 0.0249040424823761, + "objective/train/docs_used": 1736664, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0932576656341553, + "objective/train/original_loss": 3.526186943054199, + "objective/train/theoretical_loss": 3.3135148389908107, + "objective/train/tokens_used": 3086745600, + "objective/train/unlikelihood_loss": 0.16858094930648804, + "theoretical_loss": 3.3135148389908107, + "tokens_seen": 3086745600 + }, + { + "epoch": 0.94, + "learning_rate": 3.275827482447342e-05, + "loss": 1.225, + "theoretical_loss": 3.3135148389908107, + "tokens_seen": 3086745600 + }, + { + "epoch": 0.94, + "learning_rate": 3.2738214643931796e-05, + "loss": 1.2256, + "theoretical_loss": 3.3135040837587377, + "tokens_seen": 3086876672 + }, + { + "epoch": 0.94, + "learning_rate": 3.271815446339017e-05, + "loss": 1.2906, + "theoretical_loss": 3.313493329111197, + "tokens_seen": 3087007744 + }, + { + "epoch": 0.94, + "learning_rate": 3.269809428284855e-05, + "loss": 1.4157, + "theoretical_loss": 3.3134825750481323, + "tokens_seen": 3087138816 + }, + { + "epoch": 0.94, + "learning_rate": 3.2678034102306924e-05, + "loss": 1.2614, + "theoretical_loss": 3.3134718215694865, + "tokens_seen": 3087269888 + }, + { + "epoch": 0.94, + "learning_rate": 3.26579739217653e-05, + "loss": 1.2845, + "theoretical_loss": 3.3134610686752035, + "tokens_seen": 3087400960 + }, + { + "epoch": 0.94, + "learning_rate": 3.263791374122367e-05, + "loss": 1.3623, + "theoretical_loss": 3.313450316365227, + "tokens_seen": 3087532032 + }, + { + "epoch": 0.94, + "learning_rate": 3.2617853560682044e-05, + "loss": 1.1238, + "theoretical_loss": 3.3134395646394994, + "tokens_seen": 3087663104 + }, + { + "epoch": 0.94, + "learning_rate": 3.259779338014042e-05, + "loss": 1.2465, + "theoretical_loss": 3.3134288134979655, + "tokens_seen": 3087794176 + }, + { + "epoch": 0.94, + "learning_rate": 3.2577733199598796e-05, + "loss": 1.3536, + "theoretical_loss": 3.313418062940568, + "tokens_seen": 3087925248 + }, + { + "epoch": 0.94, + "learning_rate": 3.255767301905718e-05, + "loss": 1.3001, + "theoretical_loss": 3.31340731296725, + "tokens_seen": 3088056320 + }, + { + "epoch": 0.94, + "learning_rate": 3.253761283851555e-05, + "loss": 1.218, + "theoretical_loss": 3.3133965635779554, + "tokens_seen": 3088187392 + }, + { + "epoch": 0.94, + "learning_rate": 3.2517552657973923e-05, + "loss": 1.1399, + "theoretical_loss": 3.3133858147726283, + "tokens_seen": 3088318464 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.3925953209400177, + "objective/train/avg_token_score": 0.028241345658898354, + "objective/train/docs_used": 1737965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2260733842849731, + "objective/train/original_loss": 3.9560346603393555, + "objective/train/theoretical_loss": 3.313380440588934, + "objective/train/tokens_used": 3088384000, + "objective/train/unlikelihood_loss": 0.13155029714107513, + "theoretical_loss": 3.313380440588934, + "tokens_seen": 3088384000 + }, + { + "epoch": 0.94, + "learning_rate": 3.24974924774323e-05, + "loss": 1.162, + "theoretical_loss": 3.3133750665512114, + "tokens_seen": 3088449536 + }, + { + "epoch": 0.94, + "learning_rate": 3.2477432296890675e-05, + "loss": 1.2919, + "theoretical_loss": 3.3133643189136484, + "tokens_seen": 3088580608 + }, + { + "epoch": 0.94, + "learning_rate": 3.2457372116349044e-05, + "loss": 1.2526, + "theoretical_loss": 3.3133535718598823, + "tokens_seen": 3088711680 + }, + { + "epoch": 0.94, + "learning_rate": 3.243731193580742e-05, + "loss": 1.2215, + "theoretical_loss": 3.313342825389858, + "tokens_seen": 3088842752 + }, + { + "epoch": 0.94, + "learning_rate": 3.2417251755265796e-05, + "loss": 1.2667, + "theoretical_loss": 3.3133320795035175, + "tokens_seen": 3088973824 + }, + { + "epoch": 0.94, + "learning_rate": 3.239719157472417e-05, + "loss": 1.405, + "theoretical_loss": 3.313321334200805, + "tokens_seen": 3089104896 + }, + { + "epoch": 0.94, + "learning_rate": 3.237713139418255e-05, + "loss": 1.1684, + "theoretical_loss": 3.3133105894816643, + "tokens_seen": 3089235968 + }, + { + "epoch": 0.94, + "learning_rate": 3.235707121364092e-05, + "loss": 1.3908, + "theoretical_loss": 3.3132998453460387, + "tokens_seen": 3089367040 + }, + { + "epoch": 0.94, + "learning_rate": 3.23370110330993e-05, + "loss": 1.4308, + "theoretical_loss": 3.3132891017938717, + "tokens_seen": 3089498112 + }, + { + "epoch": 0.94, + "learning_rate": 3.2316950852557675e-05, + "loss": 1.2294, + "theoretical_loss": 3.313278358825107, + "tokens_seen": 3089629184 + }, + { + "epoch": 0.94, + "learning_rate": 3.229689067201605e-05, + "loss": 1.2827, + "theoretical_loss": 3.313267616439688, + "tokens_seen": 3089760256 + }, + { + "epoch": 0.94, + "learning_rate": 3.2276830491474426e-05, + "loss": 1.2165, + "theoretical_loss": 3.3132568746375584, + "tokens_seen": 3089891328 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.4759897291660309, + "objective/train/avg_token_score": 0.017334921285510063, + "objective/train/docs_used": 1738543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3473492860794067, + "objective/train/original_loss": 3.1063318252563477, + "objective/train/theoretical_loss": 3.3132461334186614, + "objective/train/tokens_used": 3090022400, + "objective/train/unlikelihood_loss": 0.26199209690093994, + "theoretical_loss": 3.3132461334186614, + "tokens_seen": 3090022400 + }, + { + "epoch": 0.94, + "learning_rate": 3.2256770310932795e-05, + "loss": 1.3479, + "theoretical_loss": 3.3132461334186614, + "tokens_seen": 3090022400 + }, + { + "epoch": 0.94, + "learning_rate": 3.223671013039117e-05, + "loss": 1.2227, + "theoretical_loss": 3.3132353927829414, + "tokens_seen": 3090153472 + }, + { + "epoch": 0.94, + "learning_rate": 3.221664994984955e-05, + "loss": 1.2916, + "theoretical_loss": 3.3132246527303413, + "tokens_seen": 3090284544 + }, + { + "epoch": 0.94, + "learning_rate": 3.219658976930792e-05, + "loss": 1.2317, + "theoretical_loss": 3.313213913260805, + "tokens_seen": 3090415616 + }, + { + "epoch": 0.94, + "learning_rate": 3.2176529588766305e-05, + "loss": 1.2001, + "theoretical_loss": 3.3132031743742765, + "tokens_seen": 3090546688 + }, + { + "epoch": 0.94, + "learning_rate": 3.2156469408224674e-05, + "loss": 1.1872, + "theoretical_loss": 3.3131924360706986, + "tokens_seen": 3090677760 + }, + { + "epoch": 0.94, + "learning_rate": 3.213640922768305e-05, + "loss": 1.2416, + "theoretical_loss": 3.3131816983500153, + "tokens_seen": 3090808832 + }, + { + "epoch": 0.94, + "learning_rate": 3.2116349047141426e-05, + "loss": 1.1081, + "theoretical_loss": 3.3131709612121703, + "tokens_seen": 3090939904 + }, + { + "epoch": 0.94, + "learning_rate": 3.20962888665998e-05, + "loss": 1.1849, + "theoretical_loss": 3.313160224657107, + "tokens_seen": 3091070976 + }, + { + "epoch": 0.94, + "learning_rate": 3.207622868605818e-05, + "loss": 1.3433, + "theoretical_loss": 3.3131494886847697, + "tokens_seen": 3091202048 + }, + { + "epoch": 0.94, + "learning_rate": 3.2056168505516546e-05, + "loss": 1.3858, + "theoretical_loss": 3.3131387532951013, + "tokens_seen": 3091333120 + }, + { + "epoch": 0.94, + "learning_rate": 3.203610832497492e-05, + "loss": 1.3099, + "theoretical_loss": 3.313128018488046, + "tokens_seen": 3091464192 + }, + { + "epoch": 0.94, + "learning_rate": 3.20160481444333e-05, + "loss": 1.1995, + "theoretical_loss": 3.3131172842635475, + "tokens_seen": 3091595264 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.4114125072956085, + "objective/train/avg_token_score": 0.007580574136227369, + "objective/train/docs_used": 1739219, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2885496616363525, + "objective/train/original_loss": 3.666783332824707, + "objective/train/theoretical_loss": 3.3131119173697394, + "objective/train/tokens_used": 3091660800, + "objective/train/unlikelihood_loss": 0.1768007129430771, + "theoretical_loss": 3.3131119173697394, + "tokens_seen": 3091660800 + }, + { + "epoch": 0.94, + "learning_rate": 3.199598796389168e-05, + "loss": 1.353, + "theoretical_loss": 3.313106550621549, + "tokens_seen": 3091726336 + }, + { + "epoch": 0.94, + "learning_rate": 3.1975927783350056e-05, + "loss": 1.348, + "theoretical_loss": 3.3130958175619947, + "tokens_seen": 3091857408 + }, + { + "epoch": 0.94, + "learning_rate": 3.1955867602808425e-05, + "loss": 1.2681, + "theoretical_loss": 3.313085085084828, + "tokens_seen": 3091988480 + }, + { + "epoch": 0.94, + "learning_rate": 3.19358074222668e-05, + "loss": 1.2646, + "theoretical_loss": 3.3130743531899927, + "tokens_seen": 3092119552 + }, + { + "epoch": 0.94, + "learning_rate": 3.191574724172518e-05, + "loss": 1.4245, + "theoretical_loss": 3.3130636218774328, + "tokens_seen": 3092250624 + }, + { + "epoch": 0.94, + "learning_rate": 3.189568706118355e-05, + "loss": 1.2948, + "theoretical_loss": 3.3130528911470916, + "tokens_seen": 3092381696 + }, + { + "epoch": 0.94, + "learning_rate": 3.187562688064193e-05, + "loss": 1.2785, + "theoretical_loss": 3.3130421609989127, + "tokens_seen": 3092512768 + }, + { + "epoch": 0.94, + "learning_rate": 3.18555667001003e-05, + "loss": 1.2712, + "theoretical_loss": 3.3130314314328406, + "tokens_seen": 3092643840 + }, + { + "epoch": 0.94, + "learning_rate": 3.183550651955867e-05, + "loss": 1.246, + "theoretical_loss": 3.313020702448818, + "tokens_seen": 3092774912 + }, + { + "epoch": 0.94, + "learning_rate": 3.181544633901705e-05, + "loss": 1.3392, + "theoretical_loss": 3.31300997404679, + "tokens_seen": 3092905984 + }, + { + "epoch": 0.94, + "learning_rate": 3.179538615847543e-05, + "loss": 1.2976, + "theoretical_loss": 3.312999246226699, + "tokens_seen": 3093037056 + }, + { + "epoch": 0.94, + "learning_rate": 3.177532597793381e-05, + "loss": 1.3687, + "theoretical_loss": 3.31298851898849, + "tokens_seen": 3093168128 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.3720674514770508, + "objective/train/avg_token_score": 0.026238325983285904, + "objective/train/docs_used": 1739859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0324097871780396, + "objective/train/original_loss": 3.8444571495056152, + "objective/train/theoretical_loss": 3.3129777923321058, + "objective/train/tokens_used": 3093299200, + "objective/train/unlikelihood_loss": 0.16645687818527222, + "theoretical_loss": 3.3129777923321058, + "tokens_seen": 3093299200 + }, + { + "epoch": 0.94, + "learning_rate": 3.1755265797392176e-05, + "loss": 1.2017, + "theoretical_loss": 3.3129777923321058, + "tokens_seen": 3093299200 + }, + { + "epoch": 0.94, + "learning_rate": 3.173520561685055e-05, + "loss": 1.303, + "theoretical_loss": 3.3129670662574906, + "tokens_seen": 3093430272 + }, + { + "epoch": 0.94, + "learning_rate": 3.171514543630893e-05, + "loss": 1.1964, + "theoretical_loss": 3.312956340764588, + "tokens_seen": 3093561344 + }, + { + "epoch": 0.94, + "learning_rate": 3.1695085255767304e-05, + "loss": 1.3905, + "theoretical_loss": 3.3129456158533426, + "tokens_seen": 3093692416 + }, + { + "epoch": 0.94, + "learning_rate": 3.167502507522567e-05, + "loss": 1.2397, + "theoretical_loss": 3.312934891523697, + "tokens_seen": 3093823488 + }, + { + "epoch": 0.94, + "learning_rate": 3.165496489468405e-05, + "loss": 1.3661, + "theoretical_loss": 3.312924167775596, + "tokens_seen": 3093954560 + }, + { + "epoch": 0.94, + "learning_rate": 3.1634904714142424e-05, + "loss": 1.1107, + "theoretical_loss": 3.3129134446089825, + "tokens_seen": 3094085632 + }, + { + "epoch": 0.94, + "learning_rate": 3.161484453360081e-05, + "loss": 1.3048, + "theoretical_loss": 3.3129027220238014, + "tokens_seen": 3094216704 + }, + { + "epoch": 0.94, + "learning_rate": 3.159478435305918e-05, + "loss": 1.2508, + "theoretical_loss": 3.312892000019996, + "tokens_seen": 3094347776 + }, + { + "epoch": 0.94, + "learning_rate": 3.157472417251755e-05, + "loss": 1.1921, + "theoretical_loss": 3.31288127859751, + "tokens_seen": 3094478848 + }, + { + "epoch": 0.94, + "learning_rate": 3.155466399197593e-05, + "loss": 1.4263, + "theoretical_loss": 3.3128705577562876, + "tokens_seen": 3094609920 + }, + { + "epoch": 0.94, + "learning_rate": 3.1534603811434303e-05, + "loss": 1.2844, + "theoretical_loss": 3.3128598374962723, + "tokens_seen": 3094740992 + }, + { + "epoch": 0.94, + "learning_rate": 3.151454363089268e-05, + "loss": 1.339, + "theoretical_loss": 3.312849117817408, + "tokens_seen": 3094872064 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.37438905239105225, + "objective/train/avg_token_score": 0.014798027463257313, + "objective/train/docs_used": 1741115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1371474266052246, + "objective/train/original_loss": 3.8924810886383057, + "objective/train/theoretical_loss": 3.3128437581958905, + "objective/train/tokens_used": 3094937600, + "objective/train/unlikelihood_loss": 0.14895901083946228, + "theoretical_loss": 3.3128437581958905, + "tokens_seen": 3094937600 + }, + { + "epoch": 0.94, + "learning_rate": 3.1494483450351055e-05, + "loss": 1.2403, + "theoretical_loss": 3.3128383987196397, + "tokens_seen": 3095003136 + }, + { + "epoch": 0.94, + "learning_rate": 3.1474423269809424e-05, + "loss": 1.1867, + "theoretical_loss": 3.3128276802029095, + "tokens_seen": 3095134208 + }, + { + "epoch": 0.94, + "learning_rate": 3.14543630892678e-05, + "loss": 1.3042, + "theoretical_loss": 3.3128169622671626, + "tokens_seen": 3095265280 + }, + { + "epoch": 0.94, + "learning_rate": 3.143430290872618e-05, + "loss": 1.4162, + "theoretical_loss": 3.312806244912342, + "tokens_seen": 3095396352 + }, + { + "epoch": 0.94, + "learning_rate": 3.141424272818456e-05, + "loss": 1.3032, + "theoretical_loss": 3.312795528138393, + "tokens_seen": 3095527424 + }, + { + "epoch": 0.94, + "learning_rate": 3.1394182547642934e-05, + "loss": 1.2032, + "theoretical_loss": 3.312784811945258, + "tokens_seen": 3095658496 + }, + { + "epoch": 0.94, + "learning_rate": 3.13741223671013e-05, + "loss": 1.2768, + "theoretical_loss": 3.3127740963328813, + "tokens_seen": 3095789568 + }, + { + "epoch": 0.94, + "learning_rate": 3.135406218655968e-05, + "loss": 1.2772, + "theoretical_loss": 3.3127633813012074, + "tokens_seen": 3095920640 + }, + { + "epoch": 0.94, + "learning_rate": 3.1334002006018055e-05, + "loss": 1.3538, + "theoretical_loss": 3.31275266685018, + "tokens_seen": 3096051712 + }, + { + "epoch": 0.94, + "learning_rate": 3.131394182547643e-05, + "loss": 1.3629, + "theoretical_loss": 3.312741952979743, + "tokens_seen": 3096182784 + }, + { + "epoch": 0.94, + "learning_rate": 3.1293881644934806e-05, + "loss": 1.2161, + "theoretical_loss": 3.3127312396898403, + "tokens_seen": 3096313856 + }, + { + "epoch": 0.94, + "learning_rate": 3.1273821464393175e-05, + "loss": 1.2455, + "theoretical_loss": 3.312720526980416, + "tokens_seen": 3096444928 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.4919965863227844, + "objective/train/avg_token_score": 0.01394337136298418, + "objective/train/docs_used": 1741744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4911632537841797, + "objective/train/original_loss": 3.3799147605895996, + "objective/train/theoretical_loss": 3.312709814851414, + "objective/train/tokens_used": 3096576000, + "objective/train/unlikelihood_loss": 0.17638273537158966, + "theoretical_loss": 3.312709814851414, + "tokens_seen": 3096576000 + }, + { + "epoch": 0.94, + "learning_rate": 3.125376128385155e-05, + "loss": 1.3673, + "theoretical_loss": 3.312709814851414, + "tokens_seen": 3096576000 + }, + { + "epoch": 0.94, + "learning_rate": 3.123370110330993e-05, + "loss": 1.2996, + "theoretical_loss": 3.312699103302778, + "tokens_seen": 3096707072 + }, + { + "epoch": 0.94, + "learning_rate": 3.12136409227683e-05, + "loss": 1.2769, + "theoretical_loss": 3.3126883923344526, + "tokens_seen": 3096838144 + }, + { + "epoch": 0.94, + "learning_rate": 3.1193580742226685e-05, + "loss": 1.2214, + "theoretical_loss": 3.3126776819463815, + "tokens_seen": 3096969216 + }, + { + "epoch": 0.94, + "learning_rate": 3.1173520561685054e-05, + "loss": 1.358, + "theoretical_loss": 3.3126669721385085, + "tokens_seen": 3097100288 + }, + { + "epoch": 0.94, + "learning_rate": 3.115346038114343e-05, + "loss": 1.3054, + "theoretical_loss": 3.312656262910778, + "tokens_seen": 3097231360 + }, + { + "epoch": 0.94, + "learning_rate": 3.1133400200601806e-05, + "loss": 1.2828, + "theoretical_loss": 3.312645554263134, + "tokens_seen": 3097362432 + }, + { + "epoch": 0.94, + "learning_rate": 3.111334002006018e-05, + "loss": 1.2705, + "theoretical_loss": 3.31263484619552, + "tokens_seen": 3097493504 + }, + { + "epoch": 0.94, + "learning_rate": 3.109327983951856e-05, + "loss": 1.2244, + "theoretical_loss": 3.3126241387078803, + "tokens_seen": 3097624576 + }, + { + "epoch": 0.94, + "learning_rate": 3.107321965897693e-05, + "loss": 1.3834, + "theoretical_loss": 3.3126134318001594, + "tokens_seen": 3097755648 + }, + { + "epoch": 0.94, + "learning_rate": 3.105315947843531e-05, + "loss": 1.285, + "theoretical_loss": 3.312602725472301, + "tokens_seen": 3097886720 + }, + { + "epoch": 0.94, + "learning_rate": 3.103309929789368e-05, + "loss": 1.3934, + "theoretical_loss": 3.3125920197242493, + "tokens_seen": 3098017792 + }, + { + "epoch": 0.94, + "learning_rate": 3.101303911735206e-05, + "loss": 1.2955, + "theoretical_loss": 3.312581314555948, + "tokens_seen": 3098148864 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.428519070148468, + "objective/train/avg_token_score": 0.008675279095768929, + "objective/train/docs_used": 1742265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3498865365982056, + "objective/train/original_loss": 3.6586172580718994, + "objective/train/theoretical_loss": 3.3125759621891864, + "objective/train/tokens_used": 3098214400, + "objective/train/unlikelihood_loss": 0.18009120225906372, + "theoretical_loss": 3.3125759621891864, + "tokens_seen": 3098214400 + }, + { + "epoch": 0.94, + "learning_rate": 3.099297893681043e-05, + "loss": 1.1761, + "theoretical_loss": 3.3125706099673415, + "tokens_seen": 3098279936 + }, + { + "epoch": 0.94, + "learning_rate": 3.0972918756268805e-05, + "loss": 1.1619, + "theoretical_loss": 3.312559905958374, + "tokens_seen": 3098411008 + }, + { + "epoch": 0.94, + "learning_rate": 3.095285857572718e-05, + "loss": 1.2353, + "theoretical_loss": 3.3125492025289893, + "tokens_seen": 3098542080 + }, + { + "epoch": 0.94, + "learning_rate": 3.093279839518556e-05, + "loss": 1.0896, + "theoretical_loss": 3.3125384996791314, + "tokens_seen": 3098673152 + }, + { + "epoch": 0.94, + "learning_rate": 3.091273821464393e-05, + "loss": 1.3563, + "theoretical_loss": 3.3125277974087446, + "tokens_seen": 3098804224 + }, + { + "epoch": 0.94, + "learning_rate": 3.089267803410231e-05, + "loss": 1.3369, + "theoretical_loss": 3.3125170957177734, + "tokens_seen": 3098935296 + }, + { + "epoch": 0.94, + "learning_rate": 3.0872617853560684e-05, + "loss": 1.1495, + "theoretical_loss": 3.3125063946061615, + "tokens_seen": 3099066368 + }, + { + "epoch": 0.94, + "learning_rate": 3.085255767301906e-05, + "loss": 1.3082, + "theoretical_loss": 3.3124956940738532, + "tokens_seen": 3099197440 + }, + { + "epoch": 0.94, + "learning_rate": 3.083249749247743e-05, + "loss": 1.2467, + "theoretical_loss": 3.3124849941207923, + "tokens_seen": 3099328512 + }, + { + "epoch": 0.94, + "learning_rate": 3.081243731193581e-05, + "loss": 1.2821, + "theoretical_loss": 3.3124742947469232, + "tokens_seen": 3099459584 + }, + { + "epoch": 0.94, + "learning_rate": 3.079237713139418e-05, + "loss": 1.3662, + "theoretical_loss": 3.31246359595219, + "tokens_seen": 3099590656 + }, + { + "epoch": 0.94, + "learning_rate": 3.0772316950852556e-05, + "loss": 1.2873, + "theoretical_loss": 3.312452897736537, + "tokens_seen": 3099721728 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.5181451439857483, + "objective/train/avg_token_score": 0.005962103605270386, + "objective/train/docs_used": 1743460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4834048748016357, + "objective/train/original_loss": 3.659877061843872, + "objective/train/theoretical_loss": 3.3124422000999085, + "objective/train/tokens_used": 3099852800, + "objective/train/unlikelihood_loss": 0.160153329372406, + "theoretical_loss": 3.3124422000999085, + "tokens_seen": 3099852800 + }, + { + "epoch": 0.94, + "learning_rate": 3.075225677031093e-05, + "loss": 1.2333, + "theoretical_loss": 3.3124422000999085, + "tokens_seen": 3099852800 + }, + { + "epoch": 0.94, + "learning_rate": 3.073219658976931e-05, + "loss": 1.3913, + "theoretical_loss": 3.3124315030422484, + "tokens_seen": 3099983872 + }, + { + "epoch": 0.94, + "learning_rate": 3.0712136409227684e-05, + "loss": 1.3212, + "theoretical_loss": 3.312420806563501, + "tokens_seen": 3100114944 + }, + { + "epoch": 0.94, + "learning_rate": 3.069207622868606e-05, + "loss": 1.2802, + "theoretical_loss": 3.31241011066361, + "tokens_seen": 3100246016 + }, + { + "epoch": 0.94, + "learning_rate": 3.0672016048144435e-05, + "loss": 1.3061, + "theoretical_loss": 3.3123994153425205, + "tokens_seen": 3100377088 + }, + { + "epoch": 0.94, + "learning_rate": 3.0651955867602804e-05, + "loss": 1.3987, + "theoretical_loss": 3.3123887206001763, + "tokens_seen": 3100508160 + }, + { + "epoch": 0.94, + "learning_rate": 3.063189568706119e-05, + "loss": 1.2209, + "theoretical_loss": 3.3123780264365212, + "tokens_seen": 3100639232 + }, + { + "epoch": 0.94, + "learning_rate": 3.061183550651956e-05, + "loss": 1.2864, + "theoretical_loss": 3.3123673328515, + "tokens_seen": 3100770304 + }, + { + "epoch": 0.94, + "learning_rate": 3.059177532597793e-05, + "loss": 1.2466, + "theoretical_loss": 3.3123566398450572, + "tokens_seen": 3100901376 + }, + { + "epoch": 0.94, + "learning_rate": 3.057171514543631e-05, + "loss": 1.4749, + "theoretical_loss": 3.312345947417136, + "tokens_seen": 3101032448 + }, + { + "epoch": 0.94, + "learning_rate": 3.055165496489468e-05, + "loss": 1.3994, + "theoretical_loss": 3.3123352555676813, + "tokens_seen": 3101163520 + }, + { + "epoch": 0.94, + "learning_rate": 3.053159478435306e-05, + "loss": 1.241, + "theoretical_loss": 3.3123245642966377, + "tokens_seen": 3101294592 + }, + { + "epoch": 0.94, + "learning_rate": 3.0511534603811435e-05, + "loss": 1.195, + "theoretical_loss": 3.312313873603949, + "tokens_seen": 3101425664 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.34292522072792053, + "objective/train/avg_token_score": 0.024552121758461, + "objective/train/docs_used": 1744846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0607516765594482, + "objective/train/original_loss": 4.288986682891846, + "objective/train/theoretical_loss": 3.3123085284744698, + "objective/train/tokens_used": 3101491200, + "objective/train/unlikelihood_loss": 0.16645298898220062, + "theoretical_loss": 3.3123085284744698, + "tokens_seen": 3101491200 + }, + { + "epoch": 0.94, + "learning_rate": 3.049147442326981e-05, + "loss": 1.298, + "theoretical_loss": 3.312303183489559, + "tokens_seen": 3101556736 + }, + { + "epoch": 0.94, + "learning_rate": 3.0471414242728183e-05, + "loss": 1.2266, + "theoretical_loss": 3.3122924939534126, + "tokens_seen": 3101687808 + }, + { + "epoch": 0.94, + "learning_rate": 3.0451354062186562e-05, + "loss": 1.2621, + "theoretical_loss": 3.3122818049954543, + "tokens_seen": 3101818880 + }, + { + "epoch": 0.94, + "learning_rate": 3.0431293881644935e-05, + "loss": 1.2649, + "theoretical_loss": 3.3122711166156282, + "tokens_seen": 3101949952 + }, + { + "epoch": 0.94, + "learning_rate": 3.041123370110331e-05, + "loss": 1.2573, + "theoretical_loss": 3.3122604288138784, + "tokens_seen": 3102081024 + }, + { + "epoch": 0.94, + "learning_rate": 3.0391173520561686e-05, + "loss": 1.3104, + "theoretical_loss": 3.3122497415901493, + "tokens_seen": 3102212096 + }, + { + "epoch": 0.94, + "learning_rate": 3.037111334002006e-05, + "loss": 1.2674, + "theoretical_loss": 3.312239054944385, + "tokens_seen": 3102343168 + }, + { + "epoch": 0.94, + "learning_rate": 3.0351053159478438e-05, + "loss": 1.4155, + "theoretical_loss": 3.3122283688765304, + "tokens_seen": 3102474240 + }, + { + "epoch": 0.94, + "learning_rate": 3.033099297893681e-05, + "loss": 1.2584, + "theoretical_loss": 3.3122176833865296, + "tokens_seen": 3102605312 + }, + { + "epoch": 0.94, + "learning_rate": 3.0310932798395186e-05, + "loss": 1.3983, + "theoretical_loss": 3.3122069984743265, + "tokens_seen": 3102736384 + }, + { + "epoch": 0.94, + "learning_rate": 3.0290872617853562e-05, + "loss": 1.304, + "theoretical_loss": 3.312196314139866, + "tokens_seen": 3102867456 + }, + { + "epoch": 0.94, + "learning_rate": 3.0270812437311934e-05, + "loss": 1.43, + "theoretical_loss": 3.312185630383092, + "tokens_seen": 3102998528 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.41904935240745544, + "objective/train/avg_token_score": 0.01015518605709076, + "objective/train/docs_used": 1745390, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2144376039505005, + "objective/train/original_loss": 3.507963180541992, + "objective/train/theoretical_loss": 3.3121749472039492, + "objective/train/tokens_used": 3103129600, + "objective/train/unlikelihood_loss": 0.20057956874370575, + "theoretical_loss": 3.3121749472039492, + "tokens_seen": 3103129600 + }, + { + "epoch": 0.94, + "learning_rate": 3.0250752256770314e-05, + "loss": 1.2558, + "theoretical_loss": 3.3121749472039492, + "tokens_seen": 3103129600 + }, + { + "epoch": 0.94, + "learning_rate": 3.0230692076228686e-05, + "loss": 1.2933, + "theoretical_loss": 3.312164264602382, + "tokens_seen": 3103260672 + }, + { + "epoch": 0.94, + "learning_rate": 3.0210631895687062e-05, + "loss": 1.2219, + "theoretical_loss": 3.312153582578335, + "tokens_seen": 3103391744 + }, + { + "epoch": 0.94, + "learning_rate": 3.0190571715145438e-05, + "loss": 1.4491, + "theoretical_loss": 3.3121429011317516, + "tokens_seen": 3103522816 + }, + { + "epoch": 0.94, + "learning_rate": 3.0170511534603813e-05, + "loss": 1.3508, + "theoretical_loss": 3.312132220262577, + "tokens_seen": 3103653888 + }, + { + "epoch": 0.94, + "learning_rate": 3.015045135406219e-05, + "loss": 1.1506, + "theoretical_loss": 3.3121215399707555, + "tokens_seen": 3103784960 + }, + { + "epoch": 0.94, + "learning_rate": 3.013039117352056e-05, + "loss": 1.4621, + "theoretical_loss": 3.3121108602562317, + "tokens_seen": 3103916032 + }, + { + "epoch": 0.94, + "learning_rate": 3.0110330992978937e-05, + "loss": 1.3153, + "theoretical_loss": 3.3121001811189497, + "tokens_seen": 3104047104 + }, + { + "epoch": 0.94, + "learning_rate": 3.009027081243731e-05, + "loss": 1.2217, + "theoretical_loss": 3.312089502558854, + "tokens_seen": 3104178176 + }, + { + "epoch": 0.94, + "learning_rate": 3.007021063189569e-05, + "loss": 1.2082, + "theoretical_loss": 3.312078824575889, + "tokens_seen": 3104309248 + }, + { + "epoch": 0.94, + "learning_rate": 3.0050150451354065e-05, + "loss": 1.4061, + "theoretical_loss": 3.3120681471699993, + "tokens_seen": 3104440320 + }, + { + "epoch": 0.94, + "learning_rate": 3.0030090270812437e-05, + "loss": 1.2172, + "theoretical_loss": 3.312057470341129, + "tokens_seen": 3104571392 + }, + { + "epoch": 0.94, + "learning_rate": 3.0010030090270813e-05, + "loss": 1.3721, + "theoretical_loss": 3.312046794089223, + "tokens_seen": 3104702464 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.2804863154888153, + "objective/train/avg_token_score": 0.0348673053085804, + "objective/train/docs_used": 1745987, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8927942514419556, + "objective/train/original_loss": 4.808982849121094, + "objective/train/theoretical_loss": 3.3120414561796143, + "objective/train/tokens_used": 3104768000, + "objective/train/unlikelihood_loss": 0.11761842668056488, + "theoretical_loss": 3.3120414561796143, + "tokens_seen": 3104768000 + }, + { + "epoch": 0.94, + "learning_rate": 2.9989969909729185e-05, + "loss": 1.1407, + "theoretical_loss": 3.3120361184142255, + "tokens_seen": 3104833536 + }, + { + "epoch": 0.94, + "learning_rate": 2.9969909729187564e-05, + "loss": 1.2902, + "theoretical_loss": 3.312025443316081, + "tokens_seen": 3104964608 + }, + { + "epoch": 0.94, + "learning_rate": 2.994984954864594e-05, + "loss": 1.3448, + "theoretical_loss": 3.312014768794734, + "tokens_seen": 3105095680 + }, + { + "epoch": 0.94, + "learning_rate": 2.9929789368104313e-05, + "loss": 1.1538, + "theoretical_loss": 3.3120040948501295, + "tokens_seen": 3105226752 + }, + { + "epoch": 0.94, + "learning_rate": 2.990972918756269e-05, + "loss": 1.3767, + "theoretical_loss": 3.311993421482211, + "tokens_seen": 3105357824 + }, + { + "epoch": 0.94, + "learning_rate": 2.988966900702106e-05, + "loss": 1.4416, + "theoretical_loss": 3.3119827486909235, + "tokens_seen": 3105488896 + }, + { + "epoch": 0.94, + "learning_rate": 2.986960882647944e-05, + "loss": 1.1336, + "theoretical_loss": 3.3119720764762115, + "tokens_seen": 3105619968 + }, + { + "epoch": 0.94, + "learning_rate": 2.9849548645937816e-05, + "loss": 1.2073, + "theoretical_loss": 3.3119614048380197, + "tokens_seen": 3105751040 + }, + { + "epoch": 0.94, + "learning_rate": 2.9829488465396188e-05, + "loss": 1.3106, + "theoretical_loss": 3.3119507337762926, + "tokens_seen": 3105882112 + }, + { + "epoch": 0.94, + "learning_rate": 2.9809428284854564e-05, + "loss": 1.199, + "theoretical_loss": 3.3119400632909746, + "tokens_seen": 3106013184 + }, + { + "epoch": 0.94, + "learning_rate": 2.978936810431294e-05, + "loss": 1.2063, + "theoretical_loss": 3.3119293933820098, + "tokens_seen": 3106144256 + }, + { + "epoch": 0.94, + "learning_rate": 2.9769307923771316e-05, + "loss": 1.1934, + "theoretical_loss": 3.3119187240493435, + "tokens_seen": 3106275328 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.5098362565040588, + "objective/train/avg_token_score": 0.015404353849589825, + "objective/train/docs_used": 1746871, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6450471878051758, + "objective/train/original_loss": 3.5115885734558105, + "objective/train/theoretical_loss": 3.31190805529292, + "objective/train/tokens_used": 3106406400, + "objective/train/unlikelihood_loss": 0.19817888736724854, + "theoretical_loss": 3.31190805529292, + "tokens_seen": 3106406400 + }, + { + "epoch": 0.94, + "learning_rate": 2.974924774322969e-05, + "loss": 1.269, + "theoretical_loss": 3.31190805529292, + "tokens_seen": 3106406400 + }, + { + "epoch": 0.94, + "learning_rate": 2.9729187562688064e-05, + "loss": 1.34, + "theoretical_loss": 3.3118973871126838, + "tokens_seen": 3106537472 + }, + { + "epoch": 0.94, + "learning_rate": 2.970912738214644e-05, + "loss": 1.2644, + "theoretical_loss": 3.3118867195085793, + "tokens_seen": 3106668544 + }, + { + "epoch": 0.94, + "learning_rate": 2.9689067201604815e-05, + "loss": 1.2288, + "theoretical_loss": 3.3118760524805517, + "tokens_seen": 3106799616 + }, + { + "epoch": 0.94, + "learning_rate": 2.966900702106319e-05, + "loss": 1.3708, + "theoretical_loss": 3.3118653860285447, + "tokens_seen": 3106930688 + }, + { + "epoch": 0.94, + "learning_rate": 2.9648946840521564e-05, + "loss": 1.2954, + "theoretical_loss": 3.3118547201525037, + "tokens_seen": 3107061760 + }, + { + "epoch": 0.94, + "learning_rate": 2.962888665997994e-05, + "loss": 1.4462, + "theoretical_loss": 3.3118440548523727, + "tokens_seen": 3107192832 + }, + { + "epoch": 0.94, + "learning_rate": 2.9608826479438315e-05, + "loss": 1.3957, + "theoretical_loss": 3.3118333901280965, + "tokens_seen": 3107323904 + }, + { + "epoch": 0.94, + "learning_rate": 2.958876629889669e-05, + "loss": 1.3087, + "theoretical_loss": 3.3118227259796202, + "tokens_seen": 3107454976 + }, + { + "epoch": 0.94, + "learning_rate": 2.9568706118355067e-05, + "loss": 1.2973, + "theoretical_loss": 3.3118120624068874, + "tokens_seen": 3107586048 + }, + { + "epoch": 0.94, + "learning_rate": 2.954864593781344e-05, + "loss": 1.3233, + "theoretical_loss": 3.311801399409844, + "tokens_seen": 3107717120 + }, + { + "epoch": 0.94, + "learning_rate": 2.9528585757271815e-05, + "loss": 1.2958, + "theoretical_loss": 3.311790736988434, + "tokens_seen": 3107848192 + }, + { + "epoch": 0.94, + "learning_rate": 2.9508525576730194e-05, + "loss": 1.3379, + "theoretical_loss": 3.3117800751426016, + "tokens_seen": 3107979264 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.5563905239105225, + "objective/train/avg_token_score": 0.004627614747732878, + "objective/train/docs_used": 1748010, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6964725255966187, + "objective/train/original_loss": 3.4210283756256104, + "objective/train/theoretical_loss": 3.31177474443551, + "objective/train/tokens_used": 3108044800, + "objective/train/unlikelihood_loss": 0.1128830537199974, + "theoretical_loss": 3.31177474443551, + "tokens_seen": 3108044800 + }, + { + "epoch": 0.94, + "learning_rate": 2.9488465396188567e-05, + "loss": 1.3896, + "theoretical_loss": 3.311769413872292, + "tokens_seen": 3108110336 + }, + { + "epoch": 0.94, + "learning_rate": 2.9468405215646942e-05, + "loss": 1.2537, + "theoretical_loss": 3.31175875317745, + "tokens_seen": 3108241408 + }, + { + "epoch": 0.94, + "learning_rate": 2.9448345035105315e-05, + "loss": 1.4829, + "theoretical_loss": 3.3117480930580196, + "tokens_seen": 3108372480 + }, + { + "epoch": 0.94, + "learning_rate": 2.942828485456369e-05, + "loss": 1.3584, + "theoretical_loss": 3.3117374335139464, + "tokens_seen": 3108503552 + }, + { + "epoch": 0.94, + "learning_rate": 2.940822467402207e-05, + "loss": 1.4039, + "theoretical_loss": 3.3117267745451744, + "tokens_seen": 3108634624 + }, + { + "epoch": 0.94, + "learning_rate": 2.9388164493480442e-05, + "loss": 1.3242, + "theoretical_loss": 3.311716116151649, + "tokens_seen": 3108765696 + }, + { + "epoch": 0.94, + "learning_rate": 2.9368104312938818e-05, + "loss": 1.3273, + "theoretical_loss": 3.3117054583333134, + "tokens_seen": 3108896768 + }, + { + "epoch": 0.94, + "learning_rate": 2.934804413239719e-05, + "loss": 1.2619, + "theoretical_loss": 3.311694801090114, + "tokens_seen": 3109027840 + }, + { + "epoch": 0.94, + "learning_rate": 2.9327983951855566e-05, + "loss": 1.2832, + "theoretical_loss": 3.311684144421995, + "tokens_seen": 3109158912 + }, + { + "epoch": 0.94, + "learning_rate": 2.9307923771313942e-05, + "loss": 1.2758, + "theoretical_loss": 3.3116734883289003, + "tokens_seen": 3109289984 + }, + { + "epoch": 0.94, + "learning_rate": 2.9287863590772318e-05, + "loss": 1.3052, + "theoretical_loss": 3.311662832810776, + "tokens_seen": 3109421056 + }, + { + "epoch": 0.94, + "learning_rate": 2.9267803410230693e-05, + "loss": 1.1836, + "theoretical_loss": 3.3116521778675656, + "tokens_seen": 3109552128 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.3836754560470581, + "objective/train/avg_token_score": 0.02375662699341774, + "objective/train/docs_used": 1748667, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.180324912071228, + "objective/train/original_loss": 3.7345690727233887, + "objective/train/theoretical_loss": 3.3116415234992145, + "objective/train/tokens_used": 3109683200, + "objective/train/unlikelihood_loss": 0.15164701640605927, + "theoretical_loss": 3.3116415234992145, + "tokens_seen": 3109683200 + }, + { + "epoch": 0.94, + "learning_rate": 2.9247743229689066e-05, + "loss": 1.3454, + "theoretical_loss": 3.3116415234992145, + "tokens_seen": 3109683200 + }, + { + "epoch": 0.94, + "learning_rate": 2.9227683049147445e-05, + "loss": 1.3474, + "theoretical_loss": 3.3116308697056676, + "tokens_seen": 3109814272 + }, + { + "epoch": 0.94, + "learning_rate": 2.9207622868605817e-05, + "loss": 1.346, + "theoretical_loss": 3.311620216486869, + "tokens_seen": 3109945344 + }, + { + "epoch": 0.94, + "learning_rate": 2.9187562688064193e-05, + "loss": 1.2184, + "theoretical_loss": 3.3116095638427643, + "tokens_seen": 3110076416 + }, + { + "epoch": 0.94, + "learning_rate": 2.916750250752257e-05, + "loss": 1.336, + "theoretical_loss": 3.311598911773298, + "tokens_seen": 3110207488 + }, + { + "epoch": 0.94, + "learning_rate": 2.914744232698094e-05, + "loss": 1.3076, + "theoretical_loss": 3.311588260278414, + "tokens_seen": 3110338560 + }, + { + "epoch": 0.94, + "learning_rate": 2.912738214643932e-05, + "loss": 1.4682, + "theoretical_loss": 3.3115776093580584, + "tokens_seen": 3110469632 + }, + { + "epoch": 0.94, + "learning_rate": 2.9107321965897693e-05, + "loss": 1.2598, + "theoretical_loss": 3.3115669590121755, + "tokens_seen": 3110600704 + }, + { + "epoch": 0.94, + "learning_rate": 2.908726178535607e-05, + "loss": 1.2013, + "theoretical_loss": 3.31155630924071, + "tokens_seen": 3110731776 + }, + { + "epoch": 0.94, + "learning_rate": 2.9067201604814445e-05, + "loss": 1.2753, + "theoretical_loss": 3.311545660043606, + "tokens_seen": 3110862848 + }, + { + "epoch": 0.94, + "learning_rate": 2.9047141424272817e-05, + "loss": 1.3308, + "theoretical_loss": 3.31153501142081, + "tokens_seen": 3110993920 + }, + { + "epoch": 0.94, + "learning_rate": 2.9027081243731196e-05, + "loss": 1.313, + "theoretical_loss": 3.311524363372266, + "tokens_seen": 3111124992 + }, + { + "epoch": 0.94, + "learning_rate": 2.900702106318957e-05, + "loss": 1.3634, + "theoretical_loss": 3.3115137158979184, + "tokens_seen": 3111256064 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.3955278694629669, + "objective/train/avg_token_score": 0.00613910797983408, + "objective/train/docs_used": 1749977, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1175273656845093, + "objective/train/original_loss": 3.0260732173919678, + "objective/train/theoretical_loss": 3.311508392376051, + "objective/train/tokens_used": 3111321600, + "objective/train/unlikelihood_loss": 0.22188349068164825, + "theoretical_loss": 3.311508392376051, + "tokens_seen": 3111321600 + }, + { + "epoch": 0.94, + "learning_rate": 2.8986960882647944e-05, + "loss": 1.2902, + "theoretical_loss": 3.3115030689977125, + "tokens_seen": 3111387136 + }, + { + "epoch": 0.94, + "learning_rate": 2.8966900702106317e-05, + "loss": 1.3795, + "theoretical_loss": 3.311492422671593, + "tokens_seen": 3111518208 + }, + { + "epoch": 0.94, + "learning_rate": 2.8946840521564696e-05, + "loss": 1.0394, + "theoretical_loss": 3.311481776919505, + "tokens_seen": 3111649280 + }, + { + "epoch": 0.94, + "learning_rate": 2.8926780341023072e-05, + "loss": 1.3332, + "theoretical_loss": 3.311471131741393, + "tokens_seen": 3111780352 + }, + { + "epoch": 0.94, + "learning_rate": 2.8906720160481444e-05, + "loss": 1.3456, + "theoretical_loss": 3.3114604871372024, + "tokens_seen": 3111911424 + }, + { + "epoch": 0.94, + "learning_rate": 2.888665997993982e-05, + "loss": 1.3186, + "theoretical_loss": 3.3114498431068773, + "tokens_seen": 3112042496 + }, + { + "epoch": 0.94, + "learning_rate": 2.8866599799398192e-05, + "loss": 1.3501, + "theoretical_loss": 3.311439199650364, + "tokens_seen": 3112173568 + }, + { + "epoch": 0.94, + "learning_rate": 2.884653961885657e-05, + "loss": 1.235, + "theoretical_loss": 3.3114285567676056, + "tokens_seen": 3112304640 + }, + { + "epoch": 0.94, + "learning_rate": 2.8826479438314947e-05, + "loss": 1.4288, + "theoretical_loss": 3.311417914458548, + "tokens_seen": 3112435712 + }, + { + "epoch": 0.94, + "learning_rate": 2.880641925777332e-05, + "loss": 1.4646, + "theoretical_loss": 3.311407272723136, + "tokens_seen": 3112566784 + }, + { + "epoch": 0.94, + "learning_rate": 2.8786359077231696e-05, + "loss": 1.2709, + "theoretical_loss": 3.3113966315613146, + "tokens_seen": 3112697856 + }, + { + "epoch": 0.94, + "learning_rate": 2.8766298896690068e-05, + "loss": 1.3542, + "theoretical_loss": 3.311385990973029, + "tokens_seen": 3112828928 + }, + { + "debugging/length": 441.5, + "debugging/num_segments": 6, + "epoch": 0.94, + "objective/train/aligned_token_count": 0.263868510723114, + "objective/train/avg_token_score": 0.011473271064460278, + "objective/train/docs_used": 1750566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.837683916091919, + "objective/train/original_loss": 3.693058729171753, + "objective/train/theoretical_loss": 3.3113753509582233, + "objective/train/tokens_used": 3112960000, + "objective/train/unlikelihood_loss": 0.23616741597652435, + "theoretical_loss": 3.3113753509582233, + "tokens_seen": 3112960000 + }, + { + "epoch": 0.94, + "learning_rate": 2.8746238716148447e-05, + "loss": 1.3104, + "theoretical_loss": 3.3113753509582233, + "tokens_seen": 3112960000 + }, + { + "epoch": 0.94, + "learning_rate": 2.8726178535606823e-05, + "loss": 1.3791, + "theoretical_loss": 3.3113647115168434, + "tokens_seen": 3113091072 + }, + { + "epoch": 0.94, + "learning_rate": 2.8706118355065195e-05, + "loss": 1.1546, + "theoretical_loss": 3.3113540726488333, + "tokens_seen": 3113222144 + }, + { + "epoch": 0.94, + "learning_rate": 2.868605817452357e-05, + "loss": 1.2548, + "theoretical_loss": 3.3113434343541392, + "tokens_seen": 3113353216 + }, + { + "epoch": 0.94, + "learning_rate": 2.8665997993981947e-05, + "loss": 1.1659, + "theoretical_loss": 3.311332796632705, + "tokens_seen": 3113484288 + }, + { + "epoch": 0.94, + "learning_rate": 2.8645937813440323e-05, + "loss": 1.2846, + "theoretical_loss": 3.3113221594844755, + "tokens_seen": 3113615360 + }, + { + "epoch": 0.94, + "learning_rate": 2.8625877632898695e-05, + "loss": 1.2792, + "theoretical_loss": 3.3113115229093966, + "tokens_seen": 3113746432 + }, + { + "epoch": 0.94, + "learning_rate": 2.860581745235707e-05, + "loss": 1.3923, + "theoretical_loss": 3.311300886907413, + "tokens_seen": 3113877504 + }, + { + "epoch": 0.94, + "learning_rate": 2.8585757271815447e-05, + "loss": 1.446, + "theoretical_loss": 3.3112902514784697, + "tokens_seen": 3114008576 + }, + { + "epoch": 0.94, + "learning_rate": 2.8565697091273823e-05, + "loss": 1.3146, + "theoretical_loss": 3.311279616622511, + "tokens_seen": 3114139648 + }, + { + "epoch": 0.94, + "learning_rate": 2.85456369107322e-05, + "loss": 1.243, + "theoretical_loss": 3.3112689823394827, + "tokens_seen": 3114270720 + }, + { + "epoch": 0.94, + "learning_rate": 2.852557673019057e-05, + "loss": 1.4244, + "theoretical_loss": 3.31125834862933, + "tokens_seen": 3114401792 + }, + { + "epoch": 0.94, + "learning_rate": 2.8505516549648947e-05, + "loss": 1.3248, + "theoretical_loss": 3.3112477154919975, + "tokens_seen": 3114532864 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.4362170100212097, + "objective/train/avg_token_score": 0.011590166948735714, + "objective/train/docs_used": 1751732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3674795627593994, + "objective/train/original_loss": 3.928029775619507, + "objective/train/theoretical_loss": 3.3112423991381217, + "objective/train/tokens_used": 3114598400, + "objective/train/unlikelihood_loss": 0.1687440574169159, + "theoretical_loss": 3.3112423991381217, + "tokens_seen": 3114598400 + }, + { + "epoch": 0.94, + "learning_rate": 2.8485456369107322e-05, + "loss": 1.4706, + "theoretical_loss": 3.31123708292743, + "tokens_seen": 3114663936 + }, + { + "epoch": 0.94, + "learning_rate": 2.8465396188565698e-05, + "loss": 1.2743, + "theoretical_loss": 3.311226450935573, + "tokens_seen": 3114795008 + }, + { + "epoch": 0.94, + "learning_rate": 2.8445336008024074e-05, + "loss": 1.4145, + "theoretical_loss": 3.3112158195163715, + "tokens_seen": 3114926080 + }, + { + "epoch": 0.94, + "learning_rate": 2.8425275827482446e-05, + "loss": 1.3789, + "theoretical_loss": 3.3112051886697706, + "tokens_seen": 3115057152 + }, + { + "epoch": 0.94, + "learning_rate": 2.8405215646940822e-05, + "loss": 1.3391, + "theoretical_loss": 3.311194558395715, + "tokens_seen": 3115188224 + }, + { + "epoch": 0.94, + "learning_rate": 2.83851554663992e-05, + "loss": 1.3781, + "theoretical_loss": 3.31118392869415, + "tokens_seen": 3115319296 + }, + { + "epoch": 0.94, + "learning_rate": 2.8365095285857574e-05, + "loss": 1.3162, + "theoretical_loss": 3.311173299565021, + "tokens_seen": 3115450368 + }, + { + "epoch": 0.94, + "learning_rate": 2.834503510531595e-05, + "loss": 1.4241, + "theoretical_loss": 3.3111626710082724, + "tokens_seen": 3115581440 + }, + { + "epoch": 0.94, + "learning_rate": 2.8324974924774322e-05, + "loss": 1.4074, + "theoretical_loss": 3.3111520430238497, + "tokens_seen": 3115712512 + }, + { + "epoch": 0.94, + "learning_rate": 2.8304914744232698e-05, + "loss": 1.3773, + "theoretical_loss": 3.311141415611698, + "tokens_seen": 3115843584 + }, + { + "epoch": 0.94, + "learning_rate": 2.8284854563691077e-05, + "loss": 1.3177, + "theoretical_loss": 3.311130788771763, + "tokens_seen": 3115974656 + }, + { + "epoch": 0.94, + "learning_rate": 2.826479438314945e-05, + "loss": 1.1578, + "theoretical_loss": 3.3111201625039888, + "tokens_seen": 3116105728 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.3832477927207947, + "objective/train/avg_token_score": 0.013052796944975853, + "objective/train/docs_used": 1752323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1217942237854004, + "objective/train/original_loss": 3.6574783325195312, + "objective/train/theoretical_loss": 3.3111095368083205, + "objective/train/tokens_used": 3116236800, + "objective/train/unlikelihood_loss": 0.21597155928611755, + "theoretical_loss": 3.3111095368083205, + "tokens_seen": 3116236800 + }, + { + "epoch": 0.94, + "learning_rate": 2.8244734202607825e-05, + "loss": 1.2358, + "theoretical_loss": 3.3111095368083205, + "tokens_seen": 3116236800 + }, + { + "epoch": 0.94, + "learning_rate": 2.8224674022066197e-05, + "loss": 1.2542, + "theoretical_loss": 3.311098911684704, + "tokens_seen": 3116367872 + }, + { + "epoch": 0.94, + "learning_rate": 2.8204613841524573e-05, + "loss": 1.1801, + "theoretical_loss": 3.3110882871330847, + "tokens_seen": 3116498944 + }, + { + "epoch": 0.94, + "learning_rate": 2.818455366098295e-05, + "loss": 1.3935, + "theoretical_loss": 3.311077663153407, + "tokens_seen": 3116630016 + }, + { + "epoch": 0.94, + "learning_rate": 2.8164493480441325e-05, + "loss": 1.3619, + "theoretical_loss": 3.3110670397456157, + "tokens_seen": 3116761088 + }, + { + "epoch": 0.94, + "learning_rate": 2.81444332998997e-05, + "loss": 1.3866, + "theoretical_loss": 3.311056416909657, + "tokens_seen": 3116892160 + }, + { + "epoch": 0.94, + "learning_rate": 2.8124373119358073e-05, + "loss": 1.3481, + "theoretical_loss": 3.311045794645475, + "tokens_seen": 3117023232 + }, + { + "epoch": 0.94, + "learning_rate": 2.8104312938816452e-05, + "loss": 1.2821, + "theoretical_loss": 3.311035172953016, + "tokens_seen": 3117154304 + }, + { + "epoch": 0.94, + "learning_rate": 2.8084252758274825e-05, + "loss": 1.3961, + "theoretical_loss": 3.3110245518322245, + "tokens_seen": 3117285376 + }, + { + "epoch": 0.94, + "learning_rate": 2.80641925777332e-05, + "loss": 1.3531, + "theoretical_loss": 3.311013931283046, + "tokens_seen": 3117416448 + }, + { + "epoch": 0.94, + "learning_rate": 2.8044132397191576e-05, + "loss": 1.2486, + "theoretical_loss": 3.311003311305426, + "tokens_seen": 3117547520 + }, + { + "epoch": 0.94, + "learning_rate": 2.802407221664995e-05, + "loss": 1.4592, + "theoretical_loss": 3.3109926918993087, + "tokens_seen": 3117678592 + }, + { + "epoch": 0.94, + "learning_rate": 2.8004012036108328e-05, + "loss": 1.4165, + "theoretical_loss": 3.3109820730646398, + "tokens_seen": 3117809664 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.5139296054840088, + "objective/train/avg_token_score": 0.016362521797418594, + "objective/train/docs_used": 1753489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5182254314422607, + "objective/train/original_loss": 3.2164061069488525, + "objective/train/theoretical_loss": 3.3109767638615817, + "objective/train/tokens_used": 3117875200, + "objective/train/unlikelihood_loss": 0.18409715592861176, + "theoretical_loss": 3.3109767638615817, + "tokens_seen": 3117875200 + }, + { + "epoch": 0.94, + "learning_rate": 2.79839518555667e-05, + "loss": 1.3893, + "theoretical_loss": 3.310971454801365, + "tokens_seen": 3117940736 + }, + { + "epoch": 0.94, + "learning_rate": 2.7963891675025076e-05, + "loss": 1.3268, + "theoretical_loss": 3.3109608371094286, + "tokens_seen": 3118071808 + }, + { + "epoch": 0.94, + "learning_rate": 2.7943831494483452e-05, + "loss": 1.4354, + "theoretical_loss": 3.310950219988777, + "tokens_seen": 3118202880 + }, + { + "epoch": 0.94, + "learning_rate": 2.7923771313941824e-05, + "loss": 1.4322, + "theoretical_loss": 3.310939603439355, + "tokens_seen": 3118333952 + }, + { + "epoch": 0.94, + "learning_rate": 2.7903711133400203e-05, + "loss": 1.3463, + "theoretical_loss": 3.310928987461107, + "tokens_seen": 3118465024 + }, + { + "epoch": 0.95, + "learning_rate": 2.7883650952858576e-05, + "loss": 1.1906, + "theoretical_loss": 3.3109183720539797, + "tokens_seen": 3118596096 + }, + { + "epoch": 0.95, + "learning_rate": 2.786359077231695e-05, + "loss": 1.3313, + "theoretical_loss": 3.310907757217917, + "tokens_seen": 3118727168 + }, + { + "epoch": 0.95, + "learning_rate": 2.7843530591775324e-05, + "loss": 1.4774, + "theoretical_loss": 3.3108971429528653, + "tokens_seen": 3118858240 + }, + { + "epoch": 0.95, + "learning_rate": 2.7823470411233703e-05, + "loss": 1.2478, + "theoretical_loss": 3.3108865292587693, + "tokens_seen": 3118989312 + }, + { + "epoch": 0.95, + "learning_rate": 2.780341023069208e-05, + "loss": 1.2726, + "theoretical_loss": 3.3108759161355743, + "tokens_seen": 3119120384 + }, + { + "epoch": 0.95, + "learning_rate": 2.778335005015045e-05, + "loss": 1.2753, + "theoretical_loss": 3.310865303583226, + "tokens_seen": 3119251456 + }, + { + "epoch": 0.95, + "learning_rate": 2.7763289869608827e-05, + "loss": 1.4342, + "theoretical_loss": 3.3108546916016692, + "tokens_seen": 3119382528 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.5128298997879028, + "objective/train/avg_token_score": 0.01106333639472723, + "objective/train/docs_used": 1754072, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5822468996047974, + "objective/train/original_loss": 3.4792442321777344, + "objective/train/theoretical_loss": 3.3108440801908494, + "objective/train/tokens_used": 3119513600, + "objective/train/unlikelihood_loss": 0.17587760090827942, + "theoretical_loss": 3.3108440801908494, + "tokens_seen": 3119513600 + }, + { + "epoch": 0.95, + "learning_rate": 2.77432296890672e-05, + "loss": 1.4915, + "theoretical_loss": 3.3108440801908494, + "tokens_seen": 3119513600 + }, + { + "epoch": 0.95, + "learning_rate": 2.772316950852558e-05, + "loss": 1.4756, + "theoretical_loss": 3.310833469350712, + "tokens_seen": 3119644672 + }, + { + "epoch": 0.95, + "learning_rate": 2.7703109327983955e-05, + "loss": 1.4008, + "theoretical_loss": 3.3108228590812026, + "tokens_seen": 3119775744 + }, + { + "epoch": 0.95, + "learning_rate": 2.7683049147442327e-05, + "loss": 1.1335, + "theoretical_loss": 3.3108122493822663, + "tokens_seen": 3119906816 + }, + { + "epoch": 0.95, + "learning_rate": 2.7662988966900703e-05, + "loss": 1.2978, + "theoretical_loss": 3.310801640253848, + "tokens_seen": 3120037888 + }, + { + "epoch": 0.95, + "learning_rate": 2.7642928786359075e-05, + "loss": 1.2425, + "theoretical_loss": 3.3107910316958935, + "tokens_seen": 3120168960 + }, + { + "epoch": 0.95, + "learning_rate": 2.7622868605817454e-05, + "loss": 1.273, + "theoretical_loss": 3.3107804237083482, + "tokens_seen": 3120300032 + }, + { + "epoch": 0.95, + "learning_rate": 2.760280842527583e-05, + "loss": 1.1667, + "theoretical_loss": 3.310769816291158, + "tokens_seen": 3120431104 + }, + { + "epoch": 0.95, + "learning_rate": 2.7582748244734203e-05, + "loss": 1.3716, + "theoretical_loss": 3.310759209444267, + "tokens_seen": 3120562176 + }, + { + "epoch": 0.95, + "learning_rate": 2.756268806419258e-05, + "loss": 1.1919, + "theoretical_loss": 3.310748603167621, + "tokens_seen": 3120693248 + }, + { + "epoch": 0.95, + "learning_rate": 2.7542627883650954e-05, + "loss": 1.2105, + "theoretical_loss": 3.310737997461166, + "tokens_seen": 3120824320 + }, + { + "epoch": 0.95, + "learning_rate": 2.752256770310933e-05, + "loss": 1.2669, + "theoretical_loss": 3.3107273923248473, + "tokens_seen": 3120955392 + }, + { + "epoch": 0.95, + "learning_rate": 2.7502507522567702e-05, + "loss": 1.582, + "theoretical_loss": 3.3107167877586097, + "tokens_seen": 3121086464 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.4464198350906372, + "objective/train/avg_token_score": 0.01945679821074009, + "objective/train/docs_used": 1754741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4922667741775513, + "objective/train/original_loss": 3.8883469104766846, + "objective/train/theoretical_loss": 3.3107114856892546, + "objective/train/tokens_used": 3121152000, + "objective/train/unlikelihood_loss": 0.13449367880821228, + "theoretical_loss": 3.3107114856892546, + "tokens_seen": 3121152000 + }, + { + "epoch": 0.95, + "learning_rate": 2.7482447342026078e-05, + "loss": 1.4813, + "theoretical_loss": 3.310706183762399, + "tokens_seen": 3121217536 + }, + { + "epoch": 0.95, + "learning_rate": 2.7462387161484454e-05, + "loss": 1.2602, + "theoretical_loss": 3.310695580336161, + "tokens_seen": 3121348608 + }, + { + "epoch": 0.95, + "learning_rate": 2.744232698094283e-05, + "loss": 1.3292, + "theoretical_loss": 3.3106849774798404, + "tokens_seen": 3121479680 + }, + { + "epoch": 0.95, + "learning_rate": 2.7422266800401205e-05, + "loss": 1.3664, + "theoretical_loss": 3.3106743751933827, + "tokens_seen": 3121610752 + }, + { + "epoch": 0.95, + "learning_rate": 2.7402206619859578e-05, + "loss": 1.3761, + "theoretical_loss": 3.310663773476734, + "tokens_seen": 3121741824 + }, + { + "epoch": 0.95, + "learning_rate": 2.7382146439317954e-05, + "loss": 1.1171, + "theoretical_loss": 3.3106531723298396, + "tokens_seen": 3121872896 + }, + { + "epoch": 0.95, + "learning_rate": 2.736208625877633e-05, + "loss": 1.4605, + "theoretical_loss": 3.310642571752644, + "tokens_seen": 3122003968 + }, + { + "epoch": 0.95, + "learning_rate": 2.7342026078234705e-05, + "loss": 1.2455, + "theoretical_loss": 3.310631971745094, + "tokens_seen": 3122135040 + }, + { + "epoch": 0.95, + "learning_rate": 2.732196589769308e-05, + "loss": 1.3478, + "theoretical_loss": 3.3106213723071343, + "tokens_seen": 3122266112 + }, + { + "epoch": 0.95, + "learning_rate": 2.7301905717151453e-05, + "loss": 1.2708, + "theoretical_loss": 3.3106107734387105, + "tokens_seen": 3122397184 + }, + { + "epoch": 0.95, + "learning_rate": 2.728184553660983e-05, + "loss": 1.3194, + "theoretical_loss": 3.3106001751397685, + "tokens_seen": 3122528256 + }, + { + "epoch": 0.95, + "learning_rate": 2.7261785356068205e-05, + "loss": 1.2427, + "theoretical_loss": 3.3105895774102527, + "tokens_seen": 3122659328 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.39748290181159973, + "objective/train/avg_token_score": 0.024448160082101822, + "objective/train/docs_used": 1755986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1395535469055176, + "objective/train/original_loss": 3.7769341468811035, + "objective/train/theoretical_loss": 3.31057898025011, + "objective/train/tokens_used": 3122790400, + "objective/train/unlikelihood_loss": 0.1543806940317154, + "theoretical_loss": 3.31057898025011, + "tokens_seen": 3122790400 + }, + { + "epoch": 0.95, + "learning_rate": 2.724172517552658e-05, + "loss": 1.3047, + "theoretical_loss": 3.31057898025011, + "tokens_seen": 3122790400 + }, + { + "epoch": 0.95, + "learning_rate": 2.7221664994984957e-05, + "loss": 1.3779, + "theoretical_loss": 3.310568383659285, + "tokens_seen": 3122921472 + }, + { + "epoch": 0.95, + "learning_rate": 2.720160481444333e-05, + "loss": 1.2046, + "theoretical_loss": 3.3105577876377237, + "tokens_seen": 3123052544 + }, + { + "epoch": 0.95, + "learning_rate": 2.7181544633901705e-05, + "loss": 1.2223, + "theoretical_loss": 3.3105471921853713, + "tokens_seen": 3123183616 + }, + { + "epoch": 0.95, + "learning_rate": 2.716148445336008e-05, + "loss": 1.2037, + "theoretical_loss": 3.3105365973021734, + "tokens_seen": 3123314688 + }, + { + "epoch": 0.95, + "learning_rate": 2.7141424272818456e-05, + "loss": 1.3952, + "theoretical_loss": 3.310526002988076, + "tokens_seen": 3123445760 + }, + { + "epoch": 0.95, + "learning_rate": 2.7121364092276832e-05, + "loss": 1.4072, + "theoretical_loss": 3.3105154092430236, + "tokens_seen": 3123576832 + }, + { + "epoch": 0.95, + "learning_rate": 2.7101303911735205e-05, + "loss": 1.2572, + "theoretical_loss": 3.3105048160669623, + "tokens_seen": 3123707904 + }, + { + "epoch": 0.95, + "learning_rate": 2.708124373119358e-05, + "loss": 1.332, + "theoretical_loss": 3.3104942234598385, + "tokens_seen": 3123838976 + }, + { + "epoch": 0.95, + "learning_rate": 2.7061183550651956e-05, + "loss": 1.2287, + "theoretical_loss": 3.310483631421597, + "tokens_seen": 3123970048 + }, + { + "epoch": 0.95, + "learning_rate": 2.7041123370110332e-05, + "loss": 1.3077, + "theoretical_loss": 3.310473039952183, + "tokens_seen": 3124101120 + }, + { + "epoch": 0.95, + "learning_rate": 2.7021063189568708e-05, + "loss": 1.3725, + "theoretical_loss": 3.3104624490515424, + "tokens_seen": 3124232192 + }, + { + "epoch": 0.95, + "learning_rate": 2.700100300902708e-05, + "loss": 1.1977, + "theoretical_loss": 3.310451858719621, + "tokens_seen": 3124363264 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.23619256913661957, + "objective/train/avg_token_score": 0.01244492270052433, + "objective/train/docs_used": 1756707, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7695622444152832, + "objective/train/original_loss": 4.114345550537109, + "objective/train/theoretical_loss": 3.3104465637669134, + "objective/train/tokens_used": 3124428800, + "objective/train/unlikelihood_loss": 0.15097959339618683, + "theoretical_loss": 3.3104465637669134, + "tokens_seen": 3124428800 + }, + { + "epoch": 0.95, + "learning_rate": 2.6980942828485456e-05, + "loss": 1.298, + "theoretical_loss": 3.3104412689563647, + "tokens_seen": 3124494336 + }, + { + "epoch": 0.95, + "learning_rate": 2.6960882647943832e-05, + "loss": 1.259, + "theoretical_loss": 3.3104306797617187, + "tokens_seen": 3124625408 + }, + { + "epoch": 0.95, + "learning_rate": 2.6940822467402208e-05, + "loss": 1.3016, + "theoretical_loss": 3.3104200911356285, + "tokens_seen": 3124756480 + }, + { + "epoch": 0.95, + "learning_rate": 2.6920762286860583e-05, + "loss": 1.3738, + "theoretical_loss": 3.31040950307804, + "tokens_seen": 3124887552 + }, + { + "epoch": 0.95, + "learning_rate": 2.6900702106318956e-05, + "loss": 1.3396, + "theoretical_loss": 3.310398915588898, + "tokens_seen": 3125018624 + }, + { + "epoch": 0.95, + "learning_rate": 2.6880641925777335e-05, + "loss": 1.2204, + "theoretical_loss": 3.3103883286681497, + "tokens_seen": 3125149696 + }, + { + "epoch": 0.95, + "learning_rate": 2.6860581745235707e-05, + "loss": 1.2782, + "theoretical_loss": 3.31037774231574, + "tokens_seen": 3125280768 + }, + { + "epoch": 0.95, + "learning_rate": 2.6840521564694083e-05, + "loss": 1.2434, + "theoretical_loss": 3.3103671565316137, + "tokens_seen": 3125411840 + }, + { + "epoch": 0.95, + "learning_rate": 2.6820461384152456e-05, + "loss": 1.2838, + "theoretical_loss": 3.310356571315718, + "tokens_seen": 3125542912 + }, + { + "epoch": 0.95, + "learning_rate": 2.680040120361083e-05, + "loss": 1.3342, + "theoretical_loss": 3.310345986667997, + "tokens_seen": 3125673984 + }, + { + "epoch": 0.95, + "learning_rate": 2.678034102306921e-05, + "loss": 1.5192, + "theoretical_loss": 3.3103354025883975, + "tokens_seen": 3125805056 + }, + { + "epoch": 0.95, + "learning_rate": 2.6760280842527583e-05, + "loss": 1.2433, + "theoretical_loss": 3.310324819076865, + "tokens_seen": 3125936128 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.4528347849845886, + "objective/train/avg_token_score": 0.005143969319760799, + "objective/train/docs_used": 1757671, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4440703392028809, + "objective/train/original_loss": 3.842324733734131, + "objective/train/theoretical_loss": 3.310314236133345, + "objective/train/tokens_used": 3126067200, + "objective/train/unlikelihood_loss": 0.15456371009349823, + "theoretical_loss": 3.310314236133345, + "tokens_seen": 3126067200 + }, + { + "epoch": 0.95, + "learning_rate": 2.674022066198596e-05, + "loss": 1.2994, + "theoretical_loss": 3.310314236133345, + "tokens_seen": 3126067200 + }, + { + "epoch": 0.95, + "learning_rate": 2.672016048144433e-05, + "loss": 1.2086, + "theoretical_loss": 3.3103036537577832, + "tokens_seen": 3126198272 + }, + { + "epoch": 0.95, + "learning_rate": 2.6700100300902707e-05, + "loss": 1.1239, + "theoretical_loss": 3.3102930719501256, + "tokens_seen": 3126329344 + }, + { + "epoch": 0.95, + "learning_rate": 2.6680040120361086e-05, + "loss": 1.1899, + "theoretical_loss": 3.310282490710317, + "tokens_seen": 3126460416 + }, + { + "epoch": 0.95, + "learning_rate": 2.665997993981946e-05, + "loss": 1.1926, + "theoretical_loss": 3.3102719100383045, + "tokens_seen": 3126591488 + }, + { + "epoch": 0.95, + "learning_rate": 2.6639919759277834e-05, + "loss": 1.2769, + "theoretical_loss": 3.3102613299340327, + "tokens_seen": 3126722560 + }, + { + "epoch": 0.95, + "learning_rate": 2.6619859578736207e-05, + "loss": 1.4347, + "theoretical_loss": 3.310250750397448, + "tokens_seen": 3126853632 + }, + { + "epoch": 0.95, + "learning_rate": 2.6599799398194586e-05, + "loss": 1.1863, + "theoretical_loss": 3.3102401714284957, + "tokens_seen": 3126984704 + }, + { + "epoch": 0.95, + "learning_rate": 2.657973921765296e-05, + "loss": 1.2932, + "theoretical_loss": 3.310229593027122, + "tokens_seen": 3127115776 + }, + { + "epoch": 0.95, + "learning_rate": 2.6559679037111334e-05, + "loss": 1.2279, + "theoretical_loss": 3.3102190151932724, + "tokens_seen": 3127246848 + }, + { + "epoch": 0.95, + "learning_rate": 2.653961885656971e-05, + "loss": 1.2047, + "theoretical_loss": 3.310208437926893, + "tokens_seen": 3127377920 + }, + { + "epoch": 0.95, + "learning_rate": 2.6519558676028082e-05, + "loss": 1.2958, + "theoretical_loss": 3.3101978612279286, + "tokens_seen": 3127508992 + }, + { + "epoch": 0.95, + "learning_rate": 2.649949849548646e-05, + "loss": 1.3367, + "theoretical_loss": 3.310187285096326, + "tokens_seen": 3127640064 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.3992546498775482, + "objective/train/avg_token_score": 0.016105500981211662, + "objective/train/docs_used": 1758353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.263795256614685, + "objective/train/original_loss": 3.5287060737609863, + "objective/train/theoretical_loss": 3.310181997243268, + "objective/train/tokens_used": 3127705600, + "objective/train/unlikelihood_loss": 0.20558471977710724, + "theoretical_loss": 3.310181997243268, + "tokens_seen": 3127705600 + }, + { + "epoch": 0.95, + "learning_rate": 2.6479438314944837e-05, + "loss": 1.3245, + "theoretical_loss": 3.3101767095320307, + "tokens_seen": 3127771136 + }, + { + "epoch": 0.95, + "learning_rate": 2.645937813440321e-05, + "loss": 1.345, + "theoretical_loss": 3.310166134534988, + "tokens_seen": 3127902208 + }, + { + "epoch": 0.95, + "learning_rate": 2.6439317953861585e-05, + "loss": 1.3813, + "theoretical_loss": 3.3101555601051444, + "tokens_seen": 3128033280 + }, + { + "epoch": 0.95, + "learning_rate": 2.6419257773319958e-05, + "loss": 1.3033, + "theoretical_loss": 3.3101449862424457, + "tokens_seen": 3128164352 + }, + { + "epoch": 0.95, + "learning_rate": 2.6399197592778337e-05, + "loss": 1.2148, + "theoretical_loss": 3.3101344129468373, + "tokens_seen": 3128295424 + }, + { + "epoch": 0.95, + "learning_rate": 2.637913741223671e-05, + "loss": 1.261, + "theoretical_loss": 3.310123840218265, + "tokens_seen": 3128426496 + }, + { + "epoch": 0.95, + "learning_rate": 2.6359077231695085e-05, + "loss": 1.3397, + "theoretical_loss": 3.3101132680566754, + "tokens_seen": 3128557568 + }, + { + "epoch": 0.95, + "learning_rate": 2.633901705115346e-05, + "loss": 1.3086, + "theoretical_loss": 3.310102696462013, + "tokens_seen": 3128688640 + }, + { + "epoch": 0.95, + "learning_rate": 2.6318956870611837e-05, + "loss": 1.0798, + "theoretical_loss": 3.310092125434225, + "tokens_seen": 3128819712 + }, + { + "epoch": 0.95, + "learning_rate": 2.6298896690070213e-05, + "loss": 1.2386, + "theoretical_loss": 3.3100815549732565, + "tokens_seen": 3128950784 + }, + { + "epoch": 0.95, + "learning_rate": 2.6278836509528585e-05, + "loss": 1.1688, + "theoretical_loss": 3.3100709850790535, + "tokens_seen": 3129081856 + }, + { + "epoch": 0.95, + "learning_rate": 2.625877632898696e-05, + "loss": 1.2236, + "theoretical_loss": 3.310060415751562, + "tokens_seen": 3129212928 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.34084799885749817, + "objective/train/avg_token_score": 0.02545267529785633, + "objective/train/docs_used": 1759339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9250695109367371, + "objective/train/original_loss": 3.95068097114563, + "objective/train/theoretical_loss": 3.310049846990728, + "objective/train/tokens_used": 3129344000, + "objective/train/unlikelihood_loss": 0.11857158690690994, + "theoretical_loss": 3.310049846990728, + "tokens_seen": 3129344000 + }, + { + "epoch": 0.95, + "learning_rate": 2.6238716148445337e-05, + "loss": 1.1453, + "theoretical_loss": 3.310049846990728, + "tokens_seen": 3129344000 + }, + { + "epoch": 0.95, + "learning_rate": 2.6218655967903712e-05, + "loss": 1.1724, + "theoretical_loss": 3.310039278796497, + "tokens_seen": 3129475072 + }, + { + "epoch": 0.95, + "learning_rate": 2.6198595787362088e-05, + "loss": 1.2074, + "theoretical_loss": 3.3100287111688154, + "tokens_seen": 3129606144 + }, + { + "epoch": 0.95, + "learning_rate": 2.617853560682046e-05, + "loss": 1.3432, + "theoretical_loss": 3.3100181441076284, + "tokens_seen": 3129737216 + }, + { + "epoch": 0.95, + "learning_rate": 2.6158475426278836e-05, + "loss": 1.2328, + "theoretical_loss": 3.310007577612882, + "tokens_seen": 3129868288 + }, + { + "epoch": 0.95, + "learning_rate": 2.6138415245737212e-05, + "loss": 1.4185, + "theoretical_loss": 3.309997011684523, + "tokens_seen": 3129999360 + }, + { + "epoch": 0.95, + "learning_rate": 2.6118355065195588e-05, + "loss": 1.3057, + "theoretical_loss": 3.3099864463224966, + "tokens_seen": 3130130432 + }, + { + "epoch": 0.95, + "learning_rate": 2.6098294884653964e-05, + "loss": 1.2645, + "theoretical_loss": 3.309975881526749, + "tokens_seen": 3130261504 + }, + { + "epoch": 0.95, + "learning_rate": 2.6078234704112336e-05, + "loss": 1.2858, + "theoretical_loss": 3.309965317297226, + "tokens_seen": 3130392576 + }, + { + "epoch": 0.95, + "learning_rate": 2.6058174523570712e-05, + "loss": 1.2167, + "theoretical_loss": 3.309954753633873, + "tokens_seen": 3130523648 + }, + { + "epoch": 0.95, + "learning_rate": 2.6038114343029088e-05, + "loss": 1.3721, + "theoretical_loss": 3.309944190536637, + "tokens_seen": 3130654720 + }, + { + "epoch": 0.95, + "learning_rate": 2.6018054162487464e-05, + "loss": 1.2712, + "theoretical_loss": 3.309933628005463, + "tokens_seen": 3130785792 + }, + { + "epoch": 0.95, + "learning_rate": 2.599799398194584e-05, + "loss": 1.2735, + "theoretical_loss": 3.309923066040298, + "tokens_seen": 3130916864 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.3224584460258484, + "objective/train/avg_token_score": 0.015580384060740471, + "objective/train/docs_used": 1760143, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9630290269851685, + "objective/train/original_loss": 3.708519458770752, + "objective/train/theoretical_loss": 3.3099177852699517, + "objective/train/tokens_used": 3130982400, + "objective/train/unlikelihood_loss": 0.19792874157428741, + "theoretical_loss": 3.3099177852699517, + "tokens_seen": 3130982400 + }, + { + "epoch": 0.95, + "learning_rate": 2.5977933801404212e-05, + "loss": 1.1666, + "theoretical_loss": 3.3099125046410873, + "tokens_seen": 3131047936 + }, + { + "epoch": 0.95, + "learning_rate": 2.5957873620862588e-05, + "loss": 1.2596, + "theoretical_loss": 3.309901943807777, + "tokens_seen": 3131179008 + }, + { + "epoch": 0.95, + "learning_rate": 2.5937813440320963e-05, + "loss": 1.31, + "theoretical_loss": 3.3098913835403128, + "tokens_seen": 3131310080 + }, + { + "epoch": 0.95, + "learning_rate": 2.591775325977934e-05, + "loss": 1.4627, + "theoretical_loss": 3.3098808238386415, + "tokens_seen": 3131441152 + }, + { + "epoch": 0.95, + "learning_rate": 2.5897693079237715e-05, + "loss": 1.1827, + "theoretical_loss": 3.3098702647027083, + "tokens_seen": 3131572224 + }, + { + "epoch": 0.95, + "learning_rate": 2.5877632898696087e-05, + "loss": 1.5079, + "theoretical_loss": 3.3098597061324595, + "tokens_seen": 3131703296 + }, + { + "epoch": 0.95, + "learning_rate": 2.5857572718154463e-05, + "loss": 1.2239, + "theoretical_loss": 3.3098491481278414, + "tokens_seen": 3131834368 + }, + { + "epoch": 0.95, + "learning_rate": 2.583751253761284e-05, + "loss": 1.2559, + "theoretical_loss": 3.3098385906887993, + "tokens_seen": 3131965440 + }, + { + "epoch": 0.95, + "learning_rate": 2.5817452357071215e-05, + "loss": 1.3476, + "theoretical_loss": 3.30982803381528, + "tokens_seen": 3132096512 + }, + { + "epoch": 0.95, + "learning_rate": 2.579739217652959e-05, + "loss": 1.3475, + "theoretical_loss": 3.309817477507229, + "tokens_seen": 3132227584 + }, + { + "epoch": 0.95, + "learning_rate": 2.5777331995987963e-05, + "loss": 1.3091, + "theoretical_loss": 3.3098069217645927, + "tokens_seen": 3132358656 + }, + { + "epoch": 0.95, + "learning_rate": 2.5757271815446342e-05, + "loss": 1.0783, + "theoretical_loss": 3.3097963665873174, + "tokens_seen": 3132489728 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.5775293111801147, + "objective/train/avg_token_score": 0.008554896339774132, + "objective/train/docs_used": 1761098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.881266713142395, + "objective/train/original_loss": 3.5834779739379883, + "objective/train/theoretical_loss": 3.3097858119753485, + "objective/train/tokens_used": 3132620800, + "objective/train/unlikelihood_loss": 0.11567112058401108, + "theoretical_loss": 3.3097858119753485, + "tokens_seen": 3132620800 + }, + { + "epoch": 0.95, + "learning_rate": 2.5737211634904715e-05, + "loss": 1.3772, + "theoretical_loss": 3.3097858119753485, + "tokens_seen": 3132620800 + }, + { + "epoch": 0.95, + "learning_rate": 2.571715145436309e-05, + "loss": 1.1509, + "theoretical_loss": 3.3097752579286324, + "tokens_seen": 3132751872 + }, + { + "epoch": 0.95, + "learning_rate": 2.5697091273821463e-05, + "loss": 1.4024, + "theoretical_loss": 3.3097647044471152, + "tokens_seen": 3132882944 + }, + { + "epoch": 0.95, + "learning_rate": 2.567703109327984e-05, + "loss": 1.1147, + "theoretical_loss": 3.309754151530743, + "tokens_seen": 3133014016 + }, + { + "epoch": 0.95, + "learning_rate": 2.5656970912738218e-05, + "loss": 1.1587, + "theoretical_loss": 3.3097435991794617, + "tokens_seen": 3133145088 + }, + { + "epoch": 0.95, + "learning_rate": 2.563691073219659e-05, + "loss": 1.4364, + "theoretical_loss": 3.3097330473932174, + "tokens_seen": 3133276160 + }, + { + "epoch": 0.95, + "learning_rate": 2.5616850551654966e-05, + "loss": 1.3694, + "theoretical_loss": 3.3097224961719567, + "tokens_seen": 3133407232 + }, + { + "epoch": 0.95, + "learning_rate": 2.5596790371113338e-05, + "loss": 1.374, + "theoretical_loss": 3.309711945515625, + "tokens_seen": 3133538304 + }, + { + "epoch": 0.95, + "learning_rate": 2.5576730190571714e-05, + "loss": 1.2516, + "theoretical_loss": 3.3097013954241694, + "tokens_seen": 3133669376 + }, + { + "epoch": 0.95, + "learning_rate": 2.5556670010030093e-05, + "loss": 1.2394, + "theoretical_loss": 3.309690845897535, + "tokens_seen": 3133800448 + }, + { + "epoch": 0.95, + "learning_rate": 2.5536609829488466e-05, + "loss": 1.3511, + "theoretical_loss": 3.3096802969356682, + "tokens_seen": 3133931520 + }, + { + "epoch": 0.95, + "learning_rate": 2.551654964894684e-05, + "loss": 1.2069, + "theoretical_loss": 3.3096697485385156, + "tokens_seen": 3134062592 + }, + { + "epoch": 0.95, + "learning_rate": 2.5496489468405214e-05, + "loss": 1.3134, + "theoretical_loss": 3.3096592007060233, + "tokens_seen": 3134193664 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.3030303120613098, + "objective/train/avg_token_score": 0.015524367801845074, + "objective/train/docs_used": 1761812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9519380331039429, + "objective/train/original_loss": 4.5459771156311035, + "objective/train/theoretical_loss": 3.3096539270015075, + "objective/train/tokens_used": 3134259200, + "objective/train/unlikelihood_loss": 0.1828564703464508, + "theoretical_loss": 3.3096539270015075, + "tokens_seen": 3134259200 + }, + { + "epoch": 0.95, + "learning_rate": 2.5476429287863593e-05, + "loss": 1.1597, + "theoretical_loss": 3.3096486534381366, + "tokens_seen": 3134324736 + }, + { + "epoch": 0.95, + "learning_rate": 2.545636910732197e-05, + "loss": 1.2887, + "theoretical_loss": 3.309638106734803, + "tokens_seen": 3134455808 + }, + { + "epoch": 0.95, + "learning_rate": 2.543630892678034e-05, + "loss": 1.3395, + "theoretical_loss": 3.3096275605959677, + "tokens_seen": 3134586880 + }, + { + "epoch": 0.95, + "learning_rate": 2.5416248746238717e-05, + "loss": 1.2199, + "theoretical_loss": 3.309617015021577, + "tokens_seen": 3134717952 + }, + { + "epoch": 0.95, + "learning_rate": 2.539618856569709e-05, + "loss": 1.3303, + "theoretical_loss": 3.3096064700115773, + "tokens_seen": 3134849024 + }, + { + "epoch": 0.95, + "learning_rate": 2.537612838515547e-05, + "loss": 1.2337, + "theoretical_loss": 3.309595925565915, + "tokens_seen": 3134980096 + }, + { + "epoch": 0.95, + "learning_rate": 2.535606820461384e-05, + "loss": 1.4324, + "theoretical_loss": 3.3095853816845358, + "tokens_seen": 3135111168 + }, + { + "epoch": 0.95, + "learning_rate": 2.5336008024072217e-05, + "loss": 1.313, + "theoretical_loss": 3.309574838367386, + "tokens_seen": 3135242240 + }, + { + "epoch": 0.95, + "learning_rate": 2.5315947843530593e-05, + "loss": 1.3348, + "theoretical_loss": 3.3095642956144125, + "tokens_seen": 3135373312 + }, + { + "epoch": 0.95, + "learning_rate": 2.5295887662988965e-05, + "loss": 1.0907, + "theoretical_loss": 3.309553753425561, + "tokens_seen": 3135504384 + }, + { + "epoch": 0.95, + "learning_rate": 2.5275827482447344e-05, + "loss": 1.2933, + "theoretical_loss": 3.3095432118007775, + "tokens_seen": 3135635456 + }, + { + "epoch": 0.95, + "learning_rate": 2.5255767301905717e-05, + "loss": 1.2498, + "theoretical_loss": 3.3095326707400083, + "tokens_seen": 3135766528 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.3876466155052185, + "objective/train/avg_token_score": 0.009929314255714417, + "objective/train/docs_used": 1763087, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2481223344802856, + "objective/train/original_loss": 3.9242730140686035, + "objective/train/theoretical_loss": 3.3095221302432, + "objective/train/tokens_used": 3135897600, + "objective/train/unlikelihood_loss": 0.2026147097349167, + "theoretical_loss": 3.3095221302432, + "tokens_seen": 3135897600 + }, + { + "epoch": 0.95, + "learning_rate": 2.5235707121364092e-05, + "loss": 1.1594, + "theoretical_loss": 3.3095221302432, + "tokens_seen": 3135897600 + }, + { + "epoch": 0.95, + "learning_rate": 2.5215646940822468e-05, + "loss": 1.2804, + "theoretical_loss": 3.3095115903102985, + "tokens_seen": 3136028672 + }, + { + "epoch": 0.95, + "learning_rate": 2.5195586760280844e-05, + "loss": 1.0834, + "theoretical_loss": 3.309501050941251, + "tokens_seen": 3136159744 + }, + { + "epoch": 0.95, + "learning_rate": 2.517552657973922e-05, + "loss": 1.2961, + "theoretical_loss": 3.3094905121360023, + "tokens_seen": 3136290816 + }, + { + "epoch": 0.95, + "learning_rate": 2.5155466399197592e-05, + "loss": 1.359, + "theoretical_loss": 3.3094799738944998, + "tokens_seen": 3136421888 + }, + { + "epoch": 0.95, + "learning_rate": 2.5135406218655968e-05, + "loss": 1.3057, + "theoretical_loss": 3.309469436216689, + "tokens_seen": 3136552960 + }, + { + "epoch": 0.95, + "learning_rate": 2.5115346038114344e-05, + "loss": 1.3402, + "theoretical_loss": 3.309458899102517, + "tokens_seen": 3136684032 + }, + { + "epoch": 0.95, + "learning_rate": 2.509528585757272e-05, + "loss": 1.1405, + "theoretical_loss": 3.3094483625519295, + "tokens_seen": 3136815104 + }, + { + "epoch": 0.95, + "learning_rate": 2.5075225677031095e-05, + "loss": 1.2348, + "theoretical_loss": 3.3094378265648734, + "tokens_seen": 3136946176 + }, + { + "epoch": 0.95, + "learning_rate": 2.5055165496489468e-05, + "loss": 1.3835, + "theoretical_loss": 3.309427291141294, + "tokens_seen": 3137077248 + }, + { + "epoch": 0.95, + "learning_rate": 2.5035105315947844e-05, + "loss": 1.3606, + "theoretical_loss": 3.3094167562811387, + "tokens_seen": 3137208320 + }, + { + "epoch": 0.95, + "learning_rate": 2.5015045135406216e-05, + "loss": 1.2995, + "theoretical_loss": 3.309406221984353, + "tokens_seen": 3137339392 + }, + { + "epoch": 0.95, + "learning_rate": 2.4994984954864595e-05, + "loss": 1.2148, + "theoretical_loss": 3.309395688250884, + "tokens_seen": 3137470464 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.4299853444099426, + "objective/train/avg_token_score": 0.033343832939863205, + "objective/train/docs_used": 1763593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3803222179412842, + "objective/train/original_loss": 4.617800235748291, + "objective/train/theoretical_loss": 3.309390421595376, + "objective/train/tokens_used": 3137536000, + "objective/train/unlikelihood_loss": 0.11386729031801224, + "theoretical_loss": 3.309390421595376, + "tokens_seen": 3137536000 + }, + { + "epoch": 0.95, + "learning_rate": 2.497492477432297e-05, + "loss": 1.298, + "theoretical_loss": 3.3093851550806774, + "tokens_seen": 3137601536 + }, + { + "epoch": 0.95, + "learning_rate": 2.4954864593781343e-05, + "loss": 1.2675, + "theoretical_loss": 3.3093746224736798, + "tokens_seen": 3137732608 + }, + { + "epoch": 0.95, + "learning_rate": 2.493480441323972e-05, + "loss": 1.3369, + "theoretical_loss": 3.309364090429838, + "tokens_seen": 3137863680 + }, + { + "epoch": 0.95, + "learning_rate": 2.4914744232698095e-05, + "loss": 1.2961, + "theoretical_loss": 3.3093535589490974, + "tokens_seen": 3137994752 + }, + { + "epoch": 0.95, + "learning_rate": 2.489468405215647e-05, + "loss": 1.4422, + "theoretical_loss": 3.3093430280314053, + "tokens_seen": 3138125824 + }, + { + "epoch": 0.95, + "learning_rate": 2.4874623871614847e-05, + "loss": 1.3315, + "theoretical_loss": 3.3093324976767073, + "tokens_seen": 3138256896 + }, + { + "epoch": 0.95, + "learning_rate": 2.485456369107322e-05, + "loss": 1.2621, + "theoretical_loss": 3.3093219678849506, + "tokens_seen": 3138387968 + }, + { + "epoch": 0.95, + "learning_rate": 2.4834503510531595e-05, + "loss": 1.2639, + "theoretical_loss": 3.3093114386560814, + "tokens_seen": 3138519040 + }, + { + "epoch": 0.95, + "learning_rate": 2.481444332998997e-05, + "loss": 1.4686, + "theoretical_loss": 3.3093009099900454, + "tokens_seen": 3138650112 + }, + { + "epoch": 0.95, + "learning_rate": 2.4794383149448346e-05, + "loss": 1.189, + "theoretical_loss": 3.3092903818867896, + "tokens_seen": 3138781184 + }, + { + "epoch": 0.95, + "learning_rate": 2.4774322968906722e-05, + "loss": 1.3248, + "theoretical_loss": 3.3092798543462605, + "tokens_seen": 3138912256 + }, + { + "epoch": 0.95, + "learning_rate": 2.4754262788365095e-05, + "loss": 1.2361, + "theoretical_loss": 3.309269327368404, + "tokens_seen": 3139043328 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.4492301940917969, + "objective/train/avg_token_score": 0.01409667544066906, + "objective/train/docs_used": 1764333, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4260575771331787, + "objective/train/original_loss": 3.8597805500030518, + "objective/train/theoretical_loss": 3.309258800953167, + "objective/train/tokens_used": 3139174400, + "objective/train/unlikelihood_loss": 0.14392858743667603, + "theoretical_loss": 3.309258800953167, + "tokens_seen": 3139174400 + }, + { + "epoch": 0.95, + "learning_rate": 2.473420260782347e-05, + "loss": 1.2482, + "theoretical_loss": 3.309258800953167, + "tokens_seen": 3139174400 + }, + { + "epoch": 0.95, + "learning_rate": 2.4714142427281846e-05, + "loss": 1.3532, + "theoretical_loss": 3.309248275100496, + "tokens_seen": 3139305472 + }, + { + "epoch": 0.95, + "learning_rate": 2.4694082246740222e-05, + "loss": 1.3555, + "theoretical_loss": 3.309237749810337, + "tokens_seen": 3139436544 + }, + { + "epoch": 0.95, + "learning_rate": 2.4674022066198598e-05, + "loss": 1.4076, + "theoretical_loss": 3.309227225082637, + "tokens_seen": 3139567616 + }, + { + "epoch": 0.95, + "learning_rate": 2.465396188565697e-05, + "loss": 1.2807, + "theoretical_loss": 3.309216700917342, + "tokens_seen": 3139698688 + }, + { + "epoch": 0.95, + "learning_rate": 2.463390170511535e-05, + "loss": 1.2793, + "theoretical_loss": 3.309206177314399, + "tokens_seen": 3139829760 + }, + { + "epoch": 0.95, + "learning_rate": 2.461384152457372e-05, + "loss": 1.1829, + "theoretical_loss": 3.3091956542737537, + "tokens_seen": 3139960832 + }, + { + "epoch": 0.95, + "learning_rate": 2.4593781344032097e-05, + "loss": 1.2105, + "theoretical_loss": 3.309185131795353, + "tokens_seen": 3140091904 + }, + { + "epoch": 0.95, + "learning_rate": 2.457372116349047e-05, + "loss": 1.2991, + "theoretical_loss": 3.309174609879144, + "tokens_seen": 3140222976 + }, + { + "epoch": 0.95, + "learning_rate": 2.4553660982948846e-05, + "loss": 1.2621, + "theoretical_loss": 3.309164088525072, + "tokens_seen": 3140354048 + }, + { + "epoch": 0.95, + "learning_rate": 2.4533600802407225e-05, + "loss": 1.3881, + "theoretical_loss": 3.3091535677330848, + "tokens_seen": 3140485120 + }, + { + "epoch": 0.95, + "learning_rate": 2.4513540621865597e-05, + "loss": 1.1534, + "theoretical_loss": 3.3091430475031274, + "tokens_seen": 3140616192 + }, + { + "epoch": 0.95, + "learning_rate": 2.4493480441323973e-05, + "loss": 1.1368, + "theoretical_loss": 3.3091325278351476, + "tokens_seen": 3140747264 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.28879520297050476, + "objective/train/avg_token_score": 0.03210514411330223, + "objective/train/docs_used": 1765173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9873993396759033, + "objective/train/original_loss": 4.502580642700195, + "objective/train/theoretical_loss": 3.3091272682118826, + "objective/train/tokens_used": 3140812800, + "objective/train/unlikelihood_loss": 0.11855122447013855, + "theoretical_loss": 3.3091272682118826, + "tokens_seen": 3140812800 + }, + { + "epoch": 0.95, + "learning_rate": 2.4473420260782345e-05, + "loss": 1.2654, + "theoretical_loss": 3.3091220087290916, + "tokens_seen": 3140878336 + }, + { + "epoch": 0.95, + "learning_rate": 2.445336008024072e-05, + "loss": 1.1355, + "theoretical_loss": 3.3091114901849057, + "tokens_seen": 3141009408 + }, + { + "epoch": 0.95, + "learning_rate": 2.44332998996991e-05, + "loss": 1.2757, + "theoretical_loss": 3.3091009722025366, + "tokens_seen": 3141140480 + }, + { + "epoch": 0.95, + "learning_rate": 2.4413239719157473e-05, + "loss": 1.2247, + "theoretical_loss": 3.3090904547819306, + "tokens_seen": 3141271552 + }, + { + "epoch": 0.95, + "learning_rate": 2.439317953861585e-05, + "loss": 1.3745, + "theoretical_loss": 3.309079937923035, + "tokens_seen": 3141402624 + }, + { + "epoch": 0.95, + "learning_rate": 2.437311935807422e-05, + "loss": 1.3142, + "theoretical_loss": 3.309069421625795, + "tokens_seen": 3141533696 + }, + { + "epoch": 0.95, + "learning_rate": 2.4353059177532597e-05, + "loss": 1.1989, + "theoretical_loss": 3.3090589058901587, + "tokens_seen": 3141664768 + }, + { + "epoch": 0.95, + "learning_rate": 2.4332998996990976e-05, + "loss": 1.3719, + "theoretical_loss": 3.3090483907160717, + "tokens_seen": 3141795840 + }, + { + "epoch": 0.95, + "learning_rate": 2.431293881644935e-05, + "loss": 1.4176, + "theoretical_loss": 3.309037876103481, + "tokens_seen": 3141926912 + }, + { + "epoch": 0.95, + "learning_rate": 2.4292878635907724e-05, + "loss": 1.2374, + "theoretical_loss": 3.309027362052333, + "tokens_seen": 3142057984 + }, + { + "epoch": 0.95, + "learning_rate": 2.4272818455366097e-05, + "loss": 1.1611, + "theoretical_loss": 3.3090168485625746, + "tokens_seen": 3142189056 + }, + { + "epoch": 0.95, + "learning_rate": 2.4252758274824476e-05, + "loss": 1.3511, + "theoretical_loss": 3.3090063356341517, + "tokens_seen": 3142320128 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.43982160091400146, + "objective/train/avg_token_score": 0.05821988731622696, + "objective/train/docs_used": 1765556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4028240442276, + "objective/train/original_loss": 3.9841055870056152, + "objective/train/theoretical_loss": 3.3089958232670122, + "objective/train/tokens_used": 3142451200, + "objective/train/unlikelihood_loss": 0.16510650515556335, + "theoretical_loss": 3.3089958232670122, + "tokens_seen": 3142451200 + }, + { + "epoch": 0.95, + "learning_rate": 2.4232698094282848e-05, + "loss": 1.3068, + "theoretical_loss": 3.3089958232670122, + "tokens_seen": 3142451200 + }, + { + "epoch": 0.95, + "learning_rate": 2.4212637913741224e-05, + "loss": 1.4032, + "theoretical_loss": 3.308985311461101, + "tokens_seen": 3142582272 + }, + { + "epoch": 0.95, + "learning_rate": 2.41925777331996e-05, + "loss": 1.2458, + "theoretical_loss": 3.308974800216366, + "tokens_seen": 3142713344 + }, + { + "epoch": 0.95, + "learning_rate": 2.4172517552657972e-05, + "loss": 1.3038, + "theoretical_loss": 3.308964289532754, + "tokens_seen": 3142844416 + }, + { + "epoch": 0.95, + "learning_rate": 2.415245737211635e-05, + "loss": 1.1359, + "theoretical_loss": 3.308953779410211, + "tokens_seen": 3142975488 + }, + { + "epoch": 0.95, + "learning_rate": 2.4132397191574724e-05, + "loss": 1.1565, + "theoretical_loss": 3.308943269848683, + "tokens_seen": 3143106560 + }, + { + "epoch": 0.95, + "learning_rate": 2.41123370110331e-05, + "loss": 1.2971, + "theoretical_loss": 3.3089327608481183, + "tokens_seen": 3143237632 + }, + { + "epoch": 0.95, + "learning_rate": 2.4092276830491475e-05, + "loss": 1.405, + "theoretical_loss": 3.3089222524084625, + "tokens_seen": 3143368704 + }, + { + "epoch": 0.95, + "learning_rate": 2.4072216649949848e-05, + "loss": 1.2928, + "theoretical_loss": 3.308911744529662, + "tokens_seen": 3143499776 + }, + { + "epoch": 0.95, + "learning_rate": 2.4052156469408227e-05, + "loss": 1.3989, + "theoretical_loss": 3.3089012372116646, + "tokens_seen": 3143630848 + }, + { + "epoch": 0.95, + "learning_rate": 2.40320962888666e-05, + "loss": 1.2269, + "theoretical_loss": 3.308890730454416, + "tokens_seen": 3143761920 + }, + { + "epoch": 0.95, + "learning_rate": 2.4012036108324975e-05, + "loss": 1.2142, + "theoretical_loss": 3.3088802242578637, + "tokens_seen": 3143892992 + }, + { + "epoch": 0.95, + "learning_rate": 2.399197592778335e-05, + "loss": 1.2552, + "theoretical_loss": 3.3088697186219536, + "tokens_seen": 3144024064 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.36589688062667847, + "objective/train/avg_token_score": 0.014658571220934391, + "objective/train/docs_used": 1766057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0987006425857544, + "objective/train/original_loss": 3.914013624191284, + "objective/train/theoretical_loss": 3.308864466014223, + "objective/train/tokens_used": 3144089600, + "objective/train/unlikelihood_loss": 0.16888949275016785, + "theoretical_loss": 3.308864466014223, + "tokens_seen": 3144089600 + }, + { + "epoch": 0.95, + "learning_rate": 2.3971915747241727e-05, + "loss": 1.337, + "theoretical_loss": 3.3088592135466333, + "tokens_seen": 3144155136 + }, + { + "epoch": 0.95, + "learning_rate": 2.3951855566700103e-05, + "loss": 1.1346, + "theoretical_loss": 3.3088487090318486, + "tokens_seen": 3144286208 + }, + { + "epoch": 0.95, + "learning_rate": 2.3931795386158475e-05, + "loss": 1.2426, + "theoretical_loss": 3.3088382050775467, + "tokens_seen": 3144417280 + }, + { + "epoch": 0.95, + "learning_rate": 2.391173520561685e-05, + "loss": 1.2907, + "theoretical_loss": 3.308827701683674, + "tokens_seen": 3144548352 + }, + { + "epoch": 0.95, + "learning_rate": 2.3891675025075223e-05, + "loss": 1.3356, + "theoretical_loss": 3.3088171988501776, + "tokens_seen": 3144679424 + }, + { + "epoch": 0.95, + "learning_rate": 2.3871614844533602e-05, + "loss": 1.2553, + "theoretical_loss": 3.3088066965770047, + "tokens_seen": 3144810496 + }, + { + "epoch": 0.95, + "learning_rate": 2.3851554663991978e-05, + "loss": 1.2687, + "theoretical_loss": 3.308796194864101, + "tokens_seen": 3144941568 + }, + { + "epoch": 0.95, + "learning_rate": 2.383149448345035e-05, + "loss": 1.0919, + "theoretical_loss": 3.3087856937114135, + "tokens_seen": 3145072640 + }, + { + "epoch": 0.95, + "learning_rate": 2.3811434302908726e-05, + "loss": 1.18, + "theoretical_loss": 3.3087751931188896, + "tokens_seen": 3145203712 + }, + { + "epoch": 0.95, + "learning_rate": 2.37913741223671e-05, + "loss": 1.2424, + "theoretical_loss": 3.308764693086476, + "tokens_seen": 3145334784 + }, + { + "epoch": 0.95, + "learning_rate": 2.3771313941825478e-05, + "loss": 1.1983, + "theoretical_loss": 3.308754193614119, + "tokens_seen": 3145465856 + }, + { + "epoch": 0.95, + "learning_rate": 2.3751253761283854e-05, + "loss": 1.2288, + "theoretical_loss": 3.308743694701765, + "tokens_seen": 3145596928 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.4585777223110199, + "objective/train/avg_token_score": 0.04008569195866585, + "objective/train/docs_used": 1767374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3940527439117432, + "objective/train/original_loss": 3.731656551361084, + "objective/train/theoretical_loss": 3.308733196349362, + "objective/train/tokens_used": 3145728000, + "objective/train/unlikelihood_loss": 0.14402715861797333, + "theoretical_loss": 3.308733196349362, + "tokens_seen": 3145728000 + }, + { + "epoch": 0.95, + "learning_rate": 2.3731193580742226e-05, + "loss": 1.2063, + "theoretical_loss": 3.308733196349362, + "tokens_seen": 3145728000 + }, + { + "epoch": 0.95, + "learning_rate": 2.3711133400200602e-05, + "loss": 1.2602, + "theoretical_loss": 3.3087226985568563, + "tokens_seen": 3145859072 + }, + { + "epoch": 0.95, + "learning_rate": 2.3691073219658978e-05, + "loss": 1.3823, + "theoretical_loss": 3.3087122013241945, + "tokens_seen": 3145990144 + }, + { + "epoch": 0.95, + "learning_rate": 2.3671013039117353e-05, + "loss": 1.2031, + "theoretical_loss": 3.308701704651323, + "tokens_seen": 3146121216 + }, + { + "epoch": 0.95, + "learning_rate": 2.365095285857573e-05, + "loss": 1.1689, + "theoretical_loss": 3.30869120853819, + "tokens_seen": 3146252288 + }, + { + "epoch": 0.95, + "learning_rate": 2.36308926780341e-05, + "loss": 1.1775, + "theoretical_loss": 3.308680712984741, + "tokens_seen": 3146383360 + }, + { + "epoch": 0.95, + "learning_rate": 2.3610832497492477e-05, + "loss": 1.1499, + "theoretical_loss": 3.3086702179909233, + "tokens_seen": 3146514432 + }, + { + "epoch": 0.95, + "learning_rate": 2.3590772316950853e-05, + "loss": 1.2253, + "theoretical_loss": 3.308659723556684, + "tokens_seen": 3146645504 + }, + { + "epoch": 0.95, + "learning_rate": 2.357071213640923e-05, + "loss": 1.1907, + "theoretical_loss": 3.3086492296819694, + "tokens_seen": 3146776576 + }, + { + "epoch": 0.95, + "learning_rate": 2.3550651955867605e-05, + "loss": 1.2912, + "theoretical_loss": 3.308638736366727, + "tokens_seen": 3146907648 + }, + { + "epoch": 0.95, + "learning_rate": 2.3530591775325977e-05, + "loss": 1.4005, + "theoretical_loss": 3.3086282436109036, + "tokens_seen": 3147038720 + }, + { + "epoch": 0.95, + "learning_rate": 2.3510531594784353e-05, + "loss": 1.3125, + "theoretical_loss": 3.3086177514144453, + "tokens_seen": 3147169792 + }, + { + "epoch": 0.95, + "learning_rate": 2.349047141424273e-05, + "loss": 1.2397, + "theoretical_loss": 3.3086072597773, + "tokens_seen": 3147300864 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.5498533844947815, + "objective/train/avg_token_score": 0.008938722312450409, + "objective/train/docs_used": 1768009, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.52300226688385, + "objective/train/original_loss": 3.1133251190185547, + "objective/train/theoretical_loss": 3.3086020141684527, + "objective/train/tokens_used": 3147366400, + "objective/train/unlikelihood_loss": 0.21915902197360992, + "theoretical_loss": 3.3086020141684527, + "tokens_seen": 3147366400 + }, + { + "epoch": 0.95, + "learning_rate": 2.3470411233701105e-05, + "loss": 1.2972, + "theoretical_loss": 3.308596768699414, + "tokens_seen": 3147431936 + }, + { + "epoch": 0.95, + "learning_rate": 2.3450351053159477e-05, + "loss": 1.3668, + "theoretical_loss": 3.3085862781807345, + "tokens_seen": 3147563008 + }, + { + "epoch": 0.95, + "learning_rate": 2.3430290872617853e-05, + "loss": 1.1726, + "theoretical_loss": 3.3085757882212077, + "tokens_seen": 3147694080 + }, + { + "epoch": 0.95, + "learning_rate": 2.3410230692076232e-05, + "loss": 1.2182, + "theoretical_loss": 3.3085652988207817, + "tokens_seen": 3147825152 + }, + { + "epoch": 0.95, + "learning_rate": 2.3390170511534604e-05, + "loss": 1.2002, + "theoretical_loss": 3.3085548099794027, + "tokens_seen": 3147956224 + }, + { + "epoch": 0.95, + "learning_rate": 2.337011033099298e-05, + "loss": 1.1943, + "theoretical_loss": 3.308544321697017, + "tokens_seen": 3148087296 + }, + { + "epoch": 0.95, + "learning_rate": 2.3350050150451353e-05, + "loss": 1.2879, + "theoretical_loss": 3.308533833973573, + "tokens_seen": 3148218368 + }, + { + "epoch": 0.95, + "learning_rate": 2.332998996990973e-05, + "loss": 1.3837, + "theoretical_loss": 3.3085233468090167, + "tokens_seen": 3148349440 + }, + { + "epoch": 0.95, + "learning_rate": 2.3309929789368108e-05, + "loss": 1.3483, + "theoretical_loss": 3.308512860203295, + "tokens_seen": 3148480512 + }, + { + "epoch": 0.95, + "learning_rate": 2.328986960882648e-05, + "loss": 1.2258, + "theoretical_loss": 3.3085023741563555, + "tokens_seen": 3148611584 + }, + { + "epoch": 0.95, + "learning_rate": 2.3269809428284856e-05, + "loss": 1.2983, + "theoretical_loss": 3.3084918886681445, + "tokens_seen": 3148742656 + }, + { + "epoch": 0.95, + "learning_rate": 2.3249749247743228e-05, + "loss": 1.2712, + "theoretical_loss": 3.308481403738609, + "tokens_seen": 3148873728 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.348240464925766, + "objective/train/avg_token_score": 0.024917595088481903, + "objective/train/docs_used": 1768862, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9741582274436951, + "objective/train/original_loss": 4.079895973205566, + "objective/train/theoretical_loss": 3.308470919367697, + "objective/train/tokens_used": 3149004800, + "objective/train/unlikelihood_loss": 0.2020789384841919, + "theoretical_loss": 3.308470919367697, + "tokens_seen": 3149004800 + }, + { + "epoch": 0.95, + "learning_rate": 2.3229689067201604e-05, + "loss": 1.1934, + "theoretical_loss": 3.308470919367697, + "tokens_seen": 3149004800 + }, + { + "epoch": 0.95, + "learning_rate": 2.3209628886659983e-05, + "loss": 1.2356, + "theoretical_loss": 3.308460435555354, + "tokens_seen": 3149135872 + }, + { + "epoch": 0.95, + "learning_rate": 2.3189568706118356e-05, + "loss": 1.1818, + "theoretical_loss": 3.3084499523015283, + "tokens_seen": 3149266944 + }, + { + "epoch": 0.95, + "learning_rate": 2.316950852557673e-05, + "loss": 1.2411, + "theoretical_loss": 3.308439469606166, + "tokens_seen": 3149398016 + }, + { + "epoch": 0.95, + "learning_rate": 2.3149448345035104e-05, + "loss": 1.3069, + "theoretical_loss": 3.308428987469214, + "tokens_seen": 3149529088 + }, + { + "epoch": 0.95, + "learning_rate": 2.3129388164493483e-05, + "loss": 1.33, + "theoretical_loss": 3.3084185058906206, + "tokens_seen": 3149660160 + }, + { + "epoch": 0.95, + "learning_rate": 2.3109327983951855e-05, + "loss": 1.3356, + "theoretical_loss": 3.3084080248703316, + "tokens_seen": 3149791232 + }, + { + "epoch": 0.95, + "learning_rate": 2.308926780341023e-05, + "loss": 1.338, + "theoretical_loss": 3.308397544408294, + "tokens_seen": 3149922304 + }, + { + "epoch": 0.95, + "learning_rate": 2.3069207622868607e-05, + "loss": 1.2522, + "theoretical_loss": 3.3083870645044553, + "tokens_seen": 3150053376 + }, + { + "epoch": 0.95, + "learning_rate": 2.304914744232698e-05, + "loss": 1.0621, + "theoretical_loss": 3.3083765851587628, + "tokens_seen": 3150184448 + }, + { + "epoch": 0.95, + "learning_rate": 2.302908726178536e-05, + "loss": 1.2123, + "theoretical_loss": 3.308366106371163, + "tokens_seen": 3150315520 + }, + { + "epoch": 0.95, + "learning_rate": 2.300902708124373e-05, + "loss": 1.3711, + "theoretical_loss": 3.3083556281416033, + "tokens_seen": 3150446592 + }, + { + "epoch": 0.95, + "learning_rate": 2.2988966900702107e-05, + "loss": 1.2951, + "theoretical_loss": 3.3083451504700307, + "tokens_seen": 3150577664 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.46462610363960266, + "objective/train/avg_token_score": 0.032059360295534134, + "objective/train/docs_used": 1770077, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4072775840759277, + "objective/train/original_loss": 3.62638258934021, + "objective/train/theoretical_loss": 3.308339911843473, + "objective/train/tokens_used": 3150643200, + "objective/train/unlikelihood_loss": 0.15498240292072296, + "theoretical_loss": 3.308339911843473, + "tokens_seen": 3150643200 + }, + { + "epoch": 0.95, + "learning_rate": 2.2968906720160482e-05, + "loss": 1.2665, + "theoretical_loss": 3.3083346733563923, + "tokens_seen": 3150708736 + }, + { + "epoch": 0.95, + "learning_rate": 2.2948846539618855e-05, + "loss": 1.1788, + "theoretical_loss": 3.3083241968006347, + "tokens_seen": 3150839808 + }, + { + "epoch": 0.95, + "learning_rate": 2.2928786359077234e-05, + "loss": 1.2799, + "theoretical_loss": 3.3083137208027056, + "tokens_seen": 3150970880 + }, + { + "epoch": 0.95, + "learning_rate": 2.2908726178535606e-05, + "loss": 1.346, + "theoretical_loss": 3.308303245362552, + "tokens_seen": 3151101952 + }, + { + "epoch": 0.95, + "learning_rate": 2.2888665997993982e-05, + "loss": 1.4803, + "theoretical_loss": 3.308292770480121, + "tokens_seen": 3151233024 + }, + { + "epoch": 0.95, + "learning_rate": 2.2868605817452358e-05, + "loss": 1.3253, + "theoretical_loss": 3.308282296155359, + "tokens_seen": 3151364096 + }, + { + "epoch": 0.95, + "learning_rate": 2.2848545636910734e-05, + "loss": 1.322, + "theoretical_loss": 3.3082718223882144, + "tokens_seen": 3151495168 + }, + { + "epoch": 0.96, + "learning_rate": 2.282848545636911e-05, + "loss": 1.2439, + "theoretical_loss": 3.3082613491786335, + "tokens_seen": 3151626240 + }, + { + "epoch": 0.96, + "learning_rate": 2.2808425275827482e-05, + "loss": 1.3147, + "theoretical_loss": 3.3082508765265635, + "tokens_seen": 3151757312 + }, + { + "epoch": 0.96, + "learning_rate": 2.2788365095285858e-05, + "loss": 1.2377, + "theoretical_loss": 3.3082404044319516, + "tokens_seen": 3151888384 + }, + { + "epoch": 0.96, + "learning_rate": 2.276830491474423e-05, + "loss": 1.3124, + "theoretical_loss": 3.308229932894745, + "tokens_seen": 3152019456 + }, + { + "epoch": 0.96, + "learning_rate": 2.274824473420261e-05, + "loss": 1.4442, + "theoretical_loss": 3.3082194619148906, + "tokens_seen": 3152150528 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.4214320480823517, + "objective/train/avg_token_score": 0.017990946769714355, + "objective/train/docs_used": 1770788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.211387276649475, + "objective/train/original_loss": 3.924201488494873, + "objective/train/theoretical_loss": 3.308208991492336, + "objective/train/tokens_used": 3152281600, + "objective/train/unlikelihood_loss": 0.14398783445358276, + "theoretical_loss": 3.308208991492336, + "tokens_seen": 3152281600 + }, + { + "epoch": 0.96, + "learning_rate": 2.2728184553660985e-05, + "loss": 1.2983, + "theoretical_loss": 3.308208991492336, + "tokens_seen": 3152281600 + }, + { + "epoch": 0.96, + "learning_rate": 2.2708124373119358e-05, + "loss": 1.4133, + "theoretical_loss": 3.3081985216270278, + "tokens_seen": 3152412672 + }, + { + "epoch": 0.96, + "learning_rate": 2.2688064192577733e-05, + "loss": 1.2764, + "theoretical_loss": 3.308188052318914, + "tokens_seen": 3152543744 + }, + { + "epoch": 0.96, + "learning_rate": 2.2668004012036106e-05, + "loss": 1.1452, + "theoretical_loss": 3.3081775835679412, + "tokens_seen": 3152674816 + }, + { + "epoch": 0.96, + "learning_rate": 2.2647943831494485e-05, + "loss": 1.3852, + "theoretical_loss": 3.3081671153740566, + "tokens_seen": 3152805888 + }, + { + "epoch": 0.96, + "learning_rate": 2.262788365095286e-05, + "loss": 1.2045, + "theoretical_loss": 3.3081566477372073, + "tokens_seen": 3152936960 + }, + { + "epoch": 0.96, + "learning_rate": 2.2607823470411233e-05, + "loss": 1.2597, + "theoretical_loss": 3.3081461806573413, + "tokens_seen": 3153068032 + }, + { + "epoch": 0.96, + "learning_rate": 2.258776328986961e-05, + "loss": 1.4029, + "theoretical_loss": 3.3081357141344045, + "tokens_seen": 3153199104 + }, + { + "epoch": 0.96, + "learning_rate": 2.2567703109327985e-05, + "loss": 1.3023, + "theoretical_loss": 3.3081252481683454, + "tokens_seen": 3153330176 + }, + { + "epoch": 0.96, + "learning_rate": 2.254764292878636e-05, + "loss": 1.2434, + "theoretical_loss": 3.30811478275911, + "tokens_seen": 3153461248 + }, + { + "epoch": 0.96, + "learning_rate": 2.2527582748244736e-05, + "loss": 1.1998, + "theoretical_loss": 3.308104317906647, + "tokens_seen": 3153592320 + }, + { + "epoch": 0.96, + "learning_rate": 2.250752256770311e-05, + "loss": 1.3185, + "theoretical_loss": 3.3080938536109024, + "tokens_seen": 3153723392 + }, + { + "epoch": 0.96, + "learning_rate": 2.2487462387161485e-05, + "loss": 1.16, + "theoretical_loss": 3.3080833898718236, + "tokens_seen": 3153854464 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.37567204236984253, + "objective/train/avg_token_score": 0.060888320207595825, + "objective/train/docs_used": 1772131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2158774137496948, + "objective/train/original_loss": 4.130688190460205, + "objective/train/theoretical_loss": 3.3080781582110177, + "objective/train/tokens_used": 3153920000, + "objective/train/unlikelihood_loss": 0.14108607172966003, + "theoretical_loss": 3.3080781582110177, + "tokens_seen": 3153920000 + }, + { + "epoch": 0.96, + "learning_rate": 2.246740220661986e-05, + "loss": 1.2996, + "theoretical_loss": 3.3080729266893583, + "tokens_seen": 3153985536 + }, + { + "epoch": 0.96, + "learning_rate": 2.2447342026078236e-05, + "loss": 1.2054, + "theoretical_loss": 3.3080624640634535, + "tokens_seen": 3154116608 + }, + { + "epoch": 0.96, + "learning_rate": 2.242728184553661e-05, + "loss": 1.1948, + "theoretical_loss": 3.308052001994057, + "tokens_seen": 3154247680 + }, + { + "epoch": 0.96, + "learning_rate": 2.2407221664994984e-05, + "loss": 1.2955, + "theoretical_loss": 3.308041540481115, + "tokens_seen": 3154378752 + }, + { + "epoch": 0.96, + "learning_rate": 2.238716148445336e-05, + "loss": 1.2449, + "theoretical_loss": 3.308031079524576, + "tokens_seen": 3154509824 + }, + { + "epoch": 0.96, + "learning_rate": 2.2367101303911736e-05, + "loss": 1.2005, + "theoretical_loss": 3.3080206191243864, + "tokens_seen": 3154640896 + }, + { + "epoch": 0.96, + "learning_rate": 2.2347041123370112e-05, + "loss": 1.3604, + "theoretical_loss": 3.308010159280494, + "tokens_seen": 3154771968 + }, + { + "epoch": 0.96, + "learning_rate": 2.2326980942828484e-05, + "loss": 1.3058, + "theoretical_loss": 3.3079996999928456, + "tokens_seen": 3154903040 + }, + { + "epoch": 0.96, + "learning_rate": 2.230692076228686e-05, + "loss": 1.2948, + "theoretical_loss": 3.3079892412613887, + "tokens_seen": 3155034112 + }, + { + "epoch": 0.96, + "learning_rate": 2.228686058174524e-05, + "loss": 1.3152, + "theoretical_loss": 3.3079787830860714, + "tokens_seen": 3155165184 + }, + { + "epoch": 0.96, + "learning_rate": 2.226680040120361e-05, + "loss": 1.2096, + "theoretical_loss": 3.30796832546684, + "tokens_seen": 3155296256 + }, + { + "epoch": 0.96, + "learning_rate": 2.2246740220661987e-05, + "loss": 1.2731, + "theoretical_loss": 3.3079578684036424, + "tokens_seen": 3155427328 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.5427052974700928, + "objective/train/avg_token_score": 0.017049681395292282, + "objective/train/docs_used": 1772760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4243935346603394, + "objective/train/original_loss": 3.553797483444214, + "objective/train/theoretical_loss": 3.307947411896425, + "objective/train/tokens_used": 3155558400, + "objective/train/unlikelihood_loss": 0.10115451365709305, + "theoretical_loss": 3.307947411896425, + "tokens_seen": 3155558400 + }, + { + "epoch": 0.96, + "learning_rate": 2.222668004012036e-05, + "loss": 1.207, + "theoretical_loss": 3.307947411896425, + "tokens_seen": 3155558400 + }, + { + "epoch": 0.96, + "learning_rate": 2.2206619859578736e-05, + "loss": 1.3106, + "theoretical_loss": 3.3079369559451366, + "tokens_seen": 3155689472 + }, + { + "epoch": 0.96, + "learning_rate": 2.2186559679037115e-05, + "loss": 1.179, + "theoretical_loss": 3.3079265005497236, + "tokens_seen": 3155820544 + }, + { + "epoch": 0.96, + "learning_rate": 2.2166499498495487e-05, + "loss": 1.3902, + "theoretical_loss": 3.3079160457101335, + "tokens_seen": 3155951616 + }, + { + "epoch": 0.96, + "learning_rate": 2.2146439317953863e-05, + "loss": 1.2776, + "theoretical_loss": 3.307905591426314, + "tokens_seen": 3156082688 + }, + { + "epoch": 0.96, + "learning_rate": 2.2126379137412235e-05, + "loss": 1.2039, + "theoretical_loss": 3.307895137698212, + "tokens_seen": 3156213760 + }, + { + "epoch": 0.96, + "learning_rate": 2.210631895687061e-05, + "loss": 1.3092, + "theoretical_loss": 3.3078846845257757, + "tokens_seen": 3156344832 + }, + { + "epoch": 0.96, + "learning_rate": 2.208625877632899e-05, + "loss": 1.2199, + "theoretical_loss": 3.3078742319089516, + "tokens_seen": 3156475904 + }, + { + "epoch": 0.96, + "learning_rate": 2.2066198595787363e-05, + "loss": 1.073, + "theoretical_loss": 3.307863779847687, + "tokens_seen": 3156606976 + }, + { + "epoch": 0.96, + "learning_rate": 2.204613841524574e-05, + "loss": 1.4172, + "theoretical_loss": 3.3078533283419302, + "tokens_seen": 3156738048 + }, + { + "epoch": 0.96, + "learning_rate": 2.202607823470411e-05, + "loss": 1.2976, + "theoretical_loss": 3.3078428773916277, + "tokens_seen": 3156869120 + }, + { + "epoch": 0.96, + "learning_rate": 2.200601805416249e-05, + "loss": 1.0062, + "theoretical_loss": 3.307832426996728, + "tokens_seen": 3157000192 + }, + { + "epoch": 0.96, + "learning_rate": 2.1985957873620862e-05, + "loss": 1.1436, + "theoretical_loss": 3.307821977157177, + "tokens_seen": 3157131264 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.25659823417663574, + "objective/train/avg_token_score": 0.008527098223567009, + "objective/train/docs_used": 1773921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.775353193283081, + "objective/train/original_loss": 3.569100856781006, + "objective/train/theoretical_loss": 3.3078167524456417, + "objective/train/tokens_used": 3157196800, + "objective/train/unlikelihood_loss": 0.23370854556560516, + "theoretical_loss": 3.3078167524456417, + "tokens_seen": 3157196800 + }, + { + "epoch": 0.96, + "learning_rate": 2.1965897693079238e-05, + "loss": 1.2654, + "theoretical_loss": 3.307811527872924, + "tokens_seen": 3157262336 + }, + { + "epoch": 0.96, + "learning_rate": 2.1945837512537614e-05, + "loss": 1.3595, + "theoretical_loss": 3.307801079143915, + "tokens_seen": 3157393408 + }, + { + "epoch": 0.96, + "learning_rate": 2.1925777331995986e-05, + "loss": 1.1177, + "theoretical_loss": 3.3077906309700977, + "tokens_seen": 3157524480 + }, + { + "epoch": 0.96, + "learning_rate": 2.1905717151454366e-05, + "loss": 1.2007, + "theoretical_loss": 3.30778018335142, + "tokens_seen": 3157655552 + }, + { + "epoch": 0.96, + "learning_rate": 2.1885656970912738e-05, + "loss": 1.3159, + "theoretical_loss": 3.3077697362878293, + "tokens_seen": 3157786624 + }, + { + "epoch": 0.96, + "learning_rate": 2.1865596790371114e-05, + "loss": 1.2775, + "theoretical_loss": 3.3077592897792725, + "tokens_seen": 3157917696 + }, + { + "epoch": 0.96, + "learning_rate": 2.184553660982949e-05, + "loss": 1.3785, + "theoretical_loss": 3.3077488438256974, + "tokens_seen": 3158048768 + }, + { + "epoch": 0.96, + "learning_rate": 2.1825476429287862e-05, + "loss": 1.3767, + "theoretical_loss": 3.307738398427052, + "tokens_seen": 3158179840 + }, + { + "epoch": 0.96, + "learning_rate": 2.180541624874624e-05, + "loss": 1.1838, + "theoretical_loss": 3.3077279535832833, + "tokens_seen": 3158310912 + }, + { + "epoch": 0.96, + "learning_rate": 2.1785356068204614e-05, + "loss": 1.2776, + "theoretical_loss": 3.3077175092943385, + "tokens_seen": 3158441984 + }, + { + "epoch": 0.96, + "learning_rate": 2.176529588766299e-05, + "loss": 1.247, + "theoretical_loss": 3.307707065560166, + "tokens_seen": 3158573056 + }, + { + "epoch": 0.96, + "learning_rate": 2.1745235707121365e-05, + "loss": 1.3428, + "theoretical_loss": 3.3076966223807123, + "tokens_seen": 3158704128 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.36540812253952026, + "objective/train/avg_token_score": 0.02244303561747074, + "objective/train/docs_used": 1774627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1629581451416016, + "objective/train/original_loss": 4.315372467041016, + "objective/train/theoretical_loss": 3.307686179755925, + "objective/train/tokens_used": 3158835200, + "objective/train/unlikelihood_loss": 0.0962921604514122, + "theoretical_loss": 3.307686179755925, + "tokens_seen": 3158835200 + }, + { + "epoch": 0.96, + "learning_rate": 2.1725175526579738e-05, + "loss": 1.296, + "theoretical_loss": 3.307686179755925, + "tokens_seen": 3158835200 + }, + { + "epoch": 0.96, + "learning_rate": 2.1705115346038117e-05, + "loss": 1.4005, + "theoretical_loss": 3.3076757376857526, + "tokens_seen": 3158966272 + }, + { + "epoch": 0.96, + "learning_rate": 2.168505516549649e-05, + "loss": 1.2747, + "theoretical_loss": 3.3076652961701423, + "tokens_seen": 3159097344 + }, + { + "epoch": 0.96, + "learning_rate": 2.1664994984954865e-05, + "loss": 1.3111, + "theoretical_loss": 3.307654855209041, + "tokens_seen": 3159228416 + }, + { + "epoch": 0.96, + "learning_rate": 2.1644934804413237e-05, + "loss": 1.1484, + "theoretical_loss": 3.3076444148023967, + "tokens_seen": 3159359488 + }, + { + "epoch": 0.96, + "learning_rate": 2.1624874623871617e-05, + "loss": 1.1508, + "theoretical_loss": 3.3076339749501567, + "tokens_seen": 3159490560 + }, + { + "epoch": 0.96, + "learning_rate": 2.1604814443329992e-05, + "loss": 1.3958, + "theoretical_loss": 3.3076235356522687, + "tokens_seen": 3159621632 + }, + { + "epoch": 0.96, + "learning_rate": 2.1584754262788365e-05, + "loss": 1.2157, + "theoretical_loss": 3.3076130969086806, + "tokens_seen": 3159752704 + }, + { + "epoch": 0.96, + "learning_rate": 2.156469408224674e-05, + "loss": 1.3377, + "theoretical_loss": 3.3076026587193397, + "tokens_seen": 3159883776 + }, + { + "epoch": 0.96, + "learning_rate": 2.1544633901705113e-05, + "loss": 1.2603, + "theoretical_loss": 3.3075922210841933, + "tokens_seen": 3160014848 + }, + { + "epoch": 0.96, + "learning_rate": 2.1524573721163492e-05, + "loss": 1.2302, + "theoretical_loss": 3.3075817840031894, + "tokens_seen": 3160145920 + }, + { + "epoch": 0.96, + "learning_rate": 2.1504513540621868e-05, + "loss": 1.3141, + "theoretical_loss": 3.3075713474762756, + "tokens_seen": 3160276992 + }, + { + "epoch": 0.96, + "learning_rate": 2.148445336008024e-05, + "loss": 1.2724, + "theoretical_loss": 3.307560911503399, + "tokens_seen": 3160408064 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.3727394938468933, + "objective/train/avg_token_score": 0.007216854020953178, + "objective/train/docs_used": 1775993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0242873430252075, + "objective/train/original_loss": 3.401794195175171, + "objective/train/theoretical_loss": 3.3075556937247086, + "objective/train/tokens_used": 3160473600, + "objective/train/unlikelihood_loss": 0.187776118516922, + "theoretical_loss": 3.3075556937247086, + "tokens_seen": 3160473600 + }, + { + "epoch": 0.96, + "learning_rate": 2.1464393179538616e-05, + "loss": 1.3471, + "theoretical_loss": 3.307550476084508, + "tokens_seen": 3160539136 + }, + { + "epoch": 0.96, + "learning_rate": 2.144433299899699e-05, + "loss": 1.1811, + "theoretical_loss": 3.3075400412195495, + "tokens_seen": 3160670208 + }, + { + "epoch": 0.96, + "learning_rate": 2.1424272818455368e-05, + "loss": 1.3196, + "theoretical_loss": 3.3075296069084716, + "tokens_seen": 3160801280 + }, + { + "epoch": 0.96, + "learning_rate": 2.1404212637913744e-05, + "loss": 1.1921, + "theoretical_loss": 3.3075191731512215, + "tokens_seen": 3160932352 + }, + { + "epoch": 0.96, + "learning_rate": 2.1384152457372116e-05, + "loss": 1.2573, + "theoretical_loss": 3.307508739947748, + "tokens_seen": 3161063424 + }, + { + "epoch": 0.96, + "learning_rate": 2.1364092276830492e-05, + "loss": 1.2482, + "theoretical_loss": 3.3074983072979967, + "tokens_seen": 3161194496 + }, + { + "epoch": 0.96, + "learning_rate": 2.1344032096288868e-05, + "loss": 1.256, + "theoretical_loss": 3.307487875201917, + "tokens_seen": 3161325568 + }, + { + "epoch": 0.96, + "learning_rate": 2.1323971915747243e-05, + "loss": 1.1493, + "theoretical_loss": 3.307477443659456, + "tokens_seen": 3161456640 + }, + { + "epoch": 0.96, + "learning_rate": 2.1303911735205616e-05, + "loss": 1.3196, + "theoretical_loss": 3.307467012670561, + "tokens_seen": 3161587712 + }, + { + "epoch": 0.96, + "learning_rate": 2.128385155466399e-05, + "loss": 1.3473, + "theoretical_loss": 3.3074565822351802, + "tokens_seen": 3161718784 + }, + { + "epoch": 0.96, + "learning_rate": 2.1263791374122367e-05, + "loss": 1.3298, + "theoretical_loss": 3.3074461523532612, + "tokens_seen": 3161849856 + }, + { + "epoch": 0.96, + "learning_rate": 2.1243731193580743e-05, + "loss": 1.081, + "theoretical_loss": 3.3074357230247515, + "tokens_seen": 3161980928 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.3655914068222046, + "objective/train/avg_token_score": 0.0324498675763607, + "objective/train/docs_used": 1776652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0920934677124023, + "objective/train/original_loss": 3.7628581523895264, + "objective/train/theoretical_loss": 3.3074252942495987, + "objective/train/tokens_used": 3162112000, + "objective/train/unlikelihood_loss": 0.2500436305999756, + "theoretical_loss": 3.3074252942495987, + "tokens_seen": 3162112000 + }, + { + "epoch": 0.96, + "learning_rate": 2.122367101303912e-05, + "loss": 1.2649, + "theoretical_loss": 3.3074252942495987, + "tokens_seen": 3162112000 + }, + { + "epoch": 0.96, + "learning_rate": 2.120361083249749e-05, + "loss": 1.3032, + "theoretical_loss": 3.307414866027751, + "tokens_seen": 3162243072 + }, + { + "epoch": 0.96, + "learning_rate": 2.1183550651955867e-05, + "loss": 1.3109, + "theoretical_loss": 3.3074044383591557, + "tokens_seen": 3162374144 + }, + { + "epoch": 0.96, + "learning_rate": 2.1163490471414243e-05, + "loss": 1.2819, + "theoretical_loss": 3.3073940112437605, + "tokens_seen": 3162505216 + }, + { + "epoch": 0.96, + "learning_rate": 2.114343029087262e-05, + "loss": 1.2012, + "theoretical_loss": 3.3073835846815136, + "tokens_seen": 3162636288 + }, + { + "epoch": 0.96, + "learning_rate": 2.1123370110330994e-05, + "loss": 1.3251, + "theoretical_loss": 3.307373158672362, + "tokens_seen": 3162767360 + }, + { + "epoch": 0.96, + "learning_rate": 2.1103309929789367e-05, + "loss": 1.2026, + "theoretical_loss": 3.307362733216254, + "tokens_seen": 3162898432 + }, + { + "epoch": 0.96, + "learning_rate": 2.1083249749247743e-05, + "loss": 1.4004, + "theoretical_loss": 3.307352308313137, + "tokens_seen": 3163029504 + }, + { + "epoch": 0.96, + "learning_rate": 2.1063189568706122e-05, + "loss": 1.3116, + "theoretical_loss": 3.307341883962959, + "tokens_seen": 3163160576 + }, + { + "epoch": 0.96, + "learning_rate": 2.1043129388164494e-05, + "loss": 1.2842, + "theoretical_loss": 3.307331460165668, + "tokens_seen": 3163291648 + }, + { + "epoch": 0.96, + "learning_rate": 2.102306920762287e-05, + "loss": 1.2275, + "theoretical_loss": 3.307321036921211, + "tokens_seen": 3163422720 + }, + { + "epoch": 0.96, + "learning_rate": 2.1003009027081242e-05, + "loss": 1.1169, + "theoretical_loss": 3.3073106142295363, + "tokens_seen": 3163553792 + }, + { + "epoch": 0.96, + "learning_rate": 2.0982948846539618e-05, + "loss": 1.2264, + "theoretical_loss": 3.307300192090592, + "tokens_seen": 3163684864 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.3938172161579132, + "objective/train/avg_token_score": 0.011000901460647583, + "objective/train/docs_used": 1777365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.288009762763977, + "objective/train/original_loss": 4.111855983734131, + "objective/train/theoretical_loss": 3.307294981228377, + "objective/train/tokens_used": 3163750400, + "objective/train/unlikelihood_loss": 0.13649454712867737, + "theoretical_loss": 3.307294981228377, + "tokens_seen": 3163750400 + }, + { + "epoch": 0.96, + "learning_rate": 2.0962888665997994e-05, + "loss": 1.4416, + "theoretical_loss": 3.307289770504325, + "tokens_seen": 3163815936 + }, + { + "epoch": 0.96, + "learning_rate": 2.094282848545637e-05, + "loss": 1.1381, + "theoretical_loss": 3.307279349470684, + "tokens_seen": 3163947008 + }, + { + "epoch": 0.96, + "learning_rate": 2.0922768304914746e-05, + "loss": 1.3367, + "theoretical_loss": 3.307268928989616, + "tokens_seen": 3164078080 + }, + { + "epoch": 0.96, + "learning_rate": 2.0902708124373118e-05, + "loss": 1.263, + "theoretical_loss": 3.3072585090610698, + "tokens_seen": 3164209152 + }, + { + "epoch": 0.96, + "learning_rate": 2.0882647943831494e-05, + "loss": 1.4154, + "theoretical_loss": 3.3072480896849923, + "tokens_seen": 3164340224 + }, + { + "epoch": 0.96, + "learning_rate": 2.086258776328987e-05, + "loss": 1.2333, + "theoretical_loss": 3.3072376708613316, + "tokens_seen": 3164471296 + }, + { + "epoch": 0.96, + "learning_rate": 2.0842527582748245e-05, + "loss": 1.3313, + "theoretical_loss": 3.3072272525900352, + "tokens_seen": 3164602368 + }, + { + "epoch": 0.96, + "learning_rate": 2.082246740220662e-05, + "loss": 1.3067, + "theoretical_loss": 3.307216834871052, + "tokens_seen": 3164733440 + }, + { + "epoch": 0.96, + "learning_rate": 2.0802407221664994e-05, + "loss": 1.3493, + "theoretical_loss": 3.3072064177043288, + "tokens_seen": 3164864512 + }, + { + "epoch": 0.96, + "learning_rate": 2.0782347041123373e-05, + "loss": 1.4497, + "theoretical_loss": 3.307196001089814, + "tokens_seen": 3164995584 + }, + { + "epoch": 0.96, + "learning_rate": 2.0762286860581745e-05, + "loss": 1.3618, + "theoretical_loss": 3.3071855850274554, + "tokens_seen": 3165126656 + }, + { + "epoch": 0.96, + "learning_rate": 2.074222668004012e-05, + "loss": 1.2231, + "theoretical_loss": 3.3071751695172007, + "tokens_seen": 3165257728 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.49468475580215454, + "objective/train/avg_token_score": 0.009586219675838947, + "objective/train/docs_used": 1778677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5401928424835205, + "objective/train/original_loss": 3.7870664596557617, + "objective/train/theoretical_loss": 3.3071647545589977, + "objective/train/tokens_used": 3165388800, + "objective/train/unlikelihood_loss": 0.1408991664648056, + "theoretical_loss": 3.3071647545589977, + "tokens_seen": 3165388800 + }, + { + "epoch": 0.96, + "learning_rate": 2.0722166499498497e-05, + "loss": 1.3768, + "theoretical_loss": 3.3071647545589977, + "tokens_seen": 3165388800 + }, + { + "epoch": 0.96, + "learning_rate": 2.070210631895687e-05, + "loss": 1.1747, + "theoretical_loss": 3.3071543401527945, + "tokens_seen": 3165519872 + }, + { + "epoch": 0.96, + "learning_rate": 2.068204613841525e-05, + "loss": 1.3358, + "theoretical_loss": 3.307143926298539, + "tokens_seen": 3165650944 + }, + { + "epoch": 0.96, + "learning_rate": 2.066198595787362e-05, + "loss": 1.0867, + "theoretical_loss": 3.3071335129961787, + "tokens_seen": 3165782016 + }, + { + "epoch": 0.96, + "learning_rate": 2.0641925777331997e-05, + "loss": 1.2788, + "theoretical_loss": 3.3071231002456623, + "tokens_seen": 3165913088 + }, + { + "epoch": 0.96, + "learning_rate": 2.062186559679037e-05, + "loss": 1.1352, + "theoretical_loss": 3.307112688046937, + "tokens_seen": 3166044160 + }, + { + "epoch": 0.96, + "learning_rate": 2.0601805416248745e-05, + "loss": 1.3101, + "theoretical_loss": 3.3071022763999505, + "tokens_seen": 3166175232 + }, + { + "epoch": 0.96, + "learning_rate": 2.0581745235707124e-05, + "loss": 1.3269, + "theoretical_loss": 3.3070918653046517, + "tokens_seen": 3166306304 + }, + { + "epoch": 0.96, + "learning_rate": 2.0561685055165496e-05, + "loss": 1.2641, + "theoretical_loss": 3.3070814547609877, + "tokens_seen": 3166437376 + }, + { + "epoch": 0.96, + "learning_rate": 2.0541624874623872e-05, + "loss": 1.2807, + "theoretical_loss": 3.307071044768907, + "tokens_seen": 3166568448 + }, + { + "epoch": 0.96, + "learning_rate": 2.0521564694082245e-05, + "loss": 1.2675, + "theoretical_loss": 3.307060635328357, + "tokens_seen": 3166699520 + }, + { + "epoch": 0.96, + "learning_rate": 2.0501504513540624e-05, + "loss": 1.3739, + "theoretical_loss": 3.3070502264392863, + "tokens_seen": 3166830592 + }, + { + "epoch": 0.96, + "learning_rate": 2.0481444332999e-05, + "loss": 1.1292, + "theoretical_loss": 3.307039818101642, + "tokens_seen": 3166961664 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.3878909945487976, + "objective/train/avg_token_score": 0.024321170523762703, + "objective/train/docs_used": 1779388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2710319757461548, + "objective/train/original_loss": 4.05733060836792, + "objective/train/theoretical_loss": 3.3070346141395888, + "objective/train/tokens_used": 3167027200, + "objective/train/unlikelihood_loss": 0.1194189041852951, + "theoretical_loss": 3.3070346141395888, + "tokens_seen": 3167027200 + }, + { + "epoch": 0.96, + "learning_rate": 2.0461384152457372e-05, + "loss": 1.241, + "theoretical_loss": 3.307029410315373, + "tokens_seen": 3167092736 + }, + { + "epoch": 0.96, + "learning_rate": 2.0441323971915748e-05, + "loss": 1.3422, + "theoretical_loss": 3.307019003080427, + "tokens_seen": 3167223808 + }, + { + "epoch": 0.96, + "learning_rate": 2.042126379137412e-05, + "loss": 1.4395, + "theoretical_loss": 3.3070085963967513, + "tokens_seen": 3167354880 + }, + { + "epoch": 0.96, + "learning_rate": 2.04012036108325e-05, + "loss": 1.1869, + "theoretical_loss": 3.306998190264294, + "tokens_seen": 3167485952 + }, + { + "epoch": 0.96, + "learning_rate": 2.0381143430290875e-05, + "loss": 1.4348, + "theoretical_loss": 3.3069877846830043, + "tokens_seen": 3167617024 + }, + { + "epoch": 0.96, + "learning_rate": 2.0361083249749248e-05, + "loss": 1.2444, + "theoretical_loss": 3.306977379652829, + "tokens_seen": 3167748096 + }, + { + "epoch": 0.96, + "learning_rate": 2.0341023069207623e-05, + "loss": 1.1292, + "theoretical_loss": 3.306966975173717, + "tokens_seen": 3167879168 + }, + { + "epoch": 0.96, + "learning_rate": 2.0320962888665996e-05, + "loss": 1.3502, + "theoretical_loss": 3.3069565712456157, + "tokens_seen": 3168010240 + }, + { + "epoch": 0.96, + "learning_rate": 2.0300902708124375e-05, + "loss": 1.1157, + "theoretical_loss": 3.306946167868473, + "tokens_seen": 3168141312 + }, + { + "epoch": 0.96, + "learning_rate": 2.028084252758275e-05, + "loss": 1.4061, + "theoretical_loss": 3.306935765042237, + "tokens_seen": 3168272384 + }, + { + "epoch": 0.96, + "learning_rate": 2.0260782347041123e-05, + "loss": 1.2139, + "theoretical_loss": 3.3069253627668562, + "tokens_seen": 3168403456 + }, + { + "epoch": 0.96, + "learning_rate": 2.02407221664995e-05, + "loss": 1.2351, + "theoretical_loss": 3.306914961042278, + "tokens_seen": 3168534528 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.3707233667373657, + "objective/train/avg_token_score": 0.025662295520305634, + "objective/train/docs_used": 1780657, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1419682502746582, + "objective/train/original_loss": 3.8702306747436523, + "objective/train/theoretical_loss": 3.3069045598684514, + "objective/train/tokens_used": 3168665600, + "objective/train/unlikelihood_loss": 0.13920243084430695, + "theoretical_loss": 3.3069045598684514, + "tokens_seen": 3168665600 + }, + { + "epoch": 0.96, + "learning_rate": 2.0220661985957875e-05, + "loss": 1.234, + "theoretical_loss": 3.3069045598684514, + "tokens_seen": 3168665600 + }, + { + "epoch": 0.96, + "learning_rate": 2.020060180541625e-05, + "loss": 1.2831, + "theoretical_loss": 3.3068941592453234, + "tokens_seen": 3168796672 + }, + { + "epoch": 0.96, + "learning_rate": 2.0180541624874623e-05, + "loss": 1.3588, + "theoretical_loss": 3.306883759172843, + "tokens_seen": 3168927744 + }, + { + "epoch": 0.96, + "learning_rate": 2.0160481444333e-05, + "loss": 1.3867, + "theoretical_loss": 3.3068733596509574, + "tokens_seen": 3169058816 + }, + { + "epoch": 0.96, + "learning_rate": 2.0140421263791374e-05, + "loss": 1.183, + "theoretical_loss": 3.306862960679615, + "tokens_seen": 3169189888 + }, + { + "epoch": 0.96, + "learning_rate": 2.012036108324975e-05, + "loss": 1.2972, + "theoretical_loss": 3.306852562258764, + "tokens_seen": 3169320960 + }, + { + "epoch": 0.96, + "learning_rate": 2.0100300902708126e-05, + "loss": 1.2421, + "theoretical_loss": 3.306842164388353, + "tokens_seen": 3169452032 + }, + { + "epoch": 0.96, + "learning_rate": 2.00802407221665e-05, + "loss": 1.2309, + "theoretical_loss": 3.306831767068329, + "tokens_seen": 3169583104 + }, + { + "epoch": 0.96, + "learning_rate": 2.0060180541624874e-05, + "loss": 1.1049, + "theoretical_loss": 3.3068213702986413, + "tokens_seen": 3169714176 + }, + { + "epoch": 0.96, + "learning_rate": 2.004012036108325e-05, + "loss": 1.1236, + "theoretical_loss": 3.306810974079237, + "tokens_seen": 3169845248 + }, + { + "epoch": 0.96, + "learning_rate": 2.0020060180541626e-05, + "loss": 1.3014, + "theoretical_loss": 3.3068005784100647, + "tokens_seen": 3169976320 + }, + { + "epoch": 0.96, + "learning_rate": 2e-05, + "loss": 1.2155, + "theoretical_loss": 3.3067901832910724, + "tokens_seen": 3170107392 + }, + { + "epoch": 0.96, + "learning_rate": 1.9979939819458374e-05, + "loss": 1.2969, + "theoretical_loss": 3.306779788722208, + "tokens_seen": 3170238464 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.1931818127632141, + "objective/train/avg_token_score": 0.043262939900159836, + "objective/train/docs_used": 1781310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.5665502548217773, + "objective/train/original_loss": 4.588189125061035, + "objective/train/theoretical_loss": 3.306774591644058, + "objective/train/tokens_used": 3170304000, + "objective/train/unlikelihood_loss": 0.15187542140483856, + "theoretical_loss": 3.306774591644058, + "tokens_seen": 3170304000 + }, + { + "epoch": 0.96, + "learning_rate": 1.995987963891675e-05, + "loss": 1.0942, + "theoretical_loss": 3.3067693947034202, + "tokens_seen": 3170369536 + }, + { + "epoch": 0.96, + "learning_rate": 1.993981945837513e-05, + "loss": 1.2607, + "theoretical_loss": 3.306759001234657, + "tokens_seen": 3170500608 + }, + { + "epoch": 0.96, + "learning_rate": 1.99197592778335e-05, + "loss": 1.1891, + "theoretical_loss": 3.3067486083158664, + "tokens_seen": 3170631680 + }, + { + "epoch": 0.96, + "learning_rate": 1.9899699097291877e-05, + "loss": 1.231, + "theoretical_loss": 3.3067382159469965, + "tokens_seen": 3170762752 + }, + { + "epoch": 0.96, + "learning_rate": 1.987963891675025e-05, + "loss": 1.2161, + "theoretical_loss": 3.3067278241279956, + "tokens_seen": 3170893824 + }, + { + "epoch": 0.96, + "learning_rate": 1.9859578736208625e-05, + "loss": 1.4253, + "theoretical_loss": 3.3067174328588114, + "tokens_seen": 3171024896 + }, + { + "epoch": 0.96, + "learning_rate": 1.9839518555667e-05, + "loss": 1.373, + "theoretical_loss": 3.306707042139393, + "tokens_seen": 3171155968 + }, + { + "epoch": 0.96, + "learning_rate": 1.9819458375125377e-05, + "loss": 1.2903, + "theoretical_loss": 3.306696651969688, + "tokens_seen": 3171287040 + }, + { + "epoch": 0.96, + "learning_rate": 1.9799398194583753e-05, + "loss": 1.2448, + "theoretical_loss": 3.306686262349645, + "tokens_seen": 3171418112 + }, + { + "epoch": 0.96, + "learning_rate": 1.9779338014042125e-05, + "loss": 1.3442, + "theoretical_loss": 3.306675873279212, + "tokens_seen": 3171549184 + }, + { + "epoch": 0.96, + "learning_rate": 1.97592778335005e-05, + "loss": 1.1593, + "theoretical_loss": 3.3066654847583368, + "tokens_seen": 3171680256 + }, + { + "epoch": 0.96, + "learning_rate": 1.9739217652958877e-05, + "loss": 1.3065, + "theoretical_loss": 3.306655096786968, + "tokens_seen": 3171811328 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.3057795763015747, + "objective/train/avg_token_score": 0.008834543637931347, + "objective/train/docs_used": 1782644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9707791805267334, + "objective/train/original_loss": 4.2016777992248535, + "objective/train/theoretical_loss": 3.3066447093650537, + "objective/train/tokens_used": 3171942400, + "objective/train/unlikelihood_loss": 0.15854141116142273, + "theoretical_loss": 3.3066447093650537, + "tokens_seen": 3171942400 + }, + { + "epoch": 0.96, + "learning_rate": 1.9719157472417253e-05, + "loss": 1.3318, + "theoretical_loss": 3.3066447093650537, + "tokens_seen": 3171942400 + }, + { + "epoch": 0.96, + "learning_rate": 1.969909729187563e-05, + "loss": 1.3191, + "theoretical_loss": 3.3066343224925423, + "tokens_seen": 3172073472 + }, + { + "epoch": 0.96, + "learning_rate": 1.9679037111334e-05, + "loss": 1.289, + "theoretical_loss": 3.306623936169382, + "tokens_seen": 3172204544 + }, + { + "epoch": 0.96, + "learning_rate": 1.965897693079238e-05, + "loss": 1.1644, + "theoretical_loss": 3.306613550395521, + "tokens_seen": 3172335616 + }, + { + "epoch": 0.96, + "learning_rate": 1.9638916750250752e-05, + "loss": 1.2308, + "theoretical_loss": 3.3066031651709076, + "tokens_seen": 3172466688 + }, + { + "epoch": 0.96, + "learning_rate": 1.9618856569709128e-05, + "loss": 1.2688, + "theoretical_loss": 3.30659278049549, + "tokens_seen": 3172597760 + }, + { + "epoch": 0.96, + "learning_rate": 1.9598796389167504e-05, + "loss": 1.2262, + "theoretical_loss": 3.3065823963692167, + "tokens_seen": 3172728832 + }, + { + "epoch": 0.96, + "learning_rate": 1.9578736208625876e-05, + "loss": 1.3657, + "theoretical_loss": 3.3065720127920355, + "tokens_seen": 3172859904 + }, + { + "epoch": 0.96, + "learning_rate": 1.9558676028084256e-05, + "loss": 1.2565, + "theoretical_loss": 3.306561629763895, + "tokens_seen": 3172990976 + }, + { + "epoch": 0.96, + "learning_rate": 1.9538615847542628e-05, + "loss": 1.3393, + "theoretical_loss": 3.3065512472847436, + "tokens_seen": 3173122048 + }, + { + "epoch": 0.96, + "learning_rate": 1.9518555667001004e-05, + "loss": 1.2188, + "theoretical_loss": 3.30654086535453, + "tokens_seen": 3173253120 + }, + { + "epoch": 0.96, + "learning_rate": 1.9498495486459376e-05, + "loss": 1.2667, + "theoretical_loss": 3.306530483973201, + "tokens_seen": 3173384192 + }, + { + "epoch": 0.96, + "learning_rate": 1.9478435305917752e-05, + "loss": 1.1919, + "theoretical_loss": 3.3065201031407065, + "tokens_seen": 3173515264 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.34610214829444885, + "objective/train/avg_token_score": 0.020223746076226234, + "objective/train/docs_used": 1783141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1582860946655273, + "objective/train/original_loss": 3.989136219024658, + "objective/train/theoretical_loss": 3.306514912930256, + "objective/train/tokens_used": 3173580800, + "objective/train/unlikelihood_loss": 0.14818865060806274, + "theoretical_loss": 3.306514912930256, + "tokens_seen": 3173580800 + }, + { + "epoch": 0.96, + "learning_rate": 1.945837512537613e-05, + "loss": 1.2581, + "theoretical_loss": 3.306509722856994, + "tokens_seen": 3173646336 + }, + { + "epoch": 0.96, + "learning_rate": 1.9438314944834504e-05, + "loss": 1.3264, + "theoretical_loss": 3.3064993431220118, + "tokens_seen": 3173777408 + }, + { + "epoch": 0.96, + "learning_rate": 1.941825476429288e-05, + "loss": 1.2092, + "theoretical_loss": 3.3064889639357085, + "tokens_seen": 3173908480 + }, + { + "epoch": 0.96, + "learning_rate": 1.9398194583751252e-05, + "loss": 1.4408, + "theoretical_loss": 3.3064785852980325, + "tokens_seen": 3174039552 + }, + { + "epoch": 0.96, + "learning_rate": 1.937813440320963e-05, + "loss": 1.3371, + "theoretical_loss": 3.306468207208932, + "tokens_seen": 3174170624 + }, + { + "epoch": 0.96, + "learning_rate": 1.9358074222668007e-05, + "loss": 1.3123, + "theoretical_loss": 3.3064578296683553, + "tokens_seen": 3174301696 + }, + { + "epoch": 0.96, + "learning_rate": 1.933801404212638e-05, + "loss": 1.2094, + "theoretical_loss": 3.306447452676251, + "tokens_seen": 3174432768 + }, + { + "epoch": 0.96, + "learning_rate": 1.9317953861584755e-05, + "loss": 1.3234, + "theoretical_loss": 3.306437076232567, + "tokens_seen": 3174563840 + }, + { + "epoch": 0.96, + "learning_rate": 1.9297893681043127e-05, + "loss": 1.2122, + "theoretical_loss": 3.3064267003372527, + "tokens_seen": 3174694912 + }, + { + "epoch": 0.96, + "learning_rate": 1.9277833500501506e-05, + "loss": 1.2734, + "theoretical_loss": 3.306416324990255, + "tokens_seen": 3174825984 + }, + { + "epoch": 0.96, + "learning_rate": 1.9257773319959882e-05, + "loss": 1.2165, + "theoretical_loss": 3.3064059501915235, + "tokens_seen": 3174957056 + }, + { + "epoch": 0.96, + "learning_rate": 1.9237713139418255e-05, + "loss": 1.2444, + "theoretical_loss": 3.306395575941006, + "tokens_seen": 3175088128 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.45992180705070496, + "objective/train/avg_token_score": 0.006896538194268942, + "objective/train/docs_used": 1784538, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3687525987625122, + "objective/train/original_loss": 3.515979766845703, + "objective/train/theoretical_loss": 3.3063852022386513, + "objective/train/tokens_used": 3175219200, + "objective/train/unlikelihood_loss": 0.24086008965969086, + "theoretical_loss": 3.3063852022386513, + "tokens_seen": 3175219200 + }, + { + "epoch": 0.96, + "learning_rate": 1.921765295887663e-05, + "loss": 1.2873, + "theoretical_loss": 3.3063852022386513, + "tokens_seen": 3175219200 + }, + { + "epoch": 0.96, + "learning_rate": 1.9197592778335003e-05, + "loss": 1.2101, + "theoretical_loss": 3.3063748290844073, + "tokens_seen": 3175350272 + }, + { + "epoch": 0.96, + "learning_rate": 1.9177532597793382e-05, + "loss": 1.22, + "theoretical_loss": 3.306364456478223, + "tokens_seen": 3175481344 + }, + { + "epoch": 0.96, + "learning_rate": 1.9157472417251758e-05, + "loss": 1.2971, + "theoretical_loss": 3.3063540844200463, + "tokens_seen": 3175612416 + }, + { + "epoch": 0.96, + "learning_rate": 1.913741223671013e-05, + "loss": 1.3221, + "theoretical_loss": 3.306343712909826, + "tokens_seen": 3175743488 + }, + { + "epoch": 0.96, + "learning_rate": 1.9117352056168506e-05, + "loss": 1.3488, + "theoretical_loss": 3.30633334194751, + "tokens_seen": 3175874560 + }, + { + "epoch": 0.96, + "learning_rate": 1.909729187562688e-05, + "loss": 1.3456, + "theoretical_loss": 3.3063229715330475, + "tokens_seen": 3176005632 + }, + { + "epoch": 0.96, + "learning_rate": 1.9077231695085258e-05, + "loss": 1.3215, + "theoretical_loss": 3.306312601666386, + "tokens_seen": 3176136704 + }, + { + "epoch": 0.96, + "learning_rate": 1.905717151454363e-05, + "loss": 1.3231, + "theoretical_loss": 3.306302232347475, + "tokens_seen": 3176267776 + }, + { + "epoch": 0.96, + "learning_rate": 1.9037111334002006e-05, + "loss": 1.3568, + "theoretical_loss": 3.306291863576263, + "tokens_seen": 3176398848 + }, + { + "epoch": 0.96, + "learning_rate": 1.901705115346038e-05, + "loss": 1.275, + "theoretical_loss": 3.306281495352697, + "tokens_seen": 3176529920 + }, + { + "epoch": 0.96, + "learning_rate": 1.8996990972918757e-05, + "loss": 1.3647, + "theoretical_loss": 3.3062711276767267, + "tokens_seen": 3176660992 + }, + { + "epoch": 0.96, + "learning_rate": 1.8976930792377133e-05, + "loss": 1.1731, + "theoretical_loss": 3.3062607605483008, + "tokens_seen": 3176792064 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.5745967626571655, + "objective/train/avg_token_score": 0.006223473697900772, + "objective/train/docs_used": 1785217, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6324126720428467, + "objective/train/original_loss": 3.296638250350952, + "objective/train/theoretical_loss": 3.3062555771894004, + "objective/train/tokens_used": 3176857600, + "objective/train/unlikelihood_loss": 0.14616720378398895, + "theoretical_loss": 3.3062555771894004, + "tokens_seen": 3176857600 + }, + { + "epoch": 0.96, + "learning_rate": 1.8956870611835506e-05, + "loss": 1.3445, + "theoretical_loss": 3.3062503939673666, + "tokens_seen": 3176923136 + }, + { + "epoch": 0.96, + "learning_rate": 1.893681043129388e-05, + "loss": 1.2067, + "theoretical_loss": 3.3062400279338737, + "tokens_seen": 3177054208 + }, + { + "epoch": 0.96, + "learning_rate": 1.8916750250752257e-05, + "loss": 1.2245, + "theoretical_loss": 3.3062296624477705, + "tokens_seen": 3177185280 + }, + { + "epoch": 0.96, + "learning_rate": 1.8896690070210633e-05, + "loss": 1.2691, + "theoretical_loss": 3.306219297509005, + "tokens_seen": 3177316352 + }, + { + "epoch": 0.96, + "learning_rate": 1.887662988966901e-05, + "loss": 1.3476, + "theoretical_loss": 3.306208933117526, + "tokens_seen": 3177447424 + }, + { + "epoch": 0.96, + "learning_rate": 1.885656970912738e-05, + "loss": 1.4493, + "theoretical_loss": 3.306198569273282, + "tokens_seen": 3177578496 + }, + { + "epoch": 0.96, + "learning_rate": 1.8836509528585757e-05, + "loss": 1.3657, + "theoretical_loss": 3.3061882059762215, + "tokens_seen": 3177709568 + }, + { + "epoch": 0.96, + "learning_rate": 1.881644934804413e-05, + "loss": 1.3334, + "theoretical_loss": 3.306177843226293, + "tokens_seen": 3177840640 + }, + { + "epoch": 0.96, + "learning_rate": 1.879638916750251e-05, + "loss": 1.1374, + "theoretical_loss": 3.3061674810234454, + "tokens_seen": 3177971712 + }, + { + "epoch": 0.96, + "learning_rate": 1.8776328986960884e-05, + "loss": 1.1446, + "theoretical_loss": 3.3061571193676267, + "tokens_seen": 3178102784 + }, + { + "epoch": 0.96, + "learning_rate": 1.8756268806419257e-05, + "loss": 1.4226, + "theoretical_loss": 3.306146758258786, + "tokens_seen": 3178233856 + }, + { + "epoch": 0.96, + "learning_rate": 1.8736208625877633e-05, + "loss": 1.4949, + "theoretical_loss": 3.3061363976968714, + "tokens_seen": 3178364928 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.4261363744735718, + "objective/train/avg_token_score": 0.010226249694824219, + "objective/train/docs_used": 1786825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3086800575256348, + "objective/train/original_loss": 3.971858263015747, + "objective/train/theoretical_loss": 3.3061260376818318, + "objective/train/tokens_used": 3178496000, + "objective/train/unlikelihood_loss": 0.17575375735759735, + "theoretical_loss": 3.3061260376818318, + "tokens_seen": 3178496000 + }, + { + "epoch": 0.96, + "learning_rate": 1.871614844533601e-05, + "loss": 1.3301, + "theoretical_loss": 3.3061260376818318, + "tokens_seen": 3178496000 + }, + { + "epoch": 0.96, + "learning_rate": 1.8696088264794384e-05, + "loss": 1.2672, + "theoretical_loss": 3.3061156782136156, + "tokens_seen": 3178627072 + }, + { + "epoch": 0.96, + "learning_rate": 1.867602808425276e-05, + "loss": 1.3072, + "theoretical_loss": 3.306105319292172, + "tokens_seen": 3178758144 + }, + { + "epoch": 0.96, + "learning_rate": 1.8655967903711132e-05, + "loss": 1.1951, + "theoretical_loss": 3.306094960917448, + "tokens_seen": 3178889216 + }, + { + "epoch": 0.96, + "learning_rate": 1.8635907723169508e-05, + "loss": 1.3682, + "theoretical_loss": 3.3060846030893942, + "tokens_seen": 3179020288 + }, + { + "epoch": 0.96, + "learning_rate": 1.8615847542627884e-05, + "loss": 1.3064, + "theoretical_loss": 3.3060742458079577, + "tokens_seen": 3179151360 + }, + { + "epoch": 0.96, + "learning_rate": 1.859578736208626e-05, + "loss": 1.3938, + "theoretical_loss": 3.306063889073088, + "tokens_seen": 3179282432 + }, + { + "epoch": 0.96, + "learning_rate": 1.8575727181544636e-05, + "loss": 1.3889, + "theoretical_loss": 3.306053532884733, + "tokens_seen": 3179413504 + }, + { + "epoch": 0.96, + "learning_rate": 1.8555667001003008e-05, + "loss": 1.3563, + "theoretical_loss": 3.3060431772428425, + "tokens_seen": 3179544576 + }, + { + "epoch": 0.96, + "learning_rate": 1.8535606820461384e-05, + "loss": 1.3166, + "theoretical_loss": 3.306032822147364, + "tokens_seen": 3179675648 + }, + { + "epoch": 0.96, + "learning_rate": 1.851554663991976e-05, + "loss": 1.3824, + "theoretical_loss": 3.3060224675982464, + "tokens_seen": 3179806720 + }, + { + "epoch": 0.96, + "learning_rate": 1.8495486459378135e-05, + "loss": 1.3407, + "theoretical_loss": 3.306012113595439, + "tokens_seen": 3179937792 + }, + { + "epoch": 0.96, + "learning_rate": 1.847542627883651e-05, + "loss": 1.1926, + "theoretical_loss": 3.3060017601388894, + "tokens_seen": 3180068864 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.3489125072956085, + "objective/train/avg_token_score": 0.022432202473282814, + "objective/train/docs_used": 1787361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0673106908798218, + "objective/train/original_loss": 4.137094020843506, + "objective/train/theoretical_loss": 3.3059965836154457, + "objective/train/tokens_used": 3180134400, + "objective/train/unlikelihood_loss": 0.16577844321727753, + "theoretical_loss": 3.3059965836154457, + "tokens_seen": 3180134400 + }, + { + "epoch": 0.96, + "learning_rate": 1.8455366098294884e-05, + "loss": 1.2486, + "theoretical_loss": 3.305991407228547, + "tokens_seen": 3180199936 + }, + { + "epoch": 0.96, + "learning_rate": 1.8435305917753263e-05, + "loss": 1.3195, + "theoretical_loss": 3.3059810548643602, + "tokens_seen": 3180331008 + }, + { + "epoch": 0.96, + "learning_rate": 1.8415245737211635e-05, + "loss": 1.4166, + "theoretical_loss": 3.305970703046278, + "tokens_seen": 3180462080 + }, + { + "epoch": 0.96, + "learning_rate": 1.839518555667001e-05, + "loss": 1.1746, + "theoretical_loss": 3.3059603517742486, + "tokens_seen": 3180593152 + }, + { + "epoch": 0.96, + "learning_rate": 1.8375125376128383e-05, + "loss": 1.4192, + "theoretical_loss": 3.3059500010482212, + "tokens_seen": 3180724224 + }, + { + "epoch": 0.96, + "learning_rate": 1.835506519558676e-05, + "loss": 1.3103, + "theoretical_loss": 3.3059396508681442, + "tokens_seen": 3180855296 + }, + { + "epoch": 0.96, + "learning_rate": 1.8335005015045138e-05, + "loss": 1.19, + "theoretical_loss": 3.3059293012339666, + "tokens_seen": 3180986368 + }, + { + "epoch": 0.96, + "learning_rate": 1.831494483450351e-05, + "loss": 1.2583, + "theoretical_loss": 3.3059189521456362, + "tokens_seen": 3181117440 + }, + { + "epoch": 0.96, + "learning_rate": 1.8294884653961886e-05, + "loss": 1.1843, + "theoretical_loss": 3.305908603603103, + "tokens_seen": 3181248512 + }, + { + "epoch": 0.96, + "learning_rate": 1.827482447342026e-05, + "loss": 1.4185, + "theoretical_loss": 3.3058982556063152, + "tokens_seen": 3181379584 + }, + { + "epoch": 0.96, + "learning_rate": 1.8254764292878635e-05, + "loss": 1.2536, + "theoretical_loss": 3.3058879081552215, + "tokens_seen": 3181510656 + }, + { + "epoch": 0.96, + "learning_rate": 1.8234704112337014e-05, + "loss": 1.366, + "theoretical_loss": 3.30587756124977, + "tokens_seen": 3181641728 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.6080767512321472, + "objective/train/avg_token_score": 0.006552429404109716, + "objective/train/docs_used": 1788594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.706471562385559, + "objective/train/original_loss": 3.280109167098999, + "objective/train/theoretical_loss": 3.3058672148899104, + "objective/train/tokens_used": 3181772800, + "objective/train/unlikelihood_loss": 0.14258672297000885, + "theoretical_loss": 3.3058672148899104, + "tokens_seen": 3181772800 + }, + { + "epoch": 0.96, + "learning_rate": 1.8214643931795386e-05, + "loss": 1.2514, + "theoretical_loss": 3.3058672148899104, + "tokens_seen": 3181772800 + }, + { + "epoch": 0.96, + "learning_rate": 1.8194583751253762e-05, + "loss": 1.2262, + "theoretical_loss": 3.3058568690755914, + "tokens_seen": 3181903872 + }, + { + "epoch": 0.96, + "learning_rate": 1.8174523570712134e-05, + "loss": 1.3646, + "theoretical_loss": 3.3058465238067614, + "tokens_seen": 3182034944 + }, + { + "epoch": 0.96, + "learning_rate": 1.8154463390170514e-05, + "loss": 1.3562, + "theoretical_loss": 3.3058361790833692, + "tokens_seen": 3182166016 + }, + { + "epoch": 0.96, + "learning_rate": 1.813440320962889e-05, + "loss": 1.295, + "theoretical_loss": 3.3058258349053635, + "tokens_seen": 3182297088 + }, + { + "epoch": 0.96, + "learning_rate": 1.8114343029087262e-05, + "loss": 1.308, + "theoretical_loss": 3.3058154912726936, + "tokens_seen": 3182428160 + }, + { + "epoch": 0.96, + "learning_rate": 1.8094282848545638e-05, + "loss": 1.3688, + "theoretical_loss": 3.3058051481853075, + "tokens_seen": 3182559232 + }, + { + "epoch": 0.96, + "learning_rate": 1.807422266800401e-05, + "loss": 1.236, + "theoretical_loss": 3.3057948056431545, + "tokens_seen": 3182690304 + }, + { + "epoch": 0.96, + "learning_rate": 1.805416248746239e-05, + "loss": 1.3945, + "theoretical_loss": 3.3057844636461833, + "tokens_seen": 3182821376 + }, + { + "epoch": 0.96, + "learning_rate": 1.803410230692076e-05, + "loss": 1.4073, + "theoretical_loss": 3.305774122194343, + "tokens_seen": 3182952448 + }, + { + "epoch": 0.96, + "learning_rate": 1.8014042126379137e-05, + "loss": 1.3692, + "theoretical_loss": 3.305763781287582, + "tokens_seen": 3183083520 + }, + { + "epoch": 0.96, + "learning_rate": 1.7993981945837513e-05, + "loss": 1.3439, + "theoretical_loss": 3.3057534409258493, + "tokens_seen": 3183214592 + }, + { + "epoch": 0.96, + "learning_rate": 1.7973921765295886e-05, + "loss": 1.227, + "theoretical_loss": 3.3057431011090936, + "tokens_seen": 3183345664 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.3182429075241089, + "objective/train/avg_token_score": 0.015484828501939774, + "objective/train/docs_used": 1789261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9438214302062988, + "objective/train/original_loss": 4.183161735534668, + "objective/train/theoretical_loss": 3.305737931405066, + "objective/train/tokens_used": 3183411200, + "objective/train/unlikelihood_loss": 0.1640411913394928, + "theoretical_loss": 3.305737931405066, + "tokens_seen": 3183411200 + }, + { + "epoch": 0.96, + "learning_rate": 1.7953861584754265e-05, + "loss": 1.3055, + "theoretical_loss": 3.305732761837264, + "tokens_seen": 3183476736 + }, + { + "epoch": 0.96, + "learning_rate": 1.7933801404212637e-05, + "loss": 1.4163, + "theoretical_loss": 3.305722423110309, + "tokens_seen": 3183607808 + }, + { + "epoch": 0.96, + "learning_rate": 1.7913741223671013e-05, + "loss": 1.3203, + "theoretical_loss": 3.305712084928178, + "tokens_seen": 3183738880 + }, + { + "epoch": 0.96, + "learning_rate": 1.789368104312939e-05, + "loss": 1.1536, + "theoretical_loss": 3.3057017472908194, + "tokens_seen": 3183869952 + }, + { + "epoch": 0.96, + "learning_rate": 1.7873620862587765e-05, + "loss": 1.3617, + "theoretical_loss": 3.305691410198182, + "tokens_seen": 3184001024 + }, + { + "epoch": 0.96, + "learning_rate": 1.785356068204614e-05, + "loss": 1.2609, + "theoretical_loss": 3.3056810736502147, + "tokens_seen": 3184132096 + }, + { + "epoch": 0.96, + "learning_rate": 1.7833500501504513e-05, + "loss": 1.2634, + "theoretical_loss": 3.305670737646867, + "tokens_seen": 3184263168 + }, + { + "epoch": 0.96, + "learning_rate": 1.781344032096289e-05, + "loss": 1.2014, + "theoretical_loss": 3.305660402188087, + "tokens_seen": 3184394240 + }, + { + "epoch": 0.97, + "learning_rate": 1.7793380140421264e-05, + "loss": 1.2194, + "theoretical_loss": 3.3056500672738243, + "tokens_seen": 3184525312 + }, + { + "epoch": 0.97, + "learning_rate": 1.777331995987964e-05, + "loss": 1.1613, + "theoretical_loss": 3.305639732904027, + "tokens_seen": 3184656384 + }, + { + "epoch": 0.97, + "learning_rate": 1.7753259779338016e-05, + "loss": 1.2695, + "theoretical_loss": 3.3056293990786445, + "tokens_seen": 3184787456 + }, + { + "epoch": 0.97, + "learning_rate": 1.773319959879639e-05, + "loss": 1.0997, + "theoretical_loss": 3.305619065797626, + "tokens_seen": 3184918528 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.5313416123390198, + "objective/train/avg_token_score": 0.005801832769066095, + "objective/train/docs_used": 1789954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5411800146102905, + "objective/train/original_loss": 3.2077465057373047, + "objective/train/theoretical_loss": 3.30560873306092, + "objective/train/tokens_used": 3185049600, + "objective/train/unlikelihood_loss": 0.1956406831741333, + "theoretical_loss": 3.30560873306092, + "tokens_seen": 3185049600 + }, + { + "epoch": 0.97, + "learning_rate": 1.7713139418254764e-05, + "loss": 1.2446, + "theoretical_loss": 3.30560873306092, + "tokens_seen": 3185049600 + }, + { + "epoch": 0.97, + "learning_rate": 1.7693079237713137e-05, + "loss": 1.2252, + "theoretical_loss": 3.305598400868475, + "tokens_seen": 3185180672 + }, + { + "epoch": 0.97, + "learning_rate": 1.7673019057171516e-05, + "loss": 1.1846, + "theoretical_loss": 3.3055880692202413, + "tokens_seen": 3185311744 + }, + { + "epoch": 0.97, + "learning_rate": 1.765295887662989e-05, + "loss": 1.2285, + "theoretical_loss": 3.3055777381161664, + "tokens_seen": 3185442816 + }, + { + "epoch": 0.97, + "learning_rate": 1.7632898696088264e-05, + "loss": 1.2733, + "theoretical_loss": 3.3055674075562, + "tokens_seen": 3185573888 + }, + { + "epoch": 0.97, + "learning_rate": 1.761283851554664e-05, + "loss": 1.2316, + "theoretical_loss": 3.305557077540291, + "tokens_seen": 3185704960 + }, + { + "epoch": 0.97, + "learning_rate": 1.7592778335005016e-05, + "loss": 1.169, + "theoretical_loss": 3.3055467480683878, + "tokens_seen": 3185836032 + }, + { + "epoch": 0.97, + "learning_rate": 1.757271815446339e-05, + "loss": 1.2089, + "theoretical_loss": 3.30553641914044, + "tokens_seen": 3185967104 + }, + { + "epoch": 0.97, + "learning_rate": 1.7552657973921767e-05, + "loss": 1.4084, + "theoretical_loss": 3.3055260907563966, + "tokens_seen": 3186098176 + }, + { + "epoch": 0.97, + "learning_rate": 1.753259779338014e-05, + "loss": 1.3693, + "theoretical_loss": 3.3055157629162064, + "tokens_seen": 3186229248 + }, + { + "epoch": 0.97, + "learning_rate": 1.7512537612838515e-05, + "loss": 1.2847, + "theoretical_loss": 3.3055054356198186, + "tokens_seen": 3186360320 + }, + { + "epoch": 0.97, + "learning_rate": 1.749247743229689e-05, + "loss": 1.2059, + "theoretical_loss": 3.3054951088671816, + "tokens_seen": 3186491392 + }, + { + "epoch": 0.97, + "learning_rate": 1.7472417251755267e-05, + "loss": 1.3385, + "theoretical_loss": 3.3054847826582447, + "tokens_seen": 3186622464 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.25708699226379395, + "objective/train/avg_token_score": 0.061789318919181824, + "objective/train/docs_used": 1791181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8450136184692383, + "objective/train/original_loss": 4.618594646453857, + "objective/train/theoretical_loss": 3.3054796197576484, + "objective/train/tokens_used": 3186688000, + "objective/train/unlikelihood_loss": 0.11923852562904358, + "theoretical_loss": 3.3054796197576484, + "tokens_seen": 3186688000 + }, + { + "epoch": 0.97, + "learning_rate": 1.7452357071213643e-05, + "loss": 1.2899, + "theoretical_loss": 3.3054744569929575, + "tokens_seen": 3186753536 + }, + { + "epoch": 0.97, + "learning_rate": 1.7432296890672015e-05, + "loss": 1.3945, + "theoretical_loss": 3.3054641318712683, + "tokens_seen": 3186884608 + }, + { + "epoch": 0.97, + "learning_rate": 1.741223671013039e-05, + "loss": 1.293, + "theoretical_loss": 3.305453807293126, + "tokens_seen": 3187015680 + }, + { + "epoch": 0.97, + "learning_rate": 1.7392176529588767e-05, + "loss": 1.1407, + "theoretical_loss": 3.3054434832584807, + "tokens_seen": 3187146752 + }, + { + "epoch": 0.97, + "learning_rate": 1.7372116349047142e-05, + "loss": 1.2993, + "theoretical_loss": 3.30543315976728, + "tokens_seen": 3187277824 + }, + { + "epoch": 0.97, + "learning_rate": 1.7352056168505518e-05, + "loss": 1.1421, + "theoretical_loss": 3.3054228368194742, + "tokens_seen": 3187408896 + }, + { + "epoch": 0.97, + "learning_rate": 1.733199598796389e-05, + "loss": 1.3461, + "theoretical_loss": 3.3054125144150115, + "tokens_seen": 3187539968 + }, + { + "epoch": 0.97, + "learning_rate": 1.731193580742227e-05, + "loss": 1.3632, + "theoretical_loss": 3.3054021925538413, + "tokens_seen": 3187671040 + }, + { + "epoch": 0.97, + "learning_rate": 1.7291875626880642e-05, + "loss": 1.2981, + "theoretical_loss": 3.305391871235913, + "tokens_seen": 3187802112 + }, + { + "epoch": 0.97, + "learning_rate": 1.7271815446339018e-05, + "loss": 1.3475, + "theoretical_loss": 3.305381550461175, + "tokens_seen": 3187933184 + }, + { + "epoch": 0.97, + "learning_rate": 1.725175526579739e-05, + "loss": 1.211, + "theoretical_loss": 3.305371230229577, + "tokens_seen": 3188064256 + }, + { + "epoch": 0.97, + "learning_rate": 1.7231695085255766e-05, + "loss": 1.3239, + "theoretical_loss": 3.3053609105410677, + "tokens_seen": 3188195328 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.3896627426147461, + "objective/train/avg_token_score": 0.018728867173194885, + "objective/train/docs_used": 1792456, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1549485921859741, + "objective/train/original_loss": 4.037176609039307, + "objective/train/theoretical_loss": 3.3053505913955963, + "objective/train/tokens_used": 3188326400, + "objective/train/unlikelihood_loss": 0.14679159224033356, + "theoretical_loss": 3.3053505913955963, + "tokens_seen": 3188326400 + }, + { + "epoch": 0.97, + "learning_rate": 1.7211634904714145e-05, + "loss": 1.2497, + "theoretical_loss": 3.3053505913955963, + "tokens_seen": 3188326400 + }, + { + "epoch": 0.97, + "learning_rate": 1.7191574724172518e-05, + "loss": 1.2863, + "theoretical_loss": 3.3053402727931114, + "tokens_seen": 3188457472 + }, + { + "epoch": 0.97, + "learning_rate": 1.7171514543630894e-05, + "loss": 1.3021, + "theoretical_loss": 3.305329954733563, + "tokens_seen": 3188588544 + }, + { + "epoch": 0.97, + "learning_rate": 1.7151454363089266e-05, + "loss": 1.3053, + "theoretical_loss": 3.3053196372169, + "tokens_seen": 3188719616 + }, + { + "epoch": 0.97, + "learning_rate": 1.7131394182547642e-05, + "loss": 1.1789, + "theoretical_loss": 3.305309320243071, + "tokens_seen": 3188850688 + }, + { + "epoch": 0.97, + "learning_rate": 1.711133400200602e-05, + "loss": 1.2908, + "theoretical_loss": 3.305299003812026, + "tokens_seen": 3188981760 + }, + { + "epoch": 0.97, + "learning_rate": 1.7091273821464393e-05, + "loss": 1.4002, + "theoretical_loss": 3.305288687923713, + "tokens_seen": 3189112832 + }, + { + "epoch": 0.97, + "learning_rate": 1.707121364092277e-05, + "loss": 1.3156, + "theoretical_loss": 3.305278372578082, + "tokens_seen": 3189243904 + }, + { + "epoch": 0.97, + "learning_rate": 1.705115346038114e-05, + "loss": 1.1852, + "theoretical_loss": 3.305268057775082, + "tokens_seen": 3189374976 + }, + { + "epoch": 0.97, + "learning_rate": 1.703109327983952e-05, + "loss": 1.31, + "theoretical_loss": 3.305257743514662, + "tokens_seen": 3189506048 + }, + { + "epoch": 0.97, + "learning_rate": 1.7011033099297897e-05, + "loss": 1.1198, + "theoretical_loss": 3.305247429796771, + "tokens_seen": 3189637120 + }, + { + "epoch": 0.97, + "learning_rate": 1.699097291875627e-05, + "loss": 1.2086, + "theoretical_loss": 3.3052371166213588, + "tokens_seen": 3189768192 + }, + { + "epoch": 0.97, + "learning_rate": 1.6970912738214645e-05, + "loss": 1.2897, + "theoretical_loss": 3.305226803988374, + "tokens_seen": 3189899264 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.39919355511665344, + "objective/train/avg_token_score": 0.03530009463429451, + "objective/train/docs_used": 1793072, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1333123445510864, + "objective/train/original_loss": 4.221036434173584, + "objective/train/theoretical_loss": 3.3052216478752756, + "objective/train/tokens_used": 3189964800, + "objective/train/unlikelihood_loss": 0.1322452872991562, + "theoretical_loss": 3.3052216478752756, + "tokens_seen": 3189964800 + }, + { + "epoch": 0.97, + "learning_rate": 1.6950852557673017e-05, + "loss": 1.2948, + "theoretical_loss": 3.305216491897766, + "tokens_seen": 3190030336 + }, + { + "epoch": 0.97, + "learning_rate": 1.6930792377131396e-05, + "loss": 1.3987, + "theoretical_loss": 3.305206180349484, + "tokens_seen": 3190161408 + }, + { + "epoch": 0.97, + "learning_rate": 1.691073219658977e-05, + "loss": 1.1489, + "theoretical_loss": 3.3051958693434766, + "tokens_seen": 3190292480 + }, + { + "epoch": 0.97, + "learning_rate": 1.6890672016048145e-05, + "loss": 1.1552, + "theoretical_loss": 3.305185558879694, + "tokens_seen": 3190423552 + }, + { + "epoch": 0.97, + "learning_rate": 1.687061183550652e-05, + "loss": 1.2391, + "theoretical_loss": 3.305175248958085, + "tokens_seen": 3190554624 + }, + { + "epoch": 0.97, + "learning_rate": 1.6850551654964893e-05, + "loss": 1.2171, + "theoretical_loss": 3.3051649395785985, + "tokens_seen": 3190685696 + }, + { + "epoch": 0.97, + "learning_rate": 1.6830491474423272e-05, + "loss": 1.338, + "theoretical_loss": 3.3051546307411845, + "tokens_seen": 3190816768 + }, + { + "epoch": 0.97, + "learning_rate": 1.6810431293881644e-05, + "loss": 1.0808, + "theoretical_loss": 3.3051443224457913, + "tokens_seen": 3190947840 + }, + { + "epoch": 0.97, + "learning_rate": 1.679037111334002e-05, + "loss": 1.2701, + "theoretical_loss": 3.305134014692369, + "tokens_seen": 3191078912 + }, + { + "epoch": 0.97, + "learning_rate": 1.6770310932798396e-05, + "loss": 1.3131, + "theoretical_loss": 3.305123707480866, + "tokens_seen": 3191209984 + }, + { + "epoch": 0.97, + "learning_rate": 1.6750250752256772e-05, + "loss": 1.1062, + "theoretical_loss": 3.3051134008112317, + "tokens_seen": 3191341056 + }, + { + "epoch": 0.97, + "learning_rate": 1.6730190571715148e-05, + "loss": 1.2793, + "theoretical_loss": 3.305103094683416, + "tokens_seen": 3191472128 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.30669599771499634, + "objective/train/avg_token_score": 0.015499619767069817, + "objective/train/docs_used": 1793729, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9740338325500488, + "objective/train/original_loss": 4.146705627441406, + "objective/train/theoretical_loss": 3.305092789097368, + "objective/train/tokens_used": 3191603200, + "objective/train/unlikelihood_loss": 0.11558662354946136, + "theoretical_loss": 3.305092789097368, + "tokens_seen": 3191603200 + }, + { + "epoch": 0.97, + "learning_rate": 1.671013039117352e-05, + "loss": 1.2897, + "theoretical_loss": 3.305092789097368, + "tokens_seen": 3191603200 + }, + { + "epoch": 0.97, + "learning_rate": 1.6690070210631896e-05, + "loss": 1.1584, + "theoretical_loss": 3.305082484053037, + "tokens_seen": 3191734272 + }, + { + "epoch": 0.97, + "learning_rate": 1.667001003009027e-05, + "loss": 1.2071, + "theoretical_loss": 3.3050721795503715, + "tokens_seen": 3191865344 + }, + { + "epoch": 0.97, + "learning_rate": 1.6649949849548647e-05, + "loss": 1.2597, + "theoretical_loss": 3.3050618755893213, + "tokens_seen": 3191996416 + }, + { + "epoch": 0.97, + "learning_rate": 1.6629889669007023e-05, + "loss": 1.1597, + "theoretical_loss": 3.305051572169836, + "tokens_seen": 3192127488 + }, + { + "epoch": 0.97, + "learning_rate": 1.6609829488465395e-05, + "loss": 1.0123, + "theoretical_loss": 3.3050412692918645, + "tokens_seen": 3192258560 + }, + { + "epoch": 0.97, + "learning_rate": 1.658976930792377e-05, + "loss": 1.2382, + "theoretical_loss": 3.3050309669553566, + "tokens_seen": 3192389632 + }, + { + "epoch": 0.97, + "learning_rate": 1.6569709127382144e-05, + "loss": 1.2281, + "theoretical_loss": 3.305020665160261, + "tokens_seen": 3192520704 + }, + { + "epoch": 0.97, + "learning_rate": 1.6549648946840523e-05, + "loss": 1.1938, + "theoretical_loss": 3.3050103639065274, + "tokens_seen": 3192651776 + }, + { + "epoch": 0.97, + "learning_rate": 1.65295887662989e-05, + "loss": 1.2547, + "theoretical_loss": 3.3050000631941048, + "tokens_seen": 3192782848 + }, + { + "epoch": 0.97, + "learning_rate": 1.650952858575727e-05, + "loss": 1.2674, + "theoretical_loss": 3.3049897630229426, + "tokens_seen": 3192913920 + }, + { + "epoch": 0.97, + "learning_rate": 1.6489468405215647e-05, + "loss": 1.1795, + "theoretical_loss": 3.3049794633929905, + "tokens_seen": 3193044992 + }, + { + "epoch": 0.97, + "learning_rate": 1.6469408224674023e-05, + "loss": 1.1787, + "theoretical_loss": 3.3049691643041976, + "tokens_seen": 3193176064 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.38630253076553345, + "objective/train/avg_token_score": 0.011545122601091862, + "objective/train/docs_used": 1795075, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2225216627120972, + "objective/train/original_loss": 4.214339256286621, + "objective/train/theoretical_loss": 3.30496401496272, + "objective/train/tokens_used": 3193241600, + "objective/train/unlikelihood_loss": 0.13777171075344086, + "theoretical_loss": 3.30496401496272, + "tokens_seen": 3193241600 + }, + { + "epoch": 0.97, + "learning_rate": 1.64493480441324e-05, + "loss": 1.2641, + "theoretical_loss": 3.304958865756513, + "tokens_seen": 3193307136 + }, + { + "epoch": 0.97, + "learning_rate": 1.6429287863590774e-05, + "loss": 1.3561, + "theoretical_loss": 3.3049485677498867, + "tokens_seen": 3193438208 + }, + { + "epoch": 0.97, + "learning_rate": 1.6409227683049147e-05, + "loss": 1.3281, + "theoretical_loss": 3.304938270284268, + "tokens_seen": 3193569280 + }, + { + "epoch": 0.97, + "learning_rate": 1.6389167502507522e-05, + "loss": 1.4086, + "theoretical_loss": 3.3049279733596055, + "tokens_seen": 3193700352 + }, + { + "epoch": 0.97, + "learning_rate": 1.6369107321965898e-05, + "loss": 1.3357, + "theoretical_loss": 3.3049176769758493, + "tokens_seen": 3193831424 + }, + { + "epoch": 0.97, + "learning_rate": 1.6349047141424274e-05, + "loss": 1.4169, + "theoretical_loss": 3.3049073811329484, + "tokens_seen": 3193962496 + }, + { + "epoch": 0.97, + "learning_rate": 1.632898696088265e-05, + "loss": 1.2552, + "theoretical_loss": 3.3048970858308526, + "tokens_seen": 3194093568 + }, + { + "epoch": 0.97, + "learning_rate": 1.6308926780341022e-05, + "loss": 1.2648, + "theoretical_loss": 3.304886791069511, + "tokens_seen": 3194224640 + }, + { + "epoch": 0.97, + "learning_rate": 1.6288866599799398e-05, + "loss": 1.1475, + "theoretical_loss": 3.304876496848873, + "tokens_seen": 3194355712 + }, + { + "epoch": 0.97, + "learning_rate": 1.6268806419257774e-05, + "loss": 1.3144, + "theoretical_loss": 3.304866203168888, + "tokens_seen": 3194486784 + }, + { + "epoch": 0.97, + "learning_rate": 1.624874623871615e-05, + "loss": 1.2806, + "theoretical_loss": 3.3048559100295054, + "tokens_seen": 3194617856 + }, + { + "epoch": 0.97, + "learning_rate": 1.6228686058174522e-05, + "loss": 1.3394, + "theoretical_loss": 3.304845617430675, + "tokens_seen": 3194748928 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.4272971749305725, + "objective/train/avg_token_score": 0.020062921568751335, + "objective/train/docs_used": 1795447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3320684432983398, + "objective/train/original_loss": 4.0758056640625, + "objective/train/theoretical_loss": 3.3048353253723457, + "objective/train/tokens_used": 3194880000, + "objective/train/unlikelihood_loss": 0.1355256736278534, + "theoretical_loss": 3.3048353253723457, + "tokens_seen": 3194880000 + }, + { + "epoch": 0.97, + "learning_rate": 1.6208625877632898e-05, + "loss": 1.317, + "theoretical_loss": 3.3048353253723457, + "tokens_seen": 3194880000 + }, + { + "epoch": 0.97, + "learning_rate": 1.6188565697091274e-05, + "loss": 1.2436, + "theoretical_loss": 3.304825033854468, + "tokens_seen": 3195011072 + }, + { + "epoch": 0.97, + "learning_rate": 1.616850551654965e-05, + "loss": 1.3273, + "theoretical_loss": 3.3048147428769896, + "tokens_seen": 3195142144 + }, + { + "epoch": 0.97, + "learning_rate": 1.6148445336008025e-05, + "loss": 1.3027, + "theoretical_loss": 3.3048044524398614, + "tokens_seen": 3195273216 + }, + { + "epoch": 0.97, + "learning_rate": 1.6128385155466398e-05, + "loss": 1.2185, + "theoretical_loss": 3.304794162543032, + "tokens_seen": 3195404288 + }, + { + "epoch": 0.97, + "learning_rate": 1.6108324974924773e-05, + "loss": 1.1838, + "theoretical_loss": 3.304783873186452, + "tokens_seen": 3195535360 + }, + { + "epoch": 0.97, + "learning_rate": 1.6088264794383153e-05, + "loss": 1.3613, + "theoretical_loss": 3.3047735843700696, + "tokens_seen": 3195666432 + }, + { + "epoch": 0.97, + "learning_rate": 1.6068204613841525e-05, + "loss": 1.1841, + "theoretical_loss": 3.304763296093835, + "tokens_seen": 3195797504 + }, + { + "epoch": 0.97, + "learning_rate": 1.60481444332999e-05, + "loss": 1.2229, + "theoretical_loss": 3.3047530083576975, + "tokens_seen": 3195928576 + }, + { + "epoch": 0.97, + "learning_rate": 1.6028084252758273e-05, + "loss": 1.2846, + "theoretical_loss": 3.3047427211616065, + "tokens_seen": 3196059648 + }, + { + "epoch": 0.97, + "learning_rate": 1.600802407221665e-05, + "loss": 1.347, + "theoretical_loss": 3.304732434505512, + "tokens_seen": 3196190720 + }, + { + "epoch": 0.97, + "learning_rate": 1.5987963891675028e-05, + "loss": 1.2671, + "theoretical_loss": 3.3047221483893625, + "tokens_seen": 3196321792 + }, + { + "epoch": 0.97, + "learning_rate": 1.59679037111334e-05, + "loss": 1.1645, + "theoretical_loss": 3.3047118628131087, + "tokens_seen": 3196452864 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.3459799587726593, + "objective/train/avg_token_score": 0.009255710989236832, + "objective/train/docs_used": 1796802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0003851652145386, + "objective/train/original_loss": 3.741623640060425, + "objective/train/theoretical_loss": 3.3047067202274265, + "objective/train/tokens_used": 3196518400, + "objective/train/unlikelihood_loss": 0.21289655566215515, + "theoretical_loss": 3.3047067202274265, + "tokens_seen": 3196518400 + }, + { + "epoch": 0.97, + "learning_rate": 1.5947843530591776e-05, + "loss": 1.0985, + "theoretical_loss": 3.304701577776699, + "tokens_seen": 3196583936 + }, + { + "epoch": 0.97, + "learning_rate": 1.592778335005015e-05, + "loss": 1.3536, + "theoretical_loss": 3.3046912932800843, + "tokens_seen": 3196715008 + }, + { + "epoch": 0.97, + "learning_rate": 1.5907723169508525e-05, + "loss": 1.2797, + "theoretical_loss": 3.304681009323213, + "tokens_seen": 3196846080 + }, + { + "epoch": 0.97, + "learning_rate": 1.5887662988966904e-05, + "loss": 1.2332, + "theoretical_loss": 3.3046707259060346, + "tokens_seen": 3196977152 + }, + { + "epoch": 0.97, + "learning_rate": 1.5867602808425276e-05, + "loss": 1.3318, + "theoretical_loss": 3.3046604430284994, + "tokens_seen": 3197108224 + }, + { + "epoch": 0.97, + "learning_rate": 1.5847542627883652e-05, + "loss": 1.2557, + "theoretical_loss": 3.3046501606905565, + "tokens_seen": 3197239296 + }, + { + "epoch": 0.97, + "learning_rate": 1.5827482447342024e-05, + "loss": 1.3146, + "theoretical_loss": 3.304639878892156, + "tokens_seen": 3197370368 + }, + { + "epoch": 0.97, + "learning_rate": 1.5807422266800404e-05, + "loss": 1.2819, + "theoretical_loss": 3.3046295976332463, + "tokens_seen": 3197501440 + }, + { + "epoch": 0.97, + "learning_rate": 1.5787362086258776e-05, + "loss": 1.1968, + "theoretical_loss": 3.304619316913778, + "tokens_seen": 3197632512 + }, + { + "epoch": 0.97, + "learning_rate": 1.5767301905717152e-05, + "loss": 1.3045, + "theoretical_loss": 3.3046090367337007, + "tokens_seen": 3197763584 + }, + { + "epoch": 0.97, + "learning_rate": 1.5747241725175527e-05, + "loss": 1.2576, + "theoretical_loss": 3.3045987570929634, + "tokens_seen": 3197894656 + }, + { + "epoch": 0.97, + "learning_rate": 1.57271815446339e-05, + "loss": 1.0814, + "theoretical_loss": 3.304588477991516, + "tokens_seen": 3198025728 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.4404325485229492, + "objective/train/avg_token_score": 0.016093246638774872, + "objective/train/docs_used": 1797432, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3605873584747314, + "objective/train/original_loss": 3.6509079933166504, + "objective/train/theoretical_loss": 3.3045781994293084, + "objective/train/tokens_used": 3198156800, + "objective/train/unlikelihood_loss": 0.14039313793182373, + "theoretical_loss": 3.3045781994293084, + "tokens_seen": 3198156800 + }, + { + "epoch": 0.97, + "learning_rate": 1.570712136409228e-05, + "loss": 1.2421, + "theoretical_loss": 3.3045781994293084, + "tokens_seen": 3198156800 + }, + { + "epoch": 0.97, + "learning_rate": 1.568706118355065e-05, + "loss": 1.2489, + "theoretical_loss": 3.3045679214062895, + "tokens_seen": 3198287872 + }, + { + "epoch": 0.97, + "learning_rate": 1.5667001003009027e-05, + "loss": 1.3578, + "theoretical_loss": 3.3045576439224096, + "tokens_seen": 3198418944 + }, + { + "epoch": 0.97, + "learning_rate": 1.5646940822467403e-05, + "loss": 1.4532, + "theoretical_loss": 3.3045473669776184, + "tokens_seen": 3198550016 + }, + { + "epoch": 0.97, + "learning_rate": 1.5626880641925775e-05, + "loss": 1.2441, + "theoretical_loss": 3.3045370905718645, + "tokens_seen": 3198681088 + }, + { + "epoch": 0.97, + "learning_rate": 1.560682046138415e-05, + "loss": 1.5288, + "theoretical_loss": 3.304526814705099, + "tokens_seen": 3198812160 + }, + { + "epoch": 0.97, + "learning_rate": 1.5586760280842527e-05, + "loss": 1.1523, + "theoretical_loss": 3.30451653937727, + "tokens_seen": 3198943232 + }, + { + "epoch": 0.97, + "learning_rate": 1.5566700100300903e-05, + "loss": 1.2733, + "theoretical_loss": 3.3045062645883285, + "tokens_seen": 3199074304 + }, + { + "epoch": 0.97, + "learning_rate": 1.554663991975928e-05, + "loss": 1.2259, + "theoretical_loss": 3.304495990338223, + "tokens_seen": 3199205376 + }, + { + "epoch": 0.97, + "learning_rate": 1.5526579739217654e-05, + "loss": 1.1695, + "theoretical_loss": 3.3044857166269046, + "tokens_seen": 3199336448 + }, + { + "epoch": 0.97, + "learning_rate": 1.550651955867603e-05, + "loss": 1.2006, + "theoretical_loss": 3.3044754434543218, + "tokens_seen": 3199467520 + }, + { + "epoch": 0.97, + "learning_rate": 1.5486459378134403e-05, + "loss": 1.3381, + "theoretical_loss": 3.3044651708204245, + "tokens_seen": 3199598592 + }, + { + "epoch": 0.97, + "learning_rate": 1.546639919759278e-05, + "loss": 1.2207, + "theoretical_loss": 3.3044548987251625, + "tokens_seen": 3199729664 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.45668378472328186, + "objective/train/avg_token_score": 0.015230822376906872, + "objective/train/docs_used": 1798843, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.373724102973938, + "objective/train/original_loss": 4.246476650238037, + "objective/train/theoretical_loss": 3.304449762879504, + "objective/train/tokens_used": 3199795200, + "objective/train/unlikelihood_loss": 0.10861189663410187, + "theoretical_loss": 3.304449762879504, + "tokens_seen": 3199795200 + }, + { + "epoch": 0.97, + "learning_rate": 1.5446339017051154e-05, + "loss": 1.5315, + "theoretical_loss": 3.3044446271684857, + "tokens_seen": 3199860736 + }, + { + "epoch": 0.97, + "learning_rate": 1.542627883650953e-05, + "loss": 0.9553, + "theoretical_loss": 3.3044343561503435, + "tokens_seen": 3199991808 + }, + { + "epoch": 0.97, + "learning_rate": 1.5406218655967906e-05, + "loss": 1.3231, + "theoretical_loss": 3.3044240856706857, + "tokens_seen": 3200122880 + }, + { + "epoch": 0.97, + "learning_rate": 1.5386158475426278e-05, + "loss": 1.3879, + "theoretical_loss": 3.304413815729462, + "tokens_seen": 3200253952 + }, + { + "epoch": 0.97, + "learning_rate": 1.5366098294884654e-05, + "loss": 1.1844, + "theoretical_loss": 3.3044035463266224, + "tokens_seen": 3200385024 + }, + { + "epoch": 0.97, + "learning_rate": 1.534603811434303e-05, + "loss": 1.2337, + "theoretical_loss": 3.304393277462116, + "tokens_seen": 3200516096 + }, + { + "epoch": 0.97, + "learning_rate": 1.5325977933801402e-05, + "loss": 1.2044, + "theoretical_loss": 3.304383009135893, + "tokens_seen": 3200647168 + }, + { + "epoch": 0.97, + "learning_rate": 1.530591775325978e-05, + "loss": 1.1373, + "theoretical_loss": 3.3043727413479034, + "tokens_seen": 3200778240 + }, + { + "epoch": 0.97, + "learning_rate": 1.5285857572718154e-05, + "loss": 1.3466, + "theoretical_loss": 3.304362474098096, + "tokens_seen": 3200909312 + }, + { + "epoch": 0.97, + "learning_rate": 1.526579739217653e-05, + "loss": 1.2238, + "theoretical_loss": 3.3043522073864215, + "tokens_seen": 3201040384 + }, + { + "epoch": 0.97, + "learning_rate": 1.5245737211634905e-05, + "loss": 1.1614, + "theoretical_loss": 3.30434194121283, + "tokens_seen": 3201171456 + }, + { + "epoch": 0.97, + "learning_rate": 1.5225677031093281e-05, + "loss": 1.1513, + "theoretical_loss": 3.3043316755772696, + "tokens_seen": 3201302528 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.42369258403778076, + "objective/train/avg_token_score": 0.02147451601922512, + "objective/train/docs_used": 1799567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3248203992843628, + "objective/train/original_loss": 3.7724480628967285, + "objective/train/theoretical_loss": 3.3043214104796914, + "objective/train/tokens_used": 3201433600, + "objective/train/unlikelihood_loss": 0.13887332379817963, + "theoretical_loss": 3.3043214104796914, + "tokens_seen": 3201433600 + }, + { + "epoch": 0.97, + "learning_rate": 1.5205616850551655e-05, + "loss": 1.2914, + "theoretical_loss": 3.3043214104796914, + "tokens_seen": 3201433600 + }, + { + "epoch": 0.97, + "learning_rate": 1.518555667001003e-05, + "loss": 1.1867, + "theoretical_loss": 3.304311145920045, + "tokens_seen": 3201564672 + }, + { + "epoch": 0.97, + "learning_rate": 1.5165496489468405e-05, + "loss": 1.3141, + "theoretical_loss": 3.3043008818982798, + "tokens_seen": 3201695744 + }, + { + "epoch": 0.97, + "learning_rate": 1.5145436308926781e-05, + "loss": 1.413, + "theoretical_loss": 3.304290618414346, + "tokens_seen": 3201826816 + }, + { + "epoch": 0.97, + "learning_rate": 1.5125376128385157e-05, + "loss": 1.3418, + "theoretical_loss": 3.3042803554681934, + "tokens_seen": 3201957888 + }, + { + "epoch": 0.97, + "learning_rate": 1.5105315947843531e-05, + "loss": 1.1225, + "theoretical_loss": 3.3042700930597713, + "tokens_seen": 3202088960 + }, + { + "epoch": 0.97, + "learning_rate": 1.5085255767301907e-05, + "loss": 1.3348, + "theoretical_loss": 3.30425983118903, + "tokens_seen": 3202220032 + }, + { + "epoch": 0.97, + "learning_rate": 1.506519558676028e-05, + "loss": 1.3144, + "theoretical_loss": 3.3042495698559193, + "tokens_seen": 3202351104 + }, + { + "epoch": 0.97, + "learning_rate": 1.5045135406218655e-05, + "loss": 1.2262, + "theoretical_loss": 3.3042393090603888, + "tokens_seen": 3202482176 + }, + { + "epoch": 0.97, + "learning_rate": 1.5025075225677032e-05, + "loss": 1.4347, + "theoretical_loss": 3.304229048802388, + "tokens_seen": 3202613248 + }, + { + "epoch": 0.97, + "learning_rate": 1.5005015045135406e-05, + "loss": 1.2773, + "theoretical_loss": 3.304218789081868, + "tokens_seen": 3202744320 + }, + { + "epoch": 0.97, + "learning_rate": 1.4984954864593782e-05, + "loss": 1.1482, + "theoretical_loss": 3.304208529898778, + "tokens_seen": 3202875392 + }, + { + "epoch": 0.97, + "learning_rate": 1.4964894684052156e-05, + "loss": 1.1963, + "theoretical_loss": 3.304198271253067, + "tokens_seen": 3203006464 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.49144673347473145, + "objective/train/avg_token_score": 0.018495580181479454, + "objective/train/docs_used": 1800690, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.348713755607605, + "objective/train/original_loss": 3.483724355697632, + "objective/train/theoretical_loss": 3.3041931421317132, + "objective/train/tokens_used": 3203072000, + "objective/train/unlikelihood_loss": 0.16879183053970337, + "theoretical_loss": 3.3041931421317132, + "tokens_seen": 3203072000 + }, + { + "epoch": 0.97, + "learning_rate": 1.494483450351053e-05, + "loss": 1.4033, + "theoretical_loss": 3.3041880131446857, + "tokens_seen": 3203137536 + }, + { + "epoch": 0.97, + "learning_rate": 1.4924774322968908e-05, + "loss": 1.1525, + "theoretical_loss": 3.304177755573584, + "tokens_seen": 3203268608 + }, + { + "epoch": 0.97, + "learning_rate": 1.4904714142427282e-05, + "loss": 1.4135, + "theoretical_loss": 3.3041674985397114, + "tokens_seen": 3203399680 + }, + { + "epoch": 0.97, + "learning_rate": 1.4884653961885658e-05, + "loss": 1.2854, + "theoretical_loss": 3.304157242043018, + "tokens_seen": 3203530752 + }, + { + "epoch": 0.97, + "learning_rate": 1.4864593781344032e-05, + "loss": 1.3932, + "theoretical_loss": 3.3041469860834543, + "tokens_seen": 3203661824 + }, + { + "epoch": 0.97, + "learning_rate": 1.4844533600802408e-05, + "loss": 1.4815, + "theoretical_loss": 3.304136730660969, + "tokens_seen": 3203792896 + }, + { + "epoch": 0.97, + "learning_rate": 1.4824473420260782e-05, + "loss": 1.3703, + "theoretical_loss": 3.3041264757755124, + "tokens_seen": 3203923968 + }, + { + "epoch": 0.97, + "learning_rate": 1.4804413239719158e-05, + "loss": 1.2684, + "theoretical_loss": 3.304116221427035, + "tokens_seen": 3204055040 + }, + { + "epoch": 0.97, + "learning_rate": 1.4784353059177533e-05, + "loss": 1.288, + "theoretical_loss": 3.3041059676154863, + "tokens_seen": 3204186112 + }, + { + "epoch": 0.97, + "learning_rate": 1.4764292878635907e-05, + "loss": 1.3632, + "theoretical_loss": 3.304095714340816, + "tokens_seen": 3204317184 + }, + { + "epoch": 0.97, + "learning_rate": 1.4744232698094283e-05, + "loss": 1.2619, + "theoretical_loss": 3.3040854616029747, + "tokens_seen": 3204448256 + }, + { + "epoch": 0.97, + "learning_rate": 1.4724172517552657e-05, + "loss": 1.2637, + "theoretical_loss": 3.3040752094019115, + "tokens_seen": 3204579328 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.3913123309612274, + "objective/train/avg_token_score": 0.032182399183511734, + "objective/train/docs_used": 1802019, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.178242564201355, + "objective/train/original_loss": 3.3910341262817383, + "objective/train/theoretical_loss": 3.3040649577375767, + "objective/train/tokens_used": 3204710400, + "objective/train/unlikelihood_loss": 0.2589299976825714, + "theoretical_loss": 3.3040649577375767, + "tokens_seen": 3204710400 + }, + { + "epoch": 0.97, + "learning_rate": 1.4704112337011035e-05, + "loss": 1.2416, + "theoretical_loss": 3.3040649577375767, + "tokens_seen": 3204710400 + }, + { + "epoch": 0.97, + "learning_rate": 1.4684052156469409e-05, + "loss": 1.1656, + "theoretical_loss": 3.3040547066099206, + "tokens_seen": 3204841472 + }, + { + "epoch": 0.97, + "learning_rate": 1.4663991975927783e-05, + "loss": 1.3952, + "theoretical_loss": 3.304044456018893, + "tokens_seen": 3204972544 + }, + { + "epoch": 0.97, + "learning_rate": 1.4643931795386159e-05, + "loss": 1.2347, + "theoretical_loss": 3.3040342059644434, + "tokens_seen": 3205103616 + }, + { + "epoch": 0.97, + "learning_rate": 1.4623871614844533e-05, + "loss": 1.4147, + "theoretical_loss": 3.304023956446522, + "tokens_seen": 3205234688 + }, + { + "epoch": 0.97, + "learning_rate": 1.4603811434302909e-05, + "loss": 1.2806, + "theoretical_loss": 3.304013707465079, + "tokens_seen": 3205365760 + }, + { + "epoch": 0.97, + "learning_rate": 1.4583751253761285e-05, + "loss": 1.3504, + "theoretical_loss": 3.3040034590200644, + "tokens_seen": 3205496832 + }, + { + "epoch": 0.97, + "learning_rate": 1.456369107321966e-05, + "loss": 1.2373, + "theoretical_loss": 3.303993211111428, + "tokens_seen": 3205627904 + }, + { + "epoch": 0.97, + "learning_rate": 1.4543630892678034e-05, + "loss": 1.2789, + "theoretical_loss": 3.3039829637391196, + "tokens_seen": 3205758976 + }, + { + "epoch": 0.97, + "learning_rate": 1.4523570712136409e-05, + "loss": 1.1504, + "theoretical_loss": 3.3039727169030897, + "tokens_seen": 3205890048 + }, + { + "epoch": 0.97, + "learning_rate": 1.4503510531594784e-05, + "loss": 1.2732, + "theoretical_loss": 3.3039624706032877, + "tokens_seen": 3206021120 + }, + { + "epoch": 0.97, + "learning_rate": 1.4483450351053158e-05, + "loss": 1.2134, + "theoretical_loss": 3.3039522248396644, + "tokens_seen": 3206152192 + }, + { + "epoch": 0.97, + "learning_rate": 1.4463390170511536e-05, + "loss": 1.307, + "theoretical_loss": 3.303941979612169, + "tokens_seen": 3206283264 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.2804863154888153, + "objective/train/avg_token_score": 0.028898051008582115, + "objective/train/docs_used": 1802648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.868274450302124, + "objective/train/original_loss": 4.002166748046875, + "objective/train/theoretical_loss": 3.303936857199454, + "objective/train/tokens_used": 3206348800, + "objective/train/unlikelihood_loss": 0.22286373376846313, + "theoretical_loss": 3.303936857199454, + "tokens_seen": 3206348800 + }, + { + "epoch": 0.97, + "learning_rate": 1.444332998996991e-05, + "loss": 1.1273, + "theoretical_loss": 3.303931734920752, + "tokens_seen": 3206414336 + }, + { + "epoch": 0.97, + "learning_rate": 1.4423269809428286e-05, + "loss": 1.2266, + "theoretical_loss": 3.303921490765364, + "tokens_seen": 3206545408 + }, + { + "epoch": 0.97, + "learning_rate": 1.440320962888666e-05, + "loss": 1.2056, + "theoretical_loss": 3.3039112471459537, + "tokens_seen": 3206676480 + }, + { + "epoch": 0.97, + "learning_rate": 1.4383149448345034e-05, + "loss": 1.2649, + "theoretical_loss": 3.303901004062472, + "tokens_seen": 3206807552 + }, + { + "epoch": 0.97, + "learning_rate": 1.4363089267803411e-05, + "loss": 1.2836, + "theoretical_loss": 3.303890761514869, + "tokens_seen": 3206938624 + }, + { + "epoch": 0.97, + "learning_rate": 1.4343029087261786e-05, + "loss": 1.1228, + "theoretical_loss": 3.3038805195030947, + "tokens_seen": 3207069696 + }, + { + "epoch": 0.97, + "learning_rate": 1.4322968906720161e-05, + "loss": 1.162, + "theoretical_loss": 3.303870278027099, + "tokens_seen": 3207200768 + }, + { + "epoch": 0.97, + "learning_rate": 1.4302908726178535e-05, + "loss": 1.3582, + "theoretical_loss": 3.3038600370868316, + "tokens_seen": 3207331840 + }, + { + "epoch": 0.97, + "learning_rate": 1.4282848545636911e-05, + "loss": 1.4053, + "theoretical_loss": 3.3038497966822433, + "tokens_seen": 3207462912 + }, + { + "epoch": 0.97, + "learning_rate": 1.4262788365095285e-05, + "loss": 1.3911, + "theoretical_loss": 3.3038395568132843, + "tokens_seen": 3207593984 + }, + { + "epoch": 0.97, + "learning_rate": 1.4242728184553661e-05, + "loss": 1.243, + "theoretical_loss": 3.3038293174799036, + "tokens_seen": 3207725056 + }, + { + "epoch": 0.97, + "learning_rate": 1.4222668004012037e-05, + "loss": 1.2026, + "theoretical_loss": 3.3038190786820527, + "tokens_seen": 3207856128 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.3851417303085327, + "objective/train/avg_token_score": 0.017263926565647125, + "objective/train/docs_used": 1803892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1330887079238892, + "objective/train/original_loss": 3.8268911838531494, + "objective/train/theoretical_loss": 3.3038088404196806, + "objective/train/tokens_used": 3207987200, + "objective/train/unlikelihood_loss": 0.16789372265338898, + "theoretical_loss": 3.3038088404196806, + "tokens_seen": 3207987200 + }, + { + "epoch": 0.97, + "learning_rate": 1.4202607823470411e-05, + "loss": 1.2005, + "theoretical_loss": 3.3038088404196806, + "tokens_seen": 3207987200 + }, + { + "epoch": 0.97, + "learning_rate": 1.4182547642928787e-05, + "loss": 1.1934, + "theoretical_loss": 3.303798602692738, + "tokens_seen": 3208118272 + }, + { + "epoch": 0.97, + "learning_rate": 1.4162487462387161e-05, + "loss": 1.2386, + "theoretical_loss": 3.3037883655011746, + "tokens_seen": 3208249344 + }, + { + "epoch": 0.97, + "learning_rate": 1.4142427281845538e-05, + "loss": 1.2043, + "theoretical_loss": 3.303778128844941, + "tokens_seen": 3208380416 + }, + { + "epoch": 0.97, + "learning_rate": 1.4122367101303913e-05, + "loss": 1.1957, + "theoretical_loss": 3.303767892723987, + "tokens_seen": 3208511488 + }, + { + "epoch": 0.97, + "learning_rate": 1.4102306920762287e-05, + "loss": 1.1943, + "theoretical_loss": 3.303757657138263, + "tokens_seen": 3208642560 + }, + { + "epoch": 0.97, + "learning_rate": 1.4082246740220662e-05, + "loss": 1.276, + "theoretical_loss": 3.303747422087719, + "tokens_seen": 3208773632 + }, + { + "epoch": 0.97, + "learning_rate": 1.4062186559679037e-05, + "loss": 1.3196, + "theoretical_loss": 3.303737187572305, + "tokens_seen": 3208904704 + }, + { + "epoch": 0.97, + "learning_rate": 1.4042126379137412e-05, + "loss": 1.1806, + "theoretical_loss": 3.3037269535919718, + "tokens_seen": 3209035776 + }, + { + "epoch": 0.97, + "learning_rate": 1.4022066198595788e-05, + "loss": 1.2807, + "theoretical_loss": 3.303716720146669, + "tokens_seen": 3209166848 + }, + { + "epoch": 0.97, + "learning_rate": 1.4002006018054164e-05, + "loss": 1.2452, + "theoretical_loss": 3.303706487236347, + "tokens_seen": 3209297920 + }, + { + "epoch": 0.97, + "learning_rate": 1.3981945837512538e-05, + "loss": 1.288, + "theoretical_loss": 3.3036962548609554, + "tokens_seen": 3209428992 + }, + { + "epoch": 0.97, + "learning_rate": 1.3961885656970912e-05, + "loss": 1.3252, + "theoretical_loss": 3.3036860230204455, + "tokens_seen": 3209560064 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.43982160091400146, + "objective/train/avg_token_score": 0.014127448201179504, + "objective/train/docs_used": 1804624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3093814849853516, + "objective/train/original_loss": 3.5772314071655273, + "objective/train/theoretical_loss": 3.3036809073007554, + "objective/train/tokens_used": 3209625600, + "objective/train/unlikelihood_loss": 0.13695333898067474, + "theoretical_loss": 3.3036809073007554, + "tokens_seen": 3209625600 + }, + { + "epoch": 0.97, + "learning_rate": 1.3941825476429288e-05, + "loss": 1.2831, + "theoretical_loss": 3.3036757917147663, + "tokens_seen": 3209691136 + }, + { + "epoch": 0.97, + "learning_rate": 1.3921765295887662e-05, + "loss": 1.2524, + "theoretical_loss": 3.3036655609438688, + "tokens_seen": 3209822208 + }, + { + "epoch": 0.97, + "learning_rate": 1.390170511534604e-05, + "loss": 1.2111, + "theoretical_loss": 3.303655330707703, + "tokens_seen": 3209953280 + }, + { + "epoch": 0.97, + "learning_rate": 1.3881644934804414e-05, + "loss": 1.2913, + "theoretical_loss": 3.3036451010062193, + "tokens_seen": 3210084352 + }, + { + "epoch": 0.97, + "learning_rate": 1.386158475426279e-05, + "loss": 1.1389, + "theoretical_loss": 3.3036348718393675, + "tokens_seen": 3210215424 + }, + { + "epoch": 0.97, + "learning_rate": 1.3841524573721163e-05, + "loss": 1.2055, + "theoretical_loss": 3.3036246432070984, + "tokens_seen": 3210346496 + }, + { + "epoch": 0.97, + "learning_rate": 1.3821464393179538e-05, + "loss": 1.3187, + "theoretical_loss": 3.3036144151093616, + "tokens_seen": 3210477568 + }, + { + "epoch": 0.97, + "learning_rate": 1.3801404212637915e-05, + "loss": 1.1516, + "theoretical_loss": 3.303604187546108, + "tokens_seen": 3210608640 + }, + { + "epoch": 0.97, + "learning_rate": 1.378134403209629e-05, + "loss": 1.2187, + "theoretical_loss": 3.3035939605172873, + "tokens_seen": 3210739712 + }, + { + "epoch": 0.97, + "learning_rate": 1.3761283851554665e-05, + "loss": 1.1069, + "theoretical_loss": 3.3035837340228498, + "tokens_seen": 3210870784 + }, + { + "epoch": 0.97, + "learning_rate": 1.3741223671013039e-05, + "loss": 1.3001, + "theoretical_loss": 3.3035735080627457, + "tokens_seen": 3211001856 + }, + { + "epoch": 0.97, + "learning_rate": 1.3721163490471415e-05, + "loss": 1.2368, + "theoretical_loss": 3.303563282636926, + "tokens_seen": 3211132928 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.34854593873023987, + "objective/train/avg_token_score": 0.03674829751253128, + "objective/train/docs_used": 1805106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0519399642944336, + "objective/train/original_loss": 4.2592668533325195, + "objective/train/theoretical_loss": 3.3035530577453405, + "objective/train/tokens_used": 3211264000, + "objective/train/unlikelihood_loss": 0.16234083473682404, + "theoretical_loss": 3.3035530577453405, + "tokens_seen": 3211264000 + }, + { + "epoch": 0.97, + "learning_rate": 1.3701103309929789e-05, + "loss": 1.1647, + "theoretical_loss": 3.3035530577453405, + "tokens_seen": 3211264000 + }, + { + "epoch": 0.97, + "learning_rate": 1.3681043129388165e-05, + "loss": 1.2874, + "theoretical_loss": 3.3035428333879393, + "tokens_seen": 3211395072 + }, + { + "epoch": 0.97, + "learning_rate": 1.366098294884654e-05, + "loss": 1.199, + "theoretical_loss": 3.3035326095646727, + "tokens_seen": 3211526144 + }, + { + "epoch": 0.97, + "learning_rate": 1.3640922768304915e-05, + "loss": 1.2103, + "theoretical_loss": 3.303522386275491, + "tokens_seen": 3211657216 + }, + { + "epoch": 0.97, + "learning_rate": 1.362086258776329e-05, + "loss": 1.2405, + "theoretical_loss": 3.3035121635203453, + "tokens_seen": 3211788288 + }, + { + "epoch": 0.97, + "learning_rate": 1.3600802407221665e-05, + "loss": 1.3187, + "theoretical_loss": 3.303501941299185, + "tokens_seen": 3211919360 + }, + { + "epoch": 0.97, + "learning_rate": 1.358074222668004e-05, + "loss": 1.1498, + "theoretical_loss": 3.3034917196119604, + "tokens_seen": 3212050432 + }, + { + "epoch": 0.97, + "learning_rate": 1.3560682046138416e-05, + "loss": 1.2917, + "theoretical_loss": 3.3034814984586225, + "tokens_seen": 3212181504 + }, + { + "epoch": 0.97, + "learning_rate": 1.354062186559679e-05, + "loss": 1.3977, + "theoretical_loss": 3.303471277839121, + "tokens_seen": 3212312576 + }, + { + "epoch": 0.97, + "learning_rate": 1.3520561685055166e-05, + "loss": 1.2226, + "theoretical_loss": 3.3034610577534065, + "tokens_seen": 3212443648 + }, + { + "epoch": 0.97, + "learning_rate": 1.350050150451354e-05, + "loss": 1.3136, + "theoretical_loss": 3.3034508382014294, + "tokens_seen": 3212574720 + }, + { + "epoch": 0.97, + "learning_rate": 1.3480441323971916e-05, + "loss": 1.2074, + "theoretical_loss": 3.30344061918314, + "tokens_seen": 3212705792 + }, + { + "epoch": 0.97, + "learning_rate": 1.3460381143430292e-05, + "loss": 1.3978, + "theoretical_loss": 3.3034304006984887, + "tokens_seen": 3212836864 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.4102517068386078, + "objective/train/avg_token_score": 0.011543256230652332, + "objective/train/docs_used": 1806087, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0842695236206055, + "objective/train/original_loss": 3.0441412925720215, + "objective/train/theoretical_loss": 3.3034252916562616, + "objective/train/tokens_used": 3212902400, + "objective/train/unlikelihood_loss": 0.2348608523607254, + "theoretical_loss": 3.3034252916562616, + "tokens_seen": 3212902400 + }, + { + "epoch": 0.97, + "learning_rate": 1.3440320962888667e-05, + "loss": 1.2144, + "theoretical_loss": 3.3034201827474257, + "tokens_seen": 3212967936 + }, + { + "epoch": 0.97, + "learning_rate": 1.3420260782347042e-05, + "loss": 1.3058, + "theoretical_loss": 3.3034099653299016, + "tokens_seen": 3213099008 + }, + { + "epoch": 0.97, + "learning_rate": 1.3400200601805416e-05, + "loss": 1.2624, + "theoretical_loss": 3.303399748445866, + "tokens_seen": 3213230080 + }, + { + "epoch": 0.97, + "learning_rate": 1.3380140421263791e-05, + "loss": 1.2706, + "theoretical_loss": 3.3033895320952706, + "tokens_seen": 3213361152 + }, + { + "epoch": 0.97, + "learning_rate": 1.3360080240722166e-05, + "loss": 1.2149, + "theoretical_loss": 3.303379316278065, + "tokens_seen": 3213492224 + }, + { + "epoch": 0.97, + "learning_rate": 1.3340020060180543e-05, + "loss": 1.2148, + "theoretical_loss": 3.3033691009941997, + "tokens_seen": 3213623296 + }, + { + "epoch": 0.97, + "learning_rate": 1.3319959879638917e-05, + "loss": 1.262, + "theoretical_loss": 3.3033588862436254, + "tokens_seen": 3213754368 + }, + { + "epoch": 0.97, + "learning_rate": 1.3299899699097293e-05, + "loss": 1.2904, + "theoretical_loss": 3.3033486720262917, + "tokens_seen": 3213885440 + }, + { + "epoch": 0.97, + "learning_rate": 1.3279839518555667e-05, + "loss": 1.2342, + "theoretical_loss": 3.3033384583421497, + "tokens_seen": 3214016512 + }, + { + "epoch": 0.97, + "learning_rate": 1.3259779338014041e-05, + "loss": 1.2863, + "theoretical_loss": 3.3033282451911496, + "tokens_seen": 3214147584 + }, + { + "epoch": 0.97, + "learning_rate": 1.3239719157472419e-05, + "loss": 1.3431, + "theoretical_loss": 3.3033180325732423, + "tokens_seen": 3214278656 + }, + { + "epoch": 0.97, + "learning_rate": 1.3219658976930793e-05, + "loss": 1.2248, + "theoretical_loss": 3.303307820488378, + "tokens_seen": 3214409728 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.6934261918067932, + "objective/train/avg_token_score": 0.007221103645861149, + "objective/train/docs_used": 1806786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.091050386428833, + "objective/train/original_loss": 3.4449198246002197, + "objective/train/theoretical_loss": 3.3032976089365063, + "objective/train/tokens_used": 3214540800, + "objective/train/unlikelihood_loss": 0.10023477673530579, + "theoretical_loss": 3.3032976089365063, + "tokens_seen": 3214540800 + }, + { + "epoch": 0.97, + "learning_rate": 1.3199598796389169e-05, + "loss": 1.4361, + "theoretical_loss": 3.3032976089365063, + "tokens_seen": 3214540800 + }, + { + "epoch": 0.97, + "learning_rate": 1.3179538615847543e-05, + "loss": 1.1405, + "theoretical_loss": 3.303287397917579, + "tokens_seen": 3214671872 + }, + { + "epoch": 0.97, + "learning_rate": 1.3159478435305918e-05, + "loss": 1.1164, + "theoretical_loss": 3.3032771874315454, + "tokens_seen": 3214802944 + }, + { + "epoch": 0.97, + "learning_rate": 1.3139418254764293e-05, + "loss": 1.2951, + "theoretical_loss": 3.3032669774783567, + "tokens_seen": 3214934016 + }, + { + "epoch": 0.97, + "learning_rate": 1.3119358074222668e-05, + "loss": 1.2622, + "theoretical_loss": 3.303256768057963, + "tokens_seen": 3215065088 + }, + { + "epoch": 0.97, + "learning_rate": 1.3099297893681044e-05, + "loss": 1.3179, + "theoretical_loss": 3.303246559170315, + "tokens_seen": 3215196160 + }, + { + "epoch": 0.97, + "learning_rate": 1.3079237713139418e-05, + "loss": 1.2945, + "theoretical_loss": 3.303236350815363, + "tokens_seen": 3215327232 + }, + { + "epoch": 0.97, + "learning_rate": 1.3059177532597794e-05, + "loss": 1.2502, + "theoretical_loss": 3.303226142993058, + "tokens_seen": 3215458304 + }, + { + "epoch": 0.97, + "learning_rate": 1.3039117352056168e-05, + "loss": 1.2904, + "theoretical_loss": 3.3032159357033497, + "tokens_seen": 3215589376 + }, + { + "epoch": 0.97, + "learning_rate": 1.3019057171514544e-05, + "loss": 1.342, + "theoretical_loss": 3.303205728946189, + "tokens_seen": 3215720448 + }, + { + "epoch": 0.97, + "learning_rate": 1.299899699097292e-05, + "loss": 1.3153, + "theoretical_loss": 3.303195522721526, + "tokens_seen": 3215851520 + }, + { + "epoch": 0.97, + "learning_rate": 1.2978936810431294e-05, + "loss": 1.2875, + "theoretical_loss": 3.3031853170293126, + "tokens_seen": 3215982592 + }, + { + "epoch": 0.97, + "learning_rate": 1.295887662988967e-05, + "loss": 1.2286, + "theoretical_loss": 3.3031751118694976, + "tokens_seen": 3216113664 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.3753054738044739, + "objective/train/avg_token_score": 0.013739002868533134, + "objective/train/docs_used": 1808057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2807402610778809, + "objective/train/original_loss": 4.779233455657959, + "objective/train/theoretical_loss": 3.3031700094892242, + "objective/train/tokens_used": 3216179200, + "objective/train/unlikelihood_loss": 0.09806220978498459, + "theoretical_loss": 3.3031700094892242, + "tokens_seen": 3216179200 + }, + { + "epoch": 0.97, + "learning_rate": 1.2938816449348044e-05, + "loss": 1.2747, + "theoretical_loss": 3.3031649072420324, + "tokens_seen": 3216244736 + }, + { + "epoch": 0.97, + "learning_rate": 1.291875626880642e-05, + "loss": 1.2321, + "theoretical_loss": 3.3031547031468675, + "tokens_seen": 3216375808 + }, + { + "epoch": 0.97, + "learning_rate": 1.2898696088264795e-05, + "loss": 1.3967, + "theoretical_loss": 3.3031444995839534, + "tokens_seen": 3216506880 + }, + { + "epoch": 0.97, + "learning_rate": 1.2878635907723171e-05, + "loss": 1.1125, + "theoretical_loss": 3.3031342965532406, + "tokens_seen": 3216637952 + }, + { + "epoch": 0.97, + "learning_rate": 1.2858575727181545e-05, + "loss": 1.045, + "theoretical_loss": 3.3031240940546795, + "tokens_seen": 3216769024 + }, + { + "epoch": 0.97, + "learning_rate": 1.283851554663992e-05, + "loss": 1.2452, + "theoretical_loss": 3.303113892088221, + "tokens_seen": 3216900096 + }, + { + "epoch": 0.97, + "learning_rate": 1.2818455366098295e-05, + "loss": 1.306, + "theoretical_loss": 3.3031036906538156, + "tokens_seen": 3217031168 + }, + { + "epoch": 0.97, + "learning_rate": 1.2798395185556669e-05, + "loss": 1.1817, + "theoretical_loss": 3.3030934897514133, + "tokens_seen": 3217162240 + }, + { + "epoch": 0.97, + "learning_rate": 1.2778335005015047e-05, + "loss": 1.2708, + "theoretical_loss": 3.3030832893809654, + "tokens_seen": 3217293312 + }, + { + "epoch": 0.97, + "learning_rate": 1.275827482447342e-05, + "loss": 1.107, + "theoretical_loss": 3.3030730895424227, + "tokens_seen": 3217424384 + }, + { + "epoch": 0.98, + "learning_rate": 1.2738214643931797e-05, + "loss": 1.2984, + "theoretical_loss": 3.3030628902357346, + "tokens_seen": 3217555456 + }, + { + "epoch": 0.98, + "learning_rate": 1.271815446339017e-05, + "loss": 1.3485, + "theoretical_loss": 3.3030526914608527, + "tokens_seen": 3217686528 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.43682795763015747, + "objective/train/avg_token_score": 0.014645503833889961, + "objective/train/docs_used": 1808828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3192623853683472, + "objective/train/original_loss": 3.8680734634399414, + "objective/train/theoretical_loss": 3.3030424932177276, + "objective/train/tokens_used": 3217817600, + "objective/train/unlikelihood_loss": 0.1511588841676712, + "theoretical_loss": 3.3030424932177276, + "tokens_seen": 3217817600 + }, + { + "epoch": 0.98, + "learning_rate": 1.2698094282848545e-05, + "loss": 1.175, + "theoretical_loss": 3.3030424932177276, + "tokens_seen": 3217817600 + }, + { + "epoch": 0.98, + "learning_rate": 1.267803410230692e-05, + "loss": 1.1797, + "theoretical_loss": 3.303032295506309, + "tokens_seen": 3217948672 + }, + { + "epoch": 0.98, + "learning_rate": 1.2657973921765296e-05, + "loss": 1.4034, + "theoretical_loss": 3.303022098326549, + "tokens_seen": 3218079744 + }, + { + "epoch": 0.98, + "learning_rate": 1.2637913741223672e-05, + "loss": 1.2147, + "theoretical_loss": 3.303011901678397, + "tokens_seen": 3218210816 + }, + { + "epoch": 0.98, + "learning_rate": 1.2617853560682046e-05, + "loss": 1.2218, + "theoretical_loss": 3.303001705561804, + "tokens_seen": 3218341888 + }, + { + "epoch": 0.98, + "learning_rate": 1.2597793380140422e-05, + "loss": 1.1913, + "theoretical_loss": 3.302991509976721, + "tokens_seen": 3218472960 + }, + { + "epoch": 0.98, + "learning_rate": 1.2577733199598796e-05, + "loss": 1.2474, + "theoretical_loss": 3.302981314923098, + "tokens_seen": 3218604032 + }, + { + "epoch": 0.98, + "learning_rate": 1.2557673019057172e-05, + "loss": 1.2704, + "theoretical_loss": 3.3029711204008865, + "tokens_seen": 3218735104 + }, + { + "epoch": 0.98, + "learning_rate": 1.2537612838515548e-05, + "loss": 1.2079, + "theoretical_loss": 3.3029609264100364, + "tokens_seen": 3218866176 + }, + { + "epoch": 0.98, + "learning_rate": 1.2517552657973922e-05, + "loss": 1.2293, + "theoretical_loss": 3.3029507329504986, + "tokens_seen": 3218997248 + }, + { + "epoch": 0.98, + "learning_rate": 1.2497492477432298e-05, + "loss": 1.233, + "theoretical_loss": 3.3029405400222234, + "tokens_seen": 3219128320 + }, + { + "epoch": 0.98, + "learning_rate": 1.2477432296890672e-05, + "loss": 1.2707, + "theoretical_loss": 3.3029303476251624, + "tokens_seen": 3219259392 + }, + { + "epoch": 0.98, + "learning_rate": 1.2457372116349047e-05, + "loss": 1.1412, + "theoretical_loss": 3.302920155759266, + "tokens_seen": 3219390464 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.4287634491920471, + "objective/train/avg_token_score": 0.02073417790234089, + "objective/train/docs_used": 1810102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.327704668045044, + "objective/train/original_loss": 3.706601142883301, + "objective/train/theoretical_loss": 3.3029150600254886, + "objective/train/tokens_used": 3219456000, + "objective/train/unlikelihood_loss": 0.1711893528699875, + "theoretical_loss": 3.3029150600254886, + "tokens_seen": 3219456000 + }, + { + "epoch": 0.98, + "learning_rate": 1.2437311935807423e-05, + "loss": 1.3441, + "theoretical_loss": 3.302909964424484, + "tokens_seen": 3219521536 + }, + { + "epoch": 0.98, + "learning_rate": 1.2417251755265797e-05, + "loss": 1.3448, + "theoretical_loss": 3.302899773620768, + "tokens_seen": 3219652608 + }, + { + "epoch": 0.98, + "learning_rate": 1.2397191574724173e-05, + "loss": 1.304, + "theoretical_loss": 3.3028895833480685, + "tokens_seen": 3219783680 + }, + { + "epoch": 0.98, + "learning_rate": 1.2377131394182547e-05, + "loss": 1.3362, + "theoretical_loss": 3.3028793936063363, + "tokens_seen": 3219914752 + }, + { + "epoch": 0.98, + "learning_rate": 1.2357071213640923e-05, + "loss": 1.136, + "theoretical_loss": 3.302869204395522, + "tokens_seen": 3220045824 + }, + { + "epoch": 0.98, + "learning_rate": 1.2337011033099299e-05, + "loss": 1.2098, + "theoretical_loss": 3.302859015715576, + "tokens_seen": 3220176896 + }, + { + "epoch": 0.98, + "learning_rate": 1.2316950852557675e-05, + "loss": 1.2015, + "theoretical_loss": 3.30284882756645, + "tokens_seen": 3220307968 + }, + { + "epoch": 0.98, + "learning_rate": 1.2296890672016049e-05, + "loss": 1.2431, + "theoretical_loss": 3.302838639948094, + "tokens_seen": 3220439040 + }, + { + "epoch": 0.98, + "learning_rate": 1.2276830491474423e-05, + "loss": 1.2709, + "theoretical_loss": 3.302828452860459, + "tokens_seen": 3220570112 + }, + { + "epoch": 0.98, + "learning_rate": 1.2256770310932799e-05, + "loss": 1.2035, + "theoretical_loss": 3.302818266303495, + "tokens_seen": 3220701184 + }, + { + "epoch": 0.98, + "learning_rate": 1.2236710130391173e-05, + "loss": 1.206, + "theoretical_loss": 3.3028080802771536, + "tokens_seen": 3220832256 + }, + { + "epoch": 0.98, + "learning_rate": 1.221664994984955e-05, + "loss": 1.2478, + "theoretical_loss": 3.302797894781386, + "tokens_seen": 3220963328 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.23521505296230316, + "objective/train/avg_token_score": 0.022963732481002808, + "objective/train/docs_used": 1810863, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.822738766670227, + "objective/train/original_loss": 4.289976119995117, + "objective/train/theoretical_loss": 3.3027877098161413, + "objective/train/tokens_used": 3221094400, + "objective/train/unlikelihood_loss": 0.17344634234905243, + "theoretical_loss": 3.3027877098161413, + "tokens_seen": 3221094400 + }, + { + "epoch": 0.98, + "learning_rate": 1.2196589769307924e-05, + "loss": 1.2959, + "theoretical_loss": 3.3027877098161413, + "tokens_seen": 3221094400 + }, + { + "epoch": 0.98, + "learning_rate": 1.2176529588766298e-05, + "loss": 1.3482, + "theoretical_loss": 3.302777525381372, + "tokens_seen": 3221225472 + }, + { + "epoch": 0.98, + "learning_rate": 1.2156469408224674e-05, + "loss": 1.276, + "theoretical_loss": 3.302767341477028, + "tokens_seen": 3221356544 + }, + { + "epoch": 0.98, + "learning_rate": 1.2136409227683048e-05, + "loss": 1.0822, + "theoretical_loss": 3.30275715810306, + "tokens_seen": 3221487616 + }, + { + "epoch": 0.98, + "learning_rate": 1.2116349047141424e-05, + "loss": 1.0864, + "theoretical_loss": 3.30274697525942, + "tokens_seen": 3221618688 + }, + { + "epoch": 0.98, + "learning_rate": 1.20962888665998e-05, + "loss": 1.3668, + "theoretical_loss": 3.302736792946057, + "tokens_seen": 3221749760 + }, + { + "epoch": 0.98, + "learning_rate": 1.2076228686058176e-05, + "loss": 1.1559, + "theoretical_loss": 3.302726611162923, + "tokens_seen": 3221880832 + }, + { + "epoch": 0.98, + "learning_rate": 1.205616850551655e-05, + "loss": 1.2634, + "theoretical_loss": 3.3027164299099687, + "tokens_seen": 3222011904 + }, + { + "epoch": 0.98, + "learning_rate": 1.2036108324974924e-05, + "loss": 1.2898, + "theoretical_loss": 3.3027062491871444, + "tokens_seen": 3222142976 + }, + { + "epoch": 0.98, + "learning_rate": 1.20160481444333e-05, + "loss": 1.2186, + "theoretical_loss": 3.3026960689944014, + "tokens_seen": 3222274048 + }, + { + "epoch": 0.98, + "learning_rate": 1.1995987963891675e-05, + "loss": 1.3063, + "theoretical_loss": 3.302685889331691, + "tokens_seen": 3222405120 + }, + { + "epoch": 0.98, + "learning_rate": 1.1975927783350051e-05, + "loss": 1.2452, + "theoretical_loss": 3.3026757101989626, + "tokens_seen": 3222536192 + }, + { + "epoch": 0.98, + "learning_rate": 1.1955867602808425e-05, + "loss": 1.3459, + "theoretical_loss": 3.3026655315961686, + "tokens_seen": 3222667264 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.3994990289211273, + "objective/train/avg_token_score": 0.007963519543409348, + "objective/train/docs_used": 1812146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1697691679000854, + "objective/train/original_loss": 4.066588401794434, + "objective/train/theoretical_loss": 3.3026604424934813, + "objective/train/tokens_used": 3222732800, + "objective/train/unlikelihood_loss": 0.15115737915039062, + "theoretical_loss": 3.3026604424934813, + "tokens_seen": 3222732800 + }, + { + "epoch": 0.98, + "learning_rate": 1.1935807422266801e-05, + "loss": 1.2343, + "theoretical_loss": 3.302655353523259, + "tokens_seen": 3222798336 + }, + { + "epoch": 0.98, + "learning_rate": 1.1915747241725175e-05, + "loss": 1.3091, + "theoretical_loss": 3.3026451759801847, + "tokens_seen": 3222929408 + }, + { + "epoch": 0.98, + "learning_rate": 1.189568706118355e-05, + "loss": 1.191, + "theoretical_loss": 3.302634998966897, + "tokens_seen": 3223060480 + }, + { + "epoch": 0.98, + "learning_rate": 1.1875626880641927e-05, + "loss": 1.1687, + "theoretical_loss": 3.302624822483346, + "tokens_seen": 3223191552 + }, + { + "epoch": 0.98, + "learning_rate": 1.1855566700100301e-05, + "loss": 1.1395, + "theoretical_loss": 3.3026146465294834, + "tokens_seen": 3223322624 + }, + { + "epoch": 0.98, + "learning_rate": 1.1835506519558677e-05, + "loss": 1.098, + "theoretical_loss": 3.3026044711052602, + "tokens_seen": 3223453696 + }, + { + "epoch": 0.98, + "learning_rate": 1.181544633901705e-05, + "loss": 1.3231, + "theoretical_loss": 3.3025942962106263, + "tokens_seen": 3223584768 + }, + { + "epoch": 0.98, + "learning_rate": 1.1795386158475427e-05, + "loss": 1.2604, + "theoretical_loss": 3.302584121845533, + "tokens_seen": 3223715840 + }, + { + "epoch": 0.98, + "learning_rate": 1.1775325977933802e-05, + "loss": 1.3812, + "theoretical_loss": 3.302573948009932, + "tokens_seen": 3223846912 + }, + { + "epoch": 0.98, + "learning_rate": 1.1755265797392177e-05, + "loss": 1.2535, + "theoretical_loss": 3.3025637747037737, + "tokens_seen": 3223977984 + }, + { + "epoch": 0.98, + "learning_rate": 1.1735205616850552e-05, + "loss": 1.3113, + "theoretical_loss": 3.3025536019270088, + "tokens_seen": 3224109056 + }, + { + "epoch": 0.98, + "learning_rate": 1.1715145436308926e-05, + "loss": 1.2047, + "theoretical_loss": 3.302543429679588, + "tokens_seen": 3224240128 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.5299364328384399, + "objective/train/avg_token_score": 0.00746686989441514, + "objective/train/docs_used": 1812917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6563072204589844, + "objective/train/original_loss": 3.7367844581604004, + "objective/train/theoretical_loss": 3.3025332579614632, + "objective/train/tokens_used": 3224371200, + "objective/train/unlikelihood_loss": 0.15678440034389496, + "theoretical_loss": 3.3025332579614632, + "tokens_seen": 3224371200 + }, + { + "epoch": 0.98, + "learning_rate": 1.1695085255767302e-05, + "loss": 1.2926, + "theoretical_loss": 3.3025332579614632, + "tokens_seen": 3224371200 + }, + { + "epoch": 0.98, + "learning_rate": 1.1675025075225676e-05, + "loss": 1.1504, + "theoretical_loss": 3.302523086772584, + "tokens_seen": 3224502272 + }, + { + "epoch": 0.98, + "learning_rate": 1.1654964894684054e-05, + "loss": 1.1497, + "theoretical_loss": 3.3025129161129025, + "tokens_seen": 3224633344 + }, + { + "epoch": 0.98, + "learning_rate": 1.1634904714142428e-05, + "loss": 1.43, + "theoretical_loss": 3.3025027459823697, + "tokens_seen": 3224764416 + }, + { + "epoch": 0.98, + "learning_rate": 1.1614844533600802e-05, + "loss": 1.3287, + "theoretical_loss": 3.302492576380936, + "tokens_seen": 3224895488 + }, + { + "epoch": 0.98, + "learning_rate": 1.1594784353059178e-05, + "loss": 1.2741, + "theoretical_loss": 3.3024824073085517, + "tokens_seen": 3225026560 + }, + { + "epoch": 0.98, + "learning_rate": 1.1574724172517552e-05, + "loss": 1.1733, + "theoretical_loss": 3.3024722387651693, + "tokens_seen": 3225157632 + }, + { + "epoch": 0.98, + "learning_rate": 1.1554663991975928e-05, + "loss": 1.1414, + "theoretical_loss": 3.302462070750739, + "tokens_seen": 3225288704 + }, + { + "epoch": 0.98, + "learning_rate": 1.1534603811434303e-05, + "loss": 1.3573, + "theoretical_loss": 3.3024519032652115, + "tokens_seen": 3225419776 + }, + { + "epoch": 0.98, + "learning_rate": 1.151454363089268e-05, + "loss": 1.2312, + "theoretical_loss": 3.3024417363085385, + "tokens_seen": 3225550848 + }, + { + "epoch": 0.98, + "learning_rate": 1.1494483450351053e-05, + "loss": 1.1395, + "theoretical_loss": 3.3024315698806705, + "tokens_seen": 3225681920 + }, + { + "epoch": 0.98, + "learning_rate": 1.1474423269809427e-05, + "loss": 1.1756, + "theoretical_loss": 3.3024214039815587, + "tokens_seen": 3225812992 + }, + { + "epoch": 0.98, + "learning_rate": 1.1454363089267803e-05, + "loss": 1.4092, + "theoretical_loss": 3.302411238611154, + "tokens_seen": 3225944064 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.48949170112609863, + "objective/train/avg_token_score": 0.02726239152252674, + "objective/train/docs_used": 1813554, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.223509669303894, + "objective/train/original_loss": 3.6083450317382812, + "objective/train/theoretical_loss": 3.302406156124202, + "objective/train/tokens_used": 3226009600, + "objective/train/unlikelihood_loss": 0.11870256811380386, + "theoretical_loss": 3.302406156124202, + "tokens_seen": 3226009600 + }, + { + "epoch": 0.98, + "learning_rate": 1.1434302908726179e-05, + "loss": 1.2835, + "theoretical_loss": 3.302401073769408, + "tokens_seen": 3226075136 + }, + { + "epoch": 0.98, + "learning_rate": 1.1414242728184555e-05, + "loss": 1.1205, + "theoretical_loss": 3.3023909094562707, + "tokens_seen": 3226206208 + }, + { + "epoch": 0.98, + "learning_rate": 1.1394182547642929e-05, + "loss": 1.3084, + "theoretical_loss": 3.3023807456716936, + "tokens_seen": 3226337280 + }, + { + "epoch": 0.98, + "learning_rate": 1.1374122367101305e-05, + "loss": 1.2516, + "theoretical_loss": 3.3023705824156284, + "tokens_seen": 3226468352 + }, + { + "epoch": 0.98, + "learning_rate": 1.1354062186559679e-05, + "loss": 1.2256, + "theoretical_loss": 3.302360419688025, + "tokens_seen": 3226599424 + }, + { + "epoch": 0.98, + "learning_rate": 1.1334002006018053e-05, + "loss": 1.229, + "theoretical_loss": 3.3023502574888353, + "tokens_seen": 3226730496 + }, + { + "epoch": 0.98, + "learning_rate": 1.131394182547643e-05, + "loss": 1.1593, + "theoretical_loss": 3.30234009581801, + "tokens_seen": 3226861568 + }, + { + "epoch": 0.98, + "learning_rate": 1.1293881644934805e-05, + "loss": 1.2557, + "theoretical_loss": 3.3023299346755004, + "tokens_seen": 3226992640 + }, + { + "epoch": 0.98, + "learning_rate": 1.127382146439318e-05, + "loss": 1.1662, + "theoretical_loss": 3.302319774061257, + "tokens_seen": 3227123712 + }, + { + "epoch": 0.98, + "learning_rate": 1.1253761283851554e-05, + "loss": 1.1305, + "theoretical_loss": 3.3023096139752317, + "tokens_seen": 3227254784 + }, + { + "epoch": 0.98, + "learning_rate": 1.123370110330993e-05, + "loss": 1.2005, + "theoretical_loss": 3.3022994544173754, + "tokens_seen": 3227385856 + }, + { + "epoch": 0.98, + "learning_rate": 1.1213640922768304e-05, + "loss": 1.1122, + "theoretical_loss": 3.3022892953876384, + "tokens_seen": 3227516928 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.3030303120613098, + "objective/train/avg_token_score": 0.01359139196574688, + "objective/train/docs_used": 1814834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9968832731246948, + "objective/train/original_loss": 4.533849716186523, + "objective/train/theoretical_loss": 3.3022791368859727, + "objective/train/tokens_used": 3227648000, + "objective/train/unlikelihood_loss": 0.10337149351835251, + "theoretical_loss": 3.3022791368859727, + "tokens_seen": 3227648000 + }, + { + "epoch": 0.98, + "learning_rate": 1.119358074222668e-05, + "loss": 1.1373, + "theoretical_loss": 3.3022791368859727, + "tokens_seen": 3227648000 + }, + { + "epoch": 0.98, + "learning_rate": 1.1173520561685056e-05, + "loss": 1.2477, + "theoretical_loss": 3.302268978912329, + "tokens_seen": 3227779072 + }, + { + "epoch": 0.98, + "learning_rate": 1.115346038114343e-05, + "loss": 1.2898, + "theoretical_loss": 3.302258821466659, + "tokens_seen": 3227910144 + }, + { + "epoch": 0.98, + "learning_rate": 1.1133400200601806e-05, + "loss": 1.1551, + "theoretical_loss": 3.3022486645489124, + "tokens_seen": 3228041216 + }, + { + "epoch": 0.98, + "learning_rate": 1.111334002006018e-05, + "loss": 1.265, + "theoretical_loss": 3.302238508159042, + "tokens_seen": 3228172288 + }, + { + "epoch": 0.98, + "learning_rate": 1.1093279839518557e-05, + "loss": 1.1657, + "theoretical_loss": 3.302228352296998, + "tokens_seen": 3228303360 + }, + { + "epoch": 0.98, + "learning_rate": 1.1073219658976931e-05, + "loss": 1.2996, + "theoretical_loss": 3.3022181969627318, + "tokens_seen": 3228434432 + }, + { + "epoch": 0.98, + "learning_rate": 1.1053159478435306e-05, + "loss": 1.337, + "theoretical_loss": 3.3022080421561943, + "tokens_seen": 3228565504 + }, + { + "epoch": 0.98, + "learning_rate": 1.1033099297893681e-05, + "loss": 1.2642, + "theoretical_loss": 3.302197887877337, + "tokens_seen": 3228696576 + }, + { + "epoch": 0.98, + "learning_rate": 1.1013039117352055e-05, + "loss": 1.3449, + "theoretical_loss": 3.302187734126111, + "tokens_seen": 3228827648 + }, + { + "epoch": 0.98, + "learning_rate": 1.0992978936810431e-05, + "loss": 1.3111, + "theoretical_loss": 3.302177580902467, + "tokens_seen": 3228958720 + }, + { + "epoch": 0.98, + "learning_rate": 1.0972918756268807e-05, + "loss": 1.2451, + "theoretical_loss": 3.302167428206357, + "tokens_seen": 3229089792 + }, + { + "epoch": 0.98, + "learning_rate": 1.0952858575727183e-05, + "loss": 1.2224, + "theoretical_loss": 3.302157276037731, + "tokens_seen": 3229220864 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.39320626854896545, + "objective/train/avg_token_score": 0.01549724955111742, + "objective/train/docs_used": 1815538, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2420200109481812, + "objective/train/original_loss": 4.236133098602295, + "objective/train/theoretical_loss": 3.30215220015121, + "objective/train/tokens_used": 3229286400, + "objective/train/unlikelihood_loss": 0.1344296634197235, + "theoretical_loss": 3.30215220015121, + "tokens_seen": 3229286400 + }, + { + "epoch": 0.98, + "learning_rate": 1.0932798395185557e-05, + "loss": 1.283, + "theoretical_loss": 3.3021471243965417, + "tokens_seen": 3229351936 + }, + { + "epoch": 0.98, + "learning_rate": 1.0912738214643931e-05, + "loss": 1.0854, + "theoretical_loss": 3.302136973282739, + "tokens_seen": 3229483008 + }, + { + "epoch": 0.98, + "learning_rate": 1.0892678034102307e-05, + "loss": 1.2882, + "theoretical_loss": 3.302126822696275, + "tokens_seen": 3229614080 + }, + { + "epoch": 0.98, + "learning_rate": 1.0872617853560683e-05, + "loss": 1.3026, + "theoretical_loss": 3.3021166726371, + "tokens_seen": 3229745152 + }, + { + "epoch": 0.98, + "learning_rate": 1.0852557673019058e-05, + "loss": 1.3261, + "theoretical_loss": 3.3021065231051656, + "tokens_seen": 3229876224 + }, + { + "epoch": 0.98, + "learning_rate": 1.0832497492477433e-05, + "loss": 1.2036, + "theoretical_loss": 3.3020963741004237, + "tokens_seen": 3230007296 + }, + { + "epoch": 0.98, + "learning_rate": 1.0812437311935808e-05, + "loss": 1.2525, + "theoretical_loss": 3.302086225622825, + "tokens_seen": 3230138368 + }, + { + "epoch": 0.98, + "learning_rate": 1.0792377131394182e-05, + "loss": 1.3489, + "theoretical_loss": 3.30207607767232, + "tokens_seen": 3230269440 + }, + { + "epoch": 0.98, + "learning_rate": 1.0772316950852556e-05, + "loss": 1.2768, + "theoretical_loss": 3.3020659302488613, + "tokens_seen": 3230400512 + }, + { + "epoch": 0.98, + "learning_rate": 1.0752256770310934e-05, + "loss": 1.118, + "theoretical_loss": 3.302055783352399, + "tokens_seen": 3230531584 + }, + { + "epoch": 0.98, + "learning_rate": 1.0732196589769308e-05, + "loss": 1.1683, + "theoretical_loss": 3.3020456369828852, + "tokens_seen": 3230662656 + }, + { + "epoch": 0.98, + "learning_rate": 1.0712136409227684e-05, + "loss": 1.355, + "theoretical_loss": 3.3020354911402703, + "tokens_seen": 3230793728 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.4148949086666107, + "objective/train/avg_token_score": 0.01718137413263321, + "objective/train/docs_used": 1816936, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2691593170166016, + "objective/train/original_loss": 3.8008651733398438, + "objective/train/theoretical_loss": 3.3020253458245064, + "objective/train/tokens_used": 3230924800, + "objective/train/unlikelihood_loss": 0.17796172201633453, + "theoretical_loss": 3.3020253458245064, + "tokens_seen": 3230924800 + }, + { + "epoch": 0.98, + "learning_rate": 1.0692076228686058e-05, + "loss": 1.3124, + "theoretical_loss": 3.3020253458245064, + "tokens_seen": 3230924800 + }, + { + "epoch": 0.98, + "learning_rate": 1.0672016048144434e-05, + "loss": 1.2048, + "theoretical_loss": 3.3020152010355446, + "tokens_seen": 3231055872 + }, + { + "epoch": 0.98, + "learning_rate": 1.0651955867602808e-05, + "loss": 1.3301, + "theoretical_loss": 3.3020050567733357, + "tokens_seen": 3231186944 + }, + { + "epoch": 0.98, + "learning_rate": 1.0631895687061184e-05, + "loss": 1.1428, + "theoretical_loss": 3.3019949130378308, + "tokens_seen": 3231318016 + }, + { + "epoch": 0.98, + "learning_rate": 1.061183550651956e-05, + "loss": 1.1847, + "theoretical_loss": 3.3019847698289824, + "tokens_seen": 3231449088 + }, + { + "epoch": 0.98, + "learning_rate": 1.0591775325977934e-05, + "loss": 1.158, + "theoretical_loss": 3.3019746271467407, + "tokens_seen": 3231580160 + }, + { + "epoch": 0.98, + "learning_rate": 1.057171514543631e-05, + "loss": 1.3489, + "theoretical_loss": 3.3019644849910574, + "tokens_seen": 3231711232 + }, + { + "epoch": 0.98, + "learning_rate": 1.0551654964894683e-05, + "loss": 1.2097, + "theoretical_loss": 3.301954343361884, + "tokens_seen": 3231842304 + }, + { + "epoch": 0.98, + "learning_rate": 1.0531594784353061e-05, + "loss": 1.1045, + "theoretical_loss": 3.3019442022591714, + "tokens_seen": 3231973376 + }, + { + "epoch": 0.98, + "learning_rate": 1.0511534603811435e-05, + "loss": 1.0556, + "theoretical_loss": 3.301934061682871, + "tokens_seen": 3232104448 + }, + { + "epoch": 0.98, + "learning_rate": 1.0491474423269809e-05, + "loss": 1.2347, + "theoretical_loss": 3.3019239216329344, + "tokens_seen": 3232235520 + }, + { + "epoch": 0.98, + "learning_rate": 1.0471414242728185e-05, + "loss": 1.3287, + "theoretical_loss": 3.3019137821093123, + "tokens_seen": 3232366592 + }, + { + "epoch": 0.98, + "learning_rate": 1.0451354062186559e-05, + "loss": 1.1633, + "theoretical_loss": 3.3019036431119573, + "tokens_seen": 3232497664 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.4042644202709198, + "objective/train/avg_token_score": 0.02891196496784687, + "objective/train/docs_used": 1817522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.177870512008667, + "objective/train/original_loss": 3.662449598312378, + "objective/train/theoretical_loss": 3.301898573810614, + "objective/train/tokens_used": 3232563200, + "objective/train/unlikelihood_loss": 0.16236652433872223, + "theoretical_loss": 3.301898573810614, + "tokens_seen": 3232563200 + }, + { + "epoch": 0.98, + "learning_rate": 1.0431293881644935e-05, + "loss": 1.132, + "theoretical_loss": 3.3018935046408195, + "tokens_seen": 3232628736 + }, + { + "epoch": 0.98, + "learning_rate": 1.041123370110331e-05, + "loss": 1.2749, + "theoretical_loss": 3.3018833666958507, + "tokens_seen": 3232759808 + }, + { + "epoch": 0.98, + "learning_rate": 1.0391173520561686e-05, + "loss": 1.286, + "theoretical_loss": 3.3018732292770023, + "tokens_seen": 3232890880 + }, + { + "epoch": 0.98, + "learning_rate": 1.037111334002006e-05, + "loss": 1.1848, + "theoretical_loss": 3.3018630923842256, + "tokens_seen": 3233021952 + }, + { + "epoch": 0.98, + "learning_rate": 1.0351053159478435e-05, + "loss": 1.2999, + "theoretical_loss": 3.301852956017472, + "tokens_seen": 3233153024 + }, + { + "epoch": 0.98, + "learning_rate": 1.033099297893681e-05, + "loss": 0.9966, + "theoretical_loss": 3.3018428201766934, + "tokens_seen": 3233284096 + }, + { + "epoch": 0.98, + "learning_rate": 1.0310932798395184e-05, + "loss": 1.3157, + "theoretical_loss": 3.3018326848618402, + "tokens_seen": 3233415168 + }, + { + "epoch": 0.98, + "learning_rate": 1.0290872617853562e-05, + "loss": 1.2991, + "theoretical_loss": 3.301822550072864, + "tokens_seen": 3233546240 + }, + { + "epoch": 0.98, + "learning_rate": 1.0270812437311936e-05, + "loss": 1.1549, + "theoretical_loss": 3.3018124158097173, + "tokens_seen": 3233677312 + }, + { + "epoch": 0.98, + "learning_rate": 1.0250752256770312e-05, + "loss": 1.1268, + "theoretical_loss": 3.30180228207235, + "tokens_seen": 3233808384 + }, + { + "epoch": 0.98, + "learning_rate": 1.0230692076228686e-05, + "loss": 1.1316, + "theoretical_loss": 3.3017921488607143, + "tokens_seen": 3233939456 + }, + { + "epoch": 0.98, + "learning_rate": 1.021063189568706e-05, + "loss": 1.3719, + "theoretical_loss": 3.3017820161747617, + "tokens_seen": 3234070528 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.25928640365600586, + "objective/train/avg_token_score": 0.01630372181534767, + "objective/train/docs_used": 1818614, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8035547137260437, + "objective/train/original_loss": 4.249894142150879, + "objective/train/theoretical_loss": 3.3017718840144434, + "objective/train/tokens_used": 3234201600, + "objective/train/unlikelihood_loss": 0.14384348690509796, + "theoretical_loss": 3.3017718840144434, + "tokens_seen": 3234201600 + }, + { + "epoch": 0.98, + "learning_rate": 1.0190571715145438e-05, + "loss": 0.9925, + "theoretical_loss": 3.3017718840144434, + "tokens_seen": 3234201600 + }, + { + "epoch": 0.98, + "learning_rate": 1.0170511534603812e-05, + "loss": 1.1786, + "theoretical_loss": 3.3017617523797105, + "tokens_seen": 3234332672 + }, + { + "epoch": 0.98, + "learning_rate": 1.0150451354062187e-05, + "loss": 1.3454, + "theoretical_loss": 3.3017516212705154, + "tokens_seen": 3234463744 + }, + { + "epoch": 0.98, + "learning_rate": 1.0130391173520562e-05, + "loss": 1.3473, + "theoretical_loss": 3.3017414906868083, + "tokens_seen": 3234594816 + }, + { + "epoch": 0.98, + "learning_rate": 1.0110330992978937e-05, + "loss": 1.3423, + "theoretical_loss": 3.301731360628542, + "tokens_seen": 3234725888 + }, + { + "epoch": 0.98, + "learning_rate": 1.0090270812437311e-05, + "loss": 1.2811, + "theoretical_loss": 3.3017212310956667, + "tokens_seen": 3234856960 + }, + { + "epoch": 0.98, + "learning_rate": 1.0070210631895687e-05, + "loss": 1.4222, + "theoretical_loss": 3.3017111020881345, + "tokens_seen": 3234988032 + }, + { + "epoch": 0.98, + "learning_rate": 1.0050150451354063e-05, + "loss": 1.1455, + "theoretical_loss": 3.301700973605897, + "tokens_seen": 3235119104 + }, + { + "epoch": 0.98, + "learning_rate": 1.0030090270812437e-05, + "loss": 1.4415, + "theoretical_loss": 3.3016908456489054, + "tokens_seen": 3235250176 + }, + { + "epoch": 0.98, + "learning_rate": 1.0010030090270813e-05, + "loss": 1.2555, + "theoretical_loss": 3.3016807182171113, + "tokens_seen": 3235381248 + }, + { + "epoch": 0.98, + "learning_rate": 9.989969909729187e-06, + "loss": 1.2586, + "theoretical_loss": 3.301670591310466, + "tokens_seen": 3235512320 + }, + { + "epoch": 0.98, + "learning_rate": 9.969909729187565e-06, + "loss": 1.0818, + "theoretical_loss": 3.3016604649289216, + "tokens_seen": 3235643392 + }, + { + "epoch": 0.98, + "learning_rate": 9.949849548645939e-06, + "loss": 1.2739, + "theoretical_loss": 3.3016503390724288, + "tokens_seen": 3235774464 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.3831866979598999, + "objective/train/avg_token_score": 0.006559401750564575, + "objective/train/docs_used": 1819270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1648765802383423, + "objective/train/original_loss": 3.201885223388672, + "objective/train/theoretical_loss": 3.3016452763410618, + "objective/train/tokens_used": 3235840000, + "objective/train/unlikelihood_loss": 0.22325165569782257, + "theoretical_loss": 3.3016452763410618, + "tokens_seen": 3235840000 + }, + { + "epoch": 0.98, + "learning_rate": 9.929789368104313e-06, + "loss": 1.2275, + "theoretical_loss": 3.3016402137409395, + "tokens_seen": 3235905536 + }, + { + "epoch": 0.98, + "learning_rate": 9.909729187562688e-06, + "loss": 1.3823, + "theoretical_loss": 3.301630088934405, + "tokens_seen": 3236036608 + }, + { + "epoch": 0.98, + "learning_rate": 9.889669007021063e-06, + "loss": 1.2847, + "theoretical_loss": 3.3016199646527773, + "tokens_seen": 3236167680 + }, + { + "epoch": 0.98, + "learning_rate": 9.869608826479438e-06, + "loss": 1.3455, + "theoretical_loss": 3.301609840896007, + "tokens_seen": 3236298752 + }, + { + "epoch": 0.98, + "learning_rate": 9.849548645937814e-06, + "loss": 1.3131, + "theoretical_loss": 3.301599717664047, + "tokens_seen": 3236429824 + }, + { + "epoch": 0.98, + "learning_rate": 9.82948846539619e-06, + "loss": 1.3822, + "theoretical_loss": 3.3015895949568477, + "tokens_seen": 3236560896 + }, + { + "epoch": 0.98, + "learning_rate": 9.809428284854564e-06, + "loss": 1.2424, + "theoretical_loss": 3.3015794727743613, + "tokens_seen": 3236691968 + }, + { + "epoch": 0.98, + "learning_rate": 9.789368104312938e-06, + "loss": 1.2679, + "theoretical_loss": 3.301569351116539, + "tokens_seen": 3236823040 + }, + { + "epoch": 0.98, + "learning_rate": 9.769307923771314e-06, + "loss": 1.1326, + "theoretical_loss": 3.301559229983332, + "tokens_seen": 3236954112 + }, + { + "epoch": 0.98, + "learning_rate": 9.749247743229688e-06, + "loss": 1.37, + "theoretical_loss": 3.301549109374693, + "tokens_seen": 3237085184 + }, + { + "epoch": 0.98, + "learning_rate": 9.729187562688066e-06, + "loss": 1.1799, + "theoretical_loss": 3.301538989290573, + "tokens_seen": 3237216256 + }, + { + "epoch": 0.98, + "learning_rate": 9.70912738214644e-06, + "loss": 1.3118, + "theoretical_loss": 3.3015288697309226, + "tokens_seen": 3237347328 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.47611191868782043, + "objective/train/avg_token_score": 0.009868558496236801, + "objective/train/docs_used": 1820808, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.323327898979187, + "objective/train/original_loss": 3.3756208419799805, + "objective/train/theoretical_loss": 3.301518750695695, + "objective/train/tokens_used": 3237478400, + "objective/train/unlikelihood_loss": 0.2048998326063156, + "theoretical_loss": 3.301518750695695, + "tokens_seen": 3237478400 + }, + { + "epoch": 0.98, + "learning_rate": 9.689067201604815e-06, + "loss": 1.2807, + "theoretical_loss": 3.301518750695695, + "tokens_seen": 3237478400 + }, + { + "epoch": 0.98, + "learning_rate": 9.66900702106319e-06, + "loss": 1.2553, + "theoretical_loss": 3.301508632184841, + "tokens_seen": 3237609472 + }, + { + "epoch": 0.98, + "learning_rate": 9.648946840521564e-06, + "loss": 0.9759, + "theoretical_loss": 3.301498514198312, + "tokens_seen": 3237740544 + }, + { + "epoch": 0.98, + "learning_rate": 9.628886659979941e-06, + "loss": 1.2988, + "theoretical_loss": 3.30148839673606, + "tokens_seen": 3237871616 + }, + { + "epoch": 0.98, + "learning_rate": 9.608826479438315e-06, + "loss": 1.1502, + "theoretical_loss": 3.3014782797980367, + "tokens_seen": 3238002688 + }, + { + "epoch": 0.98, + "learning_rate": 9.588766298896691e-06, + "loss": 1.0966, + "theoretical_loss": 3.301468163384193, + "tokens_seen": 3238133760 + }, + { + "epoch": 0.98, + "learning_rate": 9.568706118355065e-06, + "loss": 1.3357, + "theoretical_loss": 3.3014580474944815, + "tokens_seen": 3238264832 + }, + { + "epoch": 0.98, + "learning_rate": 9.54864593781344e-06, + "loss": 1.2106, + "theoretical_loss": 3.301447932128853, + "tokens_seen": 3238395904 + }, + { + "epoch": 0.98, + "learning_rate": 9.528585757271815e-06, + "loss": 1.2971, + "theoretical_loss": 3.3014378172872596, + "tokens_seen": 3238526976 + }, + { + "epoch": 0.98, + "learning_rate": 9.50852557673019e-06, + "loss": 1.1949, + "theoretical_loss": 3.301427702969653, + "tokens_seen": 3238658048 + }, + { + "epoch": 0.98, + "learning_rate": 9.488465396188567e-06, + "loss": 1.0858, + "theoretical_loss": 3.3014175891759847, + "tokens_seen": 3238789120 + }, + { + "epoch": 0.98, + "learning_rate": 9.46840521564694e-06, + "loss": 1.0868, + "theoretical_loss": 3.301407475906206, + "tokens_seen": 3238920192 + }, + { + "epoch": 0.98, + "learning_rate": 9.448345035105316e-06, + "loss": 1.357, + "theoretical_loss": 3.3013973631602695, + "tokens_seen": 3239051264 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.4915078282356262, + "objective/train/avg_token_score": 0.014469885267317295, + "objective/train/docs_used": 1821566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4832881689071655, + "objective/train/original_loss": 3.879680871963501, + "objective/train/theoretical_loss": 3.3013923069837263, + "objective/train/tokens_used": 3239116800, + "objective/train/unlikelihood_loss": 0.14204619824886322, + "theoretical_loss": 3.3013923069837263, + "tokens_seen": 3239116800 + }, + { + "epoch": 0.98, + "learning_rate": 9.42828485456369e-06, + "loss": 1.207, + "theoretical_loss": 3.3013872509381255, + "tokens_seen": 3239182336 + }, + { + "epoch": 0.98, + "learning_rate": 9.408224674022065e-06, + "loss": 1.2402, + "theoretical_loss": 3.3013771392397273, + "tokens_seen": 3239313408 + }, + { + "epoch": 0.98, + "learning_rate": 9.388164493480442e-06, + "loss": 1.2522, + "theoretical_loss": 3.3013670280650254, + "tokens_seen": 3239444480 + }, + { + "epoch": 0.98, + "learning_rate": 9.368104312938816e-06, + "loss": 1.3547, + "theoretical_loss": 3.3013569174139716, + "tokens_seen": 3239575552 + }, + { + "epoch": 0.98, + "learning_rate": 9.348044132397192e-06, + "loss": 1.3516, + "theoretical_loss": 3.301346807286518, + "tokens_seen": 3239706624 + }, + { + "epoch": 0.98, + "learning_rate": 9.327983951855566e-06, + "loss": 1.3339, + "theoretical_loss": 3.301336697682616, + "tokens_seen": 3239837696 + }, + { + "epoch": 0.98, + "learning_rate": 9.307923771313942e-06, + "loss": 1.1626, + "theoretical_loss": 3.3013265886022176, + "tokens_seen": 3239968768 + }, + { + "epoch": 0.98, + "learning_rate": 9.287863590772318e-06, + "loss": 1.2348, + "theoretical_loss": 3.3013164800452746, + "tokens_seen": 3240099840 + }, + { + "epoch": 0.98, + "learning_rate": 9.267803410230692e-06, + "loss": 1.3194, + "theoretical_loss": 3.301306372011738, + "tokens_seen": 3240230912 + }, + { + "epoch": 0.98, + "learning_rate": 9.247743229689068e-06, + "loss": 1.0914, + "theoretical_loss": 3.30129626450156, + "tokens_seen": 3240361984 + }, + { + "epoch": 0.98, + "learning_rate": 9.227683049147442e-06, + "loss": 1.175, + "theoretical_loss": 3.3012861575146926, + "tokens_seen": 3240493056 + }, + { + "epoch": 0.98, + "learning_rate": 9.207622868605818e-06, + "loss": 1.1243, + "theoretical_loss": 3.3012760510510875, + "tokens_seen": 3240624128 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.3253909945487976, + "objective/train/avg_token_score": 0.0076591093093156815, + "objective/train/docs_used": 1822902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.01478111743927, + "objective/train/original_loss": 3.9354231357574463, + "objective/train/theoretical_loss": 3.301265945110696, + "objective/train/tokens_used": 3240755200, + "objective/train/unlikelihood_loss": 0.15454895794391632, + "theoretical_loss": 3.301265945110696, + "tokens_seen": 3240755200 + }, + { + "epoch": 0.98, + "learning_rate": 9.187562688064192e-06, + "loss": 1.3434, + "theoretical_loss": 3.301265945110696, + "tokens_seen": 3240755200 + }, + { + "epoch": 0.98, + "learning_rate": 9.167502507522569e-06, + "loss": 1.2324, + "theoretical_loss": 3.3012558396934697, + "tokens_seen": 3240886272 + }, + { + "epoch": 0.98, + "learning_rate": 9.147442326980943e-06, + "loss": 1.2982, + "theoretical_loss": 3.301245734799361, + "tokens_seen": 3241017344 + }, + { + "epoch": 0.98, + "learning_rate": 9.127382146439317e-06, + "loss": 1.1738, + "theoretical_loss": 3.3012356304283212, + "tokens_seen": 3241148416 + }, + { + "epoch": 0.98, + "learning_rate": 9.107321965897693e-06, + "loss": 1.2165, + "theoretical_loss": 3.3012255265803025, + "tokens_seen": 3241279488 + }, + { + "epoch": 0.98, + "learning_rate": 9.087261785356067e-06, + "loss": 1.2664, + "theoretical_loss": 3.3012154232552566, + "tokens_seen": 3241410560 + }, + { + "epoch": 0.98, + "learning_rate": 9.067201604814445e-06, + "loss": 1.1635, + "theoretical_loss": 3.301205320453135, + "tokens_seen": 3241541632 + }, + { + "epoch": 0.98, + "learning_rate": 9.047141424272819e-06, + "loss": 1.1767, + "theoretical_loss": 3.3011952181738895, + "tokens_seen": 3241672704 + }, + { + "epoch": 0.98, + "learning_rate": 9.027081243731195e-06, + "loss": 1.2834, + "theoretical_loss": 3.3011851164174724, + "tokens_seen": 3241803776 + }, + { + "epoch": 0.98, + "learning_rate": 9.007021063189569e-06, + "loss": 1.2013, + "theoretical_loss": 3.301175015183835, + "tokens_seen": 3241934848 + }, + { + "epoch": 0.98, + "learning_rate": 8.986960882647943e-06, + "loss": 1.3483, + "theoretical_loss": 3.301164914472929, + "tokens_seen": 3242065920 + }, + { + "epoch": 0.98, + "learning_rate": 8.966900702106319e-06, + "loss": 1.3542, + "theoretical_loss": 3.3011548142847067, + "tokens_seen": 3242196992 + }, + { + "epoch": 0.98, + "learning_rate": 8.946840521564694e-06, + "loss": 1.1825, + "theoretical_loss": 3.3011447146191193, + "tokens_seen": 3242328064 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.42815250158309937, + "objective/train/avg_token_score": 0.0052326880395412445, + "objective/train/docs_used": 1823477, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2595701217651367, + "objective/train/original_loss": 3.340034008026123, + "objective/train/theoretical_loss": 3.301139664982299, + "objective/train/tokens_used": 3242393600, + "objective/train/unlikelihood_loss": 0.18434450030326843, + "theoretical_loss": 3.301139664982299, + "tokens_seen": 3242393600 + }, + { + "epoch": 0.98, + "learning_rate": 8.92678034102307e-06, + "loss": 1.0995, + "theoretical_loss": 3.3011346154761196, + "tokens_seen": 3242459136 + }, + { + "epoch": 0.98, + "learning_rate": 8.906720160481444e-06, + "loss": 1.2528, + "theoretical_loss": 3.3011245168556584, + "tokens_seen": 3242590208 + }, + { + "epoch": 0.98, + "learning_rate": 8.88665997993982e-06, + "loss": 1.211, + "theoretical_loss": 3.301114418757688, + "tokens_seen": 3242721280 + }, + { + "epoch": 0.98, + "learning_rate": 8.866599799398194e-06, + "loss": 1.4154, + "theoretical_loss": 3.3011043211821605, + "tokens_seen": 3242852352 + }, + { + "epoch": 0.98, + "learning_rate": 8.846539618856568e-06, + "loss": 1.2242, + "theoretical_loss": 3.3010942241290273, + "tokens_seen": 3242983424 + }, + { + "epoch": 0.98, + "learning_rate": 8.826479438314946e-06, + "loss": 1.1409, + "theoretical_loss": 3.3010841275982408, + "tokens_seen": 3243114496 + }, + { + "epoch": 0.98, + "learning_rate": 8.80641925777332e-06, + "loss": 1.3246, + "theoretical_loss": 3.301074031589752, + "tokens_seen": 3243245568 + }, + { + "epoch": 0.98, + "learning_rate": 8.786359077231696e-06, + "loss": 1.2276, + "theoretical_loss": 3.301063936103514, + "tokens_seen": 3243376640 + }, + { + "epoch": 0.98, + "learning_rate": 8.76629889669007e-06, + "loss": 1.2362, + "theoretical_loss": 3.3010538411394776, + "tokens_seen": 3243507712 + }, + { + "epoch": 0.98, + "learning_rate": 8.746238716148446e-06, + "loss": 1.3278, + "theoretical_loss": 3.301043746697595, + "tokens_seen": 3243638784 + }, + { + "epoch": 0.98, + "learning_rate": 8.726178535606821e-06, + "loss": 1.2688, + "theoretical_loss": 3.301033652777818, + "tokens_seen": 3243769856 + }, + { + "epoch": 0.98, + "learning_rate": 8.706118355065195e-06, + "loss": 1.3683, + "theoretical_loss": 3.301023559380099, + "tokens_seen": 3243900928 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.51496821641922, + "objective/train/avg_token_score": 0.0051541682332754135, + "objective/train/docs_used": 1824079, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5385549068450928, + "objective/train/original_loss": 3.2369046211242676, + "objective/train/theoretical_loss": 3.3010134665043895, + "objective/train/tokens_used": 3244032000, + "objective/train/unlikelihood_loss": 0.19323194026947021, + "theoretical_loss": 3.3010134665043895, + "tokens_seen": 3244032000 + }, + { + "epoch": 0.98, + "learning_rate": 8.686058174523571e-06, + "loss": 1.2448, + "theoretical_loss": 3.3010134665043895, + "tokens_seen": 3244032000 + }, + { + "epoch": 0.98, + "learning_rate": 8.665997993981945e-06, + "loss": 1.3586, + "theoretical_loss": 3.3010033741506417, + "tokens_seen": 3244163072 + }, + { + "epoch": 0.98, + "learning_rate": 8.645937813440321e-06, + "loss": 1.2838, + "theoretical_loss": 3.3009932823188066, + "tokens_seen": 3244294144 + }, + { + "epoch": 0.98, + "learning_rate": 8.625877632898695e-06, + "loss": 1.3058, + "theoretical_loss": 3.3009831910088376, + "tokens_seen": 3244425216 + }, + { + "epoch": 0.98, + "learning_rate": 8.605817452357073e-06, + "loss": 1.3025, + "theoretical_loss": 3.3009731002206855, + "tokens_seen": 3244556288 + }, + { + "epoch": 0.98, + "learning_rate": 8.585757271815447e-06, + "loss": 1.2991, + "theoretical_loss": 3.3009630099543026, + "tokens_seen": 3244687360 + }, + { + "epoch": 0.98, + "learning_rate": 8.565697091273821e-06, + "loss": 1.468, + "theoretical_loss": 3.300952920209641, + "tokens_seen": 3244818432 + }, + { + "epoch": 0.98, + "learning_rate": 8.545636910732197e-06, + "loss": 1.3029, + "theoretical_loss": 3.3009428309866524, + "tokens_seen": 3244949504 + }, + { + "epoch": 0.98, + "learning_rate": 8.52557673019057e-06, + "loss": 1.1225, + "theoretical_loss": 3.3009327422852888, + "tokens_seen": 3245080576 + }, + { + "epoch": 0.98, + "learning_rate": 8.505516549648948e-06, + "loss": 1.198, + "theoretical_loss": 3.3009226541055026, + "tokens_seen": 3245211648 + }, + { + "epoch": 0.98, + "learning_rate": 8.485456369107322e-06, + "loss": 1.3528, + "theoretical_loss": 3.300912566447245, + "tokens_seen": 3245342720 + }, + { + "epoch": 0.98, + "learning_rate": 8.465396188565698e-06, + "loss": 1.2047, + "theoretical_loss": 3.3009024793104684, + "tokens_seen": 3245473792 + }, + { + "epoch": 0.98, + "learning_rate": 8.445336008024072e-06, + "loss": 1.3189, + "theoretical_loss": 3.300892392695125, + "tokens_seen": 3245604864 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.5360459685325623, + "objective/train/avg_token_score": 0.011536694131791592, + "objective/train/docs_used": 1824541, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5942678451538086, + "objective/train/original_loss": 3.5875446796417236, + "objective/train/theoretical_loss": 3.3008873495829754, + "objective/train/tokens_used": 3245670400, + "objective/train/unlikelihood_loss": 0.14748966693878174, + "theoretical_loss": 3.3008873495829754, + "tokens_seen": 3245670400 + }, + { + "epoch": 0.98, + "learning_rate": 8.425275827482446e-06, + "loss": 1.4119, + "theoretical_loss": 3.3008823066011663, + "tokens_seen": 3245735936 + }, + { + "epoch": 0.98, + "learning_rate": 8.405215646940822e-06, + "loss": 1.1276, + "theoretical_loss": 3.300872221028545, + "tokens_seen": 3245867008 + }, + { + "epoch": 0.98, + "learning_rate": 8.385155466399198e-06, + "loss": 1.2372, + "theoretical_loss": 3.300862135977212, + "tokens_seen": 3245998080 + }, + { + "epoch": 0.98, + "learning_rate": 8.365095285857574e-06, + "loss": 1.2127, + "theoretical_loss": 3.3008520514471202, + "tokens_seen": 3246129152 + }, + { + "epoch": 0.98, + "learning_rate": 8.345035105315948e-06, + "loss": 1.2581, + "theoretical_loss": 3.3008419674382212, + "tokens_seen": 3246260224 + }, + { + "epoch": 0.98, + "learning_rate": 8.324974924774324e-06, + "loss": 1.1702, + "theoretical_loss": 3.3008318839504676, + "tokens_seen": 3246391296 + }, + { + "epoch": 0.98, + "learning_rate": 8.304914744232698e-06, + "loss": 1.2894, + "theoretical_loss": 3.300821800983811, + "tokens_seen": 3246522368 + }, + { + "epoch": 0.98, + "learning_rate": 8.284854563691072e-06, + "loss": 1.2596, + "theoretical_loss": 3.3008117185382027, + "tokens_seen": 3246653440 + }, + { + "epoch": 0.98, + "learning_rate": 8.26479438314945e-06, + "loss": 1.1035, + "theoretical_loss": 3.300801636613596, + "tokens_seen": 3246784512 + }, + { + "epoch": 0.98, + "learning_rate": 8.244734202607823e-06, + "loss": 1.404, + "theoretical_loss": 3.3007915552099423, + "tokens_seen": 3246915584 + }, + { + "epoch": 0.98, + "learning_rate": 8.2246740220662e-06, + "loss": 1.2494, + "theoretical_loss": 3.300781474327194, + "tokens_seen": 3247046656 + }, + { + "epoch": 0.98, + "learning_rate": 8.204613841524573e-06, + "loss": 1.2417, + "theoretical_loss": 3.300771393965303, + "tokens_seen": 3247177728 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.3437194526195526, + "objective/train/avg_token_score": 0.018022146075963974, + "objective/train/docs_used": 1825604, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.930517852306366, + "objective/train/original_loss": 3.6803767681121826, + "objective/train/theoretical_loss": 3.3007613141242214, + "objective/train/tokens_used": 3247308800, + "objective/train/unlikelihood_loss": 0.15270644426345825, + "theoretical_loss": 3.3007613141242214, + "tokens_seen": 3247308800 + }, + { + "epoch": 0.98, + "learning_rate": 8.184553660982949e-06, + "loss": 1.2087, + "theoretical_loss": 3.3007613141242214, + "tokens_seen": 3247308800 + }, + { + "epoch": 0.98, + "learning_rate": 8.164493480441325e-06, + "loss": 1.3489, + "theoretical_loss": 3.3007512348039008, + "tokens_seen": 3247439872 + }, + { + "epoch": 0.98, + "learning_rate": 8.144433299899699e-06, + "loss": 1.3204, + "theoretical_loss": 3.300741156004294, + "tokens_seen": 3247570944 + }, + { + "epoch": 0.98, + "learning_rate": 8.124373119358075e-06, + "loss": 1.2387, + "theoretical_loss": 3.3007310777253522, + "tokens_seen": 3247702016 + }, + { + "epoch": 0.98, + "learning_rate": 8.104312938816449e-06, + "loss": 1.3029, + "theoretical_loss": 3.3007209999670284, + "tokens_seen": 3247833088 + }, + { + "epoch": 0.98, + "learning_rate": 8.084252758274825e-06, + "loss": 1.2775, + "theoretical_loss": 3.300710922729275, + "tokens_seen": 3247964160 + }, + { + "epoch": 0.98, + "learning_rate": 8.064192577733199e-06, + "loss": 1.4026, + "theoretical_loss": 3.3007008460120426, + "tokens_seen": 3248095232 + }, + { + "epoch": 0.98, + "learning_rate": 8.044132397191576e-06, + "loss": 1.3722, + "theoretical_loss": 3.300690769815284, + "tokens_seen": 3248226304 + }, + { + "epoch": 0.98, + "learning_rate": 8.02407221664995e-06, + "loss": 1.3528, + "theoretical_loss": 3.300680694138952, + "tokens_seen": 3248357376 + }, + { + "epoch": 0.98, + "learning_rate": 8.004012036108324e-06, + "loss": 1.3794, + "theoretical_loss": 3.300670618982998, + "tokens_seen": 3248488448 + }, + { + "epoch": 0.98, + "learning_rate": 7.9839518555667e-06, + "loss": 1.2478, + "theoretical_loss": 3.300660544347374, + "tokens_seen": 3248619520 + }, + { + "epoch": 0.98, + "learning_rate": 7.963891675025074e-06, + "loss": 1.3069, + "theoretical_loss": 3.300650470232033, + "tokens_seen": 3248750592 + }, + { + "epoch": 0.98, + "learning_rate": 7.943831494483452e-06, + "loss": 1.2499, + "theoretical_loss": 3.3006403966369264, + "tokens_seen": 3248881664 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.34836265444755554, + "objective/train/avg_token_score": 0.011814399622380733, + "objective/train/docs_used": 1826305, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0396984815597534, + "objective/train/original_loss": 4.164069175720215, + "objective/train/theoretical_loss": 3.300635360034446, + "objective/train/tokens_used": 3248947200, + "objective/train/unlikelihood_loss": 0.15111522376537323, + "theoretical_loss": 3.300635360034446, + "tokens_seen": 3248947200 + }, + { + "epoch": 0.98, + "learning_rate": 7.923771313941826e-06, + "loss": 1.3061, + "theoretical_loss": 3.3006303235620065, + "tokens_seen": 3249012736 + }, + { + "epoch": 0.98, + "learning_rate": 7.903711133400202e-06, + "loss": 1.2439, + "theoretical_loss": 3.3006202510072256, + "tokens_seen": 3249143808 + }, + { + "epoch": 0.98, + "learning_rate": 7.883650952858576e-06, + "loss": 1.2718, + "theoretical_loss": 3.3006101789725353, + "tokens_seen": 3249274880 + }, + { + "epoch": 0.98, + "learning_rate": 7.86359077231695e-06, + "loss": 1.2853, + "theoretical_loss": 3.300600107457889, + "tokens_seen": 3249405952 + }, + { + "epoch": 0.98, + "learning_rate": 7.843530591775326e-06, + "loss": 1.3068, + "theoretical_loss": 3.3005900364632375, + "tokens_seen": 3249537024 + }, + { + "epoch": 0.98, + "learning_rate": 7.823470411233702e-06, + "loss": 1.1466, + "theoretical_loss": 3.3005799659885335, + "tokens_seen": 3249668096 + }, + { + "epoch": 0.98, + "learning_rate": 7.803410230692076e-06, + "loss": 1.2776, + "theoretical_loss": 3.3005698960337297, + "tokens_seen": 3249799168 + }, + { + "epoch": 0.98, + "learning_rate": 7.783350050150451e-06, + "loss": 1.3588, + "theoretical_loss": 3.3005598265987777, + "tokens_seen": 3249930240 + }, + { + "epoch": 0.98, + "learning_rate": 7.763289869608827e-06, + "loss": 1.0413, + "theoretical_loss": 3.30054975768363, + "tokens_seen": 3250061312 + }, + { + "epoch": 0.98, + "learning_rate": 7.743229689067201e-06, + "loss": 1.3144, + "theoretical_loss": 3.3005396892882386, + "tokens_seen": 3250192384 + }, + { + "epoch": 0.98, + "learning_rate": 7.723169508525577e-06, + "loss": 1.3791, + "theoretical_loss": 3.3005296214125557, + "tokens_seen": 3250323456 + }, + { + "epoch": 0.98, + "learning_rate": 7.703109327983953e-06, + "loss": 1.2251, + "theoretical_loss": 3.3005195540565335, + "tokens_seen": 3250454528 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.2581256031990051, + "objective/train/avg_token_score": 0.057645153254270554, + "objective/train/docs_used": 1827829, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.7898641228675842, + "objective/train/original_loss": 4.039286136627197, + "objective/train/theoretical_loss": 3.3005094872201246, + "objective/train/tokens_used": 3250585600, + "objective/train/unlikelihood_loss": 0.2554488480091095, + "theoretical_loss": 3.3005094872201246, + "tokens_seen": 3250585600 + }, + { + "epoch": 0.99, + "learning_rate": 7.683049147442327e-06, + "loss": 1.1698, + "theoretical_loss": 3.3005094872201246, + "tokens_seen": 3250585600 + }, + { + "epoch": 0.99, + "learning_rate": 7.662988966900701e-06, + "loss": 1.3287, + "theoretical_loss": 3.3004994209032805, + "tokens_seen": 3250716672 + }, + { + "epoch": 0.99, + "learning_rate": 7.642928786359077e-06, + "loss": 1.3023, + "theoretical_loss": 3.300489355105954, + "tokens_seen": 3250847744 + }, + { + "epoch": 0.99, + "learning_rate": 7.622868605817453e-06, + "loss": 1.128, + "theoretical_loss": 3.3004792898280977, + "tokens_seen": 3250978816 + }, + { + "epoch": 0.99, + "learning_rate": 7.602808425275828e-06, + "loss": 1.4068, + "theoretical_loss": 3.300469225069663, + "tokens_seen": 3251109888 + }, + { + "epoch": 0.99, + "learning_rate": 7.582748244734203e-06, + "loss": 1.3171, + "theoretical_loss": 3.300459160830602, + "tokens_seen": 3251240960 + }, + { + "epoch": 0.99, + "learning_rate": 7.562688064192578e-06, + "loss": 1.2708, + "theoretical_loss": 3.300449097110868, + "tokens_seen": 3251372032 + }, + { + "epoch": 0.99, + "learning_rate": 7.542627883650953e-06, + "loss": 1.2451, + "theoretical_loss": 3.300439033910413, + "tokens_seen": 3251503104 + }, + { + "epoch": 0.99, + "learning_rate": 7.522567703109327e-06, + "loss": 1.3467, + "theoretical_loss": 3.300428971229189, + "tokens_seen": 3251634176 + }, + { + "epoch": 0.99, + "learning_rate": 7.502507522567703e-06, + "loss": 1.4002, + "theoretical_loss": 3.3004189090671483, + "tokens_seen": 3251765248 + }, + { + "epoch": 0.99, + "learning_rate": 7.482447342026078e-06, + "loss": 1.2995, + "theoretical_loss": 3.300408847424243, + "tokens_seen": 3251896320 + }, + { + "epoch": 0.99, + "learning_rate": 7.462387161484454e-06, + "loss": 1.3344, + "theoretical_loss": 3.3003987863004256, + "tokens_seen": 3252027392 + }, + { + "epoch": 0.99, + "learning_rate": 7.442326980942829e-06, + "loss": 1.384, + "theoretical_loss": 3.3003887256956483, + "tokens_seen": 3252158464 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.4664589464664459, + "objective/train/avg_token_score": 0.014311726205050945, + "objective/train/docs_used": 1828572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4141318798065186, + "objective/train/original_loss": 4.091707229614258, + "objective/train/theoretical_loss": 3.300383695587885, + "objective/train/tokens_used": 3252224000, + "objective/train/unlikelihood_loss": 0.1218198910355568, + "theoretical_loss": 3.300383695587885, + "tokens_seen": 3252224000 + }, + { + "epoch": 0.99, + "learning_rate": 7.422266800401204e-06, + "loss": 1.1845, + "theoretical_loss": 3.300378665609864, + "tokens_seen": 3252289536 + }, + { + "epoch": 0.99, + "learning_rate": 7.402206619859579e-06, + "loss": 1.407, + "theoretical_loss": 3.300368606043024, + "tokens_seen": 3252420608 + }, + { + "epoch": 0.99, + "learning_rate": 7.382146439317954e-06, + "loss": 1.2885, + "theoretical_loss": 3.3003585469950814, + "tokens_seen": 3252551680 + }, + { + "epoch": 0.99, + "learning_rate": 7.362086258776329e-06, + "loss": 1.3357, + "theoretical_loss": 3.300348488465988, + "tokens_seen": 3252682752 + }, + { + "epoch": 0.99, + "learning_rate": 7.3420260782347045e-06, + "loss": 1.3683, + "theoretical_loss": 3.300338430455697, + "tokens_seen": 3252813824 + }, + { + "epoch": 0.99, + "learning_rate": 7.3219658976930794e-06, + "loss": 1.1054, + "theoretical_loss": 3.3003283729641595, + "tokens_seen": 3252944896 + }, + { + "epoch": 0.99, + "learning_rate": 7.301905717151454e-06, + "loss": 1.2656, + "theoretical_loss": 3.3003183159913285, + "tokens_seen": 3253075968 + }, + { + "epoch": 0.99, + "learning_rate": 7.28184553660983e-06, + "loss": 1.2756, + "theoretical_loss": 3.3003082595371565, + "tokens_seen": 3253207040 + }, + { + "epoch": 0.99, + "learning_rate": 7.261785356068204e-06, + "loss": 1.2097, + "theoretical_loss": 3.3002982036015958, + "tokens_seen": 3253338112 + }, + { + "epoch": 0.99, + "learning_rate": 7.241725175526579e-06, + "loss": 1.3516, + "theoretical_loss": 3.300288148184598, + "tokens_seen": 3253469184 + }, + { + "epoch": 0.99, + "learning_rate": 7.221664994984955e-06, + "loss": 1.1818, + "theoretical_loss": 3.3002780932861167, + "tokens_seen": 3253600256 + }, + { + "epoch": 0.99, + "learning_rate": 7.20160481444333e-06, + "loss": 1.3557, + "theoretical_loss": 3.3002680389061037, + "tokens_seen": 3253731328 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.42082110047340393, + "objective/train/avg_token_score": 0.01880498416721821, + "objective/train/docs_used": 1829944, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2854583263397217, + "objective/train/original_loss": 3.4707212448120117, + "objective/train/theoretical_loss": 3.300257985044511, + "objective/train/tokens_used": 3253862400, + "objective/train/unlikelihood_loss": 0.22359290719032288, + "theoretical_loss": 3.300257985044511, + "tokens_seen": 3253862400 + }, + { + "epoch": 0.99, + "learning_rate": 7.181544633901706e-06, + "loss": 1.2787, + "theoretical_loss": 3.300257985044511, + "tokens_seen": 3253862400 + }, + { + "epoch": 0.99, + "learning_rate": 7.161484453360081e-06, + "loss": 1.2302, + "theoretical_loss": 3.300247931701292, + "tokens_seen": 3253993472 + }, + { + "epoch": 0.99, + "learning_rate": 7.141424272818456e-06, + "loss": 1.1542, + "theoretical_loss": 3.300237878876398, + "tokens_seen": 3254124544 + }, + { + "epoch": 0.99, + "learning_rate": 7.121364092276831e-06, + "loss": 1.3176, + "theoretical_loss": 3.300227826569782, + "tokens_seen": 3254255616 + }, + { + "epoch": 0.99, + "learning_rate": 7.1013039117352055e-06, + "loss": 1.4152, + "theoretical_loss": 3.300217774781396, + "tokens_seen": 3254386688 + }, + { + "epoch": 0.99, + "learning_rate": 7.0812437311935805e-06, + "loss": 1.2527, + "theoretical_loss": 3.3002077235111926, + "tokens_seen": 3254517760 + }, + { + "epoch": 0.99, + "learning_rate": 7.061183550651956e-06, + "loss": 1.2694, + "theoretical_loss": 3.3001976727591247, + "tokens_seen": 3254648832 + }, + { + "epoch": 0.99, + "learning_rate": 7.041123370110331e-06, + "loss": 1.3343, + "theoretical_loss": 3.300187622525144, + "tokens_seen": 3254779904 + }, + { + "epoch": 0.99, + "learning_rate": 7.021063189568706e-06, + "loss": 1.3018, + "theoretical_loss": 3.3001775728092033, + "tokens_seen": 3254910976 + }, + { + "epoch": 0.99, + "learning_rate": 7.001003009027082e-06, + "loss": 1.2622, + "theoretical_loss": 3.3001675236112553, + "tokens_seen": 3255042048 + }, + { + "epoch": 0.99, + "learning_rate": 6.980942828485456e-06, + "loss": 1.3183, + "theoretical_loss": 3.300157474931252, + "tokens_seen": 3255173120 + }, + { + "epoch": 0.99, + "learning_rate": 6.960882647943831e-06, + "loss": 1.4533, + "theoretical_loss": 3.3001474267691457, + "tokens_seen": 3255304192 + }, + { + "epoch": 0.99, + "learning_rate": 6.940822467402207e-06, + "loss": 1.2712, + "theoretical_loss": 3.300137379124889, + "tokens_seen": 3255435264 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.3266739845275879, + "objective/train/avg_token_score": 0.01057970430701971, + "objective/train/docs_used": 1830489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9497876763343811, + "objective/train/original_loss": 3.830942153930664, + "objective/train/theoretical_loss": 3.30013235549694, + "objective/train/tokens_used": 3255500800, + "objective/train/unlikelihood_loss": 0.20653584599494934, + "theoretical_loss": 3.30013235549694, + "tokens_seen": 3255500800 + }, + { + "epoch": 0.99, + "learning_rate": 6.920762286860582e-06, + "loss": 1.3349, + "theoretical_loss": 3.300127331998435, + "tokens_seen": 3255566336 + }, + { + "epoch": 0.99, + "learning_rate": 6.9007021063189575e-06, + "loss": 1.2954, + "theoretical_loss": 3.3001172853897356, + "tokens_seen": 3255697408 + }, + { + "epoch": 0.99, + "learning_rate": 6.8806419257773325e-06, + "loss": 1.2793, + "theoretical_loss": 3.300107239298743, + "tokens_seen": 3255828480 + }, + { + "epoch": 0.99, + "learning_rate": 6.860581745235707e-06, + "loss": 1.429, + "theoretical_loss": 3.3000971937254104, + "tokens_seen": 3255959552 + }, + { + "epoch": 0.99, + "learning_rate": 6.840521564694082e-06, + "loss": 1.2873, + "theoretical_loss": 3.30008714866969, + "tokens_seen": 3256090624 + }, + { + "epoch": 0.99, + "learning_rate": 6.820461384152457e-06, + "loss": 1.2936, + "theoretical_loss": 3.300077104131534, + "tokens_seen": 3256221696 + }, + { + "epoch": 0.99, + "learning_rate": 6.800401203610832e-06, + "loss": 1.183, + "theoretical_loss": 3.300067060110895, + "tokens_seen": 3256352768 + }, + { + "epoch": 0.99, + "learning_rate": 6.780341023069208e-06, + "loss": 1.3029, + "theoretical_loss": 3.3000570166077257, + "tokens_seen": 3256483840 + }, + { + "epoch": 0.99, + "learning_rate": 6.760280842527583e-06, + "loss": 1.1691, + "theoretical_loss": 3.3000469736219786, + "tokens_seen": 3256614912 + }, + { + "epoch": 0.99, + "learning_rate": 6.740220661985958e-06, + "loss": 1.2881, + "theoretical_loss": 3.300036931153606, + "tokens_seen": 3256745984 + }, + { + "epoch": 0.99, + "learning_rate": 6.720160481444334e-06, + "loss": 1.2273, + "theoretical_loss": 3.3000268892025604, + "tokens_seen": 3256877056 + }, + { + "epoch": 0.99, + "learning_rate": 6.700100300902708e-06, + "loss": 1.2503, + "theoretical_loss": 3.300016847768795, + "tokens_seen": 3257008128 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.3680962920188904, + "objective/train/avg_token_score": 0.014035237021744251, + "objective/train/docs_used": 1831077, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1501342058181763, + "objective/train/original_loss": 3.775256633758545, + "objective/train/theoretical_loss": 3.3000068068522617, + "objective/train/tokens_used": 3257139200, + "objective/train/unlikelihood_loss": 0.20062488317489624, + "theoretical_loss": 3.3000068068522617, + "tokens_seen": 3257139200 + }, + { + "epoch": 0.99, + "learning_rate": 6.680040120361083e-06, + "loss": 1.3975, + "theoretical_loss": 3.3000068068522617, + "tokens_seen": 3257139200 + }, + { + "epoch": 0.99, + "learning_rate": 6.659979939819459e-06, + "loss": 1.2372, + "theoretical_loss": 3.299996766452913, + "tokens_seen": 3257270272 + }, + { + "epoch": 0.99, + "learning_rate": 6.6399197592778335e-06, + "loss": 1.3662, + "theoretical_loss": 3.299986726570702, + "tokens_seen": 3257401344 + }, + { + "epoch": 0.99, + "learning_rate": 6.619859578736209e-06, + "loss": 1.3416, + "theoretical_loss": 3.2999766872055805, + "tokens_seen": 3257532416 + }, + { + "epoch": 0.99, + "learning_rate": 6.599799398194584e-06, + "loss": 1.3251, + "theoretical_loss": 3.299966648357502, + "tokens_seen": 3257663488 + }, + { + "epoch": 0.99, + "learning_rate": 6.579739217652959e-06, + "loss": 1.3873, + "theoretical_loss": 3.2999566100264177, + "tokens_seen": 3257794560 + }, + { + "epoch": 0.99, + "learning_rate": 6.559679037111334e-06, + "loss": 1.134, + "theoretical_loss": 3.299946572212282, + "tokens_seen": 3257925632 + }, + { + "epoch": 0.99, + "learning_rate": 6.539618856569709e-06, + "loss": 1.1408, + "theoretical_loss": 3.299936534915046, + "tokens_seen": 3258056704 + }, + { + "epoch": 0.99, + "learning_rate": 6.519558676028084e-06, + "loss": 1.3486, + "theoretical_loss": 3.2999264981346625, + "tokens_seen": 3258187776 + }, + { + "epoch": 0.99, + "learning_rate": 6.49949849548646e-06, + "loss": 1.2499, + "theoretical_loss": 3.2999164618710846, + "tokens_seen": 3258318848 + }, + { + "epoch": 0.99, + "learning_rate": 6.479438314944835e-06, + "loss": 1.1929, + "theoretical_loss": 3.2999064261242648, + "tokens_seen": 3258449920 + }, + { + "epoch": 0.99, + "learning_rate": 6.45937813440321e-06, + "loss": 1.3667, + "theoretical_loss": 3.2998963908941557, + "tokens_seen": 3258580992 + }, + { + "epoch": 0.99, + "learning_rate": 6.4393179538615855e-06, + "loss": 1.2556, + "theoretical_loss": 3.2998863561807097, + "tokens_seen": 3258712064 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.4258919954299927, + "objective/train/avg_token_score": 0.012067155912518501, + "objective/train/docs_used": 1832365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9690877795219421, + "objective/train/original_loss": 3.2211012840270996, + "objective/train/theoretical_loss": 3.2998813390177206, + "objective/train/tokens_used": 3258777600, + "objective/train/unlikelihood_loss": 0.2337491661310196, + "theoretical_loss": 3.2998813390177206, + "tokens_seen": 3258777600 + }, + { + "epoch": 0.99, + "learning_rate": 6.41925777331996e-06, + "loss": 1.2854, + "theoretical_loss": 3.2998763219838794, + "tokens_seen": 3258843136 + }, + { + "epoch": 0.99, + "learning_rate": 6.3991975927783346e-06, + "loss": 1.2155, + "theoretical_loss": 3.2998662883036176, + "tokens_seen": 3258974208 + }, + { + "epoch": 0.99, + "learning_rate": 6.37913741223671e-06, + "loss": 1.2718, + "theoretical_loss": 3.299856255139877, + "tokens_seen": 3259105280 + }, + { + "epoch": 0.99, + "learning_rate": 6.359077231695085e-06, + "loss": 1.2241, + "theoretical_loss": 3.2998462224926097, + "tokens_seen": 3259236352 + }, + { + "epoch": 0.99, + "learning_rate": 6.33901705115346e-06, + "loss": 1.1957, + "theoretical_loss": 3.299836190361769, + "tokens_seen": 3259367424 + }, + { + "epoch": 0.99, + "learning_rate": 6.318956870611836e-06, + "loss": 0.9524, + "theoretical_loss": 3.2998261587473077, + "tokens_seen": 3259498496 + }, + { + "epoch": 0.99, + "learning_rate": 6.298896690070211e-06, + "loss": 1.2183, + "theoretical_loss": 3.2998161276491778, + "tokens_seen": 3259629568 + }, + { + "epoch": 0.99, + "learning_rate": 6.278836509528586e-06, + "loss": 1.2079, + "theoretical_loss": 3.299806097067332, + "tokens_seen": 3259760640 + }, + { + "epoch": 0.99, + "learning_rate": 6.258776328986961e-06, + "loss": 1.4388, + "theoretical_loss": 3.2997960670017235, + "tokens_seen": 3259891712 + }, + { + "epoch": 0.99, + "learning_rate": 6.238716148445336e-06, + "loss": 1.3089, + "theoretical_loss": 3.2997860374523045, + "tokens_seen": 3260022784 + }, + { + "epoch": 0.99, + "learning_rate": 6.218655967903712e-06, + "loss": 1.0568, + "theoretical_loss": 3.299776008419028, + "tokens_seen": 3260153856 + }, + { + "epoch": 0.99, + "learning_rate": 6.1985957873620866e-06, + "loss": 1.2071, + "theoretical_loss": 3.2997659799018466, + "tokens_seen": 3260284928 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.5127077102661133, + "objective/train/avg_token_score": 0.010915274731814861, + "objective/train/docs_used": 1833018, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.624677062034607, + "objective/train/original_loss": 3.555248260498047, + "objective/train/theoretical_loss": 3.2997559519007127, + "objective/train/tokens_used": 3260416000, + "objective/train/unlikelihood_loss": 0.1842552125453949, + "theoretical_loss": 3.2997559519007127, + "tokens_seen": 3260416000 + }, + { + "epoch": 0.99, + "learning_rate": 6.1785356068204615e-06, + "loss": 1.2065, + "theoretical_loss": 3.2997559519007127, + "tokens_seen": 3260416000 + }, + { + "epoch": 0.99, + "learning_rate": 6.158475426278837e-06, + "loss": 1.2056, + "theoretical_loss": 3.29974592441558, + "tokens_seen": 3260547072 + }, + { + "epoch": 0.99, + "learning_rate": 6.138415245737211e-06, + "loss": 1.2502, + "theoretical_loss": 3.2997358974463995, + "tokens_seen": 3260678144 + }, + { + "epoch": 0.99, + "learning_rate": 6.118355065195586e-06, + "loss": 1.345, + "theoretical_loss": 3.299725870993125, + "tokens_seen": 3260809216 + }, + { + "epoch": 0.99, + "learning_rate": 6.098294884653962e-06, + "loss": 1.3406, + "theoretical_loss": 3.29971584505571, + "tokens_seen": 3260940288 + }, + { + "epoch": 0.99, + "learning_rate": 6.078234704112337e-06, + "loss": 1.3585, + "theoretical_loss": 3.2997058196341054, + "tokens_seen": 3261071360 + }, + { + "epoch": 0.99, + "learning_rate": 6.058174523570712e-06, + "loss": 1.2341, + "theoretical_loss": 3.299695794728265, + "tokens_seen": 3261202432 + }, + { + "epoch": 0.99, + "learning_rate": 6.038114343029088e-06, + "loss": 1.2956, + "theoretical_loss": 3.2996857703381415, + "tokens_seen": 3261333504 + }, + { + "epoch": 0.99, + "learning_rate": 6.018054162487462e-06, + "loss": 1.2624, + "theoretical_loss": 3.2996757464636874, + "tokens_seen": 3261464576 + }, + { + "epoch": 0.99, + "learning_rate": 5.997993981945838e-06, + "loss": 1.3744, + "theoretical_loss": 3.2996657231048556, + "tokens_seen": 3261595648 + }, + { + "epoch": 0.99, + "learning_rate": 5.977933801404213e-06, + "loss": 1.3355, + "theoretical_loss": 3.2996557002615994, + "tokens_seen": 3261726720 + }, + { + "epoch": 0.99, + "learning_rate": 5.957873620862588e-06, + "loss": 1.2662, + "theoretical_loss": 3.2996456779338703, + "tokens_seen": 3261857792 + }, + { + "epoch": 0.99, + "learning_rate": 5.937813440320963e-06, + "loss": 1.1902, + "theoretical_loss": 3.2996356561216222, + "tokens_seen": 3261988864 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.4689638316631317, + "objective/train/avg_token_score": 0.00780619727447629, + "objective/train/docs_used": 1834371, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4579510688781738, + "objective/train/original_loss": 3.858952283859253, + "objective/train/theoretical_loss": 3.2996306454087883, + "objective/train/tokens_used": 3262054400, + "objective/train/unlikelihood_loss": 0.10753054916858673, + "theoretical_loss": 3.2996306454087883, + "tokens_seen": 3262054400 + }, + { + "epoch": 0.99, + "learning_rate": 5.917753259779338e-06, + "loss": 1.3403, + "theoretical_loss": 3.299625634824807, + "tokens_seen": 3262119936 + }, + { + "epoch": 0.99, + "learning_rate": 5.897693079237713e-06, + "loss": 1.3135, + "theoretical_loss": 3.2996156140433786, + "tokens_seen": 3262251008 + }, + { + "epoch": 0.99, + "learning_rate": 5.877632898696088e-06, + "loss": 1.3059, + "theoretical_loss": 3.2996055937772883, + "tokens_seen": 3262382080 + }, + { + "epoch": 0.99, + "learning_rate": 5.857572718154463e-06, + "loss": 1.2021, + "theoretical_loss": 3.2995955740264904, + "tokens_seen": 3262513152 + }, + { + "epoch": 0.99, + "learning_rate": 5.837512537612838e-06, + "loss": 1.2623, + "theoretical_loss": 3.2995855547909367, + "tokens_seen": 3262644224 + }, + { + "epoch": 0.99, + "learning_rate": 5.817452357071214e-06, + "loss": 1.453, + "theoretical_loss": 3.2995755360705807, + "tokens_seen": 3262775296 + }, + { + "epoch": 0.99, + "learning_rate": 5.797392176529589e-06, + "loss": 1.2582, + "theoretical_loss": 3.2995655178653744, + "tokens_seen": 3262906368 + }, + { + "epoch": 0.99, + "learning_rate": 5.777331995987964e-06, + "loss": 1.4342, + "theoretical_loss": 3.299555500175271, + "tokens_seen": 3263037440 + }, + { + "epoch": 0.99, + "learning_rate": 5.75727181544634e-06, + "loss": 1.3418, + "theoretical_loss": 3.2995454830002235, + "tokens_seen": 3263168512 + }, + { + "epoch": 0.99, + "learning_rate": 5.737211634904714e-06, + "loss": 1.1794, + "theoretical_loss": 3.299535466340185, + "tokens_seen": 3263299584 + }, + { + "epoch": 0.99, + "learning_rate": 5.7171514543630895e-06, + "loss": 1.2163, + "theoretical_loss": 3.2995254501951075, + "tokens_seen": 3263430656 + }, + { + "epoch": 0.99, + "learning_rate": 5.6970912738214645e-06, + "loss": 1.3524, + "theoretical_loss": 3.2995154345649444, + "tokens_seen": 3263561728 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.4000488817691803, + "objective/train/avg_token_score": 0.011858591809868813, + "objective/train/docs_used": 1834946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.025915503501892, + "objective/train/original_loss": 2.9911141395568848, + "objective/train/theoretical_loss": 3.2995054194496483, + "objective/train/tokens_used": 3263692800, + "objective/train/unlikelihood_loss": 0.26792997121810913, + "theoretical_loss": 3.2995054194496483, + "tokens_seen": 3263692800 + }, + { + "epoch": 0.99, + "learning_rate": 5.677031093279839e-06, + "loss": 1.186, + "theoretical_loss": 3.2995054194496483, + "tokens_seen": 3263692800 + }, + { + "epoch": 0.99, + "learning_rate": 5.656970912738215e-06, + "loss": 1.2143, + "theoretical_loss": 3.2994954048491723, + "tokens_seen": 3263823872 + }, + { + "epoch": 0.99, + "learning_rate": 5.63691073219659e-06, + "loss": 1.2332, + "theoretical_loss": 3.2994853907634694, + "tokens_seen": 3263954944 + }, + { + "epoch": 0.99, + "learning_rate": 5.616850551654965e-06, + "loss": 1.3194, + "theoretical_loss": 3.2994753771924925, + "tokens_seen": 3264086016 + }, + { + "epoch": 0.99, + "learning_rate": 5.59679037111334e-06, + "loss": 1.2032, + "theoretical_loss": 3.2994653641361937, + "tokens_seen": 3264217088 + }, + { + "epoch": 0.99, + "learning_rate": 5.576730190571715e-06, + "loss": 1.2655, + "theoretical_loss": 3.2994553515945264, + "tokens_seen": 3264348160 + }, + { + "epoch": 0.99, + "learning_rate": 5.55667001003009e-06, + "loss": 1.2732, + "theoretical_loss": 3.2994453395674435, + "tokens_seen": 3264479232 + }, + { + "epoch": 0.99, + "learning_rate": 5.536609829488466e-06, + "loss": 1.335, + "theoretical_loss": 3.299435328054898, + "tokens_seen": 3264610304 + }, + { + "epoch": 0.99, + "learning_rate": 5.516549648946841e-06, + "loss": 1.295, + "theoretical_loss": 3.2994253170568424, + "tokens_seen": 3264741376 + }, + { + "epoch": 0.99, + "learning_rate": 5.496489468405216e-06, + "loss": 1.2827, + "theoretical_loss": 3.2994153065732306, + "tokens_seen": 3264872448 + }, + { + "epoch": 0.99, + "learning_rate": 5.476429287863591e-06, + "loss": 1.3332, + "theoretical_loss": 3.299405296604014, + "tokens_seen": 3265003520 + }, + { + "epoch": 0.99, + "learning_rate": 5.4563691073219655e-06, + "loss": 1.3678, + "theoretical_loss": 3.2993952871491468, + "tokens_seen": 3265134592 + }, + { + "epoch": 0.99, + "learning_rate": 5.436308926780341e-06, + "loss": 1.5148, + "theoretical_loss": 3.299385278208581, + "tokens_seen": 3265265664 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.39522239565849304, + "objective/train/avg_token_score": 0.06470374763011932, + "objective/train/docs_used": 1836396, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2212307453155518, + "objective/train/original_loss": 4.073289394378662, + "objective/train/theoretical_loss": 3.299380273931147, + "objective/train/tokens_used": 3265331200, + "objective/train/unlikelihood_loss": 0.1346449851989746, + "theoretical_loss": 3.299380273931147, + "tokens_seen": 3265331200 + }, + { + "epoch": 0.99, + "learning_rate": 5.416248746238716e-06, + "loss": 1.1309, + "theoretical_loss": 3.2993752697822702, + "tokens_seen": 3265396736 + }, + { + "epoch": 0.99, + "learning_rate": 5.396188565697091e-06, + "loss": 1.387, + "theoretical_loss": 3.299365261870167, + "tokens_seen": 3265527808 + }, + { + "epoch": 0.99, + "learning_rate": 5.376128385155467e-06, + "loss": 1.198, + "theoretical_loss": 3.299355254472225, + "tokens_seen": 3265658880 + }, + { + "epoch": 0.99, + "learning_rate": 5.356068204613842e-06, + "loss": 1.2941, + "theoretical_loss": 3.2993452475883958, + "tokens_seen": 3265789952 + }, + { + "epoch": 0.99, + "learning_rate": 5.336008024072217e-06, + "loss": 1.2242, + "theoretical_loss": 3.2993352412186336, + "tokens_seen": 3265921024 + }, + { + "epoch": 0.99, + "learning_rate": 5.315947843530592e-06, + "loss": 1.2198, + "theoretical_loss": 3.299325235362891, + "tokens_seen": 3266052096 + }, + { + "epoch": 0.99, + "learning_rate": 5.295887662988967e-06, + "loss": 1.4312, + "theoretical_loss": 3.299315230021121, + "tokens_seen": 3266183168 + }, + { + "epoch": 0.99, + "learning_rate": 5.275827482447342e-06, + "loss": 1.2249, + "theoretical_loss": 3.299305225193276, + "tokens_seen": 3266314240 + }, + { + "epoch": 0.99, + "learning_rate": 5.2557673019057175e-06, + "loss": 1.2466, + "theoretical_loss": 3.2992952208793094, + "tokens_seen": 3266445312 + }, + { + "epoch": 0.99, + "learning_rate": 5.2357071213640925e-06, + "loss": 1.2611, + "theoretical_loss": 3.2992852170791744, + "tokens_seen": 3266576384 + }, + { + "epoch": 0.99, + "learning_rate": 5.215646940822467e-06, + "loss": 1.3002, + "theoretical_loss": 3.299275213792824, + "tokens_seen": 3266707456 + }, + { + "epoch": 0.99, + "learning_rate": 5.195586760280843e-06, + "loss": 1.1935, + "theoretical_loss": 3.299265211020211, + "tokens_seen": 3266838528 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.3338831961154938, + "objective/train/avg_token_score": 0.009196695871651173, + "objective/train/docs_used": 1836962, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0502116680145264, + "objective/train/original_loss": 3.517768383026123, + "objective/train/theoretical_loss": 3.2992552087612883, + "objective/train/tokens_used": 3266969600, + "objective/train/unlikelihood_loss": 0.21316513419151306, + "theoretical_loss": 3.2992552087612883, + "tokens_seen": 3266969600 + }, + { + "epoch": 0.99, + "learning_rate": 5.175526579739217e-06, + "loss": 1.2289, + "theoretical_loss": 3.2992552087612883, + "tokens_seen": 3266969600 + }, + { + "epoch": 0.99, + "learning_rate": 5.155466399197592e-06, + "loss": 1.2784, + "theoretical_loss": 3.299245207016009, + "tokens_seen": 3267100672 + }, + { + "epoch": 0.99, + "learning_rate": 5.135406218655968e-06, + "loss": 1.4087, + "theoretical_loss": 3.2992352057843264, + "tokens_seen": 3267231744 + }, + { + "epoch": 0.99, + "learning_rate": 5.115346038114343e-06, + "loss": 1.2862, + "theoretical_loss": 3.299225205066193, + "tokens_seen": 3267362816 + }, + { + "epoch": 0.99, + "learning_rate": 5.095285857572719e-06, + "loss": 1.3745, + "theoretical_loss": 3.299215204861562, + "tokens_seen": 3267493888 + }, + { + "epoch": 0.99, + "learning_rate": 5.075225677031094e-06, + "loss": 1.2646, + "theoretical_loss": 3.299205205170387, + "tokens_seen": 3267624960 + }, + { + "epoch": 0.99, + "learning_rate": 5.055165496489469e-06, + "loss": 1.2344, + "theoretical_loss": 3.2991952059926204, + "tokens_seen": 3267756032 + }, + { + "epoch": 0.99, + "learning_rate": 5.035105315947844e-06, + "loss": 1.2789, + "theoretical_loss": 3.2991852073282155, + "tokens_seen": 3267887104 + }, + { + "epoch": 0.99, + "learning_rate": 5.0150451354062186e-06, + "loss": 1.3142, + "theoretical_loss": 3.299175209177125, + "tokens_seen": 3268018176 + }, + { + "epoch": 0.99, + "learning_rate": 4.9949849548645935e-06, + "loss": 1.2189, + "theoretical_loss": 3.2991652115393024, + "tokens_seen": 3268149248 + }, + { + "epoch": 0.99, + "learning_rate": 4.974924774322969e-06, + "loss": 1.3017, + "theoretical_loss": 3.299155214414701, + "tokens_seen": 3268280320 + }, + { + "epoch": 0.99, + "learning_rate": 4.954864593781344e-06, + "loss": 1.2428, + "theoretical_loss": 3.299145217803273, + "tokens_seen": 3268411392 + }, + { + "epoch": 0.99, + "learning_rate": 4.934804413239719e-06, + "loss": 1.2366, + "theoretical_loss": 3.2991352217049723, + "tokens_seen": 3268542464 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.35508307814598083, + "objective/train/avg_token_score": 0.01710192859172821, + "objective/train/docs_used": 1837990, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9796370267868042, + "objective/train/original_loss": 3.9521570205688477, + "objective/train/theoretical_loss": 3.2991302238482296, + "objective/train/tokens_used": 3268608000, + "objective/train/unlikelihood_loss": 0.15525589883327484, + "theoretical_loss": 3.2991302238482296, + "tokens_seen": 3268608000 + }, + { + "epoch": 0.99, + "learning_rate": 4.914744232698095e-06, + "loss": 1.4024, + "theoretical_loss": 3.2991252261197515, + "tokens_seen": 3268673536 + }, + { + "epoch": 0.99, + "learning_rate": 4.894684052156469e-06, + "loss": 1.3418, + "theoretical_loss": 3.2991152310475638, + "tokens_seen": 3268804608 + }, + { + "epoch": 0.99, + "learning_rate": 4.874623871614844e-06, + "loss": 1.1556, + "theoretical_loss": 3.299105236488362, + "tokens_seen": 3268935680 + }, + { + "epoch": 0.99, + "learning_rate": 4.85456369107322e-06, + "loss": 1.2354, + "theoretical_loss": 3.2990952424421, + "tokens_seen": 3269066752 + }, + { + "epoch": 0.99, + "learning_rate": 4.834503510531595e-06, + "loss": 1.345, + "theoretical_loss": 3.299085248908731, + "tokens_seen": 3269197824 + }, + { + "epoch": 0.99, + "learning_rate": 4.8144433299899706e-06, + "loss": 1.372, + "theoretical_loss": 3.2990752558882064, + "tokens_seen": 3269328896 + }, + { + "epoch": 0.99, + "learning_rate": 4.7943831494483455e-06, + "loss": 1.2779, + "theoretical_loss": 3.2990652633804816, + "tokens_seen": 3269459968 + }, + { + "epoch": 0.99, + "learning_rate": 4.77432296890672e-06, + "loss": 1.2311, + "theoretical_loss": 3.299055271385508, + "tokens_seen": 3269591040 + }, + { + "epoch": 0.99, + "learning_rate": 4.754262788365095e-06, + "loss": 1.2313, + "theoretical_loss": 3.2990452799032393, + "tokens_seen": 3269722112 + }, + { + "epoch": 0.99, + "learning_rate": 4.73420260782347e-06, + "loss": 1.1973, + "theoretical_loss": 3.299035288933629, + "tokens_seen": 3269853184 + }, + { + "epoch": 0.99, + "learning_rate": 4.714142427281845e-06, + "loss": 1.3271, + "theoretical_loss": 3.2990252984766295, + "tokens_seen": 3269984256 + }, + { + "epoch": 0.99, + "learning_rate": 4.694082246740221e-06, + "loss": 1.5146, + "theoretical_loss": 3.299015308532195, + "tokens_seen": 3270115328 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.4568670690059662, + "objective/train/avg_token_score": 0.007332315668463707, + "objective/train/docs_used": 1838493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2325879335403442, + "objective/train/original_loss": 3.2180941104888916, + "objective/train/theoretical_loss": 3.299005319100278, + "objective/train/tokens_used": 3270246400, + "objective/train/unlikelihood_loss": 0.16612936556339264, + "theoretical_loss": 3.299005319100278, + "tokens_seen": 3270246400 + }, + { + "epoch": 0.99, + "learning_rate": 4.674022066198596e-06, + "loss": 1.3072, + "theoretical_loss": 3.299005319100278, + "tokens_seen": 3270246400 + }, + { + "epoch": 0.99, + "learning_rate": 4.653961885656971e-06, + "loss": 1.4908, + "theoretical_loss": 3.2989953301808317, + "tokens_seen": 3270377472 + }, + { + "epoch": 0.99, + "learning_rate": 4.633901705115346e-06, + "loss": 1.1966, + "theoretical_loss": 3.298985341773809, + "tokens_seen": 3270508544 + }, + { + "epoch": 0.99, + "learning_rate": 4.613841524573721e-06, + "loss": 1.1743, + "theoretical_loss": 3.2989753538791637, + "tokens_seen": 3270639616 + }, + { + "epoch": 0.99, + "learning_rate": 4.593781344032096e-06, + "loss": 1.3282, + "theoretical_loss": 3.2989653664968484, + "tokens_seen": 3270770688 + }, + { + "epoch": 0.99, + "learning_rate": 4.573721163490472e-06, + "loss": 1.3752, + "theoretical_loss": 3.2989553796268165, + "tokens_seen": 3270901760 + }, + { + "epoch": 0.99, + "learning_rate": 4.5536609829488466e-06, + "loss": 1.1613, + "theoretical_loss": 3.2989453932690216, + "tokens_seen": 3271032832 + }, + { + "epoch": 0.99, + "learning_rate": 4.533600802407222e-06, + "loss": 1.2229, + "theoretical_loss": 3.2989354074234165, + "tokens_seen": 3271163904 + }, + { + "epoch": 0.99, + "learning_rate": 4.513540621865597e-06, + "loss": 1.2634, + "theoretical_loss": 3.2989254220899547, + "tokens_seen": 3271294976 + }, + { + "epoch": 0.99, + "learning_rate": 4.493480441323971e-06, + "loss": 1.3095, + "theoretical_loss": 3.2989154372685885, + "tokens_seen": 3271426048 + }, + { + "epoch": 0.99, + "learning_rate": 4.473420260782347e-06, + "loss": 1.2762, + "theoretical_loss": 3.2989054529592723, + "tokens_seen": 3271557120 + }, + { + "epoch": 0.99, + "learning_rate": 4.453360080240722e-06, + "loss": 1.1751, + "theoretical_loss": 3.298895469161959, + "tokens_seen": 3271688192 + }, + { + "epoch": 0.99, + "learning_rate": 4.433299899699097e-06, + "loss": 1.2623, + "theoretical_loss": 3.298885485876601, + "tokens_seen": 3271819264 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.43120723962783813, + "objective/train/avg_token_score": 0.012605240568518639, + "objective/train/docs_used": 1839586, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3208898305892944, + "objective/train/original_loss": 3.8784780502319336, + "objective/train/theoretical_loss": 3.298880494425891, + "objective/train/tokens_used": 3271884800, + "objective/train/unlikelihood_loss": 0.17215445637702942, + "theoretical_loss": 3.298880494425891, + "tokens_seen": 3271884800 + }, + { + "epoch": 0.99, + "learning_rate": 4.413239719157473e-06, + "loss": 1.1963, + "theoretical_loss": 3.2988755031031527, + "tokens_seen": 3271950336 + }, + { + "epoch": 0.99, + "learning_rate": 4.393179538615848e-06, + "loss": 1.3041, + "theoretical_loss": 3.298865520841567, + "tokens_seen": 3272081408 + }, + { + "epoch": 0.99, + "learning_rate": 4.373119358074223e-06, + "loss": 1.3272, + "theoretical_loss": 3.2988555390917966, + "tokens_seen": 3272212480 + }, + { + "epoch": 0.99, + "learning_rate": 4.353059177532598e-06, + "loss": 1.2347, + "theoretical_loss": 3.298845557853795, + "tokens_seen": 3272343552 + }, + { + "epoch": 0.99, + "learning_rate": 4.332998996990973e-06, + "loss": 1.2046, + "theoretical_loss": 3.2988355771275164, + "tokens_seen": 3272474624 + }, + { + "epoch": 0.99, + "learning_rate": 4.312938816449348e-06, + "loss": 1.3834, + "theoretical_loss": 3.2988255969129128, + "tokens_seen": 3272605696 + }, + { + "epoch": 0.99, + "learning_rate": 4.292878635907723e-06, + "loss": 1.4091, + "theoretical_loss": 3.2988156172099377, + "tokens_seen": 3272736768 + }, + { + "epoch": 0.99, + "learning_rate": 4.272818455366098e-06, + "loss": 1.2461, + "theoretical_loss": 3.2988056380185453, + "tokens_seen": 3272867840 + }, + { + "epoch": 0.99, + "learning_rate": 4.252758274824474e-06, + "loss": 1.2088, + "theoretical_loss": 3.298795659338688, + "tokens_seen": 3272998912 + }, + { + "epoch": 0.99, + "learning_rate": 4.232698094282849e-06, + "loss": 1.4023, + "theoretical_loss": 3.298785681170319, + "tokens_seen": 3273129984 + }, + { + "epoch": 0.99, + "learning_rate": 4.212637913741223e-06, + "loss": 1.2436, + "theoretical_loss": 3.2987757035133924, + "tokens_seen": 3273261056 + }, + { + "epoch": 0.99, + "learning_rate": 4.192577733199599e-06, + "loss": 1.331, + "theoretical_loss": 3.298765726367861, + "tokens_seen": 3273392128 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.28928396105766296, + "objective/train/avg_token_score": 0.010914746671915054, + "objective/train/docs_used": 1840316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9185187220573425, + "objective/train/original_loss": 3.770599126815796, + "objective/train/theoretical_loss": 3.2987557497336777, + "objective/train/tokens_used": 3273523200, + "objective/train/unlikelihood_loss": 0.22434014081954956, + "theoretical_loss": 3.2987557497336777, + "tokens_seen": 3273523200 + }, + { + "epoch": 0.99, + "learning_rate": 4.172517552657974e-06, + "loss": 1.3277, + "theoretical_loss": 3.2987557497336777, + "tokens_seen": 3273523200 + }, + { + "epoch": 0.99, + "learning_rate": 4.152457372116349e-06, + "loss": 1.3576, + "theoretical_loss": 3.298745773610797, + "tokens_seen": 3273654272 + }, + { + "epoch": 0.99, + "learning_rate": 4.132397191574725e-06, + "loss": 1.306, + "theoretical_loss": 3.298735797999171, + "tokens_seen": 3273785344 + }, + { + "epoch": 0.99, + "learning_rate": 4.1123370110331e-06, + "loss": 1.359, + "theoretical_loss": 3.2987258228987537, + "tokens_seen": 3273916416 + }, + { + "epoch": 0.99, + "learning_rate": 4.0922768304914746e-06, + "loss": 1.2345, + "theoretical_loss": 3.2987158483094983, + "tokens_seen": 3274047488 + }, + { + "epoch": 0.99, + "learning_rate": 4.0722166499498495e-06, + "loss": 1.2437, + "theoretical_loss": 3.298705874231358, + "tokens_seen": 3274178560 + }, + { + "epoch": 0.99, + "learning_rate": 4.0521564694082245e-06, + "loss": 1.2996, + "theoretical_loss": 3.298695900664286, + "tokens_seen": 3274309632 + }, + { + "epoch": 0.99, + "learning_rate": 4.032096288866599e-06, + "loss": 1.1454, + "theoretical_loss": 3.2986859276082363, + "tokens_seen": 3274440704 + }, + { + "epoch": 0.99, + "learning_rate": 4.012036108324975e-06, + "loss": 1.2866, + "theoretical_loss": 3.298675955063162, + "tokens_seen": 3274571776 + }, + { + "epoch": 0.99, + "learning_rate": 3.99197592778335e-06, + "loss": 1.4559, + "theoretical_loss": 3.298665983029016, + "tokens_seen": 3274702848 + }, + { + "epoch": 0.99, + "learning_rate": 3.971915747241726e-06, + "loss": 1.4209, + "theoretical_loss": 3.2986560115057526, + "tokens_seen": 3274833920 + }, + { + "epoch": 0.99, + "learning_rate": 3.951855566700101e-06, + "loss": 1.3973, + "theoretical_loss": 3.298646040493324, + "tokens_seen": 3274964992 + }, + { + "epoch": 0.99, + "learning_rate": 3.931795386158475e-06, + "loss": 1.0997, + "theoretical_loss": 3.2986360699916846, + "tokens_seen": 3275096064 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.3554496467113495, + "objective/train/avg_token_score": 0.03225092589855194, + "objective/train/docs_used": 1841576, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.07399582862854, + "objective/train/original_loss": 3.7793476581573486, + "objective/train/theoretical_loss": 3.298631084932396, + "objective/train/tokens_used": 3275161600, + "objective/train/unlikelihood_loss": 0.18805953860282898, + "theoretical_loss": 3.298631084932396, + "tokens_seen": 3275161600 + }, + { + "epoch": 0.99, + "learning_rate": 3.911735205616851e-06, + "loss": 1.2467, + "theoretical_loss": 3.2986261000007873, + "tokens_seen": 3275227136 + }, + { + "epoch": 0.99, + "learning_rate": 3.891675025075226e-06, + "loss": 1.1777, + "theoretical_loss": 3.2986161305205854, + "tokens_seen": 3275358208 + }, + { + "epoch": 0.99, + "learning_rate": 3.871614844533601e-06, + "loss": 1.324, + "theoretical_loss": 3.2986061615510325, + "tokens_seen": 3275489280 + }, + { + "epoch": 0.99, + "learning_rate": 3.8515546639919765e-06, + "loss": 1.2235, + "theoretical_loss": 3.2985961930920826, + "tokens_seen": 3275620352 + }, + { + "epoch": 0.99, + "learning_rate": 3.8314944834503506e-06, + "loss": 1.2987, + "theoretical_loss": 3.298586225143688, + "tokens_seen": 3275751424 + }, + { + "epoch": 0.99, + "learning_rate": 3.8114343029087263e-06, + "loss": 1.2589, + "theoretical_loss": 3.2985762577058027, + "tokens_seen": 3275882496 + }, + { + "epoch": 0.99, + "learning_rate": 3.7913741223671013e-06, + "loss": 1.0471, + "theoretical_loss": 3.29856629077838, + "tokens_seen": 3276013568 + }, + { + "epoch": 0.99, + "learning_rate": 3.7713139418254767e-06, + "loss": 1.2593, + "theoretical_loss": 3.298556324361374, + "tokens_seen": 3276144640 + }, + { + "epoch": 0.99, + "learning_rate": 3.7512537612838516e-06, + "loss": 1.293, + "theoretical_loss": 3.2985463584547365, + "tokens_seen": 3276275712 + }, + { + "epoch": 0.99, + "learning_rate": 3.731193580742227e-06, + "loss": 1.1165, + "theoretical_loss": 3.298536393058423, + "tokens_seen": 3276406784 + }, + { + "epoch": 0.99, + "learning_rate": 3.711133400200602e-06, + "loss": 1.3091, + "theoretical_loss": 3.2985264281723854, + "tokens_seen": 3276537856 + }, + { + "epoch": 0.99, + "learning_rate": 3.691073219658977e-06, + "loss": 1.3395, + "theoretical_loss": 3.298516463796578, + "tokens_seen": 3276668928 + }, + { + "debugging/length": 403.375, + "debugging/num_segments": 8, + "epoch": 0.99, + "objective/train/aligned_token_count": 0.42204299569129944, + "objective/train/avg_token_score": 0.013501611538231373, + "objective/train/docs_used": 1842402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.09568190574646, + "objective/train/original_loss": 3.3935279846191406, + "objective/train/theoretical_loss": 3.2985064999309537, + "objective/train/tokens_used": 3276800000, + "objective/train/unlikelihood_loss": 0.18385246396064758, + "theoretical_loss": 3.2985064999309537, + "tokens_seen": 3276800000 + }, + { + "epoch": 0.99, + "learning_rate": 3.6710130391173522e-06, + "loss": 1.1225, + "theoretical_loss": 3.2985064999309537, + "tokens_seen": 3276800000 + } + ], + "max_steps": 25177, + "num_train_epochs": 9223372036854775807, + "total_flos": 1.6722690048e+18, + "trial_name": null, + "trial_params": null +}