diff --git "a/checkpoint-11797/trainer_state.json" "b/checkpoint-11797/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-11797/trainer_state.json" @@ -0,0 +1,93049 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9371623768668573, + "global_step": 11797, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-07, + "loss": 2.9309, + "theoretical_loss": 3.4868973533572363, + "tokens_seen": 1650130944 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015873e-06, + "loss": 3.0573, + "theoretical_loss": 3.4868733789973354, + "tokens_seen": 1650262016 + }, + { + "epoch": 0.0, + "learning_rate": 2.3809523809523808e-06, + "loss": 2.9048, + "theoretical_loss": 3.4868494070746388, + "tokens_seen": 1650393088 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031746e-06, + "loss": 3.1852, + "theoretical_loss": 3.4868254375887053, + "tokens_seen": 1650524160 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 2.9406, + "theoretical_loss": 3.4868014705390937, + "tokens_seen": 1650655232 + }, + { + "epoch": 0.0, + "learning_rate": 4.7619047619047615e-06, + "loss": 2.9301, + "theoretical_loss": 3.4867775059253625, + "tokens_seen": 1650786304 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555556e-06, + "loss": 2.9694, + "theoretical_loss": 3.4867535437470716, + "tokens_seen": 1650917376 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-06, + "loss": 2.8537, + "theoretical_loss": 3.486729584003779, + "tokens_seen": 1651048448 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857143e-06, + "loss": 3.1275, + "theoretical_loss": 3.4867056266950454, + "tokens_seen": 1651179520 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 2.8591, + "theoretical_loss": 3.4866816718204294, + "tokens_seen": 1651310592 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-06, + "loss": 2.9591, + "theoretical_loss": 3.4866577193794903, + "tokens_seen": 1651441664 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809523e-06, + "loss": 2.9381, + "theoretical_loss": 3.486633769371788, + "tokens_seen": 1651572736 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 911303, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.985459327697754, + "objective/train/theoretical_loss": 3.486621795280263, + "objective/train/tokens_used": 22097376, + "theoretical_loss": 3.486621795280263, + "tokens_seen": 1651638272 + }, + { + "epoch": 0.0, + "learning_rate": 1.0317460317460318e-05, + "loss": 3.0636, + "theoretical_loss": 3.4866098217968826, + "tokens_seen": 1651703808 + }, + { + "epoch": 0.0, + "learning_rate": 1.1111111111111112e-05, + "loss": 2.9173, + "theoretical_loss": 3.486585876654333, + "tokens_seen": 1651834880 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 2.9235, + "theoretical_loss": 3.4865619339437, + "tokens_seen": 1651965952 + }, + { + "epoch": 0.0, + "learning_rate": 1.2698412698412699e-05, + "loss": 2.9036, + "theoretical_loss": 3.4865379936645438, + "tokens_seen": 1652097024 + }, + { + "epoch": 0.0, + "learning_rate": 1.3492063492063492e-05, + "loss": 2.8327, + "theoretical_loss": 3.486514055816424, + "tokens_seen": 1652228096 + }, + { + "epoch": 0.0, + "learning_rate": 1.4285714285714285e-05, + "loss": 2.8069, + "theoretical_loss": 3.4864901203989014, + "tokens_seen": 1652359168 + }, + { + "epoch": 0.0, + "learning_rate": 1.5079365079365079e-05, + "loss": 2.6994, + "theoretical_loss": 3.4864661874115357, + "tokens_seen": 1652490240 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 2.8277, + "theoretical_loss": 3.486442256853888, + "tokens_seen": 1652621312 + }, + { + "epoch": 0.0, + "learning_rate": 1.6666666666666667e-05, + "loss": 2.8779, + "theoretical_loss": 3.4864183287255193, + "tokens_seen": 1652752384 + }, + { + "epoch": 0.0, + "learning_rate": 1.746031746031746e-05, + "loss": 2.929, + "theoretical_loss": 3.4863944030259897, + "tokens_seen": 1652883456 + }, + { + "epoch": 0.0, + "learning_rate": 1.8253968253968254e-05, + "loss": 2.7098, + "theoretical_loss": 3.4863704797548607, + "tokens_seen": 1653014528 + }, + { + "epoch": 0.0, + "learning_rate": 1.9047619047619046e-05, + "loss": 2.7617, + "theoretical_loss": 3.4863465589116927, + "tokens_seen": 1653145600 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 912643, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1410162448883057, + "objective/train/theoretical_loss": 3.486322640496047, + "objective/train/tokens_used": 23735776, + "theoretical_loss": 3.486322640496047, + "tokens_seen": 1653276672 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 2.8668, + "theoretical_loss": 3.486322640496047, + "tokens_seen": 1653276672 + }, + { + "epoch": 0.0, + "learning_rate": 2.0634920634920636e-05, + "loss": 2.7427, + "theoretical_loss": 3.486298724507485, + "tokens_seen": 1653407744 + }, + { + "epoch": 0.0, + "learning_rate": 2.1428571428571428e-05, + "loss": 2.9108, + "theoretical_loss": 3.4862748109455675, + "tokens_seen": 1653538816 + }, + { + "epoch": 0.0, + "learning_rate": 2.2222222222222223e-05, + "loss": 2.8479, + "theoretical_loss": 3.486250899809857, + "tokens_seen": 1653669888 + }, + { + "epoch": 0.0, + "learning_rate": 2.3015873015873015e-05, + "loss": 2.9489, + "theoretical_loss": 3.4862269910999135, + "tokens_seen": 1653800960 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 2.7904, + "theoretical_loss": 3.4862030848153003, + "tokens_seen": 1653932032 + }, + { + "epoch": 0.0, + "learning_rate": 2.4603174603174602e-05, + "loss": 2.6043, + "theoretical_loss": 3.4861791809555784, + "tokens_seen": 1654063104 + }, + { + "epoch": 0.0, + "learning_rate": 2.5396825396825397e-05, + "loss": 2.9128, + "theoretical_loss": 3.48615527952031, + "tokens_seen": 1654194176 + }, + { + "epoch": 0.0, + "learning_rate": 2.6190476190476192e-05, + "loss": 2.7843, + "theoretical_loss": 3.486131380509057, + "tokens_seen": 1654325248 + }, + { + "epoch": 0.0, + "learning_rate": 2.6984126984126984e-05, + "loss": 2.8452, + "theoretical_loss": 3.4861074839213813, + "tokens_seen": 1654456320 + }, + { + "epoch": 0.0, + "learning_rate": 2.777777777777778e-05, + "loss": 2.6435, + "theoretical_loss": 3.4860835897568454, + "tokens_seen": 1654587392 + }, + { + "epoch": 0.0, + "learning_rate": 2.857142857142857e-05, + "loss": 2.7978, + "theoretical_loss": 3.4860596980150116, + "tokens_seen": 1654718464 + }, + { + "epoch": 0.0, + "learning_rate": 2.9365079365079366e-05, + "loss": 2.9673, + "theoretical_loss": 3.4860358086954424, + "tokens_seen": 1654849536 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 913892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.85264253616333, + "objective/train/theoretical_loss": 3.48602386494387, + "objective/train/tokens_used": 25374176, + "theoretical_loss": 3.48602386494387, + "tokens_seen": 1654915072 + }, + { + "epoch": 0.0, + "learning_rate": 3.0158730158730158e-05, + "loss": 2.9861, + "theoretical_loss": 3.4860119217977004, + "tokens_seen": 1654980608 + }, + { + "epoch": 0.0, + "learning_rate": 3.095238095238095e-05, + "loss": 2.9108, + "theoretical_loss": 3.485988037321348, + "tokens_seen": 1655111680 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 2.7628, + "theoretical_loss": 3.4859641552659486, + "tokens_seen": 1655242752 + }, + { + "epoch": 0.0, + "learning_rate": 3.253968253968254e-05, + "loss": 2.6715, + "theoretical_loss": 3.485940275631065, + "tokens_seen": 1655373824 + }, + { + "epoch": 0.0, + "learning_rate": 3.3333333333333335e-05, + "loss": 2.9045, + "theoretical_loss": 3.4859163984162596, + "tokens_seen": 1655504896 + }, + { + "epoch": 0.0, + "learning_rate": 3.412698412698413e-05, + "loss": 2.7567, + "theoretical_loss": 3.4858925236210965, + "tokens_seen": 1655635968 + }, + { + "epoch": 0.0, + "learning_rate": 3.492063492063492e-05, + "loss": 2.787, + "theoretical_loss": 3.4858686512451387, + "tokens_seen": 1655767040 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428572e-05, + "loss": 2.6928, + "theoretical_loss": 3.4858447812879487, + "tokens_seen": 1655898112 + }, + { + "epoch": 0.0, + "learning_rate": 3.650793650793651e-05, + "loss": 2.7383, + "theoretical_loss": 3.485820913749091, + "tokens_seen": 1656029184 + }, + { + "epoch": 0.0, + "learning_rate": 3.730158730158731e-05, + "loss": 2.6014, + "theoretical_loss": 3.4857970486281293, + "tokens_seen": 1656160256 + }, + { + "epoch": 0.0, + "learning_rate": 3.809523809523809e-05, + "loss": 2.8116, + "theoretical_loss": 3.485773185924627, + "tokens_seen": 1656291328 + }, + { + "epoch": 0.0, + "learning_rate": 3.888888888888889e-05, + "loss": 2.6689, + "theoretical_loss": 3.4857493256381473, + "tokens_seen": 1656422400 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 914561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.774996280670166, + "objective/train/theoretical_loss": 3.4857254677682548, + "objective/train/tokens_used": 27012576, + "theoretical_loss": 3.4857254677682548, + "tokens_seen": 1656553472 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 2.6872, + "theoretical_loss": 3.4857254677682548, + "tokens_seen": 1656553472 + }, + { + "epoch": 0.0, + "learning_rate": 4.047619047619048e-05, + "loss": 2.5722, + "theoretical_loss": 3.485701612314514, + "tokens_seen": 1656684544 + }, + { + "epoch": 0.0, + "learning_rate": 4.126984126984127e-05, + "loss": 2.6782, + "theoretical_loss": 3.4856777592764883, + "tokens_seen": 1656815616 + }, + { + "epoch": 0.0, + "learning_rate": 4.2063492063492065e-05, + "loss": 2.6618, + "theoretical_loss": 3.4856539086537426, + "tokens_seen": 1656946688 + }, + { + "epoch": 0.0, + "learning_rate": 4.2857142857142856e-05, + "loss": 2.6818, + "theoretical_loss": 3.485630060445841, + "tokens_seen": 1657077760 + }, + { + "epoch": 0.0, + "learning_rate": 4.3650793650793655e-05, + "loss": 2.4444, + "theoretical_loss": 3.485606214652347, + "tokens_seen": 1657208832 + }, + { + "epoch": 0.0, + "learning_rate": 4.4444444444444447e-05, + "loss": 2.5045, + "theoretical_loss": 3.4855823712728276, + "tokens_seen": 1657339904 + }, + { + "epoch": 0.0, + "learning_rate": 4.523809523809524e-05, + "loss": 2.5876, + "theoretical_loss": 3.4855585303068453, + "tokens_seen": 1657470976 + }, + { + "epoch": 0.0, + "learning_rate": 4.603174603174603e-05, + "loss": 2.5061, + "theoretical_loss": 3.4855346917539665, + "tokens_seen": 1657602048 + }, + { + "epoch": 0.0, + "learning_rate": 4.682539682539683e-05, + "loss": 2.6817, + "theoretical_loss": 3.4855108556137546, + "tokens_seen": 1657733120 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 2.6159, + "theoretical_loss": 3.4854870218857763, + "tokens_seen": 1657864192 + }, + { + "epoch": 0.0, + "learning_rate": 4.841269841269841e-05, + "loss": 2.5539, + "theoretical_loss": 3.485463190569596, + "tokens_seen": 1657995264 + }, + { + "epoch": 0.0, + "learning_rate": 4.9206349206349204e-05, + "loss": 2.6999, + "theoretical_loss": 3.485439361664779, + "tokens_seen": 1658126336 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 915678, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6932287216186523, + "objective/train/theoretical_loss": 3.4854274481164964, + "objective/train/tokens_used": 28650976, + "theoretical_loss": 3.4854274481164964, + "tokens_seen": 1658191872 + }, + { + "epoch": 0.01, + "learning_rate": 5e-05, + "loss": 2.5566, + "theoretical_loss": 3.4854155351708913, + "tokens_seen": 1658257408 + }, + { + "epoch": 0.01, + "learning_rate": 5.0793650793650794e-05, + "loss": 2.7198, + "theoretical_loss": 3.4853917110874972, + "tokens_seen": 1658388480 + }, + { + "epoch": 0.01, + "learning_rate": 5.158730158730159e-05, + "loss": 2.4993, + "theoretical_loss": 3.4853678894141638, + "tokens_seen": 1658519552 + }, + { + "epoch": 0.01, + "learning_rate": 5.2380952380952384e-05, + "loss": 2.5541, + "theoretical_loss": 3.4853440701504557, + "tokens_seen": 1658650624 + }, + { + "epoch": 0.01, + "learning_rate": 5.3174603174603176e-05, + "loss": 2.5189, + "theoretical_loss": 3.4853202532959395, + "tokens_seen": 1658781696 + }, + { + "epoch": 0.01, + "learning_rate": 5.396825396825397e-05, + "loss": 2.322, + "theoretical_loss": 3.4852964388501806, + "tokens_seen": 1658912768 + }, + { + "epoch": 0.01, + "learning_rate": 5.4761904761904766e-05, + "loss": 2.6172, + "theoretical_loss": 3.485272626812746, + "tokens_seen": 1659043840 + }, + { + "epoch": 0.01, + "learning_rate": 5.555555555555556e-05, + "loss": 2.5944, + "theoretical_loss": 3.4852488171832006, + "tokens_seen": 1659174912 + }, + { + "epoch": 0.01, + "learning_rate": 5.634920634920635e-05, + "loss": 2.5423, + "theoretical_loss": 3.4852250099611117, + "tokens_seen": 1659305984 + }, + { + "epoch": 0.01, + "learning_rate": 5.714285714285714e-05, + "loss": 2.5272, + "theoretical_loss": 3.485201205146046, + "tokens_seen": 1659437056 + }, + { + "epoch": 0.01, + "learning_rate": 5.793650793650795e-05, + "loss": 2.6078, + "theoretical_loss": 3.4851774027375693, + "tokens_seen": 1659568128 + }, + { + "epoch": 0.01, + "learning_rate": 5.873015873015873e-05, + "loss": 2.5119, + "theoretical_loss": 3.4851536027352483, + "tokens_seen": 1659699200 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 916417, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.456707239151001, + "objective/train/theoretical_loss": 3.48512980513865, + "objective/train/tokens_used": 30289376, + "theoretical_loss": 3.48512980513865, + "tokens_seen": 1659830272 + }, + { + "epoch": 0.01, + "learning_rate": 5.9523809523809524e-05, + "loss": 2.6637, + "theoretical_loss": 3.48512980513865, + "tokens_seen": 1659830272 + }, + { + "epoch": 0.01, + "learning_rate": 6.0317460317460316e-05, + "loss": 2.3894, + "theoretical_loss": 3.4851060099473417, + "tokens_seen": 1659961344 + }, + { + "epoch": 0.01, + "learning_rate": 6.111111111111112e-05, + "loss": 2.5928, + "theoretical_loss": 3.4850822171608895, + "tokens_seen": 1660092416 + }, + { + "epoch": 0.01, + "learning_rate": 6.19047619047619e-05, + "loss": 2.4428, + "theoretical_loss": 3.4850584267788607, + "tokens_seen": 1660223488 + }, + { + "epoch": 0.01, + "learning_rate": 6.26984126984127e-05, + "loss": 2.7376, + "theoretical_loss": 3.4850346388008235, + "tokens_seen": 1660354560 + }, + { + "epoch": 0.01, + "learning_rate": 6.349206349206349e-05, + "loss": 2.4582, + "theoretical_loss": 3.4850108532263437, + "tokens_seen": 1660485632 + }, + { + "epoch": 0.01, + "learning_rate": 6.428571428571429e-05, + "loss": 2.409, + "theoretical_loss": 3.48498707005499, + "tokens_seen": 1660616704 + }, + { + "epoch": 0.01, + "learning_rate": 6.507936507936509e-05, + "loss": 2.5177, + "theoretical_loss": 3.4849632892863296, + "tokens_seen": 1660747776 + }, + { + "epoch": 0.01, + "learning_rate": 6.587301587301587e-05, + "loss": 2.6281, + "theoretical_loss": 3.48493951091993, + "tokens_seen": 1660878848 + }, + { + "epoch": 0.01, + "learning_rate": 6.666666666666667e-05, + "loss": 2.5806, + "theoretical_loss": 3.4849157349553592, + "tokens_seen": 1661009920 + }, + { + "epoch": 0.01, + "learning_rate": 6.746031746031747e-05, + "loss": 2.5608, + "theoretical_loss": 3.4848919613921847, + "tokens_seen": 1661140992 + }, + { + "epoch": 0.01, + "learning_rate": 6.825396825396825e-05, + "loss": 2.5385, + "theoretical_loss": 3.484868190229975, + "tokens_seen": 1661272064 + }, + { + "epoch": 0.01, + "learning_rate": 6.904761904761905e-05, + "loss": 2.5632, + "theoretical_loss": 3.484844421468298, + "tokens_seen": 1661403136 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 917689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0480103492736816, + "objective/train/theoretical_loss": 3.484832537987524, + "objective/train/tokens_used": 31927776, + "theoretical_loss": 3.484832537987524, + "tokens_seen": 1661468672 + }, + { + "epoch": 0.01, + "learning_rate": 6.984126984126984e-05, + "loss": 2.3842, + "theoretical_loss": 3.4848206551067213, + "tokens_seen": 1661534208 + }, + { + "epoch": 0.01, + "learning_rate": 7.063492063492065e-05, + "loss": 2.4595, + "theoretical_loss": 3.484796891144814, + "tokens_seen": 1661665280 + }, + { + "epoch": 0.01, + "learning_rate": 7.142857142857143e-05, + "loss": 2.5882, + "theoretical_loss": 3.4847731295821447, + "tokens_seen": 1661796352 + }, + { + "epoch": 0.01, + "learning_rate": 7.222222222222222e-05, + "loss": 2.3931, + "theoretical_loss": 3.4847493704182817, + "tokens_seen": 1661927424 + }, + { + "epoch": 0.01, + "learning_rate": 7.301587301587302e-05, + "loss": 2.622, + "theoretical_loss": 3.4847256136527935, + "tokens_seen": 1662058496 + }, + { + "epoch": 0.01, + "learning_rate": 7.380952380952382e-05, + "loss": 2.5425, + "theoretical_loss": 3.4847018592852494, + "tokens_seen": 1662189568 + }, + { + "epoch": 0.01, + "learning_rate": 7.460317460317461e-05, + "loss": 2.601, + "theoretical_loss": 3.4846781073152173, + "tokens_seen": 1662320640 + }, + { + "epoch": 0.01, + "learning_rate": 7.53968253968254e-05, + "loss": 2.5712, + "theoretical_loss": 3.484654357742267, + "tokens_seen": 1662451712 + }, + { + "epoch": 0.01, + "learning_rate": 7.619047619047618e-05, + "loss": 2.2686, + "theoretical_loss": 3.4846306105659677, + "tokens_seen": 1662582784 + }, + { + "epoch": 0.01, + "learning_rate": 7.6984126984127e-05, + "loss": 2.3971, + "theoretical_loss": 3.4846068657858877, + "tokens_seen": 1662713856 + }, + { + "epoch": 0.01, + "learning_rate": 7.777777777777778e-05, + "loss": 2.5189, + "theoretical_loss": 3.4845831234015976, + "tokens_seen": 1662844928 + }, + { + "epoch": 0.01, + "learning_rate": 7.857142857142858e-05, + "loss": 2.3924, + "theoretical_loss": 3.484559383412666, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 918239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3232572078704834, + "objective/train/theoretical_loss": 3.4845356458186627, + "objective/train/tokens_used": 33566176, + "theoretical_loss": 3.4845356458186627, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.01, + "learning_rate": 7.936507936507937e-05, + "loss": 2.5743, + "theoretical_loss": 3.4845356458186627, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.01, + "learning_rate": 8.015873015873016e-05, + "loss": 2.4116, + "theoretical_loss": 3.4845119106191578, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.01, + "learning_rate": 8.095238095238096e-05, + "loss": 2.5243, + "theoretical_loss": 3.48448817781372, + "tokens_seen": 1663369216 + }, + { + "epoch": 0.01, + "learning_rate": 8.174603174603175e-05, + "loss": 2.4798, + "theoretical_loss": 3.484464447401921, + "tokens_seen": 1663500288 + }, + { + "epoch": 0.01, + "learning_rate": 8.253968253968255e-05, + "loss": 2.3456, + "theoretical_loss": 3.4844407193833282, + "tokens_seen": 1663631360 + }, + { + "epoch": 0.01, + "learning_rate": 8.333333333333334e-05, + "loss": 2.4678, + "theoretical_loss": 3.484416993757514, + "tokens_seen": 1663762432 + }, + { + "epoch": 0.01, + "learning_rate": 8.412698412698413e-05, + "loss": 2.4106, + "theoretical_loss": 3.4843932705240483, + "tokens_seen": 1663893504 + }, + { + "epoch": 0.01, + "learning_rate": 8.492063492063493e-05, + "loss": 2.3669, + "theoretical_loss": 3.4843695496825005, + "tokens_seen": 1664024576 + }, + { + "epoch": 0.01, + "learning_rate": 8.571428571428571e-05, + "loss": 2.5092, + "theoretical_loss": 3.4843458312324413, + "tokens_seen": 1664155648 + }, + { + "epoch": 0.01, + "learning_rate": 8.650793650793651e-05, + "loss": 2.473, + "theoretical_loss": 3.484322115173442, + "tokens_seen": 1664286720 + }, + { + "epoch": 0.01, + "learning_rate": 8.730158730158731e-05, + "loss": 2.5359, + "theoretical_loss": 3.4842984015050726, + "tokens_seen": 1664417792 + }, + { + "epoch": 0.01, + "learning_rate": 8.80952380952381e-05, + "loss": 2.6096, + "theoretical_loss": 3.484274690226904, + "tokens_seen": 1664548864 + }, + { + "epoch": 0.01, + "learning_rate": 8.888888888888889e-05, + "loss": 2.4428, + "theoretical_loss": 3.484250981338507, + "tokens_seen": 1664679936 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 919306, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.603062629699707, + "objective/train/theoretical_loss": 3.484239127790339, + "objective/train/tokens_used": 35204576, + "theoretical_loss": 3.484239127790339, + "tokens_seen": 1664745472 + }, + { + "epoch": 0.01, + "learning_rate": 8.968253968253969e-05, + "loss": 2.5448, + "theoretical_loss": 3.484227274839453, + "tokens_seen": 1664811008 + }, + { + "epoch": 0.01, + "learning_rate": 9.047619047619048e-05, + "loss": 2.5698, + "theoretical_loss": 3.484203570729313, + "tokens_seen": 1664942080 + }, + { + "epoch": 0.01, + "learning_rate": 9.126984126984128e-05, + "loss": 2.4996, + "theoretical_loss": 3.484179869007658, + "tokens_seen": 1665073152 + }, + { + "epoch": 0.01, + "learning_rate": 9.206349206349206e-05, + "loss": 2.5152, + "theoretical_loss": 3.4841561696740597, + "tokens_seen": 1665204224 + }, + { + "epoch": 0.01, + "learning_rate": 9.285714285714286e-05, + "loss": 2.4996, + "theoretical_loss": 3.4841324727280893, + "tokens_seen": 1665335296 + }, + { + "epoch": 0.01, + "learning_rate": 9.365079365079366e-05, + "loss": 2.47, + "theoretical_loss": 3.484108778169318, + "tokens_seen": 1665466368 + }, + { + "epoch": 0.01, + "learning_rate": 9.444444444444444e-05, + "loss": 2.3922, + "theoretical_loss": 3.484085085997318, + "tokens_seen": 1665597440 + }, + { + "epoch": 0.01, + "learning_rate": 9.523809523809524e-05, + "loss": 2.4767, + "theoretical_loss": 3.484061396211661, + "tokens_seen": 1665728512 + }, + { + "epoch": 0.01, + "learning_rate": 9.603174603174604e-05, + "loss": 2.5377, + "theoretical_loss": 3.4840377088119188, + "tokens_seen": 1665859584 + }, + { + "epoch": 0.01, + "learning_rate": 9.682539682539682e-05, + "loss": 2.61, + "theoretical_loss": 3.4840140237976636, + "tokens_seen": 1665990656 + }, + { + "epoch": 0.01, + "learning_rate": 9.761904761904762e-05, + "loss": 2.4506, + "theoretical_loss": 3.483990341168467, + "tokens_seen": 1666121728 + }, + { + "epoch": 0.01, + "learning_rate": 9.841269841269841e-05, + "loss": 2.4311, + "theoretical_loss": 3.483966660923902, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 919806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3898022174835205, + "objective/train/theoretical_loss": 3.4839429830635407, + "objective/train/tokens_used": 36842976, + "theoretical_loss": 3.4839429830635407, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.01, + "learning_rate": 9.920634920634922e-05, + "loss": 2.543, + "theoretical_loss": 3.4839429830635407, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0001, + "loss": 2.4395, + "theoretical_loss": 3.4839193075869543, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.01, + "learning_rate": 9.999197560584176e-05, + "loss": 2.4916, + "theoretical_loss": 3.4838956344937175, + "tokens_seen": 1666646016 + }, + { + "epoch": 0.01, + "learning_rate": 9.998395121168352e-05, + "loss": 2.3888, + "theoretical_loss": 3.4838719637834012, + "tokens_seen": 1666777088 + }, + { + "epoch": 0.01, + "learning_rate": 9.997592681752528e-05, + "loss": 2.5063, + "theoretical_loss": 3.483848295455579, + "tokens_seen": 1666908160 + }, + { + "epoch": 0.01, + "learning_rate": 9.996790242336704e-05, + "loss": 2.4951, + "theoretical_loss": 3.483824629509824, + "tokens_seen": 1667039232 + }, + { + "epoch": 0.01, + "learning_rate": 9.99598780292088e-05, + "loss": 2.4226, + "theoretical_loss": 3.4838009659457088, + "tokens_seen": 1667170304 + }, + { + "epoch": 0.01, + "learning_rate": 9.995185363505056e-05, + "loss": 2.4431, + "theoretical_loss": 3.4837773047628064, + "tokens_seen": 1667301376 + }, + { + "epoch": 0.01, + "learning_rate": 9.994382924089232e-05, + "loss": 2.4434, + "theoretical_loss": 3.4837536459606904, + "tokens_seen": 1667432448 + }, + { + "epoch": 0.01, + "learning_rate": 9.993580484673407e-05, + "loss": 2.5021, + "theoretical_loss": 3.4837299895389333, + "tokens_seen": 1667563520 + }, + { + "epoch": 0.01, + "learning_rate": 9.992778045257584e-05, + "loss": 2.5622, + "theoretical_loss": 3.48370633549711, + "tokens_seen": 1667694592 + }, + { + "epoch": 0.01, + "learning_rate": 9.991975605841759e-05, + "loss": 2.524, + "theoretical_loss": 3.4836826838347923, + "tokens_seen": 1667825664 + }, + { + "epoch": 0.01, + "learning_rate": 9.991173166425936e-05, + "loss": 2.5055, + "theoretical_loss": 3.483659034551555, + "tokens_seen": 1667956736 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 921005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6571173667907715, + "objective/train/theoretical_loss": 3.4836472108019585, + "objective/train/tokens_used": 38481376, + "theoretical_loss": 3.4836472108019585, + "tokens_seen": 1668022272 + }, + { + "epoch": 0.01, + "learning_rate": 9.990370727010111e-05, + "loss": 2.5956, + "theoretical_loss": 3.483635387646972, + "tokens_seen": 1668087808 + }, + { + "epoch": 0.01, + "learning_rate": 9.989568287594288e-05, + "loss": 2.5279, + "theoretical_loss": 3.4836117431206164, + "tokens_seen": 1668218880 + }, + { + "epoch": 0.01, + "learning_rate": 9.988765848178463e-05, + "loss": 2.45, + "theoretical_loss": 3.483588100972063, + "tokens_seen": 1668349952 + }, + { + "epoch": 0.01, + "learning_rate": 9.987963408762638e-05, + "loss": 2.5056, + "theoretical_loss": 3.483564461200885, + "tokens_seen": 1668481024 + }, + { + "epoch": 0.01, + "learning_rate": 9.987160969346815e-05, + "loss": 2.56, + "theoretical_loss": 3.4835408238066567, + "tokens_seen": 1668612096 + }, + { + "epoch": 0.01, + "learning_rate": 9.98635852993099e-05, + "loss": 2.4673, + "theoretical_loss": 3.4835171887889533, + "tokens_seen": 1668743168 + }, + { + "epoch": 0.01, + "learning_rate": 9.985556090515167e-05, + "loss": 2.3887, + "theoretical_loss": 3.483493556147349, + "tokens_seen": 1668874240 + }, + { + "epoch": 0.01, + "learning_rate": 9.984753651099342e-05, + "loss": 2.478, + "theoretical_loss": 3.4834699258814172, + "tokens_seen": 1669005312 + }, + { + "epoch": 0.01, + "learning_rate": 9.983951211683519e-05, + "loss": 2.6136, + "theoretical_loss": 3.483446297990734, + "tokens_seen": 1669136384 + }, + { + "epoch": 0.01, + "learning_rate": 9.983148772267694e-05, + "loss": 2.5876, + "theoretical_loss": 3.4834226724748736, + "tokens_seen": 1669267456 + }, + { + "epoch": 0.01, + "learning_rate": 9.982346332851871e-05, + "loss": 2.5149, + "theoretical_loss": 3.4833990493334106, + "tokens_seen": 1669398528 + }, + { + "epoch": 0.01, + "learning_rate": 9.981543893436046e-05, + "loss": 2.5755, + "theoretical_loss": 3.48337542856592, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 921680, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.329754590988159, + "objective/train/theoretical_loss": 3.4833518101719774, + "objective/train/tokens_used": 40119776, + "theoretical_loss": 3.4833518101719774, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.01, + "learning_rate": 9.980741454020222e-05, + "loss": 2.4671, + "theoretical_loss": 3.4833518101719774, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.01, + "learning_rate": 9.979939014604398e-05, + "loss": 2.3942, + "theoretical_loss": 3.4833281941511576, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 9.979136575188574e-05, + "loss": 2.4857, + "theoretical_loss": 3.4833045805030363, + "tokens_seen": 1669922816 + }, + { + "epoch": 0.01, + "learning_rate": 9.97833413577275e-05, + "loss": 2.4502, + "theoretical_loss": 3.483280969227188, + "tokens_seen": 1670053888 + }, + { + "epoch": 0.01, + "learning_rate": 9.977531696356926e-05, + "loss": 2.5831, + "theoretical_loss": 3.4832573603231887, + "tokens_seen": 1670184960 + }, + { + "epoch": 0.01, + "learning_rate": 9.976729256941102e-05, + "loss": 2.561, + "theoretical_loss": 3.4832337537906146, + "tokens_seen": 1670316032 + }, + { + "epoch": 0.01, + "learning_rate": 9.975926817525277e-05, + "loss": 2.5869, + "theoretical_loss": 3.4832101496290404, + "tokens_seen": 1670447104 + }, + { + "epoch": 0.01, + "learning_rate": 9.975124378109453e-05, + "loss": 2.4206, + "theoretical_loss": 3.4831865478380433, + "tokens_seen": 1670578176 + }, + { + "epoch": 0.01, + "learning_rate": 9.97432193869363e-05, + "loss": 2.3991, + "theoretical_loss": 3.4831629484171978, + "tokens_seen": 1670709248 + }, + { + "epoch": 0.01, + "learning_rate": 9.973519499277805e-05, + "loss": 2.3821, + "theoretical_loss": 3.4831393513660807, + "tokens_seen": 1670840320 + }, + { + "epoch": 0.01, + "learning_rate": 9.972717059861981e-05, + "loss": 2.5258, + "theoretical_loss": 3.483115756684268, + "tokens_seen": 1670971392 + }, + { + "epoch": 0.01, + "learning_rate": 9.971914620446157e-05, + "loss": 2.4604, + "theoretical_loss": 3.4830921643713366, + "tokens_seen": 1671102464 + }, + { + "epoch": 0.01, + "learning_rate": 9.971112181030333e-05, + "loss": 2.5027, + "theoretical_loss": 3.483068574426862, + "tokens_seen": 1671233536 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 922598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.674171209335327, + "objective/train/theoretical_loss": 3.4830567803426637, + "objective/train/tokens_used": 41758176, + "theoretical_loss": 3.4830567803426637, + "tokens_seen": 1671299072 + }, + { + "epoch": 0.01, + "learning_rate": 9.970309741614509e-05, + "loss": 2.5194, + "theoretical_loss": 3.483044986850421, + "tokens_seen": 1671364608 + }, + { + "epoch": 0.01, + "learning_rate": 9.969507302198684e-05, + "loss": 2.4984, + "theoretical_loss": 3.48302140164159, + "tokens_seen": 1671495680 + }, + { + "epoch": 0.01, + "learning_rate": 9.968704862782861e-05, + "loss": 2.6113, + "theoretical_loss": 3.482997818799947, + "tokens_seen": 1671626752 + }, + { + "epoch": 0.01, + "learning_rate": 9.967902423367036e-05, + "loss": 2.6013, + "theoretical_loss": 3.4829742383250673, + "tokens_seen": 1671757824 + }, + { + "epoch": 0.01, + "learning_rate": 9.967099983951213e-05, + "loss": 2.4007, + "theoretical_loss": 3.4829506602165283, + "tokens_seen": 1671888896 + }, + { + "epoch": 0.01, + "learning_rate": 9.966297544535388e-05, + "loss": 2.4473, + "theoretical_loss": 3.482927084473907, + "tokens_seen": 1672019968 + }, + { + "epoch": 0.01, + "learning_rate": 9.965495105119565e-05, + "loss": 2.3941, + "theoretical_loss": 3.482903511096781, + "tokens_seen": 1672151040 + }, + { + "epoch": 0.01, + "learning_rate": 9.96469266570374e-05, + "loss": 2.5564, + "theoretical_loss": 3.482879940084727, + "tokens_seen": 1672282112 + }, + { + "epoch": 0.01, + "learning_rate": 9.963890226287915e-05, + "loss": 2.4979, + "theoretical_loss": 3.4828563714373226, + "tokens_seen": 1672413184 + }, + { + "epoch": 0.01, + "learning_rate": 9.963087786872092e-05, + "loss": 2.4506, + "theoretical_loss": 3.4828328051541453, + "tokens_seen": 1672544256 + }, + { + "epoch": 0.01, + "learning_rate": 9.962285347456267e-05, + "loss": 2.5131, + "theoretical_loss": 3.482809241234773, + "tokens_seen": 1672675328 + }, + { + "epoch": 0.01, + "learning_rate": 9.961482908040444e-05, + "loss": 2.4079, + "theoretical_loss": 3.482785679678783, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 923236, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.478933095932007, + "objective/train/theoretical_loss": 3.482762120485753, + "objective/train/tokens_used": 43396576, + "theoretical_loss": 3.482762120485753, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.01, + "learning_rate": 9.960680468624619e-05, + "loss": 2.5005, + "theoretical_loss": 3.482762120485753, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.01, + "learning_rate": 9.959878029208796e-05, + "loss": 2.669, + "theoretical_loss": 3.482738563655261, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 9.959075589792971e-05, + "loss": 2.5706, + "theoretical_loss": 3.4827150091868853, + "tokens_seen": 1673199616 + }, + { + "epoch": 0.01, + "learning_rate": 9.958273150377147e-05, + "loss": 2.5295, + "theoretical_loss": 3.482691457080204, + "tokens_seen": 1673330688 + }, + { + "epoch": 0.01, + "learning_rate": 9.957470710961323e-05, + "loss": 2.4358, + "theoretical_loss": 3.482667907334795, + "tokens_seen": 1673461760 + }, + { + "epoch": 0.01, + "learning_rate": 9.956668271545499e-05, + "loss": 2.4467, + "theoretical_loss": 3.482644359950237, + "tokens_seen": 1673592832 + }, + { + "epoch": 0.01, + "learning_rate": 9.955865832129675e-05, + "loss": 2.3988, + "theoretical_loss": 3.4826208149261078, + "tokens_seen": 1673723904 + }, + { + "epoch": 0.01, + "learning_rate": 9.95506339271385e-05, + "loss": 2.521, + "theoretical_loss": 3.482597272261987, + "tokens_seen": 1673854976 + }, + { + "epoch": 0.01, + "learning_rate": 9.954260953298027e-05, + "loss": 2.409, + "theoretical_loss": 3.4825737319574523, + "tokens_seen": 1673986048 + }, + { + "epoch": 0.01, + "learning_rate": 9.953458513882203e-05, + "loss": 2.5548, + "theoretical_loss": 3.4825501940120835, + "tokens_seen": 1674117120 + }, + { + "epoch": 0.01, + "learning_rate": 9.952656074466378e-05, + "loss": 2.4541, + "theoretical_loss": 3.4825266584254586, + "tokens_seen": 1674248192 + }, + { + "epoch": 0.01, + "learning_rate": 9.951853635050554e-05, + "loss": 2.5549, + "theoretical_loss": 3.482503125197157, + "tokens_seen": 1674379264 + }, + { + "epoch": 0.01, + "learning_rate": 9.95105119563473e-05, + "loss": 2.4687, + "theoretical_loss": 3.4824795943267577, + "tokens_seen": 1674510336 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 924425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.032245397567749, + "objective/train/theoretical_loss": 3.48246782977564, + "objective/train/tokens_used": 45034976, + "theoretical_loss": 3.48246782977564, + "tokens_seen": 1674575872 + }, + { + "epoch": 0.01, + "learning_rate": 9.950248756218906e-05, + "loss": 2.4781, + "theoretical_loss": 3.4824560658138397, + "tokens_seen": 1674641408 + }, + { + "epoch": 0.02, + "learning_rate": 9.949446316803082e-05, + "loss": 2.6007, + "theoretical_loss": 3.482432539657983, + "tokens_seen": 1674772480 + }, + { + "epoch": 0.02, + "learning_rate": 9.948643877387258e-05, + "loss": 2.4198, + "theoretical_loss": 3.4824090158587664, + "tokens_seen": 1674903552 + }, + { + "epoch": 0.02, + "learning_rate": 9.947841437971434e-05, + "loss": 2.49, + "theoretical_loss": 3.4823854944157695, + "tokens_seen": 1675034624 + }, + { + "epoch": 0.02, + "learning_rate": 9.94703899855561e-05, + "loss": 2.4733, + "theoretical_loss": 3.482361975328572, + "tokens_seen": 1675165696 + }, + { + "epoch": 0.02, + "learning_rate": 9.946236559139786e-05, + "loss": 2.6164, + "theoretical_loss": 3.482338458596754, + "tokens_seen": 1675296768 + }, + { + "epoch": 0.02, + "learning_rate": 9.945434119723961e-05, + "loss": 2.4729, + "theoretical_loss": 3.482314944219895, + "tokens_seen": 1675427840 + }, + { + "epoch": 0.02, + "learning_rate": 9.944631680308138e-05, + "loss": 2.5106, + "theoretical_loss": 3.482291432197575, + "tokens_seen": 1675558912 + }, + { + "epoch": 0.02, + "learning_rate": 9.943829240892313e-05, + "loss": 2.4255, + "theoretical_loss": 3.482267922529374, + "tokens_seen": 1675689984 + }, + { + "epoch": 0.02, + "learning_rate": 9.94302680147649e-05, + "loss": 2.4172, + "theoretical_loss": 3.482244415214873, + "tokens_seen": 1675821056 + }, + { + "epoch": 0.02, + "learning_rate": 9.942224362060665e-05, + "loss": 2.4485, + "theoretical_loss": 3.482220910253651, + "tokens_seen": 1675952128 + }, + { + "epoch": 0.02, + "learning_rate": 9.941421922644842e-05, + "loss": 2.5783, + "theoretical_loss": 3.482197407645289, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 924948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1431517601013184, + "objective/train/theoretical_loss": 3.4821739073893676, + "objective/train/tokens_used": 46673376, + "theoretical_loss": 3.4821739073893676, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.02, + "learning_rate": 9.940619483229017e-05, + "loss": 2.4842, + "theoretical_loss": 3.4821739073893676, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.02, + "learning_rate": 9.939817043813192e-05, + "loss": 2.5048, + "theoretical_loss": 3.482150409485467, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.02, + "learning_rate": 9.939014604397369e-05, + "loss": 2.4242, + "theoretical_loss": 3.482126913933169, + "tokens_seen": 1676476416 + }, + { + "epoch": 0.02, + "learning_rate": 9.938212164981544e-05, + "loss": 2.4716, + "theoretical_loss": 3.482103420732053, + "tokens_seen": 1676607488 + }, + { + "epoch": 0.02, + "learning_rate": 9.937409725565721e-05, + "loss": 2.4006, + "theoretical_loss": 3.482079929881701, + "tokens_seen": 1676738560 + }, + { + "epoch": 0.02, + "learning_rate": 9.936607286149896e-05, + "loss": 2.5438, + "theoretical_loss": 3.482056441381694, + "tokens_seen": 1676869632 + }, + { + "epoch": 0.02, + "learning_rate": 9.935804846734073e-05, + "loss": 2.4948, + "theoretical_loss": 3.4820329552316123, + "tokens_seen": 1677000704 + }, + { + "epoch": 0.02, + "learning_rate": 9.935002407318248e-05, + "loss": 2.4135, + "theoretical_loss": 3.482009471431038, + "tokens_seen": 1677131776 + }, + { + "epoch": 0.02, + "learning_rate": 9.934199967902424e-05, + "loss": 2.4302, + "theoretical_loss": 3.4819859899795516, + "tokens_seen": 1677262848 + }, + { + "epoch": 0.02, + "learning_rate": 9.9333975284866e-05, + "loss": 2.4168, + "theoretical_loss": 3.481962510876736, + "tokens_seen": 1677393920 + }, + { + "epoch": 0.02, + "learning_rate": 9.932595089070776e-05, + "loss": 2.648, + "theoretical_loss": 3.481939034122171, + "tokens_seen": 1677524992 + }, + { + "epoch": 0.02, + "learning_rate": 9.931792649654952e-05, + "loss": 2.4332, + "theoretical_loss": 3.4819155597154396, + "tokens_seen": 1677656064 + }, + { + "epoch": 0.02, + "learning_rate": 9.930990210239128e-05, + "loss": 2.4625, + "theoretical_loss": 3.4818920876561235, + "tokens_seen": 1677787136 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 926165, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.494450569152832, + "objective/train/theoretical_loss": 3.4818803525066153, + "objective/train/tokens_used": 48311776, + "theoretical_loss": 3.4818803525066153, + "tokens_seen": 1677852672 + }, + { + "epoch": 0.02, + "learning_rate": 9.930187770823304e-05, + "loss": 2.4562, + "theoretical_loss": 3.481868617943804, + "tokens_seen": 1677918208 + }, + { + "epoch": 0.02, + "learning_rate": 9.92938533140748e-05, + "loss": 2.4934, + "theoretical_loss": 3.481845150578063, + "tokens_seen": 1678049280 + }, + { + "epoch": 0.02, + "learning_rate": 9.928582891991655e-05, + "loss": 2.4625, + "theoretical_loss": 3.481821685558484, + "tokens_seen": 1678180352 + }, + { + "epoch": 0.02, + "learning_rate": 9.927780452575832e-05, + "loss": 2.3664, + "theoretical_loss": 3.4817982228846476, + "tokens_seen": 1678311424 + }, + { + "epoch": 0.02, + "learning_rate": 9.926978013160007e-05, + "loss": 2.4554, + "theoretical_loss": 3.481774762556137, + "tokens_seen": 1678442496 + }, + { + "epoch": 0.02, + "learning_rate": 9.926175573744183e-05, + "loss": 2.6398, + "theoretical_loss": 3.4817513045725343, + "tokens_seen": 1678573568 + }, + { + "epoch": 0.02, + "learning_rate": 9.925373134328359e-05, + "loss": 2.3632, + "theoretical_loss": 3.4817278489334225, + "tokens_seen": 1678704640 + }, + { + "epoch": 0.02, + "learning_rate": 9.924570694912535e-05, + "loss": 2.6002, + "theoretical_loss": 3.481704395638383, + "tokens_seen": 1678835712 + }, + { + "epoch": 0.02, + "learning_rate": 9.923768255496711e-05, + "loss": 2.5935, + "theoretical_loss": 3.4816809446870005, + "tokens_seen": 1678966784 + }, + { + "epoch": 0.02, + "learning_rate": 9.922965816080886e-05, + "loss": 2.6889, + "theoretical_loss": 3.4816574960788564, + "tokens_seen": 1679097856 + }, + { + "epoch": 0.02, + "learning_rate": 9.922163376665063e-05, + "loss": 2.2718, + "theoretical_loss": 3.4816340498135343, + "tokens_seen": 1679228928 + }, + { + "epoch": 0.02, + "learning_rate": 9.921360937249238e-05, + "loss": 2.4179, + "theoretical_loss": 3.4816106058906175, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 926923, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7133426666259766, + "objective/train/theoretical_loss": 3.481587164309688, + "objective/train/tokens_used": 49950176, + "theoretical_loss": 3.481587164309688, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.02, + "learning_rate": 9.920558497833415e-05, + "loss": 2.4835, + "theoretical_loss": 3.481587164309688, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.02, + "learning_rate": 9.91975605841759e-05, + "loss": 2.5025, + "theoretical_loss": 3.4815637250703304, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.02, + "learning_rate": 9.918953619001767e-05, + "loss": 2.4519, + "theoretical_loss": 3.4815402881721274, + "tokens_seen": 1679753216 + }, + { + "epoch": 0.02, + "learning_rate": 9.918151179585942e-05, + "loss": 2.5021, + "theoretical_loss": 3.4815168536146626, + "tokens_seen": 1679884288 + }, + { + "epoch": 0.02, + "learning_rate": 9.917348740170119e-05, + "loss": 2.4785, + "theoretical_loss": 3.48149342139752, + "tokens_seen": 1680015360 + }, + { + "epoch": 0.02, + "learning_rate": 9.916546300754294e-05, + "loss": 2.5138, + "theoretical_loss": 3.481469991520283, + "tokens_seen": 1680146432 + }, + { + "epoch": 0.02, + "learning_rate": 9.91574386133847e-05, + "loss": 2.4019, + "theoretical_loss": 3.4814465639825354, + "tokens_seen": 1680277504 + }, + { + "epoch": 0.02, + "learning_rate": 9.914941421922646e-05, + "loss": 2.5178, + "theoretical_loss": 3.481423138783861, + "tokens_seen": 1680408576 + }, + { + "epoch": 0.02, + "learning_rate": 9.914138982506821e-05, + "loss": 2.4615, + "theoretical_loss": 3.4813997159238443, + "tokens_seen": 1680539648 + }, + { + "epoch": 0.02, + "learning_rate": 9.913336543090998e-05, + "loss": 2.533, + "theoretical_loss": 3.481376295402069, + "tokens_seen": 1680670720 + }, + { + "epoch": 0.02, + "learning_rate": 9.912534103675173e-05, + "loss": 2.5408, + "theoretical_loss": 3.4813528772181193, + "tokens_seen": 1680801792 + }, + { + "epoch": 0.02, + "learning_rate": 9.91173166425935e-05, + "loss": 2.4864, + "theoretical_loss": 3.4813294613715797, + "tokens_seen": 1680932864 + }, + { + "epoch": 0.02, + "learning_rate": 9.910929224843525e-05, + "loss": 2.567, + "theoretical_loss": 3.4813060478620352, + "tokens_seen": 1681063936 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 927976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.612856864929199, + "objective/train/theoretical_loss": 3.4812943419835065, + "objective/train/tokens_used": 51588576, + "theoretical_loss": 3.4812943419835065, + "tokens_seen": 1681129472 + }, + { + "epoch": 0.02, + "learning_rate": 9.9101267854277e-05, + "loss": 2.515, + "theoretical_loss": 3.48128263668907, + "tokens_seen": 1681195008 + }, + { + "epoch": 0.02, + "learning_rate": 9.909324346011877e-05, + "loss": 2.5095, + "theoretical_loss": 3.4812592278522687, + "tokens_seen": 1681326080 + }, + { + "epoch": 0.02, + "learning_rate": 9.908521906596053e-05, + "loss": 2.6043, + "theoretical_loss": 3.4812358213512162, + "tokens_seen": 1681457152 + }, + { + "epoch": 0.02, + "learning_rate": 9.907719467180229e-05, + "loss": 2.6276, + "theoretical_loss": 3.4812124171854966, + "tokens_seen": 1681588224 + }, + { + "epoch": 0.02, + "learning_rate": 9.906917027764405e-05, + "loss": 2.3657, + "theoretical_loss": 3.481189015354696, + "tokens_seen": 1681719296 + }, + { + "epoch": 0.02, + "learning_rate": 9.906114588348581e-05, + "loss": 2.4345, + "theoretical_loss": 3.481165615858399, + "tokens_seen": 1681850368 + }, + { + "epoch": 0.02, + "learning_rate": 9.905312148932757e-05, + "loss": 2.6024, + "theoretical_loss": 3.481142218696191, + "tokens_seen": 1681981440 + }, + { + "epoch": 0.02, + "learning_rate": 9.904509709516932e-05, + "loss": 2.5844, + "theoretical_loss": 3.481118823867657, + "tokens_seen": 1682112512 + }, + { + "epoch": 0.02, + "learning_rate": 9.903707270101109e-05, + "loss": 2.4985, + "theoretical_loss": 3.481095431372383, + "tokens_seen": 1682243584 + }, + { + "epoch": 0.02, + "learning_rate": 9.902904830685284e-05, + "loss": 2.4409, + "theoretical_loss": 3.481072041209954, + "tokens_seen": 1682374656 + }, + { + "epoch": 0.02, + "learning_rate": 9.90210239126946e-05, + "loss": 2.4786, + "theoretical_loss": 3.481048653379955, + "tokens_seen": 1682505728 + }, + { + "epoch": 0.02, + "learning_rate": 9.901299951853636e-05, + "loss": 2.4033, + "theoretical_loss": 3.4810252678819733, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 929273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6835052967071533, + "objective/train/theoretical_loss": 3.481001884715594, + "objective/train/tokens_used": 53226976, + "theoretical_loss": 3.481001884715594, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.02, + "learning_rate": 9.900497512437812e-05, + "loss": 2.5045, + "theoretical_loss": 3.481001884715594, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.02, + "learning_rate": 9.899695073021988e-05, + "loss": 2.4493, + "theoretical_loss": 3.4809785038804026, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.02, + "learning_rate": 9.898892633606163e-05, + "loss": 2.5062, + "theoretical_loss": 3.480955125375986, + "tokens_seen": 1683030016 + }, + { + "epoch": 0.02, + "learning_rate": 9.89809019419034e-05, + "loss": 2.5045, + "theoretical_loss": 3.48093174920193, + "tokens_seen": 1683161088 + }, + { + "epoch": 0.02, + "learning_rate": 9.897287754774515e-05, + "loss": 2.5248, + "theoretical_loss": 3.48090837535782, + "tokens_seen": 1683292160 + }, + { + "epoch": 0.02, + "learning_rate": 9.896485315358692e-05, + "loss": 2.5576, + "theoretical_loss": 3.4808850038432437, + "tokens_seen": 1683423232 + }, + { + "epoch": 0.02, + "learning_rate": 9.895682875942867e-05, + "loss": 2.398, + "theoretical_loss": 3.4808616346577868, + "tokens_seen": 1683554304 + }, + { + "epoch": 0.02, + "learning_rate": 9.894880436527044e-05, + "loss": 2.5147, + "theoretical_loss": 3.4808382678010363, + "tokens_seen": 1683685376 + }, + { + "epoch": 0.02, + "learning_rate": 9.894077997111219e-05, + "loss": 2.5106, + "theoretical_loss": 3.480814903272579, + "tokens_seen": 1683816448 + }, + { + "epoch": 0.02, + "learning_rate": 9.893275557695394e-05, + "loss": 2.4724, + "theoretical_loss": 3.4807915410720005, + "tokens_seen": 1683947520 + }, + { + "epoch": 0.02, + "learning_rate": 9.892473118279571e-05, + "loss": 2.5067, + "theoretical_loss": 3.4807681811988895, + "tokens_seen": 1684078592 + }, + { + "epoch": 0.02, + "learning_rate": 9.891670678863746e-05, + "loss": 2.5444, + "theoretical_loss": 3.4807448236528318, + "tokens_seen": 1684209664 + }, + { + "epoch": 0.02, + "learning_rate": 9.890868239447923e-05, + "loss": 2.5985, + "theoretical_loss": 3.4807214684334147, + "tokens_seen": 1684340736 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 929850, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568603992462158, + "objective/train/theoretical_loss": 3.480709791696068, + "objective/train/tokens_used": 54865376, + "theoretical_loss": 3.480709791696068, + "tokens_seen": 1684406272 + }, + { + "epoch": 0.02, + "learning_rate": 9.890065800032098e-05, + "loss": 2.4515, + "theoretical_loss": 3.4806981155402257, + "tokens_seen": 1684471808 + }, + { + "epoch": 0.02, + "learning_rate": 9.889263360616275e-05, + "loss": 2.4635, + "theoretical_loss": 3.4806747649728518, + "tokens_seen": 1684602880 + }, + { + "epoch": 0.02, + "learning_rate": 9.88846092120045e-05, + "loss": 2.5249, + "theoretical_loss": 3.480651416730881, + "tokens_seen": 1684733952 + }, + { + "epoch": 0.02, + "learning_rate": 9.887658481784626e-05, + "loss": 2.5362, + "theoretical_loss": 3.4806280708139, + "tokens_seen": 1684865024 + }, + { + "epoch": 0.02, + "learning_rate": 9.886856042368802e-05, + "loss": 2.4881, + "theoretical_loss": 3.480604727221497, + "tokens_seen": 1684996096 + }, + { + "epoch": 0.02, + "learning_rate": 9.886053602952978e-05, + "loss": 2.555, + "theoretical_loss": 3.4805813859532595, + "tokens_seen": 1685127168 + }, + { + "epoch": 0.02, + "learning_rate": 9.885251163537154e-05, + "loss": 2.6367, + "theoretical_loss": 3.480558047008776, + "tokens_seen": 1685258240 + }, + { + "epoch": 0.02, + "learning_rate": 9.88444872412133e-05, + "loss": 2.4256, + "theoretical_loss": 3.4805347103876327, + "tokens_seen": 1685389312 + }, + { + "epoch": 0.02, + "learning_rate": 9.883646284705506e-05, + "loss": 2.2893, + "theoretical_loss": 3.48051137608942, + "tokens_seen": 1685520384 + }, + { + "epoch": 0.02, + "learning_rate": 9.882843845289682e-05, + "loss": 2.5847, + "theoretical_loss": 3.4804880441137245, + "tokens_seen": 1685651456 + }, + { + "epoch": 0.02, + "learning_rate": 9.882041405873857e-05, + "loss": 2.6211, + "theoretical_loss": 3.4804647144601346, + "tokens_seen": 1685782528 + }, + { + "epoch": 0.02, + "learning_rate": 9.881238966458034e-05, + "loss": 2.4622, + "theoretical_loss": 3.4804413871282396, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 931063, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.578721523284912, + "objective/train/theoretical_loss": 3.480418062117627, + "objective/train/tokens_used": 56503776, + "theoretical_loss": 3.480418062117627, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.02, + "learning_rate": 9.880436527042209e-05, + "loss": 2.5689, + "theoretical_loss": 3.480418062117627, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.02, + "learning_rate": 9.879634087626386e-05, + "loss": 2.4969, + "theoretical_loss": 3.4803947394278856, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.02, + "learning_rate": 9.878831648210561e-05, + "loss": 2.5164, + "theoretical_loss": 3.4803714190586037, + "tokens_seen": 1686306816 + }, + { + "epoch": 0.02, + "learning_rate": 9.878029208794736e-05, + "loss": 2.4945, + "theoretical_loss": 3.4803481010093718, + "tokens_seen": 1686437888 + }, + { + "epoch": 0.02, + "learning_rate": 9.877226769378913e-05, + "loss": 2.399, + "theoretical_loss": 3.4803247852797767, + "tokens_seen": 1686568960 + }, + { + "epoch": 0.02, + "learning_rate": 9.876424329963088e-05, + "loss": 2.4799, + "theoretical_loss": 3.4803014718694087, + "tokens_seen": 1686700032 + }, + { + "epoch": 0.02, + "learning_rate": 9.875621890547265e-05, + "loss": 2.35, + "theoretical_loss": 3.480278160777856, + "tokens_seen": 1686831104 + }, + { + "epoch": 0.02, + "learning_rate": 9.87481945113144e-05, + "loss": 2.4886, + "theoretical_loss": 3.480254852004709, + "tokens_seen": 1686962176 + }, + { + "epoch": 0.02, + "learning_rate": 9.874017011715617e-05, + "loss": 2.5754, + "theoretical_loss": 3.4802315455495565, + "tokens_seen": 1687093248 + }, + { + "epoch": 0.02, + "learning_rate": 9.873214572299792e-05, + "loss": 2.5741, + "theoretical_loss": 3.480208241411987, + "tokens_seen": 1687224320 + }, + { + "epoch": 0.02, + "learning_rate": 9.872412132883967e-05, + "loss": 2.5755, + "theoretical_loss": 3.480184939591591, + "tokens_seen": 1687355392 + }, + { + "epoch": 0.02, + "learning_rate": 9.871609693468144e-05, + "loss": 2.4047, + "theoretical_loss": 3.4801616400879585, + "tokens_seen": 1687486464 + }, + { + "epoch": 0.02, + "learning_rate": 9.87080725405232e-05, + "loss": 2.4871, + "theoretical_loss": 3.480138342900678, + "tokens_seen": 1687617536 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 931367, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5317652225494385, + "objective/train/theoretical_loss": 3.480126695175542, + "objective/train/tokens_used": 58142176, + "theoretical_loss": 3.480126695175542, + "tokens_seen": 1687683072 + }, + { + "epoch": 0.02, + "learning_rate": 9.870004814636496e-05, + "loss": 2.6203, + "theoretical_loss": 3.48011504802934, + "tokens_seen": 1687748608 + }, + { + "epoch": 0.02, + "learning_rate": 9.869202375220671e-05, + "loss": 2.4963, + "theoretical_loss": 3.4800917554735347, + "tokens_seen": 1687879680 + }, + { + "epoch": 0.02, + "learning_rate": 9.868399935804847e-05, + "loss": 2.5753, + "theoretical_loss": 3.480068465232852, + "tokens_seen": 1688010752 + }, + { + "epoch": 0.02, + "learning_rate": 9.867597496389023e-05, + "loss": 2.546, + "theoretical_loss": 3.4800451773068817, + "tokens_seen": 1688141824 + }, + { + "epoch": 0.02, + "learning_rate": 9.866795056973199e-05, + "loss": 2.3913, + "theoretical_loss": 3.4800218916952144, + "tokens_seen": 1688272896 + }, + { + "epoch": 0.02, + "learning_rate": 9.865992617557375e-05, + "loss": 2.5884, + "theoretical_loss": 3.47999860839744, + "tokens_seen": 1688403968 + }, + { + "epoch": 0.02, + "learning_rate": 9.86519017814155e-05, + "loss": 2.5369, + "theoretical_loss": 3.4799753274131495, + "tokens_seen": 1688535040 + }, + { + "epoch": 0.02, + "learning_rate": 9.864387738725727e-05, + "loss": 2.6298, + "theoretical_loss": 3.4799520487419335, + "tokens_seen": 1688666112 + }, + { + "epoch": 0.02, + "learning_rate": 9.863585299309903e-05, + "loss": 2.567, + "theoretical_loss": 3.4799287723833823, + "tokens_seen": 1688797184 + }, + { + "epoch": 0.02, + "learning_rate": 9.862782859894078e-05, + "loss": 2.4449, + "theoretical_loss": 3.4799054983370867, + "tokens_seen": 1688928256 + }, + { + "epoch": 0.02, + "learning_rate": 9.861980420478255e-05, + "loss": 2.5075, + "theoretical_loss": 3.4798822266026384, + "tokens_seen": 1689059328 + }, + { + "epoch": 0.02, + "learning_rate": 9.86117798106243e-05, + "loss": 2.5648, + "theoretical_loss": 3.4798589571796272, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 932516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2863895893096924, + "objective/train/theoretical_loss": 3.4798356900676444, + "objective/train/tokens_used": 59780576, + "theoretical_loss": 3.4798356900676444, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 9.860375541646607e-05, + "loss": 2.4437, + "theoretical_loss": 3.4798356900676444, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 9.859573102230782e-05, + "loss": 2.4915, + "theoretical_loss": 3.479812425266282, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 9.858770662814957e-05, + "loss": 2.5486, + "theoretical_loss": 3.479789162775131, + "tokens_seen": 1689583616 + }, + { + "epoch": 0.02, + "learning_rate": 9.857968223399134e-05, + "loss": 2.4756, + "theoretical_loss": 3.479765902593782, + "tokens_seen": 1689714688 + }, + { + "epoch": 0.02, + "learning_rate": 9.857165783983309e-05, + "loss": 2.4487, + "theoretical_loss": 3.4797426447218274, + "tokens_seen": 1689845760 + }, + { + "epoch": 0.02, + "learning_rate": 9.856363344567486e-05, + "loss": 2.5253, + "theoretical_loss": 3.4797193891588583, + "tokens_seen": 1689976832 + }, + { + "epoch": 0.02, + "learning_rate": 9.855560905151661e-05, + "loss": 2.4716, + "theoretical_loss": 3.479696135904467, + "tokens_seen": 1690107904 + }, + { + "epoch": 0.02, + "learning_rate": 9.854758465735838e-05, + "loss": 2.5138, + "theoretical_loss": 3.479672884958245, + "tokens_seen": 1690238976 + }, + { + "epoch": 0.02, + "learning_rate": 9.853956026320013e-05, + "loss": 2.4722, + "theoretical_loss": 3.4796496363197837, + "tokens_seen": 1690370048 + }, + { + "epoch": 0.02, + "learning_rate": 9.853153586904188e-05, + "loss": 2.4039, + "theoretical_loss": 3.4796263899886757, + "tokens_seen": 1690501120 + }, + { + "epoch": 0.02, + "learning_rate": 9.852351147488365e-05, + "loss": 2.2929, + "theoretical_loss": 3.4796031459645134, + "tokens_seen": 1690632192 + }, + { + "epoch": 0.02, + "learning_rate": 9.85154870807254e-05, + "loss": 2.5445, + "theoretical_loss": 3.4795799042468882, + "tokens_seen": 1690763264 + }, + { + "epoch": 0.02, + "learning_rate": 9.850746268656717e-05, + "loss": 2.4633, + "theoretical_loss": 3.4795566648353935, + "tokens_seen": 1690894336 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 933032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.596432685852051, + "objective/train/theoretical_loss": 3.4795450459943167, + "objective/train/tokens_used": 61418976, + "theoretical_loss": 3.4795450459943167, + "tokens_seen": 1690959872 + }, + { + "epoch": 0.02, + "learning_rate": 9.849943829240892e-05, + "loss": 2.5766, + "theoretical_loss": 3.4795334277296206, + "tokens_seen": 1691025408 + }, + { + "epoch": 0.02, + "learning_rate": 9.849141389825068e-05, + "loss": 2.4845, + "theoretical_loss": 3.4795101929291627, + "tokens_seen": 1691156480 + }, + { + "epoch": 0.03, + "learning_rate": 9.848338950409244e-05, + "loss": 2.4419, + "theoretical_loss": 3.4794869604336123, + "tokens_seen": 1691287552 + }, + { + "epoch": 0.03, + "learning_rate": 9.84753651099342e-05, + "loss": 2.6624, + "theoretical_loss": 3.4794637302425624, + "tokens_seen": 1691418624 + }, + { + "epoch": 0.03, + "learning_rate": 9.846734071577596e-05, + "loss": 2.4082, + "theoretical_loss": 3.4794405023556054, + "tokens_seen": 1691549696 + }, + { + "epoch": 0.03, + "learning_rate": 9.845931632161772e-05, + "loss": 2.5021, + "theoretical_loss": 3.4794172767723346, + "tokens_seen": 1691680768 + }, + { + "epoch": 0.03, + "learning_rate": 9.845129192745948e-05, + "loss": 2.6212, + "theoretical_loss": 3.479394053492343, + "tokens_seen": 1691811840 + }, + { + "epoch": 0.03, + "learning_rate": 9.844326753330124e-05, + "loss": 2.4537, + "theoretical_loss": 3.4793708325152237, + "tokens_seen": 1691942912 + }, + { + "epoch": 0.03, + "learning_rate": 9.843524313914299e-05, + "loss": 2.5129, + "theoretical_loss": 3.47934761384057, + "tokens_seen": 1692073984 + }, + { + "epoch": 0.03, + "learning_rate": 9.842721874498476e-05, + "loss": 2.5562, + "theoretical_loss": 3.4793243974679755, + "tokens_seen": 1692205056 + }, + { + "epoch": 0.03, + "learning_rate": 9.841919435082651e-05, + "loss": 2.409, + "theoretical_loss": 3.479301183397033, + "tokens_seen": 1692336128 + }, + { + "epoch": 0.03, + "learning_rate": 9.841116995666828e-05, + "loss": 2.5037, + "theoretical_loss": 3.4792779716273365, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 934026, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.466470241546631, + "objective/train/theoretical_loss": 3.4792547621584804, + "objective/train/tokens_used": 63057376, + "theoretical_loss": 3.4792547621584804, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.03, + "learning_rate": 9.840314556251003e-05, + "loss": 2.4919, + "theoretical_loss": 3.4792547621584804, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.03, + "learning_rate": 9.839512116835178e-05, + "loss": 2.5155, + "theoretical_loss": 3.479231554990057, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.03, + "learning_rate": 9.838709677419355e-05, + "loss": 2.4349, + "theoretical_loss": 3.4792083501216613, + "tokens_seen": 1692860416 + }, + { + "epoch": 0.03, + "learning_rate": 9.83790723800353e-05, + "loss": 2.2746, + "theoretical_loss": 3.4791851475528874, + "tokens_seen": 1692991488 + }, + { + "epoch": 0.03, + "learning_rate": 9.837104798587707e-05, + "loss": 2.6368, + "theoretical_loss": 3.4791619472833286, + "tokens_seen": 1693122560 + }, + { + "epoch": 0.03, + "learning_rate": 9.836302359171882e-05, + "loss": 2.4889, + "theoretical_loss": 3.479138749312579, + "tokens_seen": 1693253632 + }, + { + "epoch": 0.03, + "learning_rate": 9.835499919756059e-05, + "loss": 2.5798, + "theoretical_loss": 3.4791155536402343, + "tokens_seen": 1693384704 + }, + { + "epoch": 0.03, + "learning_rate": 9.834697480340234e-05, + "loss": 2.6683, + "theoretical_loss": 3.479092360265887, + "tokens_seen": 1693515776 + }, + { + "epoch": 0.03, + "learning_rate": 9.83389504092441e-05, + "loss": 2.4851, + "theoretical_loss": 3.479069169189133, + "tokens_seen": 1693646848 + }, + { + "epoch": 0.03, + "learning_rate": 9.833092601508586e-05, + "loss": 2.3859, + "theoretical_loss": 3.4790459804095666, + "tokens_seen": 1693777920 + }, + { + "epoch": 0.03, + "learning_rate": 9.832290162092762e-05, + "loss": 2.5253, + "theoretical_loss": 3.4790227939267826, + "tokens_seen": 1693908992 + }, + { + "epoch": 0.03, + "learning_rate": 9.831487722676938e-05, + "loss": 2.2977, + "theoretical_loss": 3.478999609740375, + "tokens_seen": 1694040064 + }, + { + "epoch": 0.03, + "learning_rate": 9.830685283261113e-05, + "loss": 2.4333, + "theoretical_loss": 3.4789764278499398, + "tokens_seen": 1694171136 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 934482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2957074642181396, + "objective/train/theoretical_loss": 3.478964837765585, + "objective/train/tokens_used": 64695776, + "theoretical_loss": 3.478964837765585, + "tokens_seen": 1694236672 + }, + { + "epoch": 0.03, + "learning_rate": 9.82988284384529e-05, + "loss": 2.5303, + "theoretical_loss": 3.4789532482550714, + "tokens_seen": 1694302208 + }, + { + "epoch": 0.03, + "learning_rate": 9.829080404429465e-05, + "loss": 2.4688, + "theoretical_loss": 3.478930070955365, + "tokens_seen": 1694433280 + }, + { + "epoch": 0.03, + "learning_rate": 9.828277965013641e-05, + "loss": 2.443, + "theoretical_loss": 3.478906895950416, + "tokens_seen": 1694564352 + }, + { + "epoch": 0.03, + "learning_rate": 9.827475525597817e-05, + "loss": 2.5702, + "theoretical_loss": 3.4788837232398198, + "tokens_seen": 1694695424 + }, + { + "epoch": 0.03, + "learning_rate": 9.826673086181993e-05, + "loss": 2.3373, + "theoretical_loss": 3.4788605528231713, + "tokens_seen": 1694826496 + }, + { + "epoch": 0.03, + "learning_rate": 9.82587064676617e-05, + "loss": 2.4548, + "theoretical_loss": 3.4788373847000664, + "tokens_seen": 1694957568 + }, + { + "epoch": 0.03, + "learning_rate": 9.825068207350345e-05, + "loss": 2.6429, + "theoretical_loss": 3.478814218870101, + "tokens_seen": 1695088640 + }, + { + "epoch": 0.03, + "learning_rate": 9.824265767934521e-05, + "loss": 2.3599, + "theoretical_loss": 3.4787910553328705, + "tokens_seen": 1695219712 + }, + { + "epoch": 0.03, + "learning_rate": 9.823463328518697e-05, + "loss": 2.3772, + "theoretical_loss": 3.4787678940879707, + "tokens_seen": 1695350784 + }, + { + "epoch": 0.03, + "learning_rate": 9.822660889102873e-05, + "loss": 2.5326, + "theoretical_loss": 3.478744735134998, + "tokens_seen": 1695481856 + }, + { + "epoch": 0.03, + "learning_rate": 9.821858449687049e-05, + "loss": 2.6442, + "theoretical_loss": 3.4787215784735475, + "tokens_seen": 1695612928 + }, + { + "epoch": 0.03, + "learning_rate": 9.821056010271224e-05, + "loss": 2.5572, + "theoretical_loss": 3.478698424103216, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 935631, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5070106983184814, + "objective/train/theoretical_loss": 3.4786752720236005, + "objective/train/tokens_used": 66334176, + "theoretical_loss": 3.4786752720236005, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.03, + "learning_rate": 9.820253570855401e-05, + "loss": 2.6289, + "theoretical_loss": 3.4786752720236005, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.03, + "learning_rate": 9.819451131439576e-05, + "loss": 2.659, + "theoretical_loss": 3.478652122234296, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.03, + "learning_rate": 9.818648692023753e-05, + "loss": 2.575, + "theoretical_loss": 3.4786289747348995, + "tokens_seen": 1696137216 + }, + { + "epoch": 0.03, + "learning_rate": 9.817846252607928e-05, + "loss": 2.637, + "theoretical_loss": 3.4786058295250077, + "tokens_seen": 1696268288 + }, + { + "epoch": 0.03, + "learning_rate": 9.817043813192105e-05, + "loss": 2.5304, + "theoretical_loss": 3.4785826866042173, + "tokens_seen": 1696399360 + }, + { + "epoch": 0.03, + "learning_rate": 9.81624137377628e-05, + "loss": 2.4708, + "theoretical_loss": 3.4785595459721246, + "tokens_seen": 1696530432 + }, + { + "epoch": 0.03, + "learning_rate": 9.815438934360455e-05, + "loss": 2.5821, + "theoretical_loss": 3.4785364076283267, + "tokens_seen": 1696661504 + }, + { + "epoch": 0.03, + "learning_rate": 9.814636494944632e-05, + "loss": 2.6878, + "theoretical_loss": 3.478513271572421, + "tokens_seen": 1696792576 + }, + { + "epoch": 0.03, + "learning_rate": 9.813834055528807e-05, + "loss": 2.5641, + "theoretical_loss": 3.4784901378040036, + "tokens_seen": 1696923648 + }, + { + "epoch": 0.03, + "learning_rate": 9.813031616112984e-05, + "loss": 2.5774, + "theoretical_loss": 3.4784670063226732, + "tokens_seen": 1697054720 + }, + { + "epoch": 0.03, + "learning_rate": 9.812229176697159e-05, + "loss": 2.5069, + "theoretical_loss": 3.4784438771280257, + "tokens_seen": 1697185792 + }, + { + "epoch": 0.03, + "learning_rate": 9.811426737281336e-05, + "loss": 2.4408, + "theoretical_loss": 3.4784207502196587, + "tokens_seen": 1697316864 + }, + { + "epoch": 0.03, + "learning_rate": 9.810624297865511e-05, + "loss": 2.6084, + "theoretical_loss": 3.47839762559717, + "tokens_seen": 1697447936 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 936086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.701629877090454, + "objective/train/theoretical_loss": 3.478386064143004, + "objective/train/tokens_used": 67972576, + "theoretical_loss": 3.478386064143004, + "tokens_seen": 1697513472 + }, + { + "epoch": 0.03, + "learning_rate": 9.809821858449687e-05, + "loss": 2.5616, + "theoretical_loss": 3.4783745032601567, + "tokens_seen": 1697579008 + }, + { + "epoch": 0.03, + "learning_rate": 9.809019419033863e-05, + "loss": 2.5896, + "theoretical_loss": 3.4783513832082176, + "tokens_seen": 1697710080 + }, + { + "epoch": 0.03, + "learning_rate": 9.808216979618039e-05, + "loss": 2.4651, + "theoretical_loss": 3.4783282654409495, + "tokens_seen": 1697841152 + }, + { + "epoch": 0.03, + "learning_rate": 9.807414540202215e-05, + "loss": 2.5471, + "theoretical_loss": 3.4783051499579507, + "tokens_seen": 1697972224 + }, + { + "epoch": 0.03, + "learning_rate": 9.80661210078639e-05, + "loss": 2.5858, + "theoretical_loss": 3.4782820367588183, + "tokens_seen": 1698103296 + }, + { + "epoch": 0.03, + "learning_rate": 9.805809661370567e-05, + "loss": 2.4588, + "theoretical_loss": 3.4782589258431518, + "tokens_seen": 1698234368 + }, + { + "epoch": 0.03, + "learning_rate": 9.805007221954742e-05, + "loss": 2.4407, + "theoretical_loss": 3.4782358172105483, + "tokens_seen": 1698365440 + }, + { + "epoch": 0.03, + "learning_rate": 9.804204782538918e-05, + "loss": 2.6123, + "theoretical_loss": 3.478212710860607, + "tokens_seen": 1698496512 + }, + { + "epoch": 0.03, + "learning_rate": 9.803402343123094e-05, + "loss": 2.3523, + "theoretical_loss": 3.4781896067929257, + "tokens_seen": 1698627584 + }, + { + "epoch": 0.03, + "learning_rate": 9.80259990370727e-05, + "loss": 2.5561, + "theoretical_loss": 3.4781665050071027, + "tokens_seen": 1698758656 + }, + { + "epoch": 0.03, + "learning_rate": 9.801797464291446e-05, + "loss": 2.5973, + "theoretical_loss": 3.4781434055027365, + "tokens_seen": 1698889728 + }, + { + "epoch": 0.03, + "learning_rate": 9.800995024875622e-05, + "loss": 2.3148, + "theoretical_loss": 3.4781203082794265, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 937360, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2998149394989014, + "objective/train/theoretical_loss": 3.4780972133367714, + "objective/train/tokens_used": 69610976, + "theoretical_loss": 3.4780972133367714, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.03, + "learning_rate": 9.800192585459798e-05, + "loss": 2.5517, + "theoretical_loss": 3.4780972133367714, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.03, + "learning_rate": 9.799390146043974e-05, + "loss": 2.5891, + "theoretical_loss": 3.47807412067437, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.03, + "learning_rate": 9.798587706628149e-05, + "loss": 2.5741, + "theoretical_loss": 3.478051030291821, + "tokens_seen": 1699414016 + }, + { + "epoch": 0.03, + "learning_rate": 9.797785267212326e-05, + "loss": 2.6164, + "theoretical_loss": 3.4780279421887235, + "tokens_seen": 1699545088 + }, + { + "epoch": 0.03, + "learning_rate": 9.796982827796501e-05, + "loss": 2.6742, + "theoretical_loss": 3.478004856364677, + "tokens_seen": 1699676160 + }, + { + "epoch": 0.03, + "learning_rate": 9.796180388380678e-05, + "loss": 2.5587, + "theoretical_loss": 3.4779817728192803, + "tokens_seen": 1699807232 + }, + { + "epoch": 0.03, + "learning_rate": 9.795377948964853e-05, + "loss": 2.41, + "theoretical_loss": 3.477958691552134, + "tokens_seen": 1699938304 + }, + { + "epoch": 0.03, + "learning_rate": 9.79457550954903e-05, + "loss": 2.4044, + "theoretical_loss": 3.4779356125628365, + "tokens_seen": 1700069376 + }, + { + "epoch": 0.03, + "learning_rate": 9.793773070133205e-05, + "loss": 2.6259, + "theoretical_loss": 3.4779125358509875, + "tokens_seen": 1700200448 + }, + { + "epoch": 0.03, + "learning_rate": 9.792970630717382e-05, + "loss": 2.54, + "theoretical_loss": 3.4778894614161873, + "tokens_seen": 1700331520 + }, + { + "epoch": 0.03, + "learning_rate": 9.792168191301557e-05, + "loss": 2.4259, + "theoretical_loss": 3.4778663892580353, + "tokens_seen": 1700462592 + }, + { + "epoch": 0.03, + "learning_rate": 9.791365751885732e-05, + "loss": 2.7077, + "theoretical_loss": 3.477843319376131, + "tokens_seen": 1700593664 + }, + { + "epoch": 0.03, + "learning_rate": 9.790563312469909e-05, + "loss": 2.6439, + "theoretical_loss": 3.4778202517700754, + "tokens_seen": 1700724736 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 938428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8109610080718994, + "objective/train/theoretical_loss": 3.4778087188203655, + "objective/train/tokens_used": 71249376, + "theoretical_loss": 3.4778087188203655, + "tokens_seen": 1700790272 + }, + { + "epoch": 0.03, + "learning_rate": 9.789760873054084e-05, + "loss": 2.4903, + "theoretical_loss": 3.477797186439468, + "tokens_seen": 1700855808 + }, + { + "epoch": 0.03, + "learning_rate": 9.788958433638261e-05, + "loss": 2.6681, + "theoretical_loss": 3.477774123383909, + "tokens_seen": 1700986880 + }, + { + "epoch": 0.03, + "learning_rate": 9.788155994222436e-05, + "loss": 2.4544, + "theoretical_loss": 3.477751062602999, + "tokens_seen": 1701117952 + }, + { + "epoch": 0.03, + "learning_rate": 9.787353554806613e-05, + "loss": 2.627, + "theoretical_loss": 3.4777280040963383, + "tokens_seen": 1701249024 + }, + { + "epoch": 0.03, + "learning_rate": 9.786551115390788e-05, + "loss": 2.5838, + "theoretical_loss": 3.477704947863527, + "tokens_seen": 1701380096 + }, + { + "epoch": 0.03, + "learning_rate": 9.785748675974964e-05, + "loss": 2.5787, + "theoretical_loss": 3.4776818939041663, + "tokens_seen": 1701511168 + }, + { + "epoch": 0.03, + "learning_rate": 9.78494623655914e-05, + "loss": 2.6475, + "theoretical_loss": 3.477658842217857, + "tokens_seen": 1701642240 + }, + { + "epoch": 0.03, + "learning_rate": 9.784143797143316e-05, + "loss": 2.529, + "theoretical_loss": 3.4776357928041994, + "tokens_seen": 1701773312 + }, + { + "epoch": 0.03, + "learning_rate": 9.783341357727492e-05, + "loss": 2.4775, + "theoretical_loss": 3.4776127456627948, + "tokens_seen": 1701904384 + }, + { + "epoch": 0.03, + "learning_rate": 9.782538918311668e-05, + "loss": 2.6175, + "theoretical_loss": 3.477589700793244, + "tokens_seen": 1702035456 + }, + { + "epoch": 0.03, + "learning_rate": 9.781736478895844e-05, + "loss": 2.5555, + "theoretical_loss": 3.477566658195148, + "tokens_seen": 1702166528 + }, + { + "epoch": 0.03, + "learning_rate": 9.78093403948002e-05, + "loss": 2.5131, + "theoretical_loss": 3.4775436178681085, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 939017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7329940795898438, + "objective/train/theoretical_loss": 3.477520579811727, + "objective/train/tokens_used": 72887776, + "theoretical_loss": 3.477520579811727, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.03, + "learning_rate": 9.780131600064195e-05, + "loss": 2.5521, + "theoretical_loss": 3.477520579811727, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.03, + "learning_rate": 9.779329160648371e-05, + "loss": 2.6362, + "theoretical_loss": 3.4774975440256037, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 9.778526721232547e-05, + "loss": 2.6066, + "theoretical_loss": 3.4774745105093414, + "tokens_seen": 1702690816 + }, + { + "epoch": 0.03, + "learning_rate": 9.777724281816723e-05, + "loss": 2.5312, + "theoretical_loss": 3.477451479262541, + "tokens_seen": 1702821888 + }, + { + "epoch": 0.03, + "learning_rate": 9.776921842400899e-05, + "loss": 2.5702, + "theoretical_loss": 3.477428450284805, + "tokens_seen": 1702952960 + }, + { + "epoch": 0.03, + "learning_rate": 9.776119402985075e-05, + "loss": 2.6024, + "theoretical_loss": 3.4774054235757346, + "tokens_seen": 1703084032 + }, + { + "epoch": 0.03, + "learning_rate": 9.775316963569251e-05, + "loss": 2.7145, + "theoretical_loss": 3.4773823991349317, + "tokens_seen": 1703215104 + }, + { + "epoch": 0.03, + "learning_rate": 9.774514524153426e-05, + "loss": 2.4479, + "theoretical_loss": 3.477359376961999, + "tokens_seen": 1703346176 + }, + { + "epoch": 0.03, + "learning_rate": 9.773712084737603e-05, + "loss": 2.5557, + "theoretical_loss": 3.4773363570565374, + "tokens_seen": 1703477248 + }, + { + "epoch": 0.03, + "learning_rate": 9.772909645321778e-05, + "loss": 2.514, + "theoretical_loss": 3.47731333941815, + "tokens_seen": 1703608320 + }, + { + "epoch": 0.03, + "learning_rate": 9.772107205905955e-05, + "loss": 2.4775, + "theoretical_loss": 3.4772903240464395, + "tokens_seen": 1703739392 + }, + { + "epoch": 0.03, + "learning_rate": 9.77130476649013e-05, + "loss": 2.5532, + "theoretical_loss": 3.4772673109410075, + "tokens_seen": 1703870464 + }, + { + "epoch": 0.03, + "learning_rate": 9.770502327074307e-05, + "loss": 2.5049, + "theoretical_loss": 3.4772443001014564, + "tokens_seen": 1704001536 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 939656, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5736076831817627, + "objective/train/theoretical_loss": 3.4772327955312625, + "objective/train/tokens_used": 74526176, + "theoretical_loss": 3.4772327955312625, + "tokens_seen": 1704067072 + }, + { + "epoch": 0.03, + "learning_rate": 9.769699887658482e-05, + "loss": 2.5509, + "theoretical_loss": 3.47722129152739, + "tokens_seen": 1704132608 + }, + { + "epoch": 0.03, + "learning_rate": 9.768897448242657e-05, + "loss": 2.5471, + "theoretical_loss": 3.4771982852184102, + "tokens_seen": 1704263680 + }, + { + "epoch": 0.03, + "learning_rate": 9.768095008826834e-05, + "loss": 2.4507, + "theoretical_loss": 3.4771752811741194, + "tokens_seen": 1704394752 + }, + { + "epoch": 0.03, + "learning_rate": 9.767292569411009e-05, + "loss": 2.5674, + "theoretical_loss": 3.477152279394122, + "tokens_seen": 1704525824 + }, + { + "epoch": 0.03, + "learning_rate": 9.766490129995186e-05, + "loss": 2.618, + "theoretical_loss": 3.4771292798780196, + "tokens_seen": 1704656896 + }, + { + "epoch": 0.03, + "learning_rate": 9.765687690579361e-05, + "loss": 2.5407, + "theoretical_loss": 3.4771062826254155, + "tokens_seen": 1704787968 + }, + { + "epoch": 0.03, + "learning_rate": 9.764885251163538e-05, + "loss": 2.6236, + "theoretical_loss": 3.477083287635914, + "tokens_seen": 1704919040 + }, + { + "epoch": 0.03, + "learning_rate": 9.764082811747713e-05, + "loss": 2.7451, + "theoretical_loss": 3.4770602949091174, + "tokens_seen": 1705050112 + }, + { + "epoch": 0.03, + "learning_rate": 9.763280372331889e-05, + "loss": 2.5501, + "theoretical_loss": 3.4770373044446297, + "tokens_seen": 1705181184 + }, + { + "epoch": 0.03, + "learning_rate": 9.762477932916065e-05, + "loss": 2.6444, + "theoretical_loss": 3.4770143162420544, + "tokens_seen": 1705312256 + }, + { + "epoch": 0.03, + "learning_rate": 9.76167549350024e-05, + "loss": 2.6054, + "theoretical_loss": 3.4769913303009945, + "tokens_seen": 1705443328 + }, + { + "epoch": 0.03, + "learning_rate": 9.760873054084417e-05, + "loss": 2.6033, + "theoretical_loss": 3.4769683466210544, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 940998, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.642338752746582, + "objective/train/theoretical_loss": 3.4769453652018374, + "objective/train/tokens_used": 76164576, + "theoretical_loss": 3.4769453652018374, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.03, + "learning_rate": 9.760070614668593e-05, + "loss": 2.5956, + "theoretical_loss": 3.4769453652018374, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.03, + "learning_rate": 9.759268175252769e-05, + "loss": 2.5025, + "theoretical_loss": 3.4769223860429483, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 9.758465735836945e-05, + "loss": 2.6917, + "theoretical_loss": 3.47689940914399, + "tokens_seen": 1705967616 + }, + { + "epoch": 0.03, + "learning_rate": 9.757663296421121e-05, + "loss": 2.5622, + "theoretical_loss": 3.4768764345045677, + "tokens_seen": 1706098688 + }, + { + "epoch": 0.03, + "learning_rate": 9.756860857005296e-05, + "loss": 2.5032, + "theoretical_loss": 3.4768534621242853, + "tokens_seen": 1706229760 + }, + { + "epoch": 0.03, + "learning_rate": 9.756058417589472e-05, + "loss": 2.6986, + "theoretical_loss": 3.4768304920027466, + "tokens_seen": 1706360832 + }, + { + "epoch": 0.03, + "learning_rate": 9.755255978173648e-05, + "loss": 2.5713, + "theoretical_loss": 3.4768075241395566, + "tokens_seen": 1706491904 + }, + { + "epoch": 0.03, + "learning_rate": 9.754453538757824e-05, + "loss": 2.6232, + "theoretical_loss": 3.4767845585343196, + "tokens_seen": 1706622976 + }, + { + "epoch": 0.03, + "learning_rate": 9.753651099342e-05, + "loss": 2.5417, + "theoretical_loss": 3.47676159518664, + "tokens_seen": 1706754048 + }, + { + "epoch": 0.03, + "learning_rate": 9.752848659926176e-05, + "loss": 2.5311, + "theoretical_loss": 3.4767386340961233, + "tokens_seen": 1706885120 + }, + { + "epoch": 0.03, + "learning_rate": 9.752046220510352e-05, + "loss": 2.5849, + "theoretical_loss": 3.4767156752623736, + "tokens_seen": 1707016192 + }, + { + "epoch": 0.03, + "learning_rate": 9.751243781094528e-05, + "loss": 2.6568, + "theoretical_loss": 3.476692718684996, + "tokens_seen": 1707147264 + }, + { + "epoch": 0.03, + "learning_rate": 9.750441341678703e-05, + "loss": 2.4312, + "theoretical_loss": 3.4766697643635958, + "tokens_seen": 1707278336 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 942016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.860006809234619, + "objective/train/theoretical_loss": 3.4766582880487635, + "objective/train/tokens_used": 77802976, + "theoretical_loss": 3.4766582880487635, + "tokens_seen": 1707343872 + }, + { + "epoch": 0.03, + "learning_rate": 9.74963890226288e-05, + "loss": 2.6738, + "theoretical_loss": 3.4766468122977776, + "tokens_seen": 1707409408 + }, + { + "epoch": 0.03, + "learning_rate": 9.748836462847055e-05, + "loss": 2.507, + "theoretical_loss": 3.4766238624871475, + "tokens_seen": 1707540480 + }, + { + "epoch": 0.03, + "learning_rate": 9.748034023431232e-05, + "loss": 2.6324, + "theoretical_loss": 3.4766009149313097, + "tokens_seen": 1707671552 + }, + { + "epoch": 0.04, + "learning_rate": 9.747231584015407e-05, + "loss": 2.5417, + "theoretical_loss": 3.4765779696298704, + "tokens_seen": 1707802624 + }, + { + "epoch": 0.04, + "learning_rate": 9.746429144599584e-05, + "loss": 2.568, + "theoretical_loss": 3.476555026582435, + "tokens_seen": 1707933696 + }, + { + "epoch": 0.04, + "learning_rate": 9.745626705183759e-05, + "loss": 2.4916, + "theoretical_loss": 3.4765320857886097, + "tokens_seen": 1708064768 + }, + { + "epoch": 0.04, + "learning_rate": 9.744824265767934e-05, + "loss": 2.4715, + "theoretical_loss": 3.476509147247999, + "tokens_seen": 1708195840 + }, + { + "epoch": 0.04, + "learning_rate": 9.744021826352111e-05, + "loss": 2.4469, + "theoretical_loss": 3.4764862109602097, + "tokens_seen": 1708326912 + }, + { + "epoch": 0.04, + "learning_rate": 9.743219386936286e-05, + "loss": 2.6352, + "theoretical_loss": 3.476463276924847, + "tokens_seen": 1708457984 + }, + { + "epoch": 0.04, + "learning_rate": 9.742416947520463e-05, + "loss": 2.6409, + "theoretical_loss": 3.4764403451415173, + "tokens_seen": 1708589056 + }, + { + "epoch": 0.04, + "learning_rate": 9.741614508104638e-05, + "loss": 2.6966, + "theoretical_loss": 3.476417415609827, + "tokens_seen": 1708720128 + }, + { + "epoch": 0.04, + "learning_rate": 9.740812068688815e-05, + "loss": 2.5196, + "theoretical_loss": 3.476394488329382, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 942685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.43986439704895, + "objective/train/theoretical_loss": 3.4763715632997885, + "objective/train/tokens_used": 79441376, + "theoretical_loss": 3.4763715632997885, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.04, + "learning_rate": 9.74000962927299e-05, + "loss": 2.5859, + "theoretical_loss": 3.4763715632997885, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.04, + "learning_rate": 9.739207189857166e-05, + "loss": 2.669, + "theoretical_loss": 3.476348640520653, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.04, + "learning_rate": 9.738404750441342e-05, + "loss": 2.6132, + "theoretical_loss": 3.476325719991582, + "tokens_seen": 1709244416 + }, + { + "epoch": 0.04, + "learning_rate": 9.737602311025518e-05, + "loss": 2.5786, + "theoretical_loss": 3.4763028017121824, + "tokens_seen": 1709375488 + }, + { + "epoch": 0.04, + "learning_rate": 9.736799871609694e-05, + "loss": 2.4671, + "theoretical_loss": 3.476279885682061, + "tokens_seen": 1709506560 + }, + { + "epoch": 0.04, + "learning_rate": 9.73599743219387e-05, + "loss": 2.8103, + "theoretical_loss": 3.476256971900824, + "tokens_seen": 1709637632 + }, + { + "epoch": 0.04, + "learning_rate": 9.735194992778046e-05, + "loss": 2.6216, + "theoretical_loss": 3.476234060368079, + "tokens_seen": 1709768704 + }, + { + "epoch": 0.04, + "learning_rate": 9.734392553362222e-05, + "loss": 2.5491, + "theoretical_loss": 3.476211151083432, + "tokens_seen": 1709899776 + }, + { + "epoch": 0.04, + "learning_rate": 9.733590113946397e-05, + "loss": 2.5661, + "theoretical_loss": 3.476188244046491, + "tokens_seen": 1710030848 + }, + { + "epoch": 0.04, + "learning_rate": 9.732787674530574e-05, + "loss": 2.7132, + "theoretical_loss": 3.4761653392568634, + "tokens_seen": 1710161920 + }, + { + "epoch": 0.04, + "learning_rate": 9.731985235114749e-05, + "loss": 2.5686, + "theoretical_loss": 3.476142436714156, + "tokens_seen": 1710292992 + }, + { + "epoch": 0.04, + "learning_rate": 9.731182795698925e-05, + "loss": 2.7215, + "theoretical_loss": 3.4761195364179764, + "tokens_seen": 1710424064 + }, + { + "epoch": 0.04, + "learning_rate": 9.730380356283101e-05, + "loss": 2.4812, + "theoretical_loss": 3.476096638367932, + "tokens_seen": 1710555136 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 944008, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.545758008956909, + "objective/train/theoretical_loss": 3.4760851901850875, + "objective/train/tokens_used": 81079776, + "theoretical_loss": 3.4760851901850875, + "tokens_seen": 1710620672 + }, + { + "epoch": 0.04, + "learning_rate": 9.729577916867277e-05, + "loss": 2.5449, + "theoretical_loss": 3.4760737425636297, + "tokens_seen": 1710686208 + }, + { + "epoch": 0.04, + "learning_rate": 9.728775477451453e-05, + "loss": 2.7649, + "theoretical_loss": 3.476050849004679, + "tokens_seen": 1710817280 + }, + { + "epoch": 0.04, + "learning_rate": 9.72797303803563e-05, + "loss": 2.4776, + "theoretical_loss": 3.476027957690686, + "tokens_seen": 1710948352 + }, + { + "epoch": 0.04, + "learning_rate": 9.727170598619805e-05, + "loss": 2.4808, + "theoretical_loss": 3.4760050686212596, + "tokens_seen": 1711079424 + }, + { + "epoch": 0.04, + "learning_rate": 9.72636815920398e-05, + "loss": 2.5526, + "theoretical_loss": 3.475982181796007, + "tokens_seen": 1711210496 + }, + { + "epoch": 0.04, + "learning_rate": 9.725565719788157e-05, + "loss": 2.5859, + "theoretical_loss": 3.475959297214537, + "tokens_seen": 1711341568 + }, + { + "epoch": 0.04, + "learning_rate": 9.724763280372332e-05, + "loss": 2.6257, + "theoretical_loss": 3.4759364148764575, + "tokens_seen": 1711472640 + }, + { + "epoch": 0.04, + "learning_rate": 9.723960840956509e-05, + "loss": 2.7139, + "theoretical_loss": 3.4759135347813768, + "tokens_seen": 1711603712 + }, + { + "epoch": 0.04, + "learning_rate": 9.723158401540684e-05, + "loss": 2.6359, + "theoretical_loss": 3.4758906569289034, + "tokens_seen": 1711734784 + }, + { + "epoch": 0.04, + "learning_rate": 9.722355962124861e-05, + "loss": 2.6534, + "theoretical_loss": 3.475867781318646, + "tokens_seen": 1711865856 + }, + { + "epoch": 0.04, + "learning_rate": 9.721553522709036e-05, + "loss": 2.6935, + "theoretical_loss": 3.4758449079502123, + "tokens_seen": 1711996928 + }, + { + "epoch": 0.04, + "learning_rate": 9.720751083293211e-05, + "loss": 2.6434, + "theoretical_loss": 3.475822036823212, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 944514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2325708866119385, + "objective/train/theoretical_loss": 3.4757991679372533, + "objective/train/tokens_used": 82718176, + "theoretical_loss": 3.4757991679372533, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.04, + "learning_rate": 9.719948643877388e-05, + "loss": 2.3, + "theoretical_loss": 3.4757991679372533, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.04, + "learning_rate": 9.719146204461563e-05, + "loss": 2.7386, + "theoretical_loss": 3.4757763012919454, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.04, + "learning_rate": 9.71834376504574e-05, + "loss": 2.6512, + "theoretical_loss": 3.4757534368868974, + "tokens_seen": 1712521216 + }, + { + "epoch": 0.04, + "learning_rate": 9.717541325629915e-05, + "loss": 2.5087, + "theoretical_loss": 3.4757305747217178, + "tokens_seen": 1712652288 + }, + { + "epoch": 0.04, + "learning_rate": 9.716738886214092e-05, + "loss": 2.5342, + "theoretical_loss": 3.475707714796016, + "tokens_seen": 1712783360 + }, + { + "epoch": 0.04, + "learning_rate": 9.715936446798267e-05, + "loss": 2.4806, + "theoretical_loss": 3.4756848571094014, + "tokens_seen": 1712914432 + }, + { + "epoch": 0.04, + "learning_rate": 9.715134007382443e-05, + "loss": 2.6971, + "theoretical_loss": 3.475662001661483, + "tokens_seen": 1713045504 + }, + { + "epoch": 0.04, + "learning_rate": 9.714331567966619e-05, + "loss": 2.6806, + "theoretical_loss": 3.4756391484518714, + "tokens_seen": 1713176576 + }, + { + "epoch": 0.04, + "learning_rate": 9.713529128550795e-05, + "loss": 2.554, + "theoretical_loss": 3.475616297480175, + "tokens_seen": 1713307648 + }, + { + "epoch": 0.04, + "learning_rate": 9.712726689134971e-05, + "loss": 2.579, + "theoretical_loss": 3.4755934487460047, + "tokens_seen": 1713438720 + }, + { + "epoch": 0.04, + "learning_rate": 9.711924249719147e-05, + "loss": 2.6336, + "theoretical_loss": 3.4755706022489687, + "tokens_seen": 1713569792 + }, + { + "epoch": 0.04, + "learning_rate": 9.711121810303323e-05, + "loss": 2.4975, + "theoretical_loss": 3.4755477579886778, + "tokens_seen": 1713700864 + }, + { + "epoch": 0.04, + "learning_rate": 9.710319370887499e-05, + "loss": 2.5121, + "theoretical_loss": 3.4755249159647414, + "tokens_seen": 1713831936 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 945657, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.636183261871338, + "objective/train/theoretical_loss": 3.4755134957912848, + "objective/train/tokens_used": 84356576, + "theoretical_loss": 3.4755134957912848, + "tokens_seen": 1713897472 + }, + { + "epoch": 0.04, + "learning_rate": 9.709516931471674e-05, + "loss": 2.5273, + "theoretical_loss": 3.4755020761767703, + "tokens_seen": 1713963008 + }, + { + "epoch": 0.04, + "learning_rate": 9.70871449205585e-05, + "loss": 2.4317, + "theoretical_loss": 3.4754792386243745, + "tokens_seen": 1714094080 + }, + { + "epoch": 0.04, + "learning_rate": 9.707912052640026e-05, + "loss": 2.4492, + "theoretical_loss": 3.475456403307164, + "tokens_seen": 1714225152 + }, + { + "epoch": 0.04, + "learning_rate": 9.707109613224202e-05, + "loss": 2.6308, + "theoretical_loss": 3.475433570224749, + "tokens_seen": 1714356224 + }, + { + "epoch": 0.04, + "learning_rate": 9.706307173808378e-05, + "loss": 2.5065, + "theoretical_loss": 3.47541073937674, + "tokens_seen": 1714487296 + }, + { + "epoch": 0.04, + "learning_rate": 9.705504734392554e-05, + "loss": 2.4417, + "theoretical_loss": 3.4753879107627483, + "tokens_seen": 1714618368 + }, + { + "epoch": 0.04, + "learning_rate": 9.70470229497673e-05, + "loss": 2.5615, + "theoretical_loss": 3.4753650843823833, + "tokens_seen": 1714749440 + }, + { + "epoch": 0.04, + "learning_rate": 9.703899855560905e-05, + "loss": 2.5363, + "theoretical_loss": 3.475342260235257, + "tokens_seen": 1714880512 + }, + { + "epoch": 0.04, + "learning_rate": 9.703097416145082e-05, + "loss": 2.6402, + "theoretical_loss": 3.4753194383209793, + "tokens_seen": 1715011584 + }, + { + "epoch": 0.04, + "learning_rate": 9.702294976729257e-05, + "loss": 2.6409, + "theoretical_loss": 3.4752966186391614, + "tokens_seen": 1715142656 + }, + { + "epoch": 0.04, + "learning_rate": 9.701492537313434e-05, + "loss": 2.5318, + "theoretical_loss": 3.4752738011894144, + "tokens_seen": 1715273728 + }, + { + "epoch": 0.04, + "learning_rate": 9.700690097897609e-05, + "loss": 2.5512, + "theoretical_loss": 3.47525098597135, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 946326, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3408730030059814, + "objective/train/theoretical_loss": 3.4752281729845786, + "objective/train/tokens_used": 85994976, + "theoretical_loss": 3.4752281729845786, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.04, + "learning_rate": 9.699887658481786e-05, + "loss": 2.4938, + "theoretical_loss": 3.4752281729845786, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.04, + "learning_rate": 9.699085219065961e-05, + "loss": 2.5903, + "theoretical_loss": 3.475205362228712, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.04, + "learning_rate": 9.698282779650138e-05, + "loss": 2.6315, + "theoretical_loss": 3.4751825537033607, + "tokens_seen": 1715798016 + }, + { + "epoch": 0.04, + "learning_rate": 9.697480340234313e-05, + "loss": 2.5496, + "theoretical_loss": 3.4751597474081377, + "tokens_seen": 1715929088 + }, + { + "epoch": 0.04, + "learning_rate": 9.696677900818488e-05, + "loss": 2.5582, + "theoretical_loss": 3.4751369433426538, + "tokens_seen": 1716060160 + }, + { + "epoch": 0.04, + "learning_rate": 9.695875461402665e-05, + "loss": 2.4528, + "theoretical_loss": 3.4751141415065208, + "tokens_seen": 1716191232 + }, + { + "epoch": 0.04, + "learning_rate": 9.69507302198684e-05, + "loss": 2.4352, + "theoretical_loss": 3.4750913418993505, + "tokens_seen": 1716322304 + }, + { + "epoch": 0.04, + "learning_rate": 9.694270582571017e-05, + "loss": 2.4735, + "theoretical_loss": 3.475068544520755, + "tokens_seen": 1716453376 + }, + { + "epoch": 0.04, + "learning_rate": 9.693468143155192e-05, + "loss": 2.6383, + "theoretical_loss": 3.475045749370346, + "tokens_seen": 1716584448 + }, + { + "epoch": 0.04, + "learning_rate": 9.692665703739369e-05, + "loss": 2.6037, + "theoretical_loss": 3.475022956447736, + "tokens_seen": 1716715520 + }, + { + "epoch": 0.04, + "learning_rate": 9.691863264323544e-05, + "loss": 2.5211, + "theoretical_loss": 3.4750001657525367, + "tokens_seen": 1716846592 + }, + { + "epoch": 0.04, + "learning_rate": 9.69106082490772e-05, + "loss": 2.5279, + "theoretical_loss": 3.474977377284361, + "tokens_seen": 1716977664 + }, + { + "epoch": 0.04, + "learning_rate": 9.690258385491896e-05, + "loss": 2.471, + "theoretical_loss": 3.4749545910428212, + "tokens_seen": 1717108736 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 947520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7320938110351562, + "objective/train/theoretical_loss": 3.474943198756918, + "objective/train/tokens_used": 87633376, + "theoretical_loss": 3.474943198756918, + "tokens_seen": 1717174272 + }, + { + "epoch": 0.04, + "learning_rate": 9.689455946076072e-05, + "loss": 2.5489, + "theoretical_loss": 3.474931807027529, + "tokens_seen": 1717239808 + }, + { + "epoch": 0.04, + "learning_rate": 9.688653506660248e-05, + "loss": 2.3712, + "theoretical_loss": 3.474909025238098, + "tokens_seen": 1717370880 + }, + { + "epoch": 0.04, + "learning_rate": 9.687851067244424e-05, + "loss": 2.5661, + "theoretical_loss": 3.4748862456741403, + "tokens_seen": 1717501952 + }, + { + "epoch": 0.04, + "learning_rate": 9.6870486278286e-05, + "loss": 2.6223, + "theoretical_loss": 3.474863468335269, + "tokens_seen": 1717633024 + }, + { + "epoch": 0.04, + "learning_rate": 9.686246188412776e-05, + "loss": 2.6434, + "theoretical_loss": 3.474840693221096, + "tokens_seen": 1717764096 + }, + { + "epoch": 0.04, + "learning_rate": 9.685443748996951e-05, + "loss": 2.6146, + "theoretical_loss": 3.474817920331236, + "tokens_seen": 1717895168 + }, + { + "epoch": 0.04, + "learning_rate": 9.684641309581128e-05, + "loss": 2.5673, + "theoretical_loss": 3.474795149665301, + "tokens_seen": 1718026240 + }, + { + "epoch": 0.04, + "learning_rate": 9.683838870165303e-05, + "loss": 2.5127, + "theoretical_loss": 3.4747723812229045, + "tokens_seen": 1718157312 + }, + { + "epoch": 0.04, + "learning_rate": 9.68303643074948e-05, + "loss": 2.5503, + "theoretical_loss": 3.4747496150036596, + "tokens_seen": 1718288384 + }, + { + "epoch": 0.04, + "learning_rate": 9.682233991333655e-05, + "loss": 2.6373, + "theoretical_loss": 3.47472685100718, + "tokens_seen": 1718419456 + }, + { + "epoch": 0.04, + "learning_rate": 9.681431551917831e-05, + "loss": 2.5278, + "theoretical_loss": 3.4747040892330787, + "tokens_seen": 1718550528 + }, + { + "epoch": 0.04, + "learning_rate": 9.680629112502007e-05, + "loss": 2.5697, + "theoretical_loss": 3.474681329680969, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 948221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1624414920806885, + "objective/train/theoretical_loss": 3.4746585723504655, + "objective/train/tokens_used": 89271776, + "theoretical_loss": 3.4746585723504655, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.04, + "learning_rate": 9.679826673086182e-05, + "loss": 2.4717, + "theoretical_loss": 3.4746585723504655, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.04, + "learning_rate": 9.679024233670359e-05, + "loss": 2.5336, + "theoretical_loss": 3.4746358172411815, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 9.678221794254534e-05, + "loss": 2.5264, + "theoretical_loss": 3.4746130643527304, + "tokens_seen": 1719074816 + }, + { + "epoch": 0.04, + "learning_rate": 9.677419354838711e-05, + "loss": 2.598, + "theoretical_loss": 3.474590313684727, + "tokens_seen": 1719205888 + }, + { + "epoch": 0.04, + "learning_rate": 9.676616915422886e-05, + "loss": 2.492, + "theoretical_loss": 3.474567565236785, + "tokens_seen": 1719336960 + }, + { + "epoch": 0.04, + "learning_rate": 9.675814476007063e-05, + "loss": 2.4323, + "theoretical_loss": 3.4745448190085177, + "tokens_seen": 1719468032 + }, + { + "epoch": 0.04, + "learning_rate": 9.675012036591238e-05, + "loss": 2.5456, + "theoretical_loss": 3.4745220749995407, + "tokens_seen": 1719599104 + }, + { + "epoch": 0.04, + "learning_rate": 9.674209597175413e-05, + "loss": 2.4039, + "theoretical_loss": 3.474499333209468, + "tokens_seen": 1719730176 + }, + { + "epoch": 0.04, + "learning_rate": 9.67340715775959e-05, + "loss": 2.4198, + "theoretical_loss": 3.474476593637913, + "tokens_seen": 1719861248 + }, + { + "epoch": 0.04, + "learning_rate": 9.672604718343765e-05, + "loss": 2.6933, + "theoretical_loss": 3.4744538562844913, + "tokens_seen": 1719992320 + }, + { + "epoch": 0.04, + "learning_rate": 9.671802278927942e-05, + "loss": 2.5493, + "theoretical_loss": 3.4744311211488172, + "tokens_seen": 1720123392 + }, + { + "epoch": 0.04, + "learning_rate": 9.670999839512117e-05, + "loss": 2.6969, + "theoretical_loss": 3.4744083882305055, + "tokens_seen": 1720254464 + }, + { + "epoch": 0.04, + "learning_rate": 9.670197400096294e-05, + "loss": 2.4579, + "theoretical_loss": 3.474385657529171, + "tokens_seen": 1720385536 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 949201, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1461188793182373, + "objective/train/theoretical_loss": 3.47437429300975, + "objective/train/tokens_used": 90910176, + "theoretical_loss": 3.47437429300975, + "tokens_seen": 1720451072 + }, + { + "epoch": 0.04, + "learning_rate": 9.669394960680469e-05, + "loss": 2.4628, + "theoretical_loss": 3.4743629290444282, + "tokens_seen": 1720516608 + }, + { + "epoch": 0.04, + "learning_rate": 9.668592521264645e-05, + "loss": 2.4322, + "theoretical_loss": 3.474340202775893, + "tokens_seen": 1720647680 + }, + { + "epoch": 0.04, + "learning_rate": 9.667790081848821e-05, + "loss": 2.5498, + "theoretical_loss": 3.47431747872318, + "tokens_seen": 1720778752 + }, + { + "epoch": 0.04, + "learning_rate": 9.666987642432997e-05, + "loss": 2.5226, + "theoretical_loss": 3.474294756885904, + "tokens_seen": 1720909824 + }, + { + "epoch": 0.04, + "learning_rate": 9.666185203017173e-05, + "loss": 2.4954, + "theoretical_loss": 3.474272037263681, + "tokens_seen": 1721040896 + }, + { + "epoch": 0.04, + "learning_rate": 9.665382763601349e-05, + "loss": 2.6002, + "theoretical_loss": 3.474249319856126, + "tokens_seen": 1721171968 + }, + { + "epoch": 0.04, + "learning_rate": 9.664580324185525e-05, + "loss": 2.4917, + "theoretical_loss": 3.4742266046628543, + "tokens_seen": 1721303040 + }, + { + "epoch": 0.04, + "learning_rate": 9.6637778847697e-05, + "loss": 2.479, + "theoretical_loss": 3.474203891683482, + "tokens_seen": 1721434112 + }, + { + "epoch": 0.04, + "learning_rate": 9.662975445353877e-05, + "loss": 2.479, + "theoretical_loss": 3.474181180917624, + "tokens_seen": 1721565184 + }, + { + "epoch": 0.04, + "learning_rate": 9.662173005938053e-05, + "loss": 2.717, + "theoretical_loss": 3.4741584723648975, + "tokens_seen": 1721696256 + }, + { + "epoch": 0.04, + "learning_rate": 9.661370566522228e-05, + "loss": 2.6332, + "theoretical_loss": 3.474135766024917, + "tokens_seen": 1721827328 + }, + { + "epoch": 0.04, + "learning_rate": 9.660568127106405e-05, + "loss": 2.6836, + "theoretical_loss": 3.474113061897299, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 949909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.497939348220825, + "objective/train/theoretical_loss": 3.474090359981659, + "objective/train/tokens_used": 92548576, + "theoretical_loss": 3.474090359981659, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 9.65976568769058e-05, + "loss": 2.5073, + "theoretical_loss": 3.474090359981659, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 9.658963248274757e-05, + "loss": 2.4187, + "theoretical_loss": 3.4740676602776146, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 9.658160808858932e-05, + "loss": 2.5354, + "theoretical_loss": 3.4740449627847805, + "tokens_seen": 1722351616 + }, + { + "epoch": 0.04, + "learning_rate": 9.657358369443108e-05, + "loss": 2.428, + "theoretical_loss": 3.4740222675027734, + "tokens_seen": 1722482688 + }, + { + "epoch": 0.04, + "learning_rate": 9.656555930027284e-05, + "loss": 2.5802, + "theoretical_loss": 3.4739995744312107, + "tokens_seen": 1722613760 + }, + { + "epoch": 0.04, + "learning_rate": 9.655753490611459e-05, + "loss": 2.6047, + "theoretical_loss": 3.473976883569708, + "tokens_seen": 1722744832 + }, + { + "epoch": 0.04, + "learning_rate": 9.654951051195636e-05, + "loss": 2.6242, + "theoretical_loss": 3.4739541949178827, + "tokens_seen": 1722875904 + }, + { + "epoch": 0.04, + "learning_rate": 9.654148611779811e-05, + "loss": 2.4371, + "theoretical_loss": 3.47393150847535, + "tokens_seen": 1723006976 + }, + { + "epoch": 0.04, + "learning_rate": 9.653346172363988e-05, + "loss": 2.3743, + "theoretical_loss": 3.473908824241729, + "tokens_seen": 1723138048 + }, + { + "epoch": 0.04, + "learning_rate": 9.652543732948163e-05, + "loss": 2.7518, + "theoretical_loss": 3.4738861422166343, + "tokens_seen": 1723269120 + }, + { + "epoch": 0.04, + "learning_rate": 9.65174129353234e-05, + "loss": 2.44, + "theoretical_loss": 3.4738634623996845, + "tokens_seen": 1723400192 + }, + { + "epoch": 0.04, + "learning_rate": 9.650938854116515e-05, + "loss": 2.5699, + "theoretical_loss": 3.4738407847904966, + "tokens_seen": 1723531264 + }, + { + "epoch": 0.04, + "learning_rate": 9.65013641470069e-05, + "loss": 2.4851, + "theoretical_loss": 3.473818109388687, + "tokens_seen": 1723662336 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 951235, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2583699226379395, + "objective/train/theoretical_loss": 3.47380677251543, + "objective/train/tokens_used": 94186976, + "theoretical_loss": 3.47380677251543, + "tokens_seen": 1723727872 + }, + { + "epoch": 0.04, + "learning_rate": 9.649333975284867e-05, + "loss": 2.5272, + "theoretical_loss": 3.473795436193874, + "tokens_seen": 1723793408 + }, + { + "epoch": 0.04, + "learning_rate": 9.648531535869042e-05, + "loss": 2.5135, + "theoretical_loss": 3.4737727652056742, + "tokens_seen": 1723924480 + }, + { + "epoch": 0.04, + "learning_rate": 9.647729096453219e-05, + "loss": 2.6295, + "theoretical_loss": 3.473750096423705, + "tokens_seen": 1724055552 + }, + { + "epoch": 0.04, + "learning_rate": 9.646926657037394e-05, + "loss": 2.6414, + "theoretical_loss": 3.4737274298475844, + "tokens_seen": 1724186624 + }, + { + "epoch": 0.05, + "learning_rate": 9.646124217621571e-05, + "loss": 2.5256, + "theoretical_loss": 3.4737047654769304, + "tokens_seen": 1724317696 + }, + { + "epoch": 0.05, + "learning_rate": 9.645321778205746e-05, + "loss": 2.664, + "theoretical_loss": 3.4736821033113605, + "tokens_seen": 1724448768 + }, + { + "epoch": 0.05, + "learning_rate": 9.644519338789922e-05, + "loss": 2.5327, + "theoretical_loss": 3.473659443350493, + "tokens_seen": 1724579840 + }, + { + "epoch": 0.05, + "learning_rate": 9.643716899374098e-05, + "loss": 2.6298, + "theoretical_loss": 3.4736367855939445, + "tokens_seen": 1724710912 + }, + { + "epoch": 0.05, + "learning_rate": 9.642914459958274e-05, + "loss": 2.5402, + "theoretical_loss": 3.4736141300413337, + "tokens_seen": 1724841984 + }, + { + "epoch": 0.05, + "learning_rate": 9.64211202054245e-05, + "loss": 2.4692, + "theoretical_loss": 3.47359147669228, + "tokens_seen": 1724973056 + }, + { + "epoch": 0.05, + "learning_rate": 9.641309581126626e-05, + "loss": 2.6825, + "theoretical_loss": 3.4735688255464003, + "tokens_seen": 1725104128 + }, + { + "epoch": 0.05, + "learning_rate": 9.640507141710802e-05, + "loss": 2.4039, + "theoretical_loss": 3.4735461766033136, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 951881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6248931884765625, + "objective/train/theoretical_loss": 3.4735235298626375, + "objective/train/tokens_used": 95825376, + "theoretical_loss": 3.4735235298626375, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.05, + "learning_rate": 9.639704702294978e-05, + "loss": 2.55, + "theoretical_loss": 3.4735235298626375, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.05, + "learning_rate": 9.638902262879153e-05, + "loss": 2.3494, + "theoretical_loss": 3.473500885323992, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.05, + "learning_rate": 9.63809982346333e-05, + "loss": 2.6021, + "theoretical_loss": 3.473478242986994, + "tokens_seen": 1725628416 + }, + { + "epoch": 0.05, + "learning_rate": 9.637297384047505e-05, + "loss": 2.4924, + "theoretical_loss": 3.473455602851264, + "tokens_seen": 1725759488 + }, + { + "epoch": 0.05, + "learning_rate": 9.636494944631682e-05, + "loss": 2.4347, + "theoretical_loss": 3.4734329649164195, + "tokens_seen": 1725890560 + }, + { + "epoch": 0.05, + "learning_rate": 9.635692505215857e-05, + "loss": 2.3637, + "theoretical_loss": 3.47341032918208, + "tokens_seen": 1726021632 + }, + { + "epoch": 0.05, + "learning_rate": 9.634890065800034e-05, + "loss": 2.5177, + "theoretical_loss": 3.4733876956478644, + "tokens_seen": 1726152704 + }, + { + "epoch": 0.05, + "learning_rate": 9.634087626384209e-05, + "loss": 2.5704, + "theoretical_loss": 3.473365064313392, + "tokens_seen": 1726283776 + }, + { + "epoch": 0.05, + "learning_rate": 9.633285186968385e-05, + "loss": 2.6099, + "theoretical_loss": 3.4733424351782816, + "tokens_seen": 1726414848 + }, + { + "epoch": 0.05, + "learning_rate": 9.632482747552561e-05, + "loss": 2.6702, + "theoretical_loss": 3.4733198082421533, + "tokens_seen": 1726545920 + }, + { + "epoch": 0.05, + "learning_rate": 9.631680308136736e-05, + "loss": 2.5298, + "theoretical_loss": 3.4732971835046254, + "tokens_seen": 1726676992 + }, + { + "epoch": 0.05, + "learning_rate": 9.630877868720913e-05, + "loss": 2.4978, + "theoretical_loss": 3.4732745609653177, + "tokens_seen": 1726808064 + }, + { + "epoch": 0.05, + "learning_rate": 9.630075429305088e-05, + "loss": 2.4702, + "theoretical_loss": 3.4732519406238507, + "tokens_seen": 1726939136 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 953135, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.060582160949707, + "objective/train/theoretical_loss": 3.4732406312771884, + "objective/train/tokens_used": 97463776, + "theoretical_loss": 3.4732406312771884, + "tokens_seen": 1727004672 + }, + { + "epoch": 0.05, + "learning_rate": 9.629272989889265e-05, + "loss": 2.5876, + "theoretical_loss": 3.473229322479843, + "tokens_seen": 1727070208 + }, + { + "epoch": 0.05, + "learning_rate": 9.62847055047344e-05, + "loss": 2.6043, + "theoretical_loss": 3.473206706532915, + "tokens_seen": 1727201280 + }, + { + "epoch": 0.05, + "learning_rate": 9.627668111057617e-05, + "loss": 2.5915, + "theoretical_loss": 3.4731840927826867, + "tokens_seen": 1727332352 + }, + { + "epoch": 0.05, + "learning_rate": 9.626865671641792e-05, + "loss": 2.4664, + "theoretical_loss": 3.4731614812287774, + "tokens_seen": 1727463424 + }, + { + "epoch": 0.05, + "learning_rate": 9.626063232225967e-05, + "loss": 2.5555, + "theoretical_loss": 3.4731388718708076, + "tokens_seen": 1727594496 + }, + { + "epoch": 0.05, + "learning_rate": 9.625260792810144e-05, + "loss": 2.5768, + "theoretical_loss": 3.4731162647083975, + "tokens_seen": 1727725568 + }, + { + "epoch": 0.05, + "learning_rate": 9.62445835339432e-05, + "loss": 2.6285, + "theoretical_loss": 3.473093659741167, + "tokens_seen": 1727856640 + }, + { + "epoch": 0.05, + "learning_rate": 9.623655913978496e-05, + "loss": 2.6888, + "theoretical_loss": 3.473071056968737, + "tokens_seen": 1727987712 + }, + { + "epoch": 0.05, + "learning_rate": 9.622853474562671e-05, + "loss": 2.6184, + "theoretical_loss": 3.4730484563907273, + "tokens_seen": 1728118784 + }, + { + "epoch": 0.05, + "learning_rate": 9.622051035146848e-05, + "loss": 2.6123, + "theoretical_loss": 3.473025858006759, + "tokens_seen": 1728249856 + }, + { + "epoch": 0.05, + "learning_rate": 9.621248595731023e-05, + "loss": 2.4683, + "theoretical_loss": 3.4730032618164524, + "tokens_seen": 1728380928 + }, + { + "epoch": 0.05, + "learning_rate": 9.620446156315199e-05, + "loss": 2.4986, + "theoretical_loss": 3.472980667819428, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 953685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4338219165802, + "objective/train/theoretical_loss": 3.472958076015307, + "objective/train/tokens_used": 99102176, + "theoretical_loss": 3.472958076015307, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.05, + "learning_rate": 9.619643716899375e-05, + "loss": 2.6793, + "theoretical_loss": 3.472958076015307, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.05, + "learning_rate": 9.61884127748355e-05, + "loss": 2.3993, + "theoretical_loss": 3.4729354864037107, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.05, + "learning_rate": 9.618038838067727e-05, + "loss": 2.5727, + "theoretical_loss": 3.4729128989842595, + "tokens_seen": 1728905216 + }, + { + "epoch": 0.05, + "learning_rate": 9.617236398651903e-05, + "loss": 2.5781, + "theoretical_loss": 3.4728903137565745, + "tokens_seen": 1729036288 + }, + { + "epoch": 0.05, + "learning_rate": 9.616433959236079e-05, + "loss": 2.5062, + "theoretical_loss": 3.472867730720277, + "tokens_seen": 1729167360 + }, + { + "epoch": 0.05, + "learning_rate": 9.615631519820255e-05, + "loss": 2.6331, + "theoretical_loss": 3.4728451498749884, + "tokens_seen": 1729298432 + }, + { + "epoch": 0.05, + "learning_rate": 9.61482908040443e-05, + "loss": 2.6771, + "theoretical_loss": 3.4728225712203304, + "tokens_seen": 1729429504 + }, + { + "epoch": 0.05, + "learning_rate": 9.614026640988607e-05, + "loss": 2.562, + "theoretical_loss": 3.472799994755924, + "tokens_seen": 1729560576 + }, + { + "epoch": 0.05, + "learning_rate": 9.613224201572782e-05, + "loss": 2.5274, + "theoretical_loss": 3.4727774204813904, + "tokens_seen": 1729691648 + }, + { + "epoch": 0.05, + "learning_rate": 9.612421762156959e-05, + "loss": 2.4391, + "theoretical_loss": 3.472754848396352, + "tokens_seen": 1729822720 + }, + { + "epoch": 0.05, + "learning_rate": 9.611619322741134e-05, + "loss": 2.5883, + "theoretical_loss": 3.4727322785004304, + "tokens_seen": 1729953792 + }, + { + "epoch": 0.05, + "learning_rate": 9.610816883325309e-05, + "loss": 2.7006, + "theoretical_loss": 3.4727097107932474, + "tokens_seen": 1730084864 + }, + { + "epoch": 0.05, + "learning_rate": 9.610014443909486e-05, + "loss": 2.5706, + "theoretical_loss": 3.472687145274425, + "tokens_seen": 1730215936 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 954893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3295490741729736, + "objective/train/theoretical_loss": 3.472675863335531, + "objective/train/tokens_used": 100740576, + "theoretical_loss": 3.472675863335531, + "tokens_seen": 1730281472 + }, + { + "epoch": 0.05, + "learning_rate": 9.609212004493661e-05, + "loss": 2.5245, + "theoretical_loss": 3.472664581943585, + "tokens_seen": 1730347008 + }, + { + "epoch": 0.05, + "learning_rate": 9.608409565077838e-05, + "loss": 2.5552, + "theoretical_loss": 3.4726420208003494, + "tokens_seen": 1730478080 + }, + { + "epoch": 0.05, + "learning_rate": 9.607607125662013e-05, + "loss": 2.5185, + "theoretical_loss": 3.4726194618443413, + "tokens_seen": 1730609152 + }, + { + "epoch": 0.05, + "learning_rate": 9.60680468624619e-05, + "loss": 2.53, + "theoretical_loss": 3.472596905075182, + "tokens_seen": 1730740224 + }, + { + "epoch": 0.05, + "learning_rate": 9.606002246830365e-05, + "loss": 2.6368, + "theoretical_loss": 3.472574350492495, + "tokens_seen": 1730871296 + }, + { + "epoch": 0.05, + "learning_rate": 9.60519980741454e-05, + "loss": 2.4409, + "theoretical_loss": 3.4725517980959015, + "tokens_seen": 1731002368 + }, + { + "epoch": 0.05, + "learning_rate": 9.604397367998717e-05, + "loss": 2.6554, + "theoretical_loss": 3.472529247885025, + "tokens_seen": 1731133440 + }, + { + "epoch": 0.05, + "learning_rate": 9.603594928582892e-05, + "loss": 2.6305, + "theoretical_loss": 3.4725066998594882, + "tokens_seen": 1731264512 + }, + { + "epoch": 0.05, + "learning_rate": 9.602792489167069e-05, + "loss": 2.5356, + "theoretical_loss": 3.4724841540189137, + "tokens_seen": 1731395584 + }, + { + "epoch": 0.05, + "learning_rate": 9.601990049751244e-05, + "loss": 2.5259, + "theoretical_loss": 3.4724616103629242, + "tokens_seen": 1731526656 + }, + { + "epoch": 0.05, + "learning_rate": 9.60118761033542e-05, + "loss": 2.5755, + "theoretical_loss": 3.472439068891143, + "tokens_seen": 1731657728 + }, + { + "epoch": 0.05, + "learning_rate": 9.600385170919596e-05, + "loss": 2.6055, + "theoretical_loss": 3.4724165296031924, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 956184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5680079460144043, + "objective/train/theoretical_loss": 3.472393992498697, + "objective/train/tokens_used": 102378976, + "theoretical_loss": 3.472393992498697, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.05, + "learning_rate": 9.599582731503772e-05, + "loss": 2.6725, + "theoretical_loss": 3.472393992498697, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.05, + "learning_rate": 9.598780292087948e-05, + "loss": 2.5937, + "theoretical_loss": 3.472371457577279, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.05, + "learning_rate": 9.597977852672124e-05, + "loss": 2.613, + "theoretical_loss": 3.472348924838562, + "tokens_seen": 1732182016 + }, + { + "epoch": 0.05, + "learning_rate": 9.5971754132563e-05, + "loss": 2.5003, + "theoretical_loss": 3.4723263942821694, + "tokens_seen": 1732313088 + }, + { + "epoch": 0.05, + "learning_rate": 9.596372973840476e-05, + "loss": 2.5422, + "theoretical_loss": 3.4723038659077248, + "tokens_seen": 1732444160 + }, + { + "epoch": 0.05, + "learning_rate": 9.595570534424651e-05, + "loss": 2.5334, + "theoretical_loss": 3.4722813397148515, + "tokens_seen": 1732575232 + }, + { + "epoch": 0.05, + "learning_rate": 9.594768095008828e-05, + "loss": 2.4992, + "theoretical_loss": 3.4722588157031744, + "tokens_seen": 1732706304 + }, + { + "epoch": 0.05, + "learning_rate": 9.593965655593003e-05, + "loss": 2.5993, + "theoretical_loss": 3.4722362938723155, + "tokens_seen": 1732837376 + }, + { + "epoch": 0.05, + "learning_rate": 9.59316321617718e-05, + "loss": 2.5522, + "theoretical_loss": 3.472213774221901, + "tokens_seen": 1732968448 + }, + { + "epoch": 0.05, + "learning_rate": 9.592360776761355e-05, + "loss": 2.5903, + "theoretical_loss": 3.472191256751552, + "tokens_seen": 1733099520 + }, + { + "epoch": 0.05, + "learning_rate": 9.59155833734553e-05, + "loss": 2.55, + "theoretical_loss": 3.4721687414608953, + "tokens_seen": 1733230592 + }, + { + "epoch": 0.05, + "learning_rate": 9.590755897929707e-05, + "loss": 2.6655, + "theoretical_loss": 3.472146228349554, + "tokens_seen": 1733361664 + }, + { + "epoch": 0.05, + "learning_rate": 9.589953458513882e-05, + "loss": 2.5992, + "theoretical_loss": 3.472123717417152, + "tokens_seen": 1733492736 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 956719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.863294839859009, + "objective/train/theoretical_loss": 3.472112462767936, + "objective/train/tokens_used": 104017376, + "theoretical_loss": 3.472112462767936, + "tokens_seen": 1733558272 + }, + { + "epoch": 0.05, + "learning_rate": 9.589151019098059e-05, + "loss": 2.592, + "theoretical_loss": 3.4721012086633145, + "tokens_seen": 1733623808 + }, + { + "epoch": 0.05, + "learning_rate": 9.588348579682234e-05, + "loss": 2.6002, + "theoretical_loss": 3.472078702087665, + "tokens_seen": 1733754880 + }, + { + "epoch": 0.05, + "learning_rate": 9.587546140266411e-05, + "loss": 2.396, + "theoretical_loss": 3.4720561976898288, + "tokens_seen": 1733885952 + }, + { + "epoch": 0.05, + "learning_rate": 9.586743700850586e-05, + "loss": 2.5257, + "theoretical_loss": 3.47203369546943, + "tokens_seen": 1734017024 + }, + { + "epoch": 0.05, + "learning_rate": 9.585941261434761e-05, + "loss": 2.2733, + "theoretical_loss": 3.472011195426095, + "tokens_seen": 1734148096 + }, + { + "epoch": 0.05, + "learning_rate": 9.585138822018938e-05, + "loss": 2.5829, + "theoretical_loss": 3.4719886975594463, + "tokens_seen": 1734279168 + }, + { + "epoch": 0.05, + "learning_rate": 9.584336382603113e-05, + "loss": 2.5333, + "theoretical_loss": 3.47196620186911, + "tokens_seen": 1734410240 + }, + { + "epoch": 0.05, + "learning_rate": 9.58353394318729e-05, + "loss": 2.6064, + "theoretical_loss": 3.4719437083547113, + "tokens_seen": 1734541312 + }, + { + "epoch": 0.05, + "learning_rate": 9.582731503771465e-05, + "loss": 2.5828, + "theoretical_loss": 3.4719212170158755, + "tokens_seen": 1734672384 + }, + { + "epoch": 0.05, + "learning_rate": 9.581929064355641e-05, + "loss": 2.5385, + "theoretical_loss": 3.471898727852227, + "tokens_seen": 1734803456 + }, + { + "epoch": 0.05, + "learning_rate": 9.581126624939817e-05, + "loss": 2.5934, + "theoretical_loss": 3.4718762408633914, + "tokens_seen": 1734934528 + }, + { + "epoch": 0.05, + "learning_rate": 9.580324185523993e-05, + "loss": 2.6143, + "theoretical_loss": 3.471853756048994, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 957813, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.456376314163208, + "objective/train/theoretical_loss": 3.471831273408661, + "objective/train/tokens_used": 105655776, + "theoretical_loss": 3.471831273408661, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.05, + "learning_rate": 9.57952174610817e-05, + "loss": 2.5648, + "theoretical_loss": 3.471831273408661, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.05, + "learning_rate": 9.578719306692345e-05, + "loss": 2.6123, + "theoretical_loss": 3.471808792942017, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.05, + "learning_rate": 9.57791686727652e-05, + "loss": 2.5439, + "theoretical_loss": 3.4717863146486887, + "tokens_seen": 1735458816 + }, + { + "epoch": 0.05, + "learning_rate": 9.577114427860697e-05, + "loss": 2.5435, + "theoretical_loss": 3.471763838528301, + "tokens_seen": 1735589888 + }, + { + "epoch": 0.05, + "learning_rate": 9.576311988444872e-05, + "loss": 2.6144, + "theoretical_loss": 3.47174136458048, + "tokens_seen": 1735720960 + }, + { + "epoch": 0.05, + "learning_rate": 9.575509549029049e-05, + "loss": 2.6676, + "theoretical_loss": 3.4717188928048524, + "tokens_seen": 1735852032 + }, + { + "epoch": 0.05, + "learning_rate": 9.574707109613224e-05, + "loss": 2.6404, + "theoretical_loss": 3.471696423201043, + "tokens_seen": 1735983104 + }, + { + "epoch": 0.05, + "learning_rate": 9.5739046701974e-05, + "loss": 2.42, + "theoretical_loss": 3.4716739557686793, + "tokens_seen": 1736114176 + }, + { + "epoch": 0.05, + "learning_rate": 9.573102230781576e-05, + "loss": 2.5927, + "theoretical_loss": 3.4716514905073863, + "tokens_seen": 1736245248 + }, + { + "epoch": 0.05, + "learning_rate": 9.572299791365751e-05, + "loss": 2.5552, + "theoretical_loss": 3.4716290274167907, + "tokens_seen": 1736376320 + }, + { + "epoch": 0.05, + "learning_rate": 9.571497351949928e-05, + "loss": 2.5069, + "theoretical_loss": 3.4716065664965194, + "tokens_seen": 1736507392 + }, + { + "epoch": 0.05, + "learning_rate": 9.570694912534103e-05, + "loss": 2.6358, + "theoretical_loss": 3.4715841077461986, + "tokens_seen": 1736638464 + }, + { + "epoch": 0.05, + "learning_rate": 9.56989247311828e-05, + "loss": 2.5779, + "theoretical_loss": 3.4715616511654552, + "tokens_seen": 1736769536 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 958321, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.462451934814453, + "objective/train/theoretical_loss": 3.4715504236885577, + "objective/train/tokens_used": 107294176, + "theoretical_loss": 3.4715504236885577, + "tokens_seen": 1736835072 + }, + { + "epoch": 0.05, + "learning_rate": 9.569090033702455e-05, + "loss": 2.5436, + "theoretical_loss": 3.4715391967539153, + "tokens_seen": 1736900608 + }, + { + "epoch": 0.05, + "learning_rate": 9.568287594286632e-05, + "loss": 2.7413, + "theoretical_loss": 3.471516744511206, + "tokens_seen": 1737031680 + }, + { + "epoch": 0.05, + "learning_rate": 9.567485154870807e-05, + "loss": 2.4977, + "theoretical_loss": 3.471494294436954, + "tokens_seen": 1737162752 + }, + { + "epoch": 0.05, + "learning_rate": 9.566682715454983e-05, + "loss": 2.4846, + "theoretical_loss": 3.471471846530787, + "tokens_seen": 1737293824 + }, + { + "epoch": 0.05, + "learning_rate": 9.565880276039159e-05, + "loss": 2.5388, + "theoretical_loss": 3.4714494007923307, + "tokens_seen": 1737424896 + }, + { + "epoch": 0.05, + "learning_rate": 9.565077836623335e-05, + "loss": 2.6694, + "theoretical_loss": 3.4714269572212135, + "tokens_seen": 1737555968 + }, + { + "epoch": 0.05, + "learning_rate": 9.564275397207511e-05, + "loss": 2.5805, + "theoretical_loss": 3.4714045158170626, + "tokens_seen": 1737687040 + }, + { + "epoch": 0.05, + "learning_rate": 9.563472957791687e-05, + "loss": 2.4512, + "theoretical_loss": 3.4713820765795047, + "tokens_seen": 1737818112 + }, + { + "epoch": 0.05, + "learning_rate": 9.562670518375863e-05, + "loss": 2.4375, + "theoretical_loss": 3.471359639508168, + "tokens_seen": 1737949184 + }, + { + "epoch": 0.05, + "learning_rate": 9.561868078960038e-05, + "loss": 2.5466, + "theoretical_loss": 3.471337204602679, + "tokens_seen": 1738080256 + }, + { + "epoch": 0.05, + "learning_rate": 9.561065639544214e-05, + "loss": 2.6156, + "theoretical_loss": 3.471314771862666, + "tokens_seen": 1738211328 + }, + { + "epoch": 0.05, + "learning_rate": 9.56026320012839e-05, + "loss": 2.7741, + "theoretical_loss": 3.4712923412877563, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 959398, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5910706520080566, + "objective/train/theoretical_loss": 3.4712699128775784, + "objective/train/tokens_used": 108932576, + "theoretical_loss": 3.4712699128775784, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.05, + "learning_rate": 9.559460760712566e-05, + "loss": 2.5787, + "theoretical_loss": 3.4712699128775784, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.05, + "learning_rate": 9.558658321296742e-05, + "loss": 2.7085, + "theoretical_loss": 3.4712474866317597, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 9.557855881880918e-05, + "loss": 2.5791, + "theoretical_loss": 3.4712250625499284, + "tokens_seen": 1738735616 + }, + { + "epoch": 0.05, + "learning_rate": 9.557053442465094e-05, + "loss": 2.5154, + "theoretical_loss": 3.471202640631713, + "tokens_seen": 1738866688 + }, + { + "epoch": 0.05, + "learning_rate": 9.55625100304927e-05, + "loss": 2.6328, + "theoretical_loss": 3.4711802208767404, + "tokens_seen": 1738997760 + }, + { + "epoch": 0.05, + "learning_rate": 9.555448563633445e-05, + "loss": 2.5779, + "theoretical_loss": 3.4711578032846404, + "tokens_seen": 1739128832 + }, + { + "epoch": 0.05, + "learning_rate": 9.554646124217622e-05, + "loss": 2.4909, + "theoretical_loss": 3.47113538785504, + "tokens_seen": 1739259904 + }, + { + "epoch": 0.05, + "learning_rate": 9.553843684801797e-05, + "loss": 2.6613, + "theoretical_loss": 3.471112974587568, + "tokens_seen": 1739390976 + }, + { + "epoch": 0.05, + "learning_rate": 9.553041245385974e-05, + "loss": 2.647, + "theoretical_loss": 3.471090563481854, + "tokens_seen": 1739522048 + }, + { + "epoch": 0.05, + "learning_rate": 9.552238805970149e-05, + "loss": 2.5482, + "theoretical_loss": 3.4710681545375253, + "tokens_seen": 1739653120 + }, + { + "epoch": 0.05, + "learning_rate": 9.551436366554326e-05, + "loss": 2.6415, + "theoretical_loss": 3.471045747754211, + "tokens_seen": 1739784192 + }, + { + "epoch": 0.05, + "learning_rate": 9.550633927138501e-05, + "loss": 2.5234, + "theoretical_loss": 3.4710233431315407, + "tokens_seen": 1739915264 + }, + { + "epoch": 0.05, + "learning_rate": 9.549831487722676e-05, + "loss": 2.4932, + "theoretical_loss": 3.471000940669142, + "tokens_seen": 1740046336 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 960697, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7677383422851562, + "objective/train/theoretical_loss": 3.470989740247929, + "objective/train/tokens_used": 110570976, + "theoretical_loss": 3.470989740247929, + "tokens_seen": 1740111872 + }, + { + "epoch": 0.05, + "learning_rate": 9.549029048306853e-05, + "loss": 2.5452, + "theoretical_loss": 3.4709785403666453, + "tokens_seen": 1740177408 + }, + { + "epoch": 0.05, + "learning_rate": 9.548226608891028e-05, + "loss": 2.5708, + "theoretical_loss": 3.4709561422236783, + "tokens_seen": 1740308480 + }, + { + "epoch": 0.05, + "learning_rate": 9.547424169475205e-05, + "loss": 2.7149, + "theoretical_loss": 3.4709337462398713, + "tokens_seen": 1740439552 + }, + { + "epoch": 0.05, + "learning_rate": 9.54662173005938e-05, + "loss": 2.5015, + "theoretical_loss": 3.470911352414853, + "tokens_seen": 1740570624 + }, + { + "epoch": 0.05, + "learning_rate": 9.545819290643557e-05, + "loss": 2.6489, + "theoretical_loss": 3.470888960748253, + "tokens_seen": 1740701696 + }, + { + "epoch": 0.06, + "learning_rate": 9.545016851227732e-05, + "loss": 2.6192, + "theoretical_loss": 3.4708665712397004, + "tokens_seen": 1740832768 + }, + { + "epoch": 0.06, + "learning_rate": 9.544214411811908e-05, + "loss": 2.6038, + "theoretical_loss": 3.4708441838888255, + "tokens_seen": 1740963840 + }, + { + "epoch": 0.06, + "learning_rate": 9.543411972396084e-05, + "loss": 2.6892, + "theoretical_loss": 3.4708217986952574, + "tokens_seen": 1741094912 + }, + { + "epoch": 0.06, + "learning_rate": 9.54260953298026e-05, + "loss": 2.5735, + "theoretical_loss": 3.4707994156586253, + "tokens_seen": 1741225984 + }, + { + "epoch": 0.06, + "learning_rate": 9.541807093564436e-05, + "loss": 2.6655, + "theoretical_loss": 3.47077703477856, + "tokens_seen": 1741357056 + }, + { + "epoch": 0.06, + "learning_rate": 9.541004654148612e-05, + "loss": 2.5846, + "theoretical_loss": 3.470754656054691, + "tokens_seen": 1741488128 + }, + { + "epoch": 0.06, + "learning_rate": 9.540202214732788e-05, + "loss": 2.539, + "theoretical_loss": 3.470732279486649, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 961372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.166609525680542, + "objective/train/theoretical_loss": 3.470709905074062, + "objective/train/tokens_used": 112209376, + "theoretical_loss": 3.470709905074062, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.06, + "learning_rate": 9.539399775316964e-05, + "loss": 2.6481, + "theoretical_loss": 3.470709905074062, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.06, + "learning_rate": 9.53859733590114e-05, + "loss": 2.4939, + "theoretical_loss": 3.470687532816563, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.06, + "learning_rate": 9.537794896485315e-05, + "loss": 2.488, + "theoretical_loss": 3.4706651627137806, + "tokens_seen": 1742012416 + }, + { + "epoch": 0.06, + "learning_rate": 9.536992457069491e-05, + "loss": 2.5908, + "theoretical_loss": 3.470642794765345, + "tokens_seen": 1742143488 + }, + { + "epoch": 0.06, + "learning_rate": 9.536190017653667e-05, + "loss": 2.5974, + "theoretical_loss": 3.4706204289708875, + "tokens_seen": 1742274560 + }, + { + "epoch": 0.06, + "learning_rate": 9.535387578237843e-05, + "loss": 2.5184, + "theoretical_loss": 3.4705980653300386, + "tokens_seen": 1742405632 + }, + { + "epoch": 0.06, + "learning_rate": 9.53458513882202e-05, + "loss": 2.4658, + "theoretical_loss": 3.4705757038424285, + "tokens_seen": 1742536704 + }, + { + "epoch": 0.06, + "learning_rate": 9.533782699406195e-05, + "loss": 2.6165, + "theoretical_loss": 3.470553344507688, + "tokens_seen": 1742667776 + }, + { + "epoch": 0.06, + "learning_rate": 9.532980259990371e-05, + "loss": 2.5697, + "theoretical_loss": 3.470530987325448, + "tokens_seen": 1742798848 + }, + { + "epoch": 0.06, + "learning_rate": 9.532177820574547e-05, + "loss": 2.7383, + "theoretical_loss": 3.4705086322953393, + "tokens_seen": 1742929920 + }, + { + "epoch": 0.06, + "learning_rate": 9.531375381158722e-05, + "loss": 2.5499, + "theoretical_loss": 3.4704862794169937, + "tokens_seen": 1743060992 + }, + { + "epoch": 0.06, + "learning_rate": 9.530572941742899e-05, + "loss": 2.5394, + "theoretical_loss": 3.470463928690041, + "tokens_seen": 1743192064 + }, + { + "epoch": 0.06, + "learning_rate": 9.529770502327074e-05, + "loss": 2.5673, + "theoretical_loss": 3.4704415801141133, + "tokens_seen": 1743323136 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 962529, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.988215923309326, + "objective/train/theoretical_loss": 3.4704304066326683, + "objective/train/tokens_used": 113847776, + "theoretical_loss": 3.4704304066326683, + "tokens_seen": 1743388672 + }, + { + "epoch": 0.06, + "learning_rate": 9.528968062911251e-05, + "loss": 2.7066, + "theoretical_loss": 3.4704192336888413, + "tokens_seen": 1743454208 + }, + { + "epoch": 0.06, + "learning_rate": 9.528165623495426e-05, + "loss": 2.6361, + "theoretical_loss": 3.4703968894138573, + "tokens_seen": 1743585280 + }, + { + "epoch": 0.06, + "learning_rate": 9.527363184079603e-05, + "loss": 2.6597, + "theoretical_loss": 3.470374547288792, + "tokens_seen": 1743716352 + }, + { + "epoch": 0.06, + "learning_rate": 9.526560744663778e-05, + "loss": 2.7227, + "theoretical_loss": 3.4703522073132773, + "tokens_seen": 1743847424 + }, + { + "epoch": 0.06, + "learning_rate": 9.525758305247953e-05, + "loss": 2.4906, + "theoretical_loss": 3.470329869486944, + "tokens_seen": 1743978496 + }, + { + "epoch": 0.06, + "learning_rate": 9.52495586583213e-05, + "loss": 2.6045, + "theoretical_loss": 3.4703075338094256, + "tokens_seen": 1744109568 + }, + { + "epoch": 0.06, + "learning_rate": 9.524153426416305e-05, + "loss": 2.5417, + "theoretical_loss": 3.470285200280352, + "tokens_seen": 1744240640 + }, + { + "epoch": 0.06, + "learning_rate": 9.523350987000482e-05, + "loss": 2.6117, + "theoretical_loss": 3.470262868899357, + "tokens_seen": 1744371712 + }, + { + "epoch": 0.06, + "learning_rate": 9.522548547584657e-05, + "loss": 2.6588, + "theoretical_loss": 3.4702405396660705, + "tokens_seen": 1744502784 + }, + { + "epoch": 0.06, + "learning_rate": 9.521746108168834e-05, + "loss": 2.6923, + "theoretical_loss": 3.470218212580126, + "tokens_seen": 1744633856 + }, + { + "epoch": 0.06, + "learning_rate": 9.520943668753009e-05, + "loss": 2.7159, + "theoretical_loss": 3.4701958876411556, + "tokens_seen": 1744764928 + }, + { + "epoch": 0.06, + "learning_rate": 9.520141229337185e-05, + "loss": 2.5938, + "theoretical_loss": 3.4701735648487912, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 963086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.850843667984009, + "objective/train/theoretical_loss": 3.4701512442026656, + "objective/train/tokens_used": 115486176, + "theoretical_loss": 3.4701512442026656, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.06, + "learning_rate": 9.519338789921361e-05, + "loss": 2.575, + "theoretical_loss": 3.4701512442026656, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.06, + "learning_rate": 9.518536350505537e-05, + "loss": 2.6056, + "theoretical_loss": 3.4701289257024106, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.06, + "learning_rate": 9.517733911089713e-05, + "loss": 2.5319, + "theoretical_loss": 3.47010660934766, + "tokens_seen": 1745289216 + }, + { + "epoch": 0.06, + "learning_rate": 9.516931471673889e-05, + "loss": 2.5956, + "theoretical_loss": 3.4700842951380446, + "tokens_seen": 1745420288 + }, + { + "epoch": 0.06, + "learning_rate": 9.516129032258065e-05, + "loss": 2.5044, + "theoretical_loss": 3.4700619830731982, + "tokens_seen": 1745551360 + }, + { + "epoch": 0.06, + "learning_rate": 9.51532659284224e-05, + "loss": 2.499, + "theoretical_loss": 3.4700396731527543, + "tokens_seen": 1745682432 + }, + { + "epoch": 0.06, + "learning_rate": 9.514524153426416e-05, + "loss": 2.6185, + "theoretical_loss": 3.4700173653763446, + "tokens_seen": 1745813504 + }, + { + "epoch": 0.06, + "learning_rate": 9.513721714010593e-05, + "loss": 2.5509, + "theoretical_loss": 3.469995059743603, + "tokens_seen": 1745944576 + }, + { + "epoch": 0.06, + "learning_rate": 9.512919274594768e-05, + "loss": 2.5919, + "theoretical_loss": 3.469972756254162, + "tokens_seen": 1746075648 + }, + { + "epoch": 0.06, + "learning_rate": 9.512116835178944e-05, + "loss": 2.4592, + "theoretical_loss": 3.4699504549076545, + "tokens_seen": 1746206720 + }, + { + "epoch": 0.06, + "learning_rate": 9.51131439576312e-05, + "loss": 2.6279, + "theoretical_loss": 3.4699281557037147, + "tokens_seen": 1746337792 + }, + { + "epoch": 0.06, + "learning_rate": 9.510511956347296e-05, + "loss": 2.6709, + "theoretical_loss": 3.4699058586419755, + "tokens_seen": 1746468864 + }, + { + "epoch": 0.06, + "learning_rate": 9.509709516931472e-05, + "loss": 2.5483, + "theoretical_loss": 3.46988356372207, + "tokens_seen": 1746599936 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 964151, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.168619394302368, + "objective/train/theoretical_loss": 3.469872417065191, + "objective/train/tokens_used": 117124576, + "theoretical_loss": 3.469872417065191, + "tokens_seen": 1746665472 + }, + { + "epoch": 0.06, + "learning_rate": 9.508907077515648e-05, + "loss": 2.4492, + "theoretical_loss": 3.4698612709436327, + "tokens_seen": 1746731008 + }, + { + "epoch": 0.06, + "learning_rate": 9.508104638099824e-05, + "loss": 2.6619, + "theoretical_loss": 3.469838980306297, + "tokens_seen": 1746862080 + }, + { + "epoch": 0.06, + "learning_rate": 9.507302198683999e-05, + "loss": 2.6721, + "theoretical_loss": 3.4698166918096955, + "tokens_seen": 1746993152 + }, + { + "epoch": 0.06, + "learning_rate": 9.506499759268176e-05, + "loss": 2.502, + "theoretical_loss": 3.469794405453463, + "tokens_seen": 1747124224 + }, + { + "epoch": 0.06, + "learning_rate": 9.505697319852351e-05, + "loss": 2.599, + "theoretical_loss": 3.4697721212372334, + "tokens_seen": 1747255296 + }, + { + "epoch": 0.06, + "learning_rate": 9.504894880436528e-05, + "loss": 2.6716, + "theoretical_loss": 3.4697498391606403, + "tokens_seen": 1747386368 + }, + { + "epoch": 0.06, + "learning_rate": 9.504092441020703e-05, + "loss": 2.5568, + "theoretical_loss": 3.469727559223318, + "tokens_seen": 1747517440 + }, + { + "epoch": 0.06, + "learning_rate": 9.50329000160488e-05, + "loss": 2.6531, + "theoretical_loss": 3.469705281424901, + "tokens_seen": 1747648512 + }, + { + "epoch": 0.06, + "learning_rate": 9.502487562189055e-05, + "loss": 2.653, + "theoretical_loss": 3.469683005765023, + "tokens_seen": 1747779584 + }, + { + "epoch": 0.06, + "learning_rate": 9.50168512277323e-05, + "loss": 2.547, + "theoretical_loss": 3.469660732243319, + "tokens_seen": 1747910656 + }, + { + "epoch": 0.06, + "learning_rate": 9.500882683357407e-05, + "loss": 2.5207, + "theoretical_loss": 3.469638460859423, + "tokens_seen": 1748041728 + }, + { + "epoch": 0.06, + "learning_rate": 9.500080243941582e-05, + "loss": 2.6592, + "theoretical_loss": 3.4696161916129693, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 964837, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9922051429748535, + "objective/train/theoretical_loss": 3.469593924503593, + "objective/train/tokens_used": 118762976, + "theoretical_loss": 3.469593924503593, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.06, + "learning_rate": 9.499277804525759e-05, + "loss": 2.5553, + "theoretical_loss": 3.469593924503593, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.06, + "learning_rate": 9.498475365109934e-05, + "loss": 2.5655, + "theoretical_loss": 3.4695716595309287, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.06, + "learning_rate": 9.497672925694111e-05, + "loss": 2.6482, + "theoretical_loss": 3.469549396694611, + "tokens_seen": 1748566016 + }, + { + "epoch": 0.06, + "learning_rate": 9.496870486278286e-05, + "loss": 2.5774, + "theoretical_loss": 3.469527135994275, + "tokens_seen": 1748697088 + }, + { + "epoch": 0.06, + "learning_rate": 9.496068046862462e-05, + "loss": 2.4727, + "theoretical_loss": 3.469504877429556, + "tokens_seen": 1748828160 + }, + { + "epoch": 0.06, + "learning_rate": 9.495265607446638e-05, + "loss": 2.5101, + "theoretical_loss": 3.4694826210000884, + "tokens_seen": 1748959232 + }, + { + "epoch": 0.06, + "learning_rate": 9.494463168030814e-05, + "loss": 2.6768, + "theoretical_loss": 3.4694603667055075, + "tokens_seen": 1749090304 + }, + { + "epoch": 0.06, + "learning_rate": 9.49366072861499e-05, + "loss": 2.553, + "theoretical_loss": 3.4694381145454494, + "tokens_seen": 1749221376 + }, + { + "epoch": 0.06, + "learning_rate": 9.492858289199166e-05, + "loss": 2.4986, + "theoretical_loss": 3.469415864519548, + "tokens_seen": 1749352448 + }, + { + "epoch": 0.06, + "learning_rate": 9.492055849783342e-05, + "loss": 2.66, + "theoretical_loss": 3.469393616627441, + "tokens_seen": 1749483520 + }, + { + "epoch": 0.06, + "learning_rate": 9.491253410367518e-05, + "loss": 2.5936, + "theoretical_loss": 3.4693713708687612, + "tokens_seen": 1749614592 + }, + { + "epoch": 0.06, + "learning_rate": 9.490450970951693e-05, + "loss": 2.5735, + "theoretical_loss": 3.4693491272431456, + "tokens_seen": 1749745664 + }, + { + "epoch": 0.06, + "learning_rate": 9.48964853153587e-05, + "loss": 2.6236, + "theoretical_loss": 3.46932688575023, + "tokens_seen": 1749876736 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 965967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4741082191467285, + "objective/train/theoretical_loss": 3.4693157658034206, + "objective/train/tokens_used": 120401376, + "theoretical_loss": 3.4693157658034206, + "tokens_seen": 1749942272 + }, + { + "epoch": 0.06, + "learning_rate": 9.488846092120045e-05, + "loss": 2.6312, + "theoretical_loss": 3.46930464638965, + "tokens_seen": 1750007808 + }, + { + "epoch": 0.06, + "learning_rate": 9.488043652704221e-05, + "loss": 2.6914, + "theoretical_loss": 3.4692824091610412, + "tokens_seen": 1750138880 + }, + { + "epoch": 0.06, + "learning_rate": 9.487241213288397e-05, + "loss": 2.4969, + "theoretical_loss": 3.4692601740640403, + "tokens_seen": 1750269952 + }, + { + "epoch": 0.06, + "learning_rate": 9.486438773872573e-05, + "loss": 2.568, + "theoretical_loss": 3.4692379410982825, + "tokens_seen": 1750401024 + }, + { + "epoch": 0.06, + "learning_rate": 9.485636334456749e-05, + "loss": 2.5101, + "theoretical_loss": 3.4692157102634047, + "tokens_seen": 1750532096 + }, + { + "epoch": 0.06, + "learning_rate": 9.484833895040924e-05, + "loss": 2.6857, + "theoretical_loss": 3.4691934815590426, + "tokens_seen": 1750663168 + }, + { + "epoch": 0.06, + "learning_rate": 9.484031455625101e-05, + "loss": 2.4291, + "theoretical_loss": 3.4691712549848326, + "tokens_seen": 1750794240 + }, + { + "epoch": 0.06, + "learning_rate": 9.483229016209276e-05, + "loss": 2.6526, + "theoretical_loss": 3.469149030540412, + "tokens_seen": 1750925312 + }, + { + "epoch": 0.06, + "learning_rate": 9.482426576793453e-05, + "loss": 2.5627, + "theoretical_loss": 3.4691268082254156, + "tokens_seen": 1751056384 + }, + { + "epoch": 0.06, + "learning_rate": 9.481624137377628e-05, + "loss": 2.5353, + "theoretical_loss": 3.4691045880394817, + "tokens_seen": 1751187456 + }, + { + "epoch": 0.06, + "learning_rate": 9.480821697961805e-05, + "loss": 2.5783, + "theoretical_loss": 3.4690823699822455, + "tokens_seen": 1751318528 + }, + { + "epoch": 0.06, + "learning_rate": 9.48001925854598e-05, + "loss": 2.6184, + "theoretical_loss": 3.469060154053345, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 966884, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.789921760559082, + "objective/train/theoretical_loss": 3.4690379402524165, + "objective/train/tokens_used": 122039776, + "theoretical_loss": 3.4690379402524165, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.06, + "learning_rate": 9.479216819130155e-05, + "loss": 2.5307, + "theoretical_loss": 3.4690379402524165, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.06, + "learning_rate": 9.478414379714332e-05, + "loss": 2.5257, + "theoretical_loss": 3.4690157285790972, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 9.477611940298507e-05, + "loss": 2.561, + "theoretical_loss": 3.468993519033024, + "tokens_seen": 1751842816 + }, + { + "epoch": 0.06, + "learning_rate": 9.476809500882684e-05, + "loss": 2.3812, + "theoretical_loss": 3.468971311613834, + "tokens_seen": 1751973888 + }, + { + "epoch": 0.06, + "learning_rate": 9.476007061466859e-05, + "loss": 2.5491, + "theoretical_loss": 3.468949106321164, + "tokens_seen": 1752104960 + }, + { + "epoch": 0.06, + "learning_rate": 9.475204622051036e-05, + "loss": 2.5825, + "theoretical_loss": 3.468926903154652, + "tokens_seen": 1752236032 + }, + { + "epoch": 0.06, + "learning_rate": 9.474402182635211e-05, + "loss": 2.608, + "theoretical_loss": 3.4689047021139356, + "tokens_seen": 1752367104 + }, + { + "epoch": 0.06, + "learning_rate": 9.473599743219388e-05, + "loss": 2.6951, + "theoretical_loss": 3.4688825031986514, + "tokens_seen": 1752498176 + }, + { + "epoch": 0.06, + "learning_rate": 9.472797303803563e-05, + "loss": 2.5484, + "theoretical_loss": 3.4688603064084376, + "tokens_seen": 1752629248 + }, + { + "epoch": 0.06, + "learning_rate": 9.471994864387739e-05, + "loss": 2.7742, + "theoretical_loss": 3.4688381117429317, + "tokens_seen": 1752760320 + }, + { + "epoch": 0.06, + "learning_rate": 9.471192424971915e-05, + "loss": 2.5422, + "theoretical_loss": 3.4688159192017713, + "tokens_seen": 1752891392 + }, + { + "epoch": 0.06, + "learning_rate": 9.47038998555609e-05, + "loss": 2.637, + "theoretical_loss": 3.4687937287845947, + "tokens_seen": 1753022464 + }, + { + "epoch": 0.06, + "learning_rate": 9.469587546140267e-05, + "loss": 2.5073, + "theoretical_loss": 3.4687715404910398, + "tokens_seen": 1753153536 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 967417, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.34035325050354, + "objective/train/theoretical_loss": 3.4687604471405065, + "objective/train/tokens_used": 123678176, + "theoretical_loss": 3.4687604471405065, + "tokens_seen": 1753219072 + }, + { + "epoch": 0.06, + "learning_rate": 9.468785106724443e-05, + "loss": 2.5376, + "theoretical_loss": 3.4687493543207433, + "tokens_seen": 1753284608 + }, + { + "epoch": 0.06, + "learning_rate": 9.467982667308619e-05, + "loss": 2.6251, + "theoretical_loss": 3.468727170273345, + "tokens_seen": 1753415680 + }, + { + "epoch": 0.06, + "learning_rate": 9.467180227892795e-05, + "loss": 2.8049, + "theoretical_loss": 3.468704988348483, + "tokens_seen": 1753546752 + }, + { + "epoch": 0.06, + "learning_rate": 9.46637778847697e-05, + "loss": 2.5002, + "theoretical_loss": 3.468682808545794, + "tokens_seen": 1753677824 + }, + { + "epoch": 0.06, + "learning_rate": 9.465575349061147e-05, + "loss": 2.5396, + "theoretical_loss": 3.4686606308649184, + "tokens_seen": 1753808896 + }, + { + "epoch": 0.06, + "learning_rate": 9.464772909645322e-05, + "loss": 2.5852, + "theoretical_loss": 3.468638455305493, + "tokens_seen": 1753939968 + }, + { + "epoch": 0.06, + "learning_rate": 9.463970470229499e-05, + "loss": 2.5664, + "theoretical_loss": 3.468616281867157, + "tokens_seen": 1754071040 + }, + { + "epoch": 0.06, + "learning_rate": 9.463168030813674e-05, + "loss": 2.6855, + "theoretical_loss": 3.46859411054955, + "tokens_seen": 1754202112 + }, + { + "epoch": 0.06, + "learning_rate": 9.46236559139785e-05, + "loss": 2.6069, + "theoretical_loss": 3.468571941352309, + "tokens_seen": 1754333184 + }, + { + "epoch": 0.06, + "learning_rate": 9.461563151982026e-05, + "loss": 2.5943, + "theoretical_loss": 3.4685497742750737, + "tokens_seen": 1754464256 + }, + { + "epoch": 0.06, + "learning_rate": 9.460760712566201e-05, + "loss": 2.5968, + "theoretical_loss": 3.4685276093174835, + "tokens_seen": 1754595328 + }, + { + "epoch": 0.06, + "learning_rate": 9.459958273150378e-05, + "loss": 2.4725, + "theoretical_loss": 3.4685054464791767, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 968410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6490659713745117, + "objective/train/theoretical_loss": 3.4684832857597923, + "objective/train/tokens_used": 125316576, + "theoretical_loss": 3.4684832857597923, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 9.459155833734553e-05, + "loss": 2.5628, + "theoretical_loss": 3.4684832857597923, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 9.45835339431873e-05, + "loss": 2.5849, + "theoretical_loss": 3.46846112715897, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 9.457550954902905e-05, + "loss": 2.6675, + "theoretical_loss": 3.468438970676348, + "tokens_seen": 1755119616 + }, + { + "epoch": 0.06, + "learning_rate": 9.456748515487082e-05, + "loss": 2.5566, + "theoretical_loss": 3.468416816311567, + "tokens_seen": 1755250688 + }, + { + "epoch": 0.06, + "learning_rate": 9.455946076071257e-05, + "loss": 2.4097, + "theoretical_loss": 3.468394664064266, + "tokens_seen": 1755381760 + }, + { + "epoch": 0.06, + "learning_rate": 9.455143636655432e-05, + "loss": 2.6195, + "theoretical_loss": 3.468372513934084, + "tokens_seen": 1755512832 + }, + { + "epoch": 0.06, + "learning_rate": 9.454341197239609e-05, + "loss": 2.5105, + "theoretical_loss": 3.468350365920662, + "tokens_seen": 1755643904 + }, + { + "epoch": 0.06, + "learning_rate": 9.453538757823784e-05, + "loss": 2.5745, + "theoretical_loss": 3.468328220023638, + "tokens_seen": 1755774976 + }, + { + "epoch": 0.06, + "learning_rate": 9.452736318407961e-05, + "loss": 2.5794, + "theoretical_loss": 3.4683060762426523, + "tokens_seen": 1755906048 + }, + { + "epoch": 0.06, + "learning_rate": 9.451933878992136e-05, + "loss": 2.563, + "theoretical_loss": 3.4682839345773457, + "tokens_seen": 1756037120 + }, + { + "epoch": 0.06, + "learning_rate": 9.451131439576313e-05, + "loss": 2.5961, + "theoretical_loss": 3.468261795027357, + "tokens_seen": 1756168192 + }, + { + "epoch": 0.06, + "learning_rate": 9.450329000160488e-05, + "loss": 2.6463, + "theoretical_loss": 3.4682396575923264, + "tokens_seen": 1756299264 + }, + { + "epoch": 0.06, + "learning_rate": 9.449526560744664e-05, + "loss": 2.5798, + "theoretical_loss": 3.468217522271895, + "tokens_seen": 1756430336 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 968954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.701629638671875, + "objective/train/theoretical_loss": 3.468206455404541, + "objective/train/tokens_used": 126954976, + "theoretical_loss": 3.468206455404541, + "tokens_seen": 1756495872 + }, + { + "epoch": 0.06, + "learning_rate": 9.44872412132884e-05, + "loss": 2.4543, + "theoretical_loss": 3.468195389065702, + "tokens_seen": 1756561408 + }, + { + "epoch": 0.06, + "learning_rate": 9.447921681913016e-05, + "loss": 2.5349, + "theoretical_loss": 3.4681732579733886, + "tokens_seen": 1756692480 + }, + { + "epoch": 0.06, + "learning_rate": 9.447119242497192e-05, + "loss": 2.5532, + "theoretical_loss": 3.4681511289945943, + "tokens_seen": 1756823552 + }, + { + "epoch": 0.06, + "learning_rate": 9.446316803081368e-05, + "loss": 2.5709, + "theoretical_loss": 3.46812900212896, + "tokens_seen": 1756954624 + }, + { + "epoch": 0.06, + "learning_rate": 9.445514363665544e-05, + "loss": 2.5352, + "theoretical_loss": 3.468106877376126, + "tokens_seen": 1757085696 + }, + { + "epoch": 0.06, + "learning_rate": 9.44471192424972e-05, + "loss": 2.6435, + "theoretical_loss": 3.468084754735734, + "tokens_seen": 1757216768 + }, + { + "epoch": 0.07, + "learning_rate": 9.443909484833896e-05, + "loss": 2.6461, + "theoretical_loss": 3.468062634207424, + "tokens_seen": 1757347840 + }, + { + "epoch": 0.07, + "learning_rate": 9.443107045418072e-05, + "loss": 2.655, + "theoretical_loss": 3.4680405157908365, + "tokens_seen": 1757478912 + }, + { + "epoch": 0.07, + "learning_rate": 9.442304606002247e-05, + "loss": 2.6677, + "theoretical_loss": 3.4680183994856133, + "tokens_seen": 1757609984 + }, + { + "epoch": 0.07, + "learning_rate": 9.441502166586424e-05, + "loss": 2.6228, + "theoretical_loss": 3.4679962852913944, + "tokens_seen": 1757741056 + }, + { + "epoch": 0.07, + "learning_rate": 9.440699727170599e-05, + "loss": 2.6249, + "theoretical_loss": 3.4679741732078213, + "tokens_seen": 1757872128 + }, + { + "epoch": 0.07, + "learning_rate": 9.439897287754776e-05, + "loss": 2.5379, + "theoretical_loss": 3.467952063234536, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 970319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6145591735839844, + "objective/train/theoretical_loss": 3.4679299553711793, + "objective/train/tokens_used": 128593376, + "theoretical_loss": 3.4679299553711793, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.07, + "learning_rate": 9.439094848338951e-05, + "loss": 2.6071, + "theoretical_loss": 3.4679299553711793, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.07, + "learning_rate": 9.438292408923127e-05, + "loss": 2.4915, + "theoretical_loss": 3.467907849617392, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.07, + "learning_rate": 9.437489969507303e-05, + "loss": 2.4659, + "theoretical_loss": 3.467885745972816, + "tokens_seen": 1758396416 + }, + { + "epoch": 0.07, + "learning_rate": 9.436687530091478e-05, + "loss": 2.5481, + "theoretical_loss": 3.467863644437093, + "tokens_seen": 1758527488 + }, + { + "epoch": 0.07, + "learning_rate": 9.435885090675655e-05, + "loss": 2.6001, + "theoretical_loss": 3.4678415450098643, + "tokens_seen": 1758658560 + }, + { + "epoch": 0.07, + "learning_rate": 9.43508265125983e-05, + "loss": 2.5004, + "theoretical_loss": 3.467819447690772, + "tokens_seen": 1758789632 + }, + { + "epoch": 0.07, + "learning_rate": 9.434280211844007e-05, + "loss": 2.6864, + "theoretical_loss": 3.467797352479458, + "tokens_seen": 1758920704 + }, + { + "epoch": 0.07, + "learning_rate": 9.433477772428182e-05, + "loss": 2.6084, + "theoretical_loss": 3.4677752593755633, + "tokens_seen": 1759051776 + }, + { + "epoch": 0.07, + "learning_rate": 9.432675333012359e-05, + "loss": 2.553, + "theoretical_loss": 3.4677531683787306, + "tokens_seen": 1759182848 + }, + { + "epoch": 0.07, + "learning_rate": 9.431872893596534e-05, + "loss": 2.6378, + "theoretical_loss": 3.4677310794886025, + "tokens_seen": 1759313920 + }, + { + "epoch": 0.07, + "learning_rate": 9.43107045418071e-05, + "loss": 2.38, + "theoretical_loss": 3.46770899270482, + "tokens_seen": 1759444992 + }, + { + "epoch": 0.07, + "learning_rate": 9.430268014764886e-05, + "loss": 2.6261, + "theoretical_loss": 3.467686908027026, + "tokens_seen": 1759576064 + }, + { + "epoch": 0.07, + "learning_rate": 9.429465575349061e-05, + "loss": 2.6348, + "theoretical_loss": 3.4676648254548628, + "tokens_seen": 1759707136 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 970997, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7837326526641846, + "objective/train/theoretical_loss": 3.467653784958281, + "objective/train/tokens_used": 130231776, + "theoretical_loss": 3.467653784958281, + "tokens_seen": 1759772672 + }, + { + "epoch": 0.07, + "learning_rate": 9.428663135933238e-05, + "loss": 2.6637, + "theoretical_loss": 3.467642744987973, + "tokens_seen": 1759838208 + }, + { + "epoch": 0.07, + "learning_rate": 9.427860696517413e-05, + "loss": 2.6002, + "theoretical_loss": 3.467620666625999, + "tokens_seen": 1759969280 + }, + { + "epoch": 0.07, + "learning_rate": 9.42705825710159e-05, + "loss": 2.3965, + "theoretical_loss": 3.4675985903685826, + "tokens_seen": 1760100352 + }, + { + "epoch": 0.07, + "learning_rate": 9.426255817685765e-05, + "loss": 2.4942, + "theoretical_loss": 3.4675765162153676, + "tokens_seen": 1760231424 + }, + { + "epoch": 0.07, + "learning_rate": 9.42545337826994e-05, + "loss": 2.6354, + "theoretical_loss": 3.4675544441659962, + "tokens_seen": 1760362496 + }, + { + "epoch": 0.07, + "learning_rate": 9.424650938854117e-05, + "loss": 2.5839, + "theoretical_loss": 3.467532374220112, + "tokens_seen": 1760493568 + }, + { + "epoch": 0.07, + "learning_rate": 9.423848499438293e-05, + "loss": 2.5664, + "theoretical_loss": 3.4675103063773567, + "tokens_seen": 1760624640 + }, + { + "epoch": 0.07, + "learning_rate": 9.423046060022469e-05, + "loss": 2.6749, + "theoretical_loss": 3.4674882406373744, + "tokens_seen": 1760755712 + }, + { + "epoch": 0.07, + "learning_rate": 9.422243620606645e-05, + "loss": 2.5527, + "theoretical_loss": 3.4674661769998076, + "tokens_seen": 1760886784 + }, + { + "epoch": 0.07, + "learning_rate": 9.421441181190821e-05, + "loss": 2.4963, + "theoretical_loss": 3.4674441154643, + "tokens_seen": 1761017856 + }, + { + "epoch": 0.07, + "learning_rate": 9.420638741774997e-05, + "loss": 2.5026, + "theoretical_loss": 3.467422056030494, + "tokens_seen": 1761148928 + }, + { + "epoch": 0.07, + "learning_rate": 9.419836302359172e-05, + "loss": 2.546, + "theoretical_loss": 3.467399998698034, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 972106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7436439990997314, + "objective/train/theoretical_loss": 3.4673779434665635, + "objective/train/tokens_used": 131870176, + "theoretical_loss": 3.4673779434665635, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.07, + "learning_rate": 9.419033862943349e-05, + "loss": 2.5274, + "theoretical_loss": 3.4673779434665635, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.07, + "learning_rate": 9.418231423527524e-05, + "loss": 2.7645, + "theoretical_loss": 3.4673558903357247, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.07, + "learning_rate": 9.4174289841117e-05, + "loss": 2.498, + "theoretical_loss": 3.467333839305163, + "tokens_seen": 1761673216 + }, + { + "epoch": 0.07, + "learning_rate": 9.416626544695876e-05, + "loss": 2.6431, + "theoretical_loss": 3.467311790374521, + "tokens_seen": 1761804288 + }, + { + "epoch": 0.07, + "learning_rate": 9.415824105280053e-05, + "loss": 2.6502, + "theoretical_loss": 3.467289743543443, + "tokens_seen": 1761935360 + }, + { + "epoch": 0.07, + "learning_rate": 9.415021665864228e-05, + "loss": 2.5922, + "theoretical_loss": 3.4672676988115727, + "tokens_seen": 1762066432 + }, + { + "epoch": 0.07, + "learning_rate": 9.414219226448404e-05, + "loss": 2.5547, + "theoretical_loss": 3.4672456561785543, + "tokens_seen": 1762197504 + }, + { + "epoch": 0.07, + "learning_rate": 9.41341678703258e-05, + "loss": 2.5285, + "theoretical_loss": 3.4672236156440315, + "tokens_seen": 1762328576 + }, + { + "epoch": 0.07, + "learning_rate": 9.412614347616755e-05, + "loss": 2.7133, + "theoretical_loss": 3.467201577207649, + "tokens_seen": 1762459648 + }, + { + "epoch": 0.07, + "learning_rate": 9.411811908200932e-05, + "loss": 2.516, + "theoretical_loss": 3.4671795408690507, + "tokens_seen": 1762590720 + }, + { + "epoch": 0.07, + "learning_rate": 9.411009468785107e-05, + "loss": 2.4072, + "theoretical_loss": 3.467157506627881, + "tokens_seen": 1762721792 + }, + { + "epoch": 0.07, + "learning_rate": 9.410207029369284e-05, + "loss": 2.3683, + "theoretical_loss": 3.4671354744837837, + "tokens_seen": 1762852864 + }, + { + "epoch": 0.07, + "learning_rate": 9.409404589953459e-05, + "loss": 2.6088, + "theoretical_loss": 3.4671134444364045, + "tokens_seen": 1762983936 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 972675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4672791957855225, + "objective/train/theoretical_loss": 3.467102430198873, + "objective/train/tokens_used": 133508576, + "theoretical_loss": 3.467102430198873, + "tokens_seen": 1763049472 + }, + { + "epoch": 0.07, + "learning_rate": 9.408602150537636e-05, + "loss": 2.4902, + "theoretical_loss": 3.4670914164853874, + "tokens_seen": 1763115008 + }, + { + "epoch": 0.07, + "learning_rate": 9.407799711121811e-05, + "loss": 2.5745, + "theoretical_loss": 3.4670693906303773, + "tokens_seen": 1763246080 + }, + { + "epoch": 0.07, + "learning_rate": 9.406997271705986e-05, + "loss": 2.6415, + "theoretical_loss": 3.4670473668710184, + "tokens_seen": 1763377152 + }, + { + "epoch": 0.07, + "learning_rate": 9.406194832290163e-05, + "loss": 2.6236, + "theoretical_loss": 3.467025345206956, + "tokens_seen": 1763508224 + }, + { + "epoch": 0.07, + "learning_rate": 9.405392392874338e-05, + "loss": 2.5835, + "theoretical_loss": 3.4670033256378354, + "tokens_seen": 1763639296 + }, + { + "epoch": 0.07, + "learning_rate": 9.404589953458515e-05, + "loss": 2.5896, + "theoretical_loss": 3.4669813081633007, + "tokens_seen": 1763770368 + }, + { + "epoch": 0.07, + "learning_rate": 9.40378751404269e-05, + "loss": 2.4005, + "theoretical_loss": 3.466959292782998, + "tokens_seen": 1763901440 + }, + { + "epoch": 0.07, + "learning_rate": 9.402985074626867e-05, + "loss": 2.6698, + "theoretical_loss": 3.4669372794965723, + "tokens_seen": 1764032512 + }, + { + "epoch": 0.07, + "learning_rate": 9.402182635211042e-05, + "loss": 2.4925, + "theoretical_loss": 3.4669152683036684, + "tokens_seen": 1764163584 + }, + { + "epoch": 0.07, + "learning_rate": 9.401380195795218e-05, + "loss": 2.4732, + "theoretical_loss": 3.466893259203932, + "tokens_seen": 1764294656 + }, + { + "epoch": 0.07, + "learning_rate": 9.400577756379394e-05, + "loss": 2.4619, + "theoretical_loss": 3.4668712521970084, + "tokens_seen": 1764425728 + }, + { + "epoch": 0.07, + "learning_rate": 9.39977531696357e-05, + "loss": 2.4386, + "theoretical_loss": 3.466849247282543, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 973758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.705296039581299, + "objective/train/theoretical_loss": 3.466827244460182, + "objective/train/tokens_used": 135146976, + "theoretical_loss": 3.466827244460182, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.07, + "learning_rate": 9.398972877547746e-05, + "loss": 2.6076, + "theoretical_loss": 3.466827244460182, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.07, + "learning_rate": 9.398170438131922e-05, + "loss": 2.5514, + "theoretical_loss": 3.466805243729571, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.07, + "learning_rate": 9.397367998716098e-05, + "loss": 2.4123, + "theoretical_loss": 3.4667832450903555, + "tokens_seen": 1764950016 + }, + { + "epoch": 0.07, + "learning_rate": 9.396565559300274e-05, + "loss": 2.5775, + "theoretical_loss": 3.4667612485421815, + "tokens_seen": 1765081088 + }, + { + "epoch": 0.07, + "learning_rate": 9.395763119884449e-05, + "loss": 2.443, + "theoretical_loss": 3.466739254084695, + "tokens_seen": 1765212160 + }, + { + "epoch": 0.07, + "learning_rate": 9.394960680468626e-05, + "loss": 2.436, + "theoretical_loss": 3.4667172617175424, + "tokens_seen": 1765343232 + }, + { + "epoch": 0.07, + "learning_rate": 9.394158241052801e-05, + "loss": 2.5424, + "theoretical_loss": 3.4666952714403694, + "tokens_seen": 1765474304 + }, + { + "epoch": 0.07, + "learning_rate": 9.393355801636978e-05, + "loss": 2.4734, + "theoretical_loss": 3.466673283252822, + "tokens_seen": 1765605376 + }, + { + "epoch": 0.07, + "learning_rate": 9.392553362221153e-05, + "loss": 2.5327, + "theoretical_loss": 3.4666512971545473, + "tokens_seen": 1765736448 + }, + { + "epoch": 0.07, + "learning_rate": 9.39175092280533e-05, + "loss": 2.5953, + "theoretical_loss": 3.4666293131451917, + "tokens_seen": 1765867520 + }, + { + "epoch": 0.07, + "learning_rate": 9.390948483389505e-05, + "loss": 2.5874, + "theoretical_loss": 3.4666073312244006, + "tokens_seen": 1765998592 + }, + { + "epoch": 0.07, + "learning_rate": 9.39014604397368e-05, + "loss": 2.4698, + "theoretical_loss": 3.4665853513918217, + "tokens_seen": 1766129664 + }, + { + "epoch": 0.07, + "learning_rate": 9.389343604557857e-05, + "loss": 2.4093, + "theoretical_loss": 3.4665633736471015, + "tokens_seen": 1766260736 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 974221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.551424026489258, + "objective/train/theoretical_loss": 3.466552385557578, + "objective/train/tokens_used": 136785376, + "theoretical_loss": 3.466552385557578, + "tokens_seen": 1766326272 + }, + { + "epoch": 0.07, + "learning_rate": 9.388541165142032e-05, + "loss": 2.5791, + "theoretical_loss": 3.4665413979898863, + "tokens_seen": 1766391808 + }, + { + "epoch": 0.07, + "learning_rate": 9.387738725726209e-05, + "loss": 2.4563, + "theoretical_loss": 3.466519424419823, + "tokens_seen": 1766522880 + }, + { + "epoch": 0.07, + "learning_rate": 9.386936286310384e-05, + "loss": 2.4674, + "theoretical_loss": 3.4664974529365593, + "tokens_seen": 1766653952 + }, + { + "epoch": 0.07, + "learning_rate": 9.386133846894561e-05, + "loss": 2.6283, + "theoretical_loss": 3.4664754835397416, + "tokens_seen": 1766785024 + }, + { + "epoch": 0.07, + "learning_rate": 9.385331407478736e-05, + "loss": 2.4859, + "theoretical_loss": 3.4664535162290164, + "tokens_seen": 1766916096 + }, + { + "epoch": 0.07, + "learning_rate": 9.384528968062911e-05, + "loss": 2.4277, + "theoretical_loss": 3.466431551004032, + "tokens_seen": 1767047168 + }, + { + "epoch": 0.07, + "learning_rate": 9.383726528647088e-05, + "loss": 2.4934, + "theoretical_loss": 3.466409587864436, + "tokens_seen": 1767178240 + }, + { + "epoch": 0.07, + "learning_rate": 9.382924089231263e-05, + "loss": 2.476, + "theoretical_loss": 3.4663876268098734, + "tokens_seen": 1767309312 + }, + { + "epoch": 0.07, + "learning_rate": 9.38212164981544e-05, + "loss": 2.6492, + "theoretical_loss": 3.4663656678399946, + "tokens_seen": 1767440384 + }, + { + "epoch": 0.07, + "learning_rate": 9.381319210399615e-05, + "loss": 2.4337, + "theoretical_loss": 3.4663437109544453, + "tokens_seen": 1767571456 + }, + { + "epoch": 0.07, + "learning_rate": 9.380516770983792e-05, + "loss": 2.5528, + "theoretical_loss": 3.4663217561528734, + "tokens_seen": 1767702528 + }, + { + "epoch": 0.07, + "learning_rate": 9.379714331567967e-05, + "loss": 2.5765, + "theoretical_loss": 3.466299803434927, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 975253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.134099006652832, + "objective/train/theoretical_loss": 3.4662778528002534, + "objective/train/tokens_used": 138423776, + "theoretical_loss": 3.4662778528002534, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.07, + "learning_rate": 9.378911892152144e-05, + "loss": 2.5428, + "theoretical_loss": 3.4662778528002534, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.07, + "learning_rate": 9.37810945273632e-05, + "loss": 2.4819, + "theoretical_loss": 3.4662559042485013, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 9.377307013320495e-05, + "loss": 2.6061, + "theoretical_loss": 3.466233957779318, + "tokens_seen": 1768226816 + }, + { + "epoch": 0.07, + "learning_rate": 9.376504573904671e-05, + "loss": 2.6773, + "theoretical_loss": 3.466212013392351, + "tokens_seen": 1768357888 + }, + { + "epoch": 0.07, + "learning_rate": 9.375702134488847e-05, + "loss": 2.5121, + "theoretical_loss": 3.46619007108725, + "tokens_seen": 1768488960 + }, + { + "epoch": 0.07, + "learning_rate": 9.374899695073023e-05, + "loss": 2.5764, + "theoretical_loss": 3.466168130863662, + "tokens_seen": 1768620032 + }, + { + "epoch": 0.07, + "learning_rate": 9.374097255657199e-05, + "loss": 2.594, + "theoretical_loss": 3.466146192721235, + "tokens_seen": 1768751104 + }, + { + "epoch": 0.07, + "learning_rate": 9.373294816241375e-05, + "loss": 2.6474, + "theoretical_loss": 3.4661242566596187, + "tokens_seen": 1768882176 + }, + { + "epoch": 0.07, + "learning_rate": 9.37249237682555e-05, + "loss": 2.6483, + "theoretical_loss": 3.46610232267846, + "tokens_seen": 1769013248 + }, + { + "epoch": 0.07, + "learning_rate": 9.371689937409726e-05, + "loss": 2.4411, + "theoretical_loss": 3.4660803907774094, + "tokens_seen": 1769144320 + }, + { + "epoch": 0.07, + "learning_rate": 9.370887497993903e-05, + "loss": 2.5386, + "theoretical_loss": 3.4660584609561145, + "tokens_seen": 1769275392 + }, + { + "epoch": 0.07, + "learning_rate": 9.370085058578078e-05, + "loss": 2.427, + "theoretical_loss": 3.4660365332142233, + "tokens_seen": 1769406464 + }, + { + "epoch": 0.07, + "learning_rate": 9.369282619162255e-05, + "loss": 2.6418, + "theoretical_loss": 3.4660146075513856, + "tokens_seen": 1769537536 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 975853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6023433208465576, + "objective/train/theoretical_loss": 3.466003645499502, + "objective/train/tokens_used": 140062176, + "theoretical_loss": 3.466003645499502, + "tokens_seen": 1769603072 + }, + { + "epoch": 0.07, + "learning_rate": 9.36848017974643e-05, + "loss": 2.4855, + "theoretical_loss": 3.46599268396725, + "tokens_seen": 1769668608 + }, + { + "epoch": 0.07, + "learning_rate": 9.367677740330607e-05, + "loss": 2.5534, + "theoretical_loss": 3.465970762461466, + "tokens_seen": 1769799680 + }, + { + "epoch": 0.07, + "learning_rate": 9.366875300914782e-05, + "loss": 2.7021, + "theoretical_loss": 3.4659488430336816, + "tokens_seen": 1769930752 + }, + { + "epoch": 0.07, + "learning_rate": 9.366072861498957e-05, + "loss": 2.5897, + "theoretical_loss": 3.4659269256835468, + "tokens_seen": 1770061824 + }, + { + "epoch": 0.07, + "learning_rate": 9.365270422083134e-05, + "loss": 2.5387, + "theoretical_loss": 3.4659050104107103, + "tokens_seen": 1770192896 + }, + { + "epoch": 0.07, + "learning_rate": 9.364467982667309e-05, + "loss": 2.4854, + "theoretical_loss": 3.465883097214822, + "tokens_seen": 1770323968 + }, + { + "epoch": 0.07, + "learning_rate": 9.363665543251486e-05, + "loss": 2.6062, + "theoretical_loss": 3.465861186095531, + "tokens_seen": 1770455040 + }, + { + "epoch": 0.07, + "learning_rate": 9.362863103835661e-05, + "loss": 2.6616, + "theoretical_loss": 3.4658392770524866, + "tokens_seen": 1770586112 + }, + { + "epoch": 0.07, + "learning_rate": 9.362060664419838e-05, + "loss": 2.54, + "theoretical_loss": 3.4658173700853387, + "tokens_seen": 1770717184 + }, + { + "epoch": 0.07, + "learning_rate": 9.361258225004013e-05, + "loss": 2.4839, + "theoretical_loss": 3.465795465193737, + "tokens_seen": 1770848256 + }, + { + "epoch": 0.07, + "learning_rate": 9.360455785588188e-05, + "loss": 2.5301, + "theoretical_loss": 3.465773562377331, + "tokens_seen": 1770979328 + }, + { + "epoch": 0.07, + "learning_rate": 9.359653346172365e-05, + "loss": 2.4655, + "theoretical_loss": 3.4657516616357706, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 976931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.991319179534912, + "objective/train/theoretical_loss": 3.4657297629687056, + "objective/train/tokens_used": 141700576, + "theoretical_loss": 3.4657297629687056, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.07, + "learning_rate": 9.35885090675654e-05, + "loss": 2.6133, + "theoretical_loss": 3.4657297629687056, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.07, + "learning_rate": 9.358048467340717e-05, + "loss": 2.5932, + "theoretical_loss": 3.4657078663757868, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 9.357246027924892e-05, + "loss": 2.5936, + "theoretical_loss": 3.4656859718566633, + "tokens_seen": 1771503616 + }, + { + "epoch": 0.07, + "learning_rate": 9.356443588509069e-05, + "loss": 2.5908, + "theoretical_loss": 3.4656640794109856, + "tokens_seen": 1771634688 + }, + { + "epoch": 0.07, + "learning_rate": 9.355641149093244e-05, + "loss": 2.5438, + "theoretical_loss": 3.4656421890384044, + "tokens_seen": 1771765760 + }, + { + "epoch": 0.07, + "learning_rate": 9.35483870967742e-05, + "loss": 2.4126, + "theoretical_loss": 3.465620300738569, + "tokens_seen": 1771896832 + }, + { + "epoch": 0.07, + "learning_rate": 9.354036270261596e-05, + "loss": 2.6821, + "theoretical_loss": 3.465598414511131, + "tokens_seen": 1772027904 + }, + { + "epoch": 0.07, + "learning_rate": 9.353233830845772e-05, + "loss": 2.5196, + "theoretical_loss": 3.46557653035574, + "tokens_seen": 1772158976 + }, + { + "epoch": 0.07, + "learning_rate": 9.352431391429948e-05, + "loss": 2.6704, + "theoretical_loss": 3.4655546482720476, + "tokens_seen": 1772290048 + }, + { + "epoch": 0.07, + "learning_rate": 9.351628952014124e-05, + "loss": 2.6822, + "theoretical_loss": 3.4655327682597035, + "tokens_seen": 1772421120 + }, + { + "epoch": 0.07, + "learning_rate": 9.3508265125983e-05, + "loss": 2.4813, + "theoretical_loss": 3.465510890318359, + "tokens_seen": 1772552192 + }, + { + "epoch": 0.07, + "learning_rate": 9.350024073182476e-05, + "loss": 2.5966, + "theoretical_loss": 3.465489014447665, + "tokens_seen": 1772683264 + }, + { + "epoch": 0.07, + "learning_rate": 9.349221633766652e-05, + "loss": 2.6041, + "theoretical_loss": 3.465467140647272, + "tokens_seen": 1772814336 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 977871, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.325096368789673, + "objective/train/theoretical_loss": 3.4654562045233295, + "objective/train/tokens_used": 143338976, + "theoretical_loss": 3.4654562045233295, + "tokens_seen": 1772879872 + }, + { + "epoch": 0.07, + "learning_rate": 9.348419194350828e-05, + "loss": 2.5723, + "theoretical_loss": 3.465445268916832, + "tokens_seen": 1772945408 + }, + { + "epoch": 0.07, + "learning_rate": 9.347616754935003e-05, + "loss": 2.5643, + "theoretical_loss": 3.4654233992559944, + "tokens_seen": 1773076480 + }, + { + "epoch": 0.07, + "learning_rate": 9.34681431551918e-05, + "loss": 2.7092, + "theoretical_loss": 3.465401531664412, + "tokens_seen": 1773207552 + }, + { + "epoch": 0.07, + "learning_rate": 9.346011876103355e-05, + "loss": 2.5261, + "theoretical_loss": 3.465379666141735, + "tokens_seen": 1773338624 + }, + { + "epoch": 0.07, + "learning_rate": 9.345209436687532e-05, + "loss": 2.6486, + "theoretical_loss": 3.4653578026876155, + "tokens_seen": 1773469696 + }, + { + "epoch": 0.07, + "learning_rate": 9.344406997271707e-05, + "loss": 2.4702, + "theoretical_loss": 3.4653359413017046, + "tokens_seen": 1773600768 + }, + { + "epoch": 0.07, + "learning_rate": 9.343604557855882e-05, + "loss": 2.6074, + "theoretical_loss": 3.465314081983654, + "tokens_seen": 1773731840 + }, + { + "epoch": 0.08, + "learning_rate": 9.342802118440059e-05, + "loss": 2.7033, + "theoretical_loss": 3.4652922247331155, + "tokens_seen": 1773862912 + }, + { + "epoch": 0.08, + "learning_rate": 9.341999679024234e-05, + "loss": 2.6171, + "theoretical_loss": 3.4652703695497404, + "tokens_seen": 1773993984 + }, + { + "epoch": 0.08, + "learning_rate": 9.341197239608411e-05, + "loss": 2.6683, + "theoretical_loss": 3.4652485164331805, + "tokens_seen": 1774125056 + }, + { + "epoch": 0.08, + "learning_rate": 9.340394800192586e-05, + "loss": 2.5473, + "theoretical_loss": 3.4652266653830877, + "tokens_seen": 1774256128 + }, + { + "epoch": 0.08, + "learning_rate": 9.339592360776763e-05, + "loss": 2.503, + "theoretical_loss": 3.465204816399114, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 978597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.45396089553833, + "objective/train/theoretical_loss": 3.4651829694809115, + "objective/train/tokens_used": 144977376, + "theoretical_loss": 3.4651829694809115, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.08, + "learning_rate": 9.338789921360938e-05, + "loss": 2.3952, + "theoretical_loss": 3.4651829694809115, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.08, + "learning_rate": 9.337987481945113e-05, + "loss": 2.6399, + "theoretical_loss": 3.4651611246281324, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.08, + "learning_rate": 9.33718504252929e-05, + "loss": 2.53, + "theoretical_loss": 3.465139281840429, + "tokens_seen": 1774780416 + }, + { + "epoch": 0.08, + "learning_rate": 9.336382603113465e-05, + "loss": 2.4813, + "theoretical_loss": 3.465117441117453, + "tokens_seen": 1774911488 + }, + { + "epoch": 0.08, + "learning_rate": 9.335580163697642e-05, + "loss": 2.6267, + "theoretical_loss": 3.4650956024588577, + "tokens_seen": 1775042560 + }, + { + "epoch": 0.08, + "learning_rate": 9.334777724281817e-05, + "loss": 2.4972, + "theoretical_loss": 3.465073765864295, + "tokens_seen": 1775173632 + }, + { + "epoch": 0.08, + "learning_rate": 9.333975284865993e-05, + "loss": 2.6277, + "theoretical_loss": 3.4650519313334174, + "tokens_seen": 1775304704 + }, + { + "epoch": 0.08, + "learning_rate": 9.33317284545017e-05, + "loss": 2.5438, + "theoretical_loss": 3.465030098865877, + "tokens_seen": 1775435776 + }, + { + "epoch": 0.08, + "learning_rate": 9.332370406034345e-05, + "loss": 2.4372, + "theoretical_loss": 3.465008268461328, + "tokens_seen": 1775566848 + }, + { + "epoch": 0.08, + "learning_rate": 9.331567966618521e-05, + "loss": 2.3739, + "theoretical_loss": 3.4649864401194215, + "tokens_seen": 1775697920 + }, + { + "epoch": 0.08, + "learning_rate": 9.330765527202697e-05, + "loss": 2.4286, + "theoretical_loss": 3.464964613839812, + "tokens_seen": 1775828992 + }, + { + "epoch": 0.08, + "learning_rate": 9.329963087786872e-05, + "loss": 2.6974, + "theoretical_loss": 3.464942789622151, + "tokens_seen": 1775960064 + }, + { + "epoch": 0.08, + "learning_rate": 9.329160648371049e-05, + "loss": 2.4616, + "theoretical_loss": 3.4649209674660923, + "tokens_seen": 1776091136 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 979583, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8104419708251953, + "objective/train/theoretical_loss": 3.464910057161056, + "objective/train/tokens_used": 146615776, + "theoretical_loss": 3.464910057161056, + "tokens_seen": 1776156672 + }, + { + "epoch": 0.08, + "learning_rate": 9.328358208955224e-05, + "loss": 2.6414, + "theoretical_loss": 3.4648991473712893, + "tokens_seen": 1776222208 + }, + { + "epoch": 0.08, + "learning_rate": 9.3275557695394e-05, + "loss": 2.524, + "theoretical_loss": 3.464877329337395, + "tokens_seen": 1776353280 + }, + { + "epoch": 0.08, + "learning_rate": 9.326753330123576e-05, + "loss": 2.7874, + "theoretical_loss": 3.4648555133640624, + "tokens_seen": 1776484352 + }, + { + "epoch": 0.08, + "learning_rate": 9.325950890707753e-05, + "loss": 2.5007, + "theoretical_loss": 3.4648336994509448, + "tokens_seen": 1776615424 + }, + { + "epoch": 0.08, + "learning_rate": 9.325148451291928e-05, + "loss": 2.6567, + "theoretical_loss": 3.464811887597696, + "tokens_seen": 1776746496 + }, + { + "epoch": 0.08, + "learning_rate": 9.324346011876103e-05, + "loss": 2.5699, + "theoretical_loss": 3.4647900778039693, + "tokens_seen": 1776877568 + }, + { + "epoch": 0.08, + "learning_rate": 9.32354357246028e-05, + "loss": 2.5771, + "theoretical_loss": 3.4647682700694187, + "tokens_seen": 1777008640 + }, + { + "epoch": 0.08, + "learning_rate": 9.322741133044455e-05, + "loss": 2.6564, + "theoretical_loss": 3.4647464643936976, + "tokens_seen": 1777139712 + }, + { + "epoch": 0.08, + "learning_rate": 9.321938693628632e-05, + "loss": 2.5478, + "theoretical_loss": 3.4647246607764606, + "tokens_seen": 1777270784 + }, + { + "epoch": 0.08, + "learning_rate": 9.321136254212807e-05, + "loss": 2.5323, + "theoretical_loss": 3.4647028592173603, + "tokens_seen": 1777401856 + }, + { + "epoch": 0.08, + "learning_rate": 9.320333814796983e-05, + "loss": 2.6142, + "theoretical_loss": 3.4646810597160513, + "tokens_seen": 1777532928 + }, + { + "epoch": 0.08, + "learning_rate": 9.319531375381159e-05, + "loss": 2.6264, + "theoretical_loss": 3.4646592622721872, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 979848, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.569380044937134, + "objective/train/theoretical_loss": 3.4646374668854234, + "objective/train/tokens_used": 148254176, + "theoretical_loss": 3.4646374668854234, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.08, + "learning_rate": 9.318728935965335e-05, + "loss": 2.6984, + "theoretical_loss": 3.4646374668854234, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.08, + "learning_rate": 9.317926496549511e-05, + "loss": 2.4607, + "theoretical_loss": 3.464615673555413, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.08, + "learning_rate": 9.317124057133686e-05, + "loss": 2.5602, + "theoretical_loss": 3.4645938822818105, + "tokens_seen": 1778057216 + }, + { + "epoch": 0.08, + "learning_rate": 9.316321617717863e-05, + "loss": 2.5766, + "theoretical_loss": 3.4645720930642705, + "tokens_seen": 1778188288 + }, + { + "epoch": 0.08, + "learning_rate": 9.315519178302038e-05, + "loss": 2.5064, + "theoretical_loss": 3.4645503059024474, + "tokens_seen": 1778319360 + }, + { + "epoch": 0.08, + "learning_rate": 9.314716738886214e-05, + "loss": 2.6483, + "theoretical_loss": 3.4645285207959953, + "tokens_seen": 1778450432 + }, + { + "epoch": 0.08, + "learning_rate": 9.31391429947039e-05, + "loss": 2.437, + "theoretical_loss": 3.4645067377445695, + "tokens_seen": 1778581504 + }, + { + "epoch": 0.08, + "learning_rate": 9.313111860054566e-05, + "loss": 2.7707, + "theoretical_loss": 3.4644849567478246, + "tokens_seen": 1778712576 + }, + { + "epoch": 0.08, + "learning_rate": 9.312309420638742e-05, + "loss": 2.6932, + "theoretical_loss": 3.464463177805415, + "tokens_seen": 1778843648 + }, + { + "epoch": 0.08, + "learning_rate": 9.311506981222918e-05, + "loss": 2.5334, + "theoretical_loss": 3.4644414009169964, + "tokens_seen": 1778974720 + }, + { + "epoch": 0.08, + "learning_rate": 9.310704541807093e-05, + "loss": 2.6573, + "theoretical_loss": 3.464419626082223, + "tokens_seen": 1779105792 + }, + { + "epoch": 0.08, + "learning_rate": 9.30990210239127e-05, + "loss": 2.5421, + "theoretical_loss": 3.46439785330075, + "tokens_seen": 1779236864 + }, + { + "epoch": 0.08, + "learning_rate": 9.309099662975445e-05, + "loss": 2.6231, + "theoretical_loss": 3.464376082572233, + "tokens_seen": 1779367936 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 981185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7941625118255615, + "objective/train/theoretical_loss": 3.464365197977725, + "objective/train/tokens_used": 149892576, + "theoretical_loss": 3.464365197977725, + "tokens_seen": 1779433472 + }, + { + "epoch": 0.08, + "learning_rate": 9.308297223559622e-05, + "loss": 2.5164, + "theoretical_loss": 3.464354313896327, + "tokens_seen": 1779499008 + }, + { + "epoch": 0.08, + "learning_rate": 9.307494784143797e-05, + "loss": 2.5201, + "theoretical_loss": 3.4643325472726865, + "tokens_seen": 1779630080 + }, + { + "epoch": 0.08, + "learning_rate": 9.306692344727974e-05, + "loss": 2.4844, + "theoretical_loss": 3.4643107827009674, + "tokens_seen": 1779761152 + }, + { + "epoch": 0.08, + "learning_rate": 9.305889905312149e-05, + "loss": 2.6485, + "theoretical_loss": 3.464289020180826, + "tokens_seen": 1779892224 + }, + { + "epoch": 0.08, + "learning_rate": 9.305087465896324e-05, + "loss": 2.5563, + "theoretical_loss": 3.4642672597119173, + "tokens_seen": 1780023296 + }, + { + "epoch": 0.08, + "learning_rate": 9.304285026480501e-05, + "loss": 2.4439, + "theoretical_loss": 3.464245501293896, + "tokens_seen": 1780154368 + }, + { + "epoch": 0.08, + "learning_rate": 9.303482587064676e-05, + "loss": 2.5935, + "theoretical_loss": 3.4642237449264193, + "tokens_seen": 1780285440 + }, + { + "epoch": 0.08, + "learning_rate": 9.302680147648853e-05, + "loss": 2.6076, + "theoretical_loss": 3.4642019906091424, + "tokens_seen": 1780416512 + }, + { + "epoch": 0.08, + "learning_rate": 9.301877708233028e-05, + "loss": 2.6269, + "theoretical_loss": 3.464180238341721, + "tokens_seen": 1780547584 + }, + { + "epoch": 0.08, + "learning_rate": 9.301075268817204e-05, + "loss": 2.5585, + "theoretical_loss": 3.464158488123811, + "tokens_seen": 1780678656 + }, + { + "epoch": 0.08, + "learning_rate": 9.30027282940138e-05, + "loss": 2.6408, + "theoretical_loss": 3.464136739955069, + "tokens_seen": 1780809728 + }, + { + "epoch": 0.08, + "learning_rate": 9.299470389985556e-05, + "loss": 2.7265, + "theoretical_loss": 3.4641149938351505, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 981883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.705595016479492, + "objective/train/theoretical_loss": 3.4640932497637125, + "objective/train/tokens_used": 151530976, + "theoretical_loss": 3.4640932497637125, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.08, + "learning_rate": 9.298667950569732e-05, + "loss": 2.6347, + "theoretical_loss": 3.4640932497637125, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.08, + "learning_rate": 9.297865511153908e-05, + "loss": 2.4974, + "theoretical_loss": 3.4640715077404103, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.08, + "learning_rate": 9.297063071738084e-05, + "loss": 2.5395, + "theoretical_loss": 3.464049767764901, + "tokens_seen": 1781334016 + }, + { + "epoch": 0.08, + "learning_rate": 9.29626063232226e-05, + "loss": 2.5233, + "theoretical_loss": 3.4640280298368413, + "tokens_seen": 1781465088 + }, + { + "epoch": 0.08, + "learning_rate": 9.295458192906435e-05, + "loss": 2.5728, + "theoretical_loss": 3.4640062939558867, + "tokens_seen": 1781596160 + }, + { + "epoch": 0.08, + "learning_rate": 9.294655753490612e-05, + "loss": 2.7589, + "theoretical_loss": 3.4639845601216948, + "tokens_seen": 1781727232 + }, + { + "epoch": 0.08, + "learning_rate": 9.293853314074787e-05, + "loss": 2.6934, + "theoretical_loss": 3.4639628283339223, + "tokens_seen": 1781858304 + }, + { + "epoch": 0.08, + "learning_rate": 9.293050874658963e-05, + "loss": 2.5922, + "theoretical_loss": 3.463941098592225, + "tokens_seen": 1781989376 + }, + { + "epoch": 0.08, + "learning_rate": 9.292248435243139e-05, + "loss": 2.5203, + "theoretical_loss": 3.4639193708962615, + "tokens_seen": 1782120448 + }, + { + "epoch": 0.08, + "learning_rate": 9.291445995827315e-05, + "loss": 2.6153, + "theoretical_loss": 3.4638976452456873, + "tokens_seen": 1782251520 + }, + { + "epoch": 0.08, + "learning_rate": 9.290643556411491e-05, + "loss": 2.5865, + "theoretical_loss": 3.4638759216401596, + "tokens_seen": 1782382592 + }, + { + "epoch": 0.08, + "learning_rate": 9.289841116995666e-05, + "loss": 2.4681, + "theoretical_loss": 3.4638542000793358, + "tokens_seen": 1782513664 + }, + { + "epoch": 0.08, + "learning_rate": 9.289038677579843e-05, + "loss": 2.6832, + "theoretical_loss": 3.463832480562873, + "tokens_seen": 1782644736 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 982960, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1652934551239014, + "objective/train/theoretical_loss": 3.463821621571171, + "objective/train/tokens_used": 153169376, + "theoretical_loss": 3.463821621571171, + "tokens_seen": 1782710272 + }, + { + "epoch": 0.08, + "learning_rate": 9.288236238164018e-05, + "loss": 2.4893, + "theoretical_loss": 3.4638107630904296, + "tokens_seen": 1782775808 + }, + { + "epoch": 0.08, + "learning_rate": 9.287433798748195e-05, + "loss": 2.5598, + "theoretical_loss": 3.4637890476616615, + "tokens_seen": 1782906880 + }, + { + "epoch": 0.08, + "learning_rate": 9.28663135933237e-05, + "loss": 2.5903, + "theoretical_loss": 3.4637673342762265, + "tokens_seen": 1783037952 + }, + { + "epoch": 0.08, + "learning_rate": 9.285828919916547e-05, + "loss": 2.4675, + "theoretical_loss": 3.463745622933782, + "tokens_seen": 1783169024 + }, + { + "epoch": 0.08, + "learning_rate": 9.285026480500722e-05, + "loss": 2.426, + "theoretical_loss": 3.463723913633987, + "tokens_seen": 1783300096 + }, + { + "epoch": 0.08, + "learning_rate": 9.284224041084899e-05, + "loss": 2.5681, + "theoretical_loss": 3.4637022063764977, + "tokens_seen": 1783431168 + }, + { + "epoch": 0.08, + "learning_rate": 9.283421601669074e-05, + "loss": 2.6727, + "theoretical_loss": 3.4636805011609724, + "tokens_seen": 1783562240 + }, + { + "epoch": 0.08, + "learning_rate": 9.28261916225325e-05, + "loss": 2.682, + "theoretical_loss": 3.4636587979870685, + "tokens_seen": 1783693312 + }, + { + "epoch": 0.08, + "learning_rate": 9.281816722837426e-05, + "loss": 2.4822, + "theoretical_loss": 3.4636370968544448, + "tokens_seen": 1783824384 + }, + { + "epoch": 0.08, + "learning_rate": 9.281014283421601e-05, + "loss": 2.5931, + "theoretical_loss": 3.4636153977627586, + "tokens_seen": 1783955456 + }, + { + "epoch": 0.08, + "learning_rate": 9.280211844005778e-05, + "loss": 2.5113, + "theoretical_loss": 3.4635937007116686, + "tokens_seen": 1784086528 + }, + { + "epoch": 0.08, + "learning_rate": 9.279409404589953e-05, + "loss": 2.4104, + "theoretical_loss": 3.4635720057008332, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 984043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.182506799697876, + "objective/train/theoretical_loss": 3.4635503127299097, + "objective/train/tokens_used": 154807776, + "theoretical_loss": 3.4635503127299097, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.08, + "learning_rate": 9.27860696517413e-05, + "loss": 2.5613, + "theoretical_loss": 3.4635503127299097, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.08, + "learning_rate": 9.277804525758305e-05, + "loss": 2.5171, + "theoretical_loss": 3.4635286217985572, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.08, + "learning_rate": 9.27700208634248e-05, + "loss": 2.5051, + "theoretical_loss": 3.463506932906434, + "tokens_seen": 1784610816 + }, + { + "epoch": 0.08, + "learning_rate": 9.276199646926657e-05, + "loss": 2.6653, + "theoretical_loss": 3.4634852460531986, + "tokens_seen": 1784741888 + }, + { + "epoch": 0.08, + "learning_rate": 9.275397207510833e-05, + "loss": 2.6551, + "theoretical_loss": 3.46346356123851, + "tokens_seen": 1784872960 + }, + { + "epoch": 0.08, + "learning_rate": 9.274594768095009e-05, + "loss": 2.5599, + "theoretical_loss": 3.463441878462026, + "tokens_seen": 1785004032 + }, + { + "epoch": 0.08, + "learning_rate": 9.273792328679185e-05, + "loss": 2.4816, + "theoretical_loss": 3.463420197723406, + "tokens_seen": 1785135104 + }, + { + "epoch": 0.08, + "learning_rate": 9.272989889263361e-05, + "loss": 2.5629, + "theoretical_loss": 3.4633985190223084, + "tokens_seen": 1785266176 + }, + { + "epoch": 0.08, + "learning_rate": 9.272187449847537e-05, + "loss": 2.5634, + "theoretical_loss": 3.463376842358393, + "tokens_seen": 1785397248 + }, + { + "epoch": 0.08, + "learning_rate": 9.271385010431712e-05, + "loss": 2.6212, + "theoretical_loss": 3.4633551677313186, + "tokens_seen": 1785528320 + }, + { + "epoch": 0.08, + "learning_rate": 9.270582571015889e-05, + "loss": 2.5097, + "theoretical_loss": 3.463333495140744, + "tokens_seen": 1785659392 + }, + { + "epoch": 0.08, + "learning_rate": 9.269780131600064e-05, + "loss": 2.5634, + "theoretical_loss": 3.463311824586328, + "tokens_seen": 1785790464 + }, + { + "epoch": 0.08, + "learning_rate": 9.26897769218424e-05, + "loss": 2.545, + "theoretical_loss": 3.463290156067731, + "tokens_seen": 1785921536 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 984688, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8167052268981934, + "objective/train/theoretical_loss": 3.4632793225717573, + "objective/train/tokens_used": 156446176, + "theoretical_loss": 3.4632793225717573, + "tokens_seen": 1785987072 + }, + { + "epoch": 0.08, + "learning_rate": 9.268175252768416e-05, + "loss": 2.7015, + "theoretical_loss": 3.463268489584611, + "tokens_seen": 1786052608 + }, + { + "epoch": 0.08, + "learning_rate": 9.267372813352592e-05, + "loss": 2.6884, + "theoretical_loss": 3.463246825136628, + "tokens_seen": 1786183680 + }, + { + "epoch": 0.08, + "learning_rate": 9.266570373936768e-05, + "loss": 2.5157, + "theoretical_loss": 3.4632251627234423, + "tokens_seen": 1786314752 + }, + { + "epoch": 0.08, + "learning_rate": 9.265767934520943e-05, + "loss": 2.5907, + "theoretical_loss": 3.4632035023447125, + "tokens_seen": 1786445824 + }, + { + "epoch": 0.08, + "learning_rate": 9.26496549510512e-05, + "loss": 2.6504, + "theoretical_loss": 3.4631818440000988, + "tokens_seen": 1786576896 + }, + { + "epoch": 0.08, + "learning_rate": 9.264163055689295e-05, + "loss": 2.5668, + "theoretical_loss": 3.4631601876892604, + "tokens_seen": 1786707968 + }, + { + "epoch": 0.08, + "learning_rate": 9.263360616273472e-05, + "loss": 2.6424, + "theoretical_loss": 3.4631385334118576, + "tokens_seen": 1786839040 + }, + { + "epoch": 0.08, + "learning_rate": 9.262558176857647e-05, + "loss": 2.6048, + "theoretical_loss": 3.463116881167551, + "tokens_seen": 1786970112 + }, + { + "epoch": 0.08, + "learning_rate": 9.261755737441824e-05, + "loss": 2.4931, + "theoretical_loss": 3.463095230955999, + "tokens_seen": 1787101184 + }, + { + "epoch": 0.08, + "learning_rate": 9.260953298025999e-05, + "loss": 2.5467, + "theoretical_loss": 3.463073582776863, + "tokens_seen": 1787232256 + }, + { + "epoch": 0.08, + "learning_rate": 9.260150858610174e-05, + "loss": 2.3647, + "theoretical_loss": 3.4630519366298023, + "tokens_seen": 1787363328 + }, + { + "epoch": 0.08, + "learning_rate": 9.259348419194351e-05, + "loss": 2.4978, + "theoretical_loss": 3.4630302925144782, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 985933, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.6754649877548218, + "objective/train/theoretical_loss": 3.4630086504305497, + "objective/train/tokens_used": 158084576, + "theoretical_loss": 3.4630086504305497, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 9.258545979778526e-05, + "loss": 2.4287, + "theoretical_loss": 3.4630086504305497, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 9.257743540362703e-05, + "loss": 2.4368, + "theoretical_loss": 3.4629870103776783, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 9.256941100946878e-05, + "loss": 2.5856, + "theoretical_loss": 3.4629653723555247, + "tokens_seen": 1787887616 + }, + { + "epoch": 0.08, + "learning_rate": 9.256138661531055e-05, + "loss": 2.4903, + "theoretical_loss": 3.4629437363637483, + "tokens_seen": 1788018688 + }, + { + "epoch": 0.08, + "learning_rate": 9.25533622211523e-05, + "loss": 2.4972, + "theoretical_loss": 3.4629221024020107, + "tokens_seen": 1788149760 + }, + { + "epoch": 0.08, + "learning_rate": 9.254533782699407e-05, + "loss": 2.4449, + "theoretical_loss": 3.4629004704699726, + "tokens_seen": 1788280832 + }, + { + "epoch": 0.08, + "learning_rate": 9.253731343283582e-05, + "loss": 2.5606, + "theoretical_loss": 3.462878840567294, + "tokens_seen": 1788411904 + }, + { + "epoch": 0.08, + "learning_rate": 9.252928903867758e-05, + "loss": 2.577, + "theoretical_loss": 3.4628572126936366, + "tokens_seen": 1788542976 + }, + { + "epoch": 0.08, + "learning_rate": 9.252126464451934e-05, + "loss": 2.5612, + "theoretical_loss": 3.4628355868486613, + "tokens_seen": 1788674048 + }, + { + "epoch": 0.08, + "learning_rate": 9.25132402503611e-05, + "loss": 2.4463, + "theoretical_loss": 3.462813963032029, + "tokens_seen": 1788805120 + }, + { + "epoch": 0.08, + "learning_rate": 9.250521585620286e-05, + "loss": 2.64, + "theoretical_loss": 3.462792341243401, + "tokens_seen": 1788936192 + }, + { + "epoch": 0.08, + "learning_rate": 9.249719146204462e-05, + "loss": 2.543, + "theoretical_loss": 3.462770721482438, + "tokens_seen": 1789067264 + }, + { + "epoch": 0.08, + "learning_rate": 9.248916706788638e-05, + "loss": 2.6057, + "theoretical_loss": 3.462749103748802, + "tokens_seen": 1789198336 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 986505, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2175519466400146, + "objective/train/theoretical_loss": 3.4627382956421258, + "objective/train/tokens_used": 159722976, + "theoretical_loss": 3.4627382956421258, + "tokens_seen": 1789263872 + }, + { + "epoch": 0.08, + "learning_rate": 9.248114267372814e-05, + "loss": 2.491, + "theoretical_loss": 3.4627274880421544, + "tokens_seen": 1789329408 + }, + { + "epoch": 0.08, + "learning_rate": 9.247311827956989e-05, + "loss": 2.6415, + "theoretical_loss": 3.462705874362156, + "tokens_seen": 1789460480 + }, + { + "epoch": 0.08, + "learning_rate": 9.246509388541166e-05, + "loss": 2.3435, + "theoretical_loss": 3.4626842627084695, + "tokens_seen": 1789591552 + }, + { + "epoch": 0.08, + "learning_rate": 9.245706949125341e-05, + "loss": 2.4189, + "theoretical_loss": 3.4626626530807547, + "tokens_seen": 1789722624 + }, + { + "epoch": 0.08, + "learning_rate": 9.244904509709518e-05, + "loss": 2.4021, + "theoretical_loss": 3.4626410454786756, + "tokens_seen": 1789853696 + }, + { + "epoch": 0.08, + "learning_rate": 9.244102070293693e-05, + "loss": 2.4551, + "theoretical_loss": 3.462619439901892, + "tokens_seen": 1789984768 + }, + { + "epoch": 0.08, + "learning_rate": 9.24329963087787e-05, + "loss": 2.6548, + "theoretical_loss": 3.462597836350067, + "tokens_seen": 1790115840 + }, + { + "epoch": 0.09, + "learning_rate": 9.242497191462045e-05, + "loss": 2.5905, + "theoretical_loss": 3.462576234822863, + "tokens_seen": 1790246912 + }, + { + "epoch": 0.09, + "learning_rate": 9.24169475204622e-05, + "loss": 2.6165, + "theoretical_loss": 3.46255463531994, + "tokens_seen": 1790377984 + }, + { + "epoch": 0.09, + "learning_rate": 9.240892312630397e-05, + "loss": 2.4857, + "theoretical_loss": 3.462533037840962, + "tokens_seen": 1790509056 + }, + { + "epoch": 0.09, + "learning_rate": 9.240089873214572e-05, + "loss": 2.6329, + "theoretical_loss": 3.4625114423855905, + "tokens_seen": 1790640128 + }, + { + "epoch": 0.09, + "learning_rate": 9.239287433798749e-05, + "loss": 2.4891, + "theoretical_loss": 3.4624898489534877, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 987700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4643940925598145, + "objective/train/theoretical_loss": 3.462468257544317, + "objective/train/tokens_used": 161361376, + "theoretical_loss": 3.462468257544317, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.09, + "learning_rate": 9.238484994382924e-05, + "loss": 2.3413, + "theoretical_loss": 3.462468257544317, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.09, + "learning_rate": 9.237682554967101e-05, + "loss": 2.4009, + "theoretical_loss": 3.462446668157739, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.09, + "learning_rate": 9.236880115551276e-05, + "loss": 2.665, + "theoretical_loss": 3.462425080793418, + "tokens_seen": 1791164416 + }, + { + "epoch": 0.09, + "learning_rate": 9.236077676135451e-05, + "loss": 2.5981, + "theoretical_loss": 3.4624034954510154, + "tokens_seen": 1791295488 + }, + { + "epoch": 0.09, + "learning_rate": 9.235275236719628e-05, + "loss": 2.6992, + "theoretical_loss": 3.462381912130194, + "tokens_seen": 1791426560 + }, + { + "epoch": 0.09, + "learning_rate": 9.234472797303803e-05, + "loss": 2.6392, + "theoretical_loss": 3.462360330830618, + "tokens_seen": 1791557632 + }, + { + "epoch": 0.09, + "learning_rate": 9.23367035788798e-05, + "loss": 2.5319, + "theoretical_loss": 3.4623387515519486, + "tokens_seen": 1791688704 + }, + { + "epoch": 0.09, + "learning_rate": 9.232867918472155e-05, + "loss": 2.487, + "theoretical_loss": 3.462317174293849, + "tokens_seen": 1791819776 + }, + { + "epoch": 0.09, + "learning_rate": 9.232065479056332e-05, + "loss": 2.4134, + "theoretical_loss": 3.462295599055983, + "tokens_seen": 1791950848 + }, + { + "epoch": 0.09, + "learning_rate": 9.231263039640507e-05, + "loss": 2.387, + "theoretical_loss": 3.4622740258380134, + "tokens_seen": 1792081920 + }, + { + "epoch": 0.09, + "learning_rate": 9.230460600224683e-05, + "loss": 2.5494, + "theoretical_loss": 3.4622524546396027, + "tokens_seen": 1792212992 + }, + { + "epoch": 0.09, + "learning_rate": 9.229658160808859e-05, + "loss": 2.4921, + "theoretical_loss": 3.4622308854604156, + "tokens_seen": 1792344064 + }, + { + "epoch": 0.09, + "learning_rate": 9.228855721393035e-05, + "loss": 2.4341, + "theoretical_loss": 3.462209318300114, + "tokens_seen": 1792475136 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 988349, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3251655101776123, + "objective/train/theoretical_loss": 3.46219853547694, + "objective/train/tokens_used": 162999776, + "theoretical_loss": 3.46219853547694, + "tokens_seen": 1792540672 + }, + { + "epoch": 0.09, + "learning_rate": 9.228053281977211e-05, + "loss": 2.5972, + "theoretical_loss": 3.4621877531583616, + "tokens_seen": 1792606208 + }, + { + "epoch": 0.09, + "learning_rate": 9.227250842561387e-05, + "loss": 2.5254, + "theoretical_loss": 3.4621661900348224, + "tokens_seen": 1792737280 + }, + { + "epoch": 0.09, + "learning_rate": 9.226448403145563e-05, + "loss": 2.4832, + "theoretical_loss": 3.46214462892916, + "tokens_seen": 1792868352 + }, + { + "epoch": 0.09, + "learning_rate": 9.225645963729739e-05, + "loss": 2.4044, + "theoretical_loss": 3.462123069841038, + "tokens_seen": 1792999424 + }, + { + "epoch": 0.09, + "learning_rate": 9.224843524313914e-05, + "loss": 2.6646, + "theoretical_loss": 3.4621015127701193, + "tokens_seen": 1793130496 + }, + { + "epoch": 0.09, + "learning_rate": 9.22404108489809e-05, + "loss": 2.4458, + "theoretical_loss": 3.462079957716069, + "tokens_seen": 1793261568 + }, + { + "epoch": 0.09, + "learning_rate": 9.223238645482266e-05, + "loss": 2.4208, + "theoretical_loss": 3.46205840467855, + "tokens_seen": 1793392640 + }, + { + "epoch": 0.09, + "learning_rate": 9.222436206066443e-05, + "loss": 2.5817, + "theoretical_loss": 3.462036853657227, + "tokens_seen": 1793523712 + }, + { + "epoch": 0.09, + "learning_rate": 9.221633766650618e-05, + "loss": 2.5792, + "theoretical_loss": 3.4620153046517634, + "tokens_seen": 1793654784 + }, + { + "epoch": 0.09, + "learning_rate": 9.220831327234795e-05, + "loss": 2.5938, + "theoretical_loss": 3.461993757661824, + "tokens_seen": 1793785856 + }, + { + "epoch": 0.09, + "learning_rate": 9.22002888781897e-05, + "loss": 2.5438, + "theoretical_loss": 3.4619722126870727, + "tokens_seen": 1793916928 + }, + { + "epoch": 0.09, + "learning_rate": 9.219226448403146e-05, + "loss": 2.5115, + "theoretical_loss": 3.461950669727174, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 989683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0829696655273438, + "objective/train/theoretical_loss": 3.461929128781792, + "objective/train/tokens_used": 164638176, + "theoretical_loss": 3.461929128781792, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.09, + "learning_rate": 9.218424008987322e-05, + "loss": 2.343, + "theoretical_loss": 3.461929128781792, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.09, + "learning_rate": 9.217621569571497e-05, + "loss": 2.5634, + "theoretical_loss": 3.4619075898505915, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.09, + "learning_rate": 9.216819130155674e-05, + "loss": 2.5899, + "theoretical_loss": 3.4618860529332363, + "tokens_seen": 1794441216 + }, + { + "epoch": 0.09, + "learning_rate": 9.216016690739849e-05, + "loss": 2.4196, + "theoretical_loss": 3.461864518029392, + "tokens_seen": 1794572288 + }, + { + "epoch": 0.09, + "learning_rate": 9.215214251324026e-05, + "loss": 2.5227, + "theoretical_loss": 3.4618429851387233, + "tokens_seen": 1794703360 + }, + { + "epoch": 0.09, + "learning_rate": 9.214411811908201e-05, + "loss": 2.442, + "theoretical_loss": 3.4618214542608943, + "tokens_seen": 1794834432 + }, + { + "epoch": 0.09, + "learning_rate": 9.213609372492378e-05, + "loss": 2.5729, + "theoretical_loss": 3.46179992539557, + "tokens_seen": 1794965504 + }, + { + "epoch": 0.09, + "learning_rate": 9.212806933076553e-05, + "loss": 2.5655, + "theoretical_loss": 3.4617783985424158, + "tokens_seen": 1795096576 + }, + { + "epoch": 0.09, + "learning_rate": 9.212004493660728e-05, + "loss": 2.4973, + "theoretical_loss": 3.461756873701096, + "tokens_seen": 1795227648 + }, + { + "epoch": 0.09, + "learning_rate": 9.211202054244905e-05, + "loss": 2.7353, + "theoretical_loss": 3.461735350871277, + "tokens_seen": 1795358720 + }, + { + "epoch": 0.09, + "learning_rate": 9.21039961482908e-05, + "loss": 2.5919, + "theoretical_loss": 3.4617138300526222, + "tokens_seen": 1795489792 + }, + { + "epoch": 0.09, + "learning_rate": 9.209597175413257e-05, + "loss": 2.4516, + "theoretical_loss": 3.461692311244798, + "tokens_seen": 1795620864 + }, + { + "epoch": 0.09, + "learning_rate": 9.208794735997432e-05, + "loss": 2.4701, + "theoretical_loss": 3.4616707944474703, + "tokens_seen": 1795751936 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 990185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.836434841156006, + "objective/train/theoretical_loss": 3.4616600368026376, + "objective/train/tokens_used": 166276576, + "theoretical_loss": 3.4616600368026376, + "tokens_seen": 1795817472 + }, + { + "epoch": 0.09, + "learning_rate": 9.207992296581609e-05, + "loss": 2.5007, + "theoretical_loss": 3.4616492796603033, + "tokens_seen": 1795883008 + }, + { + "epoch": 0.09, + "learning_rate": 9.207189857165784e-05, + "loss": 2.6287, + "theoretical_loss": 3.4616277668829634, + "tokens_seen": 1796014080 + }, + { + "epoch": 0.09, + "learning_rate": 9.20638741774996e-05, + "loss": 2.6319, + "theoretical_loss": 3.461606256115116, + "tokens_seen": 1796145152 + }, + { + "epoch": 0.09, + "learning_rate": 9.205584978334136e-05, + "loss": 2.3697, + "theoretical_loss": 3.4615847473564263, + "tokens_seen": 1796276224 + }, + { + "epoch": 0.09, + "learning_rate": 9.204782538918312e-05, + "loss": 2.5783, + "theoretical_loss": 3.4615632406065604, + "tokens_seen": 1796407296 + }, + { + "epoch": 0.09, + "learning_rate": 9.203980099502488e-05, + "loss": 2.4935, + "theoretical_loss": 3.4615417358651843, + "tokens_seen": 1796538368 + }, + { + "epoch": 0.09, + "learning_rate": 9.203177660086664e-05, + "loss": 2.6099, + "theoretical_loss": 3.4615202331319637, + "tokens_seen": 1796669440 + }, + { + "epoch": 0.09, + "learning_rate": 9.20237522067084e-05, + "loss": 2.6371, + "theoretical_loss": 3.4614987324065645, + "tokens_seen": 1796800512 + }, + { + "epoch": 0.09, + "learning_rate": 9.201572781255016e-05, + "loss": 2.464, + "theoretical_loss": 3.461477233688653, + "tokens_seen": 1796931584 + }, + { + "epoch": 0.09, + "learning_rate": 9.200770341839191e-05, + "loss": 2.609, + "theoretical_loss": 3.4614557369778947, + "tokens_seen": 1797062656 + }, + { + "epoch": 0.09, + "learning_rate": 9.199967902423368e-05, + "loss": 2.5767, + "theoretical_loss": 3.461434242273957, + "tokens_seen": 1797193728 + }, + { + "epoch": 0.09, + "learning_rate": 9.199165463007543e-05, + "loss": 2.641, + "theoretical_loss": 3.461412749576505, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 991515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3866214752197266, + "objective/train/theoretical_loss": 3.4613912588852065, + "objective/train/tokens_used": 167914976, + "theoretical_loss": 3.4613912588852065, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.09, + "learning_rate": 9.19836302359172e-05, + "loss": 2.6222, + "theoretical_loss": 3.4613912588852065, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.09, + "learning_rate": 9.197560584175895e-05, + "loss": 2.6715, + "theoretical_loss": 3.4613697701997266, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.09, + "learning_rate": 9.196758144760072e-05, + "loss": 2.5473, + "theoretical_loss": 3.4613482835197322, + "tokens_seen": 1797718016 + }, + { + "epoch": 0.09, + "learning_rate": 9.195955705344247e-05, + "loss": 2.5753, + "theoretical_loss": 3.4613267988448913, + "tokens_seen": 1797849088 + }, + { + "epoch": 0.09, + "learning_rate": 9.195153265928422e-05, + "loss": 2.6014, + "theoretical_loss": 3.4613053161748684, + "tokens_seen": 1797980160 + }, + { + "epoch": 0.09, + "learning_rate": 9.194350826512599e-05, + "loss": 2.5159, + "theoretical_loss": 3.461283835509332, + "tokens_seen": 1798111232 + }, + { + "epoch": 0.09, + "learning_rate": 9.193548387096774e-05, + "loss": 2.5353, + "theoretical_loss": 3.4612623568479477, + "tokens_seen": 1798242304 + }, + { + "epoch": 0.09, + "learning_rate": 9.192745947680951e-05, + "loss": 2.5962, + "theoretical_loss": 3.4612408801903833, + "tokens_seen": 1798373376 + }, + { + "epoch": 0.09, + "learning_rate": 9.191943508265126e-05, + "loss": 2.5236, + "theoretical_loss": 3.4612194055363057, + "tokens_seen": 1798504448 + }, + { + "epoch": 0.09, + "learning_rate": 9.191141068849303e-05, + "loss": 2.5268, + "theoretical_loss": 3.461197932885382, + "tokens_seen": 1798635520 + }, + { + "epoch": 0.09, + "learning_rate": 9.190338629433478e-05, + "loss": 2.5341, + "theoretical_loss": 3.4611764622372796, + "tokens_seen": 1798766592 + }, + { + "epoch": 0.09, + "learning_rate": 9.189536190017655e-05, + "loss": 2.538, + "theoretical_loss": 3.4611549935916646, + "tokens_seen": 1798897664 + }, + { + "epoch": 0.09, + "learning_rate": 9.18873375060183e-05, + "loss": 2.6186, + "theoretical_loss": 3.461133526948206, + "tokens_seen": 1799028736 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 992149, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.483299732208252, + "objective/train/theoretical_loss": 3.4611227943771814, + "objective/train/tokens_used": 169553376, + "theoretical_loss": 3.4611227943771814, + "tokens_seen": 1799094272 + }, + { + "epoch": 0.09, + "learning_rate": 9.187931311186005e-05, + "loss": 2.565, + "theoretical_loss": 3.4611120623065705, + "tokens_seen": 1799159808 + }, + { + "epoch": 0.09, + "learning_rate": 9.187128871770182e-05, + "loss": 2.5304, + "theoretical_loss": 3.4610905996664254, + "tokens_seen": 1799290880 + }, + { + "epoch": 0.09, + "learning_rate": 9.186326432354357e-05, + "loss": 2.5325, + "theoretical_loss": 3.461069139027438, + "tokens_seen": 1799421952 + }, + { + "epoch": 0.09, + "learning_rate": 9.185523992938534e-05, + "loss": 2.6046, + "theoretical_loss": 3.461047680389277, + "tokens_seen": 1799553024 + }, + { + "epoch": 0.09, + "learning_rate": 9.18472155352271e-05, + "loss": 2.5433, + "theoretical_loss": 3.4610262237516096, + "tokens_seen": 1799684096 + }, + { + "epoch": 0.09, + "learning_rate": 9.183919114106886e-05, + "loss": 2.5845, + "theoretical_loss": 3.461004769114103, + "tokens_seen": 1799815168 + }, + { + "epoch": 0.09, + "learning_rate": 9.183116674691061e-05, + "loss": 2.5856, + "theoretical_loss": 3.460983316476426, + "tokens_seen": 1799946240 + }, + { + "epoch": 0.09, + "learning_rate": 9.182314235275237e-05, + "loss": 2.6222, + "theoretical_loss": 3.4609618658382466, + "tokens_seen": 1800077312 + }, + { + "epoch": 0.09, + "learning_rate": 9.181511795859413e-05, + "loss": 2.5011, + "theoretical_loss": 3.4609404171992324, + "tokens_seen": 1800208384 + }, + { + "epoch": 0.09, + "learning_rate": 9.180709356443589e-05, + "loss": 2.5571, + "theoretical_loss": 3.4609189705590513, + "tokens_seen": 1800339456 + }, + { + "epoch": 0.09, + "learning_rate": 9.179906917027765e-05, + "loss": 2.3876, + "theoretical_loss": 3.460897525917373, + "tokens_seen": 1800470528 + }, + { + "epoch": 0.09, + "learning_rate": 9.17910447761194e-05, + "loss": 2.5827, + "theoretical_loss": 3.4608760832738636, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 993279, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.028233766555786, + "objective/train/theoretical_loss": 3.4608546426281928, + "objective/train/tokens_used": 171191776, + "theoretical_loss": 3.4608546426281928, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.09, + "learning_rate": 9.178302038196117e-05, + "loss": 2.5501, + "theoretical_loss": 3.4608546426281928, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.09, + "learning_rate": 9.177499598780293e-05, + "loss": 2.4478, + "theoretical_loss": 3.4608332039800294, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 9.176697159364468e-05, + "loss": 2.4669, + "theoretical_loss": 3.460811767329041, + "tokens_seen": 1800994816 + }, + { + "epoch": 0.09, + "learning_rate": 9.175894719948645e-05, + "loss": 2.6295, + "theoretical_loss": 3.4607903326748968, + "tokens_seen": 1801125888 + }, + { + "epoch": 0.09, + "learning_rate": 9.17509228053282e-05, + "loss": 2.6089, + "theoretical_loss": 3.460768900017265, + "tokens_seen": 1801256960 + }, + { + "epoch": 0.09, + "learning_rate": 9.174289841116997e-05, + "loss": 2.6017, + "theoretical_loss": 3.460747469355815, + "tokens_seen": 1801388032 + }, + { + "epoch": 0.09, + "learning_rate": 9.173487401701172e-05, + "loss": 2.6287, + "theoretical_loss": 3.460726040690215, + "tokens_seen": 1801519104 + }, + { + "epoch": 0.09, + "learning_rate": 9.172684962285349e-05, + "loss": 2.5425, + "theoretical_loss": 3.460704614020135, + "tokens_seen": 1801650176 + }, + { + "epoch": 0.09, + "learning_rate": 9.171882522869524e-05, + "loss": 2.4997, + "theoretical_loss": 3.4606831893452425, + "tokens_seen": 1801781248 + }, + { + "epoch": 0.09, + "learning_rate": 9.171080083453699e-05, + "loss": 2.5585, + "theoretical_loss": 3.4606617666652078, + "tokens_seen": 1801912320 + }, + { + "epoch": 0.09, + "learning_rate": 9.170277644037876e-05, + "loss": 2.5326, + "theoretical_loss": 3.4606403459796997, + "tokens_seen": 1802043392 + }, + { + "epoch": 0.09, + "learning_rate": 9.169475204622051e-05, + "loss": 2.6015, + "theoretical_loss": 3.4606189272883876, + "tokens_seen": 1802174464 + }, + { + "epoch": 0.09, + "learning_rate": 9.168672765206228e-05, + "loss": 2.5776, + "theoretical_loss": 3.46059751059094, + "tokens_seen": 1802305536 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 993741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3329663276672363, + "objective/train/theoretical_loss": 3.4605868029898126, + "objective/train/tokens_used": 172830176, + "theoretical_loss": 3.4605868029898126, + "tokens_seen": 1802371072 + }, + { + "epoch": 0.09, + "learning_rate": 9.167870325790403e-05, + "loss": 2.5359, + "theoretical_loss": 3.4605760958870277, + "tokens_seen": 1802436608 + }, + { + "epoch": 0.09, + "learning_rate": 9.16706788637458e-05, + "loss": 2.6917, + "theoretical_loss": 3.4605546831763188, + "tokens_seen": 1802567680 + }, + { + "epoch": 0.09, + "learning_rate": 9.166265446958755e-05, + "loss": 2.5287, + "theoretical_loss": 3.460533272458484, + "tokens_seen": 1802698752 + }, + { + "epoch": 0.09, + "learning_rate": 9.16546300754293e-05, + "loss": 2.6202, + "theoretical_loss": 3.460511863733192, + "tokens_seen": 1802829824 + }, + { + "epoch": 0.09, + "learning_rate": 9.164660568127107e-05, + "loss": 2.6136, + "theoretical_loss": 3.4604904570001134, + "tokens_seen": 1802960896 + }, + { + "epoch": 0.09, + "learning_rate": 9.163858128711282e-05, + "loss": 2.7156, + "theoretical_loss": 3.460469052258917, + "tokens_seen": 1803091968 + }, + { + "epoch": 0.09, + "learning_rate": 9.163055689295459e-05, + "loss": 2.6449, + "theoretical_loss": 3.460447649509274, + "tokens_seen": 1803223040 + }, + { + "epoch": 0.09, + "learning_rate": 9.162253249879634e-05, + "loss": 2.6126, + "theoretical_loss": 3.4604262487508533, + "tokens_seen": 1803354112 + }, + { + "epoch": 0.09, + "learning_rate": 9.161450810463811e-05, + "loss": 2.6197, + "theoretical_loss": 3.460404849983325, + "tokens_seen": 1803485184 + }, + { + "epoch": 0.09, + "learning_rate": 9.160648371047986e-05, + "loss": 2.4627, + "theoretical_loss": 3.4603834532063598, + "tokens_seen": 1803616256 + }, + { + "epoch": 0.09, + "learning_rate": 9.159845931632163e-05, + "loss": 2.5741, + "theoretical_loss": 3.4603620584196273, + "tokens_seen": 1803747328 + }, + { + "epoch": 0.09, + "learning_rate": 9.159043492216338e-05, + "loss": 2.4163, + "theoretical_loss": 3.460340665622798, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 994982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4623970985412598, + "objective/train/theoretical_loss": 3.460319274815543, + "objective/train/tokens_used": 174468576, + "theoretical_loss": 3.460319274815543, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.09, + "learning_rate": 9.158241052800514e-05, + "loss": 2.5558, + "theoretical_loss": 3.460319274815543, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.09, + "learning_rate": 9.15743861338469e-05, + "loss": 2.6833, + "theoretical_loss": 3.460297885997531, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 9.156636173968866e-05, + "loss": 2.5265, + "theoretical_loss": 3.4602764991684345, + "tokens_seen": 1804271616 + }, + { + "epoch": 0.09, + "learning_rate": 9.155833734553042e-05, + "loss": 2.3257, + "theoretical_loss": 3.4602551143279228, + "tokens_seen": 1804402688 + }, + { + "epoch": 0.09, + "learning_rate": 9.155031295137218e-05, + "loss": 2.5691, + "theoretical_loss": 3.4602337314756664, + "tokens_seen": 1804533760 + }, + { + "epoch": 0.09, + "learning_rate": 9.154228855721394e-05, + "loss": 2.5498, + "theoretical_loss": 3.4602123506113376, + "tokens_seen": 1804664832 + }, + { + "epoch": 0.09, + "learning_rate": 9.15342641630557e-05, + "loss": 2.545, + "theoretical_loss": 3.460190971734605, + "tokens_seen": 1804795904 + }, + { + "epoch": 0.09, + "learning_rate": 9.152623976889745e-05, + "loss": 2.6298, + "theoretical_loss": 3.4601695948451416, + "tokens_seen": 1804926976 + }, + { + "epoch": 0.09, + "learning_rate": 9.151821537473922e-05, + "loss": 2.6275, + "theoretical_loss": 3.460148219942617, + "tokens_seen": 1805058048 + }, + { + "epoch": 0.09, + "learning_rate": 9.151019098058097e-05, + "loss": 2.5375, + "theoretical_loss": 3.4601268470267024, + "tokens_seen": 1805189120 + }, + { + "epoch": 0.09, + "learning_rate": 9.150216658642274e-05, + "loss": 2.4901, + "theoretical_loss": 3.4601054760970698, + "tokens_seen": 1805320192 + }, + { + "epoch": 0.09, + "learning_rate": 9.149414219226449e-05, + "loss": 2.6574, + "theoretical_loss": 3.4600841071533894, + "tokens_seen": 1805451264 + }, + { + "epoch": 0.09, + "learning_rate": 9.148611779810626e-05, + "loss": 2.508, + "theoretical_loss": 3.460062740195333, + "tokens_seen": 1805582336 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 995914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.526226758956909, + "objective/train/theoretical_loss": 3.4600520574608113, + "objective/train/tokens_used": 176106976, + "theoretical_loss": 3.4600520574608113, + "tokens_seen": 1805647872 + }, + { + "epoch": 0.09, + "learning_rate": 9.147809340394801e-05, + "loss": 2.4625, + "theoretical_loss": 3.4600413752225725, + "tokens_seen": 1805713408 + }, + { + "epoch": 0.09, + "learning_rate": 9.147006900978976e-05, + "loss": 2.4556, + "theoretical_loss": 3.460020012234778, + "tokens_seen": 1805844480 + }, + { + "epoch": 0.09, + "learning_rate": 9.146204461563153e-05, + "loss": 2.56, + "theoretical_loss": 3.459998651231622, + "tokens_seen": 1805975552 + }, + { + "epoch": 0.09, + "learning_rate": 9.145402022147328e-05, + "loss": 2.5144, + "theoretical_loss": 3.4599772922127765, + "tokens_seen": 1806106624 + }, + { + "epoch": 0.09, + "learning_rate": 9.144599582731505e-05, + "loss": 2.5989, + "theoretical_loss": 3.459955935177912, + "tokens_seen": 1806237696 + }, + { + "epoch": 0.09, + "learning_rate": 9.14379714331568e-05, + "loss": 2.6055, + "theoretical_loss": 3.459934580126701, + "tokens_seen": 1806368768 + }, + { + "epoch": 0.09, + "learning_rate": 9.142994703899857e-05, + "loss": 2.5955, + "theoretical_loss": 3.4599132270588155, + "tokens_seen": 1806499840 + }, + { + "epoch": 0.09, + "learning_rate": 9.142192264484032e-05, + "loss": 2.6991, + "theoretical_loss": 3.459891875973927, + "tokens_seen": 1806630912 + }, + { + "epoch": 0.1, + "learning_rate": 9.141389825068207e-05, + "loss": 2.6054, + "theoretical_loss": 3.4598705268717076, + "tokens_seen": 1806761984 + }, + { + "epoch": 0.1, + "learning_rate": 9.140587385652384e-05, + "loss": 2.6119, + "theoretical_loss": 3.4598491797518287, + "tokens_seen": 1806893056 + }, + { + "epoch": 0.1, + "learning_rate": 9.13978494623656e-05, + "loss": 2.6178, + "theoretical_loss": 3.459827834613964, + "tokens_seen": 1807024128 + }, + { + "epoch": 0.1, + "learning_rate": 9.138982506820736e-05, + "loss": 2.7865, + "theoretical_loss": 3.4598064914577846, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 996601, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5091497898101807, + "objective/train/theoretical_loss": 3.459785150282963, + "objective/train/tokens_used": 177745376, + "theoretical_loss": 3.459785150282963, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.1, + "learning_rate": 9.138180067404911e-05, + "loss": 2.6171, + "theoretical_loss": 3.459785150282963, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.1, + "learning_rate": 9.137377627989088e-05, + "loss": 2.4956, + "theoretical_loss": 3.4597638110891715, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.1, + "learning_rate": 9.136575188573263e-05, + "loss": 2.5369, + "theoretical_loss": 3.459742473876083, + "tokens_seen": 1807548416 + }, + { + "epoch": 0.1, + "learning_rate": 9.135772749157439e-05, + "loss": 2.5218, + "theoretical_loss": 3.4597211386433697, + "tokens_seen": 1807679488 + }, + { + "epoch": 0.1, + "learning_rate": 9.134970309741615e-05, + "loss": 2.54, + "theoretical_loss": 3.459699805390705, + "tokens_seen": 1807810560 + }, + { + "epoch": 0.1, + "learning_rate": 9.13416787032579e-05, + "loss": 2.5169, + "theoretical_loss": 3.45967847411776, + "tokens_seen": 1807941632 + }, + { + "epoch": 0.1, + "learning_rate": 9.133365430909967e-05, + "loss": 2.6301, + "theoretical_loss": 3.4596571448242086, + "tokens_seen": 1808072704 + }, + { + "epoch": 0.1, + "learning_rate": 9.132562991494143e-05, + "loss": 2.6115, + "theoretical_loss": 3.4596358175097235, + "tokens_seen": 1808203776 + }, + { + "epoch": 0.1, + "learning_rate": 9.131760552078319e-05, + "loss": 2.5861, + "theoretical_loss": 3.4596144921739773, + "tokens_seen": 1808334848 + }, + { + "epoch": 0.1, + "learning_rate": 9.130958112662495e-05, + "loss": 2.5045, + "theoretical_loss": 3.459593168816644, + "tokens_seen": 1808465920 + }, + { + "epoch": 0.1, + "learning_rate": 9.13015567324667e-05, + "loss": 2.571, + "theoretical_loss": 3.4595718474373953, + "tokens_seen": 1808596992 + }, + { + "epoch": 0.1, + "learning_rate": 9.129353233830847e-05, + "loss": 2.6187, + "theoretical_loss": 3.459550528035905, + "tokens_seen": 1808728064 + }, + { + "epoch": 0.1, + "learning_rate": 9.128550794415022e-05, + "loss": 2.5396, + "theoretical_loss": 3.459529210611847, + "tokens_seen": 1808859136 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 997711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7451651096343994, + "objective/train/theoretical_loss": 3.4595185526412524, + "objective/train/tokens_used": 179383776, + "theoretical_loss": 3.4595185526412524, + "tokens_seen": 1808924672 + }, + { + "epoch": 0.1, + "learning_rate": 9.127748354999199e-05, + "loss": 2.5304, + "theoretical_loss": 3.4595078951648937, + "tokens_seen": 1808990208 + }, + { + "epoch": 0.1, + "learning_rate": 9.126945915583374e-05, + "loss": 2.5649, + "theoretical_loss": 3.4594865816947182, + "tokens_seen": 1809121280 + }, + { + "epoch": 0.1, + "learning_rate": 9.12614347616755e-05, + "loss": 2.6362, + "theoretical_loss": 3.4594652702009956, + "tokens_seen": 1809252352 + }, + { + "epoch": 0.1, + "learning_rate": 9.125341036751726e-05, + "loss": 2.6765, + "theoretical_loss": 3.4594439606833975, + "tokens_seen": 1809383424 + }, + { + "epoch": 0.1, + "learning_rate": 9.124538597335903e-05, + "loss": 2.5887, + "theoretical_loss": 3.4594226531415986, + "tokens_seen": 1809514496 + }, + { + "epoch": 0.1, + "learning_rate": 9.123736157920078e-05, + "loss": 2.5145, + "theoretical_loss": 3.459401347575273, + "tokens_seen": 1809645568 + }, + { + "epoch": 0.1, + "learning_rate": 9.122933718504253e-05, + "loss": 2.6369, + "theoretical_loss": 3.4593800439840936, + "tokens_seen": 1809776640 + }, + { + "epoch": 0.1, + "learning_rate": 9.12213127908843e-05, + "loss": 2.6258, + "theoretical_loss": 3.4593587423677348, + "tokens_seen": 1809907712 + }, + { + "epoch": 0.1, + "learning_rate": 9.121328839672605e-05, + "loss": 2.3457, + "theoretical_loss": 3.45933744272587, + "tokens_seen": 1810038784 + }, + { + "epoch": 0.1, + "learning_rate": 9.120526400256782e-05, + "loss": 2.4792, + "theoretical_loss": 3.4593161450581738, + "tokens_seen": 1810169856 + }, + { + "epoch": 0.1, + "learning_rate": 9.119723960840957e-05, + "loss": 2.5877, + "theoretical_loss": 3.45929484936432, + "tokens_seen": 1810300928 + }, + { + "epoch": 0.1, + "learning_rate": 9.118921521425134e-05, + "loss": 2.4965, + "theoretical_loss": 3.459273555643983, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 998245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.329852342605591, + "objective/train/theoretical_loss": 3.459252263896837, + "objective/train/tokens_used": 181022176, + "theoretical_loss": 3.459252263896837, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.1, + "learning_rate": 9.118119082009309e-05, + "loss": 2.5769, + "theoretical_loss": 3.459252263896837, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.1, + "learning_rate": 9.117316642593484e-05, + "loss": 2.6172, + "theoretical_loss": 3.4592309741225558, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.1, + "learning_rate": 9.116514203177661e-05, + "loss": 2.6355, + "theoretical_loss": 3.4592096863208144, + "tokens_seen": 1810825216 + }, + { + "epoch": 0.1, + "learning_rate": 9.115711763761836e-05, + "loss": 2.6663, + "theoretical_loss": 3.4591884004912874, + "tokens_seen": 1810956288 + }, + { + "epoch": 0.1, + "learning_rate": 9.114909324346013e-05, + "loss": 2.4647, + "theoretical_loss": 3.4591671166336484, + "tokens_seen": 1811087360 + }, + { + "epoch": 0.1, + "learning_rate": 9.114106884930188e-05, + "loss": 2.5586, + "theoretical_loss": 3.4591458347475736, + "tokens_seen": 1811218432 + }, + { + "epoch": 0.1, + "learning_rate": 9.113304445514365e-05, + "loss": 2.6566, + "theoretical_loss": 3.4591245548327363, + "tokens_seen": 1811349504 + }, + { + "epoch": 0.1, + "learning_rate": 9.11250200609854e-05, + "loss": 2.4653, + "theoretical_loss": 3.4591032768888117, + "tokens_seen": 1811480576 + }, + { + "epoch": 0.1, + "learning_rate": 9.111699566682716e-05, + "loss": 2.5683, + "theoretical_loss": 3.459082000915475, + "tokens_seen": 1811611648 + }, + { + "epoch": 0.1, + "learning_rate": 9.110897127266892e-05, + "loss": 2.5607, + "theoretical_loss": 3.4590607269124005, + "tokens_seen": 1811742720 + }, + { + "epoch": 0.1, + "learning_rate": 9.110094687851068e-05, + "loss": 2.577, + "theoretical_loss": 3.4590394548792642, + "tokens_seen": 1811873792 + }, + { + "epoch": 0.1, + "learning_rate": 9.109292248435244e-05, + "loss": 2.5945, + "theoretical_loss": 3.459018184815741, + "tokens_seen": 1812004864 + }, + { + "epoch": 0.1, + "learning_rate": 9.10848980901942e-05, + "loss": 2.5688, + "theoretical_loss": 3.458996916721505, + "tokens_seen": 1812135936 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 999249, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.812991142272949, + "objective/train/theoretical_loss": 3.4589862834127683, + "objective/train/tokens_used": 182660576, + "theoretical_loss": 3.4589862834127683, + "tokens_seen": 1812201472 + }, + { + "epoch": 0.1, + "learning_rate": 9.107687369603596e-05, + "loss": 2.5249, + "theoretical_loss": 3.458975650596232, + "tokens_seen": 1812267008 + }, + { + "epoch": 0.1, + "learning_rate": 9.106884930187772e-05, + "loss": 2.6084, + "theoretical_loss": 3.458954386439598, + "tokens_seen": 1812398080 + }, + { + "epoch": 0.1, + "learning_rate": 9.106082490771947e-05, + "loss": 2.5619, + "theoretical_loss": 3.4589331242512777, + "tokens_seen": 1812529152 + }, + { + "epoch": 0.1, + "learning_rate": 9.105280051356124e-05, + "loss": 2.4964, + "theoretical_loss": 3.458911864030947, + "tokens_seen": 1812660224 + }, + { + "epoch": 0.1, + "learning_rate": 9.104477611940299e-05, + "loss": 2.6568, + "theoretical_loss": 3.458890605778281, + "tokens_seen": 1812791296 + }, + { + "epoch": 0.1, + "learning_rate": 9.103675172524476e-05, + "loss": 2.5938, + "theoretical_loss": 3.4588693494929563, + "tokens_seen": 1812922368 + }, + { + "epoch": 0.1, + "learning_rate": 9.102872733108651e-05, + "loss": 2.7681, + "theoretical_loss": 3.4588480951746474, + "tokens_seen": 1813053440 + }, + { + "epoch": 0.1, + "learning_rate": 9.102070293692828e-05, + "loss": 2.5224, + "theoretical_loss": 3.4588268428230307, + "tokens_seen": 1813184512 + }, + { + "epoch": 0.1, + "learning_rate": 9.101267854277003e-05, + "loss": 2.6445, + "theoretical_loss": 3.458805592437782, + "tokens_seen": 1813315584 + }, + { + "epoch": 0.1, + "learning_rate": 9.100465414861178e-05, + "loss": 2.6035, + "theoretical_loss": 3.458784344018577, + "tokens_seen": 1813446656 + }, + { + "epoch": 0.1, + "learning_rate": 9.099662975445355e-05, + "loss": 2.6548, + "theoretical_loss": 3.4587630975650923, + "tokens_seen": 1813577728 + }, + { + "epoch": 0.1, + "learning_rate": 9.09886053602953e-05, + "loss": 2.4354, + "theoretical_loss": 3.4587418530770035, + "tokens_seen": 1813708800 + }, + { + "debugging/Self-BLEU-5": 0.3156689391415616, + "debugging/distinct-1-grams": 0.796493726160272, + "debugging/distinct-2-grams": 0.9767311252314004, + "debugging/entropy-1-grams": 5.133892638116764, + "debugging/entropy-2-grams": 5.762233051450052, + "debugging/length": 475.0, + "debugging/num_segments": 5, + "debugging/score": 0.006195986574498804, + "debugging/score_std": 0.00502469901082792, + "epoch": 0.1, + "objective/train/docs_used": 999799, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.311352252960205, + "objective/train/theoretical_loss": 3.458720610553987, + "objective/train/tokens_used": 184298976, + "theoretical_loss": 3.458720610553987, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.1, + "learning_rate": 9.098058096613707e-05, + "loss": 2.6938, + "theoretical_loss": 3.458720610553987, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.1, + "learning_rate": 9.097255657197882e-05, + "loss": 2.2563, + "theoretical_loss": 3.4586993699957187, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.1, + "learning_rate": 9.096453217782059e-05, + "loss": 2.4608, + "theoretical_loss": 3.458678131401876, + "tokens_seen": 1814102016 + }, + { + "epoch": 0.1, + "learning_rate": 9.095650778366234e-05, + "loss": 2.5433, + "theoretical_loss": 3.4586568947721337, + "tokens_seen": 1814233088 + }, + { + "epoch": 0.1, + "learning_rate": 9.094848338950411e-05, + "loss": 2.5471, + "theoretical_loss": 3.4586356601061694, + "tokens_seen": 1814364160 + }, + { + "epoch": 0.1, + "learning_rate": 9.094045899534586e-05, + "loss": 2.4761, + "theoretical_loss": 3.4586144274036594, + "tokens_seen": 1814495232 + }, + { + "epoch": 0.1, + "learning_rate": 9.093243460118761e-05, + "loss": 2.4453, + "theoretical_loss": 3.4585931966642804, + "tokens_seen": 1814626304 + }, + { + "epoch": 0.1, + "learning_rate": 9.092441020702938e-05, + "loss": 2.5264, + "theoretical_loss": 3.4585719678877087, + "tokens_seen": 1814757376 + }, + { + "epoch": 0.1, + "learning_rate": 9.091638581287113e-05, + "loss": 2.4763, + "theoretical_loss": 3.458550741073622, + "tokens_seen": 1814888448 + }, + { + "epoch": 0.1, + "learning_rate": 9.09083614187129e-05, + "loss": 2.5676, + "theoretical_loss": 3.458529516221696, + "tokens_seen": 1815019520 + }, + { + "epoch": 0.1, + "learning_rate": 9.090033702455465e-05, + "loss": 2.6084, + "theoretical_loss": 3.458508293331609, + "tokens_seen": 1815150592 + }, + { + "epoch": 0.1, + "learning_rate": 9.089231263039642e-05, + "loss": 2.3971, + "theoretical_loss": 3.4584870724030363, + "tokens_seen": 1815281664 + }, + { + "epoch": 0.1, + "learning_rate": 9.088428823623817e-05, + "loss": 2.4878, + "theoretical_loss": 3.4584658534356567, + "tokens_seen": 1815412736 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1000512, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2753448486328125, + "objective/train/theoretical_loss": 3.458455244687313, + "objective/train/tokens_used": 185937376, + "theoretical_loss": 3.458455244687313, + "tokens_seen": 1815478272 + }, + { + "epoch": 0.1, + "learning_rate": 9.087626384207993e-05, + "loss": 2.5719, + "theoretical_loss": 3.458444636429146, + "tokens_seen": 1815543808 + }, + { + "epoch": 0.1, + "learning_rate": 9.08682394479217e-05, + "loss": 2.5104, + "theoretical_loss": 3.4584234213831824, + "tokens_seen": 1815674880 + }, + { + "epoch": 0.1, + "learning_rate": 9.086021505376345e-05, + "loss": 2.5449, + "theoretical_loss": 3.458402208297443, + "tokens_seen": 1815805952 + }, + { + "epoch": 0.1, + "learning_rate": 9.085219065960521e-05, + "loss": 2.3611, + "theoretical_loss": 3.458380997171605, + "tokens_seen": 1815937024 + }, + { + "epoch": 0.1, + "learning_rate": 9.084416626544697e-05, + "loss": 2.706, + "theoretical_loss": 3.458359788005346, + "tokens_seen": 1816068096 + }, + { + "epoch": 0.1, + "learning_rate": 9.083614187128873e-05, + "loss": 2.609, + "theoretical_loss": 3.458338580798343, + "tokens_seen": 1816199168 + }, + { + "epoch": 0.1, + "learning_rate": 9.082811747713049e-05, + "loss": 2.4692, + "theoretical_loss": 3.458317375550275, + "tokens_seen": 1816330240 + }, + { + "epoch": 0.1, + "learning_rate": 9.082009308297224e-05, + "loss": 2.5132, + "theoretical_loss": 3.4582961722608188, + "tokens_seen": 1816461312 + }, + { + "epoch": 0.1, + "learning_rate": 9.0812068688814e-05, + "loss": 2.6601, + "theoretical_loss": 3.4582749709296516, + "tokens_seen": 1816592384 + }, + { + "epoch": 0.1, + "learning_rate": 9.080404429465576e-05, + "loss": 2.4721, + "theoretical_loss": 3.4582537715564525, + "tokens_seen": 1816723456 + }, + { + "epoch": 0.1, + "learning_rate": 9.079601990049753e-05, + "loss": 2.5989, + "theoretical_loss": 3.458232574140899, + "tokens_seen": 1816854528 + }, + { + "epoch": 0.1, + "learning_rate": 9.078799550633928e-05, + "loss": 2.6828, + "theoretical_loss": 3.4582113786826683, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1001919, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5981926918029785, + "objective/train/theoretical_loss": 3.4581901851814396, + "objective/train/tokens_used": 187575776, + "theoretical_loss": 3.4581901851814396, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.1, + "learning_rate": 9.077997111218105e-05, + "loss": 2.5979, + "theoretical_loss": 3.4581901851814396, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.1, + "learning_rate": 9.07719467180228e-05, + "loss": 2.5523, + "theoretical_loss": 3.4581689936368907, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 9.076392232386455e-05, + "loss": 2.5556, + "theoretical_loss": 3.458147804048699, + "tokens_seen": 1817378816 + }, + { + "epoch": 0.1, + "learning_rate": 9.075589792970632e-05, + "loss": 2.611, + "theoretical_loss": 3.4581266164165445, + "tokens_seen": 1817509888 + }, + { + "epoch": 0.1, + "learning_rate": 9.074787353554807e-05, + "loss": 2.4397, + "theoretical_loss": 3.4581054307401047, + "tokens_seen": 1817640960 + }, + { + "epoch": 0.1, + "learning_rate": 9.073984914138984e-05, + "loss": 2.5673, + "theoretical_loss": 3.458084247019057, + "tokens_seen": 1817772032 + }, + { + "epoch": 0.1, + "learning_rate": 9.073182474723159e-05, + "loss": 2.7918, + "theoretical_loss": 3.458063065253082, + "tokens_seen": 1817903104 + }, + { + "epoch": 0.1, + "learning_rate": 9.072380035307334e-05, + "loss": 2.456, + "theoretical_loss": 3.4580418854418573, + "tokens_seen": 1818034176 + }, + { + "epoch": 0.1, + "learning_rate": 9.071577595891511e-05, + "loss": 2.5651, + "theoretical_loss": 3.458020707585061, + "tokens_seen": 1818165248 + }, + { + "epoch": 0.1, + "learning_rate": 9.070775156475686e-05, + "loss": 2.4939, + "theoretical_loss": 3.457999531682373, + "tokens_seen": 1818296320 + }, + { + "epoch": 0.1, + "learning_rate": 9.069972717059863e-05, + "loss": 2.6449, + "theoretical_loss": 3.457978357733471, + "tokens_seen": 1818427392 + }, + { + "epoch": 0.1, + "learning_rate": 9.069170277644038e-05, + "loss": 2.6295, + "theoretical_loss": 3.457957185738035, + "tokens_seen": 1818558464 + }, + { + "epoch": 0.1, + "learning_rate": 9.068367838228215e-05, + "loss": 2.6178, + "theoretical_loss": 3.4579360156957435, + "tokens_seen": 1818689536 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1002350, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.173239231109619, + "objective/train/theoretical_loss": 3.4579254314069265, + "objective/train/tokens_used": 189214176, + "theoretical_loss": 3.4579254314069265, + "tokens_seen": 1818755072 + }, + { + "epoch": 0.1, + "learning_rate": 9.06756539881239e-05, + "loss": 2.5297, + "theoretical_loss": 3.4579148476062755, + "tokens_seen": 1818820608 + }, + { + "epoch": 0.1, + "learning_rate": 9.066762959396566e-05, + "loss": 2.7194, + "theoretical_loss": 3.4578936814693106, + "tokens_seen": 1818951680 + }, + { + "epoch": 0.1, + "learning_rate": 9.065960519980742e-05, + "loss": 2.5981, + "theoretical_loss": 3.4578725172845273, + "tokens_seen": 1819082752 + }, + { + "epoch": 0.1, + "learning_rate": 9.065158080564918e-05, + "loss": 2.6458, + "theoretical_loss": 3.4578513550516057, + "tokens_seen": 1819213824 + }, + { + "epoch": 0.1, + "learning_rate": 9.064355641149094e-05, + "loss": 2.7472, + "theoretical_loss": 3.457830194770225, + "tokens_seen": 1819344896 + }, + { + "epoch": 0.1, + "learning_rate": 9.06355320173327e-05, + "loss": 2.4691, + "theoretical_loss": 3.457809036440064, + "tokens_seen": 1819475968 + }, + { + "epoch": 0.1, + "learning_rate": 9.062750762317445e-05, + "loss": 2.6148, + "theoretical_loss": 3.4577878800608026, + "tokens_seen": 1819607040 + }, + { + "epoch": 0.1, + "learning_rate": 9.061948322901622e-05, + "loss": 2.5812, + "theoretical_loss": 3.457766725632121, + "tokens_seen": 1819738112 + }, + { + "epoch": 0.1, + "learning_rate": 9.061145883485797e-05, + "loss": 2.7097, + "theoretical_loss": 3.457745573153698, + "tokens_seen": 1819869184 + }, + { + "epoch": 0.1, + "learning_rate": 9.060343444069974e-05, + "loss": 2.554, + "theoretical_loss": 3.4577244226252137, + "tokens_seen": 1820000256 + }, + { + "epoch": 0.1, + "learning_rate": 9.059541004654149e-05, + "loss": 2.6685, + "theoretical_loss": 3.4577032740463483, + "tokens_seen": 1820131328 + }, + { + "epoch": 0.1, + "learning_rate": 9.058738565238326e-05, + "loss": 2.4642, + "theoretical_loss": 3.4576821274167813, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1003363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7602131366729736, + "objective/train/theoretical_loss": 3.4576609827361926, + "objective/train/tokens_used": 190852576, + "theoretical_loss": 3.4576609827361926, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 9.057936125822501e-05, + "loss": 2.6968, + "theoretical_loss": 3.4576609827361926, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 9.057133686406676e-05, + "loss": 2.5316, + "theoretical_loss": 3.457639840004262, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 9.056331246990853e-05, + "loss": 2.6249, + "theoretical_loss": 3.4576186992206708, + "tokens_seen": 1820655616 + }, + { + "epoch": 0.1, + "learning_rate": 9.055528807575028e-05, + "loss": 2.7948, + "theoretical_loss": 3.457597560385098, + "tokens_seen": 1820786688 + }, + { + "epoch": 0.1, + "learning_rate": 9.054726368159205e-05, + "loss": 2.6169, + "theoretical_loss": 3.4575764234972253, + "tokens_seen": 1820917760 + }, + { + "epoch": 0.1, + "learning_rate": 9.05392392874338e-05, + "loss": 2.6168, + "theoretical_loss": 3.457555288556731, + "tokens_seen": 1821048832 + }, + { + "epoch": 0.1, + "learning_rate": 9.053121489327556e-05, + "loss": 2.6352, + "theoretical_loss": 3.4575341555632972, + "tokens_seen": 1821179904 + }, + { + "epoch": 0.1, + "learning_rate": 9.052319049911732e-05, + "loss": 2.5961, + "theoretical_loss": 3.457513024516604, + "tokens_seen": 1821310976 + }, + { + "epoch": 0.1, + "learning_rate": 9.051516610495908e-05, + "loss": 2.6443, + "theoretical_loss": 3.4574918954163314, + "tokens_seen": 1821442048 + }, + { + "epoch": 0.1, + "learning_rate": 9.050714171080084e-05, + "loss": 2.6054, + "theoretical_loss": 3.4574707682621613, + "tokens_seen": 1821573120 + }, + { + "epoch": 0.1, + "learning_rate": 9.04991173166426e-05, + "loss": 2.5453, + "theoretical_loss": 3.4574496430537733, + "tokens_seen": 1821704192 + }, + { + "epoch": 0.1, + "learning_rate": 9.049109292248436e-05, + "loss": 2.583, + "theoretical_loss": 3.4574285197908483, + "tokens_seen": 1821835264 + }, + { + "epoch": 0.1, + "learning_rate": 9.048306852832611e-05, + "loss": 2.6039, + "theoretical_loss": 3.4574073984730678, + "tokens_seen": 1821966336 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1003939, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.497610569000244, + "objective/train/theoretical_loss": 3.4573968385435068, + "objective/train/tokens_used": 192490976, + "theoretical_loss": 3.4573968385435068, + "tokens_seen": 1822031872 + }, + { + "epoch": 0.1, + "learning_rate": 9.047504413416787e-05, + "loss": 2.4516, + "theoretical_loss": 3.4573862791001124, + "tokens_seen": 1822097408 + }, + { + "epoch": 0.1, + "learning_rate": 9.046701974000963e-05, + "loss": 2.6655, + "theoretical_loss": 3.457365161671663, + "tokens_seen": 1822228480 + }, + { + "epoch": 0.1, + "learning_rate": 9.045899534585139e-05, + "loss": 2.5194, + "theoretical_loss": 3.457344046187401, + "tokens_seen": 1822359552 + }, + { + "epoch": 0.1, + "learning_rate": 9.045097095169315e-05, + "loss": 2.7839, + "theoretical_loss": 3.4573229326470076, + "tokens_seen": 1822490624 + }, + { + "epoch": 0.1, + "learning_rate": 9.044294655753491e-05, + "loss": 2.6435, + "theoretical_loss": 3.457301821050164, + "tokens_seen": 1822621696 + }, + { + "epoch": 0.1, + "learning_rate": 9.043492216337666e-05, + "loss": 2.49, + "theoretical_loss": 3.4572807113965514, + "tokens_seen": 1822752768 + }, + { + "epoch": 0.1, + "learning_rate": 9.042689776921843e-05, + "loss": 2.6145, + "theoretical_loss": 3.457259603685851, + "tokens_seen": 1822883840 + }, + { + "epoch": 0.1, + "learning_rate": 9.041887337506018e-05, + "loss": 2.575, + "theoretical_loss": 3.4572384979177455, + "tokens_seen": 1823014912 + }, + { + "epoch": 0.1, + "learning_rate": 9.041084898090195e-05, + "loss": 2.5775, + "theoretical_loss": 3.457217394091915, + "tokens_seen": 1823145984 + }, + { + "epoch": 0.11, + "learning_rate": 9.04028245867437e-05, + "loss": 2.7332, + "theoretical_loss": 3.457196292208042, + "tokens_seen": 1823277056 + }, + { + "epoch": 0.11, + "learning_rate": 9.039480019258547e-05, + "loss": 2.5805, + "theoretical_loss": 3.457175192265808, + "tokens_seen": 1823408128 + }, + { + "epoch": 0.11, + "learning_rate": 9.038677579842722e-05, + "loss": 2.6114, + "theoretical_loss": 3.4571540942648946, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1005294, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7300758361816406, + "objective/train/theoretical_loss": 3.457132998204984, + "objective/train/tokens_used": 194129376, + "theoretical_loss": 3.457132998204984, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.11, + "learning_rate": 9.037875140426897e-05, + "loss": 2.4603, + "theoretical_loss": 3.457132998204984, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.11, + "learning_rate": 9.037072701011074e-05, + "loss": 2.6913, + "theoretical_loss": 3.4571119040857576, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.11, + "learning_rate": 9.036270261595249e-05, + "loss": 2.5986, + "theoretical_loss": 3.457090811906898, + "tokens_seen": 1823932416 + }, + { + "epoch": 0.11, + "learning_rate": 9.035467822179426e-05, + "loss": 2.6846, + "theoretical_loss": 3.457069721668087, + "tokens_seen": 1824063488 + }, + { + "epoch": 0.11, + "learning_rate": 9.034665382763601e-05, + "loss": 2.4226, + "theoretical_loss": 3.4570486333690065, + "tokens_seen": 1824194560 + }, + { + "epoch": 0.11, + "learning_rate": 9.033862943347777e-05, + "loss": 2.5405, + "theoretical_loss": 3.4570275470093397, + "tokens_seen": 1824325632 + }, + { + "epoch": 0.11, + "learning_rate": 9.033060503931953e-05, + "loss": 2.5503, + "theoretical_loss": 3.457006462588768, + "tokens_seen": 1824456704 + }, + { + "epoch": 0.11, + "learning_rate": 9.032258064516129e-05, + "loss": 2.6584, + "theoretical_loss": 3.4569853801069743, + "tokens_seen": 1824587776 + }, + { + "epoch": 0.11, + "learning_rate": 9.031455625100305e-05, + "loss": 2.5576, + "theoretical_loss": 3.4569642995636407, + "tokens_seen": 1824718848 + }, + { + "epoch": 0.11, + "learning_rate": 9.03065318568448e-05, + "loss": 2.5872, + "theoretical_loss": 3.4569432209584496, + "tokens_seen": 1824849920 + }, + { + "epoch": 0.11, + "learning_rate": 9.029850746268657e-05, + "loss": 2.5797, + "theoretical_loss": 3.4569221442910845, + "tokens_seen": 1824980992 + }, + { + "epoch": 0.11, + "learning_rate": 9.029048306852833e-05, + "loss": 2.764, + "theoretical_loss": 3.4569010695612272, + "tokens_seen": 1825112064 + }, + { + "epoch": 0.11, + "learning_rate": 9.028245867437008e-05, + "loss": 2.614, + "theoretical_loss": 3.456879996768561, + "tokens_seen": 1825243136 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1005912, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9645118713378906, + "objective/train/theoretical_loss": 3.4568694610985755, + "objective/train/tokens_used": 195767776, + "theoretical_loss": 3.4568694610985755, + "tokens_seen": 1825308672 + }, + { + "epoch": 0.11, + "learning_rate": 9.027443428021185e-05, + "loss": 2.4857, + "theoretical_loss": 3.456858925912768, + "tokens_seen": 1825374208 + }, + { + "epoch": 0.11, + "learning_rate": 9.02664098860536e-05, + "loss": 2.4868, + "theoretical_loss": 3.4568378569935323, + "tokens_seen": 1825505280 + }, + { + "epoch": 0.11, + "learning_rate": 9.025838549189537e-05, + "loss": 2.5564, + "theoretical_loss": 3.456816790010536, + "tokens_seen": 1825636352 + }, + { + "epoch": 0.11, + "learning_rate": 9.025036109773712e-05, + "loss": 2.5284, + "theoretical_loss": 3.456795724963462, + "tokens_seen": 1825767424 + }, + { + "epoch": 0.11, + "learning_rate": 9.024233670357888e-05, + "loss": 2.6118, + "theoretical_loss": 3.456774661851995, + "tokens_seen": 1825898496 + }, + { + "epoch": 0.11, + "learning_rate": 9.023431230942064e-05, + "loss": 2.5575, + "theoretical_loss": 3.4567536006758157, + "tokens_seen": 1826029568 + }, + { + "epoch": 0.11, + "learning_rate": 9.022628791526239e-05, + "loss": 2.5875, + "theoretical_loss": 3.4567325414346093, + "tokens_seen": 1826160640 + }, + { + "epoch": 0.11, + "learning_rate": 9.021826352110416e-05, + "loss": 2.5273, + "theoretical_loss": 3.456711484128059, + "tokens_seen": 1826291712 + }, + { + "epoch": 0.11, + "learning_rate": 9.021023912694591e-05, + "loss": 2.6429, + "theoretical_loss": 3.456690428755847, + "tokens_seen": 1826422784 + }, + { + "epoch": 0.11, + "learning_rate": 9.020221473278768e-05, + "loss": 2.6974, + "theoretical_loss": 3.4566693753176576, + "tokens_seen": 1826553856 + }, + { + "epoch": 0.11, + "learning_rate": 9.019419033862943e-05, + "loss": 2.4875, + "theoretical_loss": 3.4566483238131753, + "tokens_seen": 1826684928 + }, + { + "epoch": 0.11, + "learning_rate": 9.01861659444712e-05, + "loss": 2.562, + "theoretical_loss": 3.4566272742420825, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1007268, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4686145782470703, + "objective/train/theoretical_loss": 3.4566062266040634, + "objective/train/tokens_used": 197406176, + "theoretical_loss": 3.4566062266040634, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.11, + "learning_rate": 9.017814155031295e-05, + "loss": 2.4638, + "theoretical_loss": 3.4566062266040634, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.11, + "learning_rate": 9.01701171561547e-05, + "loss": 2.4429, + "theoretical_loss": 3.4565851808988013, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.11, + "learning_rate": 9.016209276199647e-05, + "loss": 2.7966, + "theoretical_loss": 3.456564137125981, + "tokens_seen": 1827209216 + }, + { + "epoch": 0.11, + "learning_rate": 9.015406836783822e-05, + "loss": 2.4905, + "theoretical_loss": 3.4565430952852862, + "tokens_seen": 1827340288 + }, + { + "epoch": 0.11, + "learning_rate": 9.014604397367999e-05, + "loss": 2.517, + "theoretical_loss": 3.4565220553764004, + "tokens_seen": 1827471360 + }, + { + "epoch": 0.11, + "learning_rate": 9.013801957952174e-05, + "loss": 2.5002, + "theoretical_loss": 3.456501017399008, + "tokens_seen": 1827602432 + }, + { + "epoch": 0.11, + "learning_rate": 9.012999518536351e-05, + "loss": 2.5377, + "theoretical_loss": 3.4564799813527936, + "tokens_seen": 1827733504 + }, + { + "epoch": 0.11, + "learning_rate": 9.012197079120526e-05, + "loss": 2.5164, + "theoretical_loss": 3.4564589472374405, + "tokens_seen": 1827864576 + }, + { + "epoch": 0.11, + "learning_rate": 9.011394639704702e-05, + "loss": 2.6276, + "theoretical_loss": 3.456437915052634, + "tokens_seen": 1827995648 + }, + { + "epoch": 0.11, + "learning_rate": 9.010592200288878e-05, + "loss": 2.5136, + "theoretical_loss": 3.456416884798058, + "tokens_seen": 1828126720 + }, + { + "epoch": 0.11, + "learning_rate": 9.009789760873054e-05, + "loss": 2.4933, + "theoretical_loss": 3.456395856473397, + "tokens_seen": 1828257792 + }, + { + "epoch": 0.11, + "learning_rate": 9.00898732145723e-05, + "loss": 2.5778, + "theoretical_loss": 3.456374830078336, + "tokens_seen": 1828388864 + }, + { + "epoch": 0.11, + "learning_rate": 9.008184882041406e-05, + "loss": 2.4973, + "theoretical_loss": 3.456353805612559, + "tokens_seen": 1828519936 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1007941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.821141481399536, + "objective/train/theoretical_loss": 3.4563432941030534, + "objective/train/tokens_used": 199044576, + "theoretical_loss": 3.4563432941030534, + "tokens_seen": 1828585472 + }, + { + "epoch": 0.11, + "learning_rate": 9.007382442625582e-05, + "loss": 2.5797, + "theoretical_loss": 3.4563327830757506, + "tokens_seen": 1828651008 + }, + { + "epoch": 0.11, + "learning_rate": 9.006580003209758e-05, + "loss": 2.6534, + "theoretical_loss": 3.4563117624675965, + "tokens_seen": 1828782080 + }, + { + "epoch": 0.11, + "learning_rate": 9.005777563793933e-05, + "loss": 2.4451, + "theoretical_loss": 3.456290743787781, + "tokens_seen": 1828913152 + }, + { + "epoch": 0.11, + "learning_rate": 9.00497512437811e-05, + "loss": 2.5534, + "theoretical_loss": 3.4562697270359886, + "tokens_seen": 1829044224 + }, + { + "epoch": 0.11, + "learning_rate": 9.004172684962285e-05, + "loss": 2.6204, + "theoretical_loss": 3.456248712211905, + "tokens_seen": 1829175296 + }, + { + "epoch": 0.11, + "learning_rate": 9.003370245546462e-05, + "loss": 2.5441, + "theoretical_loss": 3.456227699315215, + "tokens_seen": 1829306368 + }, + { + "epoch": 0.11, + "learning_rate": 9.002567806130637e-05, + "loss": 2.5198, + "theoretical_loss": 3.4562066883456035, + "tokens_seen": 1829437440 + }, + { + "epoch": 0.11, + "learning_rate": 9.001765366714814e-05, + "loss": 2.3917, + "theoretical_loss": 3.4561856793027568, + "tokens_seen": 1829568512 + }, + { + "epoch": 0.11, + "learning_rate": 9.000962927298989e-05, + "loss": 2.5708, + "theoretical_loss": 3.4561646721863584, + "tokens_seen": 1829699584 + }, + { + "epoch": 0.11, + "learning_rate": 9.000160487883166e-05, + "loss": 2.6934, + "theoretical_loss": 3.4561436669960948, + "tokens_seen": 1829830656 + }, + { + "epoch": 0.11, + "learning_rate": 8.999358048467341e-05, + "loss": 2.5966, + "theoretical_loss": 3.4561226637316516, + "tokens_seen": 1829961728 + }, + { + "epoch": 0.11, + "learning_rate": 8.998555609051516e-05, + "loss": 2.5411, + "theoretical_loss": 3.456101662392714, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1009247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.360668659210205, + "objective/train/theoretical_loss": 3.456080662978967, + "objective/train/tokens_used": 200682976, + "theoretical_loss": 3.456080662978967, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.11, + "learning_rate": 8.997753169635693e-05, + "loss": 2.4355, + "theoretical_loss": 3.456080662978967, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.11, + "learning_rate": 8.996950730219868e-05, + "loss": 2.5936, + "theoretical_loss": 3.4560596654900975, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.11, + "learning_rate": 8.996148290804045e-05, + "loss": 2.5363, + "theoretical_loss": 3.45603866992579, + "tokens_seen": 1830486016 + }, + { + "epoch": 0.11, + "learning_rate": 8.99534585138822e-05, + "loss": 2.514, + "theoretical_loss": 3.4560176762857315, + "tokens_seen": 1830617088 + }, + { + "epoch": 0.11, + "learning_rate": 8.994543411972397e-05, + "loss": 2.4908, + "theoretical_loss": 3.455996684569607, + "tokens_seen": 1830748160 + }, + { + "epoch": 0.11, + "learning_rate": 8.993740972556572e-05, + "loss": 2.5477, + "theoretical_loss": 3.4559756947771025, + "tokens_seen": 1830879232 + }, + { + "epoch": 0.11, + "learning_rate": 8.992938533140747e-05, + "loss": 2.4969, + "theoretical_loss": 3.4559547069079044, + "tokens_seen": 1831010304 + }, + { + "epoch": 0.11, + "learning_rate": 8.992136093724924e-05, + "loss": 2.5748, + "theoretical_loss": 3.4559337209616987, + "tokens_seen": 1831141376 + }, + { + "epoch": 0.11, + "learning_rate": 8.9913336543091e-05, + "loss": 2.5423, + "theoretical_loss": 3.4559127369381715, + "tokens_seen": 1831272448 + }, + { + "epoch": 0.11, + "learning_rate": 8.990531214893276e-05, + "loss": 2.7193, + "theoretical_loss": 3.455891754837009, + "tokens_seen": 1831403520 + }, + { + "epoch": 0.11, + "learning_rate": 8.989728775477451e-05, + "loss": 2.4593, + "theoretical_loss": 3.455870774657898, + "tokens_seen": 1831534592 + }, + { + "epoch": 0.11, + "learning_rate": 8.988926336061628e-05, + "loss": 2.6618, + "theoretical_loss": 3.4558497964005244, + "tokens_seen": 1831665664 + }, + { + "epoch": 0.11, + "learning_rate": 8.988123896645803e-05, + "loss": 2.5454, + "theoretical_loss": 3.455828820064575, + "tokens_seen": 1831796736 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1009780, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6938259601593018, + "objective/train/theoretical_loss": 3.455818332617036, + "objective/train/tokens_used": 202321376, + "theoretical_loss": 3.455818332617036, + "tokens_seen": 1831862272 + }, + { + "epoch": 0.11, + "learning_rate": 8.987321457229979e-05, + "loss": 2.6755, + "theoretical_loss": 3.455807845649735, + "tokens_seen": 1831927808 + }, + { + "epoch": 0.11, + "learning_rate": 8.986519017814155e-05, + "loss": 2.6631, + "theoretical_loss": 3.4557868731556933, + "tokens_seen": 1832058880 + }, + { + "epoch": 0.11, + "learning_rate": 8.98571657839833e-05, + "loss": 2.7067, + "theoretical_loss": 3.4557659025821352, + "tokens_seen": 1832189952 + }, + { + "epoch": 0.11, + "learning_rate": 8.984914138982507e-05, + "loss": 2.4591, + "theoretical_loss": 3.455744933928748, + "tokens_seen": 1832321024 + }, + { + "epoch": 0.11, + "learning_rate": 8.984111699566683e-05, + "loss": 2.3704, + "theoretical_loss": 3.455723967195218, + "tokens_seen": 1832452096 + }, + { + "epoch": 0.11, + "learning_rate": 8.983309260150859e-05, + "loss": 2.5915, + "theoretical_loss": 3.4557030023812323, + "tokens_seen": 1832583168 + }, + { + "epoch": 0.11, + "learning_rate": 8.982506820735035e-05, + "loss": 2.576, + "theoretical_loss": 3.455682039486478, + "tokens_seen": 1832714240 + }, + { + "epoch": 0.11, + "learning_rate": 8.98170438131921e-05, + "loss": 2.5794, + "theoretical_loss": 3.4556610785106425, + "tokens_seen": 1832845312 + }, + { + "epoch": 0.11, + "learning_rate": 8.980901941903387e-05, + "loss": 2.6461, + "theoretical_loss": 3.4556401194534123, + "tokens_seen": 1832976384 + }, + { + "epoch": 0.11, + "learning_rate": 8.980099502487562e-05, + "loss": 2.632, + "theoretical_loss": 3.4556191623144747, + "tokens_seen": 1833107456 + }, + { + "epoch": 0.11, + "learning_rate": 8.979297063071739e-05, + "loss": 2.6446, + "theoretical_loss": 3.4555982070935176, + "tokens_seen": 1833238528 + }, + { + "epoch": 0.11, + "learning_rate": 8.978494623655914e-05, + "loss": 2.5226, + "theoretical_loss": 3.455577253790228, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1010758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.585409164428711, + "objective/train/theoretical_loss": 3.455556302404293, + "objective/train/tokens_used": 203959776, + "theoretical_loss": 3.455556302404293, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.11, + "learning_rate": 8.97769218424009e-05, + "loss": 2.5823, + "theoretical_loss": 3.455556302404293, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.11, + "learning_rate": 8.976889744824266e-05, + "loss": 2.422, + "theoretical_loss": 3.4555353529354003, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.11, + "learning_rate": 8.976087305408441e-05, + "loss": 2.6723, + "theoretical_loss": 3.455514405383237, + "tokens_seen": 1833762816 + }, + { + "epoch": 0.11, + "learning_rate": 8.975284865992618e-05, + "loss": 2.5495, + "theoretical_loss": 3.4554934597474922, + "tokens_seen": 1833893888 + }, + { + "epoch": 0.11, + "learning_rate": 8.974482426576793e-05, + "loss": 2.5599, + "theoretical_loss": 3.4554725160278528, + "tokens_seen": 1834024960 + }, + { + "epoch": 0.11, + "learning_rate": 8.97367998716097e-05, + "loss": 2.482, + "theoretical_loss": 3.4554515742240057, + "tokens_seen": 1834156032 + }, + { + "epoch": 0.11, + "learning_rate": 8.972877547745145e-05, + "loss": 2.6911, + "theoretical_loss": 3.45543063433564, + "tokens_seen": 1834287104 + }, + { + "epoch": 0.11, + "learning_rate": 8.972075108329322e-05, + "loss": 2.6872, + "theoretical_loss": 3.455409696362443, + "tokens_seen": 1834418176 + }, + { + "epoch": 0.11, + "learning_rate": 8.971272668913497e-05, + "loss": 2.5098, + "theoretical_loss": 3.4553887603041034, + "tokens_seen": 1834549248 + }, + { + "epoch": 0.11, + "learning_rate": 8.970470229497674e-05, + "loss": 2.7034, + "theoretical_loss": 3.4553678261603085, + "tokens_seen": 1834680320 + }, + { + "epoch": 0.11, + "learning_rate": 8.969667790081849e-05, + "loss": 2.482, + "theoretical_loss": 3.455346893930747, + "tokens_seen": 1834811392 + }, + { + "epoch": 0.11, + "learning_rate": 8.968865350666024e-05, + "loss": 2.5007, + "theoretical_loss": 3.4553259636151066, + "tokens_seen": 1834942464 + }, + { + "epoch": 0.11, + "learning_rate": 8.968062911250201e-05, + "loss": 2.5495, + "theoretical_loss": 3.455305035213076, + "tokens_seen": 1835073536 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1011050, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.749027729034424, + "objective/train/theoretical_loss": 3.455294571729567, + "objective/train/tokens_used": 205598176, + "theoretical_loss": 3.455294571729567, + "tokens_seen": 1835139072 + }, + { + "epoch": 0.11, + "learning_rate": 8.967260471834376e-05, + "loss": 2.7461, + "theoretical_loss": 3.455284108724344, + "tokens_seen": 1835204608 + }, + { + "epoch": 0.11, + "learning_rate": 8.966458032418553e-05, + "loss": 2.8043, + "theoretical_loss": 3.455263184148598, + "tokens_seen": 1835335680 + }, + { + "epoch": 0.11, + "learning_rate": 8.965655593002728e-05, + "loss": 2.5974, + "theoretical_loss": 3.455242261485527, + "tokens_seen": 1835466752 + }, + { + "epoch": 0.11, + "learning_rate": 8.964853153586905e-05, + "loss": 2.4865, + "theoretical_loss": 3.45522134073482, + "tokens_seen": 1835597824 + }, + { + "epoch": 0.11, + "learning_rate": 8.96405071417108e-05, + "loss": 2.6225, + "theoretical_loss": 3.455200421896165, + "tokens_seen": 1835728896 + }, + { + "epoch": 0.11, + "learning_rate": 8.963248274755256e-05, + "loss": 2.6943, + "theoretical_loss": 3.4551795049692515, + "tokens_seen": 1835859968 + }, + { + "epoch": 0.11, + "learning_rate": 8.962445835339432e-05, + "loss": 2.5456, + "theoretical_loss": 3.455158589953768, + "tokens_seen": 1835991040 + }, + { + "epoch": 0.11, + "learning_rate": 8.961643395923608e-05, + "loss": 2.5191, + "theoretical_loss": 3.455137676849403, + "tokens_seen": 1836122112 + }, + { + "epoch": 0.11, + "learning_rate": 8.960840956507784e-05, + "loss": 2.6647, + "theoretical_loss": 3.4551167656558457, + "tokens_seen": 1836253184 + }, + { + "epoch": 0.11, + "learning_rate": 8.96003851709196e-05, + "loss": 2.5848, + "theoretical_loss": 3.4550958563727856, + "tokens_seen": 1836384256 + }, + { + "epoch": 0.11, + "learning_rate": 8.959236077676136e-05, + "loss": 2.5178, + "theoretical_loss": 3.4550749489999113, + "tokens_seen": 1836515328 + }, + { + "epoch": 0.11, + "learning_rate": 8.958433638260312e-05, + "loss": 2.5931, + "theoretical_loss": 3.455054043536912, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1012285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6171936988830566, + "objective/train/theoretical_loss": 3.4550331399834766, + "objective/train/tokens_used": 207236576, + "theoretical_loss": 3.4550331399834766, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.11, + "learning_rate": 8.957631198844487e-05, + "loss": 2.5262, + "theoretical_loss": 3.4550331399834766, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.11, + "learning_rate": 8.956828759428664e-05, + "loss": 2.6416, + "theoretical_loss": 3.455012238339296, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 8.956026320012839e-05, + "loss": 2.5346, + "theoretical_loss": 3.4549913386040574, + "tokens_seen": 1837039616 + }, + { + "epoch": 0.11, + "learning_rate": 8.955223880597016e-05, + "loss": 2.6475, + "theoretical_loss": 3.4549704407774517, + "tokens_seen": 1837170688 + }, + { + "epoch": 0.11, + "learning_rate": 8.954421441181191e-05, + "loss": 2.7004, + "theoretical_loss": 3.4549495448591685, + "tokens_seen": 1837301760 + }, + { + "epoch": 0.11, + "learning_rate": 8.953619001765368e-05, + "loss": 2.6106, + "theoretical_loss": 3.4549286508488963, + "tokens_seen": 1837432832 + }, + { + "epoch": 0.11, + "learning_rate": 8.952816562349543e-05, + "loss": 2.5996, + "theoretical_loss": 3.4549077587463257, + "tokens_seen": 1837563904 + }, + { + "epoch": 0.11, + "learning_rate": 8.952014122933718e-05, + "loss": 2.8679, + "theoretical_loss": 3.4548868685511467, + "tokens_seen": 1837694976 + }, + { + "epoch": 0.11, + "learning_rate": 8.951211683517895e-05, + "loss": 2.6206, + "theoretical_loss": 3.4548659802630484, + "tokens_seen": 1837826048 + }, + { + "epoch": 0.11, + "learning_rate": 8.95040924410207e-05, + "loss": 2.4902, + "theoretical_loss": 3.4548450938817212, + "tokens_seen": 1837957120 + }, + { + "epoch": 0.11, + "learning_rate": 8.949606804686247e-05, + "loss": 2.5685, + "theoretical_loss": 3.454824209406855, + "tokens_seen": 1838088192 + }, + { + "epoch": 0.11, + "learning_rate": 8.948804365270422e-05, + "loss": 2.6072, + "theoretical_loss": 3.4548033268381397, + "tokens_seen": 1838219264 + }, + { + "epoch": 0.11, + "learning_rate": 8.948001925854599e-05, + "loss": 2.5037, + "theoretical_loss": 3.4547824461752654, + "tokens_seen": 1838350336 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1012947, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6320648193359375, + "objective/train/theoretical_loss": 3.4547720065584215, + "objective/train/tokens_used": 208874976, + "theoretical_loss": 3.4547720065584215, + "tokens_seen": 1838415872 + }, + { + "epoch": 0.11, + "learning_rate": 8.947199486438774e-05, + "loss": 2.6496, + "theoretical_loss": 3.454761567417922, + "tokens_seen": 1838481408 + }, + { + "epoch": 0.11, + "learning_rate": 8.94639704702295e-05, + "loss": 2.6935, + "theoretical_loss": 3.4547406905658002, + "tokens_seen": 1838612480 + }, + { + "epoch": 0.11, + "learning_rate": 8.945594607607126e-05, + "loss": 2.5356, + "theoretical_loss": 3.454719815618591, + "tokens_seen": 1838743552 + }, + { + "epoch": 0.11, + "learning_rate": 8.944792168191301e-05, + "loss": 2.5054, + "theoretical_loss": 3.4546989425759835, + "tokens_seen": 1838874624 + }, + { + "epoch": 0.11, + "learning_rate": 8.943989728775478e-05, + "loss": 2.663, + "theoretical_loss": 3.4546780714376695, + "tokens_seen": 1839005696 + }, + { + "epoch": 0.11, + "learning_rate": 8.943187289359653e-05, + "loss": 2.5547, + "theoretical_loss": 3.4546572022033377, + "tokens_seen": 1839136768 + }, + { + "epoch": 0.11, + "learning_rate": 8.94238484994383e-05, + "loss": 2.3293, + "theoretical_loss": 3.4546363348726805, + "tokens_seen": 1839267840 + }, + { + "epoch": 0.11, + "learning_rate": 8.941582410528005e-05, + "loss": 2.6209, + "theoretical_loss": 3.4546154694453883, + "tokens_seen": 1839398912 + }, + { + "epoch": 0.11, + "learning_rate": 8.94077997111218e-05, + "loss": 2.5797, + "theoretical_loss": 3.454594605921151, + "tokens_seen": 1839529984 + }, + { + "epoch": 0.11, + "learning_rate": 8.939977531696357e-05, + "loss": 2.6429, + "theoretical_loss": 3.454573744299661, + "tokens_seen": 1839661056 + }, + { + "epoch": 0.12, + "learning_rate": 8.939175092280533e-05, + "loss": 2.6145, + "theoretical_loss": 3.454552884580607, + "tokens_seen": 1839792128 + }, + { + "epoch": 0.12, + "learning_rate": 8.93837265286471e-05, + "loss": 2.5151, + "theoretical_loss": 3.454532026763682, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1014005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.342764139175415, + "objective/train/theoretical_loss": 3.4545111708485763, + "objective/train/tokens_used": 210513376, + "theoretical_loss": 3.4545111708485763, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.12, + "learning_rate": 8.937570213448885e-05, + "loss": 2.5643, + "theoretical_loss": 3.4545111708485763, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.12, + "learning_rate": 8.936767774033061e-05, + "loss": 2.7472, + "theoretical_loss": 3.4544903168349808, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.12, + "learning_rate": 8.935965334617237e-05, + "loss": 2.6095, + "theoretical_loss": 3.454469464722587, + "tokens_seen": 1840316416 + }, + { + "epoch": 0.12, + "learning_rate": 8.935162895201413e-05, + "loss": 2.5621, + "theoretical_loss": 3.4544486145110858, + "tokens_seen": 1840447488 + }, + { + "epoch": 0.12, + "learning_rate": 8.934360455785589e-05, + "loss": 2.4646, + "theoretical_loss": 3.4544277662001694, + "tokens_seen": 1840578560 + }, + { + "epoch": 0.12, + "learning_rate": 8.933558016369764e-05, + "loss": 2.5613, + "theoretical_loss": 3.4544069197895286, + "tokens_seen": 1840709632 + }, + { + "epoch": 0.12, + "learning_rate": 8.93275557695394e-05, + "loss": 2.6725, + "theoretical_loss": 3.454386075278855, + "tokens_seen": 1840840704 + }, + { + "epoch": 0.12, + "learning_rate": 8.931953137538116e-05, + "loss": 2.6032, + "theoretical_loss": 3.4543652326678402, + "tokens_seen": 1840971776 + }, + { + "epoch": 0.12, + "learning_rate": 8.931150698122293e-05, + "loss": 2.5783, + "theoretical_loss": 3.454344391956176, + "tokens_seen": 1841102848 + }, + { + "epoch": 0.12, + "learning_rate": 8.930348258706468e-05, + "loss": 2.541, + "theoretical_loss": 3.4543235531435537, + "tokens_seen": 1841233920 + }, + { + "epoch": 0.12, + "learning_rate": 8.929545819290645e-05, + "loss": 2.5714, + "theoretical_loss": 3.454302716229665, + "tokens_seen": 1841364992 + }, + { + "epoch": 0.12, + "learning_rate": 8.92874337987482e-05, + "loss": 2.6278, + "theoretical_loss": 3.454281881214203, + "tokens_seen": 1841496064 + }, + { + "epoch": 0.12, + "learning_rate": 8.927940940458995e-05, + "loss": 2.5964, + "theoretical_loss": 3.454261048096858, + "tokens_seen": 1841627136 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1015263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.241819381713867, + "objective/train/theoretical_loss": 3.4542506322498836, + "objective/train/tokens_used": 212151776, + "theoretical_loss": 3.4542506322498836, + "tokens_seen": 1841692672 + }, + { + "epoch": 0.12, + "learning_rate": 8.927138501043172e-05, + "loss": 2.6414, + "theoretical_loss": 3.454240216877323, + "tokens_seen": 1841758208 + }, + { + "epoch": 0.12, + "learning_rate": 8.926336061627347e-05, + "loss": 2.5861, + "theoretical_loss": 3.45421938755529, + "tokens_seen": 1841889280 + }, + { + "epoch": 0.12, + "learning_rate": 8.925533622211524e-05, + "loss": 2.6131, + "theoretical_loss": 3.4541985601304503, + "tokens_seen": 1842020352 + }, + { + "epoch": 0.12, + "learning_rate": 8.924731182795699e-05, + "loss": 2.6827, + "theoretical_loss": 3.4541777346024976, + "tokens_seen": 1842151424 + }, + { + "epoch": 0.12, + "learning_rate": 8.923928743379876e-05, + "loss": 2.583, + "theoretical_loss": 3.4541569109711228, + "tokens_seen": 1842282496 + }, + { + "epoch": 0.12, + "learning_rate": 8.923126303964051e-05, + "loss": 2.5736, + "theoretical_loss": 3.4541360892360187, + "tokens_seen": 1842413568 + }, + { + "epoch": 0.12, + "learning_rate": 8.922323864548226e-05, + "loss": 2.6615, + "theoretical_loss": 3.4541152693968784, + "tokens_seen": 1842544640 + }, + { + "epoch": 0.12, + "learning_rate": 8.921521425132403e-05, + "loss": 2.6997, + "theoretical_loss": 3.4540944514533933, + "tokens_seen": 1842675712 + }, + { + "epoch": 0.12, + "learning_rate": 8.920718985716578e-05, + "loss": 2.5494, + "theoretical_loss": 3.4540736354052575, + "tokens_seen": 1842806784 + }, + { + "epoch": 0.12, + "learning_rate": 8.919916546300755e-05, + "loss": 2.6537, + "theoretical_loss": 3.4540528212521617, + "tokens_seen": 1842937856 + }, + { + "epoch": 0.12, + "learning_rate": 8.91911410688493e-05, + "loss": 2.5037, + "theoretical_loss": 3.4540320089938, + "tokens_seen": 1843068928 + }, + { + "epoch": 0.12, + "learning_rate": 8.918311667469107e-05, + "loss": 2.5411, + "theoretical_loss": 3.454011198629865, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1015948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7754783630371094, + "objective/train/theoretical_loss": 3.453990390160049, + "objective/train/tokens_used": 213790176, + "theoretical_loss": 3.453990390160049, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.12, + "learning_rate": 8.917509228053282e-05, + "loss": 2.6023, + "theoretical_loss": 3.453990390160049, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.12, + "learning_rate": 8.916706788637458e-05, + "loss": 2.6114, + "theoretical_loss": 3.453969583584045, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.12, + "learning_rate": 8.915904349221634e-05, + "loss": 2.6495, + "theoretical_loss": 3.453948778901547, + "tokens_seen": 1843593216 + }, + { + "epoch": 0.12, + "learning_rate": 8.91510190980581e-05, + "loss": 2.5029, + "theoretical_loss": 3.4539279761122472, + "tokens_seen": 1843724288 + }, + { + "epoch": 0.12, + "learning_rate": 8.914299470389986e-05, + "loss": 2.4838, + "theoretical_loss": 3.453907175215839, + "tokens_seen": 1843855360 + }, + { + "epoch": 0.12, + "learning_rate": 8.913497030974162e-05, + "loss": 2.468, + "theoretical_loss": 3.453886376212015, + "tokens_seen": 1843986432 + }, + { + "epoch": 0.12, + "learning_rate": 8.912694591558338e-05, + "loss": 2.4687, + "theoretical_loss": 3.4538655791004693, + "tokens_seen": 1844117504 + }, + { + "epoch": 0.12, + "learning_rate": 8.911892152142514e-05, + "loss": 2.4653, + "theoretical_loss": 3.4538447838808954, + "tokens_seen": 1844248576 + }, + { + "epoch": 0.12, + "learning_rate": 8.911089712726689e-05, + "loss": 2.7463, + "theoretical_loss": 3.453823990552986, + "tokens_seen": 1844379648 + }, + { + "epoch": 0.12, + "learning_rate": 8.910287273310866e-05, + "loss": 2.459, + "theoretical_loss": 3.453803199116435, + "tokens_seen": 1844510720 + }, + { + "epoch": 0.12, + "learning_rate": 8.909484833895041e-05, + "loss": 2.4994, + "theoretical_loss": 3.4537824095709366, + "tokens_seen": 1844641792 + }, + { + "epoch": 0.12, + "learning_rate": 8.908682394479218e-05, + "loss": 2.4943, + "theoretical_loss": 3.4537616219161835, + "tokens_seen": 1844772864 + }, + { + "epoch": 0.12, + "learning_rate": 8.907879955063393e-05, + "loss": 2.5477, + "theoretical_loss": 3.4537408361518693, + "tokens_seen": 1844903936 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1017203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5314950942993164, + "objective/train/theoretical_loss": 3.4537304439785315, + "objective/train/tokens_used": 215428576, + "theoretical_loss": 3.4537304439785315, + "tokens_seen": 1844969472 + }, + { + "epoch": 0.12, + "learning_rate": 8.90707751564757e-05, + "loss": 2.4796, + "theoretical_loss": 3.4537200522776885, + "tokens_seen": 1845035008 + }, + { + "epoch": 0.12, + "learning_rate": 8.906275076231745e-05, + "loss": 2.7601, + "theoretical_loss": 3.4536992702933347, + "tokens_seen": 1845166080 + }, + { + "epoch": 0.12, + "learning_rate": 8.905472636815922e-05, + "loss": 2.4913, + "theoretical_loss": 3.4536784901985014, + "tokens_seen": 1845297152 + }, + { + "epoch": 0.12, + "learning_rate": 8.904670197400097e-05, + "loss": 2.5899, + "theoretical_loss": 3.4536577119928835, + "tokens_seen": 1845428224 + }, + { + "epoch": 0.12, + "learning_rate": 8.903867757984272e-05, + "loss": 2.3805, + "theoretical_loss": 3.4536369356761742, + "tokens_seen": 1845559296 + }, + { + "epoch": 0.12, + "learning_rate": 8.903065318568449e-05, + "loss": 2.6224, + "theoretical_loss": 3.4536161612480685, + "tokens_seen": 1845690368 + }, + { + "epoch": 0.12, + "learning_rate": 8.902262879152624e-05, + "loss": 2.5781, + "theoretical_loss": 3.4535953887082598, + "tokens_seen": 1845821440 + }, + { + "epoch": 0.12, + "learning_rate": 8.901460439736801e-05, + "loss": 2.5903, + "theoretical_loss": 3.453574618056443, + "tokens_seen": 1845952512 + }, + { + "epoch": 0.12, + "learning_rate": 8.900658000320976e-05, + "loss": 2.5429, + "theoretical_loss": 3.4535538492923123, + "tokens_seen": 1846083584 + }, + { + "epoch": 0.12, + "learning_rate": 8.899855560905153e-05, + "loss": 2.5974, + "theoretical_loss": 3.4535330824155617, + "tokens_seen": 1846214656 + }, + { + "epoch": 0.12, + "learning_rate": 8.899053121489328e-05, + "loss": 2.464, + "theoretical_loss": 3.4535123174258864, + "tokens_seen": 1846345728 + }, + { + "epoch": 0.12, + "learning_rate": 8.898250682073503e-05, + "loss": 2.6167, + "theoretical_loss": 3.4534915543229805, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1017618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7826731204986572, + "objective/train/theoretical_loss": 3.453470793106539, + "objective/train/tokens_used": 217066976, + "theoretical_loss": 3.453470793106539, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.12, + "learning_rate": 8.89744824265768e-05, + "loss": 2.5319, + "theoretical_loss": 3.453470793106539, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.12, + "learning_rate": 8.896645803241855e-05, + "loss": 2.6329, + "theoretical_loss": 3.453450033776256, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.12, + "learning_rate": 8.895843363826032e-05, + "loss": 2.4089, + "theoretical_loss": 3.4534292763318275, + "tokens_seen": 1846870016 + }, + { + "epoch": 0.12, + "learning_rate": 8.895040924410207e-05, + "loss": 2.5541, + "theoretical_loss": 3.453408520772947, + "tokens_seen": 1847001088 + }, + { + "epoch": 0.12, + "learning_rate": 8.894238484994384e-05, + "loss": 2.5275, + "theoretical_loss": 3.4533877670993096, + "tokens_seen": 1847132160 + }, + { + "epoch": 0.12, + "learning_rate": 8.89343604557856e-05, + "loss": 2.4709, + "theoretical_loss": 3.4533670153106115, + "tokens_seen": 1847263232 + }, + { + "epoch": 0.12, + "learning_rate": 8.892633606162735e-05, + "loss": 2.4936, + "theoretical_loss": 3.4533462654065463, + "tokens_seen": 1847394304 + }, + { + "epoch": 0.12, + "learning_rate": 8.891831166746911e-05, + "loss": 2.6244, + "theoretical_loss": 3.4533255173868103, + "tokens_seen": 1847525376 + }, + { + "epoch": 0.12, + "learning_rate": 8.891028727331087e-05, + "loss": 2.5891, + "theoretical_loss": 3.4533047712510982, + "tokens_seen": 1847656448 + }, + { + "epoch": 0.12, + "learning_rate": 8.890226287915263e-05, + "loss": 2.4386, + "theoretical_loss": 3.453284026999105, + "tokens_seen": 1847787520 + }, + { + "epoch": 0.12, + "learning_rate": 8.889423848499439e-05, + "loss": 2.5154, + "theoretical_loss": 3.453263284630527, + "tokens_seen": 1847918592 + }, + { + "epoch": 0.12, + "learning_rate": 8.888621409083615e-05, + "loss": 2.5096, + "theoretical_loss": 3.4532425441450587, + "tokens_seen": 1848049664 + }, + { + "epoch": 0.12, + "learning_rate": 8.88781896966779e-05, + "loss": 2.5441, + "theoretical_loss": 3.4532218055423956, + "tokens_seen": 1848180736 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1018803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3100316524505615, + "objective/train/theoretical_loss": 3.4532114369470213, + "objective/train/tokens_used": 218705376, + "theoretical_loss": 3.4532114369470213, + "tokens_seen": 1848246272 + }, + { + "epoch": 0.12, + "learning_rate": 8.887016530251966e-05, + "loss": 2.3515, + "theoretical_loss": 3.453201068822234, + "tokens_seen": 1848311808 + }, + { + "epoch": 0.12, + "learning_rate": 8.886214090836143e-05, + "loss": 2.4272, + "theoretical_loss": 3.4531803339842693, + "tokens_seen": 1848442880 + }, + { + "epoch": 0.12, + "learning_rate": 8.885411651420318e-05, + "loss": 2.4624, + "theoretical_loss": 3.4531596010281964, + "tokens_seen": 1848573952 + }, + { + "epoch": 0.12, + "learning_rate": 8.884609212004495e-05, + "loss": 2.5151, + "theoretical_loss": 3.453138869953712, + "tokens_seen": 1848705024 + }, + { + "epoch": 0.12, + "learning_rate": 8.88380677258867e-05, + "loss": 2.622, + "theoretical_loss": 3.4531181407605116, + "tokens_seen": 1848836096 + }, + { + "epoch": 0.12, + "learning_rate": 8.883004333172847e-05, + "loss": 2.6775, + "theoretical_loss": 3.4530974134482917, + "tokens_seen": 1848967168 + }, + { + "epoch": 0.12, + "learning_rate": 8.882201893757022e-05, + "loss": 2.564, + "theoretical_loss": 3.4530766880167474, + "tokens_seen": 1849098240 + }, + { + "epoch": 0.12, + "learning_rate": 8.881399454341197e-05, + "loss": 2.6534, + "theoretical_loss": 3.4530559644655754, + "tokens_seen": 1849229312 + }, + { + "epoch": 0.12, + "learning_rate": 8.880597014925374e-05, + "loss": 2.5453, + "theoretical_loss": 3.4530352427944715, + "tokens_seen": 1849360384 + }, + { + "epoch": 0.12, + "learning_rate": 8.879794575509549e-05, + "loss": 2.6126, + "theoretical_loss": 3.453014523003132, + "tokens_seen": 1849491456 + }, + { + "epoch": 0.12, + "learning_rate": 8.878992136093726e-05, + "loss": 2.5405, + "theoretical_loss": 3.452993805091253, + "tokens_seen": 1849622528 + }, + { + "epoch": 0.12, + "learning_rate": 8.878189696677901e-05, + "loss": 2.6833, + "theoretical_loss": 3.4529730890585313, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1019322, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4879612922668457, + "objective/train/theoretical_loss": 3.4529523749046627, + "objective/train/tokens_used": 220343776, + "theoretical_loss": 3.4529523749046627, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.12, + "learning_rate": 8.877387257262078e-05, + "loss": 2.5839, + "theoretical_loss": 3.4529523749046627, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.12, + "learning_rate": 8.876584817846253e-05, + "loss": 2.5506, + "theoretical_loss": 3.4529316626293443, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 8.87578237843043e-05, + "loss": 2.5309, + "theoretical_loss": 3.4529109522322727, + "tokens_seen": 1850146816 + }, + { + "epoch": 0.12, + "learning_rate": 8.874979939014605e-05, + "loss": 2.5407, + "theoretical_loss": 3.452890243713144, + "tokens_seen": 1850277888 + }, + { + "epoch": 0.12, + "learning_rate": 8.87417749959878e-05, + "loss": 2.4533, + "theoretical_loss": 3.452869537071655, + "tokens_seen": 1850408960 + }, + { + "epoch": 0.12, + "learning_rate": 8.873375060182957e-05, + "loss": 2.5144, + "theoretical_loss": 3.4528488323075024, + "tokens_seen": 1850540032 + }, + { + "epoch": 0.12, + "learning_rate": 8.872572620767132e-05, + "loss": 2.4951, + "theoretical_loss": 3.452828129420384, + "tokens_seen": 1850671104 + }, + { + "epoch": 0.12, + "learning_rate": 8.871770181351309e-05, + "loss": 2.5968, + "theoretical_loss": 3.4528074284099954, + "tokens_seen": 1850802176 + }, + { + "epoch": 0.12, + "learning_rate": 8.870967741935484e-05, + "loss": 2.7107, + "theoretical_loss": 3.452786729276034, + "tokens_seen": 1850933248 + }, + { + "epoch": 0.12, + "learning_rate": 8.870165302519661e-05, + "loss": 2.4595, + "theoretical_loss": 3.4527660320181974, + "tokens_seen": 1851064320 + }, + { + "epoch": 0.12, + "learning_rate": 8.869362863103836e-05, + "loss": 2.5775, + "theoretical_loss": 3.452745336636182, + "tokens_seen": 1851195392 + }, + { + "epoch": 0.12, + "learning_rate": 8.868560423688012e-05, + "loss": 2.627, + "theoretical_loss": 3.4527246431296854, + "tokens_seen": 1851326464 + }, + { + "epoch": 0.12, + "learning_rate": 8.867757984272188e-05, + "loss": 2.5698, + "theoretical_loss": 3.452703951498405, + "tokens_seen": 1851457536 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1019593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.545363426208496, + "objective/train/theoretical_loss": 3.452693606385876, + "objective/train/tokens_used": 221982176, + "theoretical_loss": 3.452693606385876, + "tokens_seen": 1851523072 + }, + { + "epoch": 0.12, + "learning_rate": 8.866955544856364e-05, + "loss": 2.6297, + "theoretical_loss": 3.452683261742038, + "tokens_seen": 1851588608 + }, + { + "epoch": 0.12, + "learning_rate": 8.86615310544054e-05, + "loss": 2.664, + "theoretical_loss": 3.452662573860281, + "tokens_seen": 1851719680 + }, + { + "epoch": 0.12, + "learning_rate": 8.865350666024716e-05, + "loss": 2.5098, + "theoretical_loss": 3.4526418878528333, + "tokens_seen": 1851850752 + }, + { + "epoch": 0.12, + "learning_rate": 8.864548226608892e-05, + "loss": 2.5979, + "theoretical_loss": 3.452621203719391, + "tokens_seen": 1851981824 + }, + { + "epoch": 0.12, + "learning_rate": 8.863745787193068e-05, + "loss": 2.6701, + "theoretical_loss": 3.452600521459652, + "tokens_seen": 1852112896 + }, + { + "epoch": 0.12, + "learning_rate": 8.862943347777243e-05, + "loss": 2.5514, + "theoretical_loss": 3.4525798410733133, + "tokens_seen": 1852243968 + }, + { + "epoch": 0.12, + "learning_rate": 8.86214090836142e-05, + "loss": 2.615, + "theoretical_loss": 3.4525591625600742, + "tokens_seen": 1852375040 + }, + { + "epoch": 0.12, + "learning_rate": 8.861338468945595e-05, + "loss": 2.4747, + "theoretical_loss": 3.4525384859196313, + "tokens_seen": 1852506112 + }, + { + "epoch": 0.12, + "learning_rate": 8.860536029529772e-05, + "loss": 2.5322, + "theoretical_loss": 3.4525178111516834, + "tokens_seen": 1852637184 + }, + { + "epoch": 0.12, + "learning_rate": 8.859733590113947e-05, + "loss": 2.6186, + "theoretical_loss": 3.4524971382559277, + "tokens_seen": 1852768256 + }, + { + "epoch": 0.12, + "learning_rate": 8.858931150698124e-05, + "loss": 2.5473, + "theoretical_loss": 3.4524764672320627, + "tokens_seen": 1852899328 + }, + { + "epoch": 0.12, + "learning_rate": 8.858128711282299e-05, + "loss": 2.5329, + "theoretical_loss": 3.452455798079786, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1020710, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7822999954223633, + "objective/train/theoretical_loss": 3.4524351307987966, + "objective/train/tokens_used": 223620576, + "theoretical_loss": 3.4524351307987966, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 8.857326271866474e-05, + "loss": 2.7127, + "theoretical_loss": 3.4524351307987966, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 8.856523832450651e-05, + "loss": 2.6761, + "theoretical_loss": 3.452414465388792, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 8.855721393034826e-05, + "loss": 2.5513, + "theoretical_loss": 3.4523938018494706, + "tokens_seen": 1853423616 + }, + { + "epoch": 0.12, + "learning_rate": 8.854918953619003e-05, + "loss": 2.5548, + "theoretical_loss": 3.4523731401805313, + "tokens_seen": 1853554688 + }, + { + "epoch": 0.12, + "learning_rate": 8.854116514203178e-05, + "loss": 2.7371, + "theoretical_loss": 3.4523524803816716, + "tokens_seen": 1853685760 + }, + { + "epoch": 0.12, + "learning_rate": 8.853314074787355e-05, + "loss": 2.5652, + "theoretical_loss": 3.4523318224525914, + "tokens_seen": 1853816832 + }, + { + "epoch": 0.12, + "learning_rate": 8.85251163537153e-05, + "loss": 2.6217, + "theoretical_loss": 3.452311166392988, + "tokens_seen": 1853947904 + }, + { + "epoch": 0.12, + "learning_rate": 8.851709195955705e-05, + "loss": 2.6013, + "theoretical_loss": 3.452290512202561, + "tokens_seen": 1854078976 + }, + { + "epoch": 0.12, + "learning_rate": 8.850906756539882e-05, + "loss": 2.4873, + "theoretical_loss": 3.4522698598810084, + "tokens_seen": 1854210048 + }, + { + "epoch": 0.12, + "learning_rate": 8.850104317124057e-05, + "loss": 2.5747, + "theoretical_loss": 3.4522492094280297, + "tokens_seen": 1854341120 + }, + { + "epoch": 0.12, + "learning_rate": 8.849301877708234e-05, + "loss": 2.4675, + "theoretical_loss": 3.4522285608433236, + "tokens_seen": 1854472192 + }, + { + "epoch": 0.12, + "learning_rate": 8.84849943829241e-05, + "loss": 2.6827, + "theoretical_loss": 3.4522079141265882, + "tokens_seen": 1854603264 + }, + { + "epoch": 0.12, + "learning_rate": 8.847696998876586e-05, + "loss": 2.6778, + "theoretical_loss": 3.4521872692775237, + "tokens_seen": 1854734336 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1021316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.845196485519409, + "objective/train/theoretical_loss": 3.4521769475532738, + "objective/train/tokens_used": 225258976, + "theoretical_loss": 3.4521769475532738, + "tokens_seen": 1854799872 + }, + { + "epoch": 0.12, + "learning_rate": 8.846894559460761e-05, + "loss": 2.6202, + "theoretical_loss": 3.4521666262958286, + "tokens_seen": 1854865408 + }, + { + "epoch": 0.12, + "learning_rate": 8.846092120044937e-05, + "loss": 2.4954, + "theoretical_loss": 3.452145985181202, + "tokens_seen": 1854996480 + }, + { + "epoch": 0.12, + "learning_rate": 8.845289680629113e-05, + "loss": 2.6478, + "theoretical_loss": 3.452125345933343, + "tokens_seen": 1855127552 + }, + { + "epoch": 0.12, + "learning_rate": 8.844487241213289e-05, + "loss": 2.4197, + "theoretical_loss": 3.4521047085519516, + "tokens_seen": 1855258624 + }, + { + "epoch": 0.12, + "learning_rate": 8.843684801797465e-05, + "loss": 2.4694, + "theoretical_loss": 3.4520840730367266, + "tokens_seen": 1855389696 + }, + { + "epoch": 0.12, + "learning_rate": 8.842882362381641e-05, + "loss": 2.6215, + "theoretical_loss": 3.4520634393873673, + "tokens_seen": 1855520768 + }, + { + "epoch": 0.12, + "learning_rate": 8.842079922965817e-05, + "loss": 2.4711, + "theoretical_loss": 3.452042807603574, + "tokens_seen": 1855651840 + }, + { + "epoch": 0.12, + "learning_rate": 8.841277483549993e-05, + "loss": 2.5541, + "theoretical_loss": 3.452022177685045, + "tokens_seen": 1855782912 + }, + { + "epoch": 0.12, + "learning_rate": 8.84047504413417e-05, + "loss": 2.691, + "theoretical_loss": 3.452001549631481, + "tokens_seen": 1855913984 + }, + { + "epoch": 0.12, + "learning_rate": 8.839672604718345e-05, + "loss": 2.7246, + "theoretical_loss": 3.4519809234425813, + "tokens_seen": 1856045056 + }, + { + "epoch": 0.12, + "learning_rate": 8.83887016530252e-05, + "loss": 2.3307, + "theoretical_loss": 3.451960299118046, + "tokens_seen": 1856176128 + }, + { + "epoch": 0.13, + "learning_rate": 8.838067725886697e-05, + "loss": 2.498, + "theoretical_loss": 3.4519396766575747, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1022409, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1355769634246826, + "objective/train/theoretical_loss": 3.4519190560608672, + "objective/train/tokens_used": 226897376, + "theoretical_loss": 3.4519190560608672, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.13, + "learning_rate": 8.837265286470872e-05, + "loss": 2.5153, + "theoretical_loss": 3.4519190560608672, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.13, + "learning_rate": 8.836462847055049e-05, + "loss": 2.4026, + "theoretical_loss": 3.4518984373276234, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.13, + "learning_rate": 8.835660407639224e-05, + "loss": 2.5265, + "theoretical_loss": 3.4518778204575433, + "tokens_seen": 1856700416 + }, + { + "epoch": 0.13, + "learning_rate": 8.8348579682234e-05, + "loss": 2.4881, + "theoretical_loss": 3.4518572054503283, + "tokens_seen": 1856831488 + }, + { + "epoch": 0.13, + "learning_rate": 8.834055528807576e-05, + "loss": 2.5827, + "theoretical_loss": 3.4518365923056766, + "tokens_seen": 1856962560 + }, + { + "epoch": 0.13, + "learning_rate": 8.833253089391751e-05, + "loss": 2.6521, + "theoretical_loss": 3.45181598102329, + "tokens_seen": 1857093632 + }, + { + "epoch": 0.13, + "learning_rate": 8.832450649975928e-05, + "loss": 2.5918, + "theoretical_loss": 3.451795371602868, + "tokens_seen": 1857224704 + }, + { + "epoch": 0.13, + "learning_rate": 8.831648210560103e-05, + "loss": 2.5057, + "theoretical_loss": 3.451774764044112, + "tokens_seen": 1857355776 + }, + { + "epoch": 0.13, + "learning_rate": 8.83084577114428e-05, + "loss": 2.551, + "theoretical_loss": 3.4517541583467213, + "tokens_seen": 1857486848 + }, + { + "epoch": 0.13, + "learning_rate": 8.830043331728455e-05, + "loss": 2.5057, + "theoretical_loss": 3.4517335545103967, + "tokens_seen": 1857617920 + }, + { + "epoch": 0.13, + "learning_rate": 8.829240892312632e-05, + "loss": 2.4116, + "theoretical_loss": 3.45171295253484, + "tokens_seen": 1857748992 + }, + { + "epoch": 0.13, + "learning_rate": 8.828438452896807e-05, + "loss": 2.5689, + "theoretical_loss": 3.45169235241975, + "tokens_seen": 1857880064 + }, + { + "epoch": 0.13, + "learning_rate": 8.827636013480982e-05, + "loss": 2.4917, + "theoretical_loss": 3.4516717541648285, + "tokens_seen": 1858011136 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1023672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6751821041107178, + "objective/train/theoretical_loss": 3.4516614557348375, + "objective/train/tokens_used": 228535776, + "theoretical_loss": 3.4516614557348375, + "tokens_seen": 1858076672 + }, + { + "epoch": 0.13, + "learning_rate": 8.826833574065159e-05, + "loss": 2.6894, + "theoretical_loss": 3.451651157769777, + "tokens_seen": 1858142208 + }, + { + "epoch": 0.13, + "learning_rate": 8.826031134649334e-05, + "loss": 2.6551, + "theoretical_loss": 3.4516305632342945, + "tokens_seen": 1858273280 + }, + { + "epoch": 0.13, + "learning_rate": 8.825228695233511e-05, + "loss": 2.5355, + "theoretical_loss": 3.451609970558083, + "tokens_seen": 1858404352 + }, + { + "epoch": 0.13, + "learning_rate": 8.824426255817686e-05, + "loss": 2.499, + "theoretical_loss": 3.4515893797408443, + "tokens_seen": 1858535424 + }, + { + "epoch": 0.13, + "learning_rate": 8.823623816401863e-05, + "loss": 2.6607, + "theoretical_loss": 3.4515687907822787, + "tokens_seen": 1858666496 + }, + { + "epoch": 0.13, + "learning_rate": 8.822821376986038e-05, + "loss": 2.6997, + "theoretical_loss": 3.451548203682088, + "tokens_seen": 1858797568 + }, + { + "epoch": 0.13, + "learning_rate": 8.822018937570214e-05, + "loss": 2.5268, + "theoretical_loss": 3.4515276184399717, + "tokens_seen": 1858928640 + }, + { + "epoch": 0.13, + "learning_rate": 8.82121649815439e-05, + "loss": 2.7007, + "theoretical_loss": 3.4515070350556334, + "tokens_seen": 1859059712 + }, + { + "epoch": 0.13, + "learning_rate": 8.820414058738566e-05, + "loss": 2.4948, + "theoretical_loss": 3.451486453528773, + "tokens_seen": 1859190784 + }, + { + "epoch": 0.13, + "learning_rate": 8.819611619322742e-05, + "loss": 2.494, + "theoretical_loss": 3.4514658738590915, + "tokens_seen": 1859321856 + }, + { + "epoch": 0.13, + "learning_rate": 8.818809179906918e-05, + "loss": 2.7342, + "theoretical_loss": 3.451445296046292, + "tokens_seen": 1859452928 + }, + { + "epoch": 0.13, + "learning_rate": 8.818006740491094e-05, + "loss": 2.7209, + "theoretical_loss": 3.4514247200900754, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1024297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4555611610412598, + "objective/train/theoretical_loss": 3.4514041459901432, + "objective/train/tokens_used": 230174176, + "theoretical_loss": 3.4514041459901432, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.13, + "learning_rate": 8.81720430107527e-05, + "loss": 2.6482, + "theoretical_loss": 3.4514041459901432, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.13, + "learning_rate": 8.816401861659445e-05, + "loss": 2.6735, + "theoretical_loss": 3.451383573746197, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.13, + "learning_rate": 8.815599422243622e-05, + "loss": 2.5834, + "theoretical_loss": 3.451363003357939, + "tokens_seen": 1859977216 + }, + { + "epoch": 0.13, + "learning_rate": 8.814796982827797e-05, + "loss": 2.5174, + "theoretical_loss": 3.4513424348250705, + "tokens_seen": 1860108288 + }, + { + "epoch": 0.13, + "learning_rate": 8.813994543411974e-05, + "loss": 2.6091, + "theoretical_loss": 3.451321868147294, + "tokens_seen": 1860239360 + }, + { + "epoch": 0.13, + "learning_rate": 8.813192103996149e-05, + "loss": 2.6376, + "theoretical_loss": 3.4513013033243114, + "tokens_seen": 1860370432 + }, + { + "epoch": 0.13, + "learning_rate": 8.812389664580326e-05, + "loss": 2.616, + "theoretical_loss": 3.4512807403558243, + "tokens_seen": 1860501504 + }, + { + "epoch": 0.13, + "learning_rate": 8.811587225164501e-05, + "loss": 2.5169, + "theoretical_loss": 3.451260179241536, + "tokens_seen": 1860632576 + }, + { + "epoch": 0.13, + "learning_rate": 8.810784785748678e-05, + "loss": 2.7286, + "theoretical_loss": 3.4512396199811466, + "tokens_seen": 1860763648 + }, + { + "epoch": 0.13, + "learning_rate": 8.809982346332853e-05, + "loss": 2.5962, + "theoretical_loss": 3.4512190625743604, + "tokens_seen": 1860894720 + }, + { + "epoch": 0.13, + "learning_rate": 8.809179906917028e-05, + "loss": 2.6989, + "theoretical_loss": 3.4511985070208793, + "tokens_seen": 1861025792 + }, + { + "epoch": 0.13, + "learning_rate": 8.808377467501205e-05, + "loss": 2.6519, + "theoretical_loss": 3.4511779533204043, + "tokens_seen": 1861156864 + }, + { + "epoch": 0.13, + "learning_rate": 8.80757502808538e-05, + "loss": 2.6076, + "theoretical_loss": 3.4511574014726403, + "tokens_seen": 1861287936 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1025545, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7613306045532227, + "objective/train/theoretical_loss": 3.4511471262434306, + "objective/train/tokens_used": 231812576, + "theoretical_loss": 3.4511471262434306, + "tokens_seen": 1861353472 + }, + { + "epoch": 0.13, + "learning_rate": 8.806772588669557e-05, + "loss": 2.6141, + "theoretical_loss": 3.4511368514772878, + "tokens_seen": 1861419008 + }, + { + "epoch": 0.13, + "learning_rate": 8.805970149253732e-05, + "loss": 2.6818, + "theoretical_loss": 3.4511163033340497, + "tokens_seen": 1861550080 + }, + { + "epoch": 0.13, + "learning_rate": 8.805167709837908e-05, + "loss": 2.695, + "theoretical_loss": 3.45109575704263, + "tokens_seen": 1861681152 + }, + { + "epoch": 0.13, + "learning_rate": 8.804365270422084e-05, + "loss": 2.4897, + "theoretical_loss": 3.4510752126027295, + "tokens_seen": 1861812224 + }, + { + "epoch": 0.13, + "learning_rate": 8.80356283100626e-05, + "loss": 2.526, + "theoretical_loss": 3.451054670014053, + "tokens_seen": 1861943296 + }, + { + "epoch": 0.13, + "learning_rate": 8.802760391590436e-05, + "loss": 2.564, + "theoretical_loss": 3.451034129276302, + "tokens_seen": 1862074368 + }, + { + "epoch": 0.13, + "learning_rate": 8.801957952174611e-05, + "loss": 2.4509, + "theoretical_loss": 3.45101359038918, + "tokens_seen": 1862205440 + }, + { + "epoch": 0.13, + "learning_rate": 8.801155512758788e-05, + "loss": 2.6459, + "theoretical_loss": 3.4509930533523905, + "tokens_seen": 1862336512 + }, + { + "epoch": 0.13, + "learning_rate": 8.800353073342963e-05, + "loss": 2.5574, + "theoretical_loss": 3.4509725181656354, + "tokens_seen": 1862467584 + }, + { + "epoch": 0.13, + "learning_rate": 8.799550633927139e-05, + "loss": 2.6837, + "theoretical_loss": 3.4509519848286185, + "tokens_seen": 1862598656 + }, + { + "epoch": 0.13, + "learning_rate": 8.798748194511315e-05, + "loss": 2.6577, + "theoretical_loss": 3.4509314533410436, + "tokens_seen": 1862729728 + }, + { + "epoch": 0.13, + "learning_rate": 8.797945755095491e-05, + "loss": 2.6322, + "theoretical_loss": 3.450910923702613, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1026014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.46268630027771, + "objective/train/theoretical_loss": 3.4508903959130306, + "objective/train/tokens_used": 233450976, + "theoretical_loss": 3.4508903959130306, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.13, + "learning_rate": 8.797143315679667e-05, + "loss": 2.6023, + "theoretical_loss": 3.4508903959130306, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.13, + "learning_rate": 8.796340876263843e-05, + "loss": 2.5201, + "theoretical_loss": 3.450869869972, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.13, + "learning_rate": 8.795538436848018e-05, + "loss": 2.762, + "theoretical_loss": 3.4508493458792246, + "tokens_seen": 1863254016 + }, + { + "epoch": 0.13, + "learning_rate": 8.794735997432195e-05, + "loss": 2.7066, + "theoretical_loss": 3.4508288236344073, + "tokens_seen": 1863385088 + }, + { + "epoch": 0.13, + "learning_rate": 8.79393355801637e-05, + "loss": 2.5886, + "theoretical_loss": 3.4508083032372525, + "tokens_seen": 1863516160 + }, + { + "epoch": 0.13, + "learning_rate": 8.793131118600547e-05, + "loss": 2.6404, + "theoretical_loss": 3.4507877846874635, + "tokens_seen": 1863647232 + }, + { + "epoch": 0.13, + "learning_rate": 8.792328679184722e-05, + "loss": 2.7333, + "theoretical_loss": 3.4507672679847445, + "tokens_seen": 1863778304 + }, + { + "epoch": 0.13, + "learning_rate": 8.791526239768899e-05, + "loss": 2.4031, + "theoretical_loss": 3.450746753128799, + "tokens_seen": 1863909376 + }, + { + "epoch": 0.13, + "learning_rate": 8.790723800353074e-05, + "loss": 2.5246, + "theoretical_loss": 3.4507262401193315, + "tokens_seen": 1864040448 + }, + { + "epoch": 0.13, + "learning_rate": 8.789921360937249e-05, + "loss": 2.3183, + "theoretical_loss": 3.450705728956045, + "tokens_seen": 1864171520 + }, + { + "epoch": 0.13, + "learning_rate": 8.789118921521426e-05, + "loss": 2.6952, + "theoretical_loss": 3.450685219638644, + "tokens_seen": 1864302592 + }, + { + "epoch": 0.13, + "learning_rate": 8.788316482105601e-05, + "loss": 2.5442, + "theoretical_loss": 3.4506647121668332, + "tokens_seen": 1864433664 + }, + { + "epoch": 0.13, + "learning_rate": 8.787514042689778e-05, + "loss": 2.6715, + "theoretical_loss": 3.450644206540316, + "tokens_seen": 1864564736 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1027424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5948421955108643, + "objective/train/theoretical_loss": 3.4506339544189504, + "objective/train/tokens_used": 235089376, + "theoretical_loss": 3.4506339544189504, + "tokens_seen": 1864630272 + }, + { + "epoch": 0.13, + "learning_rate": 8.786711603273953e-05, + "loss": 2.6985, + "theoretical_loss": 3.450623702758797, + "tokens_seen": 1864695808 + }, + { + "epoch": 0.13, + "learning_rate": 8.785909163858129e-05, + "loss": 2.6472, + "theoretical_loss": 3.450603200821981, + "tokens_seen": 1864826880 + }, + { + "epoch": 0.13, + "learning_rate": 8.785106724442305e-05, + "loss": 2.7063, + "theoretical_loss": 3.450582700729571, + "tokens_seen": 1864957952 + }, + { + "epoch": 0.13, + "learning_rate": 8.78430428502648e-05, + "loss": 2.7536, + "theoretical_loss": 3.4505622024812728, + "tokens_seen": 1865089024 + }, + { + "epoch": 0.13, + "learning_rate": 8.783501845610657e-05, + "loss": 2.6416, + "theoretical_loss": 3.4505417060767902, + "tokens_seen": 1865220096 + }, + { + "epoch": 0.13, + "learning_rate": 8.782699406194833e-05, + "loss": 2.7129, + "theoretical_loss": 3.450521211515828, + "tokens_seen": 1865351168 + }, + { + "epoch": 0.13, + "learning_rate": 8.781896966779009e-05, + "loss": 2.5542, + "theoretical_loss": 3.4505007187980916, + "tokens_seen": 1865482240 + }, + { + "epoch": 0.13, + "learning_rate": 8.781094527363185e-05, + "loss": 2.6308, + "theoretical_loss": 3.4504802279232845, + "tokens_seen": 1865613312 + }, + { + "epoch": 0.13, + "learning_rate": 8.78029208794736e-05, + "loss": 2.5675, + "theoretical_loss": 3.4504597388911122, + "tokens_seen": 1865744384 + }, + { + "epoch": 0.13, + "learning_rate": 8.779489648531536e-05, + "loss": 2.5833, + "theoretical_loss": 3.4504392517012796, + "tokens_seen": 1865875456 + }, + { + "epoch": 0.13, + "learning_rate": 8.778687209115712e-05, + "loss": 2.6185, + "theoretical_loss": 3.450418766353491, + "tokens_seen": 1866006528 + }, + { + "epoch": 0.13, + "learning_rate": 8.777884769699888e-05, + "loss": 2.7469, + "theoretical_loss": 3.450398282847452, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1028516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7388877868652344, + "objective/train/theoretical_loss": 3.450377801182868, + "objective/train/tokens_used": 236727776, + "theoretical_loss": 3.450377801182868, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.13, + "learning_rate": 8.777082330284064e-05, + "loss": 2.6726, + "theoretical_loss": 3.450377801182868, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.13, + "learning_rate": 8.776279890868239e-05, + "loss": 2.642, + "theoretical_loss": 3.4503573213594434, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.13, + "learning_rate": 8.775477451452416e-05, + "loss": 2.6033, + "theoretical_loss": 3.4503368433768835, + "tokens_seen": 1866530816 + }, + { + "epoch": 0.13, + "learning_rate": 8.774675012036591e-05, + "loss": 2.4442, + "theoretical_loss": 3.450316367234894, + "tokens_seen": 1866661888 + }, + { + "epoch": 0.13, + "learning_rate": 8.773872572620768e-05, + "loss": 2.513, + "theoretical_loss": 3.45029589293318, + "tokens_seen": 1866792960 + }, + { + "epoch": 0.13, + "learning_rate": 8.773070133204943e-05, + "loss": 2.5866, + "theoretical_loss": 3.450275420471447, + "tokens_seen": 1866924032 + }, + { + "epoch": 0.13, + "learning_rate": 8.77226769378912e-05, + "loss": 2.7162, + "theoretical_loss": 3.4502549498494, + "tokens_seen": 1867055104 + }, + { + "epoch": 0.13, + "learning_rate": 8.771465254373295e-05, + "loss": 2.518, + "theoretical_loss": 3.4502344810667456, + "tokens_seen": 1867186176 + }, + { + "epoch": 0.13, + "learning_rate": 8.77066281495747e-05, + "loss": 2.6779, + "theoretical_loss": 3.4502140141231887, + "tokens_seen": 1867317248 + }, + { + "epoch": 0.13, + "learning_rate": 8.769860375541647e-05, + "loss": 2.6632, + "theoretical_loss": 3.450193549018435, + "tokens_seen": 1867448320 + }, + { + "epoch": 0.13, + "learning_rate": 8.769057936125822e-05, + "loss": 2.4661, + "theoretical_loss": 3.4501730857521906, + "tokens_seen": 1867579392 + }, + { + "epoch": 0.13, + "learning_rate": 8.768255496709999e-05, + "loss": 2.5559, + "theoretical_loss": 3.4501526243241605, + "tokens_seen": 1867710464 + }, + { + "epoch": 0.13, + "learning_rate": 8.767453057294174e-05, + "loss": 2.7043, + "theoretical_loss": 3.4501321647340513, + "tokens_seen": 1867841536 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1029117, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6434731483459473, + "objective/train/theoretical_loss": 3.4501219356281254, + "objective/train/tokens_used": 238366176, + "theoretical_loss": 3.4501219356281254, + "tokens_seen": 1867907072 + }, + { + "epoch": 0.13, + "learning_rate": 8.76665061787835e-05, + "loss": 2.7618, + "theoretical_loss": 3.450111706981569, + "tokens_seen": 1867972608 + }, + { + "epoch": 0.13, + "learning_rate": 8.765848178462526e-05, + "loss": 2.5639, + "theoretical_loss": 3.4500912510664192, + "tokens_seen": 1868103680 + }, + { + "epoch": 0.13, + "learning_rate": 8.765045739046702e-05, + "loss": 2.6544, + "theoretical_loss": 3.4500707969883084, + "tokens_seen": 1868234752 + }, + { + "epoch": 0.13, + "learning_rate": 8.764243299630878e-05, + "loss": 2.5797, + "theoretical_loss": 3.450050344746942, + "tokens_seen": 1868365824 + }, + { + "epoch": 0.13, + "learning_rate": 8.763440860215054e-05, + "loss": 2.5622, + "theoretical_loss": 3.450029894342028, + "tokens_seen": 1868496896 + }, + { + "epoch": 0.13, + "learning_rate": 8.76263842079923e-05, + "loss": 2.6618, + "theoretical_loss": 3.4500094457732704, + "tokens_seen": 1868627968 + }, + { + "epoch": 0.13, + "learning_rate": 8.761835981383406e-05, + "loss": 2.7035, + "theoretical_loss": 3.449988999040377, + "tokens_seen": 1868759040 + }, + { + "epoch": 0.13, + "learning_rate": 8.761033541967581e-05, + "loss": 2.6978, + "theoretical_loss": 3.4499685541430543, + "tokens_seen": 1868890112 + }, + { + "epoch": 0.13, + "learning_rate": 8.760231102551758e-05, + "loss": 2.5531, + "theoretical_loss": 3.4499481110810075, + "tokens_seen": 1869021184 + }, + { + "epoch": 0.13, + "learning_rate": 8.759428663135933e-05, + "loss": 2.5434, + "theoretical_loss": 3.449927669853945, + "tokens_seen": 1869152256 + }, + { + "epoch": 0.13, + "learning_rate": 8.75862622372011e-05, + "loss": 2.5474, + "theoretical_loss": 3.449907230461572, + "tokens_seen": 1869283328 + }, + { + "epoch": 0.13, + "learning_rate": 8.757823784304285e-05, + "loss": 2.6269, + "theoretical_loss": 3.4498867929035955, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1029470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6646249294281006, + "objective/train/theoretical_loss": 3.449866357179723, + "objective/train/tokens_used": 240004576, + "theoretical_loss": 3.449866357179723, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.13, + "learning_rate": 8.75702134488846e-05, + "loss": 2.7099, + "theoretical_loss": 3.449866357179723, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.13, + "learning_rate": 8.756218905472637e-05, + "loss": 2.635, + "theoretical_loss": 3.44984592328966, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 8.755416466056812e-05, + "loss": 2.4857, + "theoretical_loss": 3.4498254912331148, + "tokens_seen": 1869807616 + }, + { + "epoch": 0.13, + "learning_rate": 8.754614026640989e-05, + "loss": 2.7694, + "theoretical_loss": 3.4498050610097932, + "tokens_seen": 1869938688 + }, + { + "epoch": 0.13, + "learning_rate": 8.753811587225164e-05, + "loss": 2.489, + "theoretical_loss": 3.4497846326194033, + "tokens_seen": 1870069760 + }, + { + "epoch": 0.13, + "learning_rate": 8.753009147809341e-05, + "loss": 2.479, + "theoretical_loss": 3.449764206061652, + "tokens_seen": 1870200832 + }, + { + "epoch": 0.13, + "learning_rate": 8.752206708393516e-05, + "loss": 2.6056, + "theoretical_loss": 3.4497437813362453, + "tokens_seen": 1870331904 + }, + { + "epoch": 0.13, + "learning_rate": 8.751404268977691e-05, + "loss": 2.7141, + "theoretical_loss": 3.449723358442892, + "tokens_seen": 1870462976 + }, + { + "epoch": 0.13, + "learning_rate": 8.750601829561868e-05, + "loss": 2.6245, + "theoretical_loss": 3.449702937381298, + "tokens_seen": 1870594048 + }, + { + "epoch": 0.13, + "learning_rate": 8.749799390146043e-05, + "loss": 2.6135, + "theoretical_loss": 3.449682518151172, + "tokens_seen": 1870725120 + }, + { + "epoch": 0.13, + "learning_rate": 8.74899695073022e-05, + "loss": 2.6038, + "theoretical_loss": 3.449662100752221, + "tokens_seen": 1870856192 + }, + { + "epoch": 0.13, + "learning_rate": 8.748194511314395e-05, + "loss": 2.5741, + "theoretical_loss": 3.449641685184152, + "tokens_seen": 1870987264 + }, + { + "epoch": 0.13, + "learning_rate": 8.747392071898572e-05, + "loss": 2.5113, + "theoretical_loss": 3.4496212714466727, + "tokens_seen": 1871118336 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1030778, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5349972248077393, + "objective/train/theoretical_loss": 3.449611065264313, + "objective/train/tokens_used": 241642976, + "theoretical_loss": 3.449611065264313, + "tokens_seen": 1871183872 + }, + { + "epoch": 0.13, + "learning_rate": 8.746589632482747e-05, + "loss": 2.6365, + "theoretical_loss": 3.449600859539491, + "tokens_seen": 1871249408 + }, + { + "epoch": 0.13, + "learning_rate": 8.745787193066924e-05, + "loss": 2.6784, + "theoretical_loss": 3.4495804494623146, + "tokens_seen": 1871380480 + }, + { + "epoch": 0.13, + "learning_rate": 8.7449847536511e-05, + "loss": 2.6674, + "theoretical_loss": 3.449560041214851, + "tokens_seen": 1871511552 + }, + { + "epoch": 0.13, + "learning_rate": 8.744182314235275e-05, + "loss": 2.6919, + "theoretical_loss": 3.4495396347968086, + "tokens_seen": 1871642624 + }, + { + "epoch": 0.13, + "learning_rate": 8.743379874819451e-05, + "loss": 2.5136, + "theoretical_loss": 3.449519230207895, + "tokens_seen": 1871773696 + }, + { + "epoch": 0.13, + "learning_rate": 8.742577435403627e-05, + "loss": 2.4687, + "theoretical_loss": 3.4494988274478184, + "tokens_seen": 1871904768 + }, + { + "epoch": 0.13, + "learning_rate": 8.741774995987803e-05, + "loss": 2.5254, + "theoretical_loss": 3.4494784265162863, + "tokens_seen": 1872035840 + }, + { + "epoch": 0.13, + "learning_rate": 8.740972556571979e-05, + "loss": 2.5739, + "theoretical_loss": 3.449458027413007, + "tokens_seen": 1872166912 + }, + { + "epoch": 0.13, + "learning_rate": 8.740170117156155e-05, + "loss": 2.6662, + "theoretical_loss": 3.4494376301376892, + "tokens_seen": 1872297984 + }, + { + "epoch": 0.13, + "learning_rate": 8.73936767774033e-05, + "loss": 2.3869, + "theoretical_loss": 3.4494172346900402, + "tokens_seen": 1872429056 + }, + { + "epoch": 0.13, + "learning_rate": 8.738565238324506e-05, + "loss": 2.5522, + "theoretical_loss": 3.4493968410697695, + "tokens_seen": 1872560128 + }, + { + "epoch": 0.13, + "learning_rate": 8.737762798908683e-05, + "loss": 2.5246, + "theoretical_loss": 3.449376449276584, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1031383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6211163997650146, + "objective/train/theoretical_loss": 3.449356059310194, + "objective/train/tokens_used": 243281376, + "theoretical_loss": 3.449356059310194, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.14, + "learning_rate": 8.736960359492858e-05, + "loss": 2.53, + "theoretical_loss": 3.449356059310194, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.14, + "learning_rate": 8.736157920077035e-05, + "loss": 2.6602, + "theoretical_loss": 3.4493356711703065, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.14, + "learning_rate": 8.73535548066121e-05, + "loss": 2.4317, + "theoretical_loss": 3.44931528485663, + "tokens_seen": 1873084416 + }, + { + "epoch": 0.14, + "learning_rate": 8.734553041245387e-05, + "loss": 2.6668, + "theoretical_loss": 3.4492949003688747, + "tokens_seen": 1873215488 + }, + { + "epoch": 0.14, + "learning_rate": 8.733750601829562e-05, + "loss": 2.6375, + "theoretical_loss": 3.4492745177067476, + "tokens_seen": 1873346560 + }, + { + "epoch": 0.14, + "learning_rate": 8.732948162413737e-05, + "loss": 2.6174, + "theoretical_loss": 3.4492541368699587, + "tokens_seen": 1873477632 + }, + { + "epoch": 0.14, + "learning_rate": 8.732145722997914e-05, + "loss": 2.4123, + "theoretical_loss": 3.4492337578582157, + "tokens_seen": 1873608704 + }, + { + "epoch": 0.14, + "learning_rate": 8.731343283582089e-05, + "loss": 2.5342, + "theoretical_loss": 3.449213380671229, + "tokens_seen": 1873739776 + }, + { + "epoch": 0.14, + "learning_rate": 8.730540844166266e-05, + "loss": 2.5533, + "theoretical_loss": 3.4491930053087065, + "tokens_seen": 1873870848 + }, + { + "epoch": 0.14, + "learning_rate": 8.729738404750441e-05, + "loss": 2.5133, + "theoretical_loss": 3.449172631770357, + "tokens_seen": 1874001920 + }, + { + "epoch": 0.14, + "learning_rate": 8.728935965334618e-05, + "loss": 2.657, + "theoretical_loss": 3.4491522600558904, + "tokens_seen": 1874132992 + }, + { + "epoch": 0.14, + "learning_rate": 8.728133525918793e-05, + "loss": 2.5688, + "theoretical_loss": 3.449131890165016, + "tokens_seen": 1874264064 + }, + { + "epoch": 0.14, + "learning_rate": 8.727331086502968e-05, + "loss": 2.6587, + "theoretical_loss": 3.4491115220974424, + "tokens_seen": 1874395136 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1032518, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.44793438911438, + "objective/train/theoretical_loss": 3.449101338747303, + "objective/train/tokens_used": 244919776, + "theoretical_loss": 3.449101338747303, + "tokens_seen": 1874460672 + }, + { + "epoch": 0.14, + "learning_rate": 8.726528647087145e-05, + "loss": 2.5988, + "theoretical_loss": 3.449091155852879, + "tokens_seen": 1874526208 + }, + { + "epoch": 0.14, + "learning_rate": 8.72572620767132e-05, + "loss": 2.6024, + "theoretical_loss": 3.4490707914310352, + "tokens_seen": 1874657280 + }, + { + "epoch": 0.14, + "learning_rate": 8.724923768255497e-05, + "loss": 2.5618, + "theoretical_loss": 3.4490504288316215, + "tokens_seen": 1874788352 + }, + { + "epoch": 0.14, + "learning_rate": 8.724121328839672e-05, + "loss": 2.5314, + "theoretical_loss": 3.4490300680543458, + "tokens_seen": 1874919424 + }, + { + "epoch": 0.14, + "learning_rate": 8.723318889423849e-05, + "loss": 2.5469, + "theoretical_loss": 3.4490097090989185, + "tokens_seen": 1875050496 + }, + { + "epoch": 0.14, + "learning_rate": 8.722516450008024e-05, + "loss": 2.4655, + "theoretical_loss": 3.448989351965049, + "tokens_seen": 1875181568 + }, + { + "epoch": 0.14, + "learning_rate": 8.7217140105922e-05, + "loss": 2.6071, + "theoretical_loss": 3.4489689966524475, + "tokens_seen": 1875312640 + }, + { + "epoch": 0.14, + "learning_rate": 8.720911571176376e-05, + "loss": 2.5736, + "theoretical_loss": 3.448948643160823, + "tokens_seen": 1875443712 + }, + { + "epoch": 0.14, + "learning_rate": 8.720109131760552e-05, + "loss": 2.5253, + "theoretical_loss": 3.448928291489886, + "tokens_seen": 1875574784 + }, + { + "epoch": 0.14, + "learning_rate": 8.719306692344728e-05, + "loss": 2.6315, + "theoretical_loss": 3.448907941639346, + "tokens_seen": 1875705856 + }, + { + "epoch": 0.14, + "learning_rate": 8.718504252928904e-05, + "loss": 2.6466, + "theoretical_loss": 3.448887593608914, + "tokens_seen": 1875836928 + }, + { + "epoch": 0.14, + "learning_rate": 8.71770181351308e-05, + "loss": 2.6256, + "theoretical_loss": 3.448867247398298, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1033022, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8692405223846436, + "objective/train/theoretical_loss": 3.448846903007211, + "objective/train/tokens_used": 246558176, + "theoretical_loss": 3.448846903007211, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.14, + "learning_rate": 8.716899374097256e-05, + "loss": 2.6994, + "theoretical_loss": 3.448846903007211, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.14, + "learning_rate": 8.716096934681432e-05, + "loss": 2.6321, + "theoretical_loss": 3.44882656043536, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.14, + "learning_rate": 8.715294495265608e-05, + "loss": 2.5969, + "theoretical_loss": 3.4488062196824574, + "tokens_seen": 1876361216 + }, + { + "epoch": 0.14, + "learning_rate": 8.714492055849783e-05, + "loss": 2.6608, + "theoretical_loss": 3.448785880748213, + "tokens_seen": 1876492288 + }, + { + "epoch": 0.14, + "learning_rate": 8.71368961643396e-05, + "loss": 2.6761, + "theoretical_loss": 3.4487655436323363, + "tokens_seen": 1876623360 + }, + { + "epoch": 0.14, + "learning_rate": 8.712887177018135e-05, + "loss": 2.585, + "theoretical_loss": 3.4487452083345396, + "tokens_seen": 1876754432 + }, + { + "epoch": 0.14, + "learning_rate": 8.712084737602312e-05, + "loss": 2.4546, + "theoretical_loss": 3.4487248748545314, + "tokens_seen": 1876885504 + }, + { + "epoch": 0.14, + "learning_rate": 8.711282298186487e-05, + "loss": 2.7406, + "theoretical_loss": 3.4487045431920236, + "tokens_seen": 1877016576 + }, + { + "epoch": 0.14, + "learning_rate": 8.710479858770664e-05, + "loss": 2.6284, + "theoretical_loss": 3.4486842133467266, + "tokens_seen": 1877147648 + }, + { + "epoch": 0.14, + "learning_rate": 8.709677419354839e-05, + "loss": 2.6591, + "theoretical_loss": 3.448663885318351, + "tokens_seen": 1877278720 + }, + { + "epoch": 0.14, + "learning_rate": 8.708874979939014e-05, + "loss": 2.6345, + "theoretical_loss": 3.448643559106607, + "tokens_seen": 1877409792 + }, + { + "epoch": 0.14, + "learning_rate": 8.708072540523191e-05, + "loss": 2.6332, + "theoretical_loss": 3.4486232347112065, + "tokens_seen": 1877540864 + }, + { + "epoch": 0.14, + "learning_rate": 8.707270101107366e-05, + "loss": 2.6165, + "theoretical_loss": 3.4486029121318595, + "tokens_seen": 1877671936 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1034169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.514636993408203, + "objective/train/theoretical_loss": 3.448592751523116, + "objective/train/tokens_used": 248196576, + "theoretical_loss": 3.448592751523116, + "tokens_seen": 1877737472 + }, + { + "epoch": 0.14, + "learning_rate": 8.706467661691543e-05, + "loss": 2.5063, + "theoretical_loss": 3.4485825913682775, + "tokens_seen": 1877803008 + }, + { + "epoch": 0.14, + "learning_rate": 8.705665222275718e-05, + "loss": 2.5341, + "theoretical_loss": 3.448562272420171, + "tokens_seen": 1877934080 + }, + { + "epoch": 0.14, + "learning_rate": 8.704862782859895e-05, + "loss": 2.6616, + "theoretical_loss": 3.4485419552872525, + "tokens_seen": 1878065152 + }, + { + "epoch": 0.14, + "learning_rate": 8.70406034344407e-05, + "loss": 2.5614, + "theoretical_loss": 3.4485216399692313, + "tokens_seen": 1878196224 + }, + { + "epoch": 0.14, + "learning_rate": 8.703257904028245e-05, + "loss": 2.5634, + "theoretical_loss": 3.44850132646582, + "tokens_seen": 1878327296 + }, + { + "epoch": 0.14, + "learning_rate": 8.702455464612422e-05, + "loss": 2.6708, + "theoretical_loss": 3.4484810147767293, + "tokens_seen": 1878458368 + }, + { + "epoch": 0.14, + "learning_rate": 8.701653025196597e-05, + "loss": 2.7177, + "theoretical_loss": 3.4484607049016702, + "tokens_seen": 1878589440 + }, + { + "epoch": 0.14, + "learning_rate": 8.700850585780774e-05, + "loss": 2.6269, + "theoretical_loss": 3.4484403968403554, + "tokens_seen": 1878720512 + }, + { + "epoch": 0.14, + "learning_rate": 8.70004814636495e-05, + "loss": 2.6973, + "theoretical_loss": 3.4484200905924953, + "tokens_seen": 1878851584 + }, + { + "epoch": 0.14, + "learning_rate": 8.699245706949126e-05, + "loss": 2.6511, + "theoretical_loss": 3.448399786157802, + "tokens_seen": 1878982656 + }, + { + "epoch": 0.14, + "learning_rate": 8.698443267533301e-05, + "loss": 2.6892, + "theoretical_loss": 3.448379483535987, + "tokens_seen": 1879113728 + }, + { + "epoch": 0.14, + "learning_rate": 8.697640828117477e-05, + "loss": 2.6398, + "theoretical_loss": 3.4483591827267617, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1034880, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.418511390686035, + "objective/train/theoretical_loss": 3.4483388837298383, + "objective/train/tokens_used": 249834976, + "theoretical_loss": 3.4483388837298383, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.14, + "learning_rate": 8.696838388701653e-05, + "loss": 2.6351, + "theoretical_loss": 3.4483388837298383, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.14, + "learning_rate": 8.696035949285829e-05, + "loss": 2.7967, + "theoretical_loss": 3.448318586544928, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.14, + "learning_rate": 8.695233509870005e-05, + "loss": 2.7456, + "theoretical_loss": 3.4482982911717435, + "tokens_seen": 1879638016 + }, + { + "epoch": 0.14, + "learning_rate": 8.69443107045418e-05, + "loss": 2.6476, + "theoretical_loss": 3.4482779976099964, + "tokens_seen": 1879769088 + }, + { + "epoch": 0.14, + "learning_rate": 8.693628631038357e-05, + "loss": 2.56, + "theoretical_loss": 3.448257705859399, + "tokens_seen": 1879900160 + }, + { + "epoch": 0.14, + "learning_rate": 8.692826191622533e-05, + "loss": 2.6287, + "theoretical_loss": 3.448237415919663, + "tokens_seen": 1880031232 + }, + { + "epoch": 0.14, + "learning_rate": 8.692023752206708e-05, + "loss": 2.6142, + "theoretical_loss": 3.4482171277905005, + "tokens_seen": 1880162304 + }, + { + "epoch": 0.14, + "learning_rate": 8.691221312790885e-05, + "loss": 2.5558, + "theoretical_loss": 3.448196841471624, + "tokens_seen": 1880293376 + }, + { + "epoch": 0.14, + "learning_rate": 8.69041887337506e-05, + "loss": 2.4915, + "theoretical_loss": 3.4481765569627463, + "tokens_seen": 1880424448 + }, + { + "epoch": 0.14, + "learning_rate": 8.689616433959237e-05, + "loss": 2.5497, + "theoretical_loss": 3.448156274263578, + "tokens_seen": 1880555520 + }, + { + "epoch": 0.14, + "learning_rate": 8.688813994543412e-05, + "loss": 2.507, + "theoretical_loss": 3.4481359933738336, + "tokens_seen": 1880686592 + }, + { + "epoch": 0.14, + "learning_rate": 8.688011555127589e-05, + "loss": 2.6085, + "theoretical_loss": 3.448115714293225, + "tokens_seen": 1880817664 + }, + { + "epoch": 0.14, + "learning_rate": 8.687209115711764e-05, + "loss": 2.612, + "theoretical_loss": 3.448095437021464, + "tokens_seen": 1880948736 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1035982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5726146697998047, + "objective/train/theoretical_loss": 3.448085299063812, + "objective/train/tokens_used": 251473376, + "theoretical_loss": 3.448085299063812, + "tokens_seen": 1881014272 + }, + { + "epoch": 0.14, + "learning_rate": 8.68640667629594e-05, + "loss": 2.4888, + "theoretical_loss": 3.448075161558264, + "tokens_seen": 1881079808 + }, + { + "epoch": 0.14, + "learning_rate": 8.685604236880116e-05, + "loss": 2.6203, + "theoretical_loss": 3.448054887903337, + "tokens_seen": 1881210880 + }, + { + "epoch": 0.14, + "learning_rate": 8.684801797464291e-05, + "loss": 2.797, + "theoretical_loss": 3.448034616056396, + "tokens_seen": 1881341952 + }, + { + "epoch": 0.14, + "learning_rate": 8.683999358048468e-05, + "loss": 2.6205, + "theoretical_loss": 3.4480143460171546, + "tokens_seen": 1881473024 + }, + { + "epoch": 0.14, + "learning_rate": 8.683196918632643e-05, + "loss": 2.4245, + "theoretical_loss": 3.447994077785325, + "tokens_seen": 1881604096 + }, + { + "epoch": 0.14, + "learning_rate": 8.68239447921682e-05, + "loss": 2.6329, + "theoretical_loss": 3.4479738113606198, + "tokens_seen": 1881735168 + }, + { + "epoch": 0.14, + "learning_rate": 8.681592039800995e-05, + "loss": 2.5879, + "theoretical_loss": 3.4479535467427525, + "tokens_seen": 1881866240 + }, + { + "epoch": 0.14, + "learning_rate": 8.680789600385172e-05, + "loss": 2.6507, + "theoretical_loss": 3.4479332839314365, + "tokens_seen": 1881997312 + }, + { + "epoch": 0.14, + "learning_rate": 8.679987160969347e-05, + "loss": 2.6719, + "theoretical_loss": 3.447913022926384, + "tokens_seen": 1882128384 + }, + { + "epoch": 0.14, + "learning_rate": 8.679184721553522e-05, + "loss": 2.528, + "theoretical_loss": 3.4478927637273094, + "tokens_seen": 1882259456 + }, + { + "epoch": 0.14, + "learning_rate": 8.678382282137699e-05, + "loss": 2.6766, + "theoretical_loss": 3.4478725063339244, + "tokens_seen": 1882390528 + }, + { + "epoch": 0.14, + "learning_rate": 8.677579842721874e-05, + "loss": 2.5525, + "theoretical_loss": 3.447852250745944, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1036495, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.561272144317627, + "objective/train/theoretical_loss": 3.447831996963081, + "objective/train/tokens_used": 253111776, + "theoretical_loss": 3.447831996963081, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.14, + "learning_rate": 8.676777403306051e-05, + "loss": 2.7183, + "theoretical_loss": 3.447831996963081, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.14, + "learning_rate": 8.675974963890226e-05, + "loss": 2.7235, + "theoretical_loss": 3.4478117449850485, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 8.675172524474403e-05, + "loss": 2.5957, + "theoretical_loss": 3.44779149481156, + "tokens_seen": 1882914816 + }, + { + "epoch": 0.14, + "learning_rate": 8.674370085058578e-05, + "loss": 2.5508, + "theoretical_loss": 3.44777124644233, + "tokens_seen": 1883045888 + }, + { + "epoch": 0.14, + "learning_rate": 8.673567645642754e-05, + "loss": 2.5098, + "theoretical_loss": 3.447750999877072, + "tokens_seen": 1883176960 + }, + { + "epoch": 0.14, + "learning_rate": 8.67276520622693e-05, + "loss": 2.605, + "theoretical_loss": 3.4477307551154985, + "tokens_seen": 1883308032 + }, + { + "epoch": 0.14, + "learning_rate": 8.671962766811106e-05, + "loss": 2.7175, + "theoretical_loss": 3.4477105121573244, + "tokens_seen": 1883439104 + }, + { + "epoch": 0.14, + "learning_rate": 8.671160327395282e-05, + "loss": 2.5376, + "theoretical_loss": 3.4476902710022634, + "tokens_seen": 1883570176 + }, + { + "epoch": 0.14, + "learning_rate": 8.670357887979458e-05, + "loss": 2.5942, + "theoretical_loss": 3.4476700316500293, + "tokens_seen": 1883701248 + }, + { + "epoch": 0.14, + "learning_rate": 8.669555448563634e-05, + "loss": 2.6486, + "theoretical_loss": 3.447649794100336, + "tokens_seen": 1883832320 + }, + { + "epoch": 0.14, + "learning_rate": 8.66875300914781e-05, + "loss": 2.6782, + "theoretical_loss": 3.447629558352898, + "tokens_seen": 1883963392 + }, + { + "epoch": 0.14, + "learning_rate": 8.667950569731985e-05, + "loss": 2.6249, + "theoretical_loss": 3.447609324407429, + "tokens_seen": 1884094464 + }, + { + "epoch": 0.14, + "learning_rate": 8.667148130316162e-05, + "loss": 2.5889, + "theoretical_loss": 3.4475890922636436, + "tokens_seen": 1884225536 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1037807, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6240508556365967, + "objective/train/theoretical_loss": 3.447578976867293, + "objective/train/tokens_used": 254750176, + "theoretical_loss": 3.447578976867293, + "tokens_seen": 1884291072 + }, + { + "epoch": 0.14, + "learning_rate": 8.666345690900337e-05, + "loss": 2.5063, + "theoretical_loss": 3.4475688619212557, + "tokens_seen": 1884356608 + }, + { + "epoch": 0.14, + "learning_rate": 8.665543251484514e-05, + "loss": 2.8423, + "theoretical_loss": 3.4475486333799794, + "tokens_seen": 1884487680 + }, + { + "epoch": 0.14, + "learning_rate": 8.664740812068689e-05, + "loss": 2.6742, + "theoretical_loss": 3.4475284066395293, + "tokens_seen": 1884618752 + }, + { + "epoch": 0.14, + "learning_rate": 8.663938372652866e-05, + "loss": 2.7974, + "theoretical_loss": 3.4475081816996207, + "tokens_seen": 1884749824 + }, + { + "epoch": 0.14, + "learning_rate": 8.663135933237041e-05, + "loss": 2.5465, + "theoretical_loss": 3.4474879585599667, + "tokens_seen": 1884880896 + }, + { + "epoch": 0.14, + "learning_rate": 8.662333493821216e-05, + "loss": 2.6038, + "theoretical_loss": 3.4474677372202827, + "tokens_seen": 1885011968 + }, + { + "epoch": 0.14, + "learning_rate": 8.661531054405393e-05, + "loss": 2.6861, + "theoretical_loss": 3.4474475176802835, + "tokens_seen": 1885143040 + }, + { + "epoch": 0.14, + "learning_rate": 8.660728614989568e-05, + "loss": 2.4443, + "theoretical_loss": 3.447427299939683, + "tokens_seen": 1885274112 + }, + { + "epoch": 0.14, + "learning_rate": 8.659926175573745e-05, + "loss": 2.4831, + "theoretical_loss": 3.4474070839981974, + "tokens_seen": 1885405184 + }, + { + "epoch": 0.14, + "learning_rate": 8.65912373615792e-05, + "loss": 2.6502, + "theoretical_loss": 3.4473868698555403, + "tokens_seen": 1885536256 + }, + { + "epoch": 0.14, + "learning_rate": 8.658321296742097e-05, + "loss": 2.6768, + "theoretical_loss": 3.4473666575114263, + "tokens_seen": 1885667328 + }, + { + "epoch": 0.14, + "learning_rate": 8.657518857326272e-05, + "loss": 2.5946, + "theoretical_loss": 3.4473464469655717, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1038424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4946327209472656, + "objective/train/theoretical_loss": 3.4473262382176912, + "objective/train/tokens_used": 256388576, + "theoretical_loss": 3.4473262382176912, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 8.656716417910447e-05, + "loss": 2.6475, + "theoretical_loss": 3.4473262382176912, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 8.655913978494624e-05, + "loss": 2.5066, + "theoretical_loss": 3.447306031267499, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 8.6551115390788e-05, + "loss": 2.7395, + "theoretical_loss": 3.4472858261147112, + "tokens_seen": 1886191616 + }, + { + "epoch": 0.14, + "learning_rate": 8.654309099662976e-05, + "loss": 2.6795, + "theoretical_loss": 3.4472656227590424, + "tokens_seen": 1886322688 + }, + { + "epoch": 0.14, + "learning_rate": 8.653506660247151e-05, + "loss": 2.8047, + "theoretical_loss": 3.4472454212002086, + "tokens_seen": 1886453760 + }, + { + "epoch": 0.14, + "learning_rate": 8.652704220831328e-05, + "loss": 2.4961, + "theoretical_loss": 3.4472252214379244, + "tokens_seen": 1886584832 + }, + { + "epoch": 0.14, + "learning_rate": 8.651901781415503e-05, + "loss": 2.5358, + "theoretical_loss": 3.447205023471906, + "tokens_seen": 1886715904 + }, + { + "epoch": 0.14, + "learning_rate": 8.65109934199968e-05, + "loss": 2.6046, + "theoretical_loss": 3.4471848273018684, + "tokens_seen": 1886846976 + }, + { + "epoch": 0.14, + "learning_rate": 8.650296902583855e-05, + "loss": 2.5741, + "theoretical_loss": 3.447164632927527, + "tokens_seen": 1886978048 + }, + { + "epoch": 0.14, + "learning_rate": 8.649494463168031e-05, + "loss": 2.6877, + "theoretical_loss": 3.4471444403485982, + "tokens_seen": 1887109120 + }, + { + "epoch": 0.14, + "learning_rate": 8.648692023752207e-05, + "loss": 2.652, + "theoretical_loss": 3.447124249564797, + "tokens_seen": 1887240192 + }, + { + "epoch": 0.14, + "learning_rate": 8.647889584336383e-05, + "loss": 2.5121, + "theoretical_loss": 3.447104060575839, + "tokens_seen": 1887371264 + }, + { + "epoch": 0.14, + "learning_rate": 8.64708714492056e-05, + "loss": 2.5847, + "theoretical_loss": 3.44708387338144, + "tokens_seen": 1887502336 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1039538, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6100270748138428, + "objective/train/theoretical_loss": 3.447073780457112, + "objective/train/tokens_used": 258026976, + "theoretical_loss": 3.447073780457112, + "tokens_seen": 1887567872 + }, + { + "epoch": 0.14, + "learning_rate": 8.646284705504735e-05, + "loss": 2.7654, + "theoretical_loss": 3.4470636879813172, + "tokens_seen": 1887633408 + }, + { + "epoch": 0.14, + "learning_rate": 8.645482266088911e-05, + "loss": 2.5572, + "theoretical_loss": 3.447043504375185, + "tokens_seen": 1887764480 + }, + { + "epoch": 0.14, + "learning_rate": 8.644679826673087e-05, + "loss": 2.5704, + "theoretical_loss": 3.4470233225627602, + "tokens_seen": 1887895552 + }, + { + "epoch": 0.14, + "learning_rate": 8.643877387257262e-05, + "loss": 2.6712, + "theoretical_loss": 3.4470031425437586, + "tokens_seen": 1888026624 + }, + { + "epoch": 0.14, + "learning_rate": 8.643074947841439e-05, + "loss": 2.435, + "theoretical_loss": 3.4469829643178964, + "tokens_seen": 1888157696 + }, + { + "epoch": 0.14, + "learning_rate": 8.642272508425614e-05, + "loss": 2.7374, + "theoretical_loss": 3.44696278788489, + "tokens_seen": 1888288768 + }, + { + "epoch": 0.14, + "learning_rate": 8.64147006900979e-05, + "loss": 2.6685, + "theoretical_loss": 3.446942613244455, + "tokens_seen": 1888419840 + }, + { + "epoch": 0.14, + "learning_rate": 8.640667629593966e-05, + "loss": 2.6827, + "theoretical_loss": 3.4469224403963086, + "tokens_seen": 1888550912 + }, + { + "epoch": 0.14, + "learning_rate": 8.639865190178143e-05, + "loss": 2.5129, + "theoretical_loss": 3.4469022693401667, + "tokens_seen": 1888681984 + }, + { + "epoch": 0.14, + "learning_rate": 8.639062750762318e-05, + "loss": 2.7268, + "theoretical_loss": 3.4468821000757464, + "tokens_seen": 1888813056 + }, + { + "epoch": 0.14, + "learning_rate": 8.638260311346493e-05, + "loss": 2.5836, + "theoretical_loss": 3.446861932602763, + "tokens_seen": 1888944128 + }, + { + "epoch": 0.14, + "learning_rate": 8.63745787193067e-05, + "loss": 2.5723, + "theoretical_loss": 3.446841766920934, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1040979, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.302525758743286, + "objective/train/theoretical_loss": 3.446821603029976, + "objective/train/tokens_used": 259665376, + "theoretical_loss": 3.446821603029976, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.14, + "learning_rate": 8.636655432514845e-05, + "loss": 2.5104, + "theoretical_loss": 3.446821603029976, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.15, + "learning_rate": 8.635852993099022e-05, + "loss": 2.7808, + "theoretical_loss": 3.446801440929606, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.15, + "learning_rate": 8.635050553683197e-05, + "loss": 2.6051, + "theoretical_loss": 3.4467812806195397, + "tokens_seen": 1889468416 + }, + { + "epoch": 0.15, + "learning_rate": 8.634248114267374e-05, + "loss": 2.4754, + "theoretical_loss": 3.4467611220994954, + "tokens_seen": 1889599488 + }, + { + "epoch": 0.15, + "learning_rate": 8.633445674851549e-05, + "loss": 2.5734, + "theoretical_loss": 3.4467409653691887, + "tokens_seen": 1889730560 + }, + { + "epoch": 0.15, + "learning_rate": 8.632643235435724e-05, + "loss": 2.636, + "theoretical_loss": 3.4467208104283373, + "tokens_seen": 1889861632 + }, + { + "epoch": 0.15, + "learning_rate": 8.631840796019901e-05, + "loss": 2.5367, + "theoretical_loss": 3.446700657276658, + "tokens_seen": 1889992704 + }, + { + "epoch": 0.15, + "learning_rate": 8.631038356604076e-05, + "loss": 2.6323, + "theoretical_loss": 3.446680505913868, + "tokens_seen": 1890123776 + }, + { + "epoch": 0.15, + "learning_rate": 8.630235917188253e-05, + "loss": 2.6413, + "theoretical_loss": 3.4466603563396845, + "tokens_seen": 1890254848 + }, + { + "epoch": 0.15, + "learning_rate": 8.629433477772428e-05, + "loss": 2.6791, + "theoretical_loss": 3.4466402085538252, + "tokens_seen": 1890385920 + }, + { + "epoch": 0.15, + "learning_rate": 8.628631038356605e-05, + "loss": 2.63, + "theoretical_loss": 3.4466200625560064, + "tokens_seen": 1890516992 + }, + { + "epoch": 0.15, + "learning_rate": 8.62782859894078e-05, + "loss": 2.6905, + "theoretical_loss": 3.446599918345946, + "tokens_seen": 1890648064 + }, + { + "epoch": 0.15, + "learning_rate": 8.627026159524956e-05, + "loss": 2.6284, + "theoretical_loss": 3.446579775923362, + "tokens_seen": 1890779136 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1041612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.944444417953491, + "objective/train/theoretical_loss": 3.4465697053822844, + "objective/train/tokens_used": 261303776, + "theoretical_loss": 3.4465697053822844, + "tokens_seen": 1890844672 + }, + { + "epoch": 0.15, + "learning_rate": 8.626223720109132e-05, + "loss": 2.6467, + "theoretical_loss": 3.4465596352879704, + "tokens_seen": 1890910208 + }, + { + "epoch": 0.15, + "learning_rate": 8.625421280693308e-05, + "loss": 2.4845, + "theoretical_loss": 3.44653949643949, + "tokens_seen": 1891041280 + }, + { + "epoch": 0.15, + "learning_rate": 8.624618841277484e-05, + "loss": 2.5807, + "theoretical_loss": 3.446519359377638, + "tokens_seen": 1891172352 + }, + { + "epoch": 0.15, + "learning_rate": 8.62381640186166e-05, + "loss": 2.6668, + "theoretical_loss": 3.4464992241021326, + "tokens_seen": 1891303424 + }, + { + "epoch": 0.15, + "learning_rate": 8.623013962445836e-05, + "loss": 2.4901, + "theoretical_loss": 3.4464790906126908, + "tokens_seen": 1891434496 + }, + { + "epoch": 0.15, + "learning_rate": 8.622211523030012e-05, + "loss": 2.7128, + "theoretical_loss": 3.446458958909031, + "tokens_seen": 1891565568 + }, + { + "epoch": 0.15, + "learning_rate": 8.621409083614188e-05, + "loss": 2.6439, + "theoretical_loss": 3.4464388289908703, + "tokens_seen": 1891696640 + }, + { + "epoch": 0.15, + "learning_rate": 8.620606644198364e-05, + "loss": 2.6116, + "theoretical_loss": 3.4464187008579277, + "tokens_seen": 1891827712 + }, + { + "epoch": 0.15, + "learning_rate": 8.619804204782539e-05, + "loss": 2.5137, + "theoretical_loss": 3.446398574509921, + "tokens_seen": 1891958784 + }, + { + "epoch": 0.15, + "learning_rate": 8.619001765366716e-05, + "loss": 2.5921, + "theoretical_loss": 3.4463784499465673, + "tokens_seen": 1892089856 + }, + { + "epoch": 0.15, + "learning_rate": 8.618199325950891e-05, + "loss": 2.5911, + "theoretical_loss": 3.4463583271675855, + "tokens_seen": 1892220928 + }, + { + "epoch": 0.15, + "learning_rate": 8.617396886535068e-05, + "loss": 2.5032, + "theoretical_loss": 3.446338206172694, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1042686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5551538467407227, + "objective/train/theoretical_loss": 3.44631808696161, + "objective/train/tokens_used": 262942176, + "theoretical_loss": 3.44631808696161, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.15, + "learning_rate": 8.616594447119243e-05, + "loss": 2.7494, + "theoretical_loss": 3.44631808696161, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.15, + "learning_rate": 8.61579200770342e-05, + "loss": 2.5324, + "theoretical_loss": 3.4462979695340534, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.15, + "learning_rate": 8.614989568287595e-05, + "loss": 2.5766, + "theoretical_loss": 3.446277853889742, + "tokens_seen": 1892745216 + }, + { + "epoch": 0.15, + "learning_rate": 8.61418712887177e-05, + "loss": 2.5912, + "theoretical_loss": 3.4462577400283934, + "tokens_seen": 1892876288 + }, + { + "epoch": 0.15, + "learning_rate": 8.613384689455947e-05, + "loss": 2.5151, + "theoretical_loss": 3.446237627949727, + "tokens_seen": 1893007360 + }, + { + "epoch": 0.15, + "learning_rate": 8.612582250040122e-05, + "loss": 2.7351, + "theoretical_loss": 3.4462175176534613, + "tokens_seen": 1893138432 + }, + { + "epoch": 0.15, + "learning_rate": 8.611779810624299e-05, + "loss": 2.4694, + "theoretical_loss": 3.446197409139314, + "tokens_seen": 1893269504 + }, + { + "epoch": 0.15, + "learning_rate": 8.610977371208474e-05, + "loss": 2.7383, + "theoretical_loss": 3.4461773024070057, + "tokens_seen": 1893400576 + }, + { + "epoch": 0.15, + "learning_rate": 8.610174931792651e-05, + "loss": 2.2325, + "theoretical_loss": 3.4461571974562535, + "tokens_seen": 1893531648 + }, + { + "epoch": 0.15, + "learning_rate": 8.609372492376826e-05, + "loss": 2.5378, + "theoretical_loss": 3.446137094286777, + "tokens_seen": 1893662720 + }, + { + "epoch": 0.15, + "learning_rate": 8.608570052961001e-05, + "loss": 2.6805, + "theoretical_loss": 3.446116992898295, + "tokens_seen": 1893793792 + }, + { + "epoch": 0.15, + "learning_rate": 8.607767613545178e-05, + "loss": 2.5466, + "theoretical_loss": 3.4460968932905263, + "tokens_seen": 1893924864 + }, + { + "epoch": 0.15, + "learning_rate": 8.606965174129353e-05, + "loss": 2.6135, + "theoretical_loss": 3.44607679546319, + "tokens_seen": 1894055936 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1043318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7247536182403564, + "objective/train/theoretical_loss": 3.4460667472170963, + "objective/train/tokens_used": 264580576, + "theoretical_loss": 3.4460667472170963, + "tokens_seen": 1894121472 + }, + { + "epoch": 0.15, + "learning_rate": 8.60616273471353e-05, + "loss": 2.5337, + "theoretical_loss": 3.446056699416005, + "tokens_seen": 1894187008 + }, + { + "epoch": 0.15, + "learning_rate": 8.605360295297705e-05, + "loss": 2.5264, + "theoretical_loss": 3.4460366051486915, + "tokens_seen": 1894318080 + }, + { + "epoch": 0.15, + "learning_rate": 8.604557855881882e-05, + "loss": 2.5163, + "theoretical_loss": 3.446016512660967, + "tokens_seen": 1894449152 + }, + { + "epoch": 0.15, + "learning_rate": 8.603755416466057e-05, + "loss": 2.7092, + "theoretical_loss": 3.445996421952552, + "tokens_seen": 1894580224 + }, + { + "epoch": 0.15, + "learning_rate": 8.602952977050233e-05, + "loss": 2.5607, + "theoretical_loss": 3.445976333023166, + "tokens_seen": 1894711296 + }, + { + "epoch": 0.15, + "learning_rate": 8.60215053763441e-05, + "loss": 2.5803, + "theoretical_loss": 3.4459562458725275, + "tokens_seen": 1894842368 + }, + { + "epoch": 0.15, + "learning_rate": 8.601348098218585e-05, + "loss": 2.594, + "theoretical_loss": 3.4459361605003567, + "tokens_seen": 1894973440 + }, + { + "epoch": 0.15, + "learning_rate": 8.600545658802761e-05, + "loss": 2.8107, + "theoretical_loss": 3.4459160769063724, + "tokens_seen": 1895104512 + }, + { + "epoch": 0.15, + "learning_rate": 8.599743219386937e-05, + "loss": 2.5098, + "theoretical_loss": 3.4458959950902948, + "tokens_seen": 1895235584 + }, + { + "epoch": 0.15, + "learning_rate": 8.598940779971113e-05, + "loss": 2.6863, + "theoretical_loss": 3.445875915051844, + "tokens_seen": 1895366656 + }, + { + "epoch": 0.15, + "learning_rate": 8.598138340555289e-05, + "loss": 2.6153, + "theoretical_loss": 3.445855836790739, + "tokens_seen": 1895497728 + }, + { + "epoch": 0.15, + "learning_rate": 8.597335901139464e-05, + "loss": 2.2966, + "theoretical_loss": 3.4458357603066996, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1044449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.796210289001465, + "objective/train/theoretical_loss": 3.4458156855994457, + "objective/train/tokens_used": 266218976, + "theoretical_loss": 3.4458156855994457, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.15, + "learning_rate": 8.59653346172364e-05, + "loss": 2.6243, + "theoretical_loss": 3.4458156855994457, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.15, + "learning_rate": 8.595731022307816e-05, + "loss": 2.5802, + "theoretical_loss": 3.4457956126686975, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.15, + "learning_rate": 8.594928582891993e-05, + "loss": 2.5255, + "theoretical_loss": 3.445775541514175, + "tokens_seen": 1896022016 + }, + { + "epoch": 0.15, + "learning_rate": 8.594126143476168e-05, + "loss": 2.5969, + "theoretical_loss": 3.4457554721355983, + "tokens_seen": 1896153088 + }, + { + "epoch": 0.15, + "learning_rate": 8.593323704060345e-05, + "loss": 2.5436, + "theoretical_loss": 3.4457354045326873, + "tokens_seen": 1896284160 + }, + { + "epoch": 0.15, + "learning_rate": 8.59252126464452e-05, + "loss": 2.5276, + "theoretical_loss": 3.4457153387051616, + "tokens_seen": 1896415232 + }, + { + "epoch": 0.15, + "learning_rate": 8.591718825228697e-05, + "loss": 2.4972, + "theoretical_loss": 3.4456952746527425, + "tokens_seen": 1896546304 + }, + { + "epoch": 0.15, + "learning_rate": 8.590916385812872e-05, + "loss": 2.7186, + "theoretical_loss": 3.44567521237515, + "tokens_seen": 1896677376 + }, + { + "epoch": 0.15, + "learning_rate": 8.590113946397047e-05, + "loss": 2.5623, + "theoretical_loss": 3.4456551518721037, + "tokens_seen": 1896808448 + }, + { + "epoch": 0.15, + "learning_rate": 8.589311506981224e-05, + "loss": 2.7065, + "theoretical_loss": 3.4456350931433253, + "tokens_seen": 1896939520 + }, + { + "epoch": 0.15, + "learning_rate": 8.588509067565399e-05, + "loss": 2.5867, + "theoretical_loss": 3.445615036188534, + "tokens_seen": 1897070592 + }, + { + "epoch": 0.15, + "learning_rate": 8.587706628149576e-05, + "loss": 2.5036, + "theoretical_loss": 3.4455949810074515, + "tokens_seen": 1897201664 + }, + { + "epoch": 0.15, + "learning_rate": 8.586904188733751e-05, + "loss": 2.444, + "theoretical_loss": 3.4455749275997976, + "tokens_seen": 1897332736 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1045116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.431305408477783, + "objective/train/theoretical_loss": 3.4455649015609193, + "objective/train/tokens_used": 267857376, + "theoretical_loss": 3.4455649015609193, + "tokens_seen": 1897398272 + }, + { + "epoch": 0.15, + "learning_rate": 8.586101749317928e-05, + "loss": 2.4496, + "theoretical_loss": 3.4455548759652936, + "tokens_seen": 1897463808 + }, + { + "epoch": 0.15, + "learning_rate": 8.585299309902103e-05, + "loss": 2.6165, + "theoretical_loss": 3.4455348261036596, + "tokens_seen": 1897594880 + }, + { + "epoch": 0.15, + "learning_rate": 8.584496870486278e-05, + "loss": 2.5878, + "theoretical_loss": 3.4455147780146174, + "tokens_seen": 1897725952 + }, + { + "epoch": 0.15, + "learning_rate": 8.583694431070455e-05, + "loss": 2.5, + "theoretical_loss": 3.4454947316978863, + "tokens_seen": 1897857024 + }, + { + "epoch": 0.15, + "learning_rate": 8.58289199165463e-05, + "loss": 2.6347, + "theoretical_loss": 3.445474687153189, + "tokens_seen": 1897988096 + }, + { + "epoch": 0.15, + "learning_rate": 8.582089552238807e-05, + "loss": 2.5567, + "theoretical_loss": 3.4454546443802454, + "tokens_seen": 1898119168 + }, + { + "epoch": 0.15, + "learning_rate": 8.581287112822982e-05, + "loss": 2.5346, + "theoretical_loss": 3.445434603378777, + "tokens_seen": 1898250240 + }, + { + "epoch": 0.15, + "learning_rate": 8.580484673407159e-05, + "loss": 2.698, + "theoretical_loss": 3.4454145641485043, + "tokens_seen": 1898381312 + }, + { + "epoch": 0.15, + "learning_rate": 8.579682233991334e-05, + "loss": 2.5593, + "theoretical_loss": 3.4453945266891486, + "tokens_seen": 1898512384 + }, + { + "epoch": 0.15, + "learning_rate": 8.57887979457551e-05, + "loss": 2.5686, + "theoretical_loss": 3.4453744910004325, + "tokens_seen": 1898643456 + }, + { + "epoch": 0.15, + "learning_rate": 8.578077355159686e-05, + "loss": 2.5992, + "theoretical_loss": 3.4453544570820758, + "tokens_seen": 1898774528 + }, + { + "epoch": 0.15, + "learning_rate": 8.577274915743862e-05, + "loss": 2.5495, + "theoretical_loss": 3.4453344249338005, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1046217, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568467855453491, + "objective/train/theoretical_loss": 3.4453143945553277, + "objective/train/tokens_used": 269495776, + "theoretical_loss": 3.4453143945553277, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.15, + "learning_rate": 8.576472476328038e-05, + "loss": 2.6635, + "theoretical_loss": 3.4453143945553277, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.15, + "learning_rate": 8.575670036912214e-05, + "loss": 2.591, + "theoretical_loss": 3.4452943659463795, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.15, + "learning_rate": 8.57486759749639e-05, + "loss": 2.6946, + "theoretical_loss": 3.445274339106677, + "tokens_seen": 1899298816 + }, + { + "epoch": 0.15, + "learning_rate": 8.574065158080566e-05, + "loss": 2.5766, + "theoretical_loss": 3.4452543140359415, + "tokens_seen": 1899429888 + }, + { + "epoch": 0.15, + "learning_rate": 8.573262718664741e-05, + "loss": 2.6189, + "theoretical_loss": 3.4452342907338958, + "tokens_seen": 1899560960 + }, + { + "epoch": 0.15, + "learning_rate": 8.572460279248918e-05, + "loss": 2.556, + "theoretical_loss": 3.4452142692002603, + "tokens_seen": 1899692032 + }, + { + "epoch": 0.15, + "learning_rate": 8.571657839833093e-05, + "loss": 2.5013, + "theoretical_loss": 3.445194249434758, + "tokens_seen": 1899823104 + }, + { + "epoch": 0.15, + "learning_rate": 8.57085540041727e-05, + "loss": 2.8092, + "theoretical_loss": 3.4451742314371105, + "tokens_seen": 1899954176 + }, + { + "epoch": 0.15, + "learning_rate": 8.570052961001445e-05, + "loss": 2.6131, + "theoretical_loss": 3.4451542152070393, + "tokens_seen": 1900085248 + }, + { + "epoch": 0.15, + "learning_rate": 8.569250521585622e-05, + "loss": 2.5599, + "theoretical_loss": 3.445134200744267, + "tokens_seen": 1900216320 + }, + { + "epoch": 0.15, + "learning_rate": 8.568448082169797e-05, + "loss": 2.4855, + "theoretical_loss": 3.4451141880485148, + "tokens_seen": 1900347392 + }, + { + "epoch": 0.15, + "learning_rate": 8.567645642753972e-05, + "loss": 2.5664, + "theoretical_loss": 3.445094177119505, + "tokens_seen": 1900478464 + }, + { + "epoch": 0.15, + "learning_rate": 8.566843203338149e-05, + "loss": 2.6017, + "theoretical_loss": 3.4450741679569608, + "tokens_seen": 1900609536 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1047224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.664011001586914, + "objective/train/theoretical_loss": 3.445064164038026, + "objective/train/tokens_used": 271134176, + "theoretical_loss": 3.445064164038026, + "tokens_seen": 1900675072 + }, + { + "epoch": 0.15, + "learning_rate": 8.566040763922324e-05, + "loss": 2.6229, + "theoretical_loss": 3.4450541605606038, + "tokens_seen": 1900740608 + }, + { + "epoch": 0.15, + "learning_rate": 8.565238324506501e-05, + "loss": 2.5607, + "theoretical_loss": 3.4450341549301564, + "tokens_seen": 1900871680 + }, + { + "epoch": 0.15, + "learning_rate": 8.564435885090676e-05, + "loss": 2.4586, + "theoretical_loss": 3.4450141510653403, + "tokens_seen": 1901002752 + }, + { + "epoch": 0.15, + "learning_rate": 8.563633445674853e-05, + "loss": 2.4271, + "theoretical_loss": 3.444994148965879, + "tokens_seen": 1901133824 + }, + { + "epoch": 0.15, + "learning_rate": 8.562831006259028e-05, + "loss": 2.6382, + "theoretical_loss": 3.444974148631495, + "tokens_seen": 1901264896 + }, + { + "epoch": 0.15, + "learning_rate": 8.562028566843204e-05, + "loss": 2.6207, + "theoretical_loss": 3.4449541500619096, + "tokens_seen": 1901395968 + }, + { + "epoch": 0.15, + "learning_rate": 8.56122612742738e-05, + "loss": 2.6704, + "theoretical_loss": 3.4449341532568467, + "tokens_seen": 1901527040 + }, + { + "epoch": 0.15, + "learning_rate": 8.560423688011555e-05, + "loss": 2.7467, + "theoretical_loss": 3.4449141582160285, + "tokens_seen": 1901658112 + }, + { + "epoch": 0.15, + "learning_rate": 8.559621248595732e-05, + "loss": 2.6002, + "theoretical_loss": 3.4448941649391784, + "tokens_seen": 1901789184 + }, + { + "epoch": 0.15, + "learning_rate": 8.558818809179907e-05, + "loss": 2.6505, + "theoretical_loss": 3.444874173426018, + "tokens_seen": 1901920256 + }, + { + "epoch": 0.15, + "learning_rate": 8.558016369764084e-05, + "loss": 2.5797, + "theoretical_loss": 3.444854183676271, + "tokens_seen": 1902051328 + }, + { + "epoch": 0.15, + "learning_rate": 8.55721393034826e-05, + "loss": 2.5589, + "theoretical_loss": 3.44483419568966, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1047772, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.691408395767212, + "objective/train/theoretical_loss": 3.444814209465909, + "objective/train/tokens_used": 272772576, + "theoretical_loss": 3.444814209465909, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.15, + "learning_rate": 8.556411490932436e-05, + "loss": 2.6686, + "theoretical_loss": 3.444814209465909, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.15, + "learning_rate": 8.555609051516611e-05, + "loss": 2.7427, + "theoretical_loss": 3.44479422500474, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 8.554806612100787e-05, + "loss": 2.5318, + "theoretical_loss": 3.4447742423058765, + "tokens_seen": 1902575616 + }, + { + "epoch": 0.15, + "learning_rate": 8.554004172684963e-05, + "loss": 2.5627, + "theoretical_loss": 3.4447542613690416, + "tokens_seen": 1902706688 + }, + { + "epoch": 0.15, + "learning_rate": 8.553201733269139e-05, + "loss": 2.6165, + "theoretical_loss": 3.4447342821939584, + "tokens_seen": 1902837760 + }, + { + "epoch": 0.15, + "learning_rate": 8.552399293853315e-05, + "loss": 2.651, + "theoretical_loss": 3.4447143047803506, + "tokens_seen": 1902968832 + }, + { + "epoch": 0.15, + "learning_rate": 8.551596854437491e-05, + "loss": 2.5404, + "theoretical_loss": 3.444694329127941, + "tokens_seen": 1903099904 + }, + { + "epoch": 0.15, + "learning_rate": 8.550794415021667e-05, + "loss": 2.6022, + "theoretical_loss": 3.4446743552364545, + "tokens_seen": 1903230976 + }, + { + "epoch": 0.15, + "learning_rate": 8.549991975605843e-05, + "loss": 2.5531, + "theoretical_loss": 3.444654383105613, + "tokens_seen": 1903362048 + }, + { + "epoch": 0.15, + "learning_rate": 8.549189536190018e-05, + "loss": 2.6862, + "theoretical_loss": 3.44463441273514, + "tokens_seen": 1903493120 + }, + { + "epoch": 0.15, + "learning_rate": 8.548387096774195e-05, + "loss": 2.7709, + "theoretical_loss": 3.4446144441247606, + "tokens_seen": 1903624192 + }, + { + "epoch": 0.15, + "learning_rate": 8.54758465735837e-05, + "loss": 2.5729, + "theoretical_loss": 3.4445944772741974, + "tokens_seen": 1903755264 + }, + { + "epoch": 0.15, + "learning_rate": 8.546782217942547e-05, + "loss": 2.6571, + "theoretical_loss": 3.444574512183175, + "tokens_seen": 1903886336 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1048883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7661850452423096, + "objective/train/theoretical_loss": 3.444564530297405, + "objective/train/tokens_used": 274410976, + "theoretical_loss": 3.444564530297405, + "tokens_seen": 1903951872 + }, + { + "epoch": 0.15, + "learning_rate": 8.545979778526722e-05, + "loss": 2.6316, + "theoretical_loss": 3.444554548851416, + "tokens_seen": 1904017408 + }, + { + "epoch": 0.15, + "learning_rate": 8.545177339110899e-05, + "loss": 2.3924, + "theoretical_loss": 3.4445345872786453, + "tokens_seen": 1904148480 + }, + { + "epoch": 0.15, + "learning_rate": 8.544374899695074e-05, + "loss": 2.6117, + "theoretical_loss": 3.444514627464587, + "tokens_seen": 1904279552 + }, + { + "epoch": 0.15, + "learning_rate": 8.543572460279249e-05, + "loss": 2.5, + "theoretical_loss": 3.4444946694089635, + "tokens_seen": 1904410624 + }, + { + "epoch": 0.15, + "learning_rate": 8.542770020863426e-05, + "loss": 2.6789, + "theoretical_loss": 3.444474713111501, + "tokens_seen": 1904541696 + }, + { + "epoch": 0.15, + "learning_rate": 8.541967581447601e-05, + "loss": 2.5173, + "theoretical_loss": 3.444454758571922, + "tokens_seen": 1904672768 + }, + { + "epoch": 0.15, + "learning_rate": 8.541165142031778e-05, + "loss": 2.6405, + "theoretical_loss": 3.444434805789952, + "tokens_seen": 1904803840 + }, + { + "epoch": 0.15, + "learning_rate": 8.540362702615953e-05, + "loss": 2.5492, + "theoretical_loss": 3.444414854765314, + "tokens_seen": 1904934912 + }, + { + "epoch": 0.15, + "learning_rate": 8.53956026320013e-05, + "loss": 2.4519, + "theoretical_loss": 3.4443949054977336, + "tokens_seen": 1905065984 + }, + { + "epoch": 0.15, + "learning_rate": 8.538757823784305e-05, + "loss": 2.4236, + "theoretical_loss": 3.444374957986934, + "tokens_seen": 1905197056 + }, + { + "epoch": 0.15, + "learning_rate": 8.53795538436848e-05, + "loss": 2.5186, + "theoretical_loss": 3.4443550122326405, + "tokens_seen": 1905328128 + }, + { + "epoch": 0.15, + "learning_rate": 8.537152944952657e-05, + "loss": 2.5571, + "theoretical_loss": 3.444335068234577, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1049481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8444793224334717, + "objective/train/theoretical_loss": 3.4443151259924685, + "objective/train/tokens_used": 276049376, + "theoretical_loss": 3.4443151259924685, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 8.536350505536833e-05, + "loss": 2.7023, + "theoretical_loss": 3.4443151259924685, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 8.535548066121009e-05, + "loss": 2.5393, + "theoretical_loss": 3.4442951855060393, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.16, + "learning_rate": 8.534745626705184e-05, + "loss": 2.5215, + "theoretical_loss": 3.444275246775015, + "tokens_seen": 1905852416 + }, + { + "epoch": 0.16, + "learning_rate": 8.533943187289361e-05, + "loss": 2.6303, + "theoretical_loss": 3.444255309799119, + "tokens_seen": 1905983488 + }, + { + "epoch": 0.16, + "learning_rate": 8.533140747873536e-05, + "loss": 2.6693, + "theoretical_loss": 3.4442353745780765, + "tokens_seen": 1906114560 + }, + { + "epoch": 0.16, + "learning_rate": 8.532338308457712e-05, + "loss": 2.6018, + "theoretical_loss": 3.444215441111613, + "tokens_seen": 1906245632 + }, + { + "epoch": 0.16, + "learning_rate": 8.531535869041888e-05, + "loss": 2.6154, + "theoretical_loss": 3.444195509399453, + "tokens_seen": 1906376704 + }, + { + "epoch": 0.16, + "learning_rate": 8.530733429626064e-05, + "loss": 2.6642, + "theoretical_loss": 3.444175579441322, + "tokens_seen": 1906507776 + }, + { + "epoch": 0.16, + "learning_rate": 8.52993099021024e-05, + "loss": 2.5281, + "theoretical_loss": 3.4441556512369442, + "tokens_seen": 1906638848 + }, + { + "epoch": 0.16, + "learning_rate": 8.529128550794416e-05, + "loss": 2.5748, + "theoretical_loss": 3.4441357247860456, + "tokens_seen": 1906769920 + }, + { + "epoch": 0.16, + "learning_rate": 8.528326111378591e-05, + "loss": 2.4962, + "theoretical_loss": 3.4441158000883507, + "tokens_seen": 1906900992 + }, + { + "epoch": 0.16, + "learning_rate": 8.527523671962768e-05, + "loss": 2.6538, + "theoretical_loss": 3.4440958771435852, + "tokens_seen": 1907032064 + }, + { + "epoch": 0.16, + "learning_rate": 8.526721232546943e-05, + "loss": 2.7252, + "theoretical_loss": 3.444075955951474, + "tokens_seen": 1907163136 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1050597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0899901390075684, + "objective/train/theoretical_loss": 3.4440659960125783, + "objective/train/tokens_used": 277687776, + "theoretical_loss": 3.4440659960125783, + "tokens_seen": 1907228672 + }, + { + "epoch": 0.16, + "learning_rate": 8.52591879313112e-05, + "loss": 2.5514, + "theoretical_loss": 3.4440560365117427, + "tokens_seen": 1907294208 + }, + { + "epoch": 0.16, + "learning_rate": 8.525116353715295e-05, + "loss": 2.5941, + "theoretical_loss": 3.4440361188241173, + "tokens_seen": 1907425280 + }, + { + "epoch": 0.16, + "learning_rate": 8.524313914299472e-05, + "loss": 2.58, + "theoretical_loss": 3.4440162028883226, + "tokens_seen": 1907556352 + }, + { + "epoch": 0.16, + "learning_rate": 8.523511474883647e-05, + "loss": 2.3977, + "theoretical_loss": 3.443996288704084, + "tokens_seen": 1907687424 + }, + { + "epoch": 0.16, + "learning_rate": 8.522709035467822e-05, + "loss": 2.5847, + "theoretical_loss": 3.443976376271128, + "tokens_seen": 1907818496 + }, + { + "epoch": 0.16, + "learning_rate": 8.521906596051999e-05, + "loss": 2.6012, + "theoretical_loss": 3.4439564655891797, + "tokens_seen": 1907949568 + }, + { + "epoch": 0.16, + "learning_rate": 8.521104156636174e-05, + "loss": 2.6382, + "theoretical_loss": 3.4439365566579645, + "tokens_seen": 1908080640 + }, + { + "epoch": 0.16, + "learning_rate": 8.520301717220351e-05, + "loss": 2.4888, + "theoretical_loss": 3.4439166494772087, + "tokens_seen": 1908211712 + }, + { + "epoch": 0.16, + "learning_rate": 8.519499277804526e-05, + "loss": 2.7036, + "theoretical_loss": 3.4438967440466386, + "tokens_seen": 1908342784 + }, + { + "epoch": 0.16, + "learning_rate": 8.518696838388702e-05, + "loss": 2.6248, + "theoretical_loss": 3.443876840365979, + "tokens_seen": 1908473856 + }, + { + "epoch": 0.16, + "learning_rate": 8.517894398972878e-05, + "loss": 2.6627, + "theoretical_loss": 3.443856938434957, + "tokens_seen": 1908604928 + }, + { + "epoch": 0.16, + "learning_rate": 8.517091959557054e-05, + "loss": 2.5851, + "theoretical_loss": 3.443837038253298, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1051040, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.524304151535034, + "objective/train/theoretical_loss": 3.4438171398207285, + "objective/train/tokens_used": 279326176, + "theoretical_loss": 3.4438171398207285, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.16, + "learning_rate": 8.51628952014123e-05, + "loss": 2.5677, + "theoretical_loss": 3.4438171398207285, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.16, + "learning_rate": 8.515487080725406e-05, + "loss": 2.5599, + "theoretical_loss": 3.4437972431369746, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.16, + "learning_rate": 8.514684641309582e-05, + "loss": 2.7516, + "theoretical_loss": 3.443777348201762, + "tokens_seen": 1909129216 + }, + { + "epoch": 0.16, + "learning_rate": 8.513882201893758e-05, + "loss": 2.4558, + "theoretical_loss": 3.4437574550148176, + "tokens_seen": 1909260288 + }, + { + "epoch": 0.16, + "learning_rate": 8.513079762477933e-05, + "loss": 2.6508, + "theoretical_loss": 3.4437375635758674, + "tokens_seen": 1909391360 + }, + { + "epoch": 0.16, + "learning_rate": 8.51227732306211e-05, + "loss": 2.6752, + "theoretical_loss": 3.4437176738846382, + "tokens_seen": 1909522432 + }, + { + "epoch": 0.16, + "learning_rate": 8.511474883646285e-05, + "loss": 2.6269, + "theoretical_loss": 3.4436977859408566, + "tokens_seen": 1909653504 + }, + { + "epoch": 0.16, + "learning_rate": 8.510672444230461e-05, + "loss": 2.6323, + "theoretical_loss": 3.443677899744249, + "tokens_seen": 1909784576 + }, + { + "epoch": 0.16, + "learning_rate": 8.509870004814637e-05, + "loss": 2.6092, + "theoretical_loss": 3.4436580152945413, + "tokens_seen": 1909915648 + }, + { + "epoch": 0.16, + "learning_rate": 8.509067565398812e-05, + "loss": 2.6991, + "theoretical_loss": 3.4436381325914613, + "tokens_seen": 1910046720 + }, + { + "epoch": 0.16, + "learning_rate": 8.508265125982989e-05, + "loss": 2.7202, + "theoretical_loss": 3.443618251634735, + "tokens_seen": 1910177792 + }, + { + "epoch": 0.16, + "learning_rate": 8.507462686567164e-05, + "loss": 2.397, + "theoretical_loss": 3.443598372424089, + "tokens_seen": 1910308864 + }, + { + "epoch": 0.16, + "learning_rate": 8.506660247151341e-05, + "loss": 2.4573, + "theoretical_loss": 3.443578494959251, + "tokens_seen": 1910439936 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1051849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3590188026428223, + "objective/train/theoretical_loss": 3.4435685568814245, + "objective/train/tokens_used": 280964576, + "theoretical_loss": 3.4435685568814245, + "tokens_seen": 1910505472 + }, + { + "epoch": 0.16, + "learning_rate": 8.505857807735516e-05, + "loss": 2.6063, + "theoretical_loss": 3.4435586192399477, + "tokens_seen": 1910571008 + }, + { + "epoch": 0.16, + "learning_rate": 8.505055368319693e-05, + "loss": 2.6925, + "theoretical_loss": 3.443538745265905, + "tokens_seen": 1910702080 + }, + { + "epoch": 0.16, + "learning_rate": 8.504252928903868e-05, + "loss": 2.6257, + "theoretical_loss": 3.443518873036852, + "tokens_seen": 1910833152 + }, + { + "epoch": 0.16, + "learning_rate": 8.503450489488043e-05, + "loss": 2.5778, + "theoretical_loss": 3.443499002552514, + "tokens_seen": 1910964224 + }, + { + "epoch": 0.16, + "learning_rate": 8.50264805007222e-05, + "loss": 2.5841, + "theoretical_loss": 3.4434791338126187, + "tokens_seen": 1911095296 + }, + { + "epoch": 0.16, + "learning_rate": 8.501845610656395e-05, + "loss": 2.5606, + "theoretical_loss": 3.4434592668168933, + "tokens_seen": 1911226368 + }, + { + "epoch": 0.16, + "learning_rate": 8.501043171240572e-05, + "loss": 2.7309, + "theoretical_loss": 3.4434394015650653, + "tokens_seen": 1911357440 + }, + { + "epoch": 0.16, + "learning_rate": 8.500240731824747e-05, + "loss": 2.6035, + "theoretical_loss": 3.443419538056862, + "tokens_seen": 1911488512 + }, + { + "epoch": 0.16, + "learning_rate": 8.499438292408923e-05, + "loss": 2.5485, + "theoretical_loss": 3.4433996762920107, + "tokens_seen": 1911619584 + }, + { + "epoch": 0.16, + "learning_rate": 8.498635852993099e-05, + "loss": 2.6219, + "theoretical_loss": 3.443379816270239, + "tokens_seen": 1911750656 + }, + { + "epoch": 0.16, + "learning_rate": 8.497833413577275e-05, + "loss": 2.6035, + "theoretical_loss": 3.443359957991275, + "tokens_seen": 1911881728 + }, + { + "epoch": 0.16, + "learning_rate": 8.497030974161451e-05, + "loss": 2.4346, + "theoretical_loss": 3.4433401014548446, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1052490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.299638032913208, + "objective/train/theoretical_loss": 3.4433202466606767, + "objective/train/tokens_used": 282602976, + "theoretical_loss": 3.4433202466606767, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.16, + "learning_rate": 8.496228534745627e-05, + "loss": 2.6254, + "theoretical_loss": 3.4433202466606767, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.16, + "learning_rate": 8.495426095329802e-05, + "loss": 2.4929, + "theoretical_loss": 3.4433003936084994, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.16, + "learning_rate": 8.494623655913979e-05, + "loss": 2.5459, + "theoretical_loss": 3.44328054229804, + "tokens_seen": 1912406016 + }, + { + "epoch": 0.16, + "learning_rate": 8.493821216498154e-05, + "loss": 2.4689, + "theoretical_loss": 3.4432606927290257, + "tokens_seen": 1912537088 + }, + { + "epoch": 0.16, + "learning_rate": 8.49301877708233e-05, + "loss": 2.6596, + "theoretical_loss": 3.443240844901185, + "tokens_seen": 1912668160 + }, + { + "epoch": 0.16, + "learning_rate": 8.492216337666506e-05, + "loss": 2.5501, + "theoretical_loss": 3.4432209988142457, + "tokens_seen": 1912799232 + }, + { + "epoch": 0.16, + "learning_rate": 8.491413898250683e-05, + "loss": 2.5588, + "theoretical_loss": 3.4432011544679364, + "tokens_seen": 1912930304 + }, + { + "epoch": 0.16, + "learning_rate": 8.490611458834858e-05, + "loss": 2.631, + "theoretical_loss": 3.4431813118619847, + "tokens_seen": 1913061376 + }, + { + "epoch": 0.16, + "learning_rate": 8.489809019419033e-05, + "loss": 2.6742, + "theoretical_loss": 3.4431614709961185, + "tokens_seen": 1913192448 + }, + { + "epoch": 0.16, + "learning_rate": 8.48900658000321e-05, + "loss": 2.4847, + "theoretical_loss": 3.4431416318700663, + "tokens_seen": 1913323520 + }, + { + "epoch": 0.16, + "learning_rate": 8.488204140587385e-05, + "loss": 2.4341, + "theoretical_loss": 3.443121794483557, + "tokens_seen": 1913454592 + }, + { + "epoch": 0.16, + "learning_rate": 8.487401701171562e-05, + "loss": 2.7514, + "theoretical_loss": 3.4431019588363174, + "tokens_seen": 1913585664 + }, + { + "epoch": 0.16, + "learning_rate": 8.486599261755737e-05, + "loss": 2.5306, + "theoretical_loss": 3.443082124928077, + "tokens_seen": 1913716736 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1053877, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7644686698913574, + "objective/train/theoretical_loss": 3.443072208625997, + "objective/train/tokens_used": 284241376, + "theoretical_loss": 3.443072208625997, + "tokens_seen": 1913782272 + }, + { + "epoch": 0.16, + "learning_rate": 8.485796822339914e-05, + "loss": 2.6458, + "theoretical_loss": 3.4430622927585643, + "tokens_seen": 1913847808 + }, + { + "epoch": 0.16, + "learning_rate": 8.484994382924089e-05, + "loss": 2.5744, + "theoretical_loss": 3.4430424623275075, + "tokens_seen": 1913978880 + }, + { + "epoch": 0.16, + "learning_rate": 8.484191943508264e-05, + "loss": 2.5332, + "theoretical_loss": 3.443022633634635, + "tokens_seen": 1914109952 + }, + { + "epoch": 0.16, + "learning_rate": 8.483389504092441e-05, + "loss": 2.6024, + "theoretical_loss": 3.443002806679676, + "tokens_seen": 1914241024 + }, + { + "epoch": 0.16, + "learning_rate": 8.482587064676616e-05, + "loss": 2.6787, + "theoretical_loss": 3.4429829814623583, + "tokens_seen": 1914372096 + }, + { + "epoch": 0.16, + "learning_rate": 8.481784625260793e-05, + "loss": 2.4463, + "theoretical_loss": 3.442963157982412, + "tokens_seen": 1914503168 + }, + { + "epoch": 0.16, + "learning_rate": 8.480982185844968e-05, + "loss": 2.6267, + "theoretical_loss": 3.4429433362395643, + "tokens_seen": 1914634240 + }, + { + "epoch": 0.16, + "learning_rate": 8.480179746429145e-05, + "loss": 2.3903, + "theoretical_loss": 3.442923516233545, + "tokens_seen": 1914765312 + }, + { + "epoch": 0.16, + "learning_rate": 8.47937730701332e-05, + "loss": 2.4938, + "theoretical_loss": 3.4429036979640832, + "tokens_seen": 1914896384 + }, + { + "epoch": 0.16, + "learning_rate": 8.478574867597496e-05, + "loss": 2.5209, + "theoretical_loss": 3.442883881430908, + "tokens_seen": 1915027456 + }, + { + "epoch": 0.16, + "learning_rate": 8.477772428181672e-05, + "loss": 2.6642, + "theoretical_loss": 3.442864066633747, + "tokens_seen": 1915158528 + }, + { + "epoch": 0.16, + "learning_rate": 8.476969988765848e-05, + "loss": 2.6275, + "theoretical_loss": 3.442844253572331, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1054366, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9697377681732178, + "objective/train/theoretical_loss": 3.4428244422463887, + "objective/train/tokens_used": 285879776, + "theoretical_loss": 3.4428244422463887, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.16, + "learning_rate": 8.476167549350024e-05, + "loss": 2.6293, + "theoretical_loss": 3.4428244422463887, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.16, + "learning_rate": 8.4753651099342e-05, + "loss": 2.4817, + "theoretical_loss": 3.4428046326556494, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.16, + "learning_rate": 8.474562670518376e-05, + "loss": 2.4483, + "theoretical_loss": 3.4427848247998423, + "tokens_seen": 1915682816 + }, + { + "epoch": 0.16, + "learning_rate": 8.473760231102552e-05, + "loss": 2.6185, + "theoretical_loss": 3.4427650186786964, + "tokens_seen": 1915813888 + }, + { + "epoch": 0.16, + "learning_rate": 8.472957791686727e-05, + "loss": 2.5448, + "theoretical_loss": 3.4427452142919415, + "tokens_seen": 1915944960 + }, + { + "epoch": 0.16, + "learning_rate": 8.472155352270904e-05, + "loss": 2.5756, + "theoretical_loss": 3.442725411639307, + "tokens_seen": 1916076032 + }, + { + "epoch": 0.16, + "learning_rate": 8.471352912855079e-05, + "loss": 2.6482, + "theoretical_loss": 3.4427056107205223, + "tokens_seen": 1916207104 + }, + { + "epoch": 0.16, + "learning_rate": 8.470550473439256e-05, + "loss": 2.5003, + "theoretical_loss": 3.442685811535317, + "tokens_seen": 1916338176 + }, + { + "epoch": 0.16, + "learning_rate": 8.469748034023431e-05, + "loss": 2.7789, + "theoretical_loss": 3.4426660140834215, + "tokens_seen": 1916469248 + }, + { + "epoch": 0.16, + "learning_rate": 8.468945594607608e-05, + "loss": 2.5984, + "theoretical_loss": 3.442646218364565, + "tokens_seen": 1916600320 + }, + { + "epoch": 0.16, + "learning_rate": 8.468143155191783e-05, + "loss": 2.5712, + "theoretical_loss": 3.442626424378477, + "tokens_seen": 1916731392 + }, + { + "epoch": 0.16, + "learning_rate": 8.467340715775958e-05, + "loss": 2.5202, + "theoretical_loss": 3.4426066321248876, + "tokens_seen": 1916862464 + }, + { + "epoch": 0.16, + "learning_rate": 8.466538276360135e-05, + "loss": 2.5165, + "theoretical_loss": 3.442586841603527, + "tokens_seen": 1916993536 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1055597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.398259162902832, + "objective/train/theoretical_loss": 3.4425769469923475, + "objective/train/tokens_used": 287518176, + "theoretical_loss": 3.4425769469923475, + "tokens_seen": 1917059072 + }, + { + "epoch": 0.16, + "learning_rate": 8.46573583694431e-05, + "loss": 2.6192, + "theoretical_loss": 3.442567052814124, + "tokens_seen": 1917124608 + }, + { + "epoch": 0.16, + "learning_rate": 8.464933397528487e-05, + "loss": 2.4034, + "theoretical_loss": 3.442547265756411, + "tokens_seen": 1917255680 + }, + { + "epoch": 0.16, + "learning_rate": 8.464130958112662e-05, + "loss": 2.4076, + "theoretical_loss": 3.4425274804301154, + "tokens_seen": 1917386752 + }, + { + "epoch": 0.16, + "learning_rate": 8.463328518696839e-05, + "loss": 2.6285, + "theoretical_loss": 3.442507696834969, + "tokens_seen": 1917517824 + }, + { + "epoch": 0.16, + "learning_rate": 8.462526079281014e-05, + "loss": 2.4717, + "theoretical_loss": 3.4424879149707017, + "tokens_seen": 1917648896 + }, + { + "epoch": 0.16, + "learning_rate": 8.461723639865191e-05, + "loss": 2.649, + "theoretical_loss": 3.442468134837043, + "tokens_seen": 1917779968 + }, + { + "epoch": 0.16, + "learning_rate": 8.460921200449366e-05, + "loss": 2.5644, + "theoretical_loss": 3.4424483564337245, + "tokens_seen": 1917911040 + }, + { + "epoch": 0.16, + "learning_rate": 8.460118761033541e-05, + "loss": 2.5549, + "theoretical_loss": 3.4424285797604757, + "tokens_seen": 1918042112 + }, + { + "epoch": 0.16, + "learning_rate": 8.459316321617718e-05, + "loss": 2.5357, + "theoretical_loss": 3.4424088048170276, + "tokens_seen": 1918173184 + }, + { + "epoch": 0.16, + "learning_rate": 8.458513882201893e-05, + "loss": 2.4481, + "theoretical_loss": 3.4423890316031107, + "tokens_seen": 1918304256 + }, + { + "epoch": 0.16, + "learning_rate": 8.45771144278607e-05, + "loss": 2.3806, + "theoretical_loss": 3.442369260118455, + "tokens_seen": 1918435328 + }, + { + "epoch": 0.16, + "learning_rate": 8.456909003370245e-05, + "loss": 2.5601, + "theoretical_loss": 3.4423494903627914, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1056906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.411397695541382, + "objective/train/theoretical_loss": 3.4423297223358507, + "objective/train/tokens_used": 289156576, + "theoretical_loss": 3.4423297223358507, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 8.456106563954422e-05, + "loss": 2.6341, + "theoretical_loss": 3.4423297223358507, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 8.455304124538597e-05, + "loss": 2.4965, + "theoretical_loss": 3.4423099560373633, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 8.454501685122773e-05, + "loss": 2.4888, + "theoretical_loss": 3.4422901914670607, + "tokens_seen": 1918959616 + }, + { + "epoch": 0.16, + "learning_rate": 8.45369924570695e-05, + "loss": 2.5232, + "theoretical_loss": 3.4422704286246733, + "tokens_seen": 1919090688 + }, + { + "epoch": 0.16, + "learning_rate": 8.452896806291125e-05, + "loss": 2.6028, + "theoretical_loss": 3.4422506675099322, + "tokens_seen": 1919221760 + }, + { + "epoch": 0.16, + "learning_rate": 8.452094366875301e-05, + "loss": 2.4306, + "theoretical_loss": 3.4422309081225686, + "tokens_seen": 1919352832 + }, + { + "epoch": 0.16, + "learning_rate": 8.451291927459477e-05, + "loss": 2.6496, + "theoretical_loss": 3.442211150462313, + "tokens_seen": 1919483904 + }, + { + "epoch": 0.16, + "learning_rate": 8.450489488043653e-05, + "loss": 2.539, + "theoretical_loss": 3.442191394528897, + "tokens_seen": 1919614976 + }, + { + "epoch": 0.16, + "learning_rate": 8.449687048627829e-05, + "loss": 2.485, + "theoretical_loss": 3.4421716403220515, + "tokens_seen": 1919746048 + }, + { + "epoch": 0.16, + "learning_rate": 8.448884609212004e-05, + "loss": 2.6431, + "theoretical_loss": 3.442151887841508, + "tokens_seen": 1919877120 + }, + { + "epoch": 0.16, + "learning_rate": 8.44808216979618e-05, + "loss": 2.7387, + "theoretical_loss": 3.442132137086997, + "tokens_seen": 1920008192 + }, + { + "epoch": 0.16, + "learning_rate": 8.447279730380356e-05, + "loss": 2.5542, + "theoretical_loss": 3.442112388058251, + "tokens_seen": 1920139264 + }, + { + "epoch": 0.16, + "learning_rate": 8.446477290964533e-05, + "loss": 2.557, + "theoretical_loss": 3.4420926407550008, + "tokens_seen": 1920270336 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1057603, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.503713607788086, + "objective/train/theoretical_loss": 3.442082767750353, + "objective/train/tokens_used": 290794976, + "theoretical_loss": 3.442082767750353, + "tokens_seen": 1920335872 + }, + { + "epoch": 0.16, + "learning_rate": 8.445674851548708e-05, + "loss": 2.6189, + "theoretical_loss": 3.4420728951769783, + "tokens_seen": 1920401408 + }, + { + "epoch": 0.16, + "learning_rate": 8.444872412132885e-05, + "loss": 2.5225, + "theoretical_loss": 3.4420531513239148, + "tokens_seen": 1920532480 + }, + { + "epoch": 0.16, + "learning_rate": 8.44406997271706e-05, + "loss": 2.5151, + "theoretical_loss": 3.4420334091955413, + "tokens_seen": 1920663552 + }, + { + "epoch": 0.16, + "learning_rate": 8.443267533301235e-05, + "loss": 2.6353, + "theoretical_loss": 3.4420136687915903, + "tokens_seen": 1920794624 + }, + { + "epoch": 0.16, + "learning_rate": 8.442465093885412e-05, + "loss": 2.4835, + "theoretical_loss": 3.441993930111793, + "tokens_seen": 1920925696 + }, + { + "epoch": 0.16, + "learning_rate": 8.441662654469587e-05, + "loss": 2.5794, + "theoretical_loss": 3.4419741931558816, + "tokens_seen": 1921056768 + }, + { + "epoch": 0.16, + "learning_rate": 8.440860215053764e-05, + "loss": 2.6229, + "theoretical_loss": 3.441954457923588, + "tokens_seen": 1921187840 + }, + { + "epoch": 0.16, + "learning_rate": 8.440057775637939e-05, + "loss": 2.4523, + "theoretical_loss": 3.4419347244146437, + "tokens_seen": 1921318912 + }, + { + "epoch": 0.16, + "learning_rate": 8.439255336222116e-05, + "loss": 2.6327, + "theoretical_loss": 3.4419149926287806, + "tokens_seen": 1921449984 + }, + { + "epoch": 0.16, + "learning_rate": 8.438452896806291e-05, + "loss": 2.6949, + "theoretical_loss": 3.4418952625657306, + "tokens_seen": 1921581056 + }, + { + "epoch": 0.16, + "learning_rate": 8.437650457390466e-05, + "loss": 2.5928, + "theoretical_loss": 3.4418755342252267, + "tokens_seen": 1921712128 + }, + { + "epoch": 0.16, + "learning_rate": 8.436848017974643e-05, + "loss": 2.5388, + "theoretical_loss": 3.4418558076070003, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1058944, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.295034885406494, + "objective/train/theoretical_loss": 3.4418360827107835, + "objective/train/tokens_used": 292433376, + "theoretical_loss": 3.4418360827107835, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.16, + "learning_rate": 8.436045578558818e-05, + "loss": 2.5935, + "theoretical_loss": 3.4418360827107835, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.16, + "learning_rate": 8.435243139142995e-05, + "loss": 2.6164, + "theoretical_loss": 3.4418163595363094, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 8.43444069972717e-05, + "loss": 2.6415, + "theoretical_loss": 3.441796638083309, + "tokens_seen": 1922236416 + }, + { + "epoch": 0.17, + "learning_rate": 8.433638260311347e-05, + "loss": 2.6415, + "theoretical_loss": 3.4417769183515157, + "tokens_seen": 1922367488 + }, + { + "epoch": 0.17, + "learning_rate": 8.432835820895522e-05, + "loss": 2.5213, + "theoretical_loss": 3.441757200340662, + "tokens_seen": 1922498560 + }, + { + "epoch": 0.17, + "learning_rate": 8.432033381479699e-05, + "loss": 2.7047, + "theoretical_loss": 3.4417374840504795, + "tokens_seen": 1922629632 + }, + { + "epoch": 0.17, + "learning_rate": 8.431230942063874e-05, + "loss": 2.324, + "theoretical_loss": 3.441717769480701, + "tokens_seen": 1922760704 + }, + { + "epoch": 0.17, + "learning_rate": 8.43042850264805e-05, + "loss": 2.7517, + "theoretical_loss": 3.4416980566310604, + "tokens_seen": 1922891776 + }, + { + "epoch": 0.17, + "learning_rate": 8.429626063232226e-05, + "loss": 2.5888, + "theoretical_loss": 3.4416783455012885, + "tokens_seen": 1923022848 + }, + { + "epoch": 0.17, + "learning_rate": 8.428823623816402e-05, + "loss": 2.452, + "theoretical_loss": 3.4416586360911197, + "tokens_seen": 1923153920 + }, + { + "epoch": 0.17, + "learning_rate": 8.428021184400578e-05, + "loss": 2.6019, + "theoretical_loss": 3.4416389284002853, + "tokens_seen": 1923284992 + }, + { + "epoch": 0.17, + "learning_rate": 8.427218744984754e-05, + "loss": 2.622, + "theoretical_loss": 3.4416192224285194, + "tokens_seen": 1923416064 + }, + { + "epoch": 0.17, + "learning_rate": 8.42641630556893e-05, + "loss": 2.642, + "theoretical_loss": 3.441599518175554, + "tokens_seen": 1923547136 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1059301, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.181396245956421, + "objective/train/theoretical_loss": 3.441589666693538, + "objective/train/tokens_used": 294071776, + "theoretical_loss": 3.441589666693538, + "tokens_seen": 1923612672 + }, + { + "epoch": 0.17, + "learning_rate": 8.425613866153106e-05, + "loss": 2.6953, + "theoretical_loss": 3.4415798156411226, + "tokens_seen": 1923678208 + }, + { + "epoch": 0.17, + "learning_rate": 8.424811426737281e-05, + "loss": 2.5897, + "theoretical_loss": 3.4415601148249584, + "tokens_seen": 1923809280 + }, + { + "epoch": 0.17, + "learning_rate": 8.424008987321458e-05, + "loss": 2.5453, + "theoretical_loss": 3.441540415726794, + "tokens_seen": 1923940352 + }, + { + "epoch": 0.17, + "learning_rate": 8.423206547905633e-05, + "loss": 2.4628, + "theoretical_loss": 3.4415207183463625, + "tokens_seen": 1924071424 + }, + { + "epoch": 0.17, + "learning_rate": 8.42240410848981e-05, + "loss": 2.5583, + "theoretical_loss": 3.441501022683398, + "tokens_seen": 1924202496 + }, + { + "epoch": 0.17, + "learning_rate": 8.421601669073985e-05, + "loss": 2.5335, + "theoretical_loss": 3.4414813287376327, + "tokens_seen": 1924333568 + }, + { + "epoch": 0.17, + "learning_rate": 8.420799229658162e-05, + "loss": 2.5424, + "theoretical_loss": 3.4414616365088007, + "tokens_seen": 1924464640 + }, + { + "epoch": 0.17, + "learning_rate": 8.419996790242337e-05, + "loss": 2.5134, + "theoretical_loss": 3.441441945996635, + "tokens_seen": 1924595712 + }, + { + "epoch": 0.17, + "learning_rate": 8.419194350826512e-05, + "loss": 2.5916, + "theoretical_loss": 3.441422257200869, + "tokens_seen": 1924726784 + }, + { + "epoch": 0.17, + "learning_rate": 8.418391911410689e-05, + "loss": 2.4067, + "theoretical_loss": 3.4414025701212365, + "tokens_seen": 1924857856 + }, + { + "epoch": 0.17, + "learning_rate": 8.417589471994864e-05, + "loss": 2.5449, + "theoretical_loss": 3.441382884757471, + "tokens_seen": 1924988928 + }, + { + "epoch": 0.17, + "learning_rate": 8.416787032579041e-05, + "loss": 2.6262, + "theoretical_loss": 3.441363201109305, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1060240, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.727017641067505, + "objective/train/theoretical_loss": 3.4413435191764745, + "objective/train/tokens_used": 295710176, + "theoretical_loss": 3.4413435191764745, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.17, + "learning_rate": 8.415984593163216e-05, + "loss": 2.5072, + "theoretical_loss": 3.4413435191764745, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.17, + "learning_rate": 8.415182153747393e-05, + "loss": 2.5916, + "theoretical_loss": 3.441323838958712, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.17, + "learning_rate": 8.414379714331568e-05, + "loss": 2.6457, + "theoretical_loss": 3.4413041604557506, + "tokens_seen": 1925513216 + }, + { + "epoch": 0.17, + "learning_rate": 8.413577274915743e-05, + "loss": 2.5979, + "theoretical_loss": 3.4412844836673253, + "tokens_seen": 1925644288 + }, + { + "epoch": 0.17, + "learning_rate": 8.41277483549992e-05, + "loss": 2.5639, + "theoretical_loss": 3.4412648085931696, + "tokens_seen": 1925775360 + }, + { + "epoch": 0.17, + "learning_rate": 8.411972396084095e-05, + "loss": 2.5317, + "theoretical_loss": 3.4412451352330176, + "tokens_seen": 1925906432 + }, + { + "epoch": 0.17, + "learning_rate": 8.411169956668272e-05, + "loss": 2.553, + "theoretical_loss": 3.441225463586603, + "tokens_seen": 1926037504 + }, + { + "epoch": 0.17, + "learning_rate": 8.410367517252447e-05, + "loss": 2.5041, + "theoretical_loss": 3.44120579365366, + "tokens_seen": 1926168576 + }, + { + "epoch": 0.17, + "learning_rate": 8.409565077836624e-05, + "loss": 2.4359, + "theoretical_loss": 3.4411861254339238, + "tokens_seen": 1926299648 + }, + { + "epoch": 0.17, + "learning_rate": 8.4087626384208e-05, + "loss": 2.5611, + "theoretical_loss": 3.441166458927127, + "tokens_seen": 1926430720 + }, + { + "epoch": 0.17, + "learning_rate": 8.407960199004975e-05, + "loss": 2.4728, + "theoretical_loss": 3.441146794133005, + "tokens_seen": 1926561792 + }, + { + "epoch": 0.17, + "learning_rate": 8.407157759589151e-05, + "loss": 2.4456, + "theoretical_loss": 3.4411271310512914, + "tokens_seen": 1926692864 + }, + { + "epoch": 0.17, + "learning_rate": 8.406355320173327e-05, + "loss": 2.5713, + "theoretical_loss": 3.441107469681721, + "tokens_seen": 1926823936 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1060815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9482412338256836, + "objective/train/theoretical_loss": 3.441097639638907, + "objective/train/tokens_used": 297348576, + "theoretical_loss": 3.441097639638907, + "tokens_seen": 1926889472 + }, + { + "epoch": 0.17, + "learning_rate": 8.405552880757503e-05, + "loss": 2.6823, + "theoretical_loss": 3.4410878100240287, + "tokens_seen": 1926955008 + }, + { + "epoch": 0.17, + "learning_rate": 8.404750441341679e-05, + "loss": 2.6304, + "theoretical_loss": 3.441068152077948, + "tokens_seen": 1927086080 + }, + { + "epoch": 0.17, + "learning_rate": 8.403948001925855e-05, + "loss": 2.7154, + "theoretical_loss": 3.441048495843215, + "tokens_seen": 1927217152 + }, + { + "epoch": 0.17, + "learning_rate": 8.403145562510031e-05, + "loss": 2.6143, + "theoretical_loss": 3.4410288413195627, + "tokens_seen": 1927348224 + }, + { + "epoch": 0.17, + "learning_rate": 8.402343123094206e-05, + "loss": 2.5897, + "theoretical_loss": 3.4410091885067264, + "tokens_seen": 1927479296 + }, + { + "epoch": 0.17, + "learning_rate": 8.401540683678383e-05, + "loss": 2.7404, + "theoretical_loss": 3.440989537404441, + "tokens_seen": 1927610368 + }, + { + "epoch": 0.17, + "learning_rate": 8.400738244262558e-05, + "loss": 2.4568, + "theoretical_loss": 3.4409698880124413, + "tokens_seen": 1927741440 + }, + { + "epoch": 0.17, + "learning_rate": 8.399935804846735e-05, + "loss": 2.6438, + "theoretical_loss": 3.4409502403304626, + "tokens_seen": 1927872512 + }, + { + "epoch": 0.17, + "learning_rate": 8.39913336543091e-05, + "loss": 2.471, + "theoretical_loss": 3.440930594358239, + "tokens_seen": 1928003584 + }, + { + "epoch": 0.17, + "learning_rate": 8.398330926015087e-05, + "loss": 2.4765, + "theoretical_loss": 3.440910950095506, + "tokens_seen": 1928134656 + }, + { + "epoch": 0.17, + "learning_rate": 8.397528486599262e-05, + "loss": 2.6399, + "theoretical_loss": 3.4408913075419987, + "tokens_seen": 1928265728 + }, + { + "epoch": 0.17, + "learning_rate": 8.396726047183439e-05, + "loss": 2.6799, + "theoretical_loss": 3.440871666697452, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1062086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.882824659347534, + "objective/train/theoretical_loss": 3.4408520275616006, + "objective/train/tokens_used": 298986976, + "theoretical_loss": 3.4408520275616006, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.17, + "learning_rate": 8.395923607767614e-05, + "loss": 2.5886, + "theoretical_loss": 3.4408520275616006, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.17, + "learning_rate": 8.395121168351789e-05, + "loss": 2.5707, + "theoretical_loss": 3.4408323901341813, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.17, + "learning_rate": 8.394318728935966e-05, + "loss": 2.5705, + "theoretical_loss": 3.440812754414927, + "tokens_seen": 1928790016 + }, + { + "epoch": 0.17, + "learning_rate": 8.393516289520141e-05, + "loss": 2.5066, + "theoretical_loss": 3.4407931204035753, + "tokens_seen": 1928921088 + }, + { + "epoch": 0.17, + "learning_rate": 8.392713850104318e-05, + "loss": 2.6055, + "theoretical_loss": 3.440773488099861, + "tokens_seen": 1929052160 + }, + { + "epoch": 0.17, + "learning_rate": 8.391911410688493e-05, + "loss": 2.5717, + "theoretical_loss": 3.440753857503519, + "tokens_seen": 1929183232 + }, + { + "epoch": 0.17, + "learning_rate": 8.39110897127267e-05, + "loss": 2.5541, + "theoretical_loss": 3.440734228614285, + "tokens_seen": 1929314304 + }, + { + "epoch": 0.17, + "learning_rate": 8.390306531856845e-05, + "loss": 2.587, + "theoretical_loss": 3.4407146014318943, + "tokens_seen": 1929445376 + }, + { + "epoch": 0.17, + "learning_rate": 8.38950409244102e-05, + "loss": 2.4675, + "theoretical_loss": 3.4406949759560836, + "tokens_seen": 1929576448 + }, + { + "epoch": 0.17, + "learning_rate": 8.388701653025197e-05, + "loss": 2.6033, + "theoretical_loss": 3.440675352186587, + "tokens_seen": 1929707520 + }, + { + "epoch": 0.17, + "learning_rate": 8.387899213609372e-05, + "loss": 2.6421, + "theoretical_loss": 3.4406557301231424, + "tokens_seen": 1929838592 + }, + { + "epoch": 0.17, + "learning_rate": 8.387096774193549e-05, + "loss": 2.5802, + "theoretical_loss": 3.440636109765484, + "tokens_seen": 1929969664 + }, + { + "epoch": 0.17, + "learning_rate": 8.386294334777724e-05, + "loss": 2.5423, + "theoretical_loss": 3.440616491113348, + "tokens_seen": 1930100736 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1062749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9142069816589355, + "objective/train/theoretical_loss": 3.4406066824267683, + "objective/train/tokens_used": 300625376, + "theoretical_loss": 3.4406066824267683, + "tokens_seen": 1930166272 + }, + { + "epoch": 0.17, + "learning_rate": 8.385491895361901e-05, + "loss": 2.5888, + "theoretical_loss": 3.44059687416647, + "tokens_seen": 1930231808 + }, + { + "epoch": 0.17, + "learning_rate": 8.384689455946076e-05, + "loss": 2.6012, + "theoretical_loss": 3.4405772589245873, + "tokens_seen": 1930362880 + }, + { + "epoch": 0.17, + "learning_rate": 8.383887016530252e-05, + "loss": 2.6279, + "theoretical_loss": 3.440557645387435, + "tokens_seen": 1930493952 + }, + { + "epoch": 0.17, + "learning_rate": 8.383084577114428e-05, + "loss": 2.4806, + "theoretical_loss": 3.4405380335547493, + "tokens_seen": 1930625024 + }, + { + "epoch": 0.17, + "learning_rate": 8.382282137698604e-05, + "loss": 2.426, + "theoretical_loss": 3.4405184234262656, + "tokens_seen": 1930756096 + }, + { + "epoch": 0.17, + "learning_rate": 8.38147969828278e-05, + "loss": 2.6313, + "theoretical_loss": 3.440498815001722, + "tokens_seen": 1930887168 + }, + { + "epoch": 0.17, + "learning_rate": 8.380677258866956e-05, + "loss": 2.521, + "theoretical_loss": 3.440479208280853, + "tokens_seen": 1931018240 + }, + { + "epoch": 0.17, + "learning_rate": 8.379874819451132e-05, + "loss": 2.6727, + "theoretical_loss": 3.4404596032633963, + "tokens_seen": 1931149312 + }, + { + "epoch": 0.17, + "learning_rate": 8.379072380035308e-05, + "loss": 2.4125, + "theoretical_loss": 3.440439999949087, + "tokens_seen": 1931280384 + }, + { + "epoch": 0.17, + "learning_rate": 8.378269940619483e-05, + "loss": 2.5713, + "theoretical_loss": 3.440420398337663, + "tokens_seen": 1931411456 + }, + { + "epoch": 0.17, + "learning_rate": 8.37746750120366e-05, + "loss": 2.5418, + "theoretical_loss": 3.4404007984288594, + "tokens_seen": 1931542528 + }, + { + "epoch": 0.17, + "learning_rate": 8.376665061787835e-05, + "loss": 2.4928, + "theoretical_loss": 3.4403812002224137, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1064305, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5660760402679443, + "objective/train/theoretical_loss": 3.4403616037180624, + "objective/train/tokens_used": 302263776, + "theoretical_loss": 3.4403616037180624, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.17, + "learning_rate": 8.375862622372012e-05, + "loss": 2.5156, + "theoretical_loss": 3.4403616037180624, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.17, + "learning_rate": 8.375060182956187e-05, + "loss": 2.6033, + "theoretical_loss": 3.440342008915542, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 8.374257743540364e-05, + "loss": 2.5074, + "theoretical_loss": 3.440322415814589, + "tokens_seen": 1932066816 + }, + { + "epoch": 0.17, + "learning_rate": 8.373455304124539e-05, + "loss": 2.4945, + "theoretical_loss": 3.4403028244149416, + "tokens_seen": 1932197888 + }, + { + "epoch": 0.17, + "learning_rate": 8.372652864708714e-05, + "loss": 2.5083, + "theoretical_loss": 3.440283234716335, + "tokens_seen": 1932328960 + }, + { + "epoch": 0.17, + "learning_rate": 8.371850425292891e-05, + "loss": 2.6815, + "theoretical_loss": 3.440263646718507, + "tokens_seen": 1932460032 + }, + { + "epoch": 0.17, + "learning_rate": 8.371047985877066e-05, + "loss": 2.4038, + "theoretical_loss": 3.440244060421194, + "tokens_seen": 1932591104 + }, + { + "epoch": 0.17, + "learning_rate": 8.370245546461243e-05, + "loss": 2.7489, + "theoretical_loss": 3.4402244758241336, + "tokens_seen": 1932722176 + }, + { + "epoch": 0.17, + "learning_rate": 8.369443107045418e-05, + "loss": 2.545, + "theoretical_loss": 3.4402048929270634, + "tokens_seen": 1932853248 + }, + { + "epoch": 0.17, + "learning_rate": 8.368640667629595e-05, + "loss": 2.616, + "theoretical_loss": 3.4401853117297194, + "tokens_seen": 1932984320 + }, + { + "epoch": 0.17, + "learning_rate": 8.36783822821377e-05, + "loss": 2.4904, + "theoretical_loss": 3.440165732231839, + "tokens_seen": 1933115392 + }, + { + "epoch": 0.17, + "learning_rate": 8.367035788797947e-05, + "loss": 2.5622, + "theoretical_loss": 3.440146154433161, + "tokens_seen": 1933246464 + }, + { + "epoch": 0.17, + "learning_rate": 8.366233349382122e-05, + "loss": 2.7073, + "theoretical_loss": 3.4401265783334205, + "tokens_seen": 1933377536 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1064860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6494178771972656, + "objective/train/theoretical_loss": 3.4401167909205705, + "objective/train/tokens_used": 303902176, + "theoretical_loss": 3.4401167909205705, + "tokens_seen": 1933443072 + }, + { + "epoch": 0.17, + "learning_rate": 8.365430909966297e-05, + "loss": 2.6403, + "theoretical_loss": 3.440107003932357, + "tokens_seen": 1933508608 + }, + { + "epoch": 0.17, + "learning_rate": 8.364628470550474e-05, + "loss": 2.6015, + "theoretical_loss": 3.4400874312297063, + "tokens_seen": 1933639680 + }, + { + "epoch": 0.17, + "learning_rate": 8.36382603113465e-05, + "loss": 2.5343, + "theoretical_loss": 3.4400678602252066, + "tokens_seen": 1933770752 + }, + { + "epoch": 0.17, + "learning_rate": 8.363023591718826e-05, + "loss": 2.6448, + "theoretical_loss": 3.440048290918596, + "tokens_seen": 1933901824 + }, + { + "epoch": 0.17, + "learning_rate": 8.362221152303001e-05, + "loss": 2.5473, + "theoretical_loss": 3.4400287233096107, + "tokens_seen": 1934032896 + }, + { + "epoch": 0.17, + "learning_rate": 8.361418712887178e-05, + "loss": 2.6655, + "theoretical_loss": 3.4400091573979896, + "tokens_seen": 1934163968 + }, + { + "epoch": 0.17, + "learning_rate": 8.360616273471353e-05, + "loss": 2.5688, + "theoretical_loss": 3.439989593183471, + "tokens_seen": 1934295040 + }, + { + "epoch": 0.17, + "learning_rate": 8.359813834055529e-05, + "loss": 2.6169, + "theoretical_loss": 3.439970030665791, + "tokens_seen": 1934426112 + }, + { + "epoch": 0.17, + "learning_rate": 8.359011394639705e-05, + "loss": 2.6115, + "theoretical_loss": 3.4399504698446886, + "tokens_seen": 1934557184 + }, + { + "epoch": 0.17, + "learning_rate": 8.358208955223881e-05, + "loss": 2.4359, + "theoretical_loss": 3.4399309107199016, + "tokens_seen": 1934688256 + }, + { + "epoch": 0.17, + "learning_rate": 8.357406515808057e-05, + "loss": 2.5228, + "theoretical_loss": 3.439911353291168, + "tokens_seen": 1934819328 + }, + { + "epoch": 0.17, + "learning_rate": 8.356604076392233e-05, + "loss": 2.558, + "theoretical_loss": 3.439891797558225, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1065725, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4864485263824463, + "objective/train/theoretical_loss": 3.439872243520812, + "objective/train/tokens_used": 305540576, + "theoretical_loss": 3.439872243520812, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.17, + "learning_rate": 8.35580163697641e-05, + "loss": 2.5461, + "theoretical_loss": 3.439872243520812, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.17, + "learning_rate": 8.354999197560585e-05, + "loss": 2.4294, + "theoretical_loss": 3.4398526911786664, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 8.35419675814476e-05, + "loss": 2.48, + "theoretical_loss": 3.439833140531527, + "tokens_seen": 1935343616 + }, + { + "epoch": 0.17, + "learning_rate": 8.353394318728937e-05, + "loss": 2.5107, + "theoretical_loss": 3.4398135915791315, + "tokens_seen": 1935474688 + }, + { + "epoch": 0.17, + "learning_rate": 8.352591879313112e-05, + "loss": 2.4663, + "theoretical_loss": 3.439794044321218, + "tokens_seen": 1935605760 + }, + { + "epoch": 0.17, + "learning_rate": 8.351789439897289e-05, + "loss": 2.6062, + "theoretical_loss": 3.439774498757526, + "tokens_seen": 1935736832 + }, + { + "epoch": 0.17, + "learning_rate": 8.350987000481464e-05, + "loss": 2.6143, + "theoretical_loss": 3.4397549548877926, + "tokens_seen": 1935867904 + }, + { + "epoch": 0.17, + "learning_rate": 8.35018456106564e-05, + "loss": 2.699, + "theoretical_loss": 3.439735412711757, + "tokens_seen": 1935998976 + }, + { + "epoch": 0.17, + "learning_rate": 8.349382121649816e-05, + "loss": 2.5616, + "theoretical_loss": 3.4397158722291583, + "tokens_seen": 1936130048 + }, + { + "epoch": 0.17, + "learning_rate": 8.348579682233991e-05, + "loss": 2.5023, + "theoretical_loss": 3.439696333439734, + "tokens_seen": 1936261120 + }, + { + "epoch": 0.17, + "learning_rate": 8.347777242818168e-05, + "loss": 2.5506, + "theoretical_loss": 3.4396767963432238, + "tokens_seen": 1936392192 + }, + { + "epoch": 0.17, + "learning_rate": 8.346974803402343e-05, + "loss": 2.5643, + "theoretical_loss": 3.4396572609393656, + "tokens_seen": 1936523264 + }, + { + "epoch": 0.17, + "learning_rate": 8.34617236398652e-05, + "loss": 2.4963, + "theoretical_loss": 3.439637727227899, + "tokens_seen": 1936654336 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1066923, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5231666564941406, + "objective/train/theoretical_loss": 3.439627961006731, + "objective/train/tokens_used": 307178976, + "theoretical_loss": 3.439627961006731, + "tokens_seen": 1936719872 + }, + { + "epoch": 0.17, + "learning_rate": 8.345369924570695e-05, + "loss": 2.5335, + "theoretical_loss": 3.4396181952085625, + "tokens_seen": 1936785408 + }, + { + "epoch": 0.17, + "learning_rate": 8.344567485154872e-05, + "loss": 2.4004, + "theoretical_loss": 3.4395986648810943, + "tokens_seen": 1936916480 + }, + { + "epoch": 0.17, + "learning_rate": 8.343765045739047e-05, + "loss": 2.6551, + "theoretical_loss": 3.439579136245235, + "tokens_seen": 1937047552 + }, + { + "epoch": 0.17, + "learning_rate": 8.342962606323223e-05, + "loss": 2.5923, + "theoretical_loss": 3.439559609300722, + "tokens_seen": 1937178624 + }, + { + "epoch": 0.17, + "learning_rate": 8.342160166907399e-05, + "loss": 2.5727, + "theoretical_loss": 3.4395400840472954, + "tokens_seen": 1937309696 + }, + { + "epoch": 0.17, + "learning_rate": 8.341357727491575e-05, + "loss": 2.6667, + "theoretical_loss": 3.439520560484694, + "tokens_seen": 1937440768 + }, + { + "epoch": 0.17, + "learning_rate": 8.340555288075751e-05, + "loss": 2.6872, + "theoretical_loss": 3.439501038612657, + "tokens_seen": 1937571840 + }, + { + "epoch": 0.17, + "learning_rate": 8.339752848659926e-05, + "loss": 2.6223, + "theoretical_loss": 3.4394815184309238, + "tokens_seen": 1937702912 + }, + { + "epoch": 0.17, + "learning_rate": 8.338950409244103e-05, + "loss": 2.5338, + "theoretical_loss": 3.4394619999392333, + "tokens_seen": 1937833984 + }, + { + "epoch": 0.17, + "learning_rate": 8.338147969828278e-05, + "loss": 2.6202, + "theoretical_loss": 3.4394424831373254, + "tokens_seen": 1937965056 + }, + { + "epoch": 0.17, + "learning_rate": 8.337345530412455e-05, + "loss": 2.556, + "theoretical_loss": 3.439422968024939, + "tokens_seen": 1938096128 + }, + { + "epoch": 0.17, + "learning_rate": 8.33654309099663e-05, + "loss": 2.4879, + "theoretical_loss": 3.4394034546018144, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1067414, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6843504905700684, + "objective/train/theoretical_loss": 3.439383942867691, + "objective/train/tokens_used": 308817376, + "theoretical_loss": 3.439383942867691, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 8.335740651580806e-05, + "loss": 2.6718, + "theoretical_loss": 3.439383942867691, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 8.334938212164982e-05, + "loss": 2.4463, + "theoretical_loss": 3.439364432822307, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 8.334135772749158e-05, + "loss": 2.5893, + "theoretical_loss": 3.439344924465404, + "tokens_seen": 1938620416 + }, + { + "epoch": 0.18, + "learning_rate": 8.333333333333334e-05, + "loss": 2.442, + "theoretical_loss": 3.439325417796721, + "tokens_seen": 1938751488 + }, + { + "epoch": 0.18, + "learning_rate": 8.33253089391751e-05, + "loss": 2.5347, + "theoretical_loss": 3.439305912815997, + "tokens_seen": 1938882560 + }, + { + "epoch": 0.18, + "learning_rate": 8.331728454501686e-05, + "loss": 2.6075, + "theoretical_loss": 3.439286409522973, + "tokens_seen": 1939013632 + }, + { + "epoch": 0.18, + "learning_rate": 8.330926015085862e-05, + "loss": 2.453, + "theoretical_loss": 3.4392669079173883, + "tokens_seen": 1939144704 + }, + { + "epoch": 0.18, + "learning_rate": 8.330123575670037e-05, + "loss": 2.5933, + "theoretical_loss": 3.439247407998983, + "tokens_seen": 1939275776 + }, + { + "epoch": 0.18, + "learning_rate": 8.329321136254214e-05, + "loss": 2.4927, + "theoretical_loss": 3.439227909767497, + "tokens_seen": 1939406848 + }, + { + "epoch": 0.18, + "learning_rate": 8.328518696838389e-05, + "loss": 2.5154, + "theoretical_loss": 3.439208413222671, + "tokens_seen": 1939537920 + }, + { + "epoch": 0.18, + "learning_rate": 8.327716257422566e-05, + "loss": 2.6075, + "theoretical_loss": 3.439188918364244, + "tokens_seen": 1939668992 + }, + { + "epoch": 0.18, + "learning_rate": 8.326913818006741e-05, + "loss": 2.5968, + "theoretical_loss": 3.439169425191957, + "tokens_seen": 1939800064 + }, + { + "epoch": 0.18, + "learning_rate": 8.326111378590918e-05, + "loss": 2.4055, + "theoretical_loss": 3.4391499337055498, + "tokens_seen": 1939931136 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1068508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.342050313949585, + "objective/train/theoretical_loss": 3.4391401885944703, + "objective/train/tokens_used": 310455776, + "theoretical_loss": 3.4391401885944703, + "tokens_seen": 1939996672 + }, + { + "epoch": 0.18, + "learning_rate": 8.325308939175093e-05, + "loss": 2.4388, + "theoretical_loss": 3.439130443904763, + "tokens_seen": 1940062208 + }, + { + "epoch": 0.18, + "learning_rate": 8.324506499759268e-05, + "loss": 2.5146, + "theoretical_loss": 3.439110955789338, + "tokens_seen": 1940193280 + }, + { + "epoch": 0.18, + "learning_rate": 8.323704060343445e-05, + "loss": 2.6226, + "theoretical_loss": 3.4390914693590124, + "tokens_seen": 1940324352 + }, + { + "epoch": 0.18, + "learning_rate": 8.32290162092762e-05, + "loss": 2.5135, + "theoretical_loss": 3.4390719846135296, + "tokens_seen": 1940455424 + }, + { + "epoch": 0.18, + "learning_rate": 8.322099181511797e-05, + "loss": 2.5178, + "theoretical_loss": 3.439052501552628, + "tokens_seen": 1940586496 + }, + { + "epoch": 0.18, + "learning_rate": 8.321296742095972e-05, + "loss": 2.5175, + "theoretical_loss": 3.4390330201760495, + "tokens_seen": 1940717568 + }, + { + "epoch": 0.18, + "learning_rate": 8.320494302680149e-05, + "loss": 2.4956, + "theoretical_loss": 3.439013540483535, + "tokens_seen": 1940848640 + }, + { + "epoch": 0.18, + "learning_rate": 8.319691863264324e-05, + "loss": 2.4474, + "theoretical_loss": 3.4389940624748236, + "tokens_seen": 1940979712 + }, + { + "epoch": 0.18, + "learning_rate": 8.3188894238485e-05, + "loss": 2.4981, + "theoretical_loss": 3.4389745861496577, + "tokens_seen": 1941110784 + }, + { + "epoch": 0.18, + "learning_rate": 8.318086984432676e-05, + "loss": 2.4449, + "theoretical_loss": 3.438955111507777, + "tokens_seen": 1941241856 + }, + { + "epoch": 0.18, + "learning_rate": 8.317284545016852e-05, + "loss": 2.5271, + "theoretical_loss": 3.4389356385489234, + "tokens_seen": 1941372928 + }, + { + "epoch": 0.18, + "learning_rate": 8.316482105601028e-05, + "loss": 2.5505, + "theoretical_loss": 3.4389161672728368, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1069184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.391522169113159, + "objective/train/theoretical_loss": 3.438896697679259, + "objective/train/tokens_used": 312094176, + "theoretical_loss": 3.438896697679259, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.18, + "learning_rate": 8.315679666185203e-05, + "loss": 2.5596, + "theoretical_loss": 3.438896697679259, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.18, + "learning_rate": 8.31487722676938e-05, + "loss": 2.4975, + "theoretical_loss": 3.4388772297679306, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.18, + "learning_rate": 8.314074787353555e-05, + "loss": 2.5002, + "theoretical_loss": 3.438857763538593, + "tokens_seen": 1941897216 + }, + { + "epoch": 0.18, + "learning_rate": 8.313272347937731e-05, + "loss": 2.5132, + "theoretical_loss": 3.4388382989909863, + "tokens_seen": 1942028288 + }, + { + "epoch": 0.18, + "learning_rate": 8.312469908521907e-05, + "loss": 2.5543, + "theoretical_loss": 3.4388188361248533, + "tokens_seen": 1942159360 + }, + { + "epoch": 0.18, + "learning_rate": 8.311667469106083e-05, + "loss": 2.381, + "theoretical_loss": 3.4387993749399346, + "tokens_seen": 1942290432 + }, + { + "epoch": 0.18, + "learning_rate": 8.31086502969026e-05, + "loss": 2.3192, + "theoretical_loss": 3.4387799154359717, + "tokens_seen": 1942421504 + }, + { + "epoch": 0.18, + "learning_rate": 8.310062590274435e-05, + "loss": 2.5764, + "theoretical_loss": 3.4387604576127053, + "tokens_seen": 1942552576 + }, + { + "epoch": 0.18, + "learning_rate": 8.309260150858611e-05, + "loss": 2.3756, + "theoretical_loss": 3.438741001469878, + "tokens_seen": 1942683648 + }, + { + "epoch": 0.18, + "learning_rate": 8.308457711442787e-05, + "loss": 2.4454, + "theoretical_loss": 3.43872154700723, + "tokens_seen": 1942814720 + }, + { + "epoch": 0.18, + "learning_rate": 8.307655272026962e-05, + "loss": 2.5725, + "theoretical_loss": 3.438702094224504, + "tokens_seen": 1942945792 + }, + { + "epoch": 0.18, + "learning_rate": 8.306852832611139e-05, + "loss": 2.4494, + "theoretical_loss": 3.438682643121441, + "tokens_seen": 1943076864 + }, + { + "epoch": 0.18, + "learning_rate": 8.306050393195314e-05, + "loss": 2.3641, + "theoretical_loss": 3.4386631936977827, + "tokens_seen": 1943207936 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1070317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5918591022491455, + "objective/train/theoretical_loss": 3.43865346961565, + "objective/train/tokens_used": 313732576, + "theoretical_loss": 3.43865346961565, + "tokens_seen": 1943273472 + }, + { + "epoch": 0.18, + "learning_rate": 8.305247953779491e-05, + "loss": 2.6276, + "theoretical_loss": 3.4386437459532715, + "tokens_seen": 1943339008 + }, + { + "epoch": 0.18, + "learning_rate": 8.304445514363666e-05, + "loss": 2.5815, + "theoretical_loss": 3.438624299887648, + "tokens_seen": 1943470080 + }, + { + "epoch": 0.18, + "learning_rate": 8.303643074947843e-05, + "loss": 2.3985, + "theoretical_loss": 3.438604855500655, + "tokens_seen": 1943601152 + }, + { + "epoch": 0.18, + "learning_rate": 8.302840635532018e-05, + "loss": 2.5067, + "theoretical_loss": 3.438585412792034, + "tokens_seen": 1943732224 + }, + { + "epoch": 0.18, + "learning_rate": 8.302038196116195e-05, + "loss": 2.605, + "theoretical_loss": 3.438565971761528, + "tokens_seen": 1943863296 + }, + { + "epoch": 0.18, + "learning_rate": 8.30123575670037e-05, + "loss": 2.4195, + "theoretical_loss": 3.4385465324088766, + "tokens_seen": 1943994368 + }, + { + "epoch": 0.18, + "learning_rate": 8.300433317284545e-05, + "loss": 2.5353, + "theoretical_loss": 3.4385270947338245, + "tokens_seen": 1944125440 + }, + { + "epoch": 0.18, + "learning_rate": 8.299630877868722e-05, + "loss": 2.5632, + "theoretical_loss": 3.4385076587361123, + "tokens_seen": 1944256512 + }, + { + "epoch": 0.18, + "learning_rate": 8.298828438452897e-05, + "loss": 2.4589, + "theoretical_loss": 3.438488224415483, + "tokens_seen": 1944387584 + }, + { + "epoch": 0.18, + "learning_rate": 8.298025999037074e-05, + "loss": 2.4504, + "theoretical_loss": 3.438468791771678, + "tokens_seen": 1944518656 + }, + { + "epoch": 0.18, + "learning_rate": 8.297223559621249e-05, + "loss": 2.7096, + "theoretical_loss": 3.4384493608044404, + "tokens_seen": 1944649728 + }, + { + "epoch": 0.18, + "learning_rate": 8.296421120205426e-05, + "loss": 2.494, + "theoretical_loss": 3.4384299315135123, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1070782, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.948735237121582, + "objective/train/theoretical_loss": 3.438410503898636, + "objective/train/tokens_used": 315370976, + "theoretical_loss": 3.438410503898636, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.18, + "learning_rate": 8.295618680789601e-05, + "loss": 2.5633, + "theoretical_loss": 3.438410503898636, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.18, + "learning_rate": 8.294816241373777e-05, + "loss": 2.4825, + "theoretical_loss": 3.4383910779595537, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.18, + "learning_rate": 8.294013801957953e-05, + "loss": 2.593, + "theoretical_loss": 3.438371653696009, + "tokens_seen": 1945174016 + }, + { + "epoch": 0.18, + "learning_rate": 8.293211362542129e-05, + "loss": 2.4795, + "theoretical_loss": 3.4383522311077432, + "tokens_seen": 1945305088 + }, + { + "epoch": 0.18, + "learning_rate": 8.292408923126305e-05, + "loss": 2.6767, + "theoretical_loss": 3.4383328101945, + "tokens_seen": 1945436160 + }, + { + "epoch": 0.18, + "learning_rate": 8.29160648371048e-05, + "loss": 2.5267, + "theoretical_loss": 3.4383133909560213, + "tokens_seen": 1945567232 + }, + { + "epoch": 0.18, + "learning_rate": 8.290804044294657e-05, + "loss": 2.5432, + "theoretical_loss": 3.4382939733920503, + "tokens_seen": 1945698304 + }, + { + "epoch": 0.18, + "learning_rate": 8.290001604878832e-05, + "loss": 2.5006, + "theoretical_loss": 3.4382745575023295, + "tokens_seen": 1945829376 + }, + { + "epoch": 0.18, + "learning_rate": 8.289199165463008e-05, + "loss": 2.5887, + "theoretical_loss": 3.4382551432866024, + "tokens_seen": 1945960448 + }, + { + "epoch": 0.18, + "learning_rate": 8.288396726047184e-05, + "loss": 2.4243, + "theoretical_loss": 3.438235730744611, + "tokens_seen": 1946091520 + }, + { + "epoch": 0.18, + "learning_rate": 8.28759428663136e-05, + "loss": 2.5427, + "theoretical_loss": 3.4382163198760995, + "tokens_seen": 1946222592 + }, + { + "epoch": 0.18, + "learning_rate": 8.286791847215536e-05, + "loss": 2.6092, + "theoretical_loss": 3.43819691068081, + "tokens_seen": 1946353664 + }, + { + "epoch": 0.18, + "learning_rate": 8.285989407799712e-05, + "loss": 2.4251, + "theoretical_loss": 3.4381775031584856, + "tokens_seen": 1946484736 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1072031, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5232903957366943, + "objective/train/theoretical_loss": 3.4381678000246056, + "objective/train/tokens_used": 317009376, + "theoretical_loss": 3.4381678000246056, + "tokens_seen": 1946550272 + }, + { + "epoch": 0.18, + "learning_rate": 8.285186968383888e-05, + "loss": 2.6291, + "theoretical_loss": 3.43815809730887, + "tokens_seen": 1946615808 + }, + { + "epoch": 0.18, + "learning_rate": 8.284384528968064e-05, + "loss": 2.5281, + "theoretical_loss": 3.438138693131706, + "tokens_seen": 1946746880 + }, + { + "epoch": 0.18, + "learning_rate": 8.283582089552239e-05, + "loss": 2.4736, + "theoretical_loss": 3.4381192906267373, + "tokens_seen": 1946877952 + }, + { + "epoch": 0.18, + "learning_rate": 8.282779650136416e-05, + "loss": 2.538, + "theoretical_loss": 3.4380998897937065, + "tokens_seen": 1947009024 + }, + { + "epoch": 0.18, + "learning_rate": 8.281977210720591e-05, + "loss": 2.4369, + "theoretical_loss": 3.438080490632358, + "tokens_seen": 1947140096 + }, + { + "epoch": 0.18, + "learning_rate": 8.281174771304768e-05, + "loss": 2.7668, + "theoretical_loss": 3.4380610931424345, + "tokens_seen": 1947271168 + }, + { + "epoch": 0.18, + "learning_rate": 8.280372331888943e-05, + "loss": 2.5402, + "theoretical_loss": 3.438041697323679, + "tokens_seen": 1947402240 + }, + { + "epoch": 0.18, + "learning_rate": 8.27956989247312e-05, + "loss": 2.499, + "theoretical_loss": 3.4380223031758366, + "tokens_seen": 1947533312 + }, + { + "epoch": 0.18, + "learning_rate": 8.278767453057295e-05, + "loss": 2.3226, + "theoretical_loss": 3.43800291069865, + "tokens_seen": 1947664384 + }, + { + "epoch": 0.18, + "learning_rate": 8.27796501364147e-05, + "loss": 2.5558, + "theoretical_loss": 3.4379835198918625, + "tokens_seen": 1947795456 + }, + { + "epoch": 0.18, + "learning_rate": 8.277162574225647e-05, + "loss": 2.5019, + "theoretical_loss": 3.4379641307552182, + "tokens_seen": 1947926528 + }, + { + "epoch": 0.18, + "learning_rate": 8.276360134809822e-05, + "loss": 2.51, + "theoretical_loss": 3.4379447432884613, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1072626, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2674527168273926, + "objective/train/theoretical_loss": 3.437925357491335, + "objective/train/tokens_used": 318647776, + "theoretical_loss": 3.437925357491335, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.18, + "learning_rate": 8.275557695393999e-05, + "loss": 2.5014, + "theoretical_loss": 3.437925357491335, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.18, + "learning_rate": 8.274755255978174e-05, + "loss": 2.5272, + "theoretical_loss": 3.437905973363584, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.18, + "learning_rate": 8.273952816562351e-05, + "loss": 2.659, + "theoretical_loss": 3.4378865909049514, + "tokens_seen": 1948450816 + }, + { + "epoch": 0.18, + "learning_rate": 8.273150377146526e-05, + "loss": 2.5092, + "theoretical_loss": 3.437867210115181, + "tokens_seen": 1948581888 + }, + { + "epoch": 0.18, + "learning_rate": 8.272347937730703e-05, + "loss": 2.5668, + "theoretical_loss": 3.437847830994018, + "tokens_seen": 1948712960 + }, + { + "epoch": 0.18, + "learning_rate": 8.271545498314878e-05, + "loss": 2.5462, + "theoretical_loss": 3.437828453541206, + "tokens_seen": 1948844032 + }, + { + "epoch": 0.18, + "learning_rate": 8.270743058899054e-05, + "loss": 2.7522, + "theoretical_loss": 3.437809077756489, + "tokens_seen": 1948975104 + }, + { + "epoch": 0.18, + "learning_rate": 8.26994061948323e-05, + "loss": 2.6027, + "theoretical_loss": 3.437789703639611, + "tokens_seen": 1949106176 + }, + { + "epoch": 0.18, + "learning_rate": 8.269138180067406e-05, + "loss": 2.5426, + "theoretical_loss": 3.4377703311903174, + "tokens_seen": 1949237248 + }, + { + "epoch": 0.18, + "learning_rate": 8.268335740651582e-05, + "loss": 2.6411, + "theoretical_loss": 3.437750960408351, + "tokens_seen": 1949368320 + }, + { + "epoch": 0.18, + "learning_rate": 8.267533301235758e-05, + "loss": 2.4644, + "theoretical_loss": 3.437731591293457, + "tokens_seen": 1949499392 + }, + { + "epoch": 0.18, + "learning_rate": 8.266730861819934e-05, + "loss": 2.5176, + "theoretical_loss": 3.4377122238453803, + "tokens_seen": 1949630464 + }, + { + "epoch": 0.18, + "learning_rate": 8.26592842240411e-05, + "loss": 2.6194, + "theoretical_loss": 3.4376928580638646, + "tokens_seen": 1949761536 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1073976, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5130059719085693, + "objective/train/theoretical_loss": 3.4376831757979875, + "objective/train/tokens_used": 320286176, + "theoretical_loss": 3.4376831757979875, + "tokens_seen": 1949827072 + }, + { + "epoch": 0.18, + "learning_rate": 8.265125982988285e-05, + "loss": 2.536, + "theoretical_loss": 3.437673493948655, + "tokens_seen": 1949892608 + }, + { + "epoch": 0.18, + "learning_rate": 8.264323543572461e-05, + "loss": 2.6313, + "theoretical_loss": 3.437654131499496, + "tokens_seen": 1950023680 + }, + { + "epoch": 0.18, + "learning_rate": 8.263521104156637e-05, + "loss": 2.5475, + "theoretical_loss": 3.437634770716132, + "tokens_seen": 1950154752 + }, + { + "epoch": 0.18, + "learning_rate": 8.262718664740813e-05, + "loss": 2.6455, + "theoretical_loss": 3.4376154115983084, + "tokens_seen": 1950285824 + }, + { + "epoch": 0.18, + "learning_rate": 8.261916225324989e-05, + "loss": 2.3373, + "theoretical_loss": 3.4375960541457697, + "tokens_seen": 1950416896 + }, + { + "epoch": 0.18, + "learning_rate": 8.261113785909164e-05, + "loss": 2.4703, + "theoretical_loss": 3.43757669835826, + "tokens_seen": 1950547968 + }, + { + "epoch": 0.18, + "learning_rate": 8.260311346493341e-05, + "loss": 2.5127, + "theoretical_loss": 3.4375573442355254, + "tokens_seen": 1950679040 + }, + { + "epoch": 0.18, + "learning_rate": 8.259508907077516e-05, + "loss": 2.3909, + "theoretical_loss": 3.43753799177731, + "tokens_seen": 1950810112 + }, + { + "epoch": 0.18, + "learning_rate": 8.258706467661693e-05, + "loss": 2.4477, + "theoretical_loss": 3.437518640983359, + "tokens_seen": 1950941184 + }, + { + "epoch": 0.18, + "learning_rate": 8.257904028245868e-05, + "loss": 2.3993, + "theoretical_loss": 3.4374992918534177, + "tokens_seen": 1951072256 + }, + { + "epoch": 0.18, + "learning_rate": 8.257101588830045e-05, + "loss": 2.5379, + "theoretical_loss": 3.4374799443872317, + "tokens_seen": 1951203328 + }, + { + "epoch": 0.18, + "learning_rate": 8.25629914941422e-05, + "loss": 2.5257, + "theoretical_loss": 3.4374605985845452, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1074351, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568186044692993, + "objective/train/theoretical_loss": 3.437441254445104, + "objective/train/tokens_used": 321924576, + "theoretical_loss": 3.437441254445104, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 8.255496709998395e-05, + "loss": 2.6717, + "theoretical_loss": 3.437441254445104, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 8.254694270582572e-05, + "loss": 2.6209, + "theoretical_loss": 3.437421911968653, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 8.253891831166747e-05, + "loss": 2.4163, + "theoretical_loss": 3.4374025711549385, + "tokens_seen": 1951727616 + }, + { + "epoch": 0.18, + "learning_rate": 8.253089391750924e-05, + "loss": 2.5627, + "theoretical_loss": 3.437383232003705, + "tokens_seen": 1951858688 + }, + { + "epoch": 0.18, + "learning_rate": 8.252286952335099e-05, + "loss": 2.4633, + "theoretical_loss": 3.4373638945146983, + "tokens_seen": 1951989760 + }, + { + "epoch": 0.18, + "learning_rate": 8.251484512919275e-05, + "loss": 2.5232, + "theoretical_loss": 3.4373445586876636, + "tokens_seen": 1952120832 + }, + { + "epoch": 0.18, + "learning_rate": 8.250682073503451e-05, + "loss": 2.5546, + "theoretical_loss": 3.4373252245223473, + "tokens_seen": 1952251904 + }, + { + "epoch": 0.18, + "learning_rate": 8.249879634087627e-05, + "loss": 2.5288, + "theoretical_loss": 3.437305892018494, + "tokens_seen": 1952382976 + }, + { + "epoch": 0.18, + "learning_rate": 8.249077194671803e-05, + "loss": 2.6947, + "theoretical_loss": 3.43728656117585, + "tokens_seen": 1952514048 + }, + { + "epoch": 0.18, + "learning_rate": 8.248274755255979e-05, + "loss": 2.6388, + "theoretical_loss": 3.437267231994161, + "tokens_seen": 1952645120 + }, + { + "epoch": 0.18, + "learning_rate": 8.247472315840154e-05, + "loss": 2.5857, + "theoretical_loss": 3.437247904473173, + "tokens_seen": 1952776192 + }, + { + "epoch": 0.18, + "learning_rate": 8.24666987642433e-05, + "loss": 2.5235, + "theoretical_loss": 3.4372285786126318, + "tokens_seen": 1952907264 + }, + { + "epoch": 0.18, + "learning_rate": 8.245867437008506e-05, + "loss": 2.5955, + "theoretical_loss": 3.4372092544122825, + "tokens_seen": 1953038336 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1075675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2708775997161865, + "objective/train/theoretical_loss": 3.4371995929346006, + "objective/train/tokens_used": 323562976, + "theoretical_loss": 3.4371995929346006, + "tokens_seen": 1953103872 + }, + { + "epoch": 0.18, + "learning_rate": 8.245064997592683e-05, + "loss": 2.5938, + "theoretical_loss": 3.437189931871872, + "tokens_seen": 1953169408 + }, + { + "epoch": 0.18, + "learning_rate": 8.244262558176858e-05, + "loss": 2.6016, + "theoretical_loss": 3.437170610991146, + "tokens_seen": 1953300480 + }, + { + "epoch": 0.18, + "learning_rate": 8.243460118761035e-05, + "loss": 2.3854, + "theoretical_loss": 3.437151291769851, + "tokens_seen": 1953431552 + }, + { + "epoch": 0.18, + "learning_rate": 8.24265767934521e-05, + "loss": 2.59, + "theoretical_loss": 3.4371319742077326, + "tokens_seen": 1953562624 + }, + { + "epoch": 0.18, + "learning_rate": 8.241855239929385e-05, + "loss": 2.4944, + "theoretical_loss": 3.437112658304537, + "tokens_seen": 1953693696 + }, + { + "epoch": 0.18, + "learning_rate": 8.241052800513562e-05, + "loss": 2.6604, + "theoretical_loss": 3.437093344060011, + "tokens_seen": 1953824768 + }, + { + "epoch": 0.18, + "learning_rate": 8.240250361097737e-05, + "loss": 2.4924, + "theoretical_loss": 3.437074031473901, + "tokens_seen": 1953955840 + }, + { + "epoch": 0.18, + "learning_rate": 8.239447921681914e-05, + "loss": 2.5443, + "theoretical_loss": 3.4370547205459516, + "tokens_seen": 1954086912 + }, + { + "epoch": 0.18, + "learning_rate": 8.238645482266089e-05, + "loss": 2.5714, + "theoretical_loss": 3.4370354112759114, + "tokens_seen": 1954217984 + }, + { + "epoch": 0.18, + "learning_rate": 8.237843042850264e-05, + "loss": 2.5103, + "theoretical_loss": 3.4370161036635256, + "tokens_seen": 1954349056 + }, + { + "epoch": 0.18, + "learning_rate": 8.237040603434441e-05, + "loss": 2.5072, + "theoretical_loss": 3.4369967977085416, + "tokens_seen": 1954480128 + }, + { + "epoch": 0.18, + "learning_rate": 8.236238164018616e-05, + "loss": 2.4739, + "theoretical_loss": 3.436977493410706, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1076263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2574844360351562, + "objective/train/theoretical_loss": 3.436958190769764, + "objective/train/tokens_used": 325201376, + "theoretical_loss": 3.436958190769764, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.18, + "learning_rate": 8.235435724602793e-05, + "loss": 2.5272, + "theoretical_loss": 3.436958190769764, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.18, + "learning_rate": 8.234633285186968e-05, + "loss": 2.5859, + "theoretical_loss": 3.4369388897854636, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 8.233830845771145e-05, + "loss": 2.4542, + "theoretical_loss": 3.436919590457552, + "tokens_seen": 1955004416 + }, + { + "epoch": 0.18, + "learning_rate": 8.23302840635532e-05, + "loss": 2.6658, + "theoretical_loss": 3.4369002927857744, + "tokens_seen": 1955135488 + }, + { + "epoch": 0.19, + "learning_rate": 8.232225966939496e-05, + "loss": 2.5128, + "theoretical_loss": 3.436880996769879, + "tokens_seen": 1955266560 + }, + { + "epoch": 0.19, + "learning_rate": 8.231423527523672e-05, + "loss": 2.4474, + "theoretical_loss": 3.436861702409612, + "tokens_seen": 1955397632 + }, + { + "epoch": 0.19, + "learning_rate": 8.230621088107848e-05, + "loss": 2.6982, + "theoretical_loss": 3.4368424097047208, + "tokens_seen": 1955528704 + }, + { + "epoch": 0.19, + "learning_rate": 8.229818648692024e-05, + "loss": 2.4142, + "theoretical_loss": 3.4368231186549525, + "tokens_seen": 1955659776 + }, + { + "epoch": 0.19, + "learning_rate": 8.2290162092762e-05, + "loss": 2.4942, + "theoretical_loss": 3.436803829260054, + "tokens_seen": 1955790848 + }, + { + "epoch": 0.19, + "learning_rate": 8.228213769860375e-05, + "loss": 2.5114, + "theoretical_loss": 3.4367845415197715, + "tokens_seen": 1955921920 + }, + { + "epoch": 0.19, + "learning_rate": 8.227411330444552e-05, + "loss": 2.4439, + "theoretical_loss": 3.436765255433854, + "tokens_seen": 1956052992 + }, + { + "epoch": 0.19, + "learning_rate": 8.226608891028727e-05, + "loss": 2.5427, + "theoretical_loss": 3.436745971002048, + "tokens_seen": 1956184064 + }, + { + "epoch": 0.19, + "learning_rate": 8.225806451612904e-05, + "loss": 2.5254, + "theoretical_loss": 3.4367266882241, + "tokens_seen": 1956315136 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1077404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.399874210357666, + "objective/train/theoretical_loss": 3.436717047455245, + "objective/train/tokens_used": 326839776, + "theoretical_loss": 3.436717047455245, + "tokens_seen": 1956380672 + }, + { + "epoch": 0.19, + "learning_rate": 8.225004012197079e-05, + "loss": 2.5416, + "theoretical_loss": 3.436707407099759, + "tokens_seen": 1956446208 + }, + { + "epoch": 0.19, + "learning_rate": 8.224201572781256e-05, + "loss": 2.7036, + "theoretical_loss": 3.436688127628771, + "tokens_seen": 1956577280 + }, + { + "epoch": 0.19, + "learning_rate": 8.223399133365431e-05, + "loss": 2.6167, + "theoretical_loss": 3.436668849810884, + "tokens_seen": 1956708352 + }, + { + "epoch": 0.19, + "learning_rate": 8.222596693949606e-05, + "loss": 2.5307, + "theoretical_loss": 3.4366495736458456, + "tokens_seen": 1956839424 + }, + { + "epoch": 0.19, + "learning_rate": 8.221794254533783e-05, + "loss": 2.6293, + "theoretical_loss": 3.436630299133403, + "tokens_seen": 1956970496 + }, + { + "epoch": 0.19, + "learning_rate": 8.220991815117958e-05, + "loss": 2.5141, + "theoretical_loss": 3.4366110262733045, + "tokens_seen": 1957101568 + }, + { + "epoch": 0.19, + "learning_rate": 8.220189375702135e-05, + "loss": 2.4892, + "theoretical_loss": 3.4365917550652973, + "tokens_seen": 1957232640 + }, + { + "epoch": 0.19, + "learning_rate": 8.21938693628631e-05, + "loss": 2.6137, + "theoretical_loss": 3.4365724855091293, + "tokens_seen": 1957363712 + }, + { + "epoch": 0.19, + "learning_rate": 8.218584496870485e-05, + "loss": 2.4596, + "theoretical_loss": 3.4365532176045486, + "tokens_seen": 1957494784 + }, + { + "epoch": 0.19, + "learning_rate": 8.217782057454662e-05, + "loss": 2.5889, + "theoretical_loss": 3.4365339513513025, + "tokens_seen": 1957625856 + }, + { + "epoch": 0.19, + "learning_rate": 8.216979618038837e-05, + "loss": 2.5195, + "theoretical_loss": 3.436514686749139, + "tokens_seen": 1957756928 + }, + { + "epoch": 0.19, + "learning_rate": 8.216177178623014e-05, + "loss": 2.48, + "theoretical_loss": 3.4364954237978065, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1077881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7305192947387695, + "objective/train/theoretical_loss": 3.436476162497053, + "objective/train/tokens_used": 328478176, + "theoretical_loss": 3.436476162497053, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.19, + "learning_rate": 8.21537473920719e-05, + "loss": 2.5058, + "theoretical_loss": 3.436476162497053, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.19, + "learning_rate": 8.214572299791366e-05, + "loss": 2.514, + "theoretical_loss": 3.4364569028466265, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.19, + "learning_rate": 8.213769860375541e-05, + "loss": 2.4948, + "theoretical_loss": 3.436437644846275, + "tokens_seen": 1958281216 + }, + { + "epoch": 0.19, + "learning_rate": 8.212967420959717e-05, + "loss": 2.4609, + "theoretical_loss": 3.4364183884957464, + "tokens_seen": 1958412288 + }, + { + "epoch": 0.19, + "learning_rate": 8.212164981543893e-05, + "loss": 2.6224, + "theoretical_loss": 3.43639913379479, + "tokens_seen": 1958543360 + }, + { + "epoch": 0.19, + "learning_rate": 8.211362542128069e-05, + "loss": 2.5121, + "theoretical_loss": 3.4363798807431527, + "tokens_seen": 1958674432 + }, + { + "epoch": 0.19, + "learning_rate": 8.210560102712245e-05, + "loss": 2.5246, + "theoretical_loss": 3.4363606293405837, + "tokens_seen": 1958805504 + }, + { + "epoch": 0.19, + "learning_rate": 8.209757663296421e-05, + "loss": 2.598, + "theoretical_loss": 3.436341379586832, + "tokens_seen": 1958936576 + }, + { + "epoch": 0.19, + "learning_rate": 8.208955223880597e-05, + "loss": 2.6481, + "theoretical_loss": 3.436322131481645, + "tokens_seen": 1959067648 + }, + { + "epoch": 0.19, + "learning_rate": 8.208152784464773e-05, + "loss": 2.3418, + "theoretical_loss": 3.4363028850247717, + "tokens_seen": 1959198720 + }, + { + "epoch": 0.19, + "learning_rate": 8.20735034504895e-05, + "loss": 2.378, + "theoretical_loss": 3.4362836402159607, + "tokens_seen": 1959329792 + }, + { + "epoch": 0.19, + "learning_rate": 8.206547905633125e-05, + "loss": 2.6566, + "theoretical_loss": 3.43626439705496, + "tokens_seen": 1959460864 + }, + { + "epoch": 0.19, + "learning_rate": 8.2057454662173e-05, + "loss": 2.5246, + "theoretical_loss": 3.4362451555415197, + "tokens_seen": 1959591936 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1079297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.692366600036621, + "objective/train/theoretical_loss": 3.436235535402555, + "objective/train/tokens_used": 330116576, + "theoretical_loss": 3.436235535402555, + "tokens_seen": 1959657472 + }, + { + "epoch": 0.19, + "learning_rate": 8.204943026801477e-05, + "loss": 2.4392, + "theoretical_loss": 3.4362259156753874, + "tokens_seen": 1959723008 + }, + { + "epoch": 0.19, + "learning_rate": 8.204140587385652e-05, + "loss": 2.5496, + "theoretical_loss": 3.436206677456312, + "tokens_seen": 1959854080 + }, + { + "epoch": 0.19, + "learning_rate": 8.203338147969829e-05, + "loss": 2.4338, + "theoretical_loss": 3.4361874408840425, + "tokens_seen": 1959985152 + }, + { + "epoch": 0.19, + "learning_rate": 8.202535708554004e-05, + "loss": 2.618, + "theoretical_loss": 3.436168205958328, + "tokens_seen": 1960116224 + }, + { + "epoch": 0.19, + "learning_rate": 8.20173326913818e-05, + "loss": 2.4057, + "theoretical_loss": 3.436148972678917, + "tokens_seen": 1960247296 + }, + { + "epoch": 0.19, + "learning_rate": 8.200930829722356e-05, + "loss": 2.6132, + "theoretical_loss": 3.43612974104556, + "tokens_seen": 1960378368 + }, + { + "epoch": 0.19, + "learning_rate": 8.200128390306531e-05, + "loss": 2.3999, + "theoretical_loss": 3.436110511058004, + "tokens_seen": 1960509440 + }, + { + "epoch": 0.19, + "learning_rate": 8.199325950890708e-05, + "loss": 2.5032, + "theoretical_loss": 3.4360912827159997, + "tokens_seen": 1960640512 + }, + { + "epoch": 0.19, + "learning_rate": 8.198523511474883e-05, + "loss": 2.4567, + "theoretical_loss": 3.436072056019295, + "tokens_seen": 1960771584 + }, + { + "epoch": 0.19, + "learning_rate": 8.19772107205906e-05, + "loss": 2.5605, + "theoretical_loss": 3.4360528309676406, + "tokens_seen": 1960902656 + }, + { + "epoch": 0.19, + "learning_rate": 8.196918632643235e-05, + "loss": 2.5381, + "theoretical_loss": 3.436033607560785, + "tokens_seen": 1961033728 + }, + { + "epoch": 0.19, + "learning_rate": 8.196116193227412e-05, + "loss": 2.5904, + "theoretical_loss": 3.4360143857984777, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1080387, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7845823764801025, + "objective/train/theoretical_loss": 3.4359951656804677, + "objective/train/tokens_used": 331754976, + "theoretical_loss": 3.4359951656804677, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.19, + "learning_rate": 8.195313753811587e-05, + "loss": 2.6118, + "theoretical_loss": 3.4359951656804677, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.19, + "learning_rate": 8.194511314395762e-05, + "loss": 2.5112, + "theoretical_loss": 3.435975947206505, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.19, + "learning_rate": 8.193708874979939e-05, + "loss": 2.5616, + "theoretical_loss": 3.4359567303763385, + "tokens_seen": 1961558016 + }, + { + "epoch": 0.19, + "learning_rate": 8.192906435564114e-05, + "loss": 2.472, + "theoretical_loss": 3.4359375151897185, + "tokens_seen": 1961689088 + }, + { + "epoch": 0.19, + "learning_rate": 8.192103996148291e-05, + "loss": 2.5179, + "theoretical_loss": 3.4359183016463946, + "tokens_seen": 1961820160 + }, + { + "epoch": 0.19, + "learning_rate": 8.191301556732466e-05, + "loss": 2.4378, + "theoretical_loss": 3.435899089746116, + "tokens_seen": 1961951232 + }, + { + "epoch": 0.19, + "learning_rate": 8.190499117316643e-05, + "loss": 2.3945, + "theoretical_loss": 3.435879879488633, + "tokens_seen": 1962082304 + }, + { + "epoch": 0.19, + "learning_rate": 8.189696677900818e-05, + "loss": 2.592, + "theoretical_loss": 3.4358606708736943, + "tokens_seen": 1962213376 + }, + { + "epoch": 0.19, + "learning_rate": 8.188894238484994e-05, + "loss": 2.5364, + "theoretical_loss": 3.435841463901051, + "tokens_seen": 1962344448 + }, + { + "epoch": 0.19, + "learning_rate": 8.18809179906917e-05, + "loss": 2.5239, + "theoretical_loss": 3.4358222585704525, + "tokens_seen": 1962475520 + }, + { + "epoch": 0.19, + "learning_rate": 8.187289359653346e-05, + "loss": 2.5953, + "theoretical_loss": 3.4358030548816485, + "tokens_seen": 1962606592 + }, + { + "epoch": 0.19, + "learning_rate": 8.186486920237522e-05, + "loss": 2.4442, + "theoretical_loss": 3.4357838528343896, + "tokens_seen": 1962737664 + }, + { + "epoch": 0.19, + "learning_rate": 8.185684480821698e-05, + "loss": 2.5722, + "theoretical_loss": 3.4357646524284258, + "tokens_seen": 1962868736 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1080811, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.384798049926758, + "objective/train/theoretical_loss": 3.435755052840851, + "objective/train/tokens_used": 333393376, + "theoretical_loss": 3.435755052840851, + "tokens_seen": 1962934272 + }, + { + "epoch": 0.19, + "learning_rate": 8.184882041405874e-05, + "loss": 2.4926, + "theoretical_loss": 3.4357454536635066, + "tokens_seen": 1962999808 + }, + { + "epoch": 0.19, + "learning_rate": 8.18407960199005e-05, + "loss": 2.5586, + "theoretical_loss": 3.4357262565393825, + "tokens_seen": 1963130880 + }, + { + "epoch": 0.19, + "learning_rate": 8.183277162574225e-05, + "loss": 2.5849, + "theoretical_loss": 3.4357070610558043, + "tokens_seen": 1963261952 + }, + { + "epoch": 0.19, + "learning_rate": 8.182474723158402e-05, + "loss": 2.5331, + "theoretical_loss": 3.4356878672125215, + "tokens_seen": 1963393024 + }, + { + "epoch": 0.19, + "learning_rate": 8.181672283742577e-05, + "loss": 2.433, + "theoretical_loss": 3.4356686750092846, + "tokens_seen": 1963524096 + }, + { + "epoch": 0.19, + "learning_rate": 8.180869844326754e-05, + "loss": 2.6144, + "theoretical_loss": 3.4356494844458445, + "tokens_seen": 1963655168 + }, + { + "epoch": 0.19, + "learning_rate": 8.180067404910929e-05, + "loss": 2.5068, + "theoretical_loss": 3.435630295521951, + "tokens_seen": 1963786240 + }, + { + "epoch": 0.19, + "learning_rate": 8.179264965495106e-05, + "loss": 2.4976, + "theoretical_loss": 3.4356111082373553, + "tokens_seen": 1963917312 + }, + { + "epoch": 0.19, + "learning_rate": 8.178462526079281e-05, + "loss": 2.5668, + "theoretical_loss": 3.435591922591807, + "tokens_seen": 1964048384 + }, + { + "epoch": 0.19, + "learning_rate": 8.177660086663458e-05, + "loss": 2.5539, + "theoretical_loss": 3.4355727385850576, + "tokens_seen": 1964179456 + }, + { + "epoch": 0.19, + "learning_rate": 8.176857647247633e-05, + "loss": 2.4739, + "theoretical_loss": 3.435553556216858, + "tokens_seen": 1964310528 + }, + { + "epoch": 0.19, + "learning_rate": 8.176055207831808e-05, + "loss": 2.4657, + "theoretical_loss": 3.4355343754869576, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1082085, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1325109004974365, + "objective/train/theoretical_loss": 3.4355151963951083, + "objective/train/tokens_used": 335031776, + "theoretical_loss": 3.4355151963951083, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.19, + "learning_rate": 8.175252768415985e-05, + "loss": 2.5154, + "theoretical_loss": 3.4355151963951083, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.19, + "learning_rate": 8.17445032900016e-05, + "loss": 2.586, + "theoretical_loss": 3.4354960189410604, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.19, + "learning_rate": 8.173647889584337e-05, + "loss": 2.6418, + "theoretical_loss": 3.4354768431245652, + "tokens_seen": 1964834816 + }, + { + "epoch": 0.19, + "learning_rate": 8.172845450168512e-05, + "loss": 2.566, + "theoretical_loss": 3.4354576689453733, + "tokens_seen": 1964965888 + }, + { + "epoch": 0.19, + "learning_rate": 8.172043010752689e-05, + "loss": 2.5053, + "theoretical_loss": 3.4354384964032363, + "tokens_seen": 1965096960 + }, + { + "epoch": 0.19, + "learning_rate": 8.171240571336864e-05, + "loss": 2.4657, + "theoretical_loss": 3.435419325497904, + "tokens_seen": 1965228032 + }, + { + "epoch": 0.19, + "learning_rate": 8.17043813192104e-05, + "loss": 2.5324, + "theoretical_loss": 3.4354001562291288, + "tokens_seen": 1965359104 + }, + { + "epoch": 0.19, + "learning_rate": 8.169635692505216e-05, + "loss": 2.468, + "theoretical_loss": 3.4353809885966613, + "tokens_seen": 1965490176 + }, + { + "epoch": 0.19, + "learning_rate": 8.168833253089391e-05, + "loss": 2.5399, + "theoretical_loss": 3.4353618226002527, + "tokens_seen": 1965621248 + }, + { + "epoch": 0.19, + "learning_rate": 8.168030813673568e-05, + "loss": 2.5918, + "theoretical_loss": 3.4353426582396542, + "tokens_seen": 1965752320 + }, + { + "epoch": 0.19, + "learning_rate": 8.167228374257743e-05, + "loss": 2.5126, + "theoretical_loss": 3.4353234955146177, + "tokens_seen": 1965883392 + }, + { + "epoch": 0.19, + "learning_rate": 8.16642593484192e-05, + "loss": 2.4399, + "theoretical_loss": 3.4353043344248944, + "tokens_seen": 1966014464 + }, + { + "epoch": 0.19, + "learning_rate": 8.165623495426095e-05, + "loss": 2.465, + "theoretical_loss": 3.435285174970235, + "tokens_seen": 1966145536 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1082685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4286997318267822, + "objective/train/theoretical_loss": 3.435275595855977, + "objective/train/tokens_used": 336670176, + "theoretical_loss": 3.435275595855977, + "tokens_seen": 1966211072 + }, + { + "epoch": 0.19, + "learning_rate": 8.164821056010271e-05, + "loss": 2.4441, + "theoretical_loss": 3.4352660171503913, + "tokens_seen": 1966276608 + }, + { + "epoch": 0.19, + "learning_rate": 8.164018616594447e-05, + "loss": 2.586, + "theoretical_loss": 3.4352468609651154, + "tokens_seen": 1966407680 + }, + { + "epoch": 0.19, + "learning_rate": 8.163216177178623e-05, + "loss": 2.5347, + "theoretical_loss": 3.4352277064141585, + "tokens_seen": 1966538752 + }, + { + "epoch": 0.19, + "learning_rate": 8.1624137377628e-05, + "loss": 2.4911, + "theoretical_loss": 3.4352085534972723, + "tokens_seen": 1966669824 + }, + { + "epoch": 0.19, + "learning_rate": 8.161611298346975e-05, + "loss": 2.6297, + "theoretical_loss": 3.435189402214208, + "tokens_seen": 1966800896 + }, + { + "epoch": 0.19, + "learning_rate": 8.160808858931151e-05, + "loss": 2.4255, + "theoretical_loss": 3.4351702525647183, + "tokens_seen": 1966931968 + }, + { + "epoch": 0.19, + "learning_rate": 8.160006419515327e-05, + "loss": 2.5653, + "theoretical_loss": 3.4351511045485545, + "tokens_seen": 1967063040 + }, + { + "epoch": 0.19, + "learning_rate": 8.159203980099502e-05, + "loss": 2.5436, + "theoretical_loss": 3.4351319581654685, + "tokens_seen": 1967194112 + }, + { + "epoch": 0.19, + "learning_rate": 8.158401540683679e-05, + "loss": 2.6022, + "theoretical_loss": 3.4351128134152122, + "tokens_seen": 1967325184 + }, + { + "epoch": 0.19, + "learning_rate": 8.157599101267854e-05, + "loss": 2.4349, + "theoretical_loss": 3.4350936702975376, + "tokens_seen": 1967456256 + }, + { + "epoch": 0.19, + "learning_rate": 8.15679666185203e-05, + "loss": 2.4711, + "theoretical_loss": 3.435074528812197, + "tokens_seen": 1967587328 + }, + { + "epoch": 0.19, + "learning_rate": 8.155994222436206e-05, + "loss": 2.5145, + "theoretical_loss": 3.435055388958942, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1083643, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4880809783935547, + "objective/train/theoretical_loss": 3.435036250737525, + "objective/train/tokens_used": 338308576, + "theoretical_loss": 3.435036250737525, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.19, + "learning_rate": 8.155191783020383e-05, + "loss": 2.5831, + "theoretical_loss": 3.435036250737525, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.19, + "learning_rate": 8.154389343604558e-05, + "loss": 2.5233, + "theoretical_loss": 3.4350171141476986, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 8.153586904188733e-05, + "loss": 2.4237, + "theoretical_loss": 3.4349979791892142, + "tokens_seen": 1968111616 + }, + { + "epoch": 0.19, + "learning_rate": 8.15278446477291e-05, + "loss": 2.5631, + "theoretical_loss": 3.4349788458618247, + "tokens_seen": 1968242688 + }, + { + "epoch": 0.19, + "learning_rate": 8.151982025357085e-05, + "loss": 2.4931, + "theoretical_loss": 3.434959714165282, + "tokens_seen": 1968373760 + }, + { + "epoch": 0.19, + "learning_rate": 8.151179585941262e-05, + "loss": 2.7059, + "theoretical_loss": 3.4349405840993392, + "tokens_seen": 1968504832 + }, + { + "epoch": 0.19, + "learning_rate": 8.150377146525437e-05, + "loss": 2.4426, + "theoretical_loss": 3.434921455663748, + "tokens_seen": 1968635904 + }, + { + "epoch": 0.19, + "learning_rate": 8.149574707109614e-05, + "loss": 2.5333, + "theoretical_loss": 3.4349023288582616, + "tokens_seen": 1968766976 + }, + { + "epoch": 0.19, + "learning_rate": 8.148772267693789e-05, + "loss": 2.5284, + "theoretical_loss": 3.4348832036826322, + "tokens_seen": 1968898048 + }, + { + "epoch": 0.19, + "learning_rate": 8.147969828277966e-05, + "loss": 2.5927, + "theoretical_loss": 3.434864080136612, + "tokens_seen": 1969029120 + }, + { + "epoch": 0.19, + "learning_rate": 8.147167388862141e-05, + "loss": 2.7439, + "theoretical_loss": 3.434844958219955, + "tokens_seen": 1969160192 + }, + { + "epoch": 0.19, + "learning_rate": 8.146364949446317e-05, + "loss": 2.5252, + "theoretical_loss": 3.434825837932413, + "tokens_seen": 1969291264 + }, + { + "epoch": 0.19, + "learning_rate": 8.145562510030493e-05, + "loss": 2.674, + "theoretical_loss": 3.4348067192737384, + "tokens_seen": 1969422336 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1084001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6455793380737305, + "objective/train/theoretical_loss": 3.4347971605551493, + "objective/train/tokens_used": 339946976, + "theoretical_loss": 3.4347971605551493, + "tokens_seen": 1969487872 + }, + { + "epoch": 0.19, + "learning_rate": 8.144760070614668e-05, + "loss": 2.4479, + "theoretical_loss": 3.4347876022436843, + "tokens_seen": 1969553408 + }, + { + "epoch": 0.19, + "learning_rate": 8.143957631198845e-05, + "loss": 2.5325, + "theoretical_loss": 3.434768486842005, + "tokens_seen": 1969684480 + }, + { + "epoch": 0.19, + "learning_rate": 8.14315519178302e-05, + "loss": 2.6488, + "theoretical_loss": 3.434749373068451, + "tokens_seen": 1969815552 + }, + { + "epoch": 0.19, + "learning_rate": 8.142352752367197e-05, + "loss": 2.4881, + "theoretical_loss": 3.4347302609227772, + "tokens_seen": 1969946624 + }, + { + "epoch": 0.19, + "learning_rate": 8.141550312951372e-05, + "loss": 2.4335, + "theoretical_loss": 3.434711150404736, + "tokens_seen": 1970077696 + }, + { + "epoch": 0.19, + "learning_rate": 8.140747873535548e-05, + "loss": 2.48, + "theoretical_loss": 3.4346920415140803, + "tokens_seen": 1970208768 + }, + { + "epoch": 0.19, + "learning_rate": 8.139945434119724e-05, + "loss": 2.6898, + "theoretical_loss": 3.434672934250564, + "tokens_seen": 1970339840 + }, + { + "epoch": 0.19, + "learning_rate": 8.1391429947039e-05, + "loss": 2.5244, + "theoretical_loss": 3.434653828613939, + "tokens_seen": 1970470912 + }, + { + "epoch": 0.19, + "learning_rate": 8.138340555288076e-05, + "loss": 2.4301, + "theoretical_loss": 3.434634724603961, + "tokens_seen": 1970601984 + }, + { + "epoch": 0.19, + "learning_rate": 8.137538115872252e-05, + "loss": 2.5058, + "theoretical_loss": 3.43461562222038, + "tokens_seen": 1970733056 + }, + { + "epoch": 0.19, + "learning_rate": 8.136735676456428e-05, + "loss": 2.5456, + "theoretical_loss": 3.434596521462952, + "tokens_seen": 1970864128 + }, + { + "epoch": 0.19, + "learning_rate": 8.135933237040604e-05, + "loss": 2.4504, + "theoretical_loss": 3.4345774223314294, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1085329, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.782360076904297, + "objective/train/theoretical_loss": 3.4345583248255656, + "objective/train/tokens_used": 341585376, + "theoretical_loss": 3.4345583248255656, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 8.135130797624779e-05, + "loss": 2.4819, + "theoretical_loss": 3.4345583248255656, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 8.134328358208956e-05, + "loss": 2.6041, + "theoretical_loss": 3.434539228945115, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 8.133525918793131e-05, + "loss": 2.5261, + "theoretical_loss": 3.43452013468983, + "tokens_seen": 1971388416 + }, + { + "epoch": 0.19, + "learning_rate": 8.132723479377308e-05, + "loss": 2.4584, + "theoretical_loss": 3.4345010420594653, + "tokens_seen": 1971519488 + }, + { + "epoch": 0.19, + "learning_rate": 8.131921039961483e-05, + "loss": 2.3719, + "theoretical_loss": 3.434481951053774, + "tokens_seen": 1971650560 + }, + { + "epoch": 0.2, + "learning_rate": 8.13111860054566e-05, + "loss": 2.5287, + "theoretical_loss": 3.4344628616725097, + "tokens_seen": 1971781632 + }, + { + "epoch": 0.2, + "learning_rate": 8.130316161129835e-05, + "loss": 2.7084, + "theoretical_loss": 3.4344437739154268, + "tokens_seen": 1971912704 + }, + { + "epoch": 0.2, + "learning_rate": 8.12951372171401e-05, + "loss": 2.4448, + "theoretical_loss": 3.434424687782279, + "tokens_seen": 1972043776 + }, + { + "epoch": 0.2, + "learning_rate": 8.128711282298187e-05, + "loss": 2.3985, + "theoretical_loss": 3.4344056032728196, + "tokens_seen": 1972174848 + }, + { + "epoch": 0.2, + "learning_rate": 8.127908842882362e-05, + "loss": 2.6247, + "theoretical_loss": 3.4343865203868034, + "tokens_seen": 1972305920 + }, + { + "epoch": 0.2, + "learning_rate": 8.127106403466539e-05, + "loss": 2.6056, + "theoretical_loss": 3.4343674391239842, + "tokens_seen": 1972436992 + }, + { + "epoch": 0.2, + "learning_rate": 8.126303964050714e-05, + "loss": 2.6542, + "theoretical_loss": 3.4343483594841158, + "tokens_seen": 1972568064 + }, + { + "epoch": 0.2, + "learning_rate": 8.125501524634891e-05, + "loss": 2.48, + "theoretical_loss": 3.4343292814669524, + "tokens_seen": 1972699136 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1086429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2790753841400146, + "objective/train/theoretical_loss": 3.4343197430668084, + "objective/train/tokens_used": 343223776, + "theoretical_loss": 3.4343197430668084, + "tokens_seen": 1972764672 + }, + { + "epoch": 0.2, + "learning_rate": 8.124699085219066e-05, + "loss": 2.6051, + "theoretical_loss": 3.4343102050722485, + "tokens_seen": 1972830208 + }, + { + "epoch": 0.2, + "learning_rate": 8.123896645803242e-05, + "loss": 2.5512, + "theoretical_loss": 3.434291130299758, + "tokens_seen": 1972961280 + }, + { + "epoch": 0.2, + "learning_rate": 8.123094206387418e-05, + "loss": 2.3245, + "theoretical_loss": 3.4342720571492356, + "tokens_seen": 1973092352 + }, + { + "epoch": 0.2, + "learning_rate": 8.122291766971594e-05, + "loss": 2.538, + "theoretical_loss": 3.434252985620435, + "tokens_seen": 1973223424 + }, + { + "epoch": 0.2, + "learning_rate": 8.12148932755577e-05, + "loss": 2.6185, + "theoretical_loss": 3.4342339157131114, + "tokens_seen": 1973354496 + }, + { + "epoch": 0.2, + "learning_rate": 8.120686888139945e-05, + "loss": 2.5536, + "theoretical_loss": 3.434214847427018, + "tokens_seen": 1973485568 + }, + { + "epoch": 0.2, + "learning_rate": 8.119884448724122e-05, + "loss": 2.3628, + "theoretical_loss": 3.434195780761911, + "tokens_seen": 1973616640 + }, + { + "epoch": 0.2, + "learning_rate": 8.119082009308297e-05, + "loss": 2.4919, + "theoretical_loss": 3.434176715717544, + "tokens_seen": 1973747712 + }, + { + "epoch": 0.2, + "learning_rate": 8.118279569892473e-05, + "loss": 2.5678, + "theoretical_loss": 3.4341576522936714, + "tokens_seen": 1973878784 + }, + { + "epoch": 0.2, + "learning_rate": 8.11747713047665e-05, + "loss": 2.6441, + "theoretical_loss": 3.4341385904900483, + "tokens_seen": 1974009856 + }, + { + "epoch": 0.2, + "learning_rate": 8.116674691060825e-05, + "loss": 2.5687, + "theoretical_loss": 3.4341195303064294, + "tokens_seen": 1974140928 + }, + { + "epoch": 0.2, + "learning_rate": 8.115872251645001e-05, + "loss": 2.5559, + "theoretical_loss": 3.4341004717425694, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1087105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.320133924484253, + "objective/train/theoretical_loss": 3.4340814147982233, + "objective/train/tokens_used": 344862176, + "theoretical_loss": 3.4340814147982233, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.2, + "learning_rate": 8.115069812229177e-05, + "loss": 2.4467, + "theoretical_loss": 3.4340814147982233, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.2, + "learning_rate": 8.114267372813353e-05, + "loss": 2.4401, + "theoretical_loss": 3.4340623594731454, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.2, + "learning_rate": 8.113464933397529e-05, + "loss": 2.6396, + "theoretical_loss": 3.4340433057670907, + "tokens_seen": 1974665216 + }, + { + "epoch": 0.2, + "learning_rate": 8.112662493981705e-05, + "loss": 2.6161, + "theoretical_loss": 3.434024253679815, + "tokens_seen": 1974796288 + }, + { + "epoch": 0.2, + "learning_rate": 8.111860054565881e-05, + "loss": 2.8056, + "theoretical_loss": 3.434005203211073, + "tokens_seen": 1974927360 + }, + { + "epoch": 0.2, + "learning_rate": 8.111057615150056e-05, + "loss": 2.5804, + "theoretical_loss": 3.4339861543606194, + "tokens_seen": 1975058432 + }, + { + "epoch": 0.2, + "learning_rate": 8.110255175734233e-05, + "loss": 2.4813, + "theoretical_loss": 3.4339671071282094, + "tokens_seen": 1975189504 + }, + { + "epoch": 0.2, + "learning_rate": 8.109452736318408e-05, + "loss": 2.5453, + "theoretical_loss": 3.4339480615135987, + "tokens_seen": 1975320576 + }, + { + "epoch": 0.2, + "learning_rate": 8.108650296902585e-05, + "loss": 2.618, + "theoretical_loss": 3.4339290175165424, + "tokens_seen": 1975451648 + }, + { + "epoch": 0.2, + "learning_rate": 8.10784785748676e-05, + "loss": 2.5069, + "theoretical_loss": 3.4339099751367956, + "tokens_seen": 1975582720 + }, + { + "epoch": 0.2, + "learning_rate": 8.107045418070937e-05, + "loss": 2.498, + "theoretical_loss": 3.433890934374114, + "tokens_seen": 1975713792 + }, + { + "epoch": 0.2, + "learning_rate": 8.106242978655112e-05, + "loss": 2.5357, + "theoretical_loss": 3.4338718952282523, + "tokens_seen": 1975844864 + }, + { + "epoch": 0.2, + "learning_rate": 8.105440539239287e-05, + "loss": 2.4488, + "theoretical_loss": 3.4338528576989664, + "tokens_seen": 1975975936 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1088406, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.515301465988159, + "objective/train/theoretical_loss": 3.433843339540463, + "objective/train/tokens_used": 346500576, + "theoretical_loss": 3.433843339540463, + "tokens_seen": 1976041472 + }, + { + "epoch": 0.2, + "learning_rate": 8.104638099823464e-05, + "loss": 2.4573, + "theoretical_loss": 3.4338338217860116, + "tokens_seen": 1976107008 + }, + { + "epoch": 0.2, + "learning_rate": 8.103835660407639e-05, + "loss": 2.5943, + "theoretical_loss": 3.433814787489144, + "tokens_seen": 1976238080 + }, + { + "epoch": 0.2, + "learning_rate": 8.103033220991816e-05, + "loss": 2.6775, + "theoretical_loss": 3.433795754808119, + "tokens_seen": 1976369152 + }, + { + "epoch": 0.2, + "learning_rate": 8.102230781575991e-05, + "loss": 2.5248, + "theoretical_loss": 3.4337767237426924, + "tokens_seen": 1976500224 + }, + { + "epoch": 0.2, + "learning_rate": 8.101428342160168e-05, + "loss": 2.5347, + "theoretical_loss": 3.43375769429262, + "tokens_seen": 1976631296 + }, + { + "epoch": 0.2, + "learning_rate": 8.100625902744343e-05, + "loss": 2.4765, + "theoretical_loss": 3.4337386664576566, + "tokens_seen": 1976762368 + }, + { + "epoch": 0.2, + "learning_rate": 8.099823463328519e-05, + "loss": 2.4933, + "theoretical_loss": 3.433719640237559, + "tokens_seen": 1976893440 + }, + { + "epoch": 0.2, + "learning_rate": 8.099021023912695e-05, + "loss": 2.6944, + "theoretical_loss": 3.433700615632083, + "tokens_seen": 1977024512 + }, + { + "epoch": 0.2, + "learning_rate": 8.09821858449687e-05, + "loss": 2.5308, + "theoretical_loss": 3.4336815926409843, + "tokens_seen": 1977155584 + }, + { + "epoch": 0.2, + "learning_rate": 8.097416145081047e-05, + "loss": 2.6257, + "theoretical_loss": 3.433662571264019, + "tokens_seen": 1977286656 + }, + { + "epoch": 0.2, + "learning_rate": 8.096613705665222e-05, + "loss": 2.4688, + "theoretical_loss": 3.4336435515009436, + "tokens_seen": 1977417728 + }, + { + "epoch": 0.2, + "learning_rate": 8.095811266249399e-05, + "loss": 2.4563, + "theoretical_loss": 3.433624533351513, + "tokens_seen": 1977548800 + }, + { + "debugging/Self-BLEU-5": 0.5383387038400951, + "debugging/distinct-1-grams": 0.7574669480332016, + "debugging/distinct-2-grams": 0.9486429063642564, + "debugging/entropy-1-grams": 5.78993480235564, + "debugging/entropy-2-grams": 6.620072215652616, + "debugging/length": 591.3, + "debugging/num_segments": 10, + "debugging/score": 0.004437156266881241, + "debugging/score_std": 0.0035169708188302635, + "epoch": 0.2, + "objective/train/docs_used": 1088993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6108832359313965, + "objective/train/theoretical_loss": 3.4336055168154846, + "objective/train/tokens_used": 348138976, + "theoretical_loss": 3.4336055168154846, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.2, + "learning_rate": 8.095008826833574e-05, + "loss": 2.5314, + "theoretical_loss": 3.4336055168154846, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.2, + "learning_rate": 8.09420638741775e-05, + "loss": 2.5313, + "theoretical_loss": 3.4335865018926146, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.2, + "learning_rate": 8.093403948001926e-05, + "loss": 2.3723, + "theoretical_loss": 3.433567488582658, + "tokens_seen": 1977942016 + }, + { + "epoch": 0.2, + "learning_rate": 8.092601508586102e-05, + "loss": 2.4287, + "theoretical_loss": 3.433548476885372, + "tokens_seen": 1978073088 + }, + { + "epoch": 0.2, + "learning_rate": 8.091799069170278e-05, + "loss": 2.49, + "theoretical_loss": 3.433529466800514, + "tokens_seen": 1978204160 + }, + { + "epoch": 0.2, + "learning_rate": 8.090996629754454e-05, + "loss": 2.5583, + "theoretical_loss": 3.433510458327838, + "tokens_seen": 1978335232 + }, + { + "epoch": 0.2, + "learning_rate": 8.09019419033863e-05, + "loss": 2.4996, + "theoretical_loss": 3.4334914514671024, + "tokens_seen": 1978466304 + }, + { + "epoch": 0.2, + "learning_rate": 8.089391750922806e-05, + "loss": 2.5626, + "theoretical_loss": 3.433472446218063, + "tokens_seen": 1978597376 + }, + { + "epoch": 0.2, + "learning_rate": 8.088589311506981e-05, + "loss": 2.5652, + "theoretical_loss": 3.4334534425804764, + "tokens_seen": 1978728448 + }, + { + "epoch": 0.2, + "learning_rate": 8.087786872091158e-05, + "loss": 2.4335, + "theoretical_loss": 3.4334344405541, + "tokens_seen": 1978859520 + }, + { + "epoch": 0.2, + "learning_rate": 8.086984432675333e-05, + "loss": 2.495, + "theoretical_loss": 3.433415440138689, + "tokens_seen": 1978990592 + }, + { + "epoch": 0.2, + "learning_rate": 8.08618199325951e-05, + "loss": 2.4582, + "theoretical_loss": 3.4333964413340015, + "tokens_seen": 1979121664 + }, + { + "epoch": 0.2, + "learning_rate": 8.085379553843685e-05, + "loss": 2.5634, + "theoretical_loss": 3.4333774441397935, + "tokens_seen": 1979252736 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1090226, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6920125484466553, + "objective/train/theoretical_loss": 3.4333679461465434, + "objective/train/tokens_used": 349777376, + "theoretical_loss": 3.4333679461465434, + "tokens_seen": 1979318272 + }, + { + "epoch": 0.2, + "learning_rate": 8.084577114427862e-05, + "loss": 2.6059, + "theoretical_loss": 3.433358448555822, + "tokens_seen": 1979383808 + }, + { + "epoch": 0.2, + "learning_rate": 8.083774675012037e-05, + "loss": 2.5343, + "theoretical_loss": 3.4333394545818443, + "tokens_seen": 1979514880 + }, + { + "epoch": 0.2, + "learning_rate": 8.082972235596214e-05, + "loss": 2.588, + "theoretical_loss": 3.433320462217617, + "tokens_seen": 1979645952 + }, + { + "epoch": 0.2, + "learning_rate": 8.082169796180389e-05, + "loss": 2.5355, + "theoretical_loss": 3.4333014714628973, + "tokens_seen": 1979777024 + }, + { + "epoch": 0.2, + "learning_rate": 8.081367356764564e-05, + "loss": 2.7333, + "theoretical_loss": 3.433282482317442, + "tokens_seen": 1979908096 + }, + { + "epoch": 0.2, + "learning_rate": 8.080564917348741e-05, + "loss": 2.4832, + "theoretical_loss": 3.4332634947810083, + "tokens_seen": 1980039168 + }, + { + "epoch": 0.2, + "learning_rate": 8.079762477932916e-05, + "loss": 2.5933, + "theoretical_loss": 3.4332445088533534, + "tokens_seen": 1980170240 + }, + { + "epoch": 0.2, + "learning_rate": 8.078960038517093e-05, + "loss": 2.5591, + "theoretical_loss": 3.433225524534235, + "tokens_seen": 1980301312 + }, + { + "epoch": 0.2, + "learning_rate": 8.078157599101268e-05, + "loss": 2.6487, + "theoretical_loss": 3.433206541823409, + "tokens_seen": 1980432384 + }, + { + "epoch": 0.2, + "learning_rate": 8.077355159685445e-05, + "loss": 2.4959, + "theoretical_loss": 3.433187560720634, + "tokens_seen": 1980563456 + }, + { + "epoch": 0.2, + "learning_rate": 8.07655272026962e-05, + "loss": 2.6386, + "theoretical_loss": 3.433168581225667, + "tokens_seen": 1980694528 + }, + { + "epoch": 0.2, + "learning_rate": 8.075750280853796e-05, + "loss": 2.5598, + "theoretical_loss": 3.433149603338266, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1091410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.233128309249878, + "objective/train/theoretical_loss": 3.433130627058187, + "objective/train/tokens_used": 351415776, + "theoretical_loss": 3.433130627058187, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.2, + "learning_rate": 8.074947841437972e-05, + "loss": 2.4389, + "theoretical_loss": 3.433130627058187, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.2, + "learning_rate": 8.074145402022148e-05, + "loss": 2.6606, + "theoretical_loss": 3.433111652385189, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 8.073342962606324e-05, + "loss": 2.614, + "theoretical_loss": 3.433092679319029, + "tokens_seen": 1981218816 + }, + { + "epoch": 0.2, + "learning_rate": 8.0725405231905e-05, + "loss": 2.5507, + "theoretical_loss": 3.4330737078594638, + "tokens_seen": 1981349888 + }, + { + "epoch": 0.2, + "learning_rate": 8.071738083774676e-05, + "loss": 2.7506, + "theoretical_loss": 3.433054738006253, + "tokens_seen": 1981480960 + }, + { + "epoch": 0.2, + "learning_rate": 8.070935644358851e-05, + "loss": 2.5217, + "theoretical_loss": 3.4330357697591527, + "tokens_seen": 1981612032 + }, + { + "epoch": 0.2, + "learning_rate": 8.070133204943027e-05, + "loss": 2.5975, + "theoretical_loss": 3.4330168031179213, + "tokens_seen": 1981743104 + }, + { + "epoch": 0.2, + "learning_rate": 8.069330765527203e-05, + "loss": 2.5798, + "theoretical_loss": 3.4329978380823167, + "tokens_seen": 1981874176 + }, + { + "epoch": 0.2, + "learning_rate": 8.068528326111379e-05, + "loss": 2.5297, + "theoretical_loss": 3.432978874652097, + "tokens_seen": 1982005248 + }, + { + "epoch": 0.2, + "learning_rate": 8.067725886695555e-05, + "loss": 2.5863, + "theoretical_loss": 3.4329599128270196, + "tokens_seen": 1982136320 + }, + { + "epoch": 0.2, + "learning_rate": 8.066923447279731e-05, + "loss": 2.4973, + "theoretical_loss": 3.4329409526068426, + "tokens_seen": 1982267392 + }, + { + "epoch": 0.2, + "learning_rate": 8.066121007863907e-05, + "loss": 2.4644, + "theoretical_loss": 3.4329219939913247, + "tokens_seen": 1982398464 + }, + { + "epoch": 0.2, + "learning_rate": 8.065318568448083e-05, + "loss": 2.5549, + "theoretical_loss": 3.432903036980223, + "tokens_seen": 1982529536 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1091797, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8559112548828125, + "objective/train/theoretical_loss": 3.4328935590762533, + "objective/train/tokens_used": 353054176, + "theoretical_loss": 3.4328935590762533, + "tokens_seen": 1982595072 + }, + { + "epoch": 0.2, + "learning_rate": 8.064516129032258e-05, + "loss": 2.5303, + "theoretical_loss": 3.4328840815732966, + "tokens_seen": 1982660608 + }, + { + "epoch": 0.2, + "learning_rate": 8.063713689616435e-05, + "loss": 2.521, + "theoretical_loss": 3.4328651277703033, + "tokens_seen": 1982791680 + }, + { + "epoch": 0.2, + "learning_rate": 8.06291125020061e-05, + "loss": 2.5473, + "theoretical_loss": 3.432846175571001, + "tokens_seen": 1982922752 + }, + { + "epoch": 0.2, + "learning_rate": 8.062108810784787e-05, + "loss": 2.6932, + "theoretical_loss": 3.4328272249751492, + "tokens_seen": 1983053824 + }, + { + "epoch": 0.2, + "learning_rate": 8.061306371368962e-05, + "loss": 2.6695, + "theoretical_loss": 3.432808275982505, + "tokens_seen": 1983184896 + }, + { + "epoch": 0.2, + "learning_rate": 8.060503931953139e-05, + "loss": 2.5086, + "theoretical_loss": 3.4327893285928273, + "tokens_seen": 1983315968 + }, + { + "epoch": 0.2, + "learning_rate": 8.059701492537314e-05, + "loss": 2.5303, + "theoretical_loss": 3.432770382805874, + "tokens_seen": 1983447040 + }, + { + "epoch": 0.2, + "learning_rate": 8.058899053121489e-05, + "loss": 2.5718, + "theoretical_loss": 3.432751438621405, + "tokens_seen": 1983578112 + }, + { + "epoch": 0.2, + "learning_rate": 8.058096613705666e-05, + "loss": 2.6883, + "theoretical_loss": 3.4327324960391783, + "tokens_seen": 1983709184 + }, + { + "epoch": 0.2, + "learning_rate": 8.057294174289841e-05, + "loss": 2.6956, + "theoretical_loss": 3.4327135550589514, + "tokens_seen": 1983840256 + }, + { + "epoch": 0.2, + "learning_rate": 8.056491734874018e-05, + "loss": 2.6438, + "theoretical_loss": 3.432694615680485, + "tokens_seen": 1983971328 + }, + { + "epoch": 0.2, + "learning_rate": 8.055689295458193e-05, + "loss": 2.5278, + "theoretical_loss": 3.4326756779035357, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1092754, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2613365650177, + "objective/train/theoretical_loss": 3.4326567417278637, + "objective/train/tokens_used": 354692576, + "theoretical_loss": 3.4326567417278637, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 8.05488685604237e-05, + "loss": 2.54, + "theoretical_loss": 3.4326567417278637, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 8.054084416626545e-05, + "loss": 2.4222, + "theoretical_loss": 3.432637807153228, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 8.053281977210722e-05, + "loss": 2.6055, + "theoretical_loss": 3.4326188741793864, + "tokens_seen": 1984495616 + }, + { + "epoch": 0.2, + "learning_rate": 8.052479537794897e-05, + "loss": 2.5514, + "theoretical_loss": 3.4325999428060987, + "tokens_seen": 1984626688 + }, + { + "epoch": 0.2, + "learning_rate": 8.051677098379073e-05, + "loss": 2.7175, + "theoretical_loss": 3.432581013033124, + "tokens_seen": 1984757760 + }, + { + "epoch": 0.2, + "learning_rate": 8.050874658963249e-05, + "loss": 2.4612, + "theoretical_loss": 3.43256208486022, + "tokens_seen": 1984888832 + }, + { + "epoch": 0.2, + "learning_rate": 8.050072219547425e-05, + "loss": 2.5532, + "theoretical_loss": 3.4325431582871473, + "tokens_seen": 1985019904 + }, + { + "epoch": 0.2, + "learning_rate": 8.049269780131601e-05, + "loss": 2.4733, + "theoretical_loss": 3.4325242333136643, + "tokens_seen": 1985150976 + }, + { + "epoch": 0.2, + "learning_rate": 8.048467340715777e-05, + "loss": 2.6379, + "theoretical_loss": 3.432505309939531, + "tokens_seen": 1985282048 + }, + { + "epoch": 0.2, + "learning_rate": 8.047664901299953e-05, + "loss": 2.6498, + "theoretical_loss": 3.432486388164506, + "tokens_seen": 1985413120 + }, + { + "epoch": 0.2, + "learning_rate": 8.046862461884128e-05, + "loss": 2.5589, + "theoretical_loss": 3.432467467988348, + "tokens_seen": 1985544192 + }, + { + "epoch": 0.2, + "learning_rate": 8.046060022468304e-05, + "loss": 2.6433, + "theoretical_loss": 3.4324485494108177, + "tokens_seen": 1985675264 + }, + { + "epoch": 0.2, + "learning_rate": 8.04525758305248e-05, + "loss": 2.59, + "theoretical_loss": 3.4324296324316736, + "tokens_seen": 1985806336 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1093159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2847602367401123, + "objective/train/theoretical_loss": 3.432420174541422, + "objective/train/tokens_used": 356330976, + "theoretical_loss": 3.432420174541422, + "tokens_seen": 1985871872 + }, + { + "epoch": 0.2, + "learning_rate": 8.044455143636656e-05, + "loss": 2.6555, + "theoretical_loss": 3.4324107170506757, + "tokens_seen": 1985937408 + }, + { + "epoch": 0.2, + "learning_rate": 8.043652704220832e-05, + "loss": 2.5619, + "theoretical_loss": 3.432391803267583, + "tokens_seen": 1986068480 + }, + { + "epoch": 0.2, + "learning_rate": 8.042850264805008e-05, + "loss": 2.7432, + "theoretical_loss": 3.432372891082156, + "tokens_seen": 1986199552 + }, + { + "epoch": 0.2, + "learning_rate": 8.042047825389184e-05, + "loss": 2.6317, + "theoretical_loss": 3.4323539804941534, + "tokens_seen": 1986330624 + }, + { + "epoch": 0.2, + "learning_rate": 8.04124538597336e-05, + "loss": 2.5587, + "theoretical_loss": 3.432335071503335, + "tokens_seen": 1986461696 + }, + { + "epoch": 0.2, + "learning_rate": 8.040442946557535e-05, + "loss": 2.7005, + "theoretical_loss": 3.4323161641094604, + "tokens_seen": 1986592768 + }, + { + "epoch": 0.2, + "learning_rate": 8.039640507141712e-05, + "loss": 2.6133, + "theoretical_loss": 3.4322972583122904, + "tokens_seen": 1986723840 + }, + { + "epoch": 0.2, + "learning_rate": 8.038838067725887e-05, + "loss": 2.6584, + "theoretical_loss": 3.4322783541115838, + "tokens_seen": 1986854912 + }, + { + "epoch": 0.2, + "learning_rate": 8.038035628310064e-05, + "loss": 2.6685, + "theoretical_loss": 3.4322594515071003, + "tokens_seen": 1986985984 + }, + { + "epoch": 0.2, + "learning_rate": 8.037233188894239e-05, + "loss": 2.6374, + "theoretical_loss": 3.4322405504986007, + "tokens_seen": 1987117056 + }, + { + "epoch": 0.2, + "learning_rate": 8.036430749478416e-05, + "loss": 2.6503, + "theoretical_loss": 3.432221651085845, + "tokens_seen": 1987248128 + }, + { + "epoch": 0.2, + "learning_rate": 8.035628310062591e-05, + "loss": 2.637, + "theoretical_loss": 3.4322027532685926, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1094053, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.824427366256714, + "objective/train/theoretical_loss": 3.4321838570466037, + "objective/train/tokens_used": 357969376, + "theoretical_loss": 3.4321838570466037, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.2, + "learning_rate": 8.034825870646766e-05, + "loss": 2.658, + "theoretical_loss": 3.4321838570466037, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.2, + "learning_rate": 8.034023431230943e-05, + "loss": 2.568, + "theoretical_loss": 3.4321649624196384, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 8.033220991815118e-05, + "loss": 2.4641, + "theoretical_loss": 3.432146069387458, + "tokens_seen": 1987772416 + }, + { + "epoch": 0.2, + "learning_rate": 8.032418552399295e-05, + "loss": 2.5967, + "theoretical_loss": 3.432127177949821, + "tokens_seen": 1987903488 + }, + { + "epoch": 0.2, + "learning_rate": 8.03161611298347e-05, + "loss": 2.6815, + "theoretical_loss": 3.4321082881064884, + "tokens_seen": 1988034560 + }, + { + "epoch": 0.2, + "learning_rate": 8.030813673567647e-05, + "loss": 2.6253, + "theoretical_loss": 3.4320893998572215, + "tokens_seen": 1988165632 + }, + { + "epoch": 0.21, + "learning_rate": 8.030011234151822e-05, + "loss": 2.5624, + "theoretical_loss": 3.4320705132017792, + "tokens_seen": 1988296704 + }, + { + "epoch": 0.21, + "learning_rate": 8.029208794735998e-05, + "loss": 2.6203, + "theoretical_loss": 3.4320516281399227, + "tokens_seen": 1988427776 + }, + { + "epoch": 0.21, + "learning_rate": 8.028406355320174e-05, + "loss": 2.6069, + "theoretical_loss": 3.432032744671413, + "tokens_seen": 1988558848 + }, + { + "epoch": 0.21, + "learning_rate": 8.02760391590435e-05, + "loss": 2.6759, + "theoretical_loss": 3.4320138627960097, + "tokens_seen": 1988689920 + }, + { + "epoch": 0.21, + "learning_rate": 8.026801476488526e-05, + "loss": 2.5438, + "theoretical_loss": 3.4319949825134737, + "tokens_seen": 1988820992 + }, + { + "epoch": 0.21, + "learning_rate": 8.025999037072702e-05, + "loss": 2.6477, + "theoretical_loss": 3.4319761038235663, + "tokens_seen": 1988952064 + }, + { + "epoch": 0.21, + "learning_rate": 8.025196597656878e-05, + "loss": 2.5959, + "theoretical_loss": 3.4319572267260474, + "tokens_seen": 1989083136 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1094680, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8489184379577637, + "objective/train/theoretical_loss": 3.431947788774359, + "objective/train/tokens_used": 359607776, + "theoretical_loss": 3.431947788774359, + "tokens_seen": 1989148672 + }, + { + "epoch": 0.21, + "learning_rate": 8.024394158241054e-05, + "loss": 2.6793, + "theoretical_loss": 3.431938351220678, + "tokens_seen": 1989214208 + }, + { + "epoch": 0.21, + "learning_rate": 8.023591718825229e-05, + "loss": 2.6479, + "theoretical_loss": 3.4319194773072192, + "tokens_seen": 1989345280 + }, + { + "epoch": 0.21, + "learning_rate": 8.022789279409406e-05, + "loss": 2.5235, + "theoretical_loss": 3.4319006049854313, + "tokens_seen": 1989476352 + }, + { + "epoch": 0.21, + "learning_rate": 8.021986839993581e-05, + "loss": 2.6275, + "theoretical_loss": 3.431881734255076, + "tokens_seen": 1989607424 + }, + { + "epoch": 0.21, + "learning_rate": 8.021184400577757e-05, + "loss": 2.6886, + "theoretical_loss": 3.4318628651159138, + "tokens_seen": 1989738496 + }, + { + "epoch": 0.21, + "learning_rate": 8.020381961161933e-05, + "loss": 2.5579, + "theoretical_loss": 3.4318439975677055, + "tokens_seen": 1989869568 + }, + { + "epoch": 0.21, + "learning_rate": 8.01957952174611e-05, + "loss": 2.5609, + "theoretical_loss": 3.4318251316102124, + "tokens_seen": 1990000640 + }, + { + "epoch": 0.21, + "learning_rate": 8.018777082330285e-05, + "loss": 2.5518, + "theoretical_loss": 3.4318062672431964, + "tokens_seen": 1990131712 + }, + { + "epoch": 0.21, + "learning_rate": 8.017974642914461e-05, + "loss": 2.7237, + "theoretical_loss": 3.431787404466417, + "tokens_seen": 1990262784 + }, + { + "epoch": 0.21, + "learning_rate": 8.017172203498637e-05, + "loss": 2.5871, + "theoretical_loss": 3.431768543279637, + "tokens_seen": 1990393856 + }, + { + "epoch": 0.21, + "learning_rate": 8.016369764082812e-05, + "loss": 2.4901, + "theoretical_loss": 3.431749683682617, + "tokens_seen": 1990524928 + }, + { + "epoch": 0.21, + "learning_rate": 8.015567324666989e-05, + "loss": 2.6152, + "theoretical_loss": 3.4317308256751184, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1095652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7679972648620605, + "objective/train/theoretical_loss": 3.4317119692569023, + "objective/train/tokens_used": 361246176, + "theoretical_loss": 3.4317119692569023, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.21, + "learning_rate": 8.014764885251164e-05, + "loss": 2.5604, + "theoretical_loss": 3.4317119692569023, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.21, + "learning_rate": 8.013962445835341e-05, + "loss": 2.7062, + "theoretical_loss": 3.431693114427731, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.21, + "learning_rate": 8.013160006419516e-05, + "loss": 2.6594, + "theoretical_loss": 3.431674261187365, + "tokens_seen": 1991049216 + }, + { + "epoch": 0.21, + "learning_rate": 8.012357567003693e-05, + "loss": 2.5803, + "theoretical_loss": 3.4316554095355665, + "tokens_seen": 1991180288 + }, + { + "epoch": 0.21, + "learning_rate": 8.011555127587868e-05, + "loss": 2.6, + "theoretical_loss": 3.4316365594720963, + "tokens_seen": 1991311360 + }, + { + "epoch": 0.21, + "learning_rate": 8.010752688172043e-05, + "loss": 2.5181, + "theoretical_loss": 3.431617710996717, + "tokens_seen": 1991442432 + }, + { + "epoch": 0.21, + "learning_rate": 8.00995024875622e-05, + "loss": 2.4289, + "theoretical_loss": 3.4315988641091906, + "tokens_seen": 1991573504 + }, + { + "epoch": 0.21, + "learning_rate": 8.009147809340395e-05, + "loss": 2.6387, + "theoretical_loss": 3.431580018809277, + "tokens_seen": 1991704576 + }, + { + "epoch": 0.21, + "learning_rate": 8.008345369924572e-05, + "loss": 2.6242, + "theoretical_loss": 3.4315611750967396, + "tokens_seen": 1991835648 + }, + { + "epoch": 0.21, + "learning_rate": 8.007542930508747e-05, + "loss": 2.5817, + "theoretical_loss": 3.4315423329713397, + "tokens_seen": 1991966720 + }, + { + "epoch": 0.21, + "learning_rate": 8.006740491092924e-05, + "loss": 2.5962, + "theoretical_loss": 3.4315234924328393, + "tokens_seen": 1992097792 + }, + { + "epoch": 0.21, + "learning_rate": 8.005938051677099e-05, + "loss": 2.5673, + "theoretical_loss": 3.4315046534810003, + "tokens_seen": 1992228864 + }, + { + "epoch": 0.21, + "learning_rate": 8.005135612261275e-05, + "loss": 2.4096, + "theoretical_loss": 3.4314858161155852, + "tokens_seen": 1992359936 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1096842, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.55733060836792, + "objective/train/theoretical_loss": 3.431476398027712, + "objective/train/tokens_used": 362884576, + "theoretical_loss": 3.431476398027712, + "tokens_seen": 1992425472 + }, + { + "epoch": 0.21, + "learning_rate": 8.004333172845451e-05, + "loss": 2.5752, + "theoretical_loss": 3.431466980336355, + "tokens_seen": 1992491008 + }, + { + "epoch": 0.21, + "learning_rate": 8.003530733429627e-05, + "loss": 2.6069, + "theoretical_loss": 3.4314481461430724, + "tokens_seen": 1992622080 + }, + { + "epoch": 0.21, + "learning_rate": 8.002728294013803e-05, + "loss": 2.6405, + "theoretical_loss": 3.4314293135355003, + "tokens_seen": 1992753152 + }, + { + "epoch": 0.21, + "learning_rate": 8.001925854597979e-05, + "loss": 2.6135, + "theoretical_loss": 3.4314104825133995, + "tokens_seen": 1992884224 + }, + { + "epoch": 0.21, + "learning_rate": 8.001123415182155e-05, + "loss": 2.6832, + "theoretical_loss": 3.431391653076533, + "tokens_seen": 1993015296 + }, + { + "epoch": 0.21, + "learning_rate": 8.00032097576633e-05, + "loss": 2.5489, + "theoretical_loss": 3.4313728252246634, + "tokens_seen": 1993146368 + }, + { + "epoch": 0.21, + "learning_rate": 7.999518536350506e-05, + "loss": 2.6522, + "theoretical_loss": 3.431353998957552, + "tokens_seen": 1993277440 + }, + { + "epoch": 0.21, + "learning_rate": 7.998716096934683e-05, + "loss": 2.7246, + "theoretical_loss": 3.4313351742749623, + "tokens_seen": 1993408512 + }, + { + "epoch": 0.21, + "learning_rate": 7.997913657518858e-05, + "loss": 2.7097, + "theoretical_loss": 3.4313163511766565, + "tokens_seen": 1993539584 + }, + { + "epoch": 0.21, + "learning_rate": 7.997111218103034e-05, + "loss": 2.7062, + "theoretical_loss": 3.431297529662397, + "tokens_seen": 1993670656 + }, + { + "epoch": 0.21, + "learning_rate": 7.99630877868721e-05, + "loss": 2.7169, + "theoretical_loss": 3.431278709731946, + "tokens_seen": 1993801728 + }, + { + "epoch": 0.21, + "learning_rate": 7.995506339271386e-05, + "loss": 2.5566, + "theoretical_loss": 3.431259891385067, + "tokens_seen": 1993932800 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1097514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8202970027923584, + "objective/train/theoretical_loss": 3.4312410746215214, + "objective/train/tokens_used": 364522976, + "theoretical_loss": 3.4312410746215214, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.21, + "learning_rate": 7.994703899855562e-05, + "loss": 2.5633, + "theoretical_loss": 3.4312410746215214, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.21, + "learning_rate": 7.993901460439737e-05, + "loss": 2.7012, + "theoretical_loss": 3.431222259441073, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.21, + "learning_rate": 7.993099021023914e-05, + "loss": 2.6065, + "theoretical_loss": 3.431203445843485, + "tokens_seen": 1994326016 + }, + { + "epoch": 0.21, + "learning_rate": 7.992296581608089e-05, + "loss": 2.6068, + "theoretical_loss": 3.4311846338285186, + "tokens_seen": 1994457088 + }, + { + "epoch": 0.21, + "learning_rate": 7.991494142192266e-05, + "loss": 2.527, + "theoretical_loss": 3.4311658233959372, + "tokens_seen": 1994588160 + }, + { + "epoch": 0.21, + "learning_rate": 7.990691702776441e-05, + "loss": 2.5969, + "theoretical_loss": 3.431147014545505, + "tokens_seen": 1994719232 + }, + { + "epoch": 0.21, + "learning_rate": 7.989889263360616e-05, + "loss": 2.7501, + "theoretical_loss": 3.4311282072769833, + "tokens_seen": 1994850304 + }, + { + "epoch": 0.21, + "learning_rate": 7.989086823944793e-05, + "loss": 2.5058, + "theoretical_loss": 3.4311094015901364, + "tokens_seen": 1994981376 + }, + { + "epoch": 0.21, + "learning_rate": 7.988284384528968e-05, + "loss": 2.6537, + "theoretical_loss": 3.431090597484727, + "tokens_seen": 1995112448 + }, + { + "epoch": 0.21, + "learning_rate": 7.987481945113145e-05, + "loss": 2.6582, + "theoretical_loss": 3.431071794960517, + "tokens_seen": 1995243520 + }, + { + "epoch": 0.21, + "learning_rate": 7.98667950569732e-05, + "loss": 2.6121, + "theoretical_loss": 3.431052994017272, + "tokens_seen": 1995374592 + }, + { + "epoch": 0.21, + "learning_rate": 7.985877066281497e-05, + "loss": 2.6305, + "theoretical_loss": 3.431034194654753, + "tokens_seen": 1995505664 + }, + { + "epoch": 0.21, + "learning_rate": 7.985074626865672e-05, + "loss": 2.5408, + "theoretical_loss": 3.4310153968727244, + "tokens_seen": 1995636736 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1098702, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.530543804168701, + "objective/train/theoretical_loss": 3.43100599857432, + "objective/train/tokens_used": 366161376, + "theoretical_loss": 3.43100599857432, + "tokens_seen": 1995702272 + }, + { + "epoch": 0.21, + "learning_rate": 7.984272187449848e-05, + "loss": 2.6301, + "theoretical_loss": 3.4309966006709494, + "tokens_seen": 1995767808 + }, + { + "epoch": 0.21, + "learning_rate": 7.983469748034024e-05, + "loss": 2.6202, + "theoretical_loss": 3.4309778060491913, + "tokens_seen": 1995898880 + }, + { + "epoch": 0.21, + "learning_rate": 7.9826673086182e-05, + "loss": 2.5104, + "theoretical_loss": 3.430959013007213, + "tokens_seen": 1996029952 + }, + { + "epoch": 0.21, + "learning_rate": 7.981864869202376e-05, + "loss": 2.7801, + "theoretical_loss": 3.4309402215447795, + "tokens_seen": 1996161024 + }, + { + "epoch": 0.21, + "learning_rate": 7.981062429786552e-05, + "loss": 2.6228, + "theoretical_loss": 3.4309214316616528, + "tokens_seen": 1996292096 + }, + { + "epoch": 0.21, + "learning_rate": 7.980259990370727e-05, + "loss": 2.6714, + "theoretical_loss": 3.4309026433575966, + "tokens_seen": 1996423168 + }, + { + "epoch": 0.21, + "learning_rate": 7.979457550954904e-05, + "loss": 2.7458, + "theoretical_loss": 3.4308838566323754, + "tokens_seen": 1996554240 + }, + { + "epoch": 0.21, + "learning_rate": 7.978655111539079e-05, + "loss": 2.6498, + "theoretical_loss": 3.4308650714857523, + "tokens_seen": 1996685312 + }, + { + "epoch": 0.21, + "learning_rate": 7.977852672123256e-05, + "loss": 2.5197, + "theoretical_loss": 3.4308462879174915, + "tokens_seen": 1996816384 + }, + { + "epoch": 0.21, + "learning_rate": 7.977050232707431e-05, + "loss": 2.5544, + "theoretical_loss": 3.4308275059273563, + "tokens_seen": 1996947456 + }, + { + "epoch": 0.21, + "learning_rate": 7.976247793291608e-05, + "loss": 2.6762, + "theoretical_loss": 3.430808725515111, + "tokens_seen": 1997078528 + }, + { + "epoch": 0.21, + "learning_rate": 7.975445353875783e-05, + "loss": 2.636, + "theoretical_loss": 3.4307899466805187, + "tokens_seen": 1997209600 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1099661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.868697166442871, + "objective/train/theoretical_loss": 3.4307711694233443, + "objective/train/tokens_used": 367799776, + "theoretical_loss": 3.4307711694233443, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.21, + "learning_rate": 7.974642914459958e-05, + "loss": 2.6351, + "theoretical_loss": 3.4307711694233443, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.21, + "learning_rate": 7.973840475044135e-05, + "loss": 2.5566, + "theoretical_loss": 3.430752393743351, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.21, + "learning_rate": 7.97303803562831e-05, + "loss": 2.6381, + "theoretical_loss": 3.4307336196403035, + "tokens_seen": 1997602816 + }, + { + "epoch": 0.21, + "learning_rate": 7.972235596212487e-05, + "loss": 2.7568, + "theoretical_loss": 3.4307148471139657, + "tokens_seen": 1997733888 + }, + { + "epoch": 0.21, + "learning_rate": 7.971433156796662e-05, + "loss": 2.5882, + "theoretical_loss": 3.4306960761641014, + "tokens_seen": 1997864960 + }, + { + "epoch": 0.21, + "learning_rate": 7.970630717380837e-05, + "loss": 2.578, + "theoretical_loss": 3.430677306790475, + "tokens_seen": 1997996032 + }, + { + "epoch": 0.21, + "learning_rate": 7.969828277965014e-05, + "loss": 2.5211, + "theoretical_loss": 3.430658538992851, + "tokens_seen": 1998127104 + }, + { + "epoch": 0.21, + "learning_rate": 7.96902583854919e-05, + "loss": 2.6673, + "theoretical_loss": 3.4306397727709936, + "tokens_seen": 1998258176 + }, + { + "epoch": 0.21, + "learning_rate": 7.968223399133366e-05, + "loss": 2.5972, + "theoretical_loss": 3.4306210081246666, + "tokens_seen": 1998389248 + }, + { + "epoch": 0.21, + "learning_rate": 7.967420959717541e-05, + "loss": 2.7591, + "theoretical_loss": 3.4306022450536355, + "tokens_seen": 1998520320 + }, + { + "epoch": 0.21, + "learning_rate": 7.966618520301718e-05, + "loss": 2.6882, + "theoretical_loss": 3.4305834835576636, + "tokens_seen": 1998651392 + }, + { + "epoch": 0.21, + "learning_rate": 7.965816080885893e-05, + "loss": 2.7569, + "theoretical_loss": 3.4305647236365164, + "tokens_seen": 1998782464 + }, + { + "epoch": 0.21, + "learning_rate": 7.965013641470069e-05, + "loss": 2.6635, + "theoretical_loss": 3.4305459652899577, + "tokens_seen": 1998913536 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1100646, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.641911745071411, + "objective/train/theoretical_loss": 3.4305365867070754, + "objective/train/tokens_used": 369438176, + "theoretical_loss": 3.4305365867070754, + "tokens_seen": 1998979072 + }, + { + "epoch": 0.21, + "learning_rate": 7.964211202054245e-05, + "loss": 2.6049, + "theoretical_loss": 3.430527208517752, + "tokens_seen": 1999044608 + }, + { + "epoch": 0.21, + "learning_rate": 7.96340876263842e-05, + "loss": 2.576, + "theoretical_loss": 3.430508453319665, + "tokens_seen": 1999175680 + }, + { + "epoch": 0.21, + "learning_rate": 7.962606323222597e-05, + "loss": 2.6991, + "theoretical_loss": 3.4304896996954604, + "tokens_seen": 1999306752 + }, + { + "epoch": 0.21, + "learning_rate": 7.961803883806773e-05, + "loss": 2.5727, + "theoretical_loss": 3.430470947644903, + "tokens_seen": 1999437824 + }, + { + "epoch": 0.21, + "learning_rate": 7.961001444390948e-05, + "loss": 2.7093, + "theoretical_loss": 3.4304521971677584, + "tokens_seen": 1999568896 + }, + { + "epoch": 0.21, + "learning_rate": 7.960199004975125e-05, + "loss": 2.5668, + "theoretical_loss": 3.4304334482637904, + "tokens_seen": 1999699968 + }, + { + "epoch": 0.21, + "learning_rate": 7.9593965655593e-05, + "loss": 2.6501, + "theoretical_loss": 3.430414700932765, + "tokens_seen": 1999831040 + }, + { + "epoch": 0.21, + "learning_rate": 7.958594126143477e-05, + "loss": 2.5152, + "theoretical_loss": 3.430395955174446, + "tokens_seen": 1999962112 + }, + { + "epoch": 0.21, + "learning_rate": 7.957791686727652e-05, + "loss": 2.6144, + "theoretical_loss": 3.4303772109885995, + "tokens_seen": 2000093184 + }, + { + "epoch": 0.21, + "learning_rate": 7.956989247311829e-05, + "loss": 2.5636, + "theoretical_loss": 3.43035846837499, + "tokens_seen": 2000224256 + }, + { + "epoch": 0.21, + "learning_rate": 7.956186807896004e-05, + "loss": 2.5419, + "theoretical_loss": 3.430339727333383, + "tokens_seen": 2000355328 + }, + { + "epoch": 0.21, + "learning_rate": 7.955384368480179e-05, + "loss": 2.6236, + "theoretical_loss": 3.4303209878635426, + "tokens_seen": 2000486400 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1101283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.551567792892456, + "objective/train/theoretical_loss": 3.4303022499652354, + "objective/train/tokens_used": 371076576, + "theoretical_loss": 3.4303022499652354, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.21, + "learning_rate": 7.954581929064356e-05, + "loss": 2.6409, + "theoretical_loss": 3.4303022499652354, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.21, + "learning_rate": 7.953779489648531e-05, + "loss": 2.5011, + "theoretical_loss": 3.4302835136382255, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.21, + "learning_rate": 7.952977050232708e-05, + "loss": 2.5693, + "theoretical_loss": 3.430264778882279, + "tokens_seen": 2000879616 + }, + { + "epoch": 0.21, + "learning_rate": 7.952174610816883e-05, + "loss": 2.6041, + "theoretical_loss": 3.430246045697161, + "tokens_seen": 2001010688 + }, + { + "epoch": 0.21, + "learning_rate": 7.951372171401059e-05, + "loss": 2.501, + "theoretical_loss": 3.4302273140826367, + "tokens_seen": 2001141760 + }, + { + "epoch": 0.21, + "learning_rate": 7.950569731985235e-05, + "loss": 2.7269, + "theoretical_loss": 3.430208584038472, + "tokens_seen": 2001272832 + }, + { + "epoch": 0.21, + "learning_rate": 7.94976729256941e-05, + "loss": 2.5952, + "theoretical_loss": 3.430189855564432, + "tokens_seen": 2001403904 + }, + { + "epoch": 0.21, + "learning_rate": 7.948964853153587e-05, + "loss": 2.5573, + "theoretical_loss": 3.430171128660283, + "tokens_seen": 2001534976 + }, + { + "epoch": 0.21, + "learning_rate": 7.948162413737762e-05, + "loss": 2.6459, + "theoretical_loss": 3.4301524033257893, + "tokens_seen": 2001666048 + }, + { + "epoch": 0.21, + "learning_rate": 7.947359974321939e-05, + "loss": 2.7094, + "theoretical_loss": 3.4301336795607176, + "tokens_seen": 2001797120 + }, + { + "epoch": 0.21, + "learning_rate": 7.946557534906114e-05, + "loss": 2.5831, + "theoretical_loss": 3.4301149573648333, + "tokens_seen": 2001928192 + }, + { + "epoch": 0.21, + "learning_rate": 7.94575509549029e-05, + "loss": 2.5399, + "theoretical_loss": 3.430096236737902, + "tokens_seen": 2002059264 + }, + { + "epoch": 0.21, + "learning_rate": 7.944952656074466e-05, + "loss": 2.5772, + "theoretical_loss": 3.43007751767969, + "tokens_seen": 2002190336 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1102695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5117669105529785, + "objective/train/theoretical_loss": 3.4300681587387807, + "objective/train/tokens_used": 372714976, + "theoretical_loss": 3.4300681587387807, + "tokens_seen": 2002255872 + }, + { + "epoch": 0.21, + "learning_rate": 7.944150216658642e-05, + "loss": 2.5223, + "theoretical_loss": 3.430058800189963, + "tokens_seen": 2002321408 + }, + { + "epoch": 0.21, + "learning_rate": 7.943347777242818e-05, + "loss": 2.5086, + "theoretical_loss": 3.4300400842684864, + "tokens_seen": 2002452480 + }, + { + "epoch": 0.21, + "learning_rate": 7.942545337826994e-05, + "loss": 2.5602, + "theoretical_loss": 3.430021369915027, + "tokens_seen": 2002583552 + }, + { + "epoch": 0.21, + "learning_rate": 7.94174289841117e-05, + "loss": 2.6225, + "theoretical_loss": 3.43000265712935, + "tokens_seen": 2002714624 + }, + { + "epoch": 0.21, + "learning_rate": 7.940940458995346e-05, + "loss": 2.5564, + "theoretical_loss": 3.4299839459112222, + "tokens_seen": 2002845696 + }, + { + "epoch": 0.21, + "learning_rate": 7.940138019579521e-05, + "loss": 2.5846, + "theoretical_loss": 3.4299652362604087, + "tokens_seen": 2002976768 + }, + { + "epoch": 0.21, + "learning_rate": 7.939335580163698e-05, + "loss": 2.578, + "theoretical_loss": 3.429946528176677, + "tokens_seen": 2003107840 + }, + { + "epoch": 0.21, + "learning_rate": 7.938533140747873e-05, + "loss": 2.5654, + "theoretical_loss": 3.429927821659793, + "tokens_seen": 2003238912 + }, + { + "epoch": 0.21, + "learning_rate": 7.93773070133205e-05, + "loss": 2.5886, + "theoretical_loss": 3.4299091167095215, + "tokens_seen": 2003369984 + }, + { + "epoch": 0.21, + "learning_rate": 7.936928261916225e-05, + "loss": 2.5983, + "theoretical_loss": 3.42989041332563, + "tokens_seen": 2003501056 + }, + { + "epoch": 0.21, + "learning_rate": 7.936125822500402e-05, + "loss": 2.6702, + "theoretical_loss": 3.4298717115078854, + "tokens_seen": 2003632128 + }, + { + "epoch": 0.21, + "learning_rate": 7.935323383084577e-05, + "loss": 2.6316, + "theoretical_loss": 3.4298530112560535, + "tokens_seen": 2003763200 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1103208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2795019149780273, + "objective/train/theoretical_loss": 3.4298343125699002, + "objective/train/tokens_used": 374353376, + "theoretical_loss": 3.4298343125699002, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.21, + "learning_rate": 7.934520943668752e-05, + "loss": 2.4894, + "theoretical_loss": 3.4298343125699002, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.21, + "learning_rate": 7.933718504252929e-05, + "loss": 2.4966, + "theoretical_loss": 3.4298156154491934, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.21, + "learning_rate": 7.932916064837104e-05, + "loss": 2.4743, + "theoretical_loss": 3.429796919893698, + "tokens_seen": 2004156416 + }, + { + "epoch": 0.21, + "learning_rate": 7.932113625421281e-05, + "loss": 2.5501, + "theoretical_loss": 3.429778225903182, + "tokens_seen": 2004287488 + }, + { + "epoch": 0.21, + "learning_rate": 7.931311186005456e-05, + "loss": 2.5934, + "theoretical_loss": 3.4297595334774114, + "tokens_seen": 2004418560 + }, + { + "epoch": 0.21, + "learning_rate": 7.930508746589633e-05, + "loss": 2.5649, + "theoretical_loss": 3.429740842616153, + "tokens_seen": 2004549632 + }, + { + "epoch": 0.21, + "learning_rate": 7.929706307173808e-05, + "loss": 2.5895, + "theoretical_loss": 3.4297221533191737, + "tokens_seen": 2004680704 + }, + { + "epoch": 0.22, + "learning_rate": 7.928903867757984e-05, + "loss": 2.699, + "theoretical_loss": 3.4297034655862406, + "tokens_seen": 2004811776 + }, + { + "epoch": 0.22, + "learning_rate": 7.92810142834216e-05, + "loss": 2.6369, + "theoretical_loss": 3.4296847794171197, + "tokens_seen": 2004942848 + }, + { + "epoch": 0.22, + "learning_rate": 7.927298988926336e-05, + "loss": 2.5172, + "theoretical_loss": 3.4296660948115782, + "tokens_seen": 2005073920 + }, + { + "epoch": 0.22, + "learning_rate": 7.926496549510512e-05, + "loss": 2.4895, + "theoretical_loss": 3.429647411769384, + "tokens_seen": 2005204992 + }, + { + "epoch": 0.22, + "learning_rate": 7.925694110094687e-05, + "loss": 2.5339, + "theoretical_loss": 3.429628730290303, + "tokens_seen": 2005336064 + }, + { + "epoch": 0.22, + "learning_rate": 7.924891670678864e-05, + "loss": 2.7384, + "theoretical_loss": 3.429610050374103, + "tokens_seen": 2005467136 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1104357, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6726927757263184, + "objective/train/theoretical_loss": 3.4296007110020104, + "objective/train/tokens_used": 375991776, + "theoretical_loss": 3.4296007110020104, + "tokens_seen": 2005532672 + }, + { + "epoch": 0.22, + "learning_rate": 7.92408923126304e-05, + "loss": 2.5657, + "theoretical_loss": 3.4295913720205506, + "tokens_seen": 2005598208 + }, + { + "epoch": 0.22, + "learning_rate": 7.923286791847216e-05, + "loss": 2.6706, + "theoretical_loss": 3.429572695229413, + "tokens_seen": 2005729280 + }, + { + "epoch": 0.22, + "learning_rate": 7.922484352431391e-05, + "loss": 2.65, + "theoretical_loss": 3.4295540200004577, + "tokens_seen": 2005860352 + }, + { + "epoch": 0.22, + "learning_rate": 7.921681913015567e-05, + "loss": 2.5002, + "theoretical_loss": 3.429535346333452, + "tokens_seen": 2005991424 + }, + { + "epoch": 0.22, + "learning_rate": 7.920879473599743e-05, + "loss": 2.581, + "theoretical_loss": 3.4295166742281626, + "tokens_seen": 2006122496 + }, + { + "epoch": 0.22, + "learning_rate": 7.920077034183919e-05, + "loss": 2.6358, + "theoretical_loss": 3.4294980036843583, + "tokens_seen": 2006253568 + }, + { + "epoch": 0.22, + "learning_rate": 7.919274594768095e-05, + "loss": 2.5882, + "theoretical_loss": 3.4294793347018047, + "tokens_seen": 2006384640 + }, + { + "epoch": 0.22, + "learning_rate": 7.918472155352271e-05, + "loss": 2.6364, + "theoretical_loss": 3.4294606672802703, + "tokens_seen": 2006515712 + }, + { + "epoch": 0.22, + "learning_rate": 7.917669715936447e-05, + "loss": 2.4089, + "theoretical_loss": 3.4294420014195226, + "tokens_seen": 2006646784 + }, + { + "epoch": 0.22, + "learning_rate": 7.916867276520623e-05, + "loss": 2.6042, + "theoretical_loss": 3.429423337119329, + "tokens_seen": 2006777856 + }, + { + "epoch": 0.22, + "learning_rate": 7.916064837104798e-05, + "loss": 2.5733, + "theoretical_loss": 3.429404674379457, + "tokens_seen": 2006908928 + }, + { + "epoch": 0.22, + "learning_rate": 7.915262397688975e-05, + "loss": 2.5246, + "theoretical_loss": 3.4293860131996743, + "tokens_seen": 2007040000 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1104987, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.601886749267578, + "objective/train/theoretical_loss": 3.4293673535797486, + "objective/train/tokens_used": 377630176, + "theoretical_loss": 3.4293673535797486, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.22, + "learning_rate": 7.91445995827315e-05, + "loss": 2.6193, + "theoretical_loss": 3.4293673535797486, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.22, + "learning_rate": 7.913657518857327e-05, + "loss": 2.4668, + "theoretical_loss": 3.4293486955194483, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.22, + "learning_rate": 7.912855079441502e-05, + "loss": 2.5915, + "theoretical_loss": 3.42933003901854, + "tokens_seen": 2007433216 + }, + { + "epoch": 0.22, + "learning_rate": 7.912052640025679e-05, + "loss": 2.5267, + "theoretical_loss": 3.429311384076793, + "tokens_seen": 2007564288 + }, + { + "epoch": 0.22, + "learning_rate": 7.911250200609854e-05, + "loss": 2.5647, + "theoretical_loss": 3.429292730693974, + "tokens_seen": 2007695360 + }, + { + "epoch": 0.22, + "learning_rate": 7.910447761194029e-05, + "loss": 2.6728, + "theoretical_loss": 3.4292740788698515, + "tokens_seen": 2007826432 + }, + { + "epoch": 0.22, + "learning_rate": 7.909645321778206e-05, + "loss": 2.4985, + "theoretical_loss": 3.4292554286041934, + "tokens_seen": 2007957504 + }, + { + "epoch": 0.22, + "learning_rate": 7.908842882362381e-05, + "loss": 2.5317, + "theoretical_loss": 3.4292367798967676, + "tokens_seen": 2008088576 + }, + { + "epoch": 0.22, + "learning_rate": 7.908040442946558e-05, + "loss": 2.5953, + "theoretical_loss": 3.4292181327473426, + "tokens_seen": 2008219648 + }, + { + "epoch": 0.22, + "learning_rate": 7.907238003530733e-05, + "loss": 2.493, + "theoretical_loss": 3.429199487155686, + "tokens_seen": 2008350720 + }, + { + "epoch": 0.22, + "learning_rate": 7.90643556411491e-05, + "loss": 2.5788, + "theoretical_loss": 3.429180843121567, + "tokens_seen": 2008481792 + }, + { + "epoch": 0.22, + "learning_rate": 7.905633124699085e-05, + "loss": 2.4941, + "theoretical_loss": 3.429162200644753, + "tokens_seen": 2008612864 + }, + { + "epoch": 0.22, + "learning_rate": 7.90483068528326e-05, + "loss": 2.5688, + "theoretical_loss": 3.429143559725013, + "tokens_seen": 2008743936 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1106102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6869187355041504, + "objective/train/theoretical_loss": 3.429134239848973, + "objective/train/tokens_used": 379268576, + "theoretical_loss": 3.429134239848973, + "tokens_seen": 2008809472 + }, + { + "epoch": 0.22, + "learning_rate": 7.904028245867437e-05, + "loss": 2.5055, + "theoretical_loss": 3.4291249203621144, + "tokens_seen": 2008875008 + }, + { + "epoch": 0.22, + "learning_rate": 7.903225806451613e-05, + "loss": 2.5025, + "theoretical_loss": 3.429106282555826, + "tokens_seen": 2009006080 + }, + { + "epoch": 0.22, + "learning_rate": 7.902423367035789e-05, + "loss": 2.3999, + "theoretical_loss": 3.429087646305917, + "tokens_seen": 2009137152 + }, + { + "epoch": 0.22, + "learning_rate": 7.901620927619964e-05, + "loss": 2.5506, + "theoretical_loss": 3.429069011612155, + "tokens_seen": 2009268224 + }, + { + "epoch": 0.22, + "learning_rate": 7.900818488204141e-05, + "loss": 2.51, + "theoretical_loss": 3.4290503784743085, + "tokens_seen": 2009399296 + }, + { + "epoch": 0.22, + "learning_rate": 7.900016048788316e-05, + "loss": 2.5117, + "theoretical_loss": 3.429031746892147, + "tokens_seen": 2009530368 + }, + { + "epoch": 0.22, + "learning_rate": 7.899213609372492e-05, + "loss": 2.6431, + "theoretical_loss": 3.4290131168654385, + "tokens_seen": 2009661440 + }, + { + "epoch": 0.22, + "learning_rate": 7.898411169956668e-05, + "loss": 2.5644, + "theoretical_loss": 3.4289944883939514, + "tokens_seen": 2009792512 + }, + { + "epoch": 0.22, + "learning_rate": 7.897608730540844e-05, + "loss": 2.5707, + "theoretical_loss": 3.428975861477455, + "tokens_seen": 2009923584 + }, + { + "epoch": 0.22, + "learning_rate": 7.89680629112502e-05, + "loss": 2.5151, + "theoretical_loss": 3.4289572361157186, + "tokens_seen": 2010054656 + }, + { + "epoch": 0.22, + "learning_rate": 7.896003851709196e-05, + "loss": 2.5642, + "theoretical_loss": 3.42893861230851, + "tokens_seen": 2010185728 + }, + { + "epoch": 0.22, + "learning_rate": 7.895201412293372e-05, + "loss": 2.6082, + "theoretical_loss": 3.428919990055599, + "tokens_seen": 2010316800 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1107415, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5254108905792236, + "objective/train/theoretical_loss": 3.428901369356754, + "objective/train/tokens_used": 380906976, + "theoretical_loss": 3.428901369356754, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.22, + "learning_rate": 7.894398972877548e-05, + "loss": 2.5814, + "theoretical_loss": 3.428901369356754, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.22, + "learning_rate": 7.893596533461724e-05, + "loss": 2.6239, + "theoretical_loss": 3.4288827502117436, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.22, + "learning_rate": 7.8927940940459e-05, + "loss": 2.671, + "theoretical_loss": 3.4288641326203377, + "tokens_seen": 2010710016 + }, + { + "epoch": 0.22, + "learning_rate": 7.891991654630075e-05, + "loss": 2.6009, + "theoretical_loss": 3.428845516582305, + "tokens_seen": 2010841088 + }, + { + "epoch": 0.22, + "learning_rate": 7.891189215214252e-05, + "loss": 2.6173, + "theoretical_loss": 3.428826902097415, + "tokens_seen": 2010972160 + }, + { + "epoch": 0.22, + "learning_rate": 7.890386775798427e-05, + "loss": 2.5199, + "theoretical_loss": 3.4288082891654366, + "tokens_seen": 2011103232 + }, + { + "epoch": 0.22, + "learning_rate": 7.889584336382604e-05, + "loss": 2.5374, + "theoretical_loss": 3.4287896777861384, + "tokens_seen": 2011234304 + }, + { + "epoch": 0.22, + "learning_rate": 7.888781896966779e-05, + "loss": 2.5707, + "theoretical_loss": 3.4287710679592913, + "tokens_seen": 2011365376 + }, + { + "epoch": 0.22, + "learning_rate": 7.887979457550956e-05, + "loss": 2.4322, + "theoretical_loss": 3.4287524596846635, + "tokens_seen": 2011496448 + }, + { + "epoch": 0.22, + "learning_rate": 7.887177018135131e-05, + "loss": 2.4975, + "theoretical_loss": 3.4287338529620244, + "tokens_seen": 2011627520 + }, + { + "epoch": 0.22, + "learning_rate": 7.886374578719306e-05, + "loss": 2.6787, + "theoretical_loss": 3.4287152477911436, + "tokens_seen": 2011758592 + }, + { + "epoch": 0.22, + "learning_rate": 7.885572139303483e-05, + "loss": 2.4772, + "theoretical_loss": 3.428696644171791, + "tokens_seen": 2011889664 + }, + { + "epoch": 0.22, + "learning_rate": 7.884769699887658e-05, + "loss": 2.6306, + "theoretical_loss": 3.428678042103736, + "tokens_seen": 2012020736 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1108037, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8532683849334717, + "objective/train/theoretical_loss": 3.428668741651373, + "objective/train/tokens_used": 382545376, + "theoretical_loss": 3.428668741651373, + "tokens_seen": 2012086272 + }, + { + "epoch": 0.22, + "learning_rate": 7.883967260471835e-05, + "loss": 2.5644, + "theoretical_loss": 3.4286594415867477, + "tokens_seen": 2012151808 + }, + { + "epoch": 0.22, + "learning_rate": 7.88316482105601e-05, + "loss": 2.5605, + "theoretical_loss": 3.428640842620596, + "tokens_seen": 2012282880 + }, + { + "epoch": 0.22, + "learning_rate": 7.882362381640187e-05, + "loss": 2.5797, + "theoretical_loss": 3.4286222452050508, + "tokens_seen": 2012413952 + }, + { + "epoch": 0.22, + "learning_rate": 7.881559942224362e-05, + "loss": 2.5207, + "theoretical_loss": 3.428603649339882, + "tokens_seen": 2012545024 + }, + { + "epoch": 0.22, + "learning_rate": 7.880757502808538e-05, + "loss": 2.5635, + "theoretical_loss": 3.4285850550248584, + "tokens_seen": 2012676096 + }, + { + "epoch": 0.22, + "learning_rate": 7.879955063392714e-05, + "loss": 2.7104, + "theoretical_loss": 3.4285664622597514, + "tokens_seen": 2012807168 + }, + { + "epoch": 0.22, + "learning_rate": 7.87915262397689e-05, + "loss": 2.5008, + "theoretical_loss": 3.4285478710443296, + "tokens_seen": 2012938240 + }, + { + "epoch": 0.22, + "learning_rate": 7.878350184561066e-05, + "loss": 2.5471, + "theoretical_loss": 3.4285292813783634, + "tokens_seen": 2013069312 + }, + { + "epoch": 0.22, + "learning_rate": 7.877547745145242e-05, + "loss": 2.4691, + "theoretical_loss": 3.4285106932616225, + "tokens_seen": 2013200384 + }, + { + "epoch": 0.22, + "learning_rate": 7.876745305729418e-05, + "loss": 2.6136, + "theoretical_loss": 3.428492106693878, + "tokens_seen": 2013331456 + }, + { + "epoch": 0.22, + "learning_rate": 7.875942866313593e-05, + "loss": 2.4363, + "theoretical_loss": 3.4284735216748983, + "tokens_seen": 2013462528 + }, + { + "epoch": 0.22, + "learning_rate": 7.875140426897769e-05, + "loss": 2.4772, + "theoretical_loss": 3.428454938204455, + "tokens_seen": 2013593600 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1109293, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.913437604904175, + "objective/train/theoretical_loss": 3.4284363562823175, + "objective/train/tokens_used": 384183776, + "theoretical_loss": 3.4284363562823175, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.22, + "learning_rate": 7.874337987481945e-05, + "loss": 2.6037, + "theoretical_loss": 3.4284363562823175, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.22, + "learning_rate": 7.873535548066121e-05, + "loss": 2.6943, + "theoretical_loss": 3.4284177759082564, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.22, + "learning_rate": 7.872733108650297e-05, + "loss": 2.5348, + "theoretical_loss": 3.428399197082042, + "tokens_seen": 2013986816 + }, + { + "epoch": 0.22, + "learning_rate": 7.871930669234473e-05, + "loss": 2.5262, + "theoretical_loss": 3.428380619803444, + "tokens_seen": 2014117888 + }, + { + "epoch": 0.22, + "learning_rate": 7.87112822981865e-05, + "loss": 2.6206, + "theoretical_loss": 3.4283620440722333, + "tokens_seen": 2014248960 + }, + { + "epoch": 0.22, + "learning_rate": 7.870325790402825e-05, + "loss": 2.5586, + "theoretical_loss": 3.4283434698881807, + "tokens_seen": 2014380032 + }, + { + "epoch": 0.22, + "learning_rate": 7.869523350987e-05, + "loss": 2.6104, + "theoretical_loss": 3.4283248972510556, + "tokens_seen": 2014511104 + }, + { + "epoch": 0.22, + "learning_rate": 7.868720911571177e-05, + "loss": 2.5206, + "theoretical_loss": 3.42830632616063, + "tokens_seen": 2014642176 + }, + { + "epoch": 0.22, + "learning_rate": 7.867918472155352e-05, + "loss": 2.5558, + "theoretical_loss": 3.4282877566166734, + "tokens_seen": 2014773248 + }, + { + "epoch": 0.22, + "learning_rate": 7.867116032739529e-05, + "loss": 2.5839, + "theoretical_loss": 3.4282691886189562, + "tokens_seen": 2014904320 + }, + { + "epoch": 0.22, + "learning_rate": 7.866313593323704e-05, + "loss": 2.5297, + "theoretical_loss": 3.4282506221672504, + "tokens_seen": 2015035392 + }, + { + "epoch": 0.22, + "learning_rate": 7.865511153907881e-05, + "loss": 2.3923, + "theoretical_loss": 3.4282320572613254, + "tokens_seen": 2015166464 + }, + { + "epoch": 0.22, + "learning_rate": 7.864708714492056e-05, + "loss": 2.5233, + "theoretical_loss": 3.4282134939009525, + "tokens_seen": 2015297536 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1109931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3523659706115723, + "objective/train/theoretical_loss": 3.4282042128002765, + "objective/train/tokens_used": 385822176, + "theoretical_loss": 3.4282042128002765, + "tokens_seen": 2015363072 + }, + { + "epoch": 0.22, + "learning_rate": 7.863906275076233e-05, + "loss": 2.4561, + "theoretical_loss": 3.4281949320859026, + "tokens_seen": 2015428608 + }, + { + "epoch": 0.22, + "learning_rate": 7.863103835660408e-05, + "loss": 2.6495, + "theoretical_loss": 3.4281763718159466, + "tokens_seen": 2015559680 + }, + { + "epoch": 0.22, + "learning_rate": 7.862301396244583e-05, + "loss": 2.6334, + "theoretical_loss": 3.4281578130908548, + "tokens_seen": 2015690752 + }, + { + "epoch": 0.22, + "learning_rate": 7.86149895682876e-05, + "loss": 2.5526, + "theoretical_loss": 3.4281392559103994, + "tokens_seen": 2015821824 + }, + { + "epoch": 0.22, + "learning_rate": 7.860696517412935e-05, + "loss": 2.5854, + "theoretical_loss": 3.4281207002743503, + "tokens_seen": 2015952896 + }, + { + "epoch": 0.22, + "learning_rate": 7.859894077997112e-05, + "loss": 2.5802, + "theoretical_loss": 3.428102146182479, + "tokens_seen": 2016083968 + }, + { + "epoch": 0.22, + "learning_rate": 7.859091638581287e-05, + "loss": 2.4863, + "theoretical_loss": 3.4280835936345566, + "tokens_seen": 2016215040 + }, + { + "epoch": 0.22, + "learning_rate": 7.858289199165464e-05, + "loss": 2.5008, + "theoretical_loss": 3.4280650426303536, + "tokens_seen": 2016346112 + }, + { + "epoch": 0.22, + "learning_rate": 7.857486759749639e-05, + "loss": 2.5014, + "theoretical_loss": 3.428046493169643, + "tokens_seen": 2016477184 + }, + { + "epoch": 0.22, + "learning_rate": 7.856684320333815e-05, + "loss": 2.6355, + "theoretical_loss": 3.428027945252194, + "tokens_seen": 2016608256 + }, + { + "epoch": 0.22, + "learning_rate": 7.855881880917991e-05, + "loss": 2.4802, + "theoretical_loss": 3.428009398877779, + "tokens_seen": 2016739328 + }, + { + "epoch": 0.22, + "learning_rate": 7.855079441502167e-05, + "loss": 2.4685, + "theoretical_loss": 3.427990854046169, + "tokens_seen": 2016870400 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1110878, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.68257212638855, + "objective/train/theoretical_loss": 3.427972310757136, + "objective/train/tokens_used": 387460576, + "theoretical_loss": 3.427972310757136, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.22, + "learning_rate": 7.854277002086343e-05, + "loss": 2.5736, + "theoretical_loss": 3.427972310757136, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.22, + "learning_rate": 7.853474562670519e-05, + "loss": 2.4837, + "theoretical_loss": 3.4279537690104513, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.22, + "learning_rate": 7.852672123254695e-05, + "loss": 2.5344, + "theoretical_loss": 3.427935228805886, + "tokens_seen": 2017263616 + }, + { + "epoch": 0.22, + "learning_rate": 7.85186968383887e-05, + "loss": 2.5714, + "theoretical_loss": 3.4279166901432117, + "tokens_seen": 2017394688 + }, + { + "epoch": 0.22, + "learning_rate": 7.851067244423046e-05, + "loss": 2.5783, + "theoretical_loss": 3.4278981530221997, + "tokens_seen": 2017525760 + }, + { + "epoch": 0.22, + "learning_rate": 7.850264805007222e-05, + "loss": 2.4, + "theoretical_loss": 3.4278796174426223, + "tokens_seen": 2017656832 + }, + { + "epoch": 0.22, + "learning_rate": 7.849462365591398e-05, + "loss": 2.5503, + "theoretical_loss": 3.427861083404251, + "tokens_seen": 2017787904 + }, + { + "epoch": 0.22, + "learning_rate": 7.848659926175574e-05, + "loss": 2.5524, + "theoretical_loss": 3.4278425509068575, + "tokens_seen": 2017918976 + }, + { + "epoch": 0.22, + "learning_rate": 7.84785748675975e-05, + "loss": 2.5015, + "theoretical_loss": 3.4278240199502137, + "tokens_seen": 2018050048 + }, + { + "epoch": 0.22, + "learning_rate": 7.847055047343926e-05, + "loss": 2.567, + "theoretical_loss": 3.4278054905340913, + "tokens_seen": 2018181120 + }, + { + "epoch": 0.22, + "learning_rate": 7.846252607928102e-05, + "loss": 2.4222, + "theoretical_loss": 3.427786962658262, + "tokens_seen": 2018312192 + }, + { + "epoch": 0.22, + "learning_rate": 7.845450168512277e-05, + "loss": 2.6226, + "theoretical_loss": 3.4277684363224985, + "tokens_seen": 2018443264 + }, + { + "epoch": 0.22, + "learning_rate": 7.844647729096454e-05, + "loss": 2.3817, + "theoretical_loss": 3.427749911526572, + "tokens_seen": 2018574336 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1111506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.24393630027771, + "objective/train/theoretical_loss": 3.427740649705976, + "objective/train/tokens_used": 389098976, + "theoretical_loss": 3.427740649705976, + "tokens_seen": 2018639872 + }, + { + "epoch": 0.22, + "learning_rate": 7.843845289680629e-05, + "loss": 2.5344, + "theoretical_loss": 3.4277313882702547, + "tokens_seen": 2018705408 + }, + { + "epoch": 0.22, + "learning_rate": 7.843042850264806e-05, + "loss": 2.5531, + "theoretical_loss": 3.4277128665533194, + "tokens_seen": 2018836480 + }, + { + "epoch": 0.22, + "learning_rate": 7.842240410848981e-05, + "loss": 2.4688, + "theoretical_loss": 3.427694346375537, + "tokens_seen": 2018967552 + }, + { + "epoch": 0.22, + "learning_rate": 7.841437971433158e-05, + "loss": 2.4856, + "theoretical_loss": 3.4276758277366803, + "tokens_seen": 2019098624 + }, + { + "epoch": 0.22, + "learning_rate": 7.840635532017333e-05, + "loss": 2.441, + "theoretical_loss": 3.427657310636522, + "tokens_seen": 2019229696 + }, + { + "epoch": 0.22, + "learning_rate": 7.839833092601508e-05, + "loss": 2.566, + "theoretical_loss": 3.4276387950748335, + "tokens_seen": 2019360768 + }, + { + "epoch": 0.22, + "learning_rate": 7.839030653185685e-05, + "loss": 2.4114, + "theoretical_loss": 3.427620281051388, + "tokens_seen": 2019491840 + }, + { + "epoch": 0.22, + "learning_rate": 7.83822821376986e-05, + "loss": 2.4363, + "theoretical_loss": 3.4276017685659577, + "tokens_seen": 2019622912 + }, + { + "epoch": 0.22, + "learning_rate": 7.837425774354037e-05, + "loss": 2.4333, + "theoretical_loss": 3.4275832576183145, + "tokens_seen": 2019753984 + }, + { + "epoch": 0.22, + "learning_rate": 7.836623334938212e-05, + "loss": 2.5378, + "theoretical_loss": 3.4275647482082308, + "tokens_seen": 2019885056 + }, + { + "epoch": 0.22, + "learning_rate": 7.835820895522389e-05, + "loss": 2.5506, + "theoretical_loss": 3.42754624033548, + "tokens_seen": 2020016128 + }, + { + "epoch": 0.22, + "learning_rate": 7.835018456106564e-05, + "loss": 2.5029, + "theoretical_loss": 3.427527733999834, + "tokens_seen": 2020147200 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1112541, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.220839262008667, + "objective/train/theoretical_loss": 3.427509229201066, + "objective/train/tokens_used": 390737376, + "theoretical_loss": 3.427509229201066, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.22, + "learning_rate": 7.83421601669074e-05, + "loss": 2.5173, + "theoretical_loss": 3.427509229201066, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.22, + "learning_rate": 7.833413577274916e-05, + "loss": 2.5369, + "theoretical_loss": 3.4274907259389478, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.22, + "learning_rate": 7.832611137859092e-05, + "loss": 2.5579, + "theoretical_loss": 3.4274722242132523, + "tokens_seen": 2020540416 + }, + { + "epoch": 0.22, + "learning_rate": 7.831808698443268e-05, + "loss": 2.5063, + "theoretical_loss": 3.427453724023753, + "tokens_seen": 2020671488 + }, + { + "epoch": 0.22, + "learning_rate": 7.831006259027444e-05, + "loss": 2.6443, + "theoretical_loss": 3.427435225370222, + "tokens_seen": 2020802560 + }, + { + "epoch": 0.22, + "learning_rate": 7.83020381961162e-05, + "loss": 2.6026, + "theoretical_loss": 3.427416728252433, + "tokens_seen": 2020933632 + }, + { + "epoch": 0.22, + "learning_rate": 7.829401380195796e-05, + "loss": 2.4388, + "theoretical_loss": 3.427398232670158, + "tokens_seen": 2021064704 + }, + { + "epoch": 0.22, + "learning_rate": 7.828598940779972e-05, + "loss": 2.6021, + "theoretical_loss": 3.4273797386231704, + "tokens_seen": 2021195776 + }, + { + "epoch": 0.23, + "learning_rate": 7.827796501364148e-05, + "loss": 2.5579, + "theoretical_loss": 3.427361246111243, + "tokens_seen": 2021326848 + }, + { + "epoch": 0.23, + "learning_rate": 7.826994061948323e-05, + "loss": 2.4902, + "theoretical_loss": 3.4273427551341493, + "tokens_seen": 2021457920 + }, + { + "epoch": 0.23, + "learning_rate": 7.8261916225325e-05, + "loss": 2.552, + "theoretical_loss": 3.4273242656916616, + "tokens_seen": 2021588992 + }, + { + "epoch": 0.23, + "learning_rate": 7.825389183116675e-05, + "loss": 2.5983, + "theoretical_loss": 3.4273057777835536, + "tokens_seen": 2021720064 + }, + { + "epoch": 0.23, + "learning_rate": 7.824586743700851e-05, + "loss": 2.4921, + "theoretical_loss": 3.427287291409599, + "tokens_seen": 2021851136 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1113127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.679295539855957, + "objective/train/theoretical_loss": 3.4272780487978576, + "objective/train/tokens_used": 392375776, + "theoretical_loss": 3.4272780487978576, + "tokens_seen": 2021916672 + }, + { + "epoch": 0.23, + "learning_rate": 7.823784304285027e-05, + "loss": 2.5579, + "theoretical_loss": 3.42726880656957, + "tokens_seen": 2021982208 + }, + { + "epoch": 0.23, + "learning_rate": 7.822981864869203e-05, + "loss": 2.3652, + "theoretical_loss": 3.42725032326324, + "tokens_seen": 2022113280 + }, + { + "epoch": 0.23, + "learning_rate": 7.822179425453379e-05, + "loss": 2.7038, + "theoretical_loss": 3.427231841490384, + "tokens_seen": 2022244352 + }, + { + "epoch": 0.23, + "learning_rate": 7.821376986037554e-05, + "loss": 2.4067, + "theoretical_loss": 3.4272133612507734, + "tokens_seen": 2022375424 + }, + { + "epoch": 0.23, + "learning_rate": 7.820574546621731e-05, + "loss": 2.5086, + "theoretical_loss": 3.427194882544182, + "tokens_seen": 2022506496 + }, + { + "epoch": 0.23, + "learning_rate": 7.819772107205906e-05, + "loss": 2.5402, + "theoretical_loss": 3.4271764053703837, + "tokens_seen": 2022637568 + }, + { + "epoch": 0.23, + "learning_rate": 7.818969667790083e-05, + "loss": 2.4729, + "theoretical_loss": 3.4271579297291526, + "tokens_seen": 2022768640 + }, + { + "epoch": 0.23, + "learning_rate": 7.818167228374258e-05, + "loss": 2.4496, + "theoretical_loss": 3.4271394556202615, + "tokens_seen": 2022899712 + }, + { + "epoch": 0.23, + "learning_rate": 7.817364788958435e-05, + "loss": 2.3272, + "theoretical_loss": 3.4271209830434843, + "tokens_seen": 2023030784 + }, + { + "epoch": 0.23, + "learning_rate": 7.81656234954261e-05, + "loss": 2.6453, + "theoretical_loss": 3.4271025119985943, + "tokens_seen": 2023161856 + }, + { + "epoch": 0.23, + "learning_rate": 7.815759910126785e-05, + "loss": 2.5107, + "theoretical_loss": 3.4270840424853657, + "tokens_seen": 2023292928 + }, + { + "epoch": 0.23, + "learning_rate": 7.814957470710962e-05, + "loss": 2.5706, + "theoretical_loss": 3.427065574503572, + "tokens_seen": 2023424000 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1114473, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.07741641998291, + "objective/train/theoretical_loss": 3.427047108052988, + "objective/train/tokens_used": 394014176, + "theoretical_loss": 3.427047108052988, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.23, + "learning_rate": 7.814155031295137e-05, + "loss": 2.5202, + "theoretical_loss": 3.427047108052988, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.23, + "learning_rate": 7.813352591879314e-05, + "loss": 2.4878, + "theoretical_loss": 3.4270286431333856, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.23, + "learning_rate": 7.812550152463489e-05, + "loss": 2.4707, + "theoretical_loss": 3.4270101797445403, + "tokens_seen": 2023817216 + }, + { + "epoch": 0.23, + "learning_rate": 7.811747713047666e-05, + "loss": 2.4816, + "theoretical_loss": 3.4269917178862253, + "tokens_seen": 2023948288 + }, + { + "epoch": 0.23, + "learning_rate": 7.810945273631841e-05, + "loss": 2.5005, + "theoretical_loss": 3.426973257558216, + "tokens_seen": 2024079360 + }, + { + "epoch": 0.23, + "learning_rate": 7.810142834216017e-05, + "loss": 2.58, + "theoretical_loss": 3.4269547987602844, + "tokens_seen": 2024210432 + }, + { + "epoch": 0.23, + "learning_rate": 7.809340394800193e-05, + "loss": 2.5284, + "theoretical_loss": 3.4269363414922056, + "tokens_seen": 2024341504 + }, + { + "epoch": 0.23, + "learning_rate": 7.808537955384369e-05, + "loss": 2.5707, + "theoretical_loss": 3.4269178857537543, + "tokens_seen": 2024472576 + }, + { + "epoch": 0.23, + "learning_rate": 7.807735515968545e-05, + "loss": 2.5637, + "theoretical_loss": 3.426899431544704, + "tokens_seen": 2024603648 + }, + { + "epoch": 0.23, + "learning_rate": 7.80693307655272e-05, + "loss": 2.5071, + "theoretical_loss": 3.4268809788648293, + "tokens_seen": 2024734720 + }, + { + "epoch": 0.23, + "learning_rate": 7.806130637136897e-05, + "loss": 2.4704, + "theoretical_loss": 3.4268625277139044, + "tokens_seen": 2024865792 + }, + { + "epoch": 0.23, + "learning_rate": 7.805328197721073e-05, + "loss": 2.5377, + "theoretical_loss": 3.4268440780917038, + "tokens_seen": 2024996864 + }, + { + "epoch": 0.23, + "learning_rate": 7.804525758305248e-05, + "loss": 2.5348, + "theoretical_loss": 3.4268256299980013, + "tokens_seen": 2025127936 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1115722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8050100803375244, + "objective/train/theoretical_loss": 3.4268164065242663, + "objective/train/tokens_used": 395652576, + "theoretical_loss": 3.4268164065242663, + "tokens_seen": 2025193472 + }, + { + "epoch": 0.23, + "learning_rate": 7.803723318889425e-05, + "loss": 2.5304, + "theoretical_loss": 3.4268071834325715, + "tokens_seen": 2025259008 + }, + { + "epoch": 0.23, + "learning_rate": 7.8029208794736e-05, + "loss": 2.4983, + "theoretical_loss": 3.42678873839519, + "tokens_seen": 2025390080 + }, + { + "epoch": 0.23, + "learning_rate": 7.802118440057776e-05, + "loss": 2.4642, + "theoretical_loss": 3.42677029488563, + "tokens_seen": 2025521152 + }, + { + "epoch": 0.23, + "learning_rate": 7.801316000641952e-05, + "loss": 2.5015, + "theoretical_loss": 3.426751852903667, + "tokens_seen": 2025652224 + }, + { + "epoch": 0.23, + "learning_rate": 7.800513561226128e-05, + "loss": 2.3507, + "theoretical_loss": 3.4267334124490754, + "tokens_seen": 2025783296 + }, + { + "epoch": 0.23, + "learning_rate": 7.799711121810304e-05, + "loss": 2.6905, + "theoretical_loss": 3.426714973521629, + "tokens_seen": 2025914368 + }, + { + "epoch": 0.23, + "learning_rate": 7.79890868239448e-05, + "loss": 2.538, + "theoretical_loss": 3.426696536121104, + "tokens_seen": 2026045440 + }, + { + "epoch": 0.23, + "learning_rate": 7.798106242978656e-05, + "loss": 2.598, + "theoretical_loss": 3.4266781002472744, + "tokens_seen": 2026176512 + }, + { + "epoch": 0.23, + "learning_rate": 7.797303803562831e-05, + "loss": 2.4248, + "theoretical_loss": 3.4266596658999147, + "tokens_seen": 2026307584 + }, + { + "epoch": 0.23, + "learning_rate": 7.796501364147008e-05, + "loss": 2.5141, + "theoretical_loss": 3.4266412330788008, + "tokens_seen": 2026438656 + }, + { + "epoch": 0.23, + "learning_rate": 7.795698924731183e-05, + "loss": 2.3992, + "theoretical_loss": 3.426622801783707, + "tokens_seen": 2026569728 + }, + { + "epoch": 0.23, + "learning_rate": 7.79489648531536e-05, + "loss": 2.6258, + "theoretical_loss": 3.426604372014408, + "tokens_seen": 2026700800 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1116448, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.421637535095215, + "objective/train/theoretical_loss": 3.426585943770679, + "objective/train/tokens_used": 397290976, + "theoretical_loss": 3.426585943770679, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.23, + "learning_rate": 7.794094045899535e-05, + "loss": 2.5952, + "theoretical_loss": 3.426585943770679, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.23, + "learning_rate": 7.793291606483712e-05, + "loss": 2.3854, + "theoretical_loss": 3.4265675170522956, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.23, + "learning_rate": 7.792489167067887e-05, + "loss": 2.4204, + "theoretical_loss": 3.426549091859032, + "tokens_seen": 2027094016 + }, + { + "epoch": 0.23, + "learning_rate": 7.791686727652062e-05, + "loss": 2.4967, + "theoretical_loss": 3.4265306681906647, + "tokens_seen": 2027225088 + }, + { + "epoch": 0.23, + "learning_rate": 7.790884288236239e-05, + "loss": 2.5815, + "theoretical_loss": 3.4265122460469675, + "tokens_seen": 2027356160 + }, + { + "epoch": 0.23, + "learning_rate": 7.790081848820414e-05, + "loss": 2.5191, + "theoretical_loss": 3.4264938254277166, + "tokens_seen": 2027487232 + }, + { + "epoch": 0.23, + "learning_rate": 7.789279409404591e-05, + "loss": 2.5302, + "theoretical_loss": 3.4264754063326865, + "tokens_seen": 2027618304 + }, + { + "epoch": 0.23, + "learning_rate": 7.788476969988766e-05, + "loss": 2.4264, + "theoretical_loss": 3.426456988761654, + "tokens_seen": 2027749376 + }, + { + "epoch": 0.23, + "learning_rate": 7.787674530572943e-05, + "loss": 2.5857, + "theoretical_loss": 3.426438572714393, + "tokens_seen": 2027880448 + }, + { + "epoch": 0.23, + "learning_rate": 7.786872091157118e-05, + "loss": 2.2381, + "theoretical_loss": 3.4264201581906795, + "tokens_seen": 2028011520 + }, + { + "epoch": 0.23, + "learning_rate": 7.786069651741294e-05, + "loss": 2.4836, + "theoretical_loss": 3.4264017451902893, + "tokens_seen": 2028142592 + }, + { + "epoch": 0.23, + "learning_rate": 7.78526721232547e-05, + "loss": 2.5449, + "theoretical_loss": 3.426383333712997, + "tokens_seen": 2028273664 + }, + { + "epoch": 0.23, + "learning_rate": 7.784464772909646e-05, + "loss": 2.5556, + "theoretical_loss": 3.4263649237585794, + "tokens_seen": 2028404736 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1117705, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.295840263366699, + "objective/train/theoretical_loss": 3.426355719352378, + "objective/train/tokens_used": 398929376, + "theoretical_loss": 3.426355719352378, + "tokens_seen": 2028470272 + }, + { + "epoch": 0.23, + "learning_rate": 7.783662333493822e-05, + "loss": 2.5011, + "theoretical_loss": 3.4263465153268116, + "tokens_seen": 2028535808 + }, + { + "epoch": 0.23, + "learning_rate": 7.782859894077998e-05, + "loss": 2.512, + "theoretical_loss": 3.4263281084174695, + "tokens_seen": 2028666880 + }, + { + "epoch": 0.23, + "learning_rate": 7.782057454662174e-05, + "loss": 2.5264, + "theoretical_loss": 3.426309703030328, + "tokens_seen": 2028797952 + }, + { + "epoch": 0.23, + "learning_rate": 7.78125501524635e-05, + "loss": 2.483, + "theoretical_loss": 3.4262912991651637, + "tokens_seen": 2028929024 + }, + { + "epoch": 0.23, + "learning_rate": 7.780452575830525e-05, + "loss": 2.5553, + "theoretical_loss": 3.426272896821753, + "tokens_seen": 2029060096 + }, + { + "epoch": 0.23, + "learning_rate": 7.779650136414702e-05, + "loss": 2.4207, + "theoretical_loss": 3.42625449599987, + "tokens_seen": 2029191168 + }, + { + "epoch": 0.23, + "learning_rate": 7.778847696998877e-05, + "loss": 2.5201, + "theoretical_loss": 3.4262360966992924, + "tokens_seen": 2029322240 + }, + { + "epoch": 0.23, + "learning_rate": 7.778045257583053e-05, + "loss": 2.5239, + "theoretical_loss": 3.426217698919795, + "tokens_seen": 2029453312 + }, + { + "epoch": 0.23, + "learning_rate": 7.777242818167229e-05, + "loss": 2.6701, + "theoretical_loss": 3.426199302661155, + "tokens_seen": 2029584384 + }, + { + "epoch": 0.23, + "learning_rate": 7.776440378751405e-05, + "loss": 2.6428, + "theoretical_loss": 3.426180907923147, + "tokens_seen": 2029715456 + }, + { + "epoch": 0.23, + "learning_rate": 7.775637939335581e-05, + "loss": 2.6025, + "theoretical_loss": 3.426162514705548, + "tokens_seen": 2029846528 + }, + { + "epoch": 0.23, + "learning_rate": 7.774835499919756e-05, + "loss": 2.5519, + "theoretical_loss": 3.426144123008134, + "tokens_seen": 2029977600 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1118254, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5424864292144775, + "objective/train/theoretical_loss": 3.426125732830682, + "objective/train/tokens_used": 400567776, + "theoretical_loss": 3.426125732830682, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.23, + "learning_rate": 7.774033060503933e-05, + "loss": 2.4409, + "theoretical_loss": 3.426125732830682, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.23, + "learning_rate": 7.773230621088108e-05, + "loss": 2.5376, + "theoretical_loss": 3.4261073441729666, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.23, + "learning_rate": 7.772428181672285e-05, + "loss": 2.584, + "theoretical_loss": 3.4260889570347652, + "tokens_seen": 2030370816 + }, + { + "epoch": 0.23, + "learning_rate": 7.77162574225646e-05, + "loss": 2.712, + "theoretical_loss": 3.4260705714158544, + "tokens_seen": 2030501888 + }, + { + "epoch": 0.23, + "learning_rate": 7.770823302840637e-05, + "loss": 2.4428, + "theoretical_loss": 3.42605218731601, + "tokens_seen": 2030632960 + }, + { + "epoch": 0.23, + "learning_rate": 7.770020863424812e-05, + "loss": 2.7793, + "theoretical_loss": 3.4260338047350087, + "tokens_seen": 2030764032 + }, + { + "epoch": 0.23, + "learning_rate": 7.769218424008987e-05, + "loss": 2.4869, + "theoretical_loss": 3.426015423672627, + "tokens_seen": 2030895104 + }, + { + "epoch": 0.23, + "learning_rate": 7.768415984593164e-05, + "loss": 2.64, + "theoretical_loss": 3.425997044128641, + "tokens_seen": 2031026176 + }, + { + "epoch": 0.23, + "learning_rate": 7.76761354517734e-05, + "loss": 2.5178, + "theoretical_loss": 3.4259786661028278, + "tokens_seen": 2031157248 + }, + { + "epoch": 0.23, + "learning_rate": 7.766811105761516e-05, + "loss": 2.6034, + "theoretical_loss": 3.425960289594964, + "tokens_seen": 2031288320 + }, + { + "epoch": 0.23, + "learning_rate": 7.766008666345691e-05, + "loss": 2.4537, + "theoretical_loss": 3.4259419146048264, + "tokens_seen": 2031419392 + }, + { + "epoch": 0.23, + "learning_rate": 7.765206226929868e-05, + "loss": 2.5037, + "theoretical_loss": 3.425923541132191, + "tokens_seen": 2031550464 + }, + { + "epoch": 0.23, + "learning_rate": 7.764403787514043e-05, + "loss": 2.4198, + "theoretical_loss": 3.4259051691768354, + "tokens_seen": 2031681536 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1119492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9781367778778076, + "objective/train/theoretical_loss": 3.425895983768068, + "objective/train/tokens_used": 402206176, + "theoretical_loss": 3.425895983768068, + "tokens_seen": 2031747072 + }, + { + "epoch": 0.23, + "learning_rate": 7.76360134809822e-05, + "loss": 2.6343, + "theoretical_loss": 3.425886798738537, + "tokens_seen": 2031812608 + }, + { + "epoch": 0.23, + "learning_rate": 7.762798908682395e-05, + "loss": 2.4851, + "theoretical_loss": 3.4258684298170703, + "tokens_seen": 2031943680 + }, + { + "epoch": 0.23, + "learning_rate": 7.76199646926657e-05, + "loss": 2.4681, + "theoretical_loss": 3.4258500624122146, + "tokens_seen": 2032074752 + }, + { + "epoch": 0.23, + "learning_rate": 7.761194029850747e-05, + "loss": 2.6024, + "theoretical_loss": 3.425831696523746, + "tokens_seen": 2032205824 + }, + { + "epoch": 0.23, + "learning_rate": 7.760391590434923e-05, + "loss": 2.4884, + "theoretical_loss": 3.4258133321514412, + "tokens_seen": 2032336896 + }, + { + "epoch": 0.23, + "learning_rate": 7.759589151019099e-05, + "loss": 2.526, + "theoretical_loss": 3.425794969295078, + "tokens_seen": 2032467968 + }, + { + "epoch": 0.23, + "learning_rate": 7.758786711603275e-05, + "loss": 2.591, + "theoretical_loss": 3.4257766079544334, + "tokens_seen": 2032599040 + }, + { + "epoch": 0.23, + "learning_rate": 7.757984272187451e-05, + "loss": 2.6383, + "theoretical_loss": 3.4257582481292834, + "tokens_seen": 2032730112 + }, + { + "epoch": 0.23, + "learning_rate": 7.757181832771627e-05, + "loss": 2.5295, + "theoretical_loss": 3.4257398898194067, + "tokens_seen": 2032861184 + }, + { + "epoch": 0.23, + "learning_rate": 7.756379393355802e-05, + "loss": 2.5044, + "theoretical_loss": 3.4257215330245794, + "tokens_seen": 2032992256 + }, + { + "epoch": 0.23, + "learning_rate": 7.755576953939979e-05, + "loss": 2.4814, + "theoretical_loss": 3.4257031777445803, + "tokens_seen": 2033123328 + }, + { + "epoch": 0.23, + "learning_rate": 7.754774514524154e-05, + "loss": 2.6389, + "theoretical_loss": 3.425684823979185, + "tokens_seen": 2033254400 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1120104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5154623985290527, + "objective/train/theoretical_loss": 3.4256664717281717, + "objective/train/tokens_used": 403844576, + "theoretical_loss": 3.4256664717281717, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.23, + "learning_rate": 7.75397207510833e-05, + "loss": 2.6288, + "theoretical_loss": 3.4256664717281717, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.23, + "learning_rate": 7.753169635692506e-05, + "loss": 2.3721, + "theoretical_loss": 3.425648120991318, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.23, + "learning_rate": 7.752367196276682e-05, + "loss": 2.4796, + "theoretical_loss": 3.4256297717684014, + "tokens_seen": 2033647616 + }, + { + "epoch": 0.23, + "learning_rate": 7.751564756860858e-05, + "loss": 2.4372, + "theoretical_loss": 3.4256114240591993, + "tokens_seen": 2033778688 + }, + { + "epoch": 0.23, + "learning_rate": 7.750762317445033e-05, + "loss": 2.5127, + "theoretical_loss": 3.425593077863489, + "tokens_seen": 2033909760 + }, + { + "epoch": 0.23, + "learning_rate": 7.74995987802921e-05, + "loss": 2.485, + "theoretical_loss": 3.4255747331810484, + "tokens_seen": 2034040832 + }, + { + "epoch": 0.23, + "learning_rate": 7.749157438613385e-05, + "loss": 2.4735, + "theoretical_loss": 3.4255563900116552, + "tokens_seen": 2034171904 + }, + { + "epoch": 0.23, + "learning_rate": 7.748354999197562e-05, + "loss": 2.678, + "theoretical_loss": 3.4255380483550866, + "tokens_seen": 2034302976 + }, + { + "epoch": 0.23, + "learning_rate": 7.747552559781737e-05, + "loss": 2.4402, + "theoretical_loss": 3.4255197082111213, + "tokens_seen": 2034434048 + }, + { + "epoch": 0.23, + "learning_rate": 7.746750120365914e-05, + "loss": 2.557, + "theoretical_loss": 3.4255013695795364, + "tokens_seen": 2034565120 + }, + { + "epoch": 0.23, + "learning_rate": 7.745947680950089e-05, + "loss": 2.5052, + "theoretical_loss": 3.4254830324601104, + "tokens_seen": 2034696192 + }, + { + "epoch": 0.23, + "learning_rate": 7.745145241534264e-05, + "loss": 2.4159, + "theoretical_loss": 3.425464696852621, + "tokens_seen": 2034827264 + }, + { + "epoch": 0.23, + "learning_rate": 7.744342802118441e-05, + "loss": 2.4283, + "theoretical_loss": 3.425446362756845, + "tokens_seen": 2034958336 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1121212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1749119758605957, + "objective/train/theoretical_loss": 3.425437196275781, + "objective/train/tokens_used": 405482976, + "theoretical_loss": 3.425437196275781, + "tokens_seen": 2035023872 + }, + { + "epoch": 0.23, + "learning_rate": 7.743540362702616e-05, + "loss": 2.4383, + "theoretical_loss": 3.4254280301725624, + "tokens_seen": 2035089408 + }, + { + "epoch": 0.23, + "learning_rate": 7.742737923286793e-05, + "loss": 2.679, + "theoretical_loss": 3.42540969909955, + "tokens_seen": 2035220480 + }, + { + "epoch": 0.23, + "learning_rate": 7.741935483870968e-05, + "loss": 2.5305, + "theoretical_loss": 3.4253913695375857, + "tokens_seen": 2035351552 + }, + { + "epoch": 0.23, + "learning_rate": 7.741133044455145e-05, + "loss": 2.6681, + "theoretical_loss": 3.4253730414864485, + "tokens_seen": 2035482624 + }, + { + "epoch": 0.23, + "learning_rate": 7.74033060503932e-05, + "loss": 2.4899, + "theoretical_loss": 3.4253547149459163, + "tokens_seen": 2035613696 + }, + { + "epoch": 0.23, + "learning_rate": 7.739528165623496e-05, + "loss": 2.7024, + "theoretical_loss": 3.425336389915767, + "tokens_seen": 2035744768 + }, + { + "epoch": 0.23, + "learning_rate": 7.738725726207672e-05, + "loss": 2.5845, + "theoretical_loss": 3.4253180663957794, + "tokens_seen": 2035875840 + }, + { + "epoch": 0.23, + "learning_rate": 7.737923286791848e-05, + "loss": 2.458, + "theoretical_loss": 3.425299744385731, + "tokens_seen": 2036006912 + }, + { + "epoch": 0.23, + "learning_rate": 7.737120847376024e-05, + "loss": 2.6014, + "theoretical_loss": 3.425281423885401, + "tokens_seen": 2036137984 + }, + { + "epoch": 0.23, + "learning_rate": 7.7363184079602e-05, + "loss": 2.533, + "theoretical_loss": 3.425263104894568, + "tokens_seen": 2036269056 + }, + { + "epoch": 0.23, + "learning_rate": 7.735515968544376e-05, + "loss": 2.5045, + "theoretical_loss": 3.4252447874130096, + "tokens_seen": 2036400128 + }, + { + "epoch": 0.23, + "learning_rate": 7.734713529128552e-05, + "loss": 2.6825, + "theoretical_loss": 3.4252264714405047, + "tokens_seen": 2036531200 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1122341, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8938703536987305, + "objective/train/theoretical_loss": 3.425208156976832, + "objective/train/tokens_used": 407121376, + "theoretical_loss": 3.425208156976832, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.23, + "learning_rate": 7.733911089712728e-05, + "loss": 2.7145, + "theoretical_loss": 3.425208156976832, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.23, + "learning_rate": 7.733108650296904e-05, + "loss": 2.5617, + "theoretical_loss": 3.4251898440217703, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.23, + "learning_rate": 7.732306210881079e-05, + "loss": 2.5561, + "theoretical_loss": 3.4251715325750975, + "tokens_seen": 2036924416 + }, + { + "epoch": 0.23, + "learning_rate": 7.731503771465256e-05, + "loss": 2.4023, + "theoretical_loss": 3.4251532226365935, + "tokens_seen": 2037055488 + }, + { + "epoch": 0.23, + "learning_rate": 7.730701332049431e-05, + "loss": 2.6135, + "theoretical_loss": 3.425134914206036, + "tokens_seen": 2037186560 + }, + { + "epoch": 0.23, + "learning_rate": 7.729898892633608e-05, + "loss": 2.6687, + "theoretical_loss": 3.425116607283204, + "tokens_seen": 2037317632 + }, + { + "epoch": 0.23, + "learning_rate": 7.729096453217783e-05, + "loss": 2.655, + "theoretical_loss": 3.4250983018678767, + "tokens_seen": 2037448704 + }, + { + "epoch": 0.23, + "learning_rate": 7.72829401380196e-05, + "loss": 2.4404, + "theoretical_loss": 3.4250799979598328, + "tokens_seen": 2037579776 + }, + { + "epoch": 0.23, + "learning_rate": 7.727491574386135e-05, + "loss": 2.5581, + "theoretical_loss": 3.425061695558851, + "tokens_seen": 2037710848 + }, + { + "epoch": 0.24, + "learning_rate": 7.72668913497031e-05, + "loss": 2.5581, + "theoretical_loss": 3.4250433946647107, + "tokens_seen": 2037841920 + }, + { + "epoch": 0.24, + "learning_rate": 7.725886695554487e-05, + "loss": 2.5608, + "theoretical_loss": 3.4250250952771912, + "tokens_seen": 2037972992 + }, + { + "epoch": 0.24, + "learning_rate": 7.725084256138662e-05, + "loss": 2.624, + "theoretical_loss": 3.4250067973960707, + "tokens_seen": 2038104064 + }, + { + "epoch": 0.24, + "learning_rate": 7.724281816722839e-05, + "loss": 2.5705, + "theoretical_loss": 3.4249885010211285, + "tokens_seen": 2038235136 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1123098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.203213691711426, + "objective/train/theoretical_loss": 3.4249793533984056, + "objective/train/tokens_used": 408759776, + "theoretical_loss": 3.4249793533984056, + "tokens_seen": 2038300672 + }, + { + "epoch": 0.24, + "learning_rate": 7.723479377307014e-05, + "loss": 2.4835, + "theoretical_loss": 3.4249702061521443, + "tokens_seen": 2038366208 + }, + { + "epoch": 0.24, + "learning_rate": 7.72267693789119e-05, + "loss": 2.6758, + "theoretical_loss": 3.4249519127888974, + "tokens_seen": 2038497280 + }, + { + "epoch": 0.24, + "learning_rate": 7.721874498475366e-05, + "loss": 2.4282, + "theoretical_loss": 3.4249336209311667, + "tokens_seen": 2038628352 + }, + { + "epoch": 0.24, + "learning_rate": 7.721072059059541e-05, + "loss": 2.6284, + "theoretical_loss": 3.4249153305787314, + "tokens_seen": 2038759424 + }, + { + "epoch": 0.24, + "learning_rate": 7.720269619643718e-05, + "loss": 2.627, + "theoretical_loss": 3.424897041731371, + "tokens_seen": 2038890496 + }, + { + "epoch": 0.24, + "learning_rate": 7.719467180227893e-05, + "loss": 2.6302, + "theoretical_loss": 3.4248787543888652, + "tokens_seen": 2039021568 + }, + { + "epoch": 0.24, + "learning_rate": 7.71866474081207e-05, + "loss": 2.4353, + "theoretical_loss": 3.424860468550993, + "tokens_seen": 2039152640 + }, + { + "epoch": 0.24, + "learning_rate": 7.717862301396245e-05, + "loss": 2.3627, + "theoretical_loss": 3.4248421842175336, + "tokens_seen": 2039283712 + }, + { + "epoch": 0.24, + "learning_rate": 7.71705986198042e-05, + "loss": 2.5219, + "theoretical_loss": 3.424823901388268, + "tokens_seen": 2039414784 + }, + { + "epoch": 0.24, + "learning_rate": 7.716257422564597e-05, + "loss": 2.5145, + "theoretical_loss": 3.4248056200629744, + "tokens_seen": 2039545856 + }, + { + "epoch": 0.24, + "learning_rate": 7.715454983148773e-05, + "loss": 2.599, + "theoretical_loss": 3.4247873402414326, + "tokens_seen": 2039676928 + }, + { + "epoch": 0.24, + "learning_rate": 7.714652543732949e-05, + "loss": 2.5982, + "theoretical_loss": 3.4247690619234223, + "tokens_seen": 2039808000 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1124234, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5336902141571045, + "objective/train/theoretical_loss": 3.424750785108724, + "objective/train/tokens_used": 410398176, + "theoretical_loss": 3.424750785108724, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.24, + "learning_rate": 7.713850104317125e-05, + "loss": 2.4938, + "theoretical_loss": 3.424750785108724, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.24, + "learning_rate": 7.7130476649013e-05, + "loss": 2.2913, + "theoretical_loss": 3.424732509797117, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.24, + "learning_rate": 7.712245225485477e-05, + "loss": 2.5377, + "theoretical_loss": 3.4247142359883807, + "tokens_seen": 2040201216 + }, + { + "epoch": 0.24, + "learning_rate": 7.711442786069652e-05, + "loss": 2.4873, + "theoretical_loss": 3.4246959636822956, + "tokens_seen": 2040332288 + }, + { + "epoch": 0.24, + "learning_rate": 7.710640346653829e-05, + "loss": 2.3766, + "theoretical_loss": 3.4246776928786415, + "tokens_seen": 2040463360 + }, + { + "epoch": 0.24, + "learning_rate": 7.709837907238004e-05, + "loss": 2.6614, + "theoretical_loss": 3.4246594235771983, + "tokens_seen": 2040594432 + }, + { + "epoch": 0.24, + "learning_rate": 7.70903546782218e-05, + "loss": 2.5043, + "theoretical_loss": 3.4246411557777456, + "tokens_seen": 2040725504 + }, + { + "epoch": 0.24, + "learning_rate": 7.708233028406356e-05, + "loss": 2.4483, + "theoretical_loss": 3.424622889480064, + "tokens_seen": 2040856576 + }, + { + "epoch": 0.24, + "learning_rate": 7.707430588990531e-05, + "loss": 2.4879, + "theoretical_loss": 3.4246046246839335, + "tokens_seen": 2040987648 + }, + { + "epoch": 0.24, + "learning_rate": 7.706628149574708e-05, + "loss": 2.6033, + "theoretical_loss": 3.4245863613891343, + "tokens_seen": 2041118720 + }, + { + "epoch": 0.24, + "learning_rate": 7.705825710158883e-05, + "loss": 2.4924, + "theoretical_loss": 3.424568099595446, + "tokens_seen": 2041249792 + }, + { + "epoch": 0.24, + "learning_rate": 7.70502327074306e-05, + "loss": 2.6618, + "theoretical_loss": 3.4245498393026494, + "tokens_seen": 2041380864 + }, + { + "epoch": 0.24, + "learning_rate": 7.704220831327235e-05, + "loss": 2.5485, + "theoretical_loss": 3.424531580510525, + "tokens_seen": 2041511936 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1124915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3148484230041504, + "objective/train/theoretical_loss": 3.424522451677146, + "objective/train/tokens_used": 412036576, + "theoretical_loss": 3.424522451677146, + "tokens_seen": 2041577472 + }, + { + "epoch": 0.24, + "learning_rate": 7.70341839191141e-05, + "loss": 2.4715, + "theoretical_loss": 3.4245133232188527, + "tokens_seen": 2041643008 + }, + { + "epoch": 0.24, + "learning_rate": 7.702615952495587e-05, + "loss": 2.4311, + "theoretical_loss": 3.424495067427413, + "tokens_seen": 2041774080 + }, + { + "epoch": 0.24, + "learning_rate": 7.701813513079762e-05, + "loss": 2.5216, + "theoretical_loss": 3.424476813135986, + "tokens_seen": 2041905152 + }, + { + "epoch": 0.24, + "learning_rate": 7.701011073663939e-05, + "loss": 2.6073, + "theoretical_loss": 3.4244585603443527, + "tokens_seen": 2042036224 + }, + { + "epoch": 0.24, + "learning_rate": 7.700208634248114e-05, + "loss": 2.609, + "theoretical_loss": 3.4244403090522937, + "tokens_seen": 2042167296 + }, + { + "epoch": 0.24, + "learning_rate": 7.699406194832291e-05, + "loss": 2.5124, + "theoretical_loss": 3.424422059259589, + "tokens_seen": 2042298368 + }, + { + "epoch": 0.24, + "learning_rate": 7.698603755416466e-05, + "loss": 2.4612, + "theoretical_loss": 3.4244038109660195, + "tokens_seen": 2042429440 + }, + { + "epoch": 0.24, + "learning_rate": 7.697801316000642e-05, + "loss": 2.4941, + "theoretical_loss": 3.4243855641713665, + "tokens_seen": 2042560512 + }, + { + "epoch": 0.24, + "learning_rate": 7.696998876584818e-05, + "loss": 2.7861, + "theoretical_loss": 3.4243673188754093, + "tokens_seen": 2042691584 + }, + { + "epoch": 0.24, + "learning_rate": 7.696196437168994e-05, + "loss": 2.5488, + "theoretical_loss": 3.42434907507793, + "tokens_seen": 2042822656 + }, + { + "epoch": 0.24, + "learning_rate": 7.69539399775317e-05, + "loss": 2.5045, + "theoretical_loss": 3.424330832778708, + "tokens_seen": 2042953728 + }, + { + "epoch": 0.24, + "learning_rate": 7.694591558337346e-05, + "loss": 2.592, + "theoretical_loss": 3.4243125919775257, + "tokens_seen": 2043084800 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1125983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.432866096496582, + "objective/train/theoretical_loss": 3.424294352674163, + "objective/train/tokens_used": 413674976, + "theoretical_loss": 3.424294352674163, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.24, + "learning_rate": 7.693789118921521e-05, + "loss": 2.4698, + "theoretical_loss": 3.424294352674163, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.24, + "learning_rate": 7.692986679505698e-05, + "loss": 2.5398, + "theoretical_loss": 3.424276114868401, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.24, + "learning_rate": 7.692184240089873e-05, + "loss": 2.5617, + "theoretical_loss": 3.424257878560021, + "tokens_seen": 2043478016 + }, + { + "epoch": 0.24, + "learning_rate": 7.69138180067405e-05, + "loss": 2.4823, + "theoretical_loss": 3.4242396437488036, + "tokens_seen": 2043609088 + }, + { + "epoch": 0.24, + "learning_rate": 7.690579361258225e-05, + "loss": 2.4788, + "theoretical_loss": 3.42422141043453, + "tokens_seen": 2043740160 + }, + { + "epoch": 0.24, + "learning_rate": 7.689776921842402e-05, + "loss": 2.6387, + "theoretical_loss": 3.4242031786169815, + "tokens_seen": 2043871232 + }, + { + "epoch": 0.24, + "learning_rate": 7.688974482426577e-05, + "loss": 2.5571, + "theoretical_loss": 3.424184948295939, + "tokens_seen": 2044002304 + }, + { + "epoch": 0.24, + "learning_rate": 7.688172043010752e-05, + "loss": 2.6261, + "theoretical_loss": 3.4241667194711845, + "tokens_seen": 2044133376 + }, + { + "epoch": 0.24, + "learning_rate": 7.687369603594929e-05, + "loss": 2.4709, + "theoretical_loss": 3.424148492142498, + "tokens_seen": 2044264448 + }, + { + "epoch": 0.24, + "learning_rate": 7.686567164179104e-05, + "loss": 2.561, + "theoretical_loss": 3.4241302663096613, + "tokens_seen": 2044395520 + }, + { + "epoch": 0.24, + "learning_rate": 7.685764724763281e-05, + "loss": 2.473, + "theoretical_loss": 3.4241120419724567, + "tokens_seen": 2044526592 + }, + { + "epoch": 0.24, + "learning_rate": 7.684962285347456e-05, + "loss": 2.5965, + "theoretical_loss": 3.424093819130664, + "tokens_seen": 2044657664 + }, + { + "epoch": 0.24, + "learning_rate": 7.684159845931632e-05, + "loss": 2.4805, + "theoretical_loss": 3.4240755977840656, + "tokens_seen": 2044788736 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1126576, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.342308759689331, + "objective/train/theoretical_loss": 3.4240664876713955, + "objective/train/tokens_used": 415313376, + "theoretical_loss": 3.4240664876713955, + "tokens_seen": 2044854272 + }, + { + "epoch": 0.24, + "learning_rate": 7.683357406515808e-05, + "loss": 2.5274, + "theoretical_loss": 3.424057377932443, + "tokens_seen": 2044919808 + }, + { + "epoch": 0.24, + "learning_rate": 7.682554967099984e-05, + "loss": 2.5764, + "theoretical_loss": 3.4240391595755773, + "tokens_seen": 2045050880 + }, + { + "epoch": 0.24, + "learning_rate": 7.68175252768416e-05, + "loss": 2.4526, + "theoretical_loss": 3.42402094271325, + "tokens_seen": 2045181952 + }, + { + "epoch": 0.24, + "learning_rate": 7.680950088268335e-05, + "loss": 2.5081, + "theoretical_loss": 3.424002727345244, + "tokens_seen": 2045313024 + }, + { + "epoch": 0.24, + "learning_rate": 7.680147648852512e-05, + "loss": 2.4939, + "theoretical_loss": 3.423984513471339, + "tokens_seen": 2045444096 + }, + { + "epoch": 0.24, + "learning_rate": 7.679345209436687e-05, + "loss": 2.3648, + "theoretical_loss": 3.423966301091318, + "tokens_seen": 2045575168 + }, + { + "epoch": 0.24, + "learning_rate": 7.678542770020863e-05, + "loss": 2.5418, + "theoretical_loss": 3.423948090204963, + "tokens_seen": 2045706240 + }, + { + "epoch": 0.24, + "learning_rate": 7.67774033060504e-05, + "loss": 2.4757, + "theoretical_loss": 3.4239298808120546, + "tokens_seen": 2045837312 + }, + { + "epoch": 0.24, + "learning_rate": 7.676937891189215e-05, + "loss": 2.4403, + "theoretical_loss": 3.423911672912376, + "tokens_seen": 2045968384 + }, + { + "epoch": 0.24, + "learning_rate": 7.676135451773391e-05, + "loss": 2.442, + "theoretical_loss": 3.4238934665057084, + "tokens_seen": 2046099456 + }, + { + "epoch": 0.24, + "learning_rate": 7.675333012357567e-05, + "loss": 2.3285, + "theoretical_loss": 3.4238752615918333, + "tokens_seen": 2046230528 + }, + { + "epoch": 0.24, + "learning_rate": 7.674530572941742e-05, + "loss": 2.6098, + "theoretical_loss": 3.4238570581705337, + "tokens_seen": 2046361600 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1127632, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.210137128829956, + "objective/train/theoretical_loss": 3.423838856241591, + "objective/train/tokens_used": 416951776, + "theoretical_loss": 3.423838856241591, + "tokens_seen": 2046492672 + }, + { + "epoch": 0.24, + "learning_rate": 7.673728133525919e-05, + "loss": 2.4872, + "theoretical_loss": 3.423838856241591, + "tokens_seen": 2046492672 + }, + { + "epoch": 0.24, + "learning_rate": 7.672925694110094e-05, + "loss": 2.629, + "theoretical_loss": 3.4238206558047874, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.24, + "learning_rate": 7.672123254694271e-05, + "loss": 2.4547, + "theoretical_loss": 3.423802456859905, + "tokens_seen": 2046754816 + }, + { + "epoch": 0.24, + "learning_rate": 7.671320815278446e-05, + "loss": 2.4999, + "theoretical_loss": 3.423784259406726, + "tokens_seen": 2046885888 + }, + { + "epoch": 0.24, + "learning_rate": 7.670518375862623e-05, + "loss": 2.4866, + "theoretical_loss": 3.4237660634450324, + "tokens_seen": 2047016960 + }, + { + "epoch": 0.24, + "learning_rate": 7.669715936446798e-05, + "loss": 2.4262, + "theoretical_loss": 3.4237478689746066, + "tokens_seen": 2047148032 + }, + { + "epoch": 0.24, + "learning_rate": 7.668913497030975e-05, + "loss": 2.3548, + "theoretical_loss": 3.4237296759952316, + "tokens_seen": 2047279104 + }, + { + "epoch": 0.24, + "learning_rate": 7.66811105761515e-05, + "loss": 2.546, + "theoretical_loss": 3.423711484506689, + "tokens_seen": 2047410176 + }, + { + "epoch": 0.24, + "learning_rate": 7.667308618199325e-05, + "loss": 2.5654, + "theoretical_loss": 3.4236932945087615, + "tokens_seen": 2047541248 + }, + { + "epoch": 0.24, + "learning_rate": 7.666506178783502e-05, + "loss": 2.4401, + "theoretical_loss": 3.423675106001231, + "tokens_seen": 2047672320 + }, + { + "epoch": 0.24, + "learning_rate": 7.665703739367677e-05, + "loss": 2.4974, + "theoretical_loss": 3.423656918983881, + "tokens_seen": 2047803392 + }, + { + "epoch": 0.24, + "learning_rate": 7.664901299951854e-05, + "loss": 2.4328, + "theoretical_loss": 3.4236387334564933, + "tokens_seen": 2047934464 + }, + { + "epoch": 0.24, + "learning_rate": 7.664098860536029e-05, + "loss": 2.392, + "theoretical_loss": 3.42362054941885, + "tokens_seen": 2048065536 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1128264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3604929447174072, + "objective/train/theoretical_loss": 3.423611457958615, + "objective/train/tokens_used": 418590176, + "theoretical_loss": 3.423611457958615, + "tokens_seen": 2048131072 + }, + { + "epoch": 0.24, + "learning_rate": 7.663296421120206e-05, + "loss": 2.6137, + "theoretical_loss": 3.4236023668707354, + "tokens_seen": 2048196608 + }, + { + "epoch": 0.24, + "learning_rate": 7.662493981704381e-05, + "loss": 2.468, + "theoretical_loss": 3.4235841858119302, + "tokens_seen": 2048327680 + }, + { + "epoch": 0.24, + "learning_rate": 7.661691542288557e-05, + "loss": 2.5171, + "theoretical_loss": 3.4235660062422184, + "tokens_seen": 2048458752 + }, + { + "epoch": 0.24, + "learning_rate": 7.660889102872733e-05, + "loss": 2.5218, + "theoretical_loss": 3.423547828161383, + "tokens_seen": 2048589824 + }, + { + "epoch": 0.24, + "learning_rate": 7.660086663456909e-05, + "loss": 2.4335, + "theoretical_loss": 3.423529651569206, + "tokens_seen": 2048720896 + }, + { + "epoch": 0.24, + "learning_rate": 7.659284224041085e-05, + "loss": 2.4221, + "theoretical_loss": 3.42351147646547, + "tokens_seen": 2048851968 + }, + { + "epoch": 0.24, + "learning_rate": 7.65848178462526e-05, + "loss": 2.413, + "theoretical_loss": 3.423493302849959, + "tokens_seen": 2048983040 + }, + { + "epoch": 0.24, + "learning_rate": 7.657679345209437e-05, + "loss": 2.5214, + "theoretical_loss": 3.4234751307224554, + "tokens_seen": 2049114112 + }, + { + "epoch": 0.24, + "learning_rate": 7.656876905793612e-05, + "loss": 2.4643, + "theoretical_loss": 3.423456960082742, + "tokens_seen": 2049245184 + }, + { + "epoch": 0.24, + "learning_rate": 7.656074466377788e-05, + "loss": 2.4014, + "theoretical_loss": 3.423438790930602, + "tokens_seen": 2049376256 + }, + { + "epoch": 0.24, + "learning_rate": 7.655272026961964e-05, + "loss": 2.6017, + "theoretical_loss": 3.4234206232658186, + "tokens_seen": 2049507328 + }, + { + "epoch": 0.24, + "learning_rate": 7.65446958754614e-05, + "loss": 2.4636, + "theoretical_loss": 3.4234024570881747, + "tokens_seen": 2049638400 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1129400, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676476001739502, + "objective/train/theoretical_loss": 3.4233842923974542, + "objective/train/tokens_used": 420228576, + "theoretical_loss": 3.4233842923974542, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.24, + "learning_rate": 7.653667148130316e-05, + "loss": 2.6725, + "theoretical_loss": 3.4233842923974542, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.24, + "learning_rate": 7.652864708714492e-05, + "loss": 2.5167, + "theoretical_loss": 3.423366129193439, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.24, + "learning_rate": 7.652062269298668e-05, + "loss": 2.4925, + "theoretical_loss": 3.4233479674759137, + "tokens_seen": 2050031616 + }, + { + "epoch": 0.24, + "learning_rate": 7.651259829882844e-05, + "loss": 2.3167, + "theoretical_loss": 3.4233298072446607, + "tokens_seen": 2050162688 + }, + { + "epoch": 0.24, + "learning_rate": 7.650457390467019e-05, + "loss": 2.3845, + "theoretical_loss": 3.423311648499464, + "tokens_seen": 2050293760 + }, + { + "epoch": 0.24, + "learning_rate": 7.649654951051196e-05, + "loss": 2.5006, + "theoretical_loss": 3.4232934912401065, + "tokens_seen": 2050424832 + }, + { + "epoch": 0.24, + "learning_rate": 7.648852511635371e-05, + "loss": 2.4615, + "theoretical_loss": 3.4232753354663714, + "tokens_seen": 2050555904 + }, + { + "epoch": 0.24, + "learning_rate": 7.648050072219548e-05, + "loss": 2.4585, + "theoretical_loss": 3.4232571811780437, + "tokens_seen": 2050686976 + }, + { + "epoch": 0.24, + "learning_rate": 7.647247632803723e-05, + "loss": 2.5186, + "theoretical_loss": 3.423239028374905, + "tokens_seen": 2050818048 + }, + { + "epoch": 0.24, + "learning_rate": 7.6464451933879e-05, + "loss": 2.4754, + "theoretical_loss": 3.4232208770567403, + "tokens_seen": 2050949120 + }, + { + "epoch": 0.24, + "learning_rate": 7.645642753972075e-05, + "loss": 2.5151, + "theoretical_loss": 3.4232027272233325, + "tokens_seen": 2051080192 + }, + { + "epoch": 0.24, + "learning_rate": 7.64484031455625e-05, + "loss": 2.7089, + "theoretical_loss": 3.4231845788744657, + "tokens_seen": 2051211264 + }, + { + "epoch": 0.24, + "learning_rate": 7.644037875140427e-05, + "loss": 2.5746, + "theoretical_loss": 3.423166432009923, + "tokens_seen": 2051342336 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1129773, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.70103120803833, + "objective/train/theoretical_loss": 3.4231573591342057, + "objective/train/tokens_used": 421866976, + "theoretical_loss": 3.4231573591342057, + "tokens_seen": 2051407872 + }, + { + "epoch": 0.24, + "learning_rate": 7.643235435724602e-05, + "loss": 2.5127, + "theoretical_loss": 3.4231482866294884, + "tokens_seen": 2051473408 + }, + { + "epoch": 0.24, + "learning_rate": 7.642432996308779e-05, + "loss": 2.4836, + "theoretical_loss": 3.423130142732947, + "tokens_seen": 2051604480 + }, + { + "epoch": 0.24, + "learning_rate": 7.641630556892954e-05, + "loss": 2.4251, + "theoretical_loss": 3.4231120003200806, + "tokens_seen": 2051735552 + }, + { + "epoch": 0.24, + "learning_rate": 7.640828117477131e-05, + "loss": 2.48, + "theoretical_loss": 3.4230938593906743, + "tokens_seen": 2051866624 + }, + { + "epoch": 0.24, + "learning_rate": 7.640025678061306e-05, + "loss": 2.5438, + "theoretical_loss": 3.423075719944512, + "tokens_seen": 2051997696 + }, + { + "epoch": 0.24, + "learning_rate": 7.639223238645483e-05, + "loss": 2.416, + "theoretical_loss": 3.4230575819813778, + "tokens_seen": 2052128768 + }, + { + "epoch": 0.24, + "learning_rate": 7.638420799229658e-05, + "loss": 2.6944, + "theoretical_loss": 3.423039445501055, + "tokens_seen": 2052259840 + }, + { + "epoch": 0.24, + "learning_rate": 7.637618359813834e-05, + "loss": 2.3774, + "theoretical_loss": 3.423021310503328, + "tokens_seen": 2052390912 + }, + { + "epoch": 0.24, + "learning_rate": 7.63681592039801e-05, + "loss": 2.3808, + "theoretical_loss": 3.423003176987981, + "tokens_seen": 2052521984 + }, + { + "epoch": 0.24, + "learning_rate": 7.636013480982186e-05, + "loss": 2.4532, + "theoretical_loss": 3.4229850449547987, + "tokens_seen": 2052653056 + }, + { + "epoch": 0.24, + "learning_rate": 7.635211041566362e-05, + "loss": 2.5069, + "theoretical_loss": 3.422966914403565, + "tokens_seen": 2052784128 + }, + { + "epoch": 0.24, + "learning_rate": 7.634408602150538e-05, + "loss": 2.4513, + "theoretical_loss": 3.4229487853340634, + "tokens_seen": 2052915200 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1130910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.115882396697998, + "objective/train/theoretical_loss": 3.422930657746079, + "objective/train/tokens_used": 423505376, + "theoretical_loss": 3.422930657746079, + "tokens_seen": 2053046272 + }, + { + "epoch": 0.24, + "learning_rate": 7.633606162734714e-05, + "loss": 2.514, + "theoretical_loss": 3.422930657746079, + "tokens_seen": 2053046272 + }, + { + "epoch": 0.24, + "learning_rate": 7.63280372331889e-05, + "loss": 2.5533, + "theoretical_loss": 3.4229125316393967, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.24, + "learning_rate": 7.632001283903065e-05, + "loss": 2.4303, + "theoretical_loss": 3.4228944070137994, + "tokens_seen": 2053308416 + }, + { + "epoch": 0.24, + "learning_rate": 7.631198844487241e-05, + "loss": 2.5642, + "theoretical_loss": 3.4228762838690727, + "tokens_seen": 2053439488 + }, + { + "epoch": 0.24, + "learning_rate": 7.630396405071417e-05, + "loss": 2.5084, + "theoretical_loss": 3.4228581622050007, + "tokens_seen": 2053570560 + }, + { + "epoch": 0.24, + "learning_rate": 7.629593965655593e-05, + "loss": 2.5071, + "theoretical_loss": 3.422840042021368, + "tokens_seen": 2053701632 + }, + { + "epoch": 0.24, + "learning_rate": 7.628791526239769e-05, + "loss": 2.5386, + "theoretical_loss": 3.422821923317959, + "tokens_seen": 2053832704 + }, + { + "epoch": 0.24, + "learning_rate": 7.627989086823945e-05, + "loss": 2.5596, + "theoretical_loss": 3.422803806094559, + "tokens_seen": 2053963776 + }, + { + "epoch": 0.24, + "learning_rate": 7.627186647408121e-05, + "loss": 2.6299, + "theoretical_loss": 3.4227856903509517, + "tokens_seen": 2054094848 + }, + { + "epoch": 0.24, + "learning_rate": 7.626384207992296e-05, + "loss": 2.2966, + "theoretical_loss": 3.4227675760869225, + "tokens_seen": 2054225920 + }, + { + "epoch": 0.25, + "learning_rate": 7.625581768576473e-05, + "loss": 2.5026, + "theoretical_loss": 3.4227494633022557, + "tokens_seen": 2054356992 + }, + { + "epoch": 0.25, + "learning_rate": 7.624779329160648e-05, + "loss": 2.4105, + "theoretical_loss": 3.4227313519967364, + "tokens_seen": 2054488064 + }, + { + "epoch": 0.25, + "learning_rate": 7.623976889744825e-05, + "loss": 2.467, + "theoretical_loss": 3.4227132421701496, + "tokens_seen": 2054619136 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1131479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.358306884765625, + "objective/train/theoretical_loss": 3.422704187811389, + "objective/train/tokens_used": 425143776, + "theoretical_loss": 3.422704187811389, + "tokens_seen": 2054684672 + }, + { + "epoch": 0.25, + "learning_rate": 7.623174450329e-05, + "loss": 2.499, + "theoretical_loss": 3.4226951338222804, + "tokens_seen": 2054750208 + }, + { + "epoch": 0.25, + "learning_rate": 7.622372010913177e-05, + "loss": 2.6132, + "theoretical_loss": 3.422677026952913, + "tokens_seen": 2054881280 + }, + { + "epoch": 0.25, + "learning_rate": 7.621569571497352e-05, + "loss": 2.6517, + "theoretical_loss": 3.4226589215618324, + "tokens_seen": 2055012352 + }, + { + "epoch": 0.25, + "learning_rate": 7.620767132081527e-05, + "loss": 2.5089, + "theoretical_loss": 3.4226408176488246, + "tokens_seen": 2055143424 + }, + { + "epoch": 0.25, + "learning_rate": 7.619964692665704e-05, + "loss": 2.5091, + "theoretical_loss": 3.4226227152136737, + "tokens_seen": 2055274496 + }, + { + "epoch": 0.25, + "learning_rate": 7.619162253249879e-05, + "loss": 2.397, + "theoretical_loss": 3.422604614256165, + "tokens_seen": 2055405568 + }, + { + "epoch": 0.25, + "learning_rate": 7.618359813834056e-05, + "loss": 2.5668, + "theoretical_loss": 3.422586514776084, + "tokens_seen": 2055536640 + }, + { + "epoch": 0.25, + "learning_rate": 7.617557374418231e-05, + "loss": 2.6935, + "theoretical_loss": 3.422568416773216, + "tokens_seen": 2055667712 + }, + { + "epoch": 0.25, + "learning_rate": 7.616754935002408e-05, + "loss": 2.3887, + "theoretical_loss": 3.4225503202473453, + "tokens_seen": 2055798784 + }, + { + "epoch": 0.25, + "learning_rate": 7.615952495586583e-05, + "loss": 2.48, + "theoretical_loss": 3.4225322251982586, + "tokens_seen": 2055929856 + }, + { + "epoch": 0.25, + "learning_rate": 7.615150056170759e-05, + "loss": 2.3379, + "theoretical_loss": 3.42251413162574, + "tokens_seen": 2056060928 + }, + { + "epoch": 0.25, + "learning_rate": 7.614347616754935e-05, + "loss": 2.5482, + "theoretical_loss": 3.4224960395295763, + "tokens_seen": 2056192000 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1132437, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8114593029022217, + "objective/train/theoretical_loss": 3.4224779489095516, + "objective/train/tokens_used": 426782176, + "theoretical_loss": 3.4224779489095516, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.25, + "learning_rate": 7.61354517733911e-05, + "loss": 2.4689, + "theoretical_loss": 3.4224779489095516, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.25, + "learning_rate": 7.612742737923287e-05, + "loss": 2.5355, + "theoretical_loss": 3.422459859765452, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.25, + "learning_rate": 7.611940298507463e-05, + "loss": 2.4067, + "theoretical_loss": 3.422441772097063, + "tokens_seen": 2056585216 + }, + { + "epoch": 0.25, + "learning_rate": 7.611137859091639e-05, + "loss": 2.5967, + "theoretical_loss": 3.42242368590417, + "tokens_seen": 2056716288 + }, + { + "epoch": 0.25, + "learning_rate": 7.610335419675815e-05, + "loss": 2.5954, + "theoretical_loss": 3.4224056011865587, + "tokens_seen": 2056847360 + }, + { + "epoch": 0.25, + "learning_rate": 7.609532980259991e-05, + "loss": 2.5875, + "theoretical_loss": 3.4223875179440144, + "tokens_seen": 2056978432 + }, + { + "epoch": 0.25, + "learning_rate": 7.608730540844167e-05, + "loss": 2.7053, + "theoretical_loss": 3.4223694361763233, + "tokens_seen": 2057109504 + }, + { + "epoch": 0.25, + "learning_rate": 7.607928101428342e-05, + "loss": 2.5196, + "theoretical_loss": 3.422351355883271, + "tokens_seen": 2057240576 + }, + { + "epoch": 0.25, + "learning_rate": 7.607125662012518e-05, + "loss": 2.5351, + "theoretical_loss": 3.4223332770646437, + "tokens_seen": 2057371648 + }, + { + "epoch": 0.25, + "learning_rate": 7.606323222596694e-05, + "loss": 2.3518, + "theoretical_loss": 3.422315199720227, + "tokens_seen": 2057502720 + }, + { + "epoch": 0.25, + "learning_rate": 7.60552078318087e-05, + "loss": 2.4935, + "theoretical_loss": 3.422297123849806, + "tokens_seen": 2057633792 + }, + { + "epoch": 0.25, + "learning_rate": 7.604718343765046e-05, + "loss": 2.5397, + "theoretical_loss": 3.4222790494531674, + "tokens_seen": 2057764864 + }, + { + "epoch": 0.25, + "learning_rate": 7.603915904349222e-05, + "loss": 2.4483, + "theoretical_loss": 3.4222609765300973, + "tokens_seen": 2057895936 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1133547, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.867115020751953, + "objective/train/theoretical_loss": 3.4222519406210834, + "objective/train/tokens_used": 428420576, + "theoretical_loss": 3.4222519406210834, + "tokens_seen": 2057961472 + }, + { + "epoch": 0.25, + "learning_rate": 7.603113464933398e-05, + "loss": 2.597, + "theoretical_loss": 3.422242905080381, + "tokens_seen": 2058027008 + }, + { + "epoch": 0.25, + "learning_rate": 7.602311025517573e-05, + "loss": 2.6583, + "theoretical_loss": 3.4222248351038056, + "tokens_seen": 2058158080 + }, + { + "epoch": 0.25, + "learning_rate": 7.60150858610175e-05, + "loss": 2.541, + "theoretical_loss": 3.4222067666001563, + "tokens_seen": 2058289152 + }, + { + "epoch": 0.25, + "learning_rate": 7.600706146685925e-05, + "loss": 2.3219, + "theoretical_loss": 3.4221886995692197, + "tokens_seen": 2058420224 + }, + { + "epoch": 0.25, + "learning_rate": 7.599903707270102e-05, + "loss": 2.3508, + "theoretical_loss": 3.422170634010782, + "tokens_seen": 2058551296 + }, + { + "epoch": 0.25, + "learning_rate": 7.599101267854277e-05, + "loss": 2.6189, + "theoretical_loss": 3.422152569924629, + "tokens_seen": 2058682368 + }, + { + "epoch": 0.25, + "learning_rate": 7.598298828438454e-05, + "loss": 2.4071, + "theoretical_loss": 3.422134507310548, + "tokens_seen": 2058813440 + }, + { + "epoch": 0.25, + "learning_rate": 7.597496389022629e-05, + "loss": 2.3938, + "theoretical_loss": 3.4221164461683236, + "tokens_seen": 2058944512 + }, + { + "epoch": 0.25, + "learning_rate": 7.596693949606804e-05, + "loss": 2.4802, + "theoretical_loss": 3.422098386497744, + "tokens_seen": 2059075584 + }, + { + "epoch": 0.25, + "learning_rate": 7.595891510190981e-05, + "loss": 2.5144, + "theoretical_loss": 3.4220803282985948, + "tokens_seen": 2059206656 + }, + { + "epoch": 0.25, + "learning_rate": 7.595089070775156e-05, + "loss": 2.7101, + "theoretical_loss": 3.4220622715706623, + "tokens_seen": 2059337728 + }, + { + "epoch": 0.25, + "learning_rate": 7.594286631359333e-05, + "loss": 2.3732, + "theoretical_loss": 3.4220442163137332, + "tokens_seen": 2059468800 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1134157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.846436023712158, + "objective/train/theoretical_loss": 3.4220261625275943, + "objective/train/tokens_used": 430058976, + "theoretical_loss": 3.4220261625275943, + "tokens_seen": 2059599872 + }, + { + "epoch": 0.25, + "learning_rate": 7.593484191943508e-05, + "loss": 2.6596, + "theoretical_loss": 3.4220261625275943, + "tokens_seen": 2059599872 + }, + { + "epoch": 0.25, + "learning_rate": 7.592681752527685e-05, + "loss": 2.5337, + "theoretical_loss": 3.422008110212032, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.25, + "learning_rate": 7.59187931311186e-05, + "loss": 2.5602, + "theoretical_loss": 3.4219900593668324, + "tokens_seen": 2059862016 + }, + { + "epoch": 0.25, + "learning_rate": 7.591076873696036e-05, + "loss": 2.5941, + "theoretical_loss": 3.421972009991783, + "tokens_seen": 2059993088 + }, + { + "epoch": 0.25, + "learning_rate": 7.590274434280212e-05, + "loss": 2.5179, + "theoretical_loss": 3.4219539620866706, + "tokens_seen": 2060124160 + }, + { + "epoch": 0.25, + "learning_rate": 7.589471994864388e-05, + "loss": 2.6585, + "theoretical_loss": 3.4219359156512814, + "tokens_seen": 2060255232 + }, + { + "epoch": 0.25, + "learning_rate": 7.588669555448564e-05, + "loss": 2.6361, + "theoretical_loss": 3.4219178706854025, + "tokens_seen": 2060386304 + }, + { + "epoch": 0.25, + "learning_rate": 7.58786711603274e-05, + "loss": 2.5926, + "theoretical_loss": 3.4218998271888204, + "tokens_seen": 2060517376 + }, + { + "epoch": 0.25, + "learning_rate": 7.587064676616916e-05, + "loss": 2.5422, + "theoretical_loss": 3.4218817851613226, + "tokens_seen": 2060648448 + }, + { + "epoch": 0.25, + "learning_rate": 7.586262237201092e-05, + "loss": 2.7374, + "theoretical_loss": 3.4218637446026956, + "tokens_seen": 2060779520 + }, + { + "epoch": 0.25, + "learning_rate": 7.585459797785267e-05, + "loss": 2.4504, + "theoretical_loss": 3.4218457055127267, + "tokens_seen": 2060910592 + }, + { + "epoch": 0.25, + "learning_rate": 7.584657358369444e-05, + "loss": 2.5051, + "theoretical_loss": 3.421827667891203, + "tokens_seen": 2061041664 + }, + { + "epoch": 0.25, + "learning_rate": 7.583854918953619e-05, + "loss": 2.5216, + "theoretical_loss": 3.4218096317379114, + "tokens_seen": 2061172736 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1135026, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5525903701782227, + "objective/train/theoretical_loss": 3.4218006142117856, + "objective/train/tokens_used": 431697376, + "theoretical_loss": 3.4218006142117856, + "tokens_seen": 2061238272 + }, + { + "epoch": 0.25, + "learning_rate": 7.583052479537795e-05, + "loss": 2.5612, + "theoretical_loss": 3.4217915970526387, + "tokens_seen": 2061303808 + }, + { + "epoch": 0.25, + "learning_rate": 7.582250040121971e-05, + "loss": 2.5427, + "theoretical_loss": 3.4217735638351727, + "tokens_seen": 2061434880 + }, + { + "epoch": 0.25, + "learning_rate": 7.581447600706147e-05, + "loss": 2.5848, + "theoretical_loss": 3.4217555320853004, + "tokens_seen": 2061565952 + }, + { + "epoch": 0.25, + "learning_rate": 7.580645161290323e-05, + "loss": 2.6088, + "theoretical_loss": 3.4217375018028084, + "tokens_seen": 2061697024 + }, + { + "epoch": 0.25, + "learning_rate": 7.579842721874498e-05, + "loss": 2.4747, + "theoretical_loss": 3.4217194729874856, + "tokens_seen": 2061828096 + }, + { + "epoch": 0.25, + "learning_rate": 7.579040282458675e-05, + "loss": 2.652, + "theoretical_loss": 3.4217014456391177, + "tokens_seen": 2061959168 + }, + { + "epoch": 0.25, + "learning_rate": 7.57823784304285e-05, + "loss": 2.4892, + "theoretical_loss": 3.421683419757493, + "tokens_seen": 2062090240 + }, + { + "epoch": 0.25, + "learning_rate": 7.577435403627027e-05, + "loss": 2.5593, + "theoretical_loss": 3.4216653953423988, + "tokens_seen": 2062221312 + }, + { + "epoch": 0.25, + "learning_rate": 7.576632964211202e-05, + "loss": 2.5257, + "theoretical_loss": 3.4216473723936223, + "tokens_seen": 2062352384 + }, + { + "epoch": 0.25, + "learning_rate": 7.575830524795379e-05, + "loss": 2.7893, + "theoretical_loss": 3.421629350910951, + "tokens_seen": 2062483456 + }, + { + "epoch": 0.25, + "learning_rate": 7.575028085379554e-05, + "loss": 2.51, + "theoretical_loss": 3.421611330894174, + "tokens_seen": 2062614528 + }, + { + "epoch": 0.25, + "learning_rate": 7.574225645963731e-05, + "loss": 2.6444, + "theoretical_loss": 3.4215933123430764, + "tokens_seen": 2062745600 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1136115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5874297618865967, + "objective/train/theoretical_loss": 3.421575295257447, + "objective/train/tokens_used": 433335776, + "theoretical_loss": 3.421575295257447, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.25, + "learning_rate": 7.573423206547906e-05, + "loss": 2.6763, + "theoretical_loss": 3.421575295257447, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.25, + "learning_rate": 7.572620767132081e-05, + "loss": 2.6042, + "theoretical_loss": 3.4215572796370743, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.25, + "learning_rate": 7.571818327716258e-05, + "loss": 2.534, + "theoretical_loss": 3.421539265481745, + "tokens_seen": 2063138816 + }, + { + "epoch": 0.25, + "learning_rate": 7.571015888300433e-05, + "loss": 2.4402, + "theoretical_loss": 3.4215212527912477, + "tokens_seen": 2063269888 + }, + { + "epoch": 0.25, + "learning_rate": 7.57021344888461e-05, + "loss": 2.5752, + "theoretical_loss": 3.4215032415653694, + "tokens_seen": 2063400960 + }, + { + "epoch": 0.25, + "learning_rate": 7.569411009468785e-05, + "loss": 2.5612, + "theoretical_loss": 3.4214852318038984, + "tokens_seen": 2063532032 + }, + { + "epoch": 0.25, + "learning_rate": 7.568608570052962e-05, + "loss": 2.5878, + "theoretical_loss": 3.421467223506623, + "tokens_seen": 2063663104 + }, + { + "epoch": 0.25, + "learning_rate": 7.567806130637137e-05, + "loss": 2.6003, + "theoretical_loss": 3.4214492166733304, + "tokens_seen": 2063794176 + }, + { + "epoch": 0.25, + "learning_rate": 7.567003691221313e-05, + "loss": 2.5608, + "theoretical_loss": 3.4214312113038092, + "tokens_seen": 2063925248 + }, + { + "epoch": 0.25, + "learning_rate": 7.566201251805489e-05, + "loss": 2.5979, + "theoretical_loss": 3.421413207397847, + "tokens_seen": 2064056320 + }, + { + "epoch": 0.25, + "learning_rate": 7.565398812389665e-05, + "loss": 2.4841, + "theoretical_loss": 3.4213952049552327, + "tokens_seen": 2064187392 + }, + { + "epoch": 0.25, + "learning_rate": 7.564596372973841e-05, + "loss": 2.6268, + "theoretical_loss": 3.4213772039757537, + "tokens_seen": 2064318464 + }, + { + "epoch": 0.25, + "learning_rate": 7.563793933558017e-05, + "loss": 2.5995, + "theoretical_loss": 3.4213592044591987, + "tokens_seen": 2064449536 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1136653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.423219919204712, + "objective/train/theoretical_loss": 3.421350205249451, + "objective/train/tokens_used": 434974176, + "theoretical_loss": 3.421350205249451, + "tokens_seen": 2064515072 + }, + { + "epoch": 0.25, + "learning_rate": 7.562991494142193e-05, + "loss": 2.6165, + "theoretical_loss": 3.421341206405355, + "tokens_seen": 2064580608 + }, + { + "epoch": 0.25, + "learning_rate": 7.562189054726369e-05, + "loss": 2.5891, + "theoretical_loss": 3.421323209814012, + "tokens_seen": 2064711680 + }, + { + "epoch": 0.25, + "learning_rate": 7.561386615310544e-05, + "loss": 2.7013, + "theoretical_loss": 3.421305214684958, + "tokens_seen": 2064842752 + }, + { + "epoch": 0.25, + "learning_rate": 7.56058417589472e-05, + "loss": 2.492, + "theoretical_loss": 3.4212872210179803, + "tokens_seen": 2064973824 + }, + { + "epoch": 0.25, + "learning_rate": 7.559781736478896e-05, + "loss": 2.4796, + "theoretical_loss": 3.4212692288128683, + "tokens_seen": 2065104896 + }, + { + "epoch": 0.25, + "learning_rate": 7.558979297063073e-05, + "loss": 2.5791, + "theoretical_loss": 3.4212512380694102, + "tokens_seen": 2065235968 + }, + { + "epoch": 0.25, + "learning_rate": 7.558176857647248e-05, + "loss": 2.5923, + "theoretical_loss": 3.421233248787394, + "tokens_seen": 2065367040 + }, + { + "epoch": 0.25, + "learning_rate": 7.557374418231424e-05, + "loss": 2.5457, + "theoretical_loss": 3.4212152609666093, + "tokens_seen": 2065498112 + }, + { + "epoch": 0.25, + "learning_rate": 7.5565719788156e-05, + "loss": 2.4568, + "theoretical_loss": 3.4211972746068438, + "tokens_seen": 2065629184 + }, + { + "epoch": 0.25, + "learning_rate": 7.555769539399775e-05, + "loss": 2.6654, + "theoretical_loss": 3.4211792897078865, + "tokens_seen": 2065760256 + }, + { + "epoch": 0.25, + "learning_rate": 7.554967099983952e-05, + "loss": 2.5968, + "theoretical_loss": 3.421161306269526, + "tokens_seen": 2065891328 + }, + { + "epoch": 0.25, + "learning_rate": 7.554164660568127e-05, + "loss": 2.6777, + "theoretical_loss": 3.4211433242915508, + "tokens_seen": 2066022400 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1137651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5705416202545166, + "objective/train/theoretical_loss": 3.42112534377375, + "objective/train/tokens_used": 436612576, + "theoretical_loss": 3.42112534377375, + "tokens_seen": 2066153472 + }, + { + "epoch": 0.25, + "learning_rate": 7.553362221152304e-05, + "loss": 2.4683, + "theoretical_loss": 3.42112534377375, + "tokens_seen": 2066153472 + }, + { + "epoch": 0.25, + "learning_rate": 7.552559781736479e-05, + "loss": 2.6254, + "theoretical_loss": 3.4211073647159123, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.25, + "learning_rate": 7.551757342320656e-05, + "loss": 2.5953, + "theoretical_loss": 3.4210893871178265, + "tokens_seen": 2066415616 + }, + { + "epoch": 0.25, + "learning_rate": 7.550954902904831e-05, + "loss": 2.5553, + "theoretical_loss": 3.4210714109792817, + "tokens_seen": 2066546688 + }, + { + "epoch": 0.25, + "learning_rate": 7.550152463489006e-05, + "loss": 2.7196, + "theoretical_loss": 3.421053436300067, + "tokens_seen": 2066677760 + }, + { + "epoch": 0.25, + "learning_rate": 7.549350024073183e-05, + "loss": 2.4659, + "theoretical_loss": 3.4210354630799706, + "tokens_seen": 2066808832 + }, + { + "epoch": 0.25, + "learning_rate": 7.548547584657358e-05, + "loss": 2.5927, + "theoretical_loss": 3.421017491318782, + "tokens_seen": 2066939904 + }, + { + "epoch": 0.25, + "learning_rate": 7.547745145241535e-05, + "loss": 2.6497, + "theoretical_loss": 3.4209995210162907, + "tokens_seen": 2067070976 + }, + { + "epoch": 0.25, + "learning_rate": 7.54694270582571e-05, + "loss": 2.6517, + "theoretical_loss": 3.420981552172285, + "tokens_seen": 2067202048 + }, + { + "epoch": 0.25, + "learning_rate": 7.546140266409887e-05, + "loss": 2.595, + "theoretical_loss": 3.4209635847865547, + "tokens_seen": 2067333120 + }, + { + "epoch": 0.25, + "learning_rate": 7.545337826994062e-05, + "loss": 2.6581, + "theoretical_loss": 3.420945618858889, + "tokens_seen": 2067464192 + }, + { + "epoch": 0.25, + "learning_rate": 7.544535387578239e-05, + "loss": 2.6037, + "theoretical_loss": 3.420927654389077, + "tokens_seen": 2067595264 + }, + { + "epoch": 0.25, + "learning_rate": 7.543732948162414e-05, + "loss": 2.6536, + "theoretical_loss": 3.4209096913769073, + "tokens_seen": 2067726336 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1138992, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7702243328094482, + "objective/train/theoretical_loss": 3.420900710417373, + "objective/train/tokens_used": 438250976, + "theoretical_loss": 3.420900710417373, + "tokens_seen": 2067791872 + }, + { + "epoch": 0.25, + "learning_rate": 7.54293050874659e-05, + "loss": 2.6081, + "theoretical_loss": 3.4208917298221704, + "tokens_seen": 2067857408 + }, + { + "epoch": 0.25, + "learning_rate": 7.542128069330766e-05, + "loss": 2.7191, + "theoretical_loss": 3.420873769724655, + "tokens_seen": 2067988480 + }, + { + "epoch": 0.25, + "learning_rate": 7.541325629914942e-05, + "loss": 2.6877, + "theoretical_loss": 3.4208558110841505, + "tokens_seen": 2068119552 + }, + { + "epoch": 0.25, + "learning_rate": 7.540523190499118e-05, + "loss": 2.628, + "theoretical_loss": 3.420837853900447, + "tokens_seen": 2068250624 + }, + { + "epoch": 0.25, + "learning_rate": 7.539720751083294e-05, + "loss": 2.6661, + "theoretical_loss": 3.420819898173333, + "tokens_seen": 2068381696 + }, + { + "epoch": 0.25, + "learning_rate": 7.53891831166747e-05, + "loss": 2.7085, + "theoretical_loss": 3.4208019439025987, + "tokens_seen": 2068512768 + }, + { + "epoch": 0.25, + "learning_rate": 7.538115872251646e-05, + "loss": 2.6818, + "theoretical_loss": 3.420783991088034, + "tokens_seen": 2068643840 + }, + { + "epoch": 0.25, + "learning_rate": 7.537313432835821e-05, + "loss": 2.6039, + "theoretical_loss": 3.420766039729428, + "tokens_seen": 2068774912 + }, + { + "epoch": 0.25, + "learning_rate": 7.536510993419998e-05, + "loss": 2.6936, + "theoretical_loss": 3.4207480898265707, + "tokens_seen": 2068905984 + }, + { + "epoch": 0.25, + "learning_rate": 7.535708554004173e-05, + "loss": 2.5812, + "theoretical_loss": 3.4207301413792512, + "tokens_seen": 2069037056 + }, + { + "epoch": 0.25, + "learning_rate": 7.53490611458835e-05, + "loss": 2.5736, + "theoretical_loss": 3.4207121943872605, + "tokens_seen": 2069168128 + }, + { + "epoch": 0.25, + "learning_rate": 7.534103675172525e-05, + "loss": 2.5398, + "theoretical_loss": 3.4206942488503875, + "tokens_seen": 2069299200 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1139620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4647276401519775, + "objective/train/theoretical_loss": 3.420676304768422, + "objective/train/tokens_used": 439889376, + "theoretical_loss": 3.420676304768422, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.25, + "learning_rate": 7.533301235756701e-05, + "loss": 2.4302, + "theoretical_loss": 3.420676304768422, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.25, + "learning_rate": 7.532498796340877e-05, + "loss": 2.4488, + "theoretical_loss": 3.420658362141154, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.25, + "learning_rate": 7.531696356925052e-05, + "loss": 2.548, + "theoretical_loss": 3.4206404209683736, + "tokens_seen": 2069692416 + }, + { + "epoch": 0.25, + "learning_rate": 7.530893917509229e-05, + "loss": 2.6237, + "theoretical_loss": 3.4206224812498713, + "tokens_seen": 2069823488 + }, + { + "epoch": 0.25, + "learning_rate": 7.530091478093404e-05, + "loss": 2.6025, + "theoretical_loss": 3.420604542985436, + "tokens_seen": 2069954560 + }, + { + "epoch": 0.25, + "learning_rate": 7.529289038677581e-05, + "loss": 2.6769, + "theoretical_loss": 3.420586606174859, + "tokens_seen": 2070085632 + }, + { + "epoch": 0.25, + "learning_rate": 7.528486599261756e-05, + "loss": 2.6906, + "theoretical_loss": 3.4205686708179295, + "tokens_seen": 2070216704 + }, + { + "epoch": 0.25, + "learning_rate": 7.527684159845933e-05, + "loss": 2.6362, + "theoretical_loss": 3.4205507369144383, + "tokens_seen": 2070347776 + }, + { + "epoch": 0.25, + "learning_rate": 7.526881720430108e-05, + "loss": 2.5741, + "theoretical_loss": 3.420532804464175, + "tokens_seen": 2070478848 + }, + { + "epoch": 0.25, + "learning_rate": 7.526079281014283e-05, + "loss": 2.5953, + "theoretical_loss": 3.4205148734669306, + "tokens_seen": 2070609920 + }, + { + "epoch": 0.26, + "learning_rate": 7.52527684159846e-05, + "loss": 2.5617, + "theoretical_loss": 3.420496943922495, + "tokens_seen": 2070740992 + }, + { + "epoch": 0.26, + "learning_rate": 7.524474402182635e-05, + "loss": 2.6418, + "theoretical_loss": 3.420479015830658, + "tokens_seen": 2070872064 + }, + { + "epoch": 0.26, + "learning_rate": 7.523671962766812e-05, + "loss": 2.5491, + "theoretical_loss": 3.4204610891912104, + "tokens_seen": 2071003136 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1140715, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.805562973022461, + "objective/train/theoretical_loss": 3.4204521264160674, + "objective/train/tokens_used": 441527776, + "theoretical_loss": 3.4204521264160674, + "tokens_seen": 2071068672 + }, + { + "epoch": 0.26, + "learning_rate": 7.522869523350987e-05, + "loss": 2.6671, + "theoretical_loss": 3.4204431640039434, + "tokens_seen": 2071134208 + }, + { + "epoch": 0.26, + "learning_rate": 7.522067083935164e-05, + "loss": 2.6024, + "theoretical_loss": 3.4204252402686466, + "tokens_seen": 2071265280 + }, + { + "epoch": 0.26, + "learning_rate": 7.521264644519339e-05, + "loss": 2.6613, + "theoretical_loss": 3.42040731798511, + "tokens_seen": 2071396352 + }, + { + "epoch": 0.26, + "learning_rate": 7.520462205103515e-05, + "loss": 2.5674, + "theoretical_loss": 3.4203893971531256, + "tokens_seen": 2071527424 + }, + { + "epoch": 0.26, + "learning_rate": 7.519659765687691e-05, + "loss": 2.6061, + "theoretical_loss": 3.420371477772483, + "tokens_seen": 2071658496 + }, + { + "epoch": 0.26, + "learning_rate": 7.518857326271867e-05, + "loss": 2.6192, + "theoretical_loss": 3.4203535598429737, + "tokens_seen": 2071789568 + }, + { + "epoch": 0.26, + "learning_rate": 7.518054886856043e-05, + "loss": 2.5439, + "theoretical_loss": 3.4203356433643877, + "tokens_seen": 2071920640 + }, + { + "epoch": 0.26, + "learning_rate": 7.517252447440219e-05, + "loss": 2.5913, + "theoretical_loss": 3.4203177283365154, + "tokens_seen": 2072051712 + }, + { + "epoch": 0.26, + "learning_rate": 7.516450008024395e-05, + "loss": 2.5747, + "theoretical_loss": 3.420299814759148, + "tokens_seen": 2072182784 + }, + { + "epoch": 0.26, + "learning_rate": 7.51564756860857e-05, + "loss": 2.4535, + "theoretical_loss": 3.4202819026320768, + "tokens_seen": 2072313856 + }, + { + "epoch": 0.26, + "learning_rate": 7.514845129192747e-05, + "loss": 2.5588, + "theoretical_loss": 3.4202639919550926, + "tokens_seen": 2072444928 + }, + { + "epoch": 0.26, + "learning_rate": 7.514042689776923e-05, + "loss": 2.5327, + "theoretical_loss": 3.420246082727985, + "tokens_seen": 2072576000 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1141285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8016202449798584, + "objective/train/theoretical_loss": 3.4202281749505463, + "objective/train/tokens_used": 443166176, + "theoretical_loss": 3.4202281749505463, + "tokens_seen": 2072707072 + }, + { + "epoch": 0.26, + "learning_rate": 7.513240250361098e-05, + "loss": 2.6386, + "theoretical_loss": 3.4202281749505463, + "tokens_seen": 2072707072 + }, + { + "epoch": 0.26, + "learning_rate": 7.512437810945275e-05, + "loss": 2.6437, + "theoretical_loss": 3.420210268622567, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.26, + "learning_rate": 7.51163537152945e-05, + "loss": 2.5769, + "theoretical_loss": 3.4201923637438383, + "tokens_seen": 2072969216 + }, + { + "epoch": 0.26, + "learning_rate": 7.510832932113627e-05, + "loss": 2.5697, + "theoretical_loss": 3.4201744603141515, + "tokens_seen": 2073100288 + }, + { + "epoch": 0.26, + "learning_rate": 7.510030492697802e-05, + "loss": 2.6594, + "theoretical_loss": 3.420156558333297, + "tokens_seen": 2073231360 + }, + { + "epoch": 0.26, + "learning_rate": 7.509228053281979e-05, + "loss": 2.5883, + "theoretical_loss": 3.420138657801066, + "tokens_seen": 2073362432 + }, + { + "epoch": 0.26, + "learning_rate": 7.508425613866154e-05, + "loss": 2.6873, + "theoretical_loss": 3.4201207587172506, + "tokens_seen": 2073493504 + }, + { + "epoch": 0.26, + "learning_rate": 7.507623174450329e-05, + "loss": 2.4776, + "theoretical_loss": 3.4201028610816415, + "tokens_seen": 2073624576 + }, + { + "epoch": 0.26, + "learning_rate": 7.506820735034506e-05, + "loss": 2.6836, + "theoretical_loss": 3.4200849648940306, + "tokens_seen": 2073755648 + }, + { + "epoch": 0.26, + "learning_rate": 7.506018295618681e-05, + "loss": 2.4895, + "theoretical_loss": 3.420067070154208, + "tokens_seen": 2073886720 + }, + { + "epoch": 0.26, + "learning_rate": 7.505215856202858e-05, + "loss": 2.6642, + "theoretical_loss": 3.420049176861966, + "tokens_seen": 2074017792 + }, + { + "epoch": 0.26, + "learning_rate": 7.504413416787033e-05, + "loss": 2.6899, + "theoretical_loss": 3.420031285017096, + "tokens_seen": 2074148864 + }, + { + "epoch": 0.26, + "learning_rate": 7.50361097737121e-05, + "loss": 2.5657, + "theoretical_loss": 3.4200133946193887, + "tokens_seen": 2074279936 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1142545, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676924228668213, + "objective/train/theoretical_loss": 3.420004449963156, + "objective/train/tokens_used": 444804576, + "theoretical_loss": 3.420004449963156, + "tokens_seen": 2074345472 + }, + { + "epoch": 0.26, + "learning_rate": 7.502808537955385e-05, + "loss": 2.5201, + "theoretical_loss": 3.419995505668637, + "tokens_seen": 2074411008 + }, + { + "epoch": 0.26, + "learning_rate": 7.50200609853956e-05, + "loss": 2.8034, + "theoretical_loss": 3.419977618164631, + "tokens_seen": 2074542080 + }, + { + "epoch": 0.26, + "learning_rate": 7.501203659123737e-05, + "loss": 2.6983, + "theoretical_loss": 3.419959732107163, + "tokens_seen": 2074673152 + }, + { + "epoch": 0.26, + "learning_rate": 7.500401219707912e-05, + "loss": 2.6096, + "theoretical_loss": 3.419941847496025, + "tokens_seen": 2074804224 + }, + { + "epoch": 0.26, + "learning_rate": 7.499598780292089e-05, + "loss": 2.5876, + "theoretical_loss": 3.4199239643310078, + "tokens_seen": 2074935296 + }, + { + "epoch": 0.26, + "learning_rate": 7.498796340876264e-05, + "loss": 2.69, + "theoretical_loss": 3.4199060826119037, + "tokens_seen": 2075066368 + }, + { + "epoch": 0.26, + "learning_rate": 7.497993901460441e-05, + "loss": 2.7312, + "theoretical_loss": 3.4198882023385044, + "tokens_seen": 2075197440 + }, + { + "epoch": 0.26, + "learning_rate": 7.497191462044616e-05, + "loss": 2.5642, + "theoretical_loss": 3.419870323510602, + "tokens_seen": 2075328512 + }, + { + "epoch": 0.26, + "learning_rate": 7.496389022628792e-05, + "loss": 2.6408, + "theoretical_loss": 3.4198524461279876, + "tokens_seen": 2075459584 + }, + { + "epoch": 0.26, + "learning_rate": 7.495586583212968e-05, + "loss": 2.5564, + "theoretical_loss": 3.419834570190454, + "tokens_seen": 2075590656 + }, + { + "epoch": 0.26, + "learning_rate": 7.494784143797144e-05, + "loss": 2.6452, + "theoretical_loss": 3.4198166956977927, + "tokens_seen": 2075721728 + }, + { + "epoch": 0.26, + "learning_rate": 7.49398170438132e-05, + "loss": 2.7101, + "theoretical_loss": 3.4197988226497955, + "tokens_seen": 2075852800 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1143739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.8349756002426147, + "objective/train/theoretical_loss": 3.4197809510462545, + "objective/train/tokens_used": 446442976, + "theoretical_loss": 3.4197809510462545, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.26, + "learning_rate": 7.493179264965496e-05, + "loss": 2.4385, + "theoretical_loss": 3.4197809510462545, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.26, + "learning_rate": 7.492376825549672e-05, + "loss": 2.5877, + "theoretical_loss": 3.4197630808869617, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.26, + "learning_rate": 7.491574386133848e-05, + "loss": 2.6309, + "theoretical_loss": 3.41974521217171, + "tokens_seen": 2076246016 + }, + { + "epoch": 0.26, + "learning_rate": 7.490771946718023e-05, + "loss": 2.5745, + "theoretical_loss": 3.4197273449002905, + "tokens_seen": 2076377088 + }, + { + "epoch": 0.26, + "learning_rate": 7.4899695073022e-05, + "loss": 2.5886, + "theoretical_loss": 3.419709479072496, + "tokens_seen": 2076508160 + }, + { + "epoch": 0.26, + "learning_rate": 7.489167067886375e-05, + "loss": 2.5897, + "theoretical_loss": 3.4196916146881184, + "tokens_seen": 2076639232 + }, + { + "epoch": 0.26, + "learning_rate": 7.488364628470552e-05, + "loss": 2.5993, + "theoretical_loss": 3.419673751746951, + "tokens_seen": 2076770304 + }, + { + "epoch": 0.26, + "learning_rate": 7.487562189054727e-05, + "loss": 2.5253, + "theoretical_loss": 3.4196558902487846, + "tokens_seen": 2076901376 + }, + { + "epoch": 0.26, + "learning_rate": 7.486759749638904e-05, + "loss": 2.5155, + "theoretical_loss": 3.419638030193413, + "tokens_seen": 2077032448 + }, + { + "epoch": 0.26, + "learning_rate": 7.485957310223079e-05, + "loss": 2.5878, + "theoretical_loss": 3.419620171580627, + "tokens_seen": 2077163520 + }, + { + "epoch": 0.26, + "learning_rate": 7.485154870807254e-05, + "loss": 2.5929, + "theoretical_loss": 3.419602314410221, + "tokens_seen": 2077294592 + }, + { + "epoch": 0.26, + "learning_rate": 7.484352431391431e-05, + "loss": 2.5011, + "theoretical_loss": 3.4195844586819857, + "tokens_seen": 2077425664 + }, + { + "epoch": 0.26, + "learning_rate": 7.483549991975606e-05, + "loss": 2.5385, + "theoretical_loss": 3.419566604395715, + "tokens_seen": 2077556736 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1144416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.198617935180664, + "objective/train/theoretical_loss": 3.419557677793251, + "objective/train/tokens_used": 448081376, + "theoretical_loss": 3.419557677793251, + "tokens_seen": 2077622272 + }, + { + "epoch": 0.26, + "learning_rate": 7.482747552559783e-05, + "loss": 2.745, + "theoretical_loss": 3.4195487515512006, + "tokens_seen": 2077687808 + }, + { + "epoch": 0.26, + "learning_rate": 7.481945113143958e-05, + "loss": 2.7385, + "theoretical_loss": 3.4195309001482355, + "tokens_seen": 2077818880 + }, + { + "epoch": 0.26, + "learning_rate": 7.481142673728135e-05, + "loss": 2.5999, + "theoretical_loss": 3.4195130501866124, + "tokens_seen": 2077949952 + }, + { + "epoch": 0.26, + "learning_rate": 7.48034023431231e-05, + "loss": 2.5005, + "theoretical_loss": 3.419495201666124, + "tokens_seen": 2078081024 + }, + { + "epoch": 0.26, + "learning_rate": 7.479537794896487e-05, + "loss": 2.6459, + "theoretical_loss": 3.4194773545865633, + "tokens_seen": 2078212096 + }, + { + "epoch": 0.26, + "learning_rate": 7.478735355480662e-05, + "loss": 2.5881, + "theoretical_loss": 3.419459508947723, + "tokens_seen": 2078343168 + }, + { + "epoch": 0.26, + "learning_rate": 7.477932916064837e-05, + "loss": 2.5664, + "theoretical_loss": 3.419441664749395, + "tokens_seen": 2078474240 + }, + { + "epoch": 0.26, + "learning_rate": 7.477130476649014e-05, + "loss": 2.6161, + "theoretical_loss": 3.4194238219913737, + "tokens_seen": 2078605312 + }, + { + "epoch": 0.26, + "learning_rate": 7.47632803723319e-05, + "loss": 2.4891, + "theoretical_loss": 3.4194059806734507, + "tokens_seen": 2078736384 + }, + { + "epoch": 0.26, + "learning_rate": 7.475525597817366e-05, + "loss": 2.4439, + "theoretical_loss": 3.41938814079542, + "tokens_seen": 2078867456 + }, + { + "epoch": 0.26, + "learning_rate": 7.474723158401541e-05, + "loss": 2.6349, + "theoretical_loss": 3.4193703023570743, + "tokens_seen": 2078998528 + }, + { + "epoch": 0.26, + "learning_rate": 7.473920718985718e-05, + "loss": 2.5836, + "theoretical_loss": 3.419352465358206, + "tokens_seen": 2079129600 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1145721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.877591133117676, + "objective/train/theoretical_loss": 3.4193346297986094, + "objective/train/tokens_used": 449719776, + "theoretical_loss": 3.4193346297986094, + "tokens_seen": 2079260672 + }, + { + "epoch": 0.26, + "learning_rate": 7.473118279569893e-05, + "loss": 2.5798, + "theoretical_loss": 3.4193346297986094, + "tokens_seen": 2079260672 + }, + { + "epoch": 0.26, + "learning_rate": 7.472315840154069e-05, + "loss": 2.6641, + "theoretical_loss": 3.4193167956780766, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.26, + "learning_rate": 7.471513400738245e-05, + "loss": 2.5415, + "theoretical_loss": 3.4192989629964012, + "tokens_seen": 2079522816 + }, + { + "epoch": 0.26, + "learning_rate": 7.47071096132242e-05, + "loss": 2.5896, + "theoretical_loss": 3.419281131753377, + "tokens_seen": 2079653888 + }, + { + "epoch": 0.26, + "learning_rate": 7.469908521906597e-05, + "loss": 2.5036, + "theoretical_loss": 3.4192633019487966, + "tokens_seen": 2079784960 + }, + { + "epoch": 0.26, + "learning_rate": 7.469106082490773e-05, + "loss": 2.5731, + "theoretical_loss": 3.419245473582453, + "tokens_seen": 2079916032 + }, + { + "epoch": 0.26, + "learning_rate": 7.468303643074949e-05, + "loss": 2.6267, + "theoretical_loss": 3.4192276466541403, + "tokens_seen": 2080047104 + }, + { + "epoch": 0.26, + "learning_rate": 7.467501203659125e-05, + "loss": 2.4308, + "theoretical_loss": 3.4192098211636512, + "tokens_seen": 2080178176 + }, + { + "epoch": 0.26, + "learning_rate": 7.4666987642433e-05, + "loss": 2.3999, + "theoretical_loss": 3.41919199711078, + "tokens_seen": 2080309248 + }, + { + "epoch": 0.26, + "learning_rate": 7.465896324827477e-05, + "loss": 2.392, + "theoretical_loss": 3.4191741744953195, + "tokens_seen": 2080440320 + }, + { + "epoch": 0.26, + "learning_rate": 7.465093885411652e-05, + "loss": 2.5587, + "theoretical_loss": 3.4191563533170637, + "tokens_seen": 2080571392 + }, + { + "epoch": 0.26, + "learning_rate": 7.464291445995829e-05, + "loss": 2.6061, + "theoretical_loss": 3.4191385335758055, + "tokens_seen": 2080702464 + }, + { + "epoch": 0.26, + "learning_rate": 7.463489006580004e-05, + "loss": 2.5783, + "theoretical_loss": 3.4191207152713394, + "tokens_seen": 2080833536 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1146352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6832168102264404, + "objective/train/theoretical_loss": 3.4191118066578383, + "objective/train/tokens_used": 451358176, + "theoretical_loss": 3.4191118066578383, + "tokens_seen": 2080899072 + }, + { + "epoch": 0.26, + "learning_rate": 7.46268656716418e-05, + "loss": 2.6432, + "theoretical_loss": 3.419102898403459, + "tokens_seen": 2080964608 + }, + { + "epoch": 0.26, + "learning_rate": 7.461884127748356e-05, + "loss": 2.6323, + "theoretical_loss": 3.4190850829719563, + "tokens_seen": 2081095680 + }, + { + "epoch": 0.26, + "learning_rate": 7.461081688332531e-05, + "loss": 2.6782, + "theoretical_loss": 3.4190672689766277, + "tokens_seen": 2081226752 + }, + { + "epoch": 0.26, + "learning_rate": 7.460279248916708e-05, + "loss": 2.674, + "theoretical_loss": 3.419049456417265, + "tokens_seen": 2081357824 + }, + { + "epoch": 0.26, + "learning_rate": 7.459476809500883e-05, + "loss": 2.6025, + "theoretical_loss": 3.419031645293663, + "tokens_seen": 2081488896 + }, + { + "epoch": 0.26, + "learning_rate": 7.45867437008506e-05, + "loss": 2.5709, + "theoretical_loss": 3.419013835605615, + "tokens_seen": 2081619968 + }, + { + "epoch": 0.26, + "learning_rate": 7.457871930669235e-05, + "loss": 2.6332, + "theoretical_loss": 3.4189960273529154, + "tokens_seen": 2081751040 + }, + { + "epoch": 0.26, + "learning_rate": 7.457069491253412e-05, + "loss": 2.4575, + "theoretical_loss": 3.4189782205353576, + "tokens_seen": 2081882112 + }, + { + "epoch": 0.26, + "learning_rate": 7.456267051837587e-05, + "loss": 2.4975, + "theoretical_loss": 3.4189604151527364, + "tokens_seen": 2082013184 + }, + { + "epoch": 0.26, + "learning_rate": 7.455464612421762e-05, + "loss": 2.4342, + "theoretical_loss": 3.418942611204845, + "tokens_seen": 2082144256 + }, + { + "epoch": 0.26, + "learning_rate": 7.454662173005939e-05, + "loss": 2.6555, + "theoretical_loss": 3.418924808691478, + "tokens_seen": 2082275328 + }, + { + "epoch": 0.26, + "learning_rate": 7.453859733590114e-05, + "loss": 2.6492, + "theoretical_loss": 3.4189070076124293, + "tokens_seen": 2082406400 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1147344, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.602205753326416, + "objective/train/theoretical_loss": 3.418889207967493, + "objective/train/tokens_used": 452996576, + "theoretical_loss": 3.418889207967493, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.26, + "learning_rate": 7.453057294174291e-05, + "loss": 2.6904, + "theoretical_loss": 3.418889207967493, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.26, + "learning_rate": 7.452254854758466e-05, + "loss": 2.7331, + "theoretical_loss": 3.4188714097564636, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.26, + "learning_rate": 7.451452415342643e-05, + "loss": 2.4997, + "theoretical_loss": 3.4188536129791354, + "tokens_seen": 2082799616 + }, + { + "epoch": 0.26, + "learning_rate": 7.450649975926818e-05, + "loss": 2.6038, + "theoretical_loss": 3.418835817635303, + "tokens_seen": 2082930688 + }, + { + "epoch": 0.26, + "learning_rate": 7.449847536510994e-05, + "loss": 2.5311, + "theoretical_loss": 3.41881802372476, + "tokens_seen": 2083061760 + }, + { + "epoch": 0.26, + "learning_rate": 7.44904509709517e-05, + "loss": 2.5913, + "theoretical_loss": 3.4188002312473005, + "tokens_seen": 2083192832 + }, + { + "epoch": 0.26, + "learning_rate": 7.448242657679346e-05, + "loss": 2.5041, + "theoretical_loss": 3.41878244020272, + "tokens_seen": 2083323904 + }, + { + "epoch": 0.26, + "learning_rate": 7.447440218263522e-05, + "loss": 2.5274, + "theoretical_loss": 3.418764650590812, + "tokens_seen": 2083454976 + }, + { + "epoch": 0.26, + "learning_rate": 7.446637778847698e-05, + "loss": 2.6918, + "theoretical_loss": 3.4187468624113717, + "tokens_seen": 2083586048 + }, + { + "epoch": 0.26, + "learning_rate": 7.445835339431873e-05, + "loss": 2.6158, + "theoretical_loss": 3.4187290756641935, + "tokens_seen": 2083717120 + }, + { + "epoch": 0.26, + "learning_rate": 7.44503290001605e-05, + "loss": 2.7229, + "theoretical_loss": 3.418711290349072, + "tokens_seen": 2083848192 + }, + { + "epoch": 0.26, + "learning_rate": 7.444230460600225e-05, + "loss": 2.6351, + "theoretical_loss": 3.418693506465801, + "tokens_seen": 2083979264 + }, + { + "epoch": 0.26, + "learning_rate": 7.443428021184402e-05, + "loss": 2.7173, + "theoretical_loss": 3.418675724014177, + "tokens_seen": 2084110336 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1148002, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8767805099487305, + "objective/train/theoretical_loss": 3.418666833325168, + "objective/train/tokens_used": 454634976, + "theoretical_loss": 3.418666833325168, + "tokens_seen": 2084175872 + }, + { + "epoch": 0.26, + "learning_rate": 7.442625581768577e-05, + "loss": 2.6401, + "theoretical_loss": 3.4186579429939927, + "tokens_seen": 2084241408 + }, + { + "epoch": 0.26, + "learning_rate": 7.441823142352754e-05, + "loss": 2.4244, + "theoretical_loss": 3.4186401634050445, + "tokens_seen": 2084372480 + }, + { + "epoch": 0.26, + "learning_rate": 7.441020702936929e-05, + "loss": 2.5651, + "theoretical_loss": 3.4186223852471262, + "tokens_seen": 2084503552 + }, + { + "epoch": 0.26, + "learning_rate": 7.440218263521104e-05, + "loss": 2.6412, + "theoretical_loss": 3.4186046085200332, + "tokens_seen": 2084634624 + }, + { + "epoch": 0.26, + "learning_rate": 7.439415824105281e-05, + "loss": 2.3889, + "theoretical_loss": 3.41858683322356, + "tokens_seen": 2084765696 + }, + { + "epoch": 0.26, + "learning_rate": 7.438613384689456e-05, + "loss": 2.5168, + "theoretical_loss": 3.4185690593575018, + "tokens_seen": 2084896768 + }, + { + "epoch": 0.26, + "learning_rate": 7.437810945273633e-05, + "loss": 2.6211, + "theoretical_loss": 3.418551286921653, + "tokens_seen": 2085027840 + }, + { + "epoch": 0.26, + "learning_rate": 7.437008505857808e-05, + "loss": 2.6815, + "theoretical_loss": 3.4185335159158097, + "tokens_seen": 2085158912 + }, + { + "epoch": 0.26, + "learning_rate": 7.436206066441983e-05, + "loss": 2.4658, + "theoretical_loss": 3.418515746339766, + "tokens_seen": 2085289984 + }, + { + "epoch": 0.26, + "learning_rate": 7.43540362702616e-05, + "loss": 2.5338, + "theoretical_loss": 3.4184979781933174, + "tokens_seen": 2085421056 + }, + { + "epoch": 0.26, + "learning_rate": 7.434601187610335e-05, + "loss": 2.5857, + "theoretical_loss": 3.418480211476259, + "tokens_seen": 2085552128 + }, + { + "epoch": 0.26, + "learning_rate": 7.433798748194512e-05, + "loss": 2.5221, + "theoretical_loss": 3.4184624461883857, + "tokens_seen": 2085683200 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1149018, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5401086807250977, + "objective/train/theoretical_loss": 3.4184446823294934, + "objective/train/tokens_used": 456273376, + "theoretical_loss": 3.4184446823294934, + "tokens_seen": 2085814272 + }, + { + "epoch": 0.26, + "learning_rate": 7.432996308778687e-05, + "loss": 2.5125, + "theoretical_loss": 3.4184446823294934, + "tokens_seen": 2085814272 + }, + { + "epoch": 0.26, + "learning_rate": 7.432193869362864e-05, + "loss": 2.5801, + "theoretical_loss": 3.4184269198993773, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.26, + "learning_rate": 7.43139142994704e-05, + "loss": 2.4876, + "theoretical_loss": 3.4184091588978314, + "tokens_seen": 2086076416 + }, + { + "epoch": 0.26, + "learning_rate": 7.430588990531215e-05, + "loss": 2.8154, + "theoretical_loss": 3.418391399324653, + "tokens_seen": 2086207488 + }, + { + "epoch": 0.26, + "learning_rate": 7.429786551115391e-05, + "loss": 2.5371, + "theoretical_loss": 3.4183736411796364, + "tokens_seen": 2086338560 + }, + { + "epoch": 0.26, + "learning_rate": 7.428984111699567e-05, + "loss": 2.6052, + "theoretical_loss": 3.4183558844625765, + "tokens_seen": 2086469632 + }, + { + "epoch": 0.26, + "learning_rate": 7.428181672283743e-05, + "loss": 2.4728, + "theoretical_loss": 3.41833812917327, + "tokens_seen": 2086600704 + }, + { + "epoch": 0.26, + "learning_rate": 7.427379232867919e-05, + "loss": 2.4988, + "theoretical_loss": 3.418320375311512, + "tokens_seen": 2086731776 + }, + { + "epoch": 0.26, + "learning_rate": 7.426576793452094e-05, + "loss": 2.7325, + "theoretical_loss": 3.4183026228770976, + "tokens_seen": 2086862848 + }, + { + "epoch": 0.26, + "learning_rate": 7.425774354036271e-05, + "loss": 2.6953, + "theoretical_loss": 3.4182848718698233, + "tokens_seen": 2086993920 + }, + { + "epoch": 0.26, + "learning_rate": 7.424971914620446e-05, + "loss": 2.5597, + "theoretical_loss": 3.4182671222894836, + "tokens_seen": 2087124992 + }, + { + "epoch": 0.27, + "learning_rate": 7.424169475204623e-05, + "loss": 2.5705, + "theoretical_loss": 3.4182493741358755, + "tokens_seen": 2087256064 + }, + { + "epoch": 0.27, + "learning_rate": 7.423367035788798e-05, + "loss": 2.5799, + "theoretical_loss": 3.4182316274087934, + "tokens_seen": 2087387136 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1149534, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.625910997390747, + "objective/train/theoretical_loss": 3.4182227545801362, + "objective/train/tokens_used": 457911776, + "theoretical_loss": 3.4182227545801362, + "tokens_seen": 2087452672 + }, + { + "epoch": 0.27, + "learning_rate": 7.422564596372975e-05, + "loss": 2.5105, + "theoretical_loss": 3.418213882108034, + "tokens_seen": 2087518208 + }, + { + "epoch": 0.27, + "learning_rate": 7.42176215695715e-05, + "loss": 2.5787, + "theoretical_loss": 3.4181961382333927, + "tokens_seen": 2087649280 + }, + { + "epoch": 0.27, + "learning_rate": 7.420959717541325e-05, + "loss": 2.8012, + "theoretical_loss": 3.4181783957846656, + "tokens_seen": 2087780352 + }, + { + "epoch": 0.27, + "learning_rate": 7.420157278125502e-05, + "loss": 2.6249, + "theoretical_loss": 3.4181606547616488, + "tokens_seen": 2087911424 + }, + { + "epoch": 0.27, + "learning_rate": 7.419354838709677e-05, + "loss": 2.4211, + "theoretical_loss": 3.418142915164138, + "tokens_seen": 2088042496 + }, + { + "epoch": 0.27, + "learning_rate": 7.418552399293854e-05, + "loss": 2.5906, + "theoretical_loss": 3.4181251769919285, + "tokens_seen": 2088173568 + }, + { + "epoch": 0.27, + "learning_rate": 7.417749959878029e-05, + "loss": 2.5411, + "theoretical_loss": 3.4181074402448175, + "tokens_seen": 2088304640 + }, + { + "epoch": 0.27, + "learning_rate": 7.416947520462205e-05, + "loss": 2.4751, + "theoretical_loss": 3.4180897049226004, + "tokens_seen": 2088435712 + }, + { + "epoch": 0.27, + "learning_rate": 7.416145081046381e-05, + "loss": 2.6218, + "theoretical_loss": 3.418071971025073, + "tokens_seen": 2088566784 + }, + { + "epoch": 0.27, + "learning_rate": 7.415342641630557e-05, + "loss": 2.6089, + "theoretical_loss": 3.4180542385520325, + "tokens_seen": 2088697856 + }, + { + "epoch": 0.27, + "learning_rate": 7.414540202214733e-05, + "loss": 2.607, + "theoretical_loss": 3.4180365075032744, + "tokens_seen": 2088828928 + }, + { + "epoch": 0.27, + "learning_rate": 7.413737762798909e-05, + "loss": 2.5198, + "theoretical_loss": 3.4180187778785953, + "tokens_seen": 2088960000 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1150483, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.777698040008545, + "objective/train/theoretical_loss": 3.4180010496777906, + "objective/train/tokens_used": 459550176, + "theoretical_loss": 3.4180010496777906, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.27, + "learning_rate": 7.412935323383084e-05, + "loss": 2.5472, + "theoretical_loss": 3.4180010496777906, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.27, + "learning_rate": 7.41213288396726e-05, + "loss": 2.4747, + "theoretical_loss": 3.4179833229006578, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.27, + "learning_rate": 7.411330444551436e-05, + "loss": 2.588, + "theoretical_loss": 3.417965597546992, + "tokens_seen": 2089353216 + }, + { + "epoch": 0.27, + "learning_rate": 7.410528005135612e-05, + "loss": 2.6024, + "theoretical_loss": 3.4179478736165914, + "tokens_seen": 2089484288 + }, + { + "epoch": 0.27, + "learning_rate": 7.409725565719788e-05, + "loss": 2.6249, + "theoretical_loss": 3.4179301511092506, + "tokens_seen": 2089615360 + }, + { + "epoch": 0.27, + "learning_rate": 7.408923126303964e-05, + "loss": 2.5506, + "theoretical_loss": 3.417912430024767, + "tokens_seen": 2089746432 + }, + { + "epoch": 0.27, + "learning_rate": 7.40812068688814e-05, + "loss": 2.6243, + "theoretical_loss": 3.417894710362937, + "tokens_seen": 2089877504 + }, + { + "epoch": 0.27, + "learning_rate": 7.407318247472315e-05, + "loss": 2.5144, + "theoretical_loss": 3.4178769921235572, + "tokens_seen": 2090008576 + }, + { + "epoch": 0.27, + "learning_rate": 7.406515808056492e-05, + "loss": 2.5705, + "theoretical_loss": 3.4178592753064243, + "tokens_seen": 2090139648 + }, + { + "epoch": 0.27, + "learning_rate": 7.405713368640667e-05, + "loss": 2.5887, + "theoretical_loss": 3.417841559911335, + "tokens_seen": 2090270720 + }, + { + "epoch": 0.27, + "learning_rate": 7.404910929224844e-05, + "loss": 2.55, + "theoretical_loss": 3.4178238459380856, + "tokens_seen": 2090401792 + }, + { + "epoch": 0.27, + "learning_rate": 7.404108489809019e-05, + "loss": 2.6119, + "theoretical_loss": 3.417806133386473, + "tokens_seen": 2090532864 + }, + { + "epoch": 0.27, + "learning_rate": 7.403306050393196e-05, + "loss": 2.5615, + "theoretical_loss": 3.417788422256294, + "tokens_seen": 2090663936 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1151030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4571421146392822, + "objective/train/theoretical_loss": 3.417779567224179, + "objective/train/tokens_used": 461188576, + "theoretical_loss": 3.417779567224179, + "tokens_seen": 2090729472 + }, + { + "epoch": 0.27, + "learning_rate": 7.402503610977371e-05, + "loss": 2.5956, + "theoretical_loss": 3.417770712547346, + "tokens_seen": 2090795008 + }, + { + "epoch": 0.27, + "learning_rate": 7.401701171561546e-05, + "loss": 2.6705, + "theoretical_loss": 3.417753004259425, + "tokens_seen": 2090926080 + }, + { + "epoch": 0.27, + "learning_rate": 7.400898732145723e-05, + "loss": 2.5801, + "theoretical_loss": 3.417735297392328, + "tokens_seen": 2091057152 + }, + { + "epoch": 0.27, + "learning_rate": 7.400096292729898e-05, + "loss": 2.3938, + "theoretical_loss": 3.417717591945853, + "tokens_seen": 2091188224 + }, + { + "epoch": 0.27, + "learning_rate": 7.399293853314075e-05, + "loss": 2.5603, + "theoretical_loss": 3.4176998879197957, + "tokens_seen": 2091319296 + }, + { + "epoch": 0.27, + "learning_rate": 7.39849141389825e-05, + "loss": 2.653, + "theoretical_loss": 3.4176821853139536, + "tokens_seen": 2091450368 + }, + { + "epoch": 0.27, + "learning_rate": 7.397688974482427e-05, + "loss": 2.4816, + "theoretical_loss": 3.417664484128124, + "tokens_seen": 2091581440 + }, + { + "epoch": 0.27, + "learning_rate": 7.396886535066602e-05, + "loss": 2.5028, + "theoretical_loss": 3.4176467843621037, + "tokens_seen": 2091712512 + }, + { + "epoch": 0.27, + "learning_rate": 7.396084095650778e-05, + "loss": 2.6164, + "theoretical_loss": 3.4176290860156904, + "tokens_seen": 2091843584 + }, + { + "epoch": 0.27, + "learning_rate": 7.395281656234954e-05, + "loss": 2.5514, + "theoretical_loss": 3.4176113890886803, + "tokens_seen": 2091974656 + }, + { + "epoch": 0.27, + "learning_rate": 7.39447921681913e-05, + "loss": 2.4333, + "theoretical_loss": 3.417593693580872, + "tokens_seen": 2092105728 + }, + { + "epoch": 0.27, + "learning_rate": 7.393676777403306e-05, + "loss": 2.5275, + "theoretical_loss": 3.4175759994920614, + "tokens_seen": 2092236800 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1152201, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.786438226699829, + "objective/train/theoretical_loss": 3.4175583068220465, + "objective/train/tokens_used": 462826976, + "theoretical_loss": 3.4175583068220465, + "tokens_seen": 2092367872 + }, + { + "epoch": 0.27, + "learning_rate": 7.392874337987482e-05, + "loss": 2.5778, + "theoretical_loss": 3.4175583068220465, + "tokens_seen": 2092367872 + }, + { + "epoch": 0.27, + "learning_rate": 7.392071898571658e-05, + "loss": 2.4786, + "theoretical_loss": 3.4175406155706245, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.27, + "learning_rate": 7.391269459155834e-05, + "loss": 2.4362, + "theoretical_loss": 3.4175229257375936, + "tokens_seen": 2092630016 + }, + { + "epoch": 0.27, + "learning_rate": 7.390467019740009e-05, + "loss": 2.5344, + "theoretical_loss": 3.41750523732275, + "tokens_seen": 2092761088 + }, + { + "epoch": 0.27, + "learning_rate": 7.389664580324186e-05, + "loss": 2.4331, + "theoretical_loss": 3.417487550325892, + "tokens_seen": 2092892160 + }, + { + "epoch": 0.27, + "learning_rate": 7.388862140908361e-05, + "loss": 2.5797, + "theoretical_loss": 3.417469864746817, + "tokens_seen": 2093023232 + }, + { + "epoch": 0.27, + "learning_rate": 7.388059701492537e-05, + "loss": 2.7274, + "theoretical_loss": 3.417452180585322, + "tokens_seen": 2093154304 + }, + { + "epoch": 0.27, + "learning_rate": 7.387257262076713e-05, + "loss": 2.4391, + "theoretical_loss": 3.4174344978412057, + "tokens_seen": 2093285376 + }, + { + "epoch": 0.27, + "learning_rate": 7.38645482266089e-05, + "loss": 2.5441, + "theoretical_loss": 3.417416816514265, + "tokens_seen": 2093416448 + }, + { + "epoch": 0.27, + "learning_rate": 7.385652383245065e-05, + "loss": 2.6726, + "theoretical_loss": 3.4173991366042973, + "tokens_seen": 2093547520 + }, + { + "epoch": 0.27, + "learning_rate": 7.384849943829241e-05, + "loss": 2.7121, + "theoretical_loss": 3.4173814581111017, + "tokens_seen": 2093678592 + }, + { + "epoch": 0.27, + "learning_rate": 7.384047504413417e-05, + "loss": 2.4686, + "theoretical_loss": 3.4173637810344744, + "tokens_seen": 2093809664 + }, + { + "epoch": 0.27, + "learning_rate": 7.383245064997592e-05, + "loss": 2.4801, + "theoretical_loss": 3.417346105374214, + "tokens_seen": 2093940736 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1153513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.452009439468384, + "objective/train/theoretical_loss": 3.4173372680751584, + "objective/train/tokens_used": 464465376, + "theoretical_loss": 3.4173372680751584, + "tokens_seen": 2094006272 + }, + { + "epoch": 0.27, + "learning_rate": 7.382442625581769e-05, + "loss": 2.5239, + "theoretical_loss": 3.4173284311301186, + "tokens_seen": 2094071808 + }, + { + "epoch": 0.27, + "learning_rate": 7.381640186165944e-05, + "loss": 2.5636, + "theoretical_loss": 3.4173107583019853, + "tokens_seen": 2094202880 + }, + { + "epoch": 0.27, + "learning_rate": 7.380837746750121e-05, + "loss": 2.6055, + "theoretical_loss": 3.4172930868896128, + "tokens_seen": 2094333952 + }, + { + "epoch": 0.27, + "learning_rate": 7.380035307334296e-05, + "loss": 2.4273, + "theoretical_loss": 3.4172754168927986, + "tokens_seen": 2094465024 + }, + { + "epoch": 0.27, + "learning_rate": 7.379232867918473e-05, + "loss": 2.5082, + "theoretical_loss": 3.417257748311341, + "tokens_seen": 2094596096 + }, + { + "epoch": 0.27, + "learning_rate": 7.378430428502648e-05, + "loss": 2.5065, + "theoretical_loss": 3.4172400811450387, + "tokens_seen": 2094727168 + }, + { + "epoch": 0.27, + "learning_rate": 7.377627989086823e-05, + "loss": 2.4834, + "theoretical_loss": 3.417222415393688, + "tokens_seen": 2094858240 + }, + { + "epoch": 0.27, + "learning_rate": 7.376825549671e-05, + "loss": 2.5748, + "theoretical_loss": 3.4172047510570893, + "tokens_seen": 2094989312 + }, + { + "epoch": 0.27, + "learning_rate": 7.376023110255175e-05, + "loss": 2.5213, + "theoretical_loss": 3.417187088135039, + "tokens_seen": 2095120384 + }, + { + "epoch": 0.27, + "learning_rate": 7.375220670839352e-05, + "loss": 2.5495, + "theoretical_loss": 3.4171694266273365, + "tokens_seen": 2095251456 + }, + { + "epoch": 0.27, + "learning_rate": 7.374418231423527e-05, + "loss": 2.5869, + "theoretical_loss": 3.417151766533779, + "tokens_seen": 2095382528 + }, + { + "epoch": 0.27, + "learning_rate": 7.373615792007704e-05, + "loss": 2.5185, + "theoretical_loss": 3.4171341078541664, + "tokens_seen": 2095513600 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1154141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.867384433746338, + "objective/train/theoretical_loss": 3.4171164505882956, + "objective/train/tokens_used": 466103776, + "theoretical_loss": 3.4171164505882956, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.27, + "learning_rate": 7.372813352591879e-05, + "loss": 2.6868, + "theoretical_loss": 3.4171164505882956, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.27, + "learning_rate": 7.372010913176055e-05, + "loss": 2.5989, + "theoretical_loss": 3.4170987947359652, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.27, + "learning_rate": 7.371208473760231e-05, + "loss": 2.413, + "theoretical_loss": 3.4170811402969745, + "tokens_seen": 2095906816 + }, + { + "epoch": 0.27, + "learning_rate": 7.370406034344407e-05, + "loss": 2.5521, + "theoretical_loss": 3.417063487271121, + "tokens_seen": 2096037888 + }, + { + "epoch": 0.27, + "learning_rate": 7.369603594928583e-05, + "loss": 2.6069, + "theoretical_loss": 3.417045835658204, + "tokens_seen": 2096168960 + }, + { + "epoch": 0.27, + "learning_rate": 7.368801155512759e-05, + "loss": 2.5313, + "theoretical_loss": 3.417028185458021, + "tokens_seen": 2096300032 + }, + { + "epoch": 0.27, + "learning_rate": 7.367998716096935e-05, + "loss": 2.5615, + "theoretical_loss": 3.4170105366703725, + "tokens_seen": 2096431104 + }, + { + "epoch": 0.27, + "learning_rate": 7.36719627668111e-05, + "loss": 2.5193, + "theoretical_loss": 3.416992889295055, + "tokens_seen": 2096562176 + }, + { + "epoch": 0.27, + "learning_rate": 7.366393837265286e-05, + "loss": 2.6729, + "theoretical_loss": 3.4169752433318688, + "tokens_seen": 2096693248 + }, + { + "epoch": 0.27, + "learning_rate": 7.365591397849463e-05, + "loss": 2.569, + "theoretical_loss": 3.416957598780612, + "tokens_seen": 2096824320 + }, + { + "epoch": 0.27, + "learning_rate": 7.364788958433638e-05, + "loss": 2.5365, + "theoretical_loss": 3.416939955641083, + "tokens_seen": 2096955392 + }, + { + "epoch": 0.27, + "learning_rate": 7.363986519017815e-05, + "loss": 2.612, + "theoretical_loss": 3.4169223139130813, + "tokens_seen": 2097086464 + }, + { + "epoch": 0.27, + "learning_rate": 7.36318407960199e-05, + "loss": 2.6286, + "theoretical_loss": 3.416904673596406, + "tokens_seen": 2097217536 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1155185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.522921085357666, + "objective/train/theoretical_loss": 3.416895853967252, + "objective/train/tokens_used": 467742176, + "theoretical_loss": 3.416895853967252, + "tokens_seen": 2097283072 + }, + { + "epoch": 0.27, + "learning_rate": 7.362381640186166e-05, + "loss": 2.5816, + "theoretical_loss": 3.4168870346908546, + "tokens_seen": 2097348608 + }, + { + "epoch": 0.27, + "learning_rate": 7.361579200770342e-05, + "loss": 2.6585, + "theoretical_loss": 3.416869397196227, + "tokens_seen": 2097479680 + }, + { + "epoch": 0.27, + "learning_rate": 7.360776761354517e-05, + "loss": 2.5419, + "theoretical_loss": 3.416851761112323, + "tokens_seen": 2097610752 + }, + { + "epoch": 0.27, + "learning_rate": 7.359974321938694e-05, + "loss": 2.5072, + "theoretical_loss": 3.41683412643894, + "tokens_seen": 2097741824 + }, + { + "epoch": 0.27, + "learning_rate": 7.359171882522869e-05, + "loss": 2.6212, + "theoretical_loss": 3.4168164931758778, + "tokens_seen": 2097872896 + }, + { + "epoch": 0.27, + "learning_rate": 7.358369443107046e-05, + "loss": 2.5326, + "theoretical_loss": 3.416798861322936, + "tokens_seen": 2098003968 + }, + { + "epoch": 0.27, + "learning_rate": 7.357567003691221e-05, + "loss": 2.4969, + "theoretical_loss": 3.416781230879913, + "tokens_seen": 2098135040 + }, + { + "epoch": 0.27, + "learning_rate": 7.356764564275398e-05, + "loss": 2.6687, + "theoretical_loss": 3.416763601846608, + "tokens_seen": 2098266112 + }, + { + "epoch": 0.27, + "learning_rate": 7.355962124859573e-05, + "loss": 2.6071, + "theoretical_loss": 3.416745974222821, + "tokens_seen": 2098397184 + }, + { + "epoch": 0.27, + "learning_rate": 7.35515968544375e-05, + "loss": 2.3758, + "theoretical_loss": 3.41672834800835, + "tokens_seen": 2098528256 + }, + { + "epoch": 0.27, + "learning_rate": 7.354357246027925e-05, + "loss": 2.5369, + "theoretical_loss": 3.416710723202996, + "tokens_seen": 2098659328 + }, + { + "epoch": 0.27, + "learning_rate": 7.3535548066121e-05, + "loss": 2.5719, + "theoretical_loss": 3.4166930998065568, + "tokens_seen": 2098790400 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1155992, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.662437677383423, + "objective/train/theoretical_loss": 3.416675477818832, + "objective/train/tokens_used": 469380576, + "theoretical_loss": 3.416675477818832, + "tokens_seen": 2098921472 + }, + { + "epoch": 0.27, + "learning_rate": 7.352752367196277e-05, + "loss": 2.5635, + "theoretical_loss": 3.416675477818832, + "tokens_seen": 2098921472 + }, + { + "epoch": 0.27, + "learning_rate": 7.351949927780452e-05, + "loss": 2.5932, + "theoretical_loss": 3.4166578572396222, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.27, + "learning_rate": 7.351147488364629e-05, + "loss": 2.5664, + "theoretical_loss": 3.416640238068726, + "tokens_seen": 2099183616 + }, + { + "epoch": 0.27, + "learning_rate": 7.350345048948804e-05, + "loss": 2.681, + "theoretical_loss": 3.416622620305943, + "tokens_seen": 2099314688 + }, + { + "epoch": 0.27, + "learning_rate": 7.349542609532981e-05, + "loss": 2.4963, + "theoretical_loss": 3.416605003951073, + "tokens_seen": 2099445760 + }, + { + "epoch": 0.27, + "learning_rate": 7.348740170117156e-05, + "loss": 2.552, + "theoretical_loss": 3.416587389003915, + "tokens_seen": 2099576832 + }, + { + "epoch": 0.27, + "learning_rate": 7.347937730701332e-05, + "loss": 2.5934, + "theoretical_loss": 3.416569775464269, + "tokens_seen": 2099707904 + }, + { + "epoch": 0.27, + "learning_rate": 7.347135291285508e-05, + "loss": 2.6231, + "theoretical_loss": 3.416552163331935, + "tokens_seen": 2099838976 + }, + { + "epoch": 0.27, + "learning_rate": 7.346332851869684e-05, + "loss": 2.7223, + "theoretical_loss": 3.4165345526067123, + "tokens_seen": 2099970048 + }, + { + "epoch": 0.27, + "learning_rate": 7.34553041245386e-05, + "loss": 2.4176, + "theoretical_loss": 3.416516943288401, + "tokens_seen": 2100101120 + }, + { + "epoch": 0.27, + "learning_rate": 7.344727973038036e-05, + "loss": 2.5539, + "theoretical_loss": 3.4164993353768005, + "tokens_seen": 2100232192 + }, + { + "epoch": 0.27, + "learning_rate": 7.343925533622212e-05, + "loss": 2.4916, + "theoretical_loss": 3.4164817288717106, + "tokens_seen": 2100363264 + }, + { + "epoch": 0.27, + "learning_rate": 7.343123094206388e-05, + "loss": 2.5582, + "theoretical_loss": 3.416464123772932, + "tokens_seen": 2100494336 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1157390, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.862727642059326, + "objective/train/theoretical_loss": 3.4164553217508464, + "objective/train/tokens_used": 471018976, + "theoretical_loss": 3.4164553217508464, + "tokens_seen": 2100559872 + }, + { + "epoch": 0.27, + "learning_rate": 7.342320654790563e-05, + "loss": 2.6738, + "theoretical_loss": 3.416446520080264, + "tokens_seen": 2100625408 + }, + { + "epoch": 0.27, + "learning_rate": 7.34151821537474e-05, + "loss": 2.4995, + "theoretical_loss": 3.4164289177935063, + "tokens_seen": 2100756480 + }, + { + "epoch": 0.27, + "learning_rate": 7.340715775958915e-05, + "loss": 2.5321, + "theoretical_loss": 3.4164113169124595, + "tokens_seen": 2100887552 + }, + { + "epoch": 0.27, + "learning_rate": 7.339913336543092e-05, + "loss": 2.392, + "theoretical_loss": 3.4163937174369234, + "tokens_seen": 2101018624 + }, + { + "epoch": 0.27, + "learning_rate": 7.339110897127267e-05, + "loss": 2.5493, + "theoretical_loss": 3.4163761193666975, + "tokens_seen": 2101149696 + }, + { + "epoch": 0.27, + "learning_rate": 7.338308457711443e-05, + "loss": 2.6421, + "theoretical_loss": 3.416358522701583, + "tokens_seen": 2101280768 + }, + { + "epoch": 0.27, + "learning_rate": 7.337506018295619e-05, + "loss": 2.4459, + "theoretical_loss": 3.4163409274413796, + "tokens_seen": 2101411840 + }, + { + "epoch": 0.27, + "learning_rate": 7.336703578879794e-05, + "loss": 2.5705, + "theoretical_loss": 3.416323333585888, + "tokens_seen": 2101542912 + }, + { + "epoch": 0.27, + "learning_rate": 7.335901139463971e-05, + "loss": 2.6294, + "theoretical_loss": 3.416305741134907, + "tokens_seen": 2101673984 + }, + { + "epoch": 0.27, + "learning_rate": 7.335098700048146e-05, + "loss": 2.4931, + "theoretical_loss": 3.4162881500882385, + "tokens_seen": 2101805056 + }, + { + "epoch": 0.27, + "learning_rate": 7.334296260632323e-05, + "loss": 2.6335, + "theoretical_loss": 3.416270560445682, + "tokens_seen": 2101936128 + }, + { + "epoch": 0.27, + "learning_rate": 7.333493821216498e-05, + "loss": 2.42, + "theoretical_loss": 3.4162529722070385, + "tokens_seen": 2102067200 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1158091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.529240369796753, + "objective/train/theoretical_loss": 3.416235385372107, + "objective/train/tokens_used": 472657376, + "theoretical_loss": 3.416235385372107, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.27, + "learning_rate": 7.332691381800675e-05, + "loss": 2.4835, + "theoretical_loss": 3.416235385372107, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.27, + "learning_rate": 7.33188894238485e-05, + "loss": 2.739, + "theoretical_loss": 3.41621779994069, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.27, + "learning_rate": 7.331086502969025e-05, + "loss": 2.4934, + "theoretical_loss": 3.4162002159125864, + "tokens_seen": 2102460416 + }, + { + "epoch": 0.27, + "learning_rate": 7.330284063553202e-05, + "loss": 2.552, + "theoretical_loss": 3.4161826332875975, + "tokens_seen": 2102591488 + }, + { + "epoch": 0.27, + "learning_rate": 7.329481624137377e-05, + "loss": 2.4815, + "theoretical_loss": 3.4161650520655233, + "tokens_seen": 2102722560 + }, + { + "epoch": 0.27, + "learning_rate": 7.328679184721554e-05, + "loss": 2.5107, + "theoretical_loss": 3.4161474722461653, + "tokens_seen": 2102853632 + }, + { + "epoch": 0.27, + "learning_rate": 7.32787674530573e-05, + "loss": 2.6167, + "theoretical_loss": 3.416129893829323, + "tokens_seen": 2102984704 + }, + { + "epoch": 0.27, + "learning_rate": 7.327074305889906e-05, + "loss": 2.5377, + "theoretical_loss": 3.4161123168147984, + "tokens_seen": 2103115776 + }, + { + "epoch": 0.27, + "learning_rate": 7.326271866474081e-05, + "loss": 2.4288, + "theoretical_loss": 3.416094741202391, + "tokens_seen": 2103246848 + }, + { + "epoch": 0.27, + "learning_rate": 7.325469427058258e-05, + "loss": 2.3633, + "theoretical_loss": 3.416077166991903, + "tokens_seen": 2103377920 + }, + { + "epoch": 0.27, + "learning_rate": 7.324666987642433e-05, + "loss": 2.5235, + "theoretical_loss": 3.416059594183134, + "tokens_seen": 2103508992 + }, + { + "epoch": 0.27, + "learning_rate": 7.323864548226609e-05, + "loss": 2.5012, + "theoretical_loss": 3.416042022775885, + "tokens_seen": 2103640064 + }, + { + "epoch": 0.28, + "learning_rate": 7.323062108810785e-05, + "loss": 2.6528, + "theoretical_loss": 3.4160244527699577, + "tokens_seen": 2103771136 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1159419, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5441555976867676, + "objective/train/theoretical_loss": 3.4160156682924274, + "objective/train/tokens_used": 474295776, + "theoretical_loss": 3.4160156682924274, + "tokens_seen": 2103836672 + }, + { + "epoch": 0.28, + "learning_rate": 7.32225966939496e-05, + "loss": 2.5143, + "theoretical_loss": 3.416006884165152, + "tokens_seen": 2103902208 + }, + { + "epoch": 0.28, + "learning_rate": 7.321457229979137e-05, + "loss": 2.5838, + "theoretical_loss": 3.4159893169612703, + "tokens_seen": 2104033280 + }, + { + "epoch": 0.28, + "learning_rate": 7.320654790563313e-05, + "loss": 2.4385, + "theoretical_loss": 3.4159717511581116, + "tokens_seen": 2104164352 + }, + { + "epoch": 0.28, + "learning_rate": 7.319852351147489e-05, + "loss": 2.4926, + "theoretical_loss": 3.415954186755479, + "tokens_seen": 2104295424 + }, + { + "epoch": 0.28, + "learning_rate": 7.319049911731665e-05, + "loss": 2.4845, + "theoretical_loss": 3.4159366237531725, + "tokens_seen": 2104426496 + }, + { + "epoch": 0.28, + "learning_rate": 7.31824747231584e-05, + "loss": 2.5013, + "theoretical_loss": 3.4159190621509934, + "tokens_seen": 2104557568 + }, + { + "epoch": 0.28, + "learning_rate": 7.317445032900017e-05, + "loss": 2.5026, + "theoretical_loss": 3.415901501948743, + "tokens_seen": 2104688640 + }, + { + "epoch": 0.28, + "learning_rate": 7.316642593484192e-05, + "loss": 2.6106, + "theoretical_loss": 3.4158839431462225, + "tokens_seen": 2104819712 + }, + { + "epoch": 0.28, + "learning_rate": 7.315840154068369e-05, + "loss": 2.3583, + "theoretical_loss": 3.415866385743233, + "tokens_seen": 2104950784 + }, + { + "epoch": 0.28, + "learning_rate": 7.315037714652544e-05, + "loss": 2.5919, + "theoretical_loss": 3.415848829739576, + "tokens_seen": 2105081856 + }, + { + "epoch": 0.28, + "learning_rate": 7.31423527523672e-05, + "loss": 2.559, + "theoretical_loss": 3.4158312751350532, + "tokens_seen": 2105212928 + }, + { + "epoch": 0.28, + "learning_rate": 7.313432835820896e-05, + "loss": 2.5438, + "theoretical_loss": 3.4158137219294655, + "tokens_seen": 2105344000 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1160366, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4362361431121826, + "objective/train/theoretical_loss": 3.415796170122615, + "objective/train/tokens_used": 475934176, + "theoretical_loss": 3.415796170122615, + "tokens_seen": 2105475072 + }, + { + "epoch": 0.28, + "learning_rate": 7.312630396405071e-05, + "loss": 2.5706, + "theoretical_loss": 3.415796170122615, + "tokens_seen": 2105475072 + }, + { + "epoch": 0.28, + "learning_rate": 7.311827956989248e-05, + "loss": 2.4484, + "theoretical_loss": 3.415778619714302, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.28, + "learning_rate": 7.311025517573423e-05, + "loss": 2.4628, + "theoretical_loss": 3.4157610707043284, + "tokens_seen": 2105737216 + }, + { + "epoch": 0.28, + "learning_rate": 7.3102230781576e-05, + "loss": 2.5434, + "theoretical_loss": 3.415743523092497, + "tokens_seen": 2105868288 + }, + { + "epoch": 0.28, + "learning_rate": 7.309420638741775e-05, + "loss": 2.5093, + "theoretical_loss": 3.4157259768786075, + "tokens_seen": 2105999360 + }, + { + "epoch": 0.28, + "learning_rate": 7.308618199325952e-05, + "loss": 2.443, + "theoretical_loss": 3.415708432062463, + "tokens_seen": 2106130432 + }, + { + "epoch": 0.28, + "learning_rate": 7.307815759910127e-05, + "loss": 2.6816, + "theoretical_loss": 3.415690888643865, + "tokens_seen": 2106261504 + }, + { + "epoch": 0.28, + "learning_rate": 7.307013320494302e-05, + "loss": 2.6213, + "theoretical_loss": 3.415673346622614, + "tokens_seen": 2106392576 + }, + { + "epoch": 0.28, + "learning_rate": 7.306210881078479e-05, + "loss": 2.6613, + "theoretical_loss": 3.415655805998513, + "tokens_seen": 2106523648 + }, + { + "epoch": 0.28, + "learning_rate": 7.305408441662654e-05, + "loss": 2.52, + "theoretical_loss": 3.415638266771363, + "tokens_seen": 2106654720 + }, + { + "epoch": 0.28, + "learning_rate": 7.304606002246831e-05, + "loss": 2.6228, + "theoretical_loss": 3.415620728940967, + "tokens_seen": 2106785792 + }, + { + "epoch": 0.28, + "learning_rate": 7.303803562831006e-05, + "loss": 2.4382, + "theoretical_loss": 3.4156031925071257, + "tokens_seen": 2106916864 + }, + { + "epoch": 0.28, + "learning_rate": 7.303001123415183e-05, + "loss": 2.595, + "theoretical_loss": 3.4155856574696415, + "tokens_seen": 2107047936 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1161079, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0612099170684814, + "objective/train/theoretical_loss": 3.4155768904744717, + "objective/train/tokens_used": 477572576, + "theoretical_loss": 3.4155768904744717, + "tokens_seen": 2107113472 + }, + { + "epoch": 0.28, + "learning_rate": 7.302198683999358e-05, + "loss": 2.6113, + "theoretical_loss": 3.415568123828317, + "tokens_seen": 2107179008 + }, + { + "epoch": 0.28, + "learning_rate": 7.301396244583534e-05, + "loss": 2.6254, + "theoretical_loss": 3.4155505915829525, + "tokens_seen": 2107310080 + }, + { + "epoch": 0.28, + "learning_rate": 7.30059380516771e-05, + "loss": 2.5267, + "theoretical_loss": 3.4155330607333516, + "tokens_seen": 2107441152 + }, + { + "epoch": 0.28, + "learning_rate": 7.299791365751886e-05, + "loss": 2.4114, + "theoretical_loss": 3.4155155312793157, + "tokens_seen": 2107572224 + }, + { + "epoch": 0.28, + "learning_rate": 7.298988926336062e-05, + "loss": 2.5213, + "theoretical_loss": 3.4154980032206472, + "tokens_seen": 2107703296 + }, + { + "epoch": 0.28, + "learning_rate": 7.298186486920238e-05, + "loss": 2.6768, + "theoretical_loss": 3.415480476557148, + "tokens_seen": 2107834368 + }, + { + "epoch": 0.28, + "learning_rate": 7.297384047504414e-05, + "loss": 2.4979, + "theoretical_loss": 3.4154629512886205, + "tokens_seen": 2107965440 + }, + { + "epoch": 0.28, + "learning_rate": 7.29658160808859e-05, + "loss": 2.5837, + "theoretical_loss": 3.415445427414867, + "tokens_seen": 2108096512 + }, + { + "epoch": 0.28, + "learning_rate": 7.295779168672765e-05, + "loss": 2.5184, + "theoretical_loss": 3.4154279049356897, + "tokens_seen": 2108227584 + }, + { + "epoch": 0.28, + "learning_rate": 7.294976729256942e-05, + "loss": 2.487, + "theoretical_loss": 3.4154103838508907, + "tokens_seen": 2108358656 + }, + { + "epoch": 0.28, + "learning_rate": 7.294174289841117e-05, + "loss": 2.5156, + "theoretical_loss": 3.415392864160273, + "tokens_seen": 2108489728 + }, + { + "epoch": 0.28, + "learning_rate": 7.293371850425294e-05, + "loss": 2.6961, + "theoretical_loss": 3.4153753458636382, + "tokens_seen": 2108620800 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1162446, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.430760622024536, + "objective/train/theoretical_loss": 3.415357828960789, + "objective/train/tokens_used": 479210976, + "theoretical_loss": 3.415357828960789, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.28, + "learning_rate": 7.292569411009469e-05, + "loss": 2.5458, + "theoretical_loss": 3.415357828960789, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.28, + "learning_rate": 7.291766971593646e-05, + "loss": 2.6747, + "theoretical_loss": 3.4153403134515283, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.28, + "learning_rate": 7.290964532177821e-05, + "loss": 2.5972, + "theoretical_loss": 3.415322799335658, + "tokens_seen": 2109014016 + }, + { + "epoch": 0.28, + "learning_rate": 7.290162092761998e-05, + "loss": 2.4214, + "theoretical_loss": 3.4153052866129814, + "tokens_seen": 2109145088 + }, + { + "epoch": 0.28, + "learning_rate": 7.289359653346173e-05, + "loss": 2.5883, + "theoretical_loss": 3.4152877752833, + "tokens_seen": 2109276160 + }, + { + "epoch": 0.28, + "learning_rate": 7.288557213930348e-05, + "loss": 2.6023, + "theoretical_loss": 3.4152702653464178, + "tokens_seen": 2109407232 + }, + { + "epoch": 0.28, + "learning_rate": 7.287754774514525e-05, + "loss": 2.5545, + "theoretical_loss": 3.4152527568021362, + "tokens_seen": 2109538304 + }, + { + "epoch": 0.28, + "learning_rate": 7.2869523350987e-05, + "loss": 2.5484, + "theoretical_loss": 3.4152352496502596, + "tokens_seen": 2109669376 + }, + { + "epoch": 0.28, + "learning_rate": 7.286149895682877e-05, + "loss": 2.5416, + "theoretical_loss": 3.4152177438905893, + "tokens_seen": 2109800448 + }, + { + "epoch": 0.28, + "learning_rate": 7.285347456267052e-05, + "loss": 2.501, + "theoretical_loss": 3.415200239522928, + "tokens_seen": 2109931520 + }, + { + "epoch": 0.28, + "learning_rate": 7.284545016851229e-05, + "loss": 2.663, + "theoretical_loss": 3.4151827365470795, + "tokens_seen": 2110062592 + }, + { + "epoch": 0.28, + "learning_rate": 7.283742577435404e-05, + "loss": 2.5081, + "theoretical_loss": 3.4151652349628465, + "tokens_seen": 2110193664 + }, + { + "epoch": 0.28, + "learning_rate": 7.28294013801958e-05, + "loss": 2.4971, + "theoretical_loss": 3.415147734770031, + "tokens_seen": 2110324736 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1163114, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.412344217300415, + "objective/train/theoretical_loss": 3.4151389851953438, + "objective/train/tokens_used": 480849376, + "theoretical_loss": 3.4151389851953438, + "tokens_seen": 2110390272 + }, + { + "epoch": 0.28, + "learning_rate": 7.282137698603756e-05, + "loss": 2.6189, + "theoretical_loss": 3.4151302359684372, + "tokens_seen": 2110455808 + }, + { + "epoch": 0.28, + "learning_rate": 7.281335259187931e-05, + "loss": 2.4918, + "theoretical_loss": 3.4151127385578675, + "tokens_seen": 2110586880 + }, + { + "epoch": 0.28, + "learning_rate": 7.280532819772108e-05, + "loss": 2.634, + "theoretical_loss": 3.415095242538125, + "tokens_seen": 2110717952 + }, + { + "epoch": 0.28, + "learning_rate": 7.279730380356283e-05, + "loss": 2.5236, + "theoretical_loss": 3.415077747909013, + "tokens_seen": 2110849024 + }, + { + "epoch": 0.28, + "learning_rate": 7.27892794094046e-05, + "loss": 2.4392, + "theoretical_loss": 3.415060254670334, + "tokens_seen": 2110980096 + }, + { + "epoch": 0.28, + "learning_rate": 7.278125501524635e-05, + "loss": 2.4037, + "theoretical_loss": 3.415042762821892, + "tokens_seen": 2111111168 + }, + { + "epoch": 0.28, + "learning_rate": 7.27732306210881e-05, + "loss": 2.5942, + "theoretical_loss": 3.41502527236349, + "tokens_seen": 2111242240 + }, + { + "epoch": 0.28, + "learning_rate": 7.276520622692987e-05, + "loss": 2.5675, + "theoretical_loss": 3.4150077832949304, + "tokens_seen": 2111373312 + }, + { + "epoch": 0.28, + "learning_rate": 7.275718183277163e-05, + "loss": 2.6567, + "theoretical_loss": 3.4149902956160174, + "tokens_seen": 2111504384 + }, + { + "epoch": 0.28, + "learning_rate": 7.274915743861339e-05, + "loss": 2.4766, + "theoretical_loss": 3.4149728093265543, + "tokens_seen": 2111635456 + }, + { + "epoch": 0.28, + "learning_rate": 7.274113304445515e-05, + "loss": 2.5684, + "theoretical_loss": 3.4149553244263444, + "tokens_seen": 2111766528 + }, + { + "epoch": 0.28, + "learning_rate": 7.273310865029691e-05, + "loss": 2.4782, + "theoretical_loss": 3.4149378409151905, + "tokens_seen": 2111897600 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1164441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.561600923538208, + "objective/train/theoretical_loss": 3.414920358792897, + "objective/train/tokens_used": 482487776, + "theoretical_loss": 3.414920358792897, + "tokens_seen": 2112028672 + }, + { + "epoch": 0.28, + "learning_rate": 7.272508425613867e-05, + "loss": 2.6107, + "theoretical_loss": 3.414920358792897, + "tokens_seen": 2112028672 + }, + { + "epoch": 0.28, + "learning_rate": 7.271705986198042e-05, + "loss": 2.3715, + "theoretical_loss": 3.4149028780592667, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.28, + "learning_rate": 7.270903546782219e-05, + "loss": 2.5158, + "theoretical_loss": 3.414885398714103, + "tokens_seen": 2112290816 + }, + { + "epoch": 0.28, + "learning_rate": 7.270101107366394e-05, + "loss": 2.5778, + "theoretical_loss": 3.41486792075721, + "tokens_seen": 2112421888 + }, + { + "epoch": 0.28, + "learning_rate": 7.26929866795057e-05, + "loss": 2.5923, + "theoretical_loss": 3.4148504441883913, + "tokens_seen": 2112552960 + }, + { + "epoch": 0.28, + "learning_rate": 7.268496228534746e-05, + "loss": 2.5593, + "theoretical_loss": 3.414832969007451, + "tokens_seen": 2112684032 + }, + { + "epoch": 0.28, + "learning_rate": 7.267693789118923e-05, + "loss": 2.5111, + "theoretical_loss": 3.414815495214191, + "tokens_seen": 2112815104 + }, + { + "epoch": 0.28, + "learning_rate": 7.266891349703098e-05, + "loss": 2.6016, + "theoretical_loss": 3.4147980228084167, + "tokens_seen": 2112946176 + }, + { + "epoch": 0.28, + "learning_rate": 7.266088910287273e-05, + "loss": 2.488, + "theoretical_loss": 3.414780551789931, + "tokens_seen": 2113077248 + }, + { + "epoch": 0.28, + "learning_rate": 7.26528647087145e-05, + "loss": 2.5142, + "theoretical_loss": 3.4147630821585384, + "tokens_seen": 2113208320 + }, + { + "epoch": 0.28, + "learning_rate": 7.264484031455625e-05, + "loss": 2.5584, + "theoretical_loss": 3.414745613914042, + "tokens_seen": 2113339392 + }, + { + "epoch": 0.28, + "learning_rate": 7.263681592039802e-05, + "loss": 2.5915, + "theoretical_loss": 3.4147281470562465, + "tokens_seen": 2113470464 + }, + { + "epoch": 0.28, + "learning_rate": 7.262879152623977e-05, + "loss": 2.5965, + "theoretical_loss": 3.414710681584956, + "tokens_seen": 2113601536 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1165048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.748392105102539, + "objective/train/theoretical_loss": 3.4147019493691877, + "objective/train/tokens_used": 484126176, + "theoretical_loss": 3.4147019493691877, + "tokens_seen": 2113667072 + }, + { + "epoch": 0.28, + "learning_rate": 7.262076713208154e-05, + "loss": 2.5164, + "theoretical_loss": 3.414693217499973, + "tokens_seen": 2113732608 + }, + { + "epoch": 0.28, + "learning_rate": 7.261274273792329e-05, + "loss": 2.5338, + "theoretical_loss": 3.4146757548011024, + "tokens_seen": 2113863680 + }, + { + "epoch": 0.28, + "learning_rate": 7.260471834376506e-05, + "loss": 2.5395, + "theoretical_loss": 3.4146582934881486, + "tokens_seen": 2113994752 + }, + { + "epoch": 0.28, + "learning_rate": 7.259669394960681e-05, + "loss": 2.6317, + "theoretical_loss": 3.4146408335609157, + "tokens_seen": 2114125824 + }, + { + "epoch": 0.28, + "learning_rate": 7.258866955544856e-05, + "loss": 2.4815, + "theoretical_loss": 3.4146233750192065, + "tokens_seen": 2114256896 + }, + { + "epoch": 0.28, + "learning_rate": 7.258064516129033e-05, + "loss": 2.6, + "theoretical_loss": 3.4146059178628265, + "tokens_seen": 2114387968 + }, + { + "epoch": 0.28, + "learning_rate": 7.257262076713208e-05, + "loss": 2.5097, + "theoretical_loss": 3.4145884620915803, + "tokens_seen": 2114519040 + }, + { + "epoch": 0.28, + "learning_rate": 7.256459637297385e-05, + "loss": 2.5873, + "theoretical_loss": 3.4145710077052707, + "tokens_seen": 2114650112 + }, + { + "epoch": 0.28, + "learning_rate": 7.25565719788156e-05, + "loss": 2.4999, + "theoretical_loss": 3.4145535547037023, + "tokens_seen": 2114781184 + }, + { + "epoch": 0.28, + "learning_rate": 7.254854758465737e-05, + "loss": 2.7295, + "theoretical_loss": 3.4145361030866805, + "tokens_seen": 2114912256 + }, + { + "epoch": 0.28, + "learning_rate": 7.254052319049912e-05, + "loss": 2.6493, + "theoretical_loss": 3.414518652854009, + "tokens_seen": 2115043328 + }, + { + "epoch": 0.28, + "learning_rate": 7.253249879634088e-05, + "loss": 2.5968, + "theoretical_loss": 3.414501204005492, + "tokens_seen": 2115174400 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1166321, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.903796672821045, + "objective/train/theoretical_loss": 3.4144837565409336, + "objective/train/tokens_used": 485764576, + "theoretical_loss": 3.4144837565409336, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.28, + "learning_rate": 7.252447440218264e-05, + "loss": 2.6134, + "theoretical_loss": 3.4144837565409336, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.28, + "learning_rate": 7.25164500080244e-05, + "loss": 2.4382, + "theoretical_loss": 3.4144663104601394, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.28, + "learning_rate": 7.250842561386616e-05, + "loss": 2.5556, + "theoretical_loss": 3.414448865762913, + "tokens_seen": 2115567616 + }, + { + "epoch": 0.28, + "learning_rate": 7.250040121970792e-05, + "loss": 2.5734, + "theoretical_loss": 3.4144314224490597, + "tokens_seen": 2115698688 + }, + { + "epoch": 0.28, + "learning_rate": 7.249237682554968e-05, + "loss": 2.4823, + "theoretical_loss": 3.4144139805183835, + "tokens_seen": 2115829760 + }, + { + "epoch": 0.28, + "learning_rate": 7.248435243139144e-05, + "loss": 2.659, + "theoretical_loss": 3.4143965399706895, + "tokens_seen": 2115960832 + }, + { + "epoch": 0.28, + "learning_rate": 7.247632803723319e-05, + "loss": 2.3849, + "theoretical_loss": 3.4143791008057818, + "tokens_seen": 2116091904 + }, + { + "epoch": 0.28, + "learning_rate": 7.246830364307496e-05, + "loss": 2.5886, + "theoretical_loss": 3.4143616630234654, + "tokens_seen": 2116222976 + }, + { + "epoch": 0.28, + "learning_rate": 7.246027924891671e-05, + "loss": 2.5187, + "theoretical_loss": 3.4143442266235455, + "tokens_seen": 2116354048 + }, + { + "epoch": 0.28, + "learning_rate": 7.245225485475848e-05, + "loss": 2.4991, + "theoretical_loss": 3.414326791605826, + "tokens_seen": 2116485120 + }, + { + "epoch": 0.28, + "learning_rate": 7.244423046060023e-05, + "loss": 2.6432, + "theoretical_loss": 3.414309357970113, + "tokens_seen": 2116616192 + }, + { + "epoch": 0.28, + "learning_rate": 7.2436206066442e-05, + "loss": 2.5072, + "theoretical_loss": 3.41429192571621, + "tokens_seen": 2116747264 + }, + { + "epoch": 0.28, + "learning_rate": 7.242818167228375e-05, + "loss": 2.4161, + "theoretical_loss": 3.414274494843923, + "tokens_seen": 2116878336 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1166896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.635124921798706, + "objective/train/theoretical_loss": 3.4142657799258243, + "objective/train/tokens_used": 487402976, + "theoretical_loss": 3.4142657799258243, + "tokens_seen": 2116943872 + }, + { + "epoch": 0.28, + "learning_rate": 7.24201572781255e-05, + "loss": 2.4925, + "theoretical_loss": 3.4142570653530564, + "tokens_seen": 2117009408 + }, + { + "epoch": 0.28, + "learning_rate": 7.241213288396727e-05, + "loss": 2.5845, + "theoretical_loss": 3.414239637243415, + "tokens_seen": 2117140480 + }, + { + "epoch": 0.28, + "learning_rate": 7.240410848980902e-05, + "loss": 2.4822, + "theoretical_loss": 3.414222210514805, + "tokens_seen": 2117271552 + }, + { + "epoch": 0.28, + "learning_rate": 7.239608409565079e-05, + "loss": 2.6126, + "theoretical_loss": 3.41420478516703, + "tokens_seen": 2117402624 + }, + { + "epoch": 0.28, + "learning_rate": 7.238805970149254e-05, + "loss": 2.4967, + "theoretical_loss": 3.414187361199896, + "tokens_seen": 2117533696 + }, + { + "epoch": 0.28, + "learning_rate": 7.238003530733431e-05, + "loss": 2.5921, + "theoretical_loss": 3.414169938613208, + "tokens_seen": 2117664768 + }, + { + "epoch": 0.28, + "learning_rate": 7.237201091317606e-05, + "loss": 2.5172, + "theoretical_loss": 3.4141525174067704, + "tokens_seen": 2117795840 + }, + { + "epoch": 0.28, + "learning_rate": 7.236398651901781e-05, + "loss": 2.5096, + "theoretical_loss": 3.41413509758039, + "tokens_seen": 2117926912 + }, + { + "epoch": 0.28, + "learning_rate": 7.235596212485958e-05, + "loss": 2.3977, + "theoretical_loss": 3.414117679133871, + "tokens_seen": 2118057984 + }, + { + "epoch": 0.28, + "learning_rate": 7.234793773070133e-05, + "loss": 2.5047, + "theoretical_loss": 3.414100262067019, + "tokens_seen": 2118189056 + }, + { + "epoch": 0.28, + "learning_rate": 7.23399133365431e-05, + "loss": 2.5251, + "theoretical_loss": 3.4140828463796398, + "tokens_seen": 2118320128 + }, + { + "epoch": 0.28, + "learning_rate": 7.233188894238485e-05, + "loss": 2.5052, + "theoretical_loss": 3.4140654320715376, + "tokens_seen": 2118451200 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1168203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.8257986307144165, + "objective/train/theoretical_loss": 3.414048019142519, + "objective/train/tokens_used": 489041376, + "theoretical_loss": 3.414048019142519, + "tokens_seen": 2118582272 + }, + { + "epoch": 0.28, + "learning_rate": 7.232386454822662e-05, + "loss": 2.3003, + "theoretical_loss": 3.414048019142519, + "tokens_seen": 2118582272 + }, + { + "epoch": 0.28, + "learning_rate": 7.231584015406837e-05, + "loss": 2.4695, + "theoretical_loss": 3.414030607592389, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.28, + "learning_rate": 7.230781575991014e-05, + "loss": 2.5806, + "theoretical_loss": 3.414013197420953, + "tokens_seen": 2118844416 + }, + { + "epoch": 0.28, + "learning_rate": 7.22997913657519e-05, + "loss": 2.6054, + "theoretical_loss": 3.413995788628017, + "tokens_seen": 2118975488 + }, + { + "epoch": 0.28, + "learning_rate": 7.229176697159365e-05, + "loss": 2.5005, + "theoretical_loss": 3.4139783812133864, + "tokens_seen": 2119106560 + }, + { + "epoch": 0.28, + "learning_rate": 7.228374257743541e-05, + "loss": 2.6139, + "theoretical_loss": 3.4139609751768663, + "tokens_seen": 2119237632 + }, + { + "epoch": 0.28, + "learning_rate": 7.227571818327717e-05, + "loss": 2.5732, + "theoretical_loss": 3.413943570518263, + "tokens_seen": 2119368704 + }, + { + "epoch": 0.28, + "learning_rate": 7.226769378911893e-05, + "loss": 2.5102, + "theoretical_loss": 3.413926167237382, + "tokens_seen": 2119499776 + }, + { + "epoch": 0.28, + "learning_rate": 7.225966939496069e-05, + "loss": 2.4827, + "theoretical_loss": 3.413908765334029, + "tokens_seen": 2119630848 + }, + { + "epoch": 0.28, + "learning_rate": 7.225164500080245e-05, + "loss": 2.5705, + "theoretical_loss": 3.41389136480801, + "tokens_seen": 2119761920 + }, + { + "epoch": 0.28, + "learning_rate": 7.22436206066442e-05, + "loss": 2.5917, + "theoretical_loss": 3.413873965659131, + "tokens_seen": 2119892992 + }, + { + "epoch": 0.28, + "learning_rate": 7.223559621248596e-05, + "loss": 2.3827, + "theoretical_loss": 3.4138565678871973, + "tokens_seen": 2120024064 + }, + { + "epoch": 0.28, + "learning_rate": 7.222757181832773e-05, + "loss": 2.4065, + "theoretical_loss": 3.4138391714920147, + "tokens_seen": 2120155136 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1168773, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.275192975997925, + "objective/train/theoretical_loss": 3.413830473810645, + "objective/train/tokens_used": 490679776, + "theoretical_loss": 3.413830473810645, + "tokens_seen": 2120220672 + }, + { + "epoch": 0.29, + "learning_rate": 7.221954742416948e-05, + "loss": 2.4053, + "theoretical_loss": 3.41382177647339, + "tokens_seen": 2120286208 + }, + { + "epoch": 0.29, + "learning_rate": 7.221152303001125e-05, + "loss": 2.4353, + "theoretical_loss": 3.413804382831129, + "tokens_seen": 2120417280 + }, + { + "epoch": 0.29, + "learning_rate": 7.2203498635853e-05, + "loss": 2.467, + "theoretical_loss": 3.413786990565037, + "tokens_seen": 2120548352 + }, + { + "epoch": 0.29, + "learning_rate": 7.219547424169477e-05, + "loss": 2.5341, + "theoretical_loss": 3.4137695996749207, + "tokens_seen": 2120679424 + }, + { + "epoch": 0.29, + "learning_rate": 7.218744984753652e-05, + "loss": 2.3319, + "theoretical_loss": 3.413752210160586, + "tokens_seen": 2120810496 + }, + { + "epoch": 0.29, + "learning_rate": 7.217942545337827e-05, + "loss": 2.5447, + "theoretical_loss": 3.413734822021839, + "tokens_seen": 2120941568 + }, + { + "epoch": 0.29, + "learning_rate": 7.217140105922004e-05, + "loss": 2.7, + "theoretical_loss": 3.413717435258486, + "tokens_seen": 2121072640 + }, + { + "epoch": 0.29, + "learning_rate": 7.216337666506179e-05, + "loss": 2.5168, + "theoretical_loss": 3.413700049870333, + "tokens_seen": 2121203712 + }, + { + "epoch": 0.29, + "learning_rate": 7.215535227090356e-05, + "loss": 2.4817, + "theoretical_loss": 3.413682665857187, + "tokens_seen": 2121334784 + }, + { + "epoch": 0.29, + "learning_rate": 7.214732787674531e-05, + "loss": 2.5098, + "theoretical_loss": 3.4136652832188528, + "tokens_seen": 2121465856 + }, + { + "epoch": 0.29, + "learning_rate": 7.213930348258708e-05, + "loss": 2.5541, + "theoretical_loss": 3.4136479019551382, + "tokens_seen": 2121596928 + }, + { + "epoch": 0.29, + "learning_rate": 7.213127908842883e-05, + "loss": 2.3622, + "theoretical_loss": 3.4136305220658496, + "tokens_seen": 2121728000 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1169526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4141392707824707, + "objective/train/theoretical_loss": 3.4136131435507924, + "objective/train/tokens_used": 492318176, + "theoretical_loss": 3.4136131435507924, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.29, + "learning_rate": 7.212325469427058e-05, + "loss": 2.4221, + "theoretical_loss": 3.4136131435507924, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.29, + "learning_rate": 7.211523030011235e-05, + "loss": 2.4602, + "theoretical_loss": 3.413595766409774, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.29, + "learning_rate": 7.21072059059541e-05, + "loss": 2.3975, + "theoretical_loss": 3.4135783906425994, + "tokens_seen": 2122121216 + }, + { + "epoch": 0.29, + "learning_rate": 7.209918151179587e-05, + "loss": 2.443, + "theoretical_loss": 3.4135610162490773, + "tokens_seen": 2122252288 + }, + { + "epoch": 0.29, + "learning_rate": 7.209115711763762e-05, + "loss": 2.387, + "theoretical_loss": 3.4135436432290125, + "tokens_seen": 2122383360 + }, + { + "epoch": 0.29, + "learning_rate": 7.208313272347939e-05, + "loss": 2.696, + "theoretical_loss": 3.4135262715822123, + "tokens_seen": 2122514432 + }, + { + "epoch": 0.29, + "learning_rate": 7.207510832932114e-05, + "loss": 2.6137, + "theoretical_loss": 3.4135089013084836, + "tokens_seen": 2122645504 + }, + { + "epoch": 0.29, + "learning_rate": 7.20670839351629e-05, + "loss": 2.4994, + "theoretical_loss": 3.413491532407633, + "tokens_seen": 2122776576 + }, + { + "epoch": 0.29, + "learning_rate": 7.205905954100466e-05, + "loss": 2.6495, + "theoretical_loss": 3.4134741648794664, + "tokens_seen": 2122907648 + }, + { + "epoch": 0.29, + "learning_rate": 7.205103514684642e-05, + "loss": 2.5628, + "theoretical_loss": 3.413456798723792, + "tokens_seen": 2123038720 + }, + { + "epoch": 0.29, + "learning_rate": 7.204301075268818e-05, + "loss": 2.5866, + "theoretical_loss": 3.4134394339404155, + "tokens_seen": 2123169792 + }, + { + "epoch": 0.29, + "learning_rate": 7.203498635852994e-05, + "loss": 2.5491, + "theoretical_loss": 3.413422070529144, + "tokens_seen": 2123300864 + }, + { + "epoch": 0.29, + "learning_rate": 7.20269619643717e-05, + "loss": 2.5522, + "theoretical_loss": 3.413404708489785, + "tokens_seen": 2123431936 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1170253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3250832557678223, + "objective/train/theoretical_loss": 3.4133960279845117, + "objective/train/tokens_used": 493956576, + "theoretical_loss": 3.4133960279845117, + "tokens_seen": 2123497472 + }, + { + "epoch": 0.29, + "learning_rate": 7.201893757021346e-05, + "loss": 2.3746, + "theoretical_loss": 3.4133873478221446, + "tokens_seen": 2123563008 + }, + { + "epoch": 0.29, + "learning_rate": 7.201091317605521e-05, + "loss": 2.4241, + "theoretical_loss": 3.41336998852603, + "tokens_seen": 2123694080 + }, + { + "epoch": 0.29, + "learning_rate": 7.200288878189698e-05, + "loss": 2.6194, + "theoretical_loss": 3.4133526306012483, + "tokens_seen": 2123825152 + }, + { + "epoch": 0.29, + "learning_rate": 7.199486438773873e-05, + "loss": 2.455, + "theoretical_loss": 3.4133352740476064, + "tokens_seen": 2123956224 + }, + { + "epoch": 0.29, + "learning_rate": 7.19868399935805e-05, + "loss": 2.6033, + "theoretical_loss": 3.413317918864912, + "tokens_seen": 2124087296 + }, + { + "epoch": 0.29, + "learning_rate": 7.197881559942225e-05, + "loss": 2.5824, + "theoretical_loss": 3.4133005650529715, + "tokens_seen": 2124218368 + }, + { + "epoch": 0.29, + "learning_rate": 7.197079120526402e-05, + "loss": 2.3911, + "theoretical_loss": 3.413283212611592, + "tokens_seen": 2124349440 + }, + { + "epoch": 0.29, + "learning_rate": 7.196276681110577e-05, + "loss": 2.5731, + "theoretical_loss": 3.413265861540582, + "tokens_seen": 2124480512 + }, + { + "epoch": 0.29, + "learning_rate": 7.195474241694754e-05, + "loss": 2.4229, + "theoretical_loss": 3.413248511839747, + "tokens_seen": 2124611584 + }, + { + "epoch": 0.29, + "learning_rate": 7.194671802278929e-05, + "loss": 2.4563, + "theoretical_loss": 3.413231163508895, + "tokens_seen": 2124742656 + }, + { + "epoch": 0.29, + "learning_rate": 7.193869362863104e-05, + "loss": 2.3896, + "theoretical_loss": 3.4132138165478336, + "tokens_seen": 2124873728 + }, + { + "epoch": 0.29, + "learning_rate": 7.193066923447281e-05, + "loss": 2.5088, + "theoretical_loss": 3.41319647095637, + "tokens_seen": 2125004800 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1171472, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.178572177886963, + "objective/train/theoretical_loss": 3.4131791267343115, + "objective/train/tokens_used": 495594976, + "theoretical_loss": 3.4131791267343115, + "tokens_seen": 2125135872 + }, + { + "epoch": 0.29, + "learning_rate": 7.192264484031456e-05, + "loss": 2.6308, + "theoretical_loss": 3.4131791267343115, + "tokens_seen": 2125135872 + }, + { + "epoch": 0.29, + "learning_rate": 7.191462044615633e-05, + "loss": 2.7299, + "theoretical_loss": 3.413161783881465, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.29, + "learning_rate": 7.190659605199808e-05, + "loss": 2.5909, + "theoretical_loss": 3.4131444423976394, + "tokens_seen": 2125398016 + }, + { + "epoch": 0.29, + "learning_rate": 7.189857165783985e-05, + "loss": 2.5526, + "theoretical_loss": 3.4131271022826413, + "tokens_seen": 2125529088 + }, + { + "epoch": 0.29, + "learning_rate": 7.18905472636816e-05, + "loss": 2.5848, + "theoretical_loss": 3.4131097635362777, + "tokens_seen": 2125660160 + }, + { + "epoch": 0.29, + "learning_rate": 7.188252286952335e-05, + "loss": 2.4472, + "theoretical_loss": 3.4130924261583573, + "tokens_seen": 2125791232 + }, + { + "epoch": 0.29, + "learning_rate": 7.187449847536512e-05, + "loss": 2.3733, + "theoretical_loss": 3.413075090148687, + "tokens_seen": 2125922304 + }, + { + "epoch": 0.29, + "learning_rate": 7.186647408120687e-05, + "loss": 2.59, + "theoretical_loss": 3.413057755507075, + "tokens_seen": 2126053376 + }, + { + "epoch": 0.29, + "learning_rate": 7.185844968704864e-05, + "loss": 2.6016, + "theoretical_loss": 3.413040422233328, + "tokens_seen": 2126184448 + }, + { + "epoch": 0.29, + "learning_rate": 7.18504252928904e-05, + "loss": 2.4636, + "theoretical_loss": 3.413023090327255, + "tokens_seen": 2126315520 + }, + { + "epoch": 0.29, + "learning_rate": 7.184240089873216e-05, + "loss": 2.5986, + "theoretical_loss": 3.413005759788663, + "tokens_seen": 2126446592 + }, + { + "epoch": 0.29, + "learning_rate": 7.183437650457391e-05, + "loss": 2.439, + "theoretical_loss": 3.412988430617361, + "tokens_seen": 2126577664 + }, + { + "epoch": 0.29, + "learning_rate": 7.182635211041567e-05, + "loss": 2.4675, + "theoretical_loss": 3.4129711028131546, + "tokens_seen": 2126708736 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1172068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.215710163116455, + "objective/train/theoretical_loss": 3.4129624394236533, + "objective/train/tokens_used": 497233376, + "theoretical_loss": 3.4129624394236533, + "tokens_seen": 2126774272 + }, + { + "epoch": 0.29, + "learning_rate": 7.181832771625743e-05, + "loss": 2.4494, + "theoretical_loss": 3.4129537763758537, + "tokens_seen": 2126839808 + }, + { + "epoch": 0.29, + "learning_rate": 7.181030332209919e-05, + "loss": 2.4569, + "theoretical_loss": 3.4129364513052654, + "tokens_seen": 2126970880 + }, + { + "epoch": 0.29, + "learning_rate": 7.180227892794095e-05, + "loss": 2.6965, + "theoretical_loss": 3.4129191276011985, + "tokens_seen": 2127101952 + }, + { + "epoch": 0.29, + "learning_rate": 7.17942545337827e-05, + "loss": 2.5618, + "theoretical_loss": 3.4129018052634597, + "tokens_seen": 2127233024 + }, + { + "epoch": 0.29, + "learning_rate": 7.178623013962446e-05, + "loss": 2.5163, + "theoretical_loss": 3.412884484291858, + "tokens_seen": 2127364096 + }, + { + "epoch": 0.29, + "learning_rate": 7.177820574546623e-05, + "loss": 2.3182, + "theoretical_loss": 3.412867164686201, + "tokens_seen": 2127495168 + }, + { + "epoch": 0.29, + "learning_rate": 7.177018135130798e-05, + "loss": 2.6376, + "theoretical_loss": 3.4128498464462975, + "tokens_seen": 2127626240 + }, + { + "epoch": 0.29, + "learning_rate": 7.176215695714975e-05, + "loss": 2.5248, + "theoretical_loss": 3.412832529571955, + "tokens_seen": 2127757312 + }, + { + "epoch": 0.29, + "learning_rate": 7.17541325629915e-05, + "loss": 2.6077, + "theoretical_loss": 3.4128152140629817, + "tokens_seen": 2127888384 + }, + { + "epoch": 0.29, + "learning_rate": 7.174610816883327e-05, + "loss": 2.599, + "theoretical_loss": 3.4127978999191866, + "tokens_seen": 2128019456 + }, + { + "epoch": 0.29, + "learning_rate": 7.173808377467502e-05, + "loss": 2.4737, + "theoretical_loss": 3.412780587140378, + "tokens_seen": 2128150528 + }, + { + "epoch": 0.29, + "learning_rate": 7.173005938051677e-05, + "loss": 2.668, + "theoretical_loss": 3.4127632757263626, + "tokens_seen": 2128281600 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1173293, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4033796787261963, + "objective/train/theoretical_loss": 3.4127459656769505, + "objective/train/tokens_used": 498871776, + "theoretical_loss": 3.4127459656769505, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.29, + "learning_rate": 7.172203498635854e-05, + "loss": 2.4953, + "theoretical_loss": 3.4127459656769505, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.29, + "learning_rate": 7.171401059220029e-05, + "loss": 2.5535, + "theoretical_loss": 3.4127286569919493, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.29, + "learning_rate": 7.170598619804206e-05, + "loss": 2.6874, + "theoretical_loss": 3.412711349671168, + "tokens_seen": 2128674816 + }, + { + "epoch": 0.29, + "learning_rate": 7.169796180388381e-05, + "loss": 2.7642, + "theoretical_loss": 3.4126940437144144, + "tokens_seen": 2128805888 + }, + { + "epoch": 0.29, + "learning_rate": 7.168993740972557e-05, + "loss": 2.5645, + "theoretical_loss": 3.4126767391214976, + "tokens_seen": 2128936960 + }, + { + "epoch": 0.29, + "learning_rate": 7.168191301556733e-05, + "loss": 2.432, + "theoretical_loss": 3.412659435892226, + "tokens_seen": 2129068032 + }, + { + "epoch": 0.29, + "learning_rate": 7.167388862140908e-05, + "loss": 2.6999, + "theoretical_loss": 3.412642134026408, + "tokens_seen": 2129199104 + }, + { + "epoch": 0.29, + "learning_rate": 7.166586422725085e-05, + "loss": 2.7949, + "theoretical_loss": 3.4126248335238527, + "tokens_seen": 2129330176 + }, + { + "epoch": 0.29, + "learning_rate": 7.16578398330926e-05, + "loss": 2.5417, + "theoretical_loss": 3.412607534384368, + "tokens_seen": 2129461248 + }, + { + "epoch": 0.29, + "learning_rate": 7.164981543893436e-05, + "loss": 2.44, + "theoretical_loss": 3.4125902366077634, + "tokens_seen": 2129592320 + }, + { + "epoch": 0.29, + "learning_rate": 7.164179104477612e-05, + "loss": 2.6147, + "theoretical_loss": 3.412572940193847, + "tokens_seen": 2129723392 + }, + { + "epoch": 0.29, + "learning_rate": 7.163376665061788e-05, + "loss": 2.5853, + "theoretical_loss": 3.412555645142428, + "tokens_seen": 2129854464 + }, + { + "epoch": 0.29, + "learning_rate": 7.162574225645964e-05, + "loss": 2.5282, + "theoretical_loss": 3.4125383514533154, + "tokens_seen": 2129985536 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1173914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.899099111557007, + "objective/train/theoretical_loss": 3.412529705119564, + "objective/train/tokens_used": 500510176, + "theoretical_loss": 3.412529705119564, + "tokens_seen": 2130051072 + }, + { + "epoch": 0.29, + "learning_rate": 7.16177178623014e-05, + "loss": 2.7238, + "theoretical_loss": 3.4125210591263175, + "tokens_seen": 2130116608 + }, + { + "epoch": 0.29, + "learning_rate": 7.160969346814316e-05, + "loss": 2.476, + "theoretical_loss": 3.412503768161244, + "tokens_seen": 2130247680 + }, + { + "epoch": 0.29, + "learning_rate": 7.160166907398492e-05, + "loss": 2.6226, + "theoretical_loss": 3.412486478557903, + "tokens_seen": 2130378752 + }, + { + "epoch": 0.29, + "learning_rate": 7.159364467982667e-05, + "loss": 2.5487, + "theoretical_loss": 3.412469190316104, + "tokens_seen": 2130509824 + }, + { + "epoch": 0.29, + "learning_rate": 7.158562028566844e-05, + "loss": 2.613, + "theoretical_loss": 3.412451903435656, + "tokens_seen": 2130640896 + }, + { + "epoch": 0.29, + "learning_rate": 7.157759589151019e-05, + "loss": 2.5264, + "theoretical_loss": 3.412434617916368, + "tokens_seen": 2130771968 + }, + { + "epoch": 0.29, + "learning_rate": 7.156957149735196e-05, + "loss": 2.5246, + "theoretical_loss": 3.4124173337580492, + "tokens_seen": 2130903040 + }, + { + "epoch": 0.29, + "learning_rate": 7.156154710319371e-05, + "loss": 2.5873, + "theoretical_loss": 3.412400050960508, + "tokens_seen": 2131034112 + }, + { + "epoch": 0.29, + "learning_rate": 7.155352270903546e-05, + "loss": 2.4861, + "theoretical_loss": 3.4123827695235542, + "tokens_seen": 2131165184 + }, + { + "epoch": 0.29, + "learning_rate": 7.154549831487723e-05, + "loss": 2.4371, + "theoretical_loss": 3.412365489446998, + "tokens_seen": 2131296256 + }, + { + "epoch": 0.29, + "learning_rate": 7.153747392071898e-05, + "loss": 2.6413, + "theoretical_loss": 3.412348210730647, + "tokens_seen": 2131427328 + }, + { + "epoch": 0.29, + "learning_rate": 7.152944952656075e-05, + "loss": 2.63, + "theoretical_loss": 3.412330933374311, + "tokens_seen": 2131558400 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1174844, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1257615089416504, + "objective/train/theoretical_loss": 3.4123136573777995, + "objective/train/tokens_used": 502148576, + "theoretical_loss": 3.4123136573777995, + "tokens_seen": 2131689472 + }, + { + "epoch": 0.29, + "learning_rate": 7.15214251324025e-05, + "loss": 2.4612, + "theoretical_loss": 3.4123136573777995, + "tokens_seen": 2131689472 + }, + { + "epoch": 0.29, + "learning_rate": 7.151340073824427e-05, + "loss": 2.5941, + "theoretical_loss": 3.412296382740922, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.29, + "learning_rate": 7.150537634408602e-05, + "loss": 2.627, + "theoretical_loss": 3.412279109463488, + "tokens_seen": 2131951616 + }, + { + "epoch": 0.29, + "learning_rate": 7.149735194992778e-05, + "loss": 2.6469, + "theoretical_loss": 3.412261837545307, + "tokens_seen": 2132082688 + }, + { + "epoch": 0.29, + "learning_rate": 7.148932755576954e-05, + "loss": 2.5935, + "theoretical_loss": 3.412244566986187, + "tokens_seen": 2132213760 + }, + { + "epoch": 0.29, + "learning_rate": 7.14813031616113e-05, + "loss": 2.6418, + "theoretical_loss": 3.4122272977859396, + "tokens_seen": 2132344832 + }, + { + "epoch": 0.29, + "learning_rate": 7.147327876745306e-05, + "loss": 2.3957, + "theoretical_loss": 3.412210029944374, + "tokens_seen": 2132475904 + }, + { + "epoch": 0.29, + "learning_rate": 7.146525437329482e-05, + "loss": 2.485, + "theoretical_loss": 3.4121927634612987, + "tokens_seen": 2132606976 + }, + { + "epoch": 0.29, + "learning_rate": 7.145722997913657e-05, + "loss": 2.6645, + "theoretical_loss": 3.4121754983365236, + "tokens_seen": 2132738048 + }, + { + "epoch": 0.29, + "learning_rate": 7.144920558497834e-05, + "loss": 2.559, + "theoretical_loss": 3.4121582345698593, + "tokens_seen": 2132869120 + }, + { + "epoch": 0.29, + "learning_rate": 7.144118119082009e-05, + "loss": 2.4919, + "theoretical_loss": 3.4121409721611147, + "tokens_seen": 2133000192 + }, + { + "epoch": 0.29, + "learning_rate": 7.143315679666185e-05, + "loss": 2.6485, + "theoretical_loss": 3.4121237111101, + "tokens_seen": 2133131264 + }, + { + "epoch": 0.29, + "learning_rate": 7.142513240250361e-05, + "loss": 2.671, + "theoretical_loss": 3.4121064514166246, + "tokens_seen": 2133262336 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1175941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.085177183151245, + "objective/train/theoretical_loss": 3.4120978220789047, + "objective/train/tokens_used": 503786976, + "theoretical_loss": 3.4120978220789047, + "tokens_seen": 2133327872 + }, + { + "epoch": 0.29, + "learning_rate": 7.141710800834537e-05, + "loss": 2.5607, + "theoretical_loss": 3.412089193080498, + "tokens_seen": 2133393408 + }, + { + "epoch": 0.29, + "learning_rate": 7.140908361418713e-05, + "loss": 2.4615, + "theoretical_loss": 3.4120719361015315, + "tokens_seen": 2133524480 + }, + { + "epoch": 0.29, + "learning_rate": 7.140105922002888e-05, + "loss": 2.628, + "theoretical_loss": 3.4120546804795335, + "tokens_seen": 2133655552 + }, + { + "epoch": 0.29, + "learning_rate": 7.139303482587065e-05, + "loss": 2.5597, + "theoretical_loss": 3.412037426214315, + "tokens_seen": 2133786624 + }, + { + "epoch": 0.29, + "learning_rate": 7.13850104317124e-05, + "loss": 2.605, + "theoretical_loss": 3.4120201733056854, + "tokens_seen": 2133917696 + }, + { + "epoch": 0.29, + "learning_rate": 7.137698603755417e-05, + "loss": 2.7064, + "theoretical_loss": 3.412002921753455, + "tokens_seen": 2134048768 + }, + { + "epoch": 0.29, + "learning_rate": 7.136896164339592e-05, + "loss": 2.4825, + "theoretical_loss": 3.4119856715574333, + "tokens_seen": 2134179840 + }, + { + "epoch": 0.29, + "learning_rate": 7.136093724923767e-05, + "loss": 2.6736, + "theoretical_loss": 3.411968422717431, + "tokens_seen": 2134310912 + }, + { + "epoch": 0.29, + "learning_rate": 7.135291285507944e-05, + "loss": 2.6355, + "theoretical_loss": 3.4119511752332583, + "tokens_seen": 2134441984 + }, + { + "epoch": 0.29, + "learning_rate": 7.13448884609212e-05, + "loss": 2.6273, + "theoretical_loss": 3.411933929104725, + "tokens_seen": 2134573056 + }, + { + "epoch": 0.29, + "learning_rate": 7.133686406676296e-05, + "loss": 2.4734, + "theoretical_loss": 3.4119166843316413, + "tokens_seen": 2134704128 + }, + { + "epoch": 0.29, + "learning_rate": 7.132883967260471e-05, + "loss": 2.5344, + "theoretical_loss": 3.411899440913818, + "tokens_seen": 2134835200 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1176552, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7227025032043457, + "objective/train/theoretical_loss": 3.411882198851065, + "objective/train/tokens_used": 505425376, + "theoretical_loss": 3.411882198851065, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.29, + "learning_rate": 7.132081527844648e-05, + "loss": 2.5944, + "theoretical_loss": 3.411882198851065, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.29, + "learning_rate": 7.131279088428823e-05, + "loss": 2.5057, + "theoretical_loss": 3.411864958143192, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.29, + "learning_rate": 7.130476649013e-05, + "loss": 2.6437, + "theoretical_loss": 3.4118477187900105, + "tokens_seen": 2135228416 + }, + { + "epoch": 0.29, + "learning_rate": 7.129674209597175e-05, + "loss": 2.5914, + "theoretical_loss": 3.41183048079133, + "tokens_seen": 2135359488 + }, + { + "epoch": 0.29, + "learning_rate": 7.12887177018135e-05, + "loss": 2.6825, + "theoretical_loss": 3.4118132441469617, + "tokens_seen": 2135490560 + }, + { + "epoch": 0.29, + "learning_rate": 7.128069330765527e-05, + "loss": 2.5187, + "theoretical_loss": 3.4117960088567156, + "tokens_seen": 2135621632 + }, + { + "epoch": 0.29, + "learning_rate": 7.127266891349703e-05, + "loss": 2.599, + "theoretical_loss": 3.4117787749204025, + "tokens_seen": 2135752704 + }, + { + "epoch": 0.29, + "learning_rate": 7.126464451933879e-05, + "loss": 2.5976, + "theoretical_loss": 3.4117615423378327, + "tokens_seen": 2135883776 + }, + { + "epoch": 0.29, + "learning_rate": 7.125662012518055e-05, + "loss": 2.543, + "theoretical_loss": 3.4117443111088175, + "tokens_seen": 2136014848 + }, + { + "epoch": 0.29, + "learning_rate": 7.124859573102231e-05, + "loss": 2.6293, + "theoretical_loss": 3.411727081233166, + "tokens_seen": 2136145920 + }, + { + "epoch": 0.29, + "learning_rate": 7.124057133686407e-05, + "loss": 2.4895, + "theoretical_loss": 3.41170985271069, + "tokens_seen": 2136276992 + }, + { + "epoch": 0.29, + "learning_rate": 7.123254694270582e-05, + "loss": 2.4808, + "theoretical_loss": 3.4116926255412006, + "tokens_seen": 2136408064 + }, + { + "epoch": 0.29, + "learning_rate": 7.122452254854759e-05, + "loss": 2.6091, + "theoretical_loss": 3.4116753997245075, + "tokens_seen": 2136539136 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1177438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7546355724334717, + "objective/train/theoretical_loss": 3.4116667873234006, + "objective/train/tokens_used": 507063776, + "theoretical_loss": 3.4116667873234006, + "tokens_seen": 2136604672 + }, + { + "epoch": 0.29, + "learning_rate": 7.121649815438934e-05, + "loss": 2.6309, + "theoretical_loss": 3.4116581752604223, + "tokens_seen": 2136670208 + }, + { + "epoch": 0.3, + "learning_rate": 7.12084737602311e-05, + "loss": 2.6768, + "theoretical_loss": 3.411640952148755, + "tokens_seen": 2136801280 + }, + { + "epoch": 0.3, + "learning_rate": 7.120044936607286e-05, + "loss": 2.6746, + "theoretical_loss": 3.4116237303893175, + "tokens_seen": 2136932352 + }, + { + "epoch": 0.3, + "learning_rate": 7.119242497191462e-05, + "loss": 2.5923, + "theoretical_loss": 3.4116065099819197, + "tokens_seen": 2137063424 + }, + { + "epoch": 0.3, + "learning_rate": 7.118440057775638e-05, + "loss": 2.6047, + "theoretical_loss": 3.411589290926374, + "tokens_seen": 2137194496 + }, + { + "epoch": 0.3, + "learning_rate": 7.117637618359813e-05, + "loss": 2.61, + "theoretical_loss": 3.411572073222489, + "tokens_seen": 2137325568 + }, + { + "epoch": 0.3, + "learning_rate": 7.11683517894399e-05, + "loss": 2.686, + "theoretical_loss": 3.4115548568700778, + "tokens_seen": 2137456640 + }, + { + "epoch": 0.3, + "learning_rate": 7.116032739528165e-05, + "loss": 2.5895, + "theoretical_loss": 3.411537641868951, + "tokens_seen": 2137587712 + }, + { + "epoch": 0.3, + "learning_rate": 7.115230300112342e-05, + "loss": 2.5471, + "theoretical_loss": 3.411520428218919, + "tokens_seen": 2137718784 + }, + { + "epoch": 0.3, + "learning_rate": 7.114427860696517e-05, + "loss": 2.399, + "theoretical_loss": 3.4115032159197938, + "tokens_seen": 2137849856 + }, + { + "epoch": 0.3, + "learning_rate": 7.113625421280694e-05, + "loss": 2.5753, + "theoretical_loss": 3.4114860049713855, + "tokens_seen": 2137980928 + }, + { + "epoch": 0.3, + "learning_rate": 7.112822981864869e-05, + "loss": 2.725, + "theoretical_loss": 3.4114687953735063, + "tokens_seen": 2138112000 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1177662, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.830423593521118, + "objective/train/theoretical_loss": 3.411451587125967, + "objective/train/tokens_used": 508702176, + "theoretical_loss": 3.411451587125967, + "tokens_seen": 2138243072 + }, + { + "epoch": 0.3, + "learning_rate": 7.112020542449044e-05, + "loss": 2.6082, + "theoretical_loss": 3.411451587125967, + "tokens_seen": 2138243072 + }, + { + "epoch": 0.3, + "learning_rate": 7.111218103033221e-05, + "loss": 2.5201, + "theoretical_loss": 3.411434380228579, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.3, + "learning_rate": 7.110415663617396e-05, + "loss": 2.5066, + "theoretical_loss": 3.4114171746811537, + "tokens_seen": 2138505216 + }, + { + "epoch": 0.3, + "learning_rate": 7.109613224201573e-05, + "loss": 2.6115, + "theoretical_loss": 3.4113999704835023, + "tokens_seen": 2138636288 + }, + { + "epoch": 0.3, + "learning_rate": 7.108810784785748e-05, + "loss": 2.6089, + "theoretical_loss": 3.4113827676354362, + "tokens_seen": 2138767360 + }, + { + "epoch": 0.3, + "learning_rate": 7.108008345369925e-05, + "loss": 2.7135, + "theoretical_loss": 3.411365566136767, + "tokens_seen": 2138898432 + }, + { + "epoch": 0.3, + "learning_rate": 7.1072059059541e-05, + "loss": 2.6082, + "theoretical_loss": 3.4113483659873056, + "tokens_seen": 2139029504 + }, + { + "epoch": 0.3, + "learning_rate": 7.106403466538276e-05, + "loss": 2.597, + "theoretical_loss": 3.4113311671868645, + "tokens_seen": 2139160576 + }, + { + "epoch": 0.3, + "learning_rate": 7.105601027122452e-05, + "loss": 2.7267, + "theoretical_loss": 3.4113139697352546, + "tokens_seen": 2139291648 + }, + { + "epoch": 0.3, + "learning_rate": 7.104798587706628e-05, + "loss": 2.533, + "theoretical_loss": 3.4112967736322872, + "tokens_seen": 2139422720 + }, + { + "epoch": 0.3, + "learning_rate": 7.103996148290804e-05, + "loss": 2.5938, + "theoretical_loss": 3.4112795788777746, + "tokens_seen": 2139553792 + }, + { + "epoch": 0.3, + "learning_rate": 7.10319370887498e-05, + "loss": 2.6091, + "theoretical_loss": 3.4112623854715283, + "tokens_seen": 2139684864 + }, + { + "epoch": 0.3, + "learning_rate": 7.102391269459156e-05, + "loss": 2.6935, + "theoretical_loss": 3.41124519341336, + "tokens_seen": 2139815936 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1178610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.592621088027954, + "objective/train/theoretical_loss": 3.4112365978897463, + "objective/train/tokens_used": 510340576, + "theoretical_loss": 3.4112365978897463, + "tokens_seen": 2139881472 + }, + { + "epoch": 0.3, + "learning_rate": 7.101588830043332e-05, + "loss": 2.612, + "theoretical_loss": 3.411228002703081, + "tokens_seen": 2139947008 + }, + { + "epoch": 0.3, + "learning_rate": 7.100786390627508e-05, + "loss": 2.6124, + "theoretical_loss": 3.4112108133405035, + "tokens_seen": 2140078080 + }, + { + "epoch": 0.3, + "learning_rate": 7.099983951211684e-05, + "loss": 2.5644, + "theoretical_loss": 3.4111936253254393, + "tokens_seen": 2140209152 + }, + { + "epoch": 0.3, + "learning_rate": 7.099181511795859e-05, + "loss": 2.6921, + "theoretical_loss": 3.4111764386576997, + "tokens_seen": 2140340224 + }, + { + "epoch": 0.3, + "learning_rate": 7.098379072380036e-05, + "loss": 2.5306, + "theoretical_loss": 3.4111592533370976, + "tokens_seen": 2140471296 + }, + { + "epoch": 0.3, + "learning_rate": 7.097576632964211e-05, + "loss": 2.6231, + "theoretical_loss": 3.4111420693634438, + "tokens_seen": 2140602368 + }, + { + "epoch": 0.3, + "learning_rate": 7.096774193548388e-05, + "loss": 2.5484, + "theoretical_loss": 3.411124886736551, + "tokens_seen": 2140733440 + }, + { + "epoch": 0.3, + "learning_rate": 7.095971754132563e-05, + "loss": 2.5626, + "theoretical_loss": 3.4111077054562315, + "tokens_seen": 2140864512 + }, + { + "epoch": 0.3, + "learning_rate": 7.09516931471674e-05, + "loss": 2.5805, + "theoretical_loss": 3.4110905255222965, + "tokens_seen": 2140995584 + }, + { + "epoch": 0.3, + "learning_rate": 7.094366875300915e-05, + "loss": 2.5912, + "theoretical_loss": 3.4110733469345584, + "tokens_seen": 2141126656 + }, + { + "epoch": 0.3, + "learning_rate": 7.09356443588509e-05, + "loss": 2.5502, + "theoretical_loss": 3.4110561696928294, + "tokens_seen": 2141257728 + }, + { + "epoch": 0.3, + "learning_rate": 7.092761996469267e-05, + "loss": 2.6747, + "theoretical_loss": 3.4110389937969217, + "tokens_seen": 2141388800 + }, + { + "debugging/Self-BLEU-5": 0.3562156871264047, + "debugging/distinct-1-grams": 0.8053620964343855, + "debugging/distinct-2-grams": 0.9818306010928961, + "debugging/entropy-1-grams": 5.331752904391725, + "debugging/entropy-2-grams": 5.892259632920162, + "debugging/length": 535.8, + "debugging/num_segments": 5, + "debugging/score": 0.007634856930105227, + "debugging/score_std": 0.0023886908693567656, + "epoch": 0.3, + "objective/train/docs_used": 1179232, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.219043016433716, + "objective/train/theoretical_loss": 3.4110218192466473, + "objective/train/tokens_used": 511978976, + "theoretical_loss": 3.4110218192466473, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.3, + "learning_rate": 7.091959557053442e-05, + "loss": 2.6092, + "theoretical_loss": 3.4110218192466473, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.3, + "learning_rate": 7.091157117637619e-05, + "loss": 2.7279, + "theoretical_loss": 3.4110046460418184, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.3, + "learning_rate": 7.090354678221794e-05, + "loss": 2.6784, + "theoretical_loss": 3.4109874741822477, + "tokens_seen": 2141782016 + }, + { + "epoch": 0.3, + "learning_rate": 7.089552238805971e-05, + "loss": 2.6052, + "theoretical_loss": 3.410970303667747, + "tokens_seen": 2141913088 + }, + { + "epoch": 0.3, + "learning_rate": 7.088749799390146e-05, + "loss": 2.5963, + "theoretical_loss": 3.4109531344981283, + "tokens_seen": 2142044160 + }, + { + "epoch": 0.3, + "learning_rate": 7.087947359974321e-05, + "loss": 2.6833, + "theoretical_loss": 3.4109359666732053, + "tokens_seen": 2142175232 + }, + { + "epoch": 0.3, + "learning_rate": 7.087144920558498e-05, + "loss": 2.6276, + "theoretical_loss": 3.410918800192789, + "tokens_seen": 2142306304 + }, + { + "epoch": 0.3, + "learning_rate": 7.086342481142673e-05, + "loss": 2.5189, + "theoretical_loss": 3.4109016350566934, + "tokens_seen": 2142437376 + }, + { + "epoch": 0.3, + "learning_rate": 7.08554004172685e-05, + "loss": 2.5976, + "theoretical_loss": 3.410884471264729, + "tokens_seen": 2142568448 + }, + { + "epoch": 0.3, + "learning_rate": 7.084737602311025e-05, + "loss": 2.5985, + "theoretical_loss": 3.41086730881671, + "tokens_seen": 2142699520 + }, + { + "epoch": 0.3, + "learning_rate": 7.083935162895202e-05, + "loss": 2.6729, + "theoretical_loss": 3.4108501477124484, + "tokens_seen": 2142830592 + }, + { + "epoch": 0.3, + "learning_rate": 7.083132723479377e-05, + "loss": 2.5663, + "theoretical_loss": 3.4108329879517565, + "tokens_seen": 2142961664 + }, + { + "epoch": 0.3, + "learning_rate": 7.082330284063553e-05, + "loss": 2.6607, + "theoretical_loss": 3.4108158295344473, + "tokens_seen": 2143092736 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1180509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.578857421875, + "objective/train/theoretical_loss": 3.4108072508295026, + "objective/train/tokens_used": 513617376, + "theoretical_loss": 3.4108072508295026, + "tokens_seen": 2143158272 + }, + { + "epoch": 0.3, + "learning_rate": 7.081527844647729e-05, + "loss": 2.563, + "theoretical_loss": 3.4107986724603334, + "tokens_seen": 2143223808 + }, + { + "epoch": 0.3, + "learning_rate": 7.080725405231905e-05, + "loss": 2.4593, + "theoretical_loss": 3.4107815167292275, + "tokens_seen": 2143354880 + }, + { + "epoch": 0.3, + "learning_rate": 7.079922965816081e-05, + "loss": 2.6285, + "theoretical_loss": 3.4107643623409425, + "tokens_seen": 2143485952 + }, + { + "epoch": 0.3, + "learning_rate": 7.079120526400257e-05, + "loss": 2.6234, + "theoretical_loss": 3.410747209295291, + "tokens_seen": 2143617024 + }, + { + "epoch": 0.3, + "learning_rate": 7.078318086984433e-05, + "loss": 2.558, + "theoretical_loss": 3.4107300575920854, + "tokens_seen": 2143748096 + }, + { + "epoch": 0.3, + "learning_rate": 7.077515647568609e-05, + "loss": 2.4592, + "theoretical_loss": 3.4107129072311393, + "tokens_seen": 2143879168 + }, + { + "epoch": 0.3, + "learning_rate": 7.076713208152784e-05, + "loss": 2.6204, + "theoretical_loss": 3.410695758212266, + "tokens_seen": 2144010240 + }, + { + "epoch": 0.3, + "learning_rate": 7.07591076873696e-05, + "loss": 2.7057, + "theoretical_loss": 3.4106786105352773, + "tokens_seen": 2144141312 + }, + { + "epoch": 0.3, + "learning_rate": 7.075108329321136e-05, + "loss": 2.6211, + "theoretical_loss": 3.410661464199986, + "tokens_seen": 2144272384 + }, + { + "epoch": 0.3, + "learning_rate": 7.074305889905313e-05, + "loss": 2.5688, + "theoretical_loss": 3.410644319206207, + "tokens_seen": 2144403456 + }, + { + "epoch": 0.3, + "learning_rate": 7.073503450489488e-05, + "loss": 2.6275, + "theoretical_loss": 3.4106271755537514, + "tokens_seen": 2144534528 + }, + { + "epoch": 0.3, + "learning_rate": 7.072701011073665e-05, + "loss": 2.5701, + "theoretical_loss": 3.410610033242433, + "tokens_seen": 2144665600 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1181762, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5804944038391113, + "objective/train/theoretical_loss": 3.4105928922720654, + "objective/train/tokens_used": 515255776, + "theoretical_loss": 3.4105928922720654, + "tokens_seen": 2144796672 + }, + { + "epoch": 0.3, + "learning_rate": 7.07189857165784e-05, + "loss": 2.5957, + "theoretical_loss": 3.4105928922720654, + "tokens_seen": 2144796672 + }, + { + "epoch": 0.3, + "learning_rate": 7.071096132242017e-05, + "loss": 2.6133, + "theoretical_loss": 3.410575752642461, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.3, + "learning_rate": 7.070293692826192e-05, + "loss": 2.5896, + "theoretical_loss": 3.4105586143534334, + "tokens_seen": 2145058816 + }, + { + "epoch": 0.3, + "learning_rate": 7.069491253410367e-05, + "loss": 2.4899, + "theoretical_loss": 3.410541477404796, + "tokens_seen": 2145189888 + }, + { + "epoch": 0.3, + "learning_rate": 7.068688813994544e-05, + "loss": 2.6221, + "theoretical_loss": 3.4105243417963615, + "tokens_seen": 2145320960 + }, + { + "epoch": 0.3, + "learning_rate": 7.067886374578719e-05, + "loss": 2.593, + "theoretical_loss": 3.4105072075279432, + "tokens_seen": 2145452032 + }, + { + "epoch": 0.3, + "learning_rate": 7.067083935162896e-05, + "loss": 2.6936, + "theoretical_loss": 3.410490074599356, + "tokens_seen": 2145583104 + }, + { + "epoch": 0.3, + "learning_rate": 7.066281495747071e-05, + "loss": 2.5297, + "theoretical_loss": 3.4104729430104115, + "tokens_seen": 2145714176 + }, + { + "epoch": 0.3, + "learning_rate": 7.065479056331248e-05, + "loss": 2.6632, + "theoretical_loss": 3.4104558127609232, + "tokens_seen": 2145845248 + }, + { + "epoch": 0.3, + "learning_rate": 7.064676616915423e-05, + "loss": 2.6771, + "theoretical_loss": 3.410438683850706, + "tokens_seen": 2145976320 + }, + { + "epoch": 0.3, + "learning_rate": 7.063874177499598e-05, + "loss": 2.667, + "theoretical_loss": 3.410421556279572, + "tokens_seen": 2146107392 + }, + { + "epoch": 0.3, + "learning_rate": 7.063071738083775e-05, + "loss": 2.5011, + "theoretical_loss": 3.4104044300473353, + "tokens_seen": 2146238464 + }, + { + "epoch": 0.3, + "learning_rate": 7.06226929866795e-05, + "loss": 2.626, + "theoretical_loss": 3.4103873051538094, + "tokens_seen": 2146369536 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1182263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.857555866241455, + "objective/train/theoretical_loss": 3.410378743209005, + "objective/train/tokens_used": 516894176, + "theoretical_loss": 3.410378743209005, + "tokens_seen": 2146435072 + }, + { + "epoch": 0.3, + "learning_rate": 7.061466859252127e-05, + "loss": 2.6154, + "theoretical_loss": 3.4103701815988083, + "tokens_seen": 2146500608 + }, + { + "epoch": 0.3, + "learning_rate": 7.060664419836302e-05, + "loss": 2.4926, + "theoretical_loss": 3.4103530593821447, + "tokens_seen": 2146631680 + }, + { + "epoch": 0.3, + "learning_rate": 7.059861980420479e-05, + "loss": 2.7829, + "theoretical_loss": 3.4103359385036334, + "tokens_seen": 2146762752 + }, + { + "epoch": 0.3, + "learning_rate": 7.059059541004654e-05, + "loss": 2.5032, + "theoretical_loss": 3.4103188189630878, + "tokens_seen": 2146893824 + }, + { + "epoch": 0.3, + "learning_rate": 7.05825710158883e-05, + "loss": 2.5789, + "theoretical_loss": 3.4103017007603205, + "tokens_seen": 2147024896 + }, + { + "epoch": 0.3, + "learning_rate": 7.057454662173006e-05, + "loss": 2.5056, + "theoretical_loss": 3.410284583895147, + "tokens_seen": 2147155968 + }, + { + "epoch": 0.3, + "learning_rate": 7.056652222757182e-05, + "loss": 2.5706, + "theoretical_loss": 3.4102674683673806, + "tokens_seen": 2147287040 + }, + { + "epoch": 0.3, + "learning_rate": 7.055849783341358e-05, + "loss": 2.6252, + "theoretical_loss": 3.4102503541768345, + "tokens_seen": 2147418112 + }, + { + "epoch": 0.3, + "learning_rate": 7.055047343925534e-05, + "loss": 2.5292, + "theoretical_loss": 3.4102332413233234, + "tokens_seen": 2147549184 + }, + { + "epoch": 0.3, + "learning_rate": 7.05424490450971e-05, + "loss": 2.6922, + "theoretical_loss": 3.4102161298066607, + "tokens_seen": 2147680256 + }, + { + "epoch": 0.3, + "learning_rate": 7.053442465093886e-05, + "loss": 2.6755, + "theoretical_loss": 3.410199019626661, + "tokens_seen": 2147811328 + }, + { + "epoch": 0.3, + "learning_rate": 7.052640025678061e-05, + "loss": 2.5742, + "theoretical_loss": 3.410181910783138, + "tokens_seen": 2147942400 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1183255, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4469528198242188, + "objective/train/theoretical_loss": 3.4101648032759053, + "objective/train/tokens_used": 518532576, + "theoretical_loss": 3.4101648032759053, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.3, + "learning_rate": 7.051837586262238e-05, + "loss": 2.4785, + "theoretical_loss": 3.4101648032759053, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.3, + "learning_rate": 7.051035146846413e-05, + "loss": 2.6102, + "theoretical_loss": 3.4101476971047777, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.3, + "learning_rate": 7.05023270743059e-05, + "loss": 2.5222, + "theoretical_loss": 3.410130592269569, + "tokens_seen": 2148335616 + }, + { + "epoch": 0.3, + "learning_rate": 7.049430268014765e-05, + "loss": 2.665, + "theoretical_loss": 3.4101134887700937, + "tokens_seen": 2148466688 + }, + { + "epoch": 0.3, + "learning_rate": 7.048627828598942e-05, + "loss": 2.6828, + "theoretical_loss": 3.410096386606166, + "tokens_seen": 2148597760 + }, + { + "epoch": 0.3, + "learning_rate": 7.047825389183117e-05, + "loss": 2.5421, + "theoretical_loss": 3.4100792857775994, + "tokens_seen": 2148728832 + }, + { + "epoch": 0.3, + "learning_rate": 7.047022949767292e-05, + "loss": 2.6022, + "theoretical_loss": 3.4100621862842084, + "tokens_seen": 2148859904 + }, + { + "epoch": 0.3, + "learning_rate": 7.046220510351469e-05, + "loss": 2.6454, + "theoretical_loss": 3.410045088125808, + "tokens_seen": 2148990976 + }, + { + "epoch": 0.3, + "learning_rate": 7.045418070935644e-05, + "loss": 2.5891, + "theoretical_loss": 3.4100279913022122, + "tokens_seen": 2149122048 + }, + { + "epoch": 0.3, + "learning_rate": 7.044615631519821e-05, + "loss": 2.6476, + "theoretical_loss": 3.410010895813236, + "tokens_seen": 2149253120 + }, + { + "epoch": 0.3, + "learning_rate": 7.043813192103996e-05, + "loss": 2.4586, + "theoretical_loss": 3.4099938016586924, + "tokens_seen": 2149384192 + }, + { + "epoch": 0.3, + "learning_rate": 7.043010752688173e-05, + "loss": 2.6862, + "theoretical_loss": 3.4099767088383968, + "tokens_seen": 2149515264 + }, + { + "epoch": 0.3, + "learning_rate": 7.042208313272348e-05, + "loss": 2.5898, + "theoretical_loss": 3.409959617352164, + "tokens_seen": 2149646336 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1183895, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.68380069732666, + "objective/train/theoretical_loss": 3.4099510721092625, + "objective/train/tokens_used": 520170976, + "theoretical_loss": 3.4099510721092625, + "tokens_seen": 2149711872 + }, + { + "epoch": 0.3, + "learning_rate": 7.041405873856523e-05, + "loss": 2.6714, + "theoretical_loss": 3.4099425271998074, + "tokens_seen": 2149777408 + }, + { + "epoch": 0.3, + "learning_rate": 7.0406034344407e-05, + "loss": 2.5034, + "theoretical_loss": 3.409925438381143, + "tokens_seen": 2149908480 + }, + { + "epoch": 0.3, + "learning_rate": 7.039800995024875e-05, + "loss": 2.5538, + "theoretical_loss": 3.409908350895984, + "tokens_seen": 2150039552 + }, + { + "epoch": 0.3, + "learning_rate": 7.038998555609052e-05, + "loss": 2.4173, + "theoretical_loss": 3.4098912647441466, + "tokens_seen": 2150170624 + }, + { + "epoch": 0.3, + "learning_rate": 7.038196116193227e-05, + "loss": 2.7148, + "theoretical_loss": 3.4098741799254446, + "tokens_seen": 2150301696 + }, + { + "epoch": 0.3, + "learning_rate": 7.037393676777404e-05, + "loss": 2.4907, + "theoretical_loss": 3.4098570964396924, + "tokens_seen": 2150432768 + }, + { + "epoch": 0.3, + "learning_rate": 7.03659123736158e-05, + "loss": 2.6022, + "theoretical_loss": 3.409840014286705, + "tokens_seen": 2150563840 + }, + { + "epoch": 0.3, + "learning_rate": 7.035788797945756e-05, + "loss": 2.487, + "theoretical_loss": 3.409822933466298, + "tokens_seen": 2150694912 + }, + { + "epoch": 0.3, + "learning_rate": 7.034986358529931e-05, + "loss": 2.4843, + "theoretical_loss": 3.4098058539782854, + "tokens_seen": 2150825984 + }, + { + "epoch": 0.3, + "learning_rate": 7.034183919114107e-05, + "loss": 2.5202, + "theoretical_loss": 3.4097887758224825, + "tokens_seen": 2150957056 + }, + { + "epoch": 0.3, + "learning_rate": 7.033381479698283e-05, + "loss": 2.6072, + "theoretical_loss": 3.4097716989987035, + "tokens_seen": 2151088128 + }, + { + "epoch": 0.3, + "learning_rate": 7.032579040282459e-05, + "loss": 2.613, + "theoretical_loss": 3.4097546235067644, + "tokens_seen": 2151219200 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1184816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.916866660118103, + "objective/train/theoretical_loss": 3.409737549346479, + "objective/train/tokens_used": 521809376, + "theoretical_loss": 3.409737549346479, + "tokens_seen": 2151350272 + }, + { + "epoch": 0.3, + "learning_rate": 7.031776600866635e-05, + "loss": 2.57, + "theoretical_loss": 3.409737549346479, + "tokens_seen": 2151350272 + }, + { + "epoch": 0.3, + "learning_rate": 7.03097416145081e-05, + "loss": 2.4954, + "theoretical_loss": 3.409720476517664, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.3, + "learning_rate": 7.030171722034987e-05, + "loss": 2.6051, + "theoretical_loss": 3.4097034050201334, + "tokens_seen": 2151612416 + }, + { + "epoch": 0.3, + "learning_rate": 7.029369282619163e-05, + "loss": 2.4134, + "theoretical_loss": 3.4096863348537014, + "tokens_seen": 2151743488 + }, + { + "epoch": 0.3, + "learning_rate": 7.028566843203338e-05, + "loss": 2.695, + "theoretical_loss": 3.409669266018185, + "tokens_seen": 2151874560 + }, + { + "epoch": 0.3, + "learning_rate": 7.027764403787515e-05, + "loss": 2.5432, + "theoretical_loss": 3.409652198513398, + "tokens_seen": 2152005632 + }, + { + "epoch": 0.3, + "learning_rate": 7.02696196437169e-05, + "loss": 2.6862, + "theoretical_loss": 3.4096351323391563, + "tokens_seen": 2152136704 + }, + { + "epoch": 0.3, + "learning_rate": 7.026159524955867e-05, + "loss": 2.6014, + "theoretical_loss": 3.4096180674952747, + "tokens_seen": 2152267776 + }, + { + "epoch": 0.3, + "learning_rate": 7.025357085540042e-05, + "loss": 2.6281, + "theoretical_loss": 3.409601003981569, + "tokens_seen": 2152398848 + }, + { + "epoch": 0.3, + "learning_rate": 7.024554646124219e-05, + "loss": 2.7042, + "theoretical_loss": 3.4095839417978544, + "tokens_seen": 2152529920 + }, + { + "epoch": 0.3, + "learning_rate": 7.023752206708394e-05, + "loss": 2.5141, + "theoretical_loss": 3.409566880943945, + "tokens_seen": 2152660992 + }, + { + "epoch": 0.3, + "learning_rate": 7.022949767292569e-05, + "loss": 2.5674, + "theoretical_loss": 3.4095498214196587, + "tokens_seen": 2152792064 + }, + { + "epoch": 0.3, + "learning_rate": 7.022147327876746e-05, + "loss": 2.5695, + "theoretical_loss": 3.409532763224809, + "tokens_seen": 2152923136 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1185361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4470932483673096, + "objective/train/theoretical_loss": 3.4095242346258656, + "objective/train/tokens_used": 523447776, + "theoretical_loss": 3.4095242346258656, + "tokens_seen": 2152988672 + }, + { + "epoch": 0.3, + "learning_rate": 7.021344888460921e-05, + "loss": 2.6913, + "theoretical_loss": 3.409515706359212, + "tokens_seen": 2153054208 + }, + { + "epoch": 0.3, + "learning_rate": 7.020542449045098e-05, + "loss": 2.7435, + "theoretical_loss": 3.409498650822683, + "tokens_seen": 2153185280 + }, + { + "epoch": 0.31, + "learning_rate": 7.019740009629273e-05, + "loss": 2.4549, + "theoretical_loss": 3.409481596615038, + "tokens_seen": 2153316352 + }, + { + "epoch": 0.31, + "learning_rate": 7.01893757021345e-05, + "loss": 2.5406, + "theoretical_loss": 3.4094645437360915, + "tokens_seen": 2153447424 + }, + { + "epoch": 0.31, + "learning_rate": 7.018135130797625e-05, + "loss": 2.6382, + "theoretical_loss": 3.4094474921856603, + "tokens_seen": 2153578496 + }, + { + "epoch": 0.31, + "learning_rate": 7.0173326913818e-05, + "loss": 2.6475, + "theoretical_loss": 3.4094304419635595, + "tokens_seen": 2153709568 + }, + { + "epoch": 0.31, + "learning_rate": 7.016530251965977e-05, + "loss": 2.8102, + "theoretical_loss": 3.409413393069605, + "tokens_seen": 2153840640 + }, + { + "epoch": 0.31, + "learning_rate": 7.015727812550152e-05, + "loss": 2.6188, + "theoretical_loss": 3.4093963455036125, + "tokens_seen": 2153971712 + }, + { + "epoch": 0.31, + "learning_rate": 7.014925373134329e-05, + "loss": 2.609, + "theoretical_loss": 3.4093792992653977, + "tokens_seen": 2154102784 + }, + { + "epoch": 0.31, + "learning_rate": 7.014122933718504e-05, + "loss": 2.7287, + "theoretical_loss": 3.409362254354776, + "tokens_seen": 2154233856 + }, + { + "epoch": 0.31, + "learning_rate": 7.013320494302681e-05, + "loss": 2.5641, + "theoretical_loss": 3.409345210771564, + "tokens_seen": 2154364928 + }, + { + "epoch": 0.31, + "learning_rate": 7.012518054886856e-05, + "loss": 2.5944, + "theoretical_loss": 3.409328168515577, + "tokens_seen": 2154496000 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1186484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.775474786758423, + "objective/train/theoretical_loss": 3.4093111275866312, + "objective/train/tokens_used": 525086176, + "theoretical_loss": 3.4093111275866312, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.31, + "learning_rate": 7.011715615471032e-05, + "loss": 2.6251, + "theoretical_loss": 3.4093111275866312, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.31, + "learning_rate": 7.010913176055208e-05, + "loss": 2.663, + "theoretical_loss": 3.4092940879845424, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.31, + "learning_rate": 7.010110736639384e-05, + "loss": 2.6242, + "theoretical_loss": 3.409277049709127, + "tokens_seen": 2154889216 + }, + { + "epoch": 0.31, + "learning_rate": 7.00930829722356e-05, + "loss": 2.548, + "theoretical_loss": 3.4092600127602006, + "tokens_seen": 2155020288 + }, + { + "epoch": 0.31, + "learning_rate": 7.008505857807736e-05, + "loss": 2.61, + "theoretical_loss": 3.4092429771375787, + "tokens_seen": 2155151360 + }, + { + "epoch": 0.31, + "learning_rate": 7.007703418391912e-05, + "loss": 2.5879, + "theoretical_loss": 3.4092259428410783, + "tokens_seen": 2155282432 + }, + { + "epoch": 0.31, + "learning_rate": 7.006900978976088e-05, + "loss": 2.5991, + "theoretical_loss": 3.409208909870516, + "tokens_seen": 2155413504 + }, + { + "epoch": 0.31, + "learning_rate": 7.006098539560264e-05, + "loss": 2.5815, + "theoretical_loss": 3.4091918782257062, + "tokens_seen": 2155544576 + }, + { + "epoch": 0.31, + "learning_rate": 7.00529610014444e-05, + "loss": 2.6043, + "theoretical_loss": 3.409174847906467, + "tokens_seen": 2155675648 + }, + { + "epoch": 0.31, + "learning_rate": 7.004493660728615e-05, + "loss": 2.6625, + "theoretical_loss": 3.409157818912613, + "tokens_seen": 2155806720 + }, + { + "epoch": 0.31, + "learning_rate": 7.003691221312792e-05, + "loss": 2.7572, + "theoretical_loss": 3.4091407912439617, + "tokens_seen": 2155937792 + }, + { + "epoch": 0.31, + "learning_rate": 7.002888781896967e-05, + "loss": 2.5151, + "theoretical_loss": 3.409123764900329, + "tokens_seen": 2156068864 + }, + { + "epoch": 0.31, + "learning_rate": 7.002086342481144e-05, + "loss": 2.6222, + "theoretical_loss": 3.409106739881531, + "tokens_seen": 2156199936 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1187201, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.76190185546875, + "objective/train/theoretical_loss": 3.4090982278688875, + "objective/train/tokens_used": 526724576, + "theoretical_loss": 3.4090982278688875, + "tokens_seen": 2156265472 + }, + { + "epoch": 0.31, + "learning_rate": 7.001283903065319e-05, + "loss": 2.651, + "theoretical_loss": 3.4090897161873848, + "tokens_seen": 2156331008 + }, + { + "epoch": 0.31, + "learning_rate": 7.000481463649496e-05, + "loss": 2.5518, + "theoretical_loss": 3.4090726938177056, + "tokens_seen": 2156462080 + }, + { + "epoch": 0.31, + "learning_rate": 6.999679024233671e-05, + "loss": 2.4781, + "theoretical_loss": 3.409055672772311, + "tokens_seen": 2156593152 + }, + { + "epoch": 0.31, + "learning_rate": 6.998876584817846e-05, + "loss": 2.6187, + "theoretical_loss": 3.409038653051017, + "tokens_seen": 2156724224 + }, + { + "epoch": 0.31, + "learning_rate": 6.998074145402023e-05, + "loss": 2.6385, + "theoretical_loss": 3.40902163465364, + "tokens_seen": 2156855296 + }, + { + "epoch": 0.31, + "learning_rate": 6.997271705986198e-05, + "loss": 2.6592, + "theoretical_loss": 3.4090046175799973, + "tokens_seen": 2156986368 + }, + { + "epoch": 0.31, + "learning_rate": 6.996469266570375e-05, + "loss": 2.6731, + "theoretical_loss": 3.408987601829905, + "tokens_seen": 2157117440 + }, + { + "epoch": 0.31, + "learning_rate": 6.99566682715455e-05, + "loss": 2.6475, + "theoretical_loss": 3.4089705874031795, + "tokens_seen": 2157248512 + }, + { + "epoch": 0.31, + "learning_rate": 6.994864387738727e-05, + "loss": 2.436, + "theoretical_loss": 3.4089535742996375, + "tokens_seen": 2157379584 + }, + { + "epoch": 0.31, + "learning_rate": 6.994061948322902e-05, + "loss": 2.6633, + "theoretical_loss": 3.408936562519096, + "tokens_seen": 2157510656 + }, + { + "epoch": 0.31, + "learning_rate": 6.993259508907077e-05, + "loss": 2.6681, + "theoretical_loss": 3.4089195520613718, + "tokens_seen": 2157641728 + }, + { + "epoch": 0.31, + "learning_rate": 6.992457069491254e-05, + "loss": 2.573, + "theoretical_loss": 3.4089025429262816, + "tokens_seen": 2157772800 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1188127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6738452911376953, + "objective/train/theoretical_loss": 3.4088855351136425, + "objective/train/tokens_used": 528362976, + "theoretical_loss": 3.4088855351136425, + "tokens_seen": 2157903872 + }, + { + "epoch": 0.31, + "learning_rate": 6.99165463007543e-05, + "loss": 2.5564, + "theoretical_loss": 3.4088855351136425, + "tokens_seen": 2157903872 + }, + { + "epoch": 0.31, + "learning_rate": 6.990852190659606e-05, + "loss": 2.6039, + "theoretical_loss": 3.4088685286232705, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.31, + "learning_rate": 6.990049751243781e-05, + "loss": 2.5855, + "theoretical_loss": 3.4088515234549837, + "tokens_seen": 2158166016 + }, + { + "epoch": 0.31, + "learning_rate": 6.989247311827958e-05, + "loss": 2.6119, + "theoretical_loss": 3.408834519608598, + "tokens_seen": 2158297088 + }, + { + "epoch": 0.31, + "learning_rate": 6.988444872412133e-05, + "loss": 2.5422, + "theoretical_loss": 3.408817517083931, + "tokens_seen": 2158428160 + }, + { + "epoch": 0.31, + "learning_rate": 6.987642432996309e-05, + "loss": 2.6445, + "theoretical_loss": 3.4088005158807992, + "tokens_seen": 2158559232 + }, + { + "epoch": 0.31, + "learning_rate": 6.986839993580485e-05, + "loss": 2.6504, + "theoretical_loss": 3.40878351599902, + "tokens_seen": 2158690304 + }, + { + "epoch": 0.31, + "learning_rate": 6.98603755416466e-05, + "loss": 2.6189, + "theoretical_loss": 3.408766517438411, + "tokens_seen": 2158821376 + }, + { + "epoch": 0.31, + "learning_rate": 6.985235114748837e-05, + "loss": 2.5501, + "theoretical_loss": 3.4087495201987883, + "tokens_seen": 2158952448 + }, + { + "epoch": 0.31, + "learning_rate": 6.984432675333013e-05, + "loss": 2.4872, + "theoretical_loss": 3.4087325242799693, + "tokens_seen": 2159083520 + }, + { + "epoch": 0.31, + "learning_rate": 6.98363023591719e-05, + "loss": 2.5379, + "theoretical_loss": 3.4087155296817717, + "tokens_seen": 2159214592 + }, + { + "epoch": 0.31, + "learning_rate": 6.982827796501365e-05, + "loss": 2.4032, + "theoretical_loss": 3.4086985364040125, + "tokens_seen": 2159345664 + }, + { + "epoch": 0.31, + "learning_rate": 6.98202535708554e-05, + "loss": 2.6065, + "theoretical_loss": 3.408681544446509, + "tokens_seen": 2159476736 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1188847, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7533798217773438, + "objective/train/theoretical_loss": 3.4086730489627954, + "objective/train/tokens_used": 530001376, + "theoretical_loss": 3.4086730489627954, + "tokens_seen": 2159542272 + }, + { + "epoch": 0.31, + "learning_rate": 6.981222917669717e-05, + "loss": 2.5673, + "theoretical_loss": 3.4086645538090776, + "tokens_seen": 2159607808 + }, + { + "epoch": 0.31, + "learning_rate": 6.980420478253892e-05, + "loss": 2.5625, + "theoretical_loss": 3.4086475644915373, + "tokens_seen": 2159738880 + }, + { + "epoch": 0.31, + "learning_rate": 6.979618038838069e-05, + "loss": 2.5184, + "theoretical_loss": 3.408630576493704, + "tokens_seen": 2159869952 + }, + { + "epoch": 0.31, + "learning_rate": 6.978815599422244e-05, + "loss": 2.6005, + "theoretical_loss": 3.408613589815396, + "tokens_seen": 2160001024 + }, + { + "epoch": 0.31, + "learning_rate": 6.97801316000642e-05, + "loss": 2.4839, + "theoretical_loss": 3.4085966044564304, + "tokens_seen": 2160132096 + }, + { + "epoch": 0.31, + "learning_rate": 6.977210720590596e-05, + "loss": 2.5711, + "theoretical_loss": 3.4085796204166243, + "tokens_seen": 2160263168 + }, + { + "epoch": 0.31, + "learning_rate": 6.976408281174773e-05, + "loss": 2.6037, + "theoretical_loss": 3.4085626376957965, + "tokens_seen": 2160394240 + }, + { + "epoch": 0.31, + "learning_rate": 6.975605841758948e-05, + "loss": 2.5763, + "theoretical_loss": 3.408545656293763, + "tokens_seen": 2160525312 + }, + { + "epoch": 0.31, + "learning_rate": 6.974803402343123e-05, + "loss": 2.6665, + "theoretical_loss": 3.4085286762103424, + "tokens_seen": 2160656384 + }, + { + "epoch": 0.31, + "learning_rate": 6.9740009629273e-05, + "loss": 2.4983, + "theoretical_loss": 3.408511697445352, + "tokens_seen": 2160787456 + }, + { + "epoch": 0.31, + "learning_rate": 6.973198523511475e-05, + "loss": 2.5276, + "theoretical_loss": 3.4084947199986093, + "tokens_seen": 2160918528 + }, + { + "epoch": 0.31, + "learning_rate": 6.972396084095652e-05, + "loss": 2.6489, + "theoretical_loss": 3.4084777438699323, + "tokens_seen": 2161049600 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1190045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.518320083618164, + "objective/train/theoretical_loss": 3.408460769059139, + "objective/train/tokens_used": 531639776, + "theoretical_loss": 3.408460769059139, + "tokens_seen": 2161180672 + }, + { + "epoch": 0.31, + "learning_rate": 6.971593644679827e-05, + "loss": 2.538, + "theoretical_loss": 3.408460769059139, + "tokens_seen": 2161180672 + }, + { + "epoch": 0.31, + "learning_rate": 6.970791205264004e-05, + "loss": 2.5788, + "theoretical_loss": 3.408443795566046, + "tokens_seen": 2161311744 + }, + { + "epoch": 0.31, + "learning_rate": 6.969988765848179e-05, + "loss": 2.6509, + "theoretical_loss": 3.4084268233904726, + "tokens_seen": 2161442816 + }, + { + "epoch": 0.31, + "learning_rate": 6.969186326432354e-05, + "loss": 2.578, + "theoretical_loss": 3.4084098525322357, + "tokens_seen": 2161573888 + }, + { + "epoch": 0.31, + "learning_rate": 6.968383887016531e-05, + "loss": 2.4955, + "theoretical_loss": 3.408392882991153, + "tokens_seen": 2161704960 + }, + { + "epoch": 0.31, + "learning_rate": 6.967581447600706e-05, + "loss": 2.6306, + "theoretical_loss": 3.408375914767044, + "tokens_seen": 2161836032 + }, + { + "epoch": 0.31, + "learning_rate": 6.966779008184883e-05, + "loss": 2.6397, + "theoretical_loss": 3.4083589478597247, + "tokens_seen": 2161967104 + }, + { + "epoch": 0.31, + "learning_rate": 6.965976568769058e-05, + "loss": 2.5939, + "theoretical_loss": 3.4083419822690137, + "tokens_seen": 2162098176 + }, + { + "epoch": 0.31, + "learning_rate": 6.965174129353235e-05, + "loss": 2.6069, + "theoretical_loss": 3.4083250179947293, + "tokens_seen": 2162229248 + }, + { + "epoch": 0.31, + "learning_rate": 6.96437168993741e-05, + "loss": 2.5248, + "theoretical_loss": 3.4083080550366898, + "tokens_seen": 2162360320 + }, + { + "epoch": 0.31, + "learning_rate": 6.963569250521586e-05, + "loss": 2.6181, + "theoretical_loss": 3.4082910933947126, + "tokens_seen": 2162491392 + }, + { + "epoch": 0.31, + "learning_rate": 6.962766811105762e-05, + "loss": 2.5547, + "theoretical_loss": 3.408274133068617, + "tokens_seen": 2162622464 + }, + { + "epoch": 0.31, + "learning_rate": 6.961964371689938e-05, + "loss": 2.4597, + "theoretical_loss": 3.4082571740582197, + "tokens_seen": 2162753536 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1191405, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.051525115966797, + "objective/train/theoretical_loss": 3.4082486950463515, + "objective/train/tokens_used": 533278176, + "theoretical_loss": 3.4082486950463515, + "tokens_seen": 2162819072 + }, + { + "epoch": 0.31, + "learning_rate": 6.961161932274114e-05, + "loss": 2.5429, + "theoretical_loss": 3.4082402163633394, + "tokens_seen": 2162884608 + }, + { + "epoch": 0.31, + "learning_rate": 6.96035949285829e-05, + "loss": 2.5639, + "theoretical_loss": 3.408223259983795, + "tokens_seen": 2163015680 + }, + { + "epoch": 0.31, + "learning_rate": 6.959557053442466e-05, + "loss": 2.5452, + "theoretical_loss": 3.4082063049194042, + "tokens_seen": 2163146752 + }, + { + "epoch": 0.31, + "learning_rate": 6.958754614026642e-05, + "loss": 2.5338, + "theoretical_loss": 3.4081893511699852, + "tokens_seen": 2163277824 + }, + { + "epoch": 0.31, + "learning_rate": 6.957952174610817e-05, + "loss": 2.605, + "theoretical_loss": 3.408172398735357, + "tokens_seen": 2163408896 + }, + { + "epoch": 0.31, + "learning_rate": 6.957149735194994e-05, + "loss": 2.5398, + "theoretical_loss": 3.4081554476153375, + "tokens_seen": 2163539968 + }, + { + "epoch": 0.31, + "learning_rate": 6.956347295779169e-05, + "loss": 2.5628, + "theoretical_loss": 3.4081384978097446, + "tokens_seen": 2163671040 + }, + { + "epoch": 0.31, + "learning_rate": 6.955544856363346e-05, + "loss": 2.5798, + "theoretical_loss": 3.408121549318398, + "tokens_seen": 2163802112 + }, + { + "epoch": 0.31, + "learning_rate": 6.954742416947521e-05, + "loss": 2.5445, + "theoretical_loss": 3.4081046021411154, + "tokens_seen": 2163933184 + }, + { + "epoch": 0.31, + "learning_rate": 6.953939977531698e-05, + "loss": 2.5407, + "theoretical_loss": 3.4080876562777154, + "tokens_seen": 2164064256 + }, + { + "epoch": 0.31, + "learning_rate": 6.953137538115873e-05, + "loss": 2.5254, + "theoretical_loss": 3.4080707117280165, + "tokens_seen": 2164195328 + }, + { + "epoch": 0.31, + "learning_rate": 6.952335098700048e-05, + "loss": 2.5325, + "theoretical_loss": 3.408053768491838, + "tokens_seen": 2164326400 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1192132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1488561630249023, + "objective/train/theoretical_loss": 3.4080368265689973, + "objective/train/tokens_used": 534916576, + "theoretical_loss": 3.4080368265689973, + "tokens_seen": 2164457472 + }, + { + "epoch": 0.31, + "learning_rate": 6.951532659284225e-05, + "loss": 2.4803, + "theoretical_loss": 3.4080368265689973, + "tokens_seen": 2164457472 + }, + { + "epoch": 0.31, + "learning_rate": 6.9507302198684e-05, + "loss": 2.5546, + "theoretical_loss": 3.4080198859593143, + "tokens_seen": 2164588544 + }, + { + "epoch": 0.31, + "learning_rate": 6.949927780452577e-05, + "loss": 2.723, + "theoretical_loss": 3.4080029466626067, + "tokens_seen": 2164719616 + }, + { + "epoch": 0.31, + "learning_rate": 6.949125341036752e-05, + "loss": 2.4956, + "theoretical_loss": 3.407986008678694, + "tokens_seen": 2164850688 + }, + { + "epoch": 0.31, + "learning_rate": 6.948322901620929e-05, + "loss": 2.4595, + "theoretical_loss": 3.4079690720073943, + "tokens_seen": 2164981760 + }, + { + "epoch": 0.31, + "learning_rate": 6.947520462205104e-05, + "loss": 2.5846, + "theoretical_loss": 3.4079521366485275, + "tokens_seen": 2165112832 + }, + { + "epoch": 0.31, + "learning_rate": 6.94671802278928e-05, + "loss": 2.6315, + "theoretical_loss": 3.4079352026019114, + "tokens_seen": 2165243904 + }, + { + "epoch": 0.31, + "learning_rate": 6.945915583373456e-05, + "loss": 2.595, + "theoretical_loss": 3.4079182698673653, + "tokens_seen": 2165374976 + }, + { + "epoch": 0.31, + "learning_rate": 6.945113143957631e-05, + "loss": 2.6572, + "theoretical_loss": 3.407901338444708, + "tokens_seen": 2165506048 + }, + { + "epoch": 0.31, + "learning_rate": 6.944310704541808e-05, + "loss": 2.4416, + "theoretical_loss": 3.407884408333759, + "tokens_seen": 2165637120 + }, + { + "epoch": 0.31, + "learning_rate": 6.943508265125983e-05, + "loss": 2.4823, + "theoretical_loss": 3.407867479534336, + "tokens_seen": 2165768192 + }, + { + "epoch": 0.31, + "learning_rate": 6.94270582571016e-05, + "loss": 2.4664, + "theoretical_loss": 3.40785055204626, + "tokens_seen": 2165899264 + }, + { + "epoch": 0.31, + "learning_rate": 6.941903386294335e-05, + "loss": 2.5716, + "theoretical_loss": 3.4078336258693485, + "tokens_seen": 2166030336 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1193548, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6069717407226562, + "objective/train/theoretical_loss": 3.407825163272523, + "objective/train/tokens_used": 536554976, + "theoretical_loss": 3.407825163272523, + "tokens_seen": 2166095872 + }, + { + "epoch": 0.31, + "learning_rate": 6.941100946878512e-05, + "loss": 2.4828, + "theoretical_loss": 3.407816701003421, + "tokens_seen": 2166161408 + }, + { + "epoch": 0.31, + "learning_rate": 6.940298507462687e-05, + "loss": 2.5359, + "theoretical_loss": 3.4077997774482967, + "tokens_seen": 2166292480 + }, + { + "epoch": 0.31, + "learning_rate": 6.939496068046863e-05, + "loss": 2.4822, + "theoretical_loss": 3.407782855203795, + "tokens_seen": 2166423552 + }, + { + "epoch": 0.31, + "learning_rate": 6.93869362863104e-05, + "loss": 2.5632, + "theoretical_loss": 3.4077659342697344, + "tokens_seen": 2166554624 + }, + { + "epoch": 0.31, + "learning_rate": 6.937891189215215e-05, + "loss": 2.41, + "theoretical_loss": 3.407749014645935, + "tokens_seen": 2166685696 + }, + { + "epoch": 0.31, + "learning_rate": 6.937088749799391e-05, + "loss": 2.5752, + "theoretical_loss": 3.407732096332216, + "tokens_seen": 2166816768 + }, + { + "epoch": 0.31, + "learning_rate": 6.936286310383567e-05, + "loss": 2.6156, + "theoretical_loss": 3.407715179328396, + "tokens_seen": 2166947840 + }, + { + "epoch": 0.31, + "learning_rate": 6.935483870967743e-05, + "loss": 2.512, + "theoretical_loss": 3.4076982636342956, + "tokens_seen": 2167078912 + }, + { + "epoch": 0.31, + "learning_rate": 6.934681431551919e-05, + "loss": 2.2435, + "theoretical_loss": 3.407681349249733, + "tokens_seen": 2167209984 + }, + { + "epoch": 0.31, + "learning_rate": 6.933878992136094e-05, + "loss": 2.4444, + "theoretical_loss": 3.4076644361745276, + "tokens_seen": 2167341056 + }, + { + "epoch": 0.31, + "learning_rate": 6.93307655272027e-05, + "loss": 2.6329, + "theoretical_loss": 3.4076475244084996, + "tokens_seen": 2167472128 + }, + { + "epoch": 0.31, + "learning_rate": 6.932274113304446e-05, + "loss": 2.535, + "theoretical_loss": 3.4076306139514685, + "tokens_seen": 2167603200 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1194194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.670583724975586, + "objective/train/theoretical_loss": 3.4076137048032535, + "objective/train/tokens_used": 538193376, + "theoretical_loss": 3.4076137048032535, + "tokens_seen": 2167734272 + }, + { + "epoch": 0.31, + "learning_rate": 6.931471673888623e-05, + "loss": 2.5218, + "theoretical_loss": 3.4076137048032535, + "tokens_seen": 2167734272 + }, + { + "epoch": 0.31, + "learning_rate": 6.930669234472798e-05, + "loss": 2.4784, + "theoretical_loss": 3.4075967969636745, + "tokens_seen": 2167865344 + }, + { + "epoch": 0.31, + "learning_rate": 6.929866795056975e-05, + "loss": 2.3791, + "theoretical_loss": 3.40757989043255, + "tokens_seen": 2167996416 + }, + { + "epoch": 0.31, + "learning_rate": 6.92906435564115e-05, + "loss": 2.5507, + "theoretical_loss": 3.4075629852097014, + "tokens_seen": 2168127488 + }, + { + "epoch": 0.31, + "learning_rate": 6.928261916225325e-05, + "loss": 2.6381, + "theoretical_loss": 3.4075460812949467, + "tokens_seen": 2168258560 + }, + { + "epoch": 0.31, + "learning_rate": 6.927459476809502e-05, + "loss": 2.5207, + "theoretical_loss": 3.4075291786881072, + "tokens_seen": 2168389632 + }, + { + "epoch": 0.31, + "learning_rate": 6.926657037393677e-05, + "loss": 2.4885, + "theoretical_loss": 3.407512277389001, + "tokens_seen": 2168520704 + }, + { + "epoch": 0.31, + "learning_rate": 6.925854597977854e-05, + "loss": 2.5377, + "theoretical_loss": 3.4074953773974492, + "tokens_seen": 2168651776 + }, + { + "epoch": 0.31, + "learning_rate": 6.925052158562029e-05, + "loss": 2.3995, + "theoretical_loss": 3.4074784787132715, + "tokens_seen": 2168782848 + }, + { + "epoch": 0.31, + "learning_rate": 6.924249719146206e-05, + "loss": 2.4598, + "theoretical_loss": 3.407461581336287, + "tokens_seen": 2168913920 + }, + { + "epoch": 0.31, + "learning_rate": 6.923447279730381e-05, + "loss": 2.4744, + "theoretical_loss": 3.407444685266316, + "tokens_seen": 2169044992 + }, + { + "epoch": 0.31, + "learning_rate": 6.922644840314556e-05, + "loss": 2.4483, + "theoretical_loss": 3.4074277905031782, + "tokens_seen": 2169176064 + }, + { + "epoch": 0.31, + "learning_rate": 6.921842400898733e-05, + "loss": 2.4571, + "theoretical_loss": 3.4074108970466943, + "tokens_seen": 2169307136 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1195647, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.518240451812744, + "objective/train/theoretical_loss": 3.407402450808391, + "objective/train/tokens_used": 539831776, + "theoretical_loss": 3.407402450808391, + "tokens_seen": 2169372672 + }, + { + "epoch": 0.31, + "learning_rate": 6.921039961482908e-05, + "loss": 2.5114, + "theoretical_loss": 3.4073940048966835, + "tokens_seen": 2169438208 + }, + { + "epoch": 0.31, + "learning_rate": 6.920237522067085e-05, + "loss": 2.4289, + "theoretical_loss": 3.4073771140529665, + "tokens_seen": 2169569280 + }, + { + "epoch": 0.31, + "learning_rate": 6.91943508265126e-05, + "loss": 2.5673, + "theoretical_loss": 3.4073602245153625, + "tokens_seen": 2169700352 + }, + { + "epoch": 0.32, + "learning_rate": 6.918632643235437e-05, + "loss": 2.6631, + "theoretical_loss": 3.4073433362836925, + "tokens_seen": 2169831424 + }, + { + "epoch": 0.32, + "learning_rate": 6.917830203819612e-05, + "loss": 2.4073, + "theoretical_loss": 3.407326449357776, + "tokens_seen": 2169962496 + }, + { + "epoch": 0.32, + "learning_rate": 6.917027764403788e-05, + "loss": 2.6058, + "theoretical_loss": 3.4073095637374338, + "tokens_seen": 2170093568 + }, + { + "epoch": 0.32, + "learning_rate": 6.916225324987964e-05, + "loss": 2.6248, + "theoretical_loss": 3.407292679422486, + "tokens_seen": 2170224640 + }, + { + "epoch": 0.32, + "learning_rate": 6.91542288557214e-05, + "loss": 2.614, + "theoretical_loss": 3.407275796412752, + "tokens_seen": 2170355712 + }, + { + "epoch": 0.32, + "learning_rate": 6.914620446156316e-05, + "loss": 2.5169, + "theoretical_loss": 3.407258914708053, + "tokens_seen": 2170486784 + }, + { + "epoch": 0.32, + "learning_rate": 6.913818006740492e-05, + "loss": 2.6178, + "theoretical_loss": 3.407242034308209, + "tokens_seen": 2170617856 + }, + { + "epoch": 0.32, + "learning_rate": 6.913015567324668e-05, + "loss": 2.3758, + "theoretical_loss": 3.40722515521304, + "tokens_seen": 2170748928 + }, + { + "epoch": 0.32, + "learning_rate": 6.912213127908844e-05, + "loss": 2.5497, + "theoretical_loss": 3.4072082774223675, + "tokens_seen": 2170880000 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1196267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4913604259490967, + "objective/train/theoretical_loss": 3.4071914009360107, + "objective/train/tokens_used": 541470176, + "theoretical_loss": 3.4071914009360107, + "tokens_seen": 2171011072 + }, + { + "epoch": 0.32, + "learning_rate": 6.911410688493019e-05, + "loss": 2.4815, + "theoretical_loss": 3.4071914009360107, + "tokens_seen": 2171011072 + }, + { + "epoch": 0.32, + "learning_rate": 6.910608249077196e-05, + "loss": 2.5382, + "theoretical_loss": 3.4071745257537907, + "tokens_seen": 2171142144 + }, + { + "epoch": 0.32, + "learning_rate": 6.909805809661371e-05, + "loss": 2.5229, + "theoretical_loss": 3.407157651875528, + "tokens_seen": 2171273216 + }, + { + "epoch": 0.32, + "learning_rate": 6.909003370245548e-05, + "loss": 2.3727, + "theoretical_loss": 3.407140779301043, + "tokens_seen": 2171404288 + }, + { + "epoch": 0.32, + "learning_rate": 6.908200930829723e-05, + "loss": 2.4773, + "theoretical_loss": 3.4071239080301563, + "tokens_seen": 2171535360 + }, + { + "epoch": 0.32, + "learning_rate": 6.907398491413898e-05, + "loss": 2.5604, + "theoretical_loss": 3.407107038062688, + "tokens_seen": 2171666432 + }, + { + "epoch": 0.32, + "learning_rate": 6.906596051998075e-05, + "loss": 2.4941, + "theoretical_loss": 3.40709016939846, + "tokens_seen": 2171797504 + }, + { + "epoch": 0.32, + "learning_rate": 6.90579361258225e-05, + "loss": 2.6267, + "theoretical_loss": 3.4070733020372916, + "tokens_seen": 2171928576 + }, + { + "epoch": 0.32, + "learning_rate": 6.904991173166427e-05, + "loss": 2.5602, + "theoretical_loss": 3.4070564359790048, + "tokens_seen": 2172059648 + }, + { + "epoch": 0.32, + "learning_rate": 6.904188733750602e-05, + "loss": 2.6802, + "theoretical_loss": 3.407039571223419, + "tokens_seen": 2172190720 + }, + { + "epoch": 0.32, + "learning_rate": 6.903386294334779e-05, + "loss": 2.5658, + "theoretical_loss": 3.407022707770356, + "tokens_seen": 2172321792 + }, + { + "epoch": 0.32, + "learning_rate": 6.902583854918954e-05, + "loss": 2.3892, + "theoretical_loss": 3.4070058456196364, + "tokens_seen": 2172452864 + }, + { + "epoch": 0.32, + "learning_rate": 6.90178141550313e-05, + "loss": 2.3801, + "theoretical_loss": 3.406988984771081, + "tokens_seen": 2172583936 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1197448, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.376589298248291, + "objective/train/theoretical_loss": 3.4069805548350587, + "objective/train/tokens_used": 543108576, + "theoretical_loss": 3.4069805548350587, + "tokens_seen": 2172649472 + }, + { + "epoch": 0.32, + "learning_rate": 6.900978976087306e-05, + "loss": 2.4687, + "theoretical_loss": 3.40697212522451, + "tokens_seen": 2172715008 + }, + { + "epoch": 0.32, + "learning_rate": 6.900176536671482e-05, + "loss": 2.6256, + "theoretical_loss": 3.4069552669797454, + "tokens_seen": 2172846080 + }, + { + "epoch": 0.32, + "learning_rate": 6.899374097255658e-05, + "loss": 2.5264, + "theoretical_loss": 3.4069384100366076, + "tokens_seen": 2172977152 + }, + { + "epoch": 0.32, + "learning_rate": 6.898571657839833e-05, + "loss": 2.4931, + "theoretical_loss": 3.406921554394918, + "tokens_seen": 2173108224 + }, + { + "epoch": 0.32, + "learning_rate": 6.897769218424009e-05, + "loss": 2.3636, + "theoretical_loss": 3.406904700054497, + "tokens_seen": 2173239296 + }, + { + "epoch": 0.32, + "learning_rate": 6.896966779008185e-05, + "loss": 2.4235, + "theoretical_loss": 3.4068878470151662, + "tokens_seen": 2173370368 + }, + { + "epoch": 0.32, + "learning_rate": 6.896164339592361e-05, + "loss": 2.4344, + "theoretical_loss": 3.406870995276747, + "tokens_seen": 2173501440 + }, + { + "epoch": 0.32, + "learning_rate": 6.895361900176537e-05, + "loss": 2.4578, + "theoretical_loss": 3.406854144839059, + "tokens_seen": 2173632512 + }, + { + "epoch": 0.32, + "learning_rate": 6.894559460760713e-05, + "loss": 2.5416, + "theoretical_loss": 3.406837295701925, + "tokens_seen": 2173763584 + }, + { + "epoch": 0.32, + "learning_rate": 6.89375702134489e-05, + "loss": 2.5487, + "theoretical_loss": 3.406820447865166, + "tokens_seen": 2173894656 + }, + { + "epoch": 0.32, + "learning_rate": 6.892954581929065e-05, + "loss": 2.5288, + "theoretical_loss": 3.4068036013286025, + "tokens_seen": 2174025728 + }, + { + "epoch": 0.32, + "learning_rate": 6.89215214251324e-05, + "loss": 2.4718, + "theoretical_loss": 3.406786756092056, + "tokens_seen": 2174156800 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1198116, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.546083927154541, + "objective/train/theoretical_loss": 3.4067699121553483, + "objective/train/tokens_used": 544746976, + "theoretical_loss": 3.4067699121553483, + "tokens_seen": 2174287872 + }, + { + "epoch": 0.32, + "learning_rate": 6.891349703097417e-05, + "loss": 2.4965, + "theoretical_loss": 3.4067699121553483, + "tokens_seen": 2174287872 + }, + { + "epoch": 0.32, + "learning_rate": 6.890547263681592e-05, + "loss": 2.5112, + "theoretical_loss": 3.4067530695183, + "tokens_seen": 2174418944 + }, + { + "epoch": 0.32, + "learning_rate": 6.889744824265769e-05, + "loss": 2.4467, + "theoretical_loss": 3.4067362281807334, + "tokens_seen": 2174550016 + }, + { + "epoch": 0.32, + "learning_rate": 6.888942384849944e-05, + "loss": 2.5762, + "theoretical_loss": 3.4067193881424696, + "tokens_seen": 2174681088 + }, + { + "epoch": 0.32, + "learning_rate": 6.88813994543412e-05, + "loss": 2.5666, + "theoretical_loss": 3.406702549403329, + "tokens_seen": 2174812160 + }, + { + "epoch": 0.32, + "learning_rate": 6.887337506018296e-05, + "loss": 2.4124, + "theoretical_loss": 3.406685711963135, + "tokens_seen": 2174943232 + }, + { + "epoch": 0.32, + "learning_rate": 6.886535066602471e-05, + "loss": 2.4991, + "theoretical_loss": 3.406668875821708, + "tokens_seen": 2175074304 + }, + { + "epoch": 0.32, + "learning_rate": 6.885732627186648e-05, + "loss": 2.3663, + "theoretical_loss": 3.406652040978869, + "tokens_seen": 2175205376 + }, + { + "epoch": 0.32, + "learning_rate": 6.884930187770823e-05, + "loss": 2.589, + "theoretical_loss": 3.406635207434441, + "tokens_seen": 2175336448 + }, + { + "epoch": 0.32, + "learning_rate": 6.884127748355e-05, + "loss": 2.4368, + "theoretical_loss": 3.4066183751882444, + "tokens_seen": 2175467520 + }, + { + "epoch": 0.32, + "learning_rate": 6.883325308939175e-05, + "loss": 2.568, + "theoretical_loss": 3.4066015442401016, + "tokens_seen": 2175598592 + }, + { + "epoch": 0.32, + "learning_rate": 6.88252286952335e-05, + "loss": 2.515, + "theoretical_loss": 3.406584714589834, + "tokens_seen": 2175729664 + }, + { + "epoch": 0.32, + "learning_rate": 6.881720430107527e-05, + "loss": 2.5158, + "theoretical_loss": 3.4065678862372635, + "tokens_seen": 2175860736 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1199462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0797388553619385, + "objective/train/theoretical_loss": 3.406559472547559, + "objective/train/tokens_used": 546385376, + "theoretical_loss": 3.406559472547559, + "tokens_seen": 2175926272 + }, + { + "epoch": 0.32, + "learning_rate": 6.880917990691703e-05, + "loss": 2.4784, + "theoretical_loss": 3.406551059182212, + "tokens_seen": 2175991808 + }, + { + "epoch": 0.32, + "learning_rate": 6.880115551275879e-05, + "loss": 2.5083, + "theoretical_loss": 3.406534233424501, + "tokens_seen": 2176122880 + }, + { + "epoch": 0.32, + "learning_rate": 6.879313111860055e-05, + "loss": 2.5584, + "theoretical_loss": 3.4065174089639525, + "tokens_seen": 2176253952 + }, + { + "epoch": 0.32, + "learning_rate": 6.87851067244423e-05, + "loss": 2.573, + "theoretical_loss": 3.406500585800388, + "tokens_seen": 2176385024 + }, + { + "epoch": 0.32, + "learning_rate": 6.877708233028407e-05, + "loss": 2.5617, + "theoretical_loss": 3.40648376393363, + "tokens_seen": 2176516096 + }, + { + "epoch": 0.32, + "learning_rate": 6.876905793612582e-05, + "loss": 2.5937, + "theoretical_loss": 3.4064669433635, + "tokens_seen": 2176647168 + }, + { + "epoch": 0.32, + "learning_rate": 6.876103354196759e-05, + "loss": 2.571, + "theoretical_loss": 3.4064501240898206, + "tokens_seen": 2176778240 + }, + { + "epoch": 0.32, + "learning_rate": 6.875300914780934e-05, + "loss": 2.515, + "theoretical_loss": 3.406433306112413, + "tokens_seen": 2176909312 + }, + { + "epoch": 0.32, + "learning_rate": 6.87449847536511e-05, + "loss": 2.5359, + "theoretical_loss": 3.4064164894310998, + "tokens_seen": 2177040384 + }, + { + "epoch": 0.32, + "learning_rate": 6.873696035949286e-05, + "loss": 2.5196, + "theoretical_loss": 3.406399674045703, + "tokens_seen": 2177171456 + }, + { + "epoch": 0.32, + "learning_rate": 6.872893596533461e-05, + "loss": 2.5101, + "theoretical_loss": 3.406382859956045, + "tokens_seen": 2177302528 + }, + { + "epoch": 0.32, + "learning_rate": 6.872091157117638e-05, + "loss": 2.4146, + "theoretical_loss": 3.406366047161947, + "tokens_seen": 2177433600 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1200438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.211505174636841, + "objective/train/theoretical_loss": 3.406349235663232, + "objective/train/tokens_used": 548023776, + "theoretical_loss": 3.406349235663232, + "tokens_seen": 2177564672 + }, + { + "epoch": 0.32, + "learning_rate": 6.871288717701813e-05, + "loss": 2.4019, + "theoretical_loss": 3.406349235663232, + "tokens_seen": 2177564672 + }, + { + "epoch": 0.32, + "learning_rate": 6.87048627828599e-05, + "loss": 2.4612, + "theoretical_loss": 3.4063324254597225, + "tokens_seen": 2177695744 + }, + { + "epoch": 0.32, + "learning_rate": 6.869683838870165e-05, + "loss": 2.6133, + "theoretical_loss": 3.40631561655124, + "tokens_seen": 2177826816 + }, + { + "epoch": 0.32, + "learning_rate": 6.86888139945434e-05, + "loss": 2.4651, + "theoretical_loss": 3.406298808937607, + "tokens_seen": 2177957888 + }, + { + "epoch": 0.32, + "learning_rate": 6.868078960038517e-05, + "loss": 2.4464, + "theoretical_loss": 3.406282002618646, + "tokens_seen": 2178088960 + }, + { + "epoch": 0.32, + "learning_rate": 6.867276520622692e-05, + "loss": 2.4255, + "theoretical_loss": 3.4062651975941796, + "tokens_seen": 2178220032 + }, + { + "epoch": 0.32, + "learning_rate": 6.866474081206869e-05, + "loss": 2.4586, + "theoretical_loss": 3.40624839386403, + "tokens_seen": 2178351104 + }, + { + "epoch": 0.32, + "learning_rate": 6.865671641791044e-05, + "loss": 2.4815, + "theoretical_loss": 3.4062315914280195, + "tokens_seen": 2178482176 + }, + { + "epoch": 0.32, + "learning_rate": 6.864869202375221e-05, + "loss": 2.6005, + "theoretical_loss": 3.4062147902859707, + "tokens_seen": 2178613248 + }, + { + "epoch": 0.32, + "learning_rate": 6.864066762959396e-05, + "loss": 2.5733, + "theoretical_loss": 3.4061979904377058, + "tokens_seen": 2178744320 + }, + { + "epoch": 0.32, + "learning_rate": 6.863264323543572e-05, + "loss": 2.4796, + "theoretical_loss": 3.4061811918830482, + "tokens_seen": 2178875392 + }, + { + "epoch": 0.32, + "learning_rate": 6.862461884127748e-05, + "loss": 2.5401, + "theoretical_loss": 3.406164394621819, + "tokens_seen": 2179006464 + }, + { + "epoch": 0.32, + "learning_rate": 6.861659444711924e-05, + "loss": 2.4538, + "theoretical_loss": 3.4061475986538428, + "tokens_seen": 2179137536 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1201179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.695457696914673, + "objective/train/theoretical_loss": 3.4061392011547684, + "objective/train/tokens_used": 549662176, + "theoretical_loss": 3.4061392011547684, + "tokens_seen": 2179203072 + }, + { + "epoch": 0.32, + "learning_rate": 6.8608570052961e-05, + "loss": 2.5881, + "theoretical_loss": 3.4061308039789404, + "tokens_seen": 2179268608 + }, + { + "epoch": 0.32, + "learning_rate": 6.860054565880276e-05, + "loss": 2.4436, + "theoretical_loss": 3.406114010596936, + "tokens_seen": 2179399680 + }, + { + "epoch": 0.32, + "learning_rate": 6.859252126464452e-05, + "loss": 2.5682, + "theoretical_loss": 3.406097218507651, + "tokens_seen": 2179530752 + }, + { + "epoch": 0.32, + "learning_rate": 6.858449687048628e-05, + "loss": 2.3501, + "theoretical_loss": 3.4060804277109087, + "tokens_seen": 2179661824 + }, + { + "epoch": 0.32, + "learning_rate": 6.857647247632803e-05, + "loss": 2.4841, + "theoretical_loss": 3.406063638206532, + "tokens_seen": 2179792896 + }, + { + "epoch": 0.32, + "learning_rate": 6.85684480821698e-05, + "loss": 2.4954, + "theoretical_loss": 3.4060468499943437, + "tokens_seen": 2179923968 + }, + { + "epoch": 0.32, + "learning_rate": 6.856042368801155e-05, + "loss": 2.5969, + "theoretical_loss": 3.4060300630741667, + "tokens_seen": 2180055040 + }, + { + "epoch": 0.32, + "learning_rate": 6.855239929385332e-05, + "loss": 2.5807, + "theoretical_loss": 3.4060132774458243, + "tokens_seen": 2180186112 + }, + { + "epoch": 0.32, + "learning_rate": 6.854437489969507e-05, + "loss": 2.5297, + "theoretical_loss": 3.4059964931091384, + "tokens_seen": 2180317184 + }, + { + "epoch": 0.32, + "learning_rate": 6.853635050553684e-05, + "loss": 2.5044, + "theoretical_loss": 3.4059797100639324, + "tokens_seen": 2180448256 + }, + { + "epoch": 0.32, + "learning_rate": 6.852832611137859e-05, + "loss": 2.506, + "theoretical_loss": 3.4059629283100294, + "tokens_seen": 2180579328 + }, + { + "epoch": 0.32, + "learning_rate": 6.852030171722034e-05, + "loss": 2.4052, + "theoretical_loss": 3.405946147847253, + "tokens_seen": 2180710400 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1202357, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7669031620025635, + "objective/train/theoretical_loss": 3.4059293686754253, + "objective/train/tokens_used": 551300576, + "theoretical_loss": 3.4059293686754253, + "tokens_seen": 2180841472 + }, + { + "epoch": 0.32, + "learning_rate": 6.851227732306211e-05, + "loss": 2.4613, + "theoretical_loss": 3.4059293686754253, + "tokens_seen": 2180841472 + }, + { + "epoch": 0.32, + "learning_rate": 6.850425292890386e-05, + "loss": 2.529, + "theoretical_loss": 3.40591259079437, + "tokens_seen": 2180972544 + }, + { + "epoch": 0.32, + "learning_rate": 6.849622853474563e-05, + "loss": 2.6914, + "theoretical_loss": 3.40589581420391, + "tokens_seen": 2181103616 + }, + { + "epoch": 0.32, + "learning_rate": 6.848820414058738e-05, + "loss": 2.5318, + "theoretical_loss": 3.4058790389038687, + "tokens_seen": 2181234688 + }, + { + "epoch": 0.32, + "learning_rate": 6.848017974642915e-05, + "loss": 2.4421, + "theoretical_loss": 3.4058622648940693, + "tokens_seen": 2181365760 + }, + { + "epoch": 0.32, + "learning_rate": 6.84721553522709e-05, + "loss": 2.4674, + "theoretical_loss": 3.4058454921743344, + "tokens_seen": 2181496832 + }, + { + "epoch": 0.32, + "learning_rate": 6.846413095811267e-05, + "loss": 2.3825, + "theoretical_loss": 3.405828720744488, + "tokens_seen": 2181627904 + }, + { + "epoch": 0.32, + "learning_rate": 6.845610656395442e-05, + "loss": 2.5162, + "theoretical_loss": 3.4058119506043534, + "tokens_seen": 2181758976 + }, + { + "epoch": 0.32, + "learning_rate": 6.844808216979617e-05, + "loss": 2.3782, + "theoretical_loss": 3.4057951817537537, + "tokens_seen": 2181890048 + }, + { + "epoch": 0.32, + "learning_rate": 6.844005777563794e-05, + "loss": 2.4567, + "theoretical_loss": 3.405778414192512, + "tokens_seen": 2182021120 + }, + { + "epoch": 0.32, + "learning_rate": 6.84320333814797e-05, + "loss": 2.4216, + "theoretical_loss": 3.4057616479204524, + "tokens_seen": 2182152192 + }, + { + "epoch": 0.32, + "learning_rate": 6.842400898732146e-05, + "loss": 2.4975, + "theoretical_loss": 3.405744882937398, + "tokens_seen": 2182283264 + }, + { + "epoch": 0.32, + "learning_rate": 6.841598459316321e-05, + "loss": 2.41, + "theoretical_loss": 3.405728119243172, + "tokens_seen": 2182414336 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1202887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.703336000442505, + "objective/train/theoretical_loss": 3.405719737879315, + "objective/train/tokens_used": 552938976, + "theoretical_loss": 3.405719737879315, + "tokens_seen": 2182479872 + }, + { + "epoch": 0.32, + "learning_rate": 6.840796019900498e-05, + "loss": 2.6198, + "theoretical_loss": 3.4057113568375987, + "tokens_seen": 2182545408 + }, + { + "epoch": 0.32, + "learning_rate": 6.839993580484673e-05, + "loss": 2.5698, + "theoretical_loss": 3.405694595720501, + "tokens_seen": 2182676480 + }, + { + "epoch": 0.32, + "learning_rate": 6.839191141068849e-05, + "loss": 2.4929, + "theoretical_loss": 3.4056778358917024, + "tokens_seen": 2182807552 + }, + { + "epoch": 0.32, + "learning_rate": 6.838388701653025e-05, + "loss": 2.5502, + "theoretical_loss": 3.405661077351027, + "tokens_seen": 2182938624 + }, + { + "epoch": 0.32, + "learning_rate": 6.8375862622372e-05, + "loss": 2.6634, + "theoretical_loss": 3.405644320098298, + "tokens_seen": 2183069696 + }, + { + "epoch": 0.32, + "learning_rate": 6.836783822821377e-05, + "loss": 2.5867, + "theoretical_loss": 3.40562756413334, + "tokens_seen": 2183200768 + }, + { + "epoch": 0.32, + "learning_rate": 6.835981383405553e-05, + "loss": 2.5164, + "theoretical_loss": 3.405610809455976, + "tokens_seen": 2183331840 + }, + { + "epoch": 0.32, + "learning_rate": 6.835178943989729e-05, + "loss": 2.4997, + "theoretical_loss": 3.4055940560660294, + "tokens_seen": 2183462912 + }, + { + "epoch": 0.32, + "learning_rate": 6.834376504573905e-05, + "loss": 2.4337, + "theoretical_loss": 3.405577303963325, + "tokens_seen": 2183593984 + }, + { + "epoch": 0.32, + "learning_rate": 6.83357406515808e-05, + "loss": 2.4849, + "theoretical_loss": 3.4055605531476854, + "tokens_seen": 2183725056 + }, + { + "epoch": 0.32, + "learning_rate": 6.832771625742257e-05, + "loss": 2.5738, + "theoretical_loss": 3.4055438036189356, + "tokens_seen": 2183856128 + }, + { + "epoch": 0.32, + "learning_rate": 6.831969186326432e-05, + "loss": 2.4967, + "theoretical_loss": 3.405527055376899, + "tokens_seen": 2183987200 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1204090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4657957553863525, + "objective/train/theoretical_loss": 3.4055103084213996, + "objective/train/tokens_used": 554577376, + "theoretical_loss": 3.4055103084213996, + "tokens_seen": 2184118272 + }, + { + "epoch": 0.32, + "learning_rate": 6.831166746910609e-05, + "loss": 2.5754, + "theoretical_loss": 3.4055103084213996, + "tokens_seen": 2184118272 + }, + { + "epoch": 0.32, + "learning_rate": 6.830364307494784e-05, + "loss": 2.4762, + "theoretical_loss": 3.4054935627522616, + "tokens_seen": 2184249344 + }, + { + "epoch": 0.32, + "learning_rate": 6.82956186807896e-05, + "loss": 2.5145, + "theoretical_loss": 3.4054768183693085, + "tokens_seen": 2184380416 + }, + { + "epoch": 0.32, + "learning_rate": 6.828759428663136e-05, + "loss": 2.5413, + "theoretical_loss": 3.405460075272365, + "tokens_seen": 2184511488 + }, + { + "epoch": 0.32, + "learning_rate": 6.827956989247311e-05, + "loss": 2.6336, + "theoretical_loss": 3.4054433334612546, + "tokens_seen": 2184642560 + }, + { + "epoch": 0.32, + "learning_rate": 6.827154549831488e-05, + "loss": 2.5995, + "theoretical_loss": 3.4054265929358016, + "tokens_seen": 2184773632 + }, + { + "epoch": 0.32, + "learning_rate": 6.826352110415663e-05, + "loss": 2.5385, + "theoretical_loss": 3.40540985369583, + "tokens_seen": 2184904704 + }, + { + "epoch": 0.32, + "learning_rate": 6.82554967099984e-05, + "loss": 2.3402, + "theoretical_loss": 3.4053931157411648, + "tokens_seen": 2185035776 + }, + { + "epoch": 0.32, + "learning_rate": 6.824747231584015e-05, + "loss": 2.4948, + "theoretical_loss": 3.405376379071629, + "tokens_seen": 2185166848 + }, + { + "epoch": 0.32, + "learning_rate": 6.823944792168192e-05, + "loss": 2.6132, + "theoretical_loss": 3.405359643687048, + "tokens_seen": 2185297920 + }, + { + "epoch": 0.32, + "learning_rate": 6.823142352752367e-05, + "loss": 2.4663, + "theoretical_loss": 3.405342909587245, + "tokens_seen": 2185428992 + }, + { + "epoch": 0.32, + "learning_rate": 6.822339913336542e-05, + "loss": 2.5485, + "theoretical_loss": 3.405326176772045, + "tokens_seen": 2185560064 + }, + { + "epoch": 0.32, + "learning_rate": 6.821537473920719e-05, + "loss": 2.5661, + "theoretical_loss": 3.405309445241272, + "tokens_seen": 2185691136 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1205389, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9753358364105225, + "objective/train/theoretical_loss": 3.4053010799574914, + "objective/train/tokens_used": 556215776, + "theoretical_loss": 3.4053010799574914, + "tokens_seen": 2185756672 + }, + { + "epoch": 0.32, + "learning_rate": 6.820735034504894e-05, + "loss": 2.5152, + "theoretical_loss": 3.405292714994751, + "tokens_seen": 2185822208 + }, + { + "epoch": 0.32, + "learning_rate": 6.819932595089071e-05, + "loss": 2.7423, + "theoretical_loss": 3.405275986032306, + "tokens_seen": 2185953280 + }, + { + "epoch": 0.32, + "learning_rate": 6.819130155673246e-05, + "loss": 2.5975, + "theoretical_loss": 3.4052592583537615, + "tokens_seen": 2186084352 + }, + { + "epoch": 0.32, + "learning_rate": 6.818327716257423e-05, + "loss": 2.7106, + "theoretical_loss": 3.4052425319589417, + "tokens_seen": 2186215424 + }, + { + "epoch": 0.33, + "learning_rate": 6.817525276841598e-05, + "loss": 2.5553, + "theoretical_loss": 3.4052258068476715, + "tokens_seen": 2186346496 + }, + { + "epoch": 0.33, + "learning_rate": 6.816722837425775e-05, + "loss": 2.5986, + "theoretical_loss": 3.4052090830197757, + "tokens_seen": 2186477568 + }, + { + "epoch": 0.33, + "learning_rate": 6.81592039800995e-05, + "loss": 2.6214, + "theoretical_loss": 3.405192360475078, + "tokens_seen": 2186608640 + }, + { + "epoch": 0.33, + "learning_rate": 6.815117958594126e-05, + "loss": 2.517, + "theoretical_loss": 3.405175639213404, + "tokens_seen": 2186739712 + }, + { + "epoch": 0.33, + "learning_rate": 6.814315519178302e-05, + "loss": 2.5678, + "theoretical_loss": 3.4051589192345775, + "tokens_seen": 2186870784 + }, + { + "epoch": 0.33, + "learning_rate": 6.813513079762478e-05, + "loss": 2.471, + "theoretical_loss": 3.405142200538424, + "tokens_seen": 2187001856 + }, + { + "epoch": 0.33, + "learning_rate": 6.812710640346654e-05, + "loss": 2.6426, + "theoretical_loss": 3.405125483124768, + "tokens_seen": 2187132928 + }, + { + "epoch": 0.33, + "learning_rate": 6.81190820093083e-05, + "loss": 2.5891, + "theoretical_loss": 3.4051087669934343, + "tokens_seen": 2187264000 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1206012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.534045696258545, + "objective/train/theoretical_loss": 3.405092052144247, + "objective/train/tokens_used": 557854176, + "theoretical_loss": 3.405092052144247, + "tokens_seen": 2187395072 + }, + { + "epoch": 0.33, + "learning_rate": 6.811105761515006e-05, + "loss": 2.6112, + "theoretical_loss": 3.405092052144247, + "tokens_seen": 2187395072 + }, + { + "epoch": 0.33, + "learning_rate": 6.810303322099182e-05, + "loss": 2.5409, + "theoretical_loss": 3.405075338577032, + "tokens_seen": 2187526144 + }, + { + "epoch": 0.33, + "learning_rate": 6.809500882683357e-05, + "loss": 2.596, + "theoretical_loss": 3.4050586262916136, + "tokens_seen": 2187657216 + }, + { + "epoch": 0.33, + "learning_rate": 6.808698443267534e-05, + "loss": 2.4925, + "theoretical_loss": 3.4050419152878164, + "tokens_seen": 2187788288 + }, + { + "epoch": 0.33, + "learning_rate": 6.807896003851709e-05, + "loss": 2.5335, + "theoretical_loss": 3.405025205565466, + "tokens_seen": 2187919360 + }, + { + "epoch": 0.33, + "learning_rate": 6.807093564435886e-05, + "loss": 2.5448, + "theoretical_loss": 3.405008497124387, + "tokens_seen": 2188050432 + }, + { + "epoch": 0.33, + "learning_rate": 6.806291125020061e-05, + "loss": 2.5736, + "theoretical_loss": 3.404991789964405, + "tokens_seen": 2188181504 + }, + { + "epoch": 0.33, + "learning_rate": 6.805488685604238e-05, + "loss": 2.6279, + "theoretical_loss": 3.404975084085344, + "tokens_seen": 2188312576 + }, + { + "epoch": 0.33, + "learning_rate": 6.804686246188413e-05, + "loss": 2.425, + "theoretical_loss": 3.40495837948703, + "tokens_seen": 2188443648 + }, + { + "epoch": 0.33, + "learning_rate": 6.803883806772588e-05, + "loss": 2.4579, + "theoretical_loss": 3.4049416761692872, + "tokens_seen": 2188574720 + }, + { + "epoch": 0.33, + "learning_rate": 6.803081367356765e-05, + "loss": 2.5621, + "theoretical_loss": 3.4049249741319416, + "tokens_seen": 2188705792 + }, + { + "epoch": 0.33, + "learning_rate": 6.80227892794094e-05, + "loss": 2.4914, + "theoretical_loss": 3.4049082733748186, + "tokens_seen": 2188836864 + }, + { + "epoch": 0.33, + "learning_rate": 6.801476488525117e-05, + "loss": 2.4783, + "theoretical_loss": 3.4048915738977423, + "tokens_seen": 2188967936 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1207268, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3856353759765625, + "objective/train/theoretical_loss": 3.4048832246391676, + "objective/train/tokens_used": 559492576, + "theoretical_loss": 3.4048832246391676, + "tokens_seen": 2189033472 + }, + { + "epoch": 0.33, + "learning_rate": 6.800674049109292e-05, + "loss": 2.4133, + "theoretical_loss": 3.4048748757005387, + "tokens_seen": 2189099008 + }, + { + "epoch": 0.33, + "learning_rate": 6.799871609693469e-05, + "loss": 2.5576, + "theoretical_loss": 3.4048581787830328, + "tokens_seen": 2189230080 + }, + { + "epoch": 0.33, + "learning_rate": 6.799069170277644e-05, + "loss": 2.5191, + "theoretical_loss": 3.4048414831450504, + "tokens_seen": 2189361152 + }, + { + "epoch": 0.33, + "learning_rate": 6.79826673086182e-05, + "loss": 2.5269, + "theoretical_loss": 3.4048247887864163, + "tokens_seen": 2189492224 + }, + { + "epoch": 0.33, + "learning_rate": 6.797464291445996e-05, + "loss": 2.4617, + "theoretical_loss": 3.404808095706956, + "tokens_seen": 2189623296 + }, + { + "epoch": 0.33, + "learning_rate": 6.796661852030171e-05, + "loss": 2.5635, + "theoretical_loss": 3.4047914039064953, + "tokens_seen": 2189754368 + }, + { + "epoch": 0.33, + "learning_rate": 6.795859412614348e-05, + "loss": 2.5235, + "theoretical_loss": 3.404774713384859, + "tokens_seen": 2189885440 + }, + { + "epoch": 0.33, + "learning_rate": 6.795056973198523e-05, + "loss": 2.5943, + "theoretical_loss": 3.404758024141873, + "tokens_seen": 2190016512 + }, + { + "epoch": 0.33, + "learning_rate": 6.7942545337827e-05, + "loss": 2.5848, + "theoretical_loss": 3.4047413361773633, + "tokens_seen": 2190147584 + }, + { + "epoch": 0.33, + "learning_rate": 6.793452094366875e-05, + "loss": 2.5442, + "theoretical_loss": 3.4047246494911545, + "tokens_seen": 2190278656 + }, + { + "epoch": 0.33, + "learning_rate": 6.792649654951051e-05, + "loss": 2.4471, + "theoretical_loss": 3.404707964083073, + "tokens_seen": 2190409728 + }, + { + "epoch": 0.33, + "learning_rate": 6.791847215535227e-05, + "loss": 2.3276, + "theoretical_loss": 3.4046912799529436, + "tokens_seen": 2190540800 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1207761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.551201581954956, + "objective/train/theoretical_loss": 3.404674597100593, + "objective/train/tokens_used": 561130976, + "theoretical_loss": 3.404674597100593, + "tokens_seen": 2190671872 + }, + { + "epoch": 0.33, + "learning_rate": 6.791044776119403e-05, + "loss": 2.5127, + "theoretical_loss": 3.404674597100593, + "tokens_seen": 2190671872 + }, + { + "epoch": 0.33, + "learning_rate": 6.79024233670358e-05, + "loss": 2.5596, + "theoretical_loss": 3.4046579155258456, + "tokens_seen": 2190802944 + }, + { + "epoch": 0.33, + "learning_rate": 6.789439897287755e-05, + "loss": 2.4916, + "theoretical_loss": 3.4046412352285285, + "tokens_seen": 2190934016 + }, + { + "epoch": 0.33, + "learning_rate": 6.788637457871931e-05, + "loss": 2.4777, + "theoretical_loss": 3.4046245562084665, + "tokens_seen": 2191065088 + }, + { + "epoch": 0.33, + "learning_rate": 6.787835018456107e-05, + "loss": 2.4466, + "theoretical_loss": 3.404607878465486, + "tokens_seen": 2191196160 + }, + { + "epoch": 0.33, + "learning_rate": 6.787032579040283e-05, + "loss": 2.5021, + "theoretical_loss": 3.4045912019994122, + "tokens_seen": 2191327232 + }, + { + "epoch": 0.33, + "learning_rate": 6.786230139624459e-05, + "loss": 2.5078, + "theoretical_loss": 3.4045745268100718, + "tokens_seen": 2191458304 + }, + { + "epoch": 0.33, + "learning_rate": 6.785427700208634e-05, + "loss": 2.3974, + "theoretical_loss": 3.40455785289729, + "tokens_seen": 2191589376 + }, + { + "epoch": 0.33, + "learning_rate": 6.78462526079281e-05, + "loss": 2.5153, + "theoretical_loss": 3.404541180260893, + "tokens_seen": 2191720448 + }, + { + "epoch": 0.33, + "learning_rate": 6.783822821376986e-05, + "loss": 2.6615, + "theoretical_loss": 3.404524508900707, + "tokens_seen": 2191851520 + }, + { + "epoch": 0.33, + "learning_rate": 6.783020381961163e-05, + "loss": 2.6367, + "theoretical_loss": 3.404507838816557, + "tokens_seen": 2191982592 + }, + { + "epoch": 0.33, + "learning_rate": 6.782217942545338e-05, + "loss": 2.4062, + "theoretical_loss": 3.40449117000827, + "tokens_seen": 2192113664 + }, + { + "epoch": 0.33, + "learning_rate": 6.781415503129515e-05, + "loss": 2.4533, + "theoretical_loss": 3.404474502475672, + "tokens_seen": 2192244736 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1208978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7618091106414795, + "objective/train/theoretical_loss": 3.4044661691877023, + "objective/train/tokens_used": 562769376, + "theoretical_loss": 3.4044661691877023, + "tokens_seen": 2192310272 + }, + { + "epoch": 0.33, + "learning_rate": 6.78061306371369e-05, + "loss": 2.6423, + "theoretical_loss": 3.4044578362185893, + "tokens_seen": 2192375808 + }, + { + "epoch": 0.33, + "learning_rate": 6.779810624297865e-05, + "loss": 2.3443, + "theoretical_loss": 3.404441171236847, + "tokens_seen": 2192506880 + }, + { + "epoch": 0.33, + "learning_rate": 6.779008184882042e-05, + "loss": 2.4826, + "theoretical_loss": 3.4044245075302726, + "tokens_seen": 2192637952 + }, + { + "epoch": 0.33, + "learning_rate": 6.778205745466217e-05, + "loss": 2.7126, + "theoretical_loss": 3.4044078450986914, + "tokens_seen": 2192769024 + }, + { + "epoch": 0.33, + "learning_rate": 6.777403306050394e-05, + "loss": 2.4795, + "theoretical_loss": 3.40439118394193, + "tokens_seen": 2192900096 + }, + { + "epoch": 0.33, + "learning_rate": 6.776600866634569e-05, + "loss": 2.4872, + "theoretical_loss": 3.4043745240598144, + "tokens_seen": 2193031168 + }, + { + "epoch": 0.33, + "learning_rate": 6.775798427218746e-05, + "loss": 2.5142, + "theoretical_loss": 3.4043578654521713, + "tokens_seen": 2193162240 + }, + { + "epoch": 0.33, + "learning_rate": 6.774995987802921e-05, + "loss": 2.5768, + "theoretical_loss": 3.404341208118827, + "tokens_seen": 2193293312 + }, + { + "epoch": 0.33, + "learning_rate": 6.774193548387096e-05, + "loss": 2.3759, + "theoretical_loss": 3.4043245520596077, + "tokens_seen": 2193424384 + }, + { + "epoch": 0.33, + "learning_rate": 6.773391108971273e-05, + "loss": 2.4352, + "theoretical_loss": 3.4043078972743395, + "tokens_seen": 2193555456 + }, + { + "epoch": 0.33, + "learning_rate": 6.772588669555448e-05, + "loss": 2.562, + "theoretical_loss": 3.4042912437628496, + "tokens_seen": 2193686528 + }, + { + "epoch": 0.33, + "learning_rate": 6.771786230139625e-05, + "loss": 2.5281, + "theoretical_loss": 3.4042745915249637, + "tokens_seen": 2193817600 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1209592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4534859657287598, + "objective/train/theoretical_loss": 3.404257940560509, + "objective/train/tokens_used": 564407776, + "theoretical_loss": 3.404257940560509, + "tokens_seen": 2193948672 + }, + { + "epoch": 0.33, + "learning_rate": 6.7709837907238e-05, + "loss": 2.5249, + "theoretical_loss": 3.404257940560509, + "tokens_seen": 2193948672 + }, + { + "epoch": 0.33, + "learning_rate": 6.770181351307977e-05, + "loss": 2.4806, + "theoretical_loss": 3.4042412908693116, + "tokens_seen": 2194079744 + }, + { + "epoch": 0.33, + "learning_rate": 6.769378911892152e-05, + "loss": 2.5328, + "theoretical_loss": 3.404224642451198, + "tokens_seen": 2194210816 + }, + { + "epoch": 0.33, + "learning_rate": 6.768576472476328e-05, + "loss": 2.4127, + "theoretical_loss": 3.4042079953059954, + "tokens_seen": 2194341888 + }, + { + "epoch": 0.33, + "learning_rate": 6.767774033060504e-05, + "loss": 2.3612, + "theoretical_loss": 3.40419134943353, + "tokens_seen": 2194472960 + }, + { + "epoch": 0.33, + "learning_rate": 6.76697159364468e-05, + "loss": 2.5744, + "theoretical_loss": 3.4041747048336286, + "tokens_seen": 2194604032 + }, + { + "epoch": 0.33, + "learning_rate": 6.766169154228856e-05, + "loss": 2.5248, + "theoretical_loss": 3.4041580615061178, + "tokens_seen": 2194735104 + }, + { + "epoch": 0.33, + "learning_rate": 6.765366714813032e-05, + "loss": 2.4543, + "theoretical_loss": 3.4041414194508244, + "tokens_seen": 2194866176 + }, + { + "epoch": 0.33, + "learning_rate": 6.764564275397208e-05, + "loss": 2.5248, + "theoretical_loss": 3.4041247786675752, + "tokens_seen": 2194997248 + }, + { + "epoch": 0.33, + "learning_rate": 6.763761835981384e-05, + "loss": 2.5018, + "theoretical_loss": 3.404108139156197, + "tokens_seen": 2195128320 + }, + { + "epoch": 0.33, + "learning_rate": 6.762959396565559e-05, + "loss": 2.4872, + "theoretical_loss": 3.4040915009165174, + "tokens_seen": 2195259392 + }, + { + "epoch": 0.33, + "learning_rate": 6.762156957149736e-05, + "loss": 2.5658, + "theoretical_loss": 3.404074863948362, + "tokens_seen": 2195390464 + }, + { + "epoch": 0.33, + "learning_rate": 6.761354517733911e-05, + "loss": 2.5886, + "theoretical_loss": 3.404058228251558, + "tokens_seen": 2195521536 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1211067, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.898007869720459, + "objective/train/theoretical_loss": 3.404049910879859, + "objective/train/tokens_used": 566046176, + "theoretical_loss": 3.404049910879859, + "tokens_seen": 2195587072 + }, + { + "epoch": 0.33, + "learning_rate": 6.760552078318088e-05, + "loss": 2.5062, + "theoretical_loss": 3.404041593825933, + "tokens_seen": 2195652608 + }, + { + "epoch": 0.33, + "learning_rate": 6.759749638902263e-05, + "loss": 2.4295, + "theoretical_loss": 3.4040249606713138, + "tokens_seen": 2195783680 + }, + { + "epoch": 0.33, + "learning_rate": 6.75894719948644e-05, + "loss": 2.5987, + "theoretical_loss": 3.404008328787527, + "tokens_seen": 2195914752 + }, + { + "epoch": 0.33, + "learning_rate": 6.758144760070615e-05, + "loss": 2.5701, + "theoretical_loss": 3.4039916981744005, + "tokens_seen": 2196045824 + }, + { + "epoch": 0.33, + "learning_rate": 6.75734232065479e-05, + "loss": 2.5974, + "theoretical_loss": 3.4039750688317603, + "tokens_seen": 2196176896 + }, + { + "epoch": 0.33, + "learning_rate": 6.756539881238967e-05, + "loss": 2.5712, + "theoretical_loss": 3.4039584407594345, + "tokens_seen": 2196307968 + }, + { + "epoch": 0.33, + "learning_rate": 6.755737441823142e-05, + "loss": 2.4627, + "theoretical_loss": 3.4039418139572493, + "tokens_seen": 2196439040 + }, + { + "epoch": 0.33, + "learning_rate": 6.754935002407319e-05, + "loss": 2.61, + "theoretical_loss": 3.403925188425033, + "tokens_seen": 2196570112 + }, + { + "epoch": 0.33, + "learning_rate": 6.754132562991494e-05, + "loss": 2.4044, + "theoretical_loss": 3.4039085641626117, + "tokens_seen": 2196701184 + }, + { + "epoch": 0.33, + "learning_rate": 6.753330123575671e-05, + "loss": 2.4572, + "theoretical_loss": 3.4038919411698134, + "tokens_seen": 2196832256 + }, + { + "epoch": 0.33, + "learning_rate": 6.752527684159846e-05, + "loss": 2.3802, + "theoretical_loss": 3.403875319446465, + "tokens_seen": 2196963328 + }, + { + "epoch": 0.33, + "learning_rate": 6.751725244744023e-05, + "loss": 2.538, + "theoretical_loss": 3.4038586989923942, + "tokens_seen": 2197094400 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1211765, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.350731134414673, + "objective/train/theoretical_loss": 3.4038420798074283, + "objective/train/tokens_used": 567684576, + "theoretical_loss": 3.4038420798074283, + "tokens_seen": 2197225472 + }, + { + "epoch": 0.33, + "learning_rate": 6.750922805328198e-05, + "loss": 2.6054, + "theoretical_loss": 3.4038420798074283, + "tokens_seen": 2197225472 + }, + { + "epoch": 0.33, + "learning_rate": 6.750120365912373e-05, + "loss": 2.5121, + "theoretical_loss": 3.403825461891395, + "tokens_seen": 2197356544 + }, + { + "epoch": 0.33, + "learning_rate": 6.74931792649655e-05, + "loss": 2.3871, + "theoretical_loss": 3.4038088452441206, + "tokens_seen": 2197487616 + }, + { + "epoch": 0.33, + "learning_rate": 6.748515487080725e-05, + "loss": 2.5174, + "theoretical_loss": 3.4037922298654335, + "tokens_seen": 2197618688 + }, + { + "epoch": 0.33, + "learning_rate": 6.747713047664902e-05, + "loss": 2.5223, + "theoretical_loss": 3.403775615755161, + "tokens_seen": 2197749760 + }, + { + "epoch": 0.33, + "learning_rate": 6.746910608249077e-05, + "loss": 2.4984, + "theoretical_loss": 3.4037590029131306, + "tokens_seen": 2197880832 + }, + { + "epoch": 0.33, + "learning_rate": 6.746108168833254e-05, + "loss": 2.3523, + "theoretical_loss": 3.40374239133917, + "tokens_seen": 2198011904 + }, + { + "epoch": 0.33, + "learning_rate": 6.74530572941743e-05, + "loss": 2.5873, + "theoretical_loss": 3.4037257810331063, + "tokens_seen": 2198142976 + }, + { + "epoch": 0.33, + "learning_rate": 6.744503290001605e-05, + "loss": 2.3596, + "theoretical_loss": 3.403709171994768, + "tokens_seen": 2198274048 + }, + { + "epoch": 0.33, + "learning_rate": 6.743700850585781e-05, + "loss": 2.5814, + "theoretical_loss": 3.4036925642239817, + "tokens_seen": 2198405120 + }, + { + "epoch": 0.33, + "learning_rate": 6.742898411169957e-05, + "loss": 2.3935, + "theoretical_loss": 3.403675957720576, + "tokens_seen": 2198536192 + }, + { + "epoch": 0.33, + "learning_rate": 6.742095971754133e-05, + "loss": 2.5633, + "theoretical_loss": 3.403659352484378, + "tokens_seen": 2198667264 + }, + { + "epoch": 0.33, + "learning_rate": 6.741293532338309e-05, + "loss": 2.3273, + "theoretical_loss": 3.403642748515216, + "tokens_seen": 2198798336 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1213097, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.580597162246704, + "objective/train/theoretical_loss": 3.40363444700572, + "objective/train/tokens_used": 569322976, + "theoretical_loss": 3.40363444700572, + "tokens_seen": 2198863872 + }, + { + "epoch": 0.33, + "learning_rate": 6.740491092922485e-05, + "loss": 2.4982, + "theoretical_loss": 3.403626145812918, + "tokens_seen": 2198929408 + }, + { + "epoch": 0.33, + "learning_rate": 6.73968865350666e-05, + "loss": 2.6289, + "theoretical_loss": 3.4036095443773107, + "tokens_seen": 2199060480 + }, + { + "epoch": 0.33, + "learning_rate": 6.738886214090836e-05, + "loss": 2.5871, + "theoretical_loss": 3.403592944208223, + "tokens_seen": 2199191552 + }, + { + "epoch": 0.33, + "learning_rate": 6.738083774675013e-05, + "loss": 2.4835, + "theoretical_loss": 3.403576345305482, + "tokens_seen": 2199322624 + }, + { + "epoch": 0.33, + "learning_rate": 6.737281335259188e-05, + "loss": 2.4306, + "theoretical_loss": 3.4035597476689166, + "tokens_seen": 2199453696 + }, + { + "epoch": 0.33, + "learning_rate": 6.736478895843365e-05, + "loss": 2.6061, + "theoretical_loss": 3.4035431512983543, + "tokens_seen": 2199584768 + }, + { + "epoch": 0.33, + "learning_rate": 6.73567645642754e-05, + "loss": 2.5906, + "theoretical_loss": 3.403526556193623, + "tokens_seen": 2199715840 + }, + { + "epoch": 0.33, + "learning_rate": 6.734874017011717e-05, + "loss": 2.4728, + "theoretical_loss": 3.403509962354551, + "tokens_seen": 2199846912 + }, + { + "epoch": 0.33, + "learning_rate": 6.734071577595892e-05, + "loss": 2.4921, + "theoretical_loss": 3.403493369780966, + "tokens_seen": 2199977984 + }, + { + "epoch": 0.33, + "learning_rate": 6.733269138180067e-05, + "loss": 2.6282, + "theoretical_loss": 3.403476778472696, + "tokens_seen": 2200109056 + }, + { + "epoch": 0.33, + "learning_rate": 6.732466698764244e-05, + "loss": 2.442, + "theoretical_loss": 3.4034601884295697, + "tokens_seen": 2200240128 + }, + { + "epoch": 0.33, + "learning_rate": 6.731664259348419e-05, + "loss": 2.4406, + "theoretical_loss": 3.403443599651415, + "tokens_seen": 2200371200 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1214287, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.592477560043335, + "objective/train/theoretical_loss": 3.40342701213806, + "objective/train/tokens_used": 570961376, + "theoretical_loss": 3.40342701213806, + "tokens_seen": 2200502272 + }, + { + "epoch": 0.33, + "learning_rate": 6.730861819932596e-05, + "loss": 2.493, + "theoretical_loss": 3.40342701213806, + "tokens_seen": 2200502272 + }, + { + "epoch": 0.33, + "learning_rate": 6.730059380516771e-05, + "loss": 2.3522, + "theoretical_loss": 3.403410425889333, + "tokens_seen": 2200633344 + }, + { + "epoch": 0.33, + "learning_rate": 6.729256941100948e-05, + "loss": 2.3456, + "theoretical_loss": 3.4033938409050624, + "tokens_seen": 2200764416 + }, + { + "epoch": 0.33, + "learning_rate": 6.728454501685123e-05, + "loss": 2.5727, + "theoretical_loss": 3.403377257185076, + "tokens_seen": 2200895488 + }, + { + "epoch": 0.33, + "learning_rate": 6.727652062269298e-05, + "loss": 2.4925, + "theoretical_loss": 3.4033606747292033, + "tokens_seen": 2201026560 + }, + { + "epoch": 0.33, + "learning_rate": 6.726849622853475e-05, + "loss": 2.5255, + "theoretical_loss": 3.403344093537272, + "tokens_seen": 2201157632 + }, + { + "epoch": 0.33, + "learning_rate": 6.72604718343765e-05, + "loss": 2.3977, + "theoretical_loss": 3.4033275136091095, + "tokens_seen": 2201288704 + }, + { + "epoch": 0.33, + "learning_rate": 6.725244744021827e-05, + "loss": 2.4072, + "theoretical_loss": 3.4033109349445456, + "tokens_seen": 2201419776 + }, + { + "epoch": 0.33, + "learning_rate": 6.724442304606002e-05, + "loss": 2.414, + "theoretical_loss": 3.4032943575434085, + "tokens_seen": 2201550848 + }, + { + "epoch": 0.33, + "learning_rate": 6.723639865190179e-05, + "loss": 2.488, + "theoretical_loss": 3.4032777814055266, + "tokens_seen": 2201681920 + }, + { + "epoch": 0.33, + "learning_rate": 6.722837425774354e-05, + "loss": 2.4856, + "theoretical_loss": 3.403261206530728, + "tokens_seen": 2201812992 + }, + { + "epoch": 0.33, + "learning_rate": 6.722034986358531e-05, + "loss": 2.5547, + "theoretical_loss": 3.4032446329188417, + "tokens_seen": 2201944064 + }, + { + "epoch": 0.33, + "learning_rate": 6.721232546942706e-05, + "loss": 2.5414, + "theoretical_loss": 3.4032280605696967, + "tokens_seen": 2202075136 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1214811, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5659379959106445, + "objective/train/theoretical_loss": 3.403219774868598, + "objective/train/tokens_used": 572599776, + "theoretical_loss": 3.403219774868598, + "tokens_seen": 2202140672 + }, + { + "epoch": 0.33, + "learning_rate": 6.720430107526882e-05, + "loss": 2.3994, + "theoretical_loss": 3.403211489483121, + "tokens_seen": 2202206208 + }, + { + "epoch": 0.33, + "learning_rate": 6.719627668111058e-05, + "loss": 2.5061, + "theoretical_loss": 3.4031949196589433, + "tokens_seen": 2202337280 + }, + { + "epoch": 0.33, + "learning_rate": 6.718825228695234e-05, + "loss": 2.4373, + "theoretical_loss": 3.4031783510969924, + "tokens_seen": 2202468352 + }, + { + "epoch": 0.33, + "learning_rate": 6.71802278927941e-05, + "loss": 2.4786, + "theoretical_loss": 3.4031617837970973, + "tokens_seen": 2202599424 + }, + { + "epoch": 0.34, + "learning_rate": 6.717220349863586e-05, + "loss": 2.429, + "theoretical_loss": 3.4031452177590866, + "tokens_seen": 2202730496 + }, + { + "epoch": 0.34, + "learning_rate": 6.716417910447762e-05, + "loss": 2.5734, + "theoretical_loss": 3.4031286529827893, + "tokens_seen": 2202861568 + }, + { + "epoch": 0.34, + "learning_rate": 6.715615471031938e-05, + "loss": 2.4266, + "theoretical_loss": 3.4031120894680336, + "tokens_seen": 2202992640 + }, + { + "epoch": 0.34, + "learning_rate": 6.714813031616113e-05, + "loss": 2.4192, + "theoretical_loss": 3.4030955272146493, + "tokens_seen": 2203123712 + }, + { + "epoch": 0.34, + "learning_rate": 6.71401059220029e-05, + "loss": 2.5598, + "theoretical_loss": 3.4030789662224645, + "tokens_seen": 2203254784 + }, + { + "epoch": 0.34, + "learning_rate": 6.713208152784465e-05, + "loss": 2.6017, + "theoretical_loss": 3.4030624064913084, + "tokens_seen": 2203385856 + }, + { + "epoch": 0.34, + "learning_rate": 6.712405713368642e-05, + "loss": 2.5483, + "theoretical_loss": 3.40304584802101, + "tokens_seen": 2203516928 + }, + { + "epoch": 0.34, + "learning_rate": 6.711603273952817e-05, + "loss": 2.4384, + "theoretical_loss": 3.4030292908113986, + "tokens_seen": 2203648000 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1215803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7451982498168945, + "objective/train/theoretical_loss": 3.403012734862303, + "objective/train/tokens_used": 574238176, + "theoretical_loss": 3.403012734862303, + "tokens_seen": 2203779072 + }, + { + "epoch": 0.34, + "learning_rate": 6.710800834536994e-05, + "loss": 2.5776, + "theoretical_loss": 3.403012734862303, + "tokens_seen": 2203779072 + }, + { + "epoch": 0.34, + "learning_rate": 6.709998395121169e-05, + "loss": 2.4848, + "theoretical_loss": 3.402996180173552, + "tokens_seen": 2203910144 + }, + { + "epoch": 0.34, + "learning_rate": 6.709195955705344e-05, + "loss": 2.5394, + "theoretical_loss": 3.4029796267449752, + "tokens_seen": 2204041216 + }, + { + "epoch": 0.34, + "learning_rate": 6.708393516289521e-05, + "loss": 2.5138, + "theoretical_loss": 3.4029630745764012, + "tokens_seen": 2204172288 + }, + { + "epoch": 0.34, + "learning_rate": 6.707591076873696e-05, + "loss": 2.4304, + "theoretical_loss": 3.4029465236676595, + "tokens_seen": 2204303360 + }, + { + "epoch": 0.34, + "learning_rate": 6.706788637457873e-05, + "loss": 2.3949, + "theoretical_loss": 3.40292997401858, + "tokens_seen": 2204434432 + }, + { + "epoch": 0.34, + "learning_rate": 6.705986198042048e-05, + "loss": 2.422, + "theoretical_loss": 3.4029134256289906, + "tokens_seen": 2204565504 + }, + { + "epoch": 0.34, + "learning_rate": 6.705183758626225e-05, + "loss": 2.4923, + "theoretical_loss": 3.402896878498721, + "tokens_seen": 2204696576 + }, + { + "epoch": 0.34, + "learning_rate": 6.7043813192104e-05, + "loss": 2.3677, + "theoretical_loss": 3.402880332627601, + "tokens_seen": 2204827648 + }, + { + "epoch": 0.34, + "learning_rate": 6.703578879794575e-05, + "loss": 2.5273, + "theoretical_loss": 3.4028637880154595, + "tokens_seen": 2204958720 + }, + { + "epoch": 0.34, + "learning_rate": 6.702776440378752e-05, + "loss": 2.5261, + "theoretical_loss": 3.4028472446621265, + "tokens_seen": 2205089792 + }, + { + "epoch": 0.34, + "learning_rate": 6.701974000962927e-05, + "loss": 2.4574, + "theoretical_loss": 3.4028307025674303, + "tokens_seen": 2205220864 + }, + { + "epoch": 0.34, + "learning_rate": 6.701171561547104e-05, + "loss": 2.5115, + "theoretical_loss": 3.402814161731201, + "tokens_seen": 2205351936 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1216279, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.408686637878418, + "objective/train/theoretical_loss": 3.4028058917849586, + "objective/train/tokens_used": 575876576, + "theoretical_loss": 3.4028058917849586, + "tokens_seen": 2205417472 + }, + { + "epoch": 0.34, + "learning_rate": 6.70036912213128e-05, + "loss": 2.556, + "theoretical_loss": 3.4027976221532685, + "tokens_seen": 2205483008 + }, + { + "epoch": 0.34, + "learning_rate": 6.699566682715456e-05, + "loss": 2.4738, + "theoretical_loss": 3.402781083833462, + "tokens_seen": 2205614080 + }, + { + "epoch": 0.34, + "learning_rate": 6.698764243299631e-05, + "loss": 2.4675, + "theoretical_loss": 3.4027645467716106, + "tokens_seen": 2205745152 + }, + { + "epoch": 0.34, + "learning_rate": 6.697961803883807e-05, + "loss": 2.4454, + "theoretical_loss": 3.4027480109675436, + "tokens_seen": 2205876224 + }, + { + "epoch": 0.34, + "learning_rate": 6.697159364467983e-05, + "loss": 2.6172, + "theoretical_loss": 3.402731476421092, + "tokens_seen": 2206007296 + }, + { + "epoch": 0.34, + "learning_rate": 6.696356925052159e-05, + "loss": 2.403, + "theoretical_loss": 3.402714943132084, + "tokens_seen": 2206138368 + }, + { + "epoch": 0.34, + "learning_rate": 6.695554485636335e-05, + "loss": 2.431, + "theoretical_loss": 3.40269841110035, + "tokens_seen": 2206269440 + }, + { + "epoch": 0.34, + "learning_rate": 6.694752046220511e-05, + "loss": 2.4604, + "theoretical_loss": 3.40268188032572, + "tokens_seen": 2206400512 + }, + { + "epoch": 0.34, + "learning_rate": 6.693949606804687e-05, + "loss": 2.4266, + "theoretical_loss": 3.402665350808023, + "tokens_seen": 2206531584 + }, + { + "epoch": 0.34, + "learning_rate": 6.693147167388863e-05, + "loss": 2.56, + "theoretical_loss": 3.4026488225470892, + "tokens_seen": 2206662656 + }, + { + "epoch": 0.34, + "learning_rate": 6.69234472797304e-05, + "loss": 2.5656, + "theoretical_loss": 3.402632295542748, + "tokens_seen": 2206793728 + }, + { + "epoch": 0.34, + "learning_rate": 6.691542288557215e-05, + "loss": 2.4333, + "theoretical_loss": 3.40261576979483, + "tokens_seen": 2206924800 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1217338, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7593905925750732, + "objective/train/theoretical_loss": 3.4025992453031644, + "objective/train/tokens_used": 577514976, + "theoretical_loss": 3.4025992453031644, + "tokens_seen": 2207055872 + }, + { + "epoch": 0.34, + "learning_rate": 6.69073984914139e-05, + "loss": 2.5397, + "theoretical_loss": 3.4025992453031644, + "tokens_seen": 2207055872 + }, + { + "epoch": 0.34, + "learning_rate": 6.689937409725567e-05, + "loss": 2.6356, + "theoretical_loss": 3.402582722067581, + "tokens_seen": 2207186944 + }, + { + "epoch": 0.34, + "learning_rate": 6.689134970309742e-05, + "loss": 2.5292, + "theoretical_loss": 3.4025662000879104, + "tokens_seen": 2207318016 + }, + { + "epoch": 0.34, + "learning_rate": 6.688332530893919e-05, + "loss": 2.4998, + "theoretical_loss": 3.4025496793639824, + "tokens_seen": 2207449088 + }, + { + "epoch": 0.34, + "learning_rate": 6.687530091478094e-05, + "loss": 2.4771, + "theoretical_loss": 3.4025331598956265, + "tokens_seen": 2207580160 + }, + { + "epoch": 0.34, + "learning_rate": 6.68672765206227e-05, + "loss": 2.6201, + "theoretical_loss": 3.402516641682673, + "tokens_seen": 2207711232 + }, + { + "epoch": 0.34, + "learning_rate": 6.685925212646446e-05, + "loss": 2.583, + "theoretical_loss": 3.4025001247249524, + "tokens_seen": 2207842304 + }, + { + "epoch": 0.34, + "learning_rate": 6.685122773230621e-05, + "loss": 2.4937, + "theoretical_loss": 3.402483609022294, + "tokens_seen": 2207973376 + }, + { + "epoch": 0.34, + "learning_rate": 6.684320333814798e-05, + "loss": 2.3755, + "theoretical_loss": 3.402467094574529, + "tokens_seen": 2208104448 + }, + { + "epoch": 0.34, + "learning_rate": 6.683517894398973e-05, + "loss": 2.5587, + "theoretical_loss": 3.4024505813814865, + "tokens_seen": 2208235520 + }, + { + "epoch": 0.34, + "learning_rate": 6.68271545498315e-05, + "loss": 2.5725, + "theoretical_loss": 3.402434069442997, + "tokens_seen": 2208366592 + }, + { + "epoch": 0.34, + "learning_rate": 6.681913015567325e-05, + "loss": 2.5799, + "theoretical_loss": 3.4024175587588914, + "tokens_seen": 2208497664 + }, + { + "epoch": 0.34, + "learning_rate": 6.681110576151502e-05, + "loss": 2.401, + "theoretical_loss": 3.402401049328999, + "tokens_seen": 2208628736 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1217846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5893185138702393, + "objective/train/theoretical_loss": 3.40239279508433, + "objective/train/tokens_used": 579153376, + "theoretical_loss": 3.40239279508433, + "tokens_seen": 2208694272 + }, + { + "epoch": 0.34, + "learning_rate": 6.680308136735677e-05, + "loss": 2.4925, + "theoretical_loss": 3.402384541153151, + "tokens_seen": 2208759808 + }, + { + "epoch": 0.34, + "learning_rate": 6.679505697319852e-05, + "loss": 2.4016, + "theoretical_loss": 3.402368034231177, + "tokens_seen": 2208890880 + }, + { + "epoch": 0.34, + "learning_rate": 6.678703257904029e-05, + "loss": 2.3912, + "theoretical_loss": 3.4023515285629076, + "tokens_seen": 2209021952 + }, + { + "epoch": 0.34, + "learning_rate": 6.677900818488204e-05, + "loss": 2.4989, + "theoretical_loss": 3.402335024148173, + "tokens_seen": 2209153024 + }, + { + "epoch": 0.34, + "learning_rate": 6.677098379072381e-05, + "loss": 2.553, + "theoretical_loss": 3.4023185209868045, + "tokens_seen": 2209284096 + }, + { + "epoch": 0.34, + "learning_rate": 6.676295939656556e-05, + "loss": 2.5332, + "theoretical_loss": 3.4023020190786317, + "tokens_seen": 2209415168 + }, + { + "epoch": 0.34, + "learning_rate": 6.675493500240733e-05, + "loss": 2.5386, + "theoretical_loss": 3.4022855184234855, + "tokens_seen": 2209546240 + }, + { + "epoch": 0.34, + "learning_rate": 6.674691060824908e-05, + "loss": 2.4758, + "theoretical_loss": 3.4022690190211957, + "tokens_seen": 2209677312 + }, + { + "epoch": 0.34, + "learning_rate": 6.673888621409084e-05, + "loss": 2.482, + "theoretical_loss": 3.402252520871594, + "tokens_seen": 2209808384 + }, + { + "epoch": 0.34, + "learning_rate": 6.67308618199326e-05, + "loss": 2.5679, + "theoretical_loss": 3.4022360239745106, + "tokens_seen": 2209939456 + }, + { + "epoch": 0.34, + "learning_rate": 6.672283742577436e-05, + "loss": 2.5476, + "theoretical_loss": 3.4022195283297756, + "tokens_seen": 2210070528 + }, + { + "epoch": 0.34, + "learning_rate": 6.671481303161612e-05, + "loss": 2.669, + "theoretical_loss": 3.4022030339372202, + "tokens_seen": 2210201600 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1219246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7586281299591064, + "objective/train/theoretical_loss": 3.402186540796675, + "objective/train/tokens_used": 580791776, + "theoretical_loss": 3.402186540796675, + "tokens_seen": 2210332672 + }, + { + "epoch": 0.34, + "learning_rate": 6.670678863745788e-05, + "loss": 2.4485, + "theoretical_loss": 3.402186540796675, + "tokens_seen": 2210332672 + }, + { + "epoch": 0.34, + "learning_rate": 6.669876424329964e-05, + "loss": 2.3993, + "theoretical_loss": 3.4021700489079705, + "tokens_seen": 2210463744 + }, + { + "epoch": 0.34, + "learning_rate": 6.66907398491414e-05, + "loss": 2.4686, + "theoretical_loss": 3.402153558270938, + "tokens_seen": 2210594816 + }, + { + "epoch": 0.34, + "learning_rate": 6.668271545498315e-05, + "loss": 2.4943, + "theoretical_loss": 3.4021370688854073, + "tokens_seen": 2210725888 + }, + { + "epoch": 0.34, + "learning_rate": 6.667469106082492e-05, + "loss": 2.5063, + "theoretical_loss": 3.4021205807512107, + "tokens_seen": 2210856960 + }, + { + "epoch": 0.34, + "learning_rate": 6.666666666666667e-05, + "loss": 2.4981, + "theoretical_loss": 3.4021040938681777, + "tokens_seen": 2210988032 + }, + { + "epoch": 0.34, + "learning_rate": 6.665864227250844e-05, + "loss": 2.5183, + "theoretical_loss": 3.40208760823614, + "tokens_seen": 2211119104 + }, + { + "epoch": 0.34, + "learning_rate": 6.665061787835019e-05, + "loss": 2.4552, + "theoretical_loss": 3.402071123854928, + "tokens_seen": 2211250176 + }, + { + "epoch": 0.34, + "learning_rate": 6.664259348419196e-05, + "loss": 2.5751, + "theoretical_loss": 3.4020546407243732, + "tokens_seen": 2211381248 + }, + { + "epoch": 0.34, + "learning_rate": 6.663456909003371e-05, + "loss": 2.5156, + "theoretical_loss": 3.4020381588443063, + "tokens_seen": 2211512320 + }, + { + "epoch": 0.34, + "learning_rate": 6.662654469587546e-05, + "loss": 2.5568, + "theoretical_loss": 3.402021678214558, + "tokens_seen": 2211643392 + }, + { + "epoch": 0.34, + "learning_rate": 6.661852030171723e-05, + "loss": 2.5904, + "theoretical_loss": 3.40200519883496, + "tokens_seen": 2211774464 + }, + { + "epoch": 0.34, + "learning_rate": 6.661049590755898e-05, + "loss": 2.5605, + "theoretical_loss": 3.4019887207053428, + "tokens_seen": 2211905536 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1220526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.844651937484741, + "objective/train/theoretical_loss": 3.4019804821092245, + "objective/train/tokens_used": 582430176, + "theoretical_loss": 3.4019804821092245, + "tokens_seen": 2211971072 + }, + { + "epoch": 0.34, + "learning_rate": 6.660247151340075e-05, + "loss": 2.461, + "theoretical_loss": 3.401972243825538, + "tokens_seen": 2212036608 + }, + { + "epoch": 0.34, + "learning_rate": 6.65944471192425e-05, + "loss": 2.6401, + "theoretical_loss": 3.4019557681953767, + "tokens_seen": 2212167680 + }, + { + "epoch": 0.34, + "learning_rate": 6.658642272508427e-05, + "loss": 2.4676, + "theoretical_loss": 3.4019392938146895, + "tokens_seen": 2212298752 + }, + { + "epoch": 0.34, + "learning_rate": 6.657839833092602e-05, + "loss": 2.5706, + "theoretical_loss": 3.4019228206833088, + "tokens_seen": 2212429824 + }, + { + "epoch": 0.34, + "learning_rate": 6.657037393676779e-05, + "loss": 2.3169, + "theoretical_loss": 3.401906348801065, + "tokens_seen": 2212560896 + }, + { + "epoch": 0.34, + "learning_rate": 6.656234954260954e-05, + "loss": 2.6073, + "theoretical_loss": 3.401889878167789, + "tokens_seen": 2212691968 + }, + { + "epoch": 0.34, + "learning_rate": 6.65543251484513e-05, + "loss": 2.539, + "theoretical_loss": 3.401873408783313, + "tokens_seen": 2212823040 + }, + { + "epoch": 0.34, + "learning_rate": 6.654630075429306e-05, + "loss": 2.5384, + "theoretical_loss": 3.401856940647468, + "tokens_seen": 2212954112 + }, + { + "epoch": 0.34, + "learning_rate": 6.653827636013481e-05, + "loss": 2.5948, + "theoretical_loss": 3.4018404737600854, + "tokens_seen": 2213085184 + }, + { + "epoch": 0.34, + "learning_rate": 6.653025196597658e-05, + "loss": 2.5626, + "theoretical_loss": 3.4018240081209963, + "tokens_seen": 2213216256 + }, + { + "epoch": 0.34, + "learning_rate": 6.652222757181833e-05, + "loss": 2.561, + "theoretical_loss": 3.401807543730033, + "tokens_seen": 2213347328 + }, + { + "epoch": 0.34, + "learning_rate": 6.65142031776601e-05, + "loss": 2.4798, + "theoretical_loss": 3.401791080587026, + "tokens_seen": 2213478400 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1221127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5792646408081055, + "objective/train/theoretical_loss": 3.4017746186918076, + "objective/train/tokens_used": 584068576, + "theoretical_loss": 3.4017746186918076, + "tokens_seen": 2213609472 + }, + { + "epoch": 0.34, + "learning_rate": 6.650617878350185e-05, + "loss": 2.6339, + "theoretical_loss": 3.4017746186918076, + "tokens_seen": 2213609472 + }, + { + "epoch": 0.34, + "learning_rate": 6.649815438934361e-05, + "loss": 2.4253, + "theoretical_loss": 3.401758158044209, + "tokens_seen": 2213740544 + }, + { + "epoch": 0.34, + "learning_rate": 6.649012999518537e-05, + "loss": 2.5425, + "theoretical_loss": 3.4017416986440616, + "tokens_seen": 2213871616 + }, + { + "epoch": 0.34, + "learning_rate": 6.648210560102713e-05, + "loss": 2.5688, + "theoretical_loss": 3.4017252404911975, + "tokens_seen": 2214002688 + }, + { + "epoch": 0.34, + "learning_rate": 6.64740812068689e-05, + "loss": 2.5209, + "theoretical_loss": 3.4017087835854474, + "tokens_seen": 2214133760 + }, + { + "epoch": 0.34, + "learning_rate": 6.646605681271065e-05, + "loss": 2.5333, + "theoretical_loss": 3.401692327926644, + "tokens_seen": 2214264832 + }, + { + "epoch": 0.34, + "learning_rate": 6.645803241855241e-05, + "loss": 2.4757, + "theoretical_loss": 3.4016758735146193, + "tokens_seen": 2214395904 + }, + { + "epoch": 0.34, + "learning_rate": 6.645000802439417e-05, + "loss": 2.5999, + "theoretical_loss": 3.401659420349204, + "tokens_seen": 2214526976 + }, + { + "epoch": 0.34, + "learning_rate": 6.644198363023592e-05, + "loss": 2.4715, + "theoretical_loss": 3.40164296843023, + "tokens_seen": 2214658048 + }, + { + "epoch": 0.34, + "learning_rate": 6.643395923607769e-05, + "loss": 2.4961, + "theoretical_loss": 3.40162651775753, + "tokens_seen": 2214789120 + }, + { + "epoch": 0.34, + "learning_rate": 6.642593484191944e-05, + "loss": 2.4219, + "theoretical_loss": 3.4016100683309345, + "tokens_seen": 2214920192 + }, + { + "epoch": 0.34, + "learning_rate": 6.64179104477612e-05, + "loss": 2.4605, + "theoretical_loss": 3.401593620150277, + "tokens_seen": 2215051264 + }, + { + "epoch": 0.34, + "learning_rate": 6.640988605360296e-05, + "loss": 2.5053, + "theoretical_loss": 3.401577173215388, + "tokens_seen": 2215182336 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1222538, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.598327398300171, + "objective/train/theoretical_loss": 3.4015689502150543, + "objective/train/tokens_used": 585706976, + "theoretical_loss": 3.4015689502150543, + "tokens_seen": 2215247872 + }, + { + "epoch": 0.34, + "learning_rate": 6.640186165944471e-05, + "loss": 2.5099, + "theoretical_loss": 3.4015607275260997, + "tokens_seen": 2215313408 + }, + { + "epoch": 0.34, + "learning_rate": 6.639383726528648e-05, + "loss": 2.517, + "theoretical_loss": 3.401544283082245, + "tokens_seen": 2215444480 + }, + { + "epoch": 0.34, + "learning_rate": 6.638581287112823e-05, + "loss": 2.6561, + "theoretical_loss": 3.401527839883655, + "tokens_seen": 2215575552 + }, + { + "epoch": 0.34, + "learning_rate": 6.637778847697e-05, + "loss": 2.4735, + "theoretical_loss": 3.401511397930162, + "tokens_seen": 2215706624 + }, + { + "epoch": 0.34, + "learning_rate": 6.636976408281175e-05, + "loss": 2.5296, + "theoretical_loss": 3.401494957221598, + "tokens_seen": 2215837696 + }, + { + "epoch": 0.34, + "learning_rate": 6.636173968865352e-05, + "loss": 2.6785, + "theoretical_loss": 3.4014785177577953, + "tokens_seen": 2215968768 + }, + { + "epoch": 0.34, + "learning_rate": 6.635371529449527e-05, + "loss": 2.4534, + "theoretical_loss": 3.401462079538586, + "tokens_seen": 2216099840 + }, + { + "epoch": 0.34, + "learning_rate": 6.634569090033703e-05, + "loss": 2.4605, + "theoretical_loss": 3.401445642563802, + "tokens_seen": 2216230912 + }, + { + "epoch": 0.34, + "learning_rate": 6.633766650617879e-05, + "loss": 2.4743, + "theoretical_loss": 3.401429206833276, + "tokens_seen": 2216361984 + }, + { + "epoch": 0.34, + "learning_rate": 6.632964211202055e-05, + "loss": 2.5421, + "theoretical_loss": 3.40141277234684, + "tokens_seen": 2216493056 + }, + { + "epoch": 0.34, + "learning_rate": 6.632161771786231e-05, + "loss": 2.4795, + "theoretical_loss": 3.401396339104326, + "tokens_seen": 2216624128 + }, + { + "epoch": 0.34, + "learning_rate": 6.631359332370407e-05, + "loss": 2.3755, + "theoretical_loss": 3.401379907105566, + "tokens_seen": 2216755200 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1222919, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.506207227706909, + "objective/train/theoretical_loss": 3.4013634763503937, + "objective/train/tokens_used": 587345376, + "theoretical_loss": 3.4013634763503937, + "tokens_seen": 2216886272 + }, + { + "epoch": 0.34, + "learning_rate": 6.630556892954582e-05, + "loss": 2.54, + "theoretical_loss": 3.4013634763503937, + "tokens_seen": 2216886272 + }, + { + "epoch": 0.34, + "learning_rate": 6.629754453538758e-05, + "loss": 2.6115, + "theoretical_loss": 3.40134704683864, + "tokens_seen": 2217017344 + }, + { + "epoch": 0.34, + "learning_rate": 6.628952014122934e-05, + "loss": 2.6062, + "theoretical_loss": 3.4013306185701384, + "tokens_seen": 2217148416 + }, + { + "epoch": 0.34, + "learning_rate": 6.62814957470711e-05, + "loss": 2.392, + "theoretical_loss": 3.4013141915447207, + "tokens_seen": 2217279488 + }, + { + "epoch": 0.34, + "learning_rate": 6.627347135291286e-05, + "loss": 2.62, + "theoretical_loss": 3.4012977657622194, + "tokens_seen": 2217410560 + }, + { + "epoch": 0.34, + "learning_rate": 6.626544695875462e-05, + "loss": 2.5453, + "theoretical_loss": 3.401281341222467, + "tokens_seen": 2217541632 + }, + { + "epoch": 0.34, + "learning_rate": 6.625742256459638e-05, + "loss": 2.5269, + "theoretical_loss": 3.4012649179252965, + "tokens_seen": 2217672704 + }, + { + "epoch": 0.34, + "learning_rate": 6.624939817043813e-05, + "loss": 2.4656, + "theoretical_loss": 3.4012484958705396, + "tokens_seen": 2217803776 + }, + { + "epoch": 0.34, + "learning_rate": 6.62413737762799e-05, + "loss": 2.4634, + "theoretical_loss": 3.40123207505803, + "tokens_seen": 2217934848 + }, + { + "epoch": 0.34, + "learning_rate": 6.623334938212165e-05, + "loss": 2.5549, + "theoretical_loss": 3.401215655487599, + "tokens_seen": 2218065920 + }, + { + "epoch": 0.34, + "learning_rate": 6.622532498796342e-05, + "loss": 2.4937, + "theoretical_loss": 3.4011992371590805, + "tokens_seen": 2218196992 + }, + { + "epoch": 0.34, + "learning_rate": 6.621730059380517e-05, + "loss": 2.5013, + "theoretical_loss": 3.4011828200723064, + "tokens_seen": 2218328064 + }, + { + "epoch": 0.34, + "learning_rate": 6.620927619964692e-05, + "loss": 2.6017, + "theoretical_loss": 3.40116640422711, + "tokens_seen": 2218459136 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1223993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.066260814666748, + "objective/train/theoretical_loss": 3.4011581967700506, + "objective/train/tokens_used": 588983776, + "theoretical_loss": 3.4011581967700506, + "tokens_seen": 2218524672 + }, + { + "epoch": 0.34, + "learning_rate": 6.620125180548869e-05, + "loss": 2.5717, + "theoretical_loss": 3.401149989623323, + "tokens_seen": 2218590208 + }, + { + "epoch": 0.34, + "learning_rate": 6.619322741133044e-05, + "loss": 2.7128, + "theoretical_loss": 3.4011335762607797, + "tokens_seen": 2218721280 + }, + { + "epoch": 0.34, + "learning_rate": 6.618520301717221e-05, + "loss": 2.6112, + "theoretical_loss": 3.401117164139312, + "tokens_seen": 2218852352 + }, + { + "epoch": 0.34, + "learning_rate": 6.617717862301396e-05, + "loss": 2.5601, + "theoretical_loss": 3.4011007532587527, + "tokens_seen": 2218983424 + }, + { + "epoch": 0.34, + "learning_rate": 6.616915422885573e-05, + "loss": 2.466, + "theoretical_loss": 3.401084343618935, + "tokens_seen": 2219114496 + }, + { + "epoch": 0.35, + "learning_rate": 6.616112983469748e-05, + "loss": 2.5962, + "theoretical_loss": 3.4010679352196918, + "tokens_seen": 2219245568 + }, + { + "epoch": 0.35, + "learning_rate": 6.615310544053924e-05, + "loss": 2.6519, + "theoretical_loss": 3.4010515280608558, + "tokens_seen": 2219376640 + }, + { + "epoch": 0.35, + "learning_rate": 6.6145081046381e-05, + "loss": 2.6243, + "theoretical_loss": 3.40103512214226, + "tokens_seen": 2219507712 + }, + { + "epoch": 0.35, + "learning_rate": 6.613705665222276e-05, + "loss": 2.4531, + "theoretical_loss": 3.401018717463738, + "tokens_seen": 2219638784 + }, + { + "epoch": 0.35, + "learning_rate": 6.612903225806452e-05, + "loss": 2.6186, + "theoretical_loss": 3.401002314025122, + "tokens_seen": 2219769856 + }, + { + "epoch": 0.35, + "learning_rate": 6.612100786390628e-05, + "loss": 2.3972, + "theoretical_loss": 3.4009859118262455, + "tokens_seen": 2219900928 + }, + { + "epoch": 0.35, + "learning_rate": 6.611298346974803e-05, + "loss": 2.6535, + "theoretical_loss": 3.400969510866942, + "tokens_seen": 2220032000 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1224628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7098729610443115, + "objective/train/theoretical_loss": 3.400953111147044, + "objective/train/tokens_used": 590622176, + "theoretical_loss": 3.400953111147044, + "tokens_seen": 2220163072 + }, + { + "epoch": 0.35, + "learning_rate": 6.61049590755898e-05, + "loss": 2.4975, + "theoretical_loss": 3.400953111147044, + "tokens_seen": 2220163072 + }, + { + "epoch": 0.35, + "learning_rate": 6.609693468143155e-05, + "loss": 2.5683, + "theoretical_loss": 3.4009367126663848, + "tokens_seen": 2220294144 + }, + { + "epoch": 0.35, + "learning_rate": 6.608891028727332e-05, + "loss": 2.6508, + "theoretical_loss": 3.4009203154247976, + "tokens_seen": 2220425216 + }, + { + "epoch": 0.35, + "learning_rate": 6.608088589311507e-05, + "loss": 2.6241, + "theoretical_loss": 3.4009039194221162, + "tokens_seen": 2220556288 + }, + { + "epoch": 0.35, + "learning_rate": 6.607286149895684e-05, + "loss": 2.5669, + "theoretical_loss": 3.4008875246581733, + "tokens_seen": 2220687360 + }, + { + "epoch": 0.35, + "learning_rate": 6.606483710479859e-05, + "loss": 2.4826, + "theoretical_loss": 3.400871131132802, + "tokens_seen": 2220818432 + }, + { + "epoch": 0.35, + "learning_rate": 6.605681271064034e-05, + "loss": 2.3772, + "theoretical_loss": 3.4008547388458363, + "tokens_seen": 2220949504 + }, + { + "epoch": 0.35, + "learning_rate": 6.604878831648211e-05, + "loss": 2.6465, + "theoretical_loss": 3.400838347797109, + "tokens_seen": 2221080576 + }, + { + "epoch": 0.35, + "learning_rate": 6.604076392232386e-05, + "loss": 2.5433, + "theoretical_loss": 3.4008219579864543, + "tokens_seen": 2221211648 + }, + { + "epoch": 0.35, + "learning_rate": 6.603273952816563e-05, + "loss": 2.4825, + "theoretical_loss": 3.4008055694137047, + "tokens_seen": 2221342720 + }, + { + "epoch": 0.35, + "learning_rate": 6.602471513400738e-05, + "loss": 2.5396, + "theoretical_loss": 3.400789182078694, + "tokens_seen": 2221473792 + }, + { + "epoch": 0.35, + "learning_rate": 6.601669073984913e-05, + "loss": 2.5519, + "theoretical_loss": 3.400772795981256, + "tokens_seen": 2221604864 + }, + { + "epoch": 0.35, + "learning_rate": 6.60086663456909e-05, + "loss": 2.5309, + "theoretical_loss": 3.4007564111212236, + "tokens_seen": 2221735936 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1225688, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.183415651321411, + "objective/train/theoretical_loss": 3.400748219155183, + "objective/train/tokens_used": 592260576, + "theoretical_loss": 3.400748219155183, + "tokens_seen": 2221801472 + }, + { + "epoch": 0.35, + "learning_rate": 6.600064195153265e-05, + "loss": 2.4961, + "theoretical_loss": 3.400740027498431, + "tokens_seen": 2221867008 + }, + { + "epoch": 0.35, + "learning_rate": 6.599261755737442e-05, + "loss": 2.662, + "theoretical_loss": 3.400723645112712, + "tokens_seen": 2221998080 + }, + { + "epoch": 0.35, + "learning_rate": 6.598459316321617e-05, + "loss": 2.5004, + "theoretical_loss": 3.400707263963899, + "tokens_seen": 2222129152 + }, + { + "epoch": 0.35, + "learning_rate": 6.597656876905794e-05, + "loss": 2.5408, + "theoretical_loss": 3.400690884051827, + "tokens_seen": 2222260224 + }, + { + "epoch": 0.35, + "learning_rate": 6.59685443748997e-05, + "loss": 2.4789, + "theoretical_loss": 3.4006745053763288, + "tokens_seen": 2222391296 + }, + { + "epoch": 0.35, + "learning_rate": 6.596051998074145e-05, + "loss": 2.5802, + "theoretical_loss": 3.4006581279372385, + "tokens_seen": 2222522368 + }, + { + "epoch": 0.35, + "learning_rate": 6.595249558658321e-05, + "loss": 2.4772, + "theoretical_loss": 3.4006417517343897, + "tokens_seen": 2222653440 + }, + { + "epoch": 0.35, + "learning_rate": 6.594447119242497e-05, + "loss": 2.7164, + "theoretical_loss": 3.4006253767676164, + "tokens_seen": 2222784512 + }, + { + "epoch": 0.35, + "learning_rate": 6.593644679826673e-05, + "loss": 2.5849, + "theoretical_loss": 3.4006090030367524, + "tokens_seen": 2222915584 + }, + { + "epoch": 0.35, + "learning_rate": 6.592842240410849e-05, + "loss": 2.5573, + "theoretical_loss": 3.4005926305416314, + "tokens_seen": 2223046656 + }, + { + "epoch": 0.35, + "learning_rate": 6.592039800995025e-05, + "loss": 2.5864, + "theoretical_loss": 3.4005762592820874, + "tokens_seen": 2223177728 + }, + { + "epoch": 0.35, + "learning_rate": 6.5912373615792e-05, + "loss": 2.4563, + "theoretical_loss": 3.4005598892579543, + "tokens_seen": 2223308800 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1226788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7860403060913086, + "objective/train/theoretical_loss": 3.400543520469066, + "objective/train/tokens_used": 593898976, + "theoretical_loss": 3.400543520469066, + "tokens_seen": 2223439872 + }, + { + "epoch": 0.35, + "learning_rate": 6.590434922163376e-05, + "loss": 2.7357, + "theoretical_loss": 3.400543520469066, + "tokens_seen": 2223439872 + }, + { + "epoch": 0.35, + "learning_rate": 6.589632482747553e-05, + "loss": 2.5975, + "theoretical_loss": 3.4005271529152563, + "tokens_seen": 2223570944 + }, + { + "epoch": 0.35, + "learning_rate": 6.588830043331728e-05, + "loss": 2.5676, + "theoretical_loss": 3.4005107865963597, + "tokens_seen": 2223702016 + }, + { + "epoch": 0.35, + "learning_rate": 6.588027603915905e-05, + "loss": 2.3776, + "theoretical_loss": 3.4004944215122097, + "tokens_seen": 2223833088 + }, + { + "epoch": 0.35, + "learning_rate": 6.58722516450008e-05, + "loss": 2.4822, + "theoretical_loss": 3.400478057662641, + "tokens_seen": 2223964160 + }, + { + "epoch": 0.35, + "learning_rate": 6.586422725084257e-05, + "loss": 2.6994, + "theoretical_loss": 3.400461695047487, + "tokens_seen": 2224095232 + }, + { + "epoch": 0.35, + "learning_rate": 6.585620285668432e-05, + "loss": 2.5839, + "theoretical_loss": 3.4004453336665827, + "tokens_seen": 2224226304 + }, + { + "epoch": 0.35, + "learning_rate": 6.584817846252607e-05, + "loss": 2.5777, + "theoretical_loss": 3.4004289735197615, + "tokens_seen": 2224357376 + }, + { + "epoch": 0.35, + "learning_rate": 6.584015406836784e-05, + "loss": 2.6066, + "theoretical_loss": 3.4004126146068576, + "tokens_seen": 2224488448 + }, + { + "epoch": 0.35, + "learning_rate": 6.583212967420959e-05, + "loss": 2.553, + "theoretical_loss": 3.4003962569277055, + "tokens_seen": 2224619520 + }, + { + "epoch": 0.35, + "learning_rate": 6.582410528005136e-05, + "loss": 2.5847, + "theoretical_loss": 3.40037990048214, + "tokens_seen": 2224750592 + }, + { + "epoch": 0.35, + "learning_rate": 6.581608088589311e-05, + "loss": 2.6418, + "theoretical_loss": 3.4003635452699945, + "tokens_seen": 2224881664 + }, + { + "epoch": 0.35, + "learning_rate": 6.580805649173488e-05, + "loss": 2.5826, + "theoretical_loss": 3.400347191291104, + "tokens_seen": 2225012736 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1227393, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0688304901123047, + "objective/train/theoretical_loss": 3.4003390147640773, + "objective/train/tokens_used": 595537376, + "theoretical_loss": 3.4003390147640773, + "tokens_seen": 2225078272 + }, + { + "epoch": 0.35, + "learning_rate": 6.580003209757663e-05, + "loss": 2.7564, + "theoretical_loss": 3.4003308385453024, + "tokens_seen": 2225143808 + }, + { + "epoch": 0.35, + "learning_rate": 6.579200770341838e-05, + "loss": 2.5797, + "theoretical_loss": 3.400314487032424, + "tokens_seen": 2225274880 + }, + { + "epoch": 0.35, + "learning_rate": 6.578398330926015e-05, + "loss": 2.5858, + "theoretical_loss": 3.400298136752304, + "tokens_seen": 2225405952 + }, + { + "epoch": 0.35, + "learning_rate": 6.57759589151019e-05, + "loss": 2.5401, + "theoretical_loss": 3.400281787704776, + "tokens_seen": 2225537024 + }, + { + "epoch": 0.35, + "learning_rate": 6.576793452094367e-05, + "loss": 2.6559, + "theoretical_loss": 3.4002654398896754, + "tokens_seen": 2225668096 + }, + { + "epoch": 0.35, + "learning_rate": 6.575991012678542e-05, + "loss": 2.4982, + "theoretical_loss": 3.4002490933068357, + "tokens_seen": 2225799168 + }, + { + "epoch": 0.35, + "learning_rate": 6.575188573262719e-05, + "loss": 2.4992, + "theoretical_loss": 3.4002327479560925, + "tokens_seen": 2225930240 + }, + { + "epoch": 0.35, + "learning_rate": 6.574386133846894e-05, + "loss": 2.5146, + "theoretical_loss": 3.4002164038372795, + "tokens_seen": 2226061312 + }, + { + "epoch": 0.35, + "learning_rate": 6.57358369443107e-05, + "loss": 2.6021, + "theoretical_loss": 3.4002000609502314, + "tokens_seen": 2226192384 + }, + { + "epoch": 0.35, + "learning_rate": 6.572781255015246e-05, + "loss": 2.5102, + "theoretical_loss": 3.4001837192947835, + "tokens_seen": 2226323456 + }, + { + "epoch": 0.35, + "learning_rate": 6.571978815599422e-05, + "loss": 2.5191, + "theoretical_loss": 3.40016737887077, + "tokens_seen": 2226454528 + }, + { + "epoch": 0.35, + "learning_rate": 6.571176376183598e-05, + "loss": 2.5819, + "theoretical_loss": 3.4001510396780255, + "tokens_seen": 2226585600 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1228788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.50097393989563, + "objective/train/theoretical_loss": 3.400134701716385, + "objective/train/tokens_used": 597175776, + "theoretical_loss": 3.400134701716385, + "tokens_seen": 2226716672 + }, + { + "epoch": 0.35, + "learning_rate": 6.570373936767774e-05, + "loss": 2.5784, + "theoretical_loss": 3.400134701716385, + "tokens_seen": 2226716672 + }, + { + "epoch": 0.35, + "learning_rate": 6.56957149735195e-05, + "loss": 2.359, + "theoretical_loss": 3.400118364985684, + "tokens_seen": 2226847744 + }, + { + "epoch": 0.35, + "learning_rate": 6.568769057936126e-05, + "loss": 2.6825, + "theoretical_loss": 3.4001020294857556, + "tokens_seen": 2226978816 + }, + { + "epoch": 0.35, + "learning_rate": 6.567966618520301e-05, + "loss": 2.4631, + "theoretical_loss": 3.400085695216436, + "tokens_seen": 2227109888 + }, + { + "epoch": 0.35, + "learning_rate": 6.567164179104478e-05, + "loss": 2.4789, + "theoretical_loss": 3.40006936217756, + "tokens_seen": 2227240960 + }, + { + "epoch": 0.35, + "learning_rate": 6.566361739688653e-05, + "loss": 2.3606, + "theoretical_loss": 3.400053030368962, + "tokens_seen": 2227372032 + }, + { + "epoch": 0.35, + "learning_rate": 6.56555930027283e-05, + "loss": 2.4429, + "theoretical_loss": 3.400036699790477, + "tokens_seen": 2227503104 + }, + { + "epoch": 0.35, + "learning_rate": 6.564756860857005e-05, + "loss": 2.398, + "theoretical_loss": 3.4000203704419403, + "tokens_seen": 2227634176 + }, + { + "epoch": 0.35, + "learning_rate": 6.563954421441182e-05, + "loss": 2.6781, + "theoretical_loss": 3.4000040423231863, + "tokens_seen": 2227765248 + }, + { + "epoch": 0.35, + "learning_rate": 6.563151982025357e-05, + "loss": 2.4473, + "theoretical_loss": 3.399987715434051, + "tokens_seen": 2227896320 + }, + { + "epoch": 0.35, + "learning_rate": 6.562349542609534e-05, + "loss": 2.6737, + "theoretical_loss": 3.3999713897743686, + "tokens_seen": 2228027392 + }, + { + "epoch": 0.35, + "learning_rate": 6.561547103193709e-05, + "loss": 2.5061, + "theoretical_loss": 3.399955065343975, + "tokens_seen": 2228158464 + }, + { + "epoch": 0.35, + "learning_rate": 6.560744663777884e-05, + "loss": 2.609, + "theoretical_loss": 3.3999387421427043, + "tokens_seen": 2228289536 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1229475, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.80594801902771, + "objective/train/theoretical_loss": 3.3999305810029385, + "objective/train/tokens_used": 598814176, + "theoretical_loss": 3.3999305810029385, + "tokens_seen": 2228355072 + }, + { + "epoch": 0.35, + "learning_rate": 6.559942224362061e-05, + "loss": 2.5804, + "theoretical_loss": 3.3999224201703924, + "tokens_seen": 2228420608 + }, + { + "epoch": 0.35, + "learning_rate": 6.559139784946236e-05, + "loss": 2.6122, + "theoretical_loss": 3.3999060994268744, + "tokens_seen": 2228551680 + }, + { + "epoch": 0.35, + "learning_rate": 6.558337345530413e-05, + "loss": 2.3885, + "theoretical_loss": 3.399889779911985, + "tokens_seen": 2228682752 + }, + { + "epoch": 0.35, + "learning_rate": 6.557534906114588e-05, + "loss": 2.5099, + "theoretical_loss": 3.3998734616255604, + "tokens_seen": 2228813824 + }, + { + "epoch": 0.35, + "learning_rate": 6.556732466698765e-05, + "loss": 2.57, + "theoretical_loss": 3.3998571445674353, + "tokens_seen": 2228944896 + }, + { + "epoch": 0.35, + "learning_rate": 6.55593002728294e-05, + "loss": 2.554, + "theoretical_loss": 3.399840828737445, + "tokens_seen": 2229075968 + }, + { + "epoch": 0.35, + "learning_rate": 6.555127587867115e-05, + "loss": 2.4743, + "theoretical_loss": 3.3998245141354255, + "tokens_seen": 2229207040 + }, + { + "epoch": 0.35, + "learning_rate": 6.554325148451292e-05, + "loss": 2.4917, + "theoretical_loss": 3.3998082007612114, + "tokens_seen": 2229338112 + }, + { + "epoch": 0.35, + "learning_rate": 6.553522709035467e-05, + "loss": 2.6752, + "theoretical_loss": 3.3997918886146383, + "tokens_seen": 2229469184 + }, + { + "epoch": 0.35, + "learning_rate": 6.552720269619644e-05, + "loss": 2.4935, + "theoretical_loss": 3.399775577695541, + "tokens_seen": 2229600256 + }, + { + "epoch": 0.35, + "learning_rate": 6.55191783020382e-05, + "loss": 2.5293, + "theoretical_loss": 3.3997592680037565, + "tokens_seen": 2229731328 + }, + { + "epoch": 0.35, + "learning_rate": 6.551115390787996e-05, + "loss": 2.4314, + "theoretical_loss": 3.3997429595391195, + "tokens_seen": 2229862400 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1230697, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2199413776397705, + "objective/train/theoretical_loss": 3.3997266523014655, + "objective/train/tokens_used": 600452576, + "theoretical_loss": 3.3997266523014655, + "tokens_seen": 2229993472 + }, + { + "epoch": 0.35, + "learning_rate": 6.550312951372171e-05, + "loss": 2.3866, + "theoretical_loss": 3.3997266523014655, + "tokens_seen": 2229993472 + }, + { + "epoch": 0.35, + "learning_rate": 6.549510511956347e-05, + "loss": 2.6148, + "theoretical_loss": 3.39971034629063, + "tokens_seen": 2230124544 + }, + { + "epoch": 0.35, + "learning_rate": 6.548708072540523e-05, + "loss": 2.4489, + "theoretical_loss": 3.399694041506449, + "tokens_seen": 2230255616 + }, + { + "epoch": 0.35, + "learning_rate": 6.547905633124699e-05, + "loss": 2.6336, + "theoretical_loss": 3.3996777379487577, + "tokens_seen": 2230386688 + }, + { + "epoch": 0.35, + "learning_rate": 6.547103193708875e-05, + "loss": 2.5653, + "theoretical_loss": 3.3996614356173924, + "tokens_seen": 2230517760 + }, + { + "epoch": 0.35, + "learning_rate": 6.54630075429305e-05, + "loss": 2.5572, + "theoretical_loss": 3.3996451345121876, + "tokens_seen": 2230648832 + }, + { + "epoch": 0.35, + "learning_rate": 6.545498314877227e-05, + "loss": 2.4734, + "theoretical_loss": 3.3996288346329804, + "tokens_seen": 2230779904 + }, + { + "epoch": 0.35, + "learning_rate": 6.544695875461403e-05, + "loss": 2.5562, + "theoretical_loss": 3.399612535979606, + "tokens_seen": 2230910976 + }, + { + "epoch": 0.35, + "learning_rate": 6.543893436045578e-05, + "loss": 2.5883, + "theoretical_loss": 3.3995962385519, + "tokens_seen": 2231042048 + }, + { + "epoch": 0.35, + "learning_rate": 6.543090996629755e-05, + "loss": 2.6082, + "theoretical_loss": 3.399579942349698, + "tokens_seen": 2231173120 + }, + { + "epoch": 0.35, + "learning_rate": 6.54228855721393e-05, + "loss": 2.5796, + "theoretical_loss": 3.399563647372837, + "tokens_seen": 2231304192 + }, + { + "epoch": 0.35, + "learning_rate": 6.541486117798107e-05, + "loss": 2.5459, + "theoretical_loss": 3.3995473536211516, + "tokens_seen": 2231435264 + }, + { + "epoch": 0.35, + "learning_rate": 6.540683678382282e-05, + "loss": 2.6118, + "theoretical_loss": 3.3995310610944784, + "tokens_seen": 2231566336 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1231344, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.63580322265625, + "objective/train/theoretical_loss": 3.3995229152904702, + "objective/train/tokens_used": 602090976, + "theoretical_loss": 3.3995229152904702, + "tokens_seen": 2231631872 + }, + { + "epoch": 0.35, + "learning_rate": 6.539881238966459e-05, + "loss": 2.5398, + "theoretical_loss": 3.3995147697926535, + "tokens_seen": 2231697408 + }, + { + "epoch": 0.35, + "learning_rate": 6.539078799550634e-05, + "loss": 2.4789, + "theoretical_loss": 3.399498479715512, + "tokens_seen": 2231828480 + }, + { + "epoch": 0.35, + "learning_rate": 6.538276360134809e-05, + "loss": 2.7088, + "theoretical_loss": 3.399482190862891, + "tokens_seen": 2231959552 + }, + { + "epoch": 0.35, + "learning_rate": 6.537473920718986e-05, + "loss": 2.4904, + "theoretical_loss": 3.3994659032346264, + "tokens_seen": 2232090624 + }, + { + "epoch": 0.35, + "learning_rate": 6.536671481303161e-05, + "loss": 2.6163, + "theoretical_loss": 3.3994496168305535, + "tokens_seen": 2232221696 + }, + { + "epoch": 0.35, + "learning_rate": 6.535869041887338e-05, + "loss": 2.6065, + "theoretical_loss": 3.399433331650509, + "tokens_seen": 2232352768 + }, + { + "epoch": 0.35, + "learning_rate": 6.535066602471513e-05, + "loss": 2.6058, + "theoretical_loss": 3.399417047694329, + "tokens_seen": 2232483840 + }, + { + "epoch": 0.35, + "learning_rate": 6.53426416305569e-05, + "loss": 2.5078, + "theoretical_loss": 3.3994007649618494, + "tokens_seen": 2232614912 + }, + { + "epoch": 0.35, + "learning_rate": 6.533461723639865e-05, + "loss": 2.5577, + "theoretical_loss": 3.399384483452907, + "tokens_seen": 2232745984 + }, + { + "epoch": 0.35, + "learning_rate": 6.532659284224042e-05, + "loss": 2.5347, + "theoretical_loss": 3.3993682031673376, + "tokens_seen": 2232877056 + }, + { + "epoch": 0.35, + "learning_rate": 6.531856844808217e-05, + "loss": 2.6003, + "theoretical_loss": 3.399351924104977, + "tokens_seen": 2233008128 + }, + { + "epoch": 0.35, + "learning_rate": 6.531054405392392e-05, + "loss": 2.4763, + "theoretical_loss": 3.399335646265663, + "tokens_seen": 2233139200 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1232383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.546870708465576, + "objective/train/theoretical_loss": 3.3993193696492305, + "objective/train/tokens_used": 603729376, + "theoretical_loss": 3.3993193696492305, + "tokens_seen": 2233270272 + }, + { + "epoch": 0.35, + "learning_rate": 6.530251965976569e-05, + "loss": 2.5945, + "theoretical_loss": 3.3993193696492305, + "tokens_seen": 2233270272 + }, + { + "epoch": 0.35, + "learning_rate": 6.529449526560744e-05, + "loss": 2.5399, + "theoretical_loss": 3.399303094255516, + "tokens_seen": 2233401344 + }, + { + "epoch": 0.35, + "learning_rate": 6.528647087144921e-05, + "loss": 2.5345, + "theoretical_loss": 3.3992868200843565, + "tokens_seen": 2233532416 + }, + { + "epoch": 0.35, + "learning_rate": 6.527844647729096e-05, + "loss": 2.5707, + "theoretical_loss": 3.399270547135588, + "tokens_seen": 2233663488 + }, + { + "epoch": 0.35, + "learning_rate": 6.527042208313273e-05, + "loss": 2.4072, + "theoretical_loss": 3.3992542754090476, + "tokens_seen": 2233794560 + }, + { + "epoch": 0.35, + "learning_rate": 6.526239768897448e-05, + "loss": 2.6307, + "theoretical_loss": 3.399238004904571, + "tokens_seen": 2233925632 + }, + { + "epoch": 0.35, + "learning_rate": 6.525437329481624e-05, + "loss": 2.5343, + "theoretical_loss": 3.399221735621995, + "tokens_seen": 2234056704 + }, + { + "epoch": 0.35, + "learning_rate": 6.5246348900658e-05, + "loss": 2.5621, + "theoretical_loss": 3.3992054675611563, + "tokens_seen": 2234187776 + }, + { + "epoch": 0.35, + "learning_rate": 6.523832450649976e-05, + "loss": 2.5342, + "theoretical_loss": 3.3991892007218913, + "tokens_seen": 2234318848 + }, + { + "epoch": 0.35, + "learning_rate": 6.523030011234152e-05, + "loss": 2.6233, + "theoretical_loss": 3.399172935104037, + "tokens_seen": 2234449920 + }, + { + "epoch": 0.35, + "learning_rate": 6.522227571818328e-05, + "loss": 2.464, + "theoretical_loss": 3.3991566707074297, + "tokens_seen": 2234580992 + }, + { + "epoch": 0.35, + "learning_rate": 6.521425132402504e-05, + "loss": 2.5425, + "theoretical_loss": 3.399140407531906, + "tokens_seen": 2234712064 + }, + { + "epoch": 0.35, + "learning_rate": 6.52062269298668e-05, + "loss": 2.6549, + "theoretical_loss": 3.3991241455773027, + "tokens_seen": 2234843136 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1233495, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.886728525161743, + "objective/train/theoretical_loss": 3.3991160150577953, + "objective/train/tokens_used": 605367776, + "theoretical_loss": 3.3991160150577953, + "tokens_seen": 2234908672 + }, + { + "epoch": 0.35, + "learning_rate": 6.519820253570855e-05, + "loss": 2.5483, + "theoretical_loss": 3.3991078848434566, + "tokens_seen": 2234974208 + }, + { + "epoch": 0.35, + "learning_rate": 6.519017814155032e-05, + "loss": 2.5349, + "theoretical_loss": 3.399091625330205, + "tokens_seen": 2235105280 + }, + { + "epoch": 0.35, + "learning_rate": 6.518215374739207e-05, + "loss": 2.5666, + "theoretical_loss": 3.3990753670373834, + "tokens_seen": 2235236352 + }, + { + "epoch": 0.35, + "learning_rate": 6.517412935323384e-05, + "loss": 2.554, + "theoretical_loss": 3.3990591099648295, + "tokens_seen": 2235367424 + }, + { + "epoch": 0.35, + "learning_rate": 6.516610495907559e-05, + "loss": 2.4128, + "theoretical_loss": 3.3990428541123805, + "tokens_seen": 2235498496 + }, + { + "epoch": 0.35, + "learning_rate": 6.515808056491736e-05, + "loss": 2.6454, + "theoretical_loss": 3.399026599479873, + "tokens_seen": 2235629568 + }, + { + "epoch": 0.36, + "learning_rate": 6.515005617075911e-05, + "loss": 2.5925, + "theoretical_loss": 3.3990103460671435, + "tokens_seen": 2235760640 + }, + { + "epoch": 0.36, + "learning_rate": 6.514203177660086e-05, + "loss": 2.4704, + "theoretical_loss": 3.398994093874029, + "tokens_seen": 2235891712 + }, + { + "epoch": 0.36, + "learning_rate": 6.513400738244263e-05, + "loss": 2.4929, + "theoretical_loss": 3.398977842900367, + "tokens_seen": 2236022784 + }, + { + "epoch": 0.36, + "learning_rate": 6.512598298828438e-05, + "loss": 2.7658, + "theoretical_loss": 3.3989615931459944, + "tokens_seen": 2236153856 + }, + { + "epoch": 0.36, + "learning_rate": 6.511795859412615e-05, + "loss": 2.6872, + "theoretical_loss": 3.3989453446107483, + "tokens_seen": 2236284928 + }, + { + "epoch": 0.36, + "learning_rate": 6.51099341999679e-05, + "loss": 2.4934, + "theoretical_loss": 3.3989290972944657, + "tokens_seen": 2236416000 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1234013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2317135334014893, + "objective/train/theoretical_loss": 3.3989128511969833, + "objective/train/tokens_used": 607006176, + "theoretical_loss": 3.3989128511969833, + "tokens_seen": 2236547072 + }, + { + "epoch": 0.36, + "learning_rate": 6.510190980580967e-05, + "loss": 2.5527, + "theoretical_loss": 3.3989128511969833, + "tokens_seen": 2236547072 + }, + { + "epoch": 0.36, + "learning_rate": 6.509388541165142e-05, + "loss": 2.527, + "theoretical_loss": 3.398896606318139, + "tokens_seen": 2236678144 + }, + { + "epoch": 0.36, + "learning_rate": 6.508586101749317e-05, + "loss": 2.5679, + "theoretical_loss": 3.3988803626577697, + "tokens_seen": 2236809216 + }, + { + "epoch": 0.36, + "learning_rate": 6.507783662333494e-05, + "loss": 2.615, + "theoretical_loss": 3.398864120215712, + "tokens_seen": 2236940288 + }, + { + "epoch": 0.36, + "learning_rate": 6.50698122291767e-05, + "loss": 2.3885, + "theoretical_loss": 3.398847878991804, + "tokens_seen": 2237071360 + }, + { + "epoch": 0.36, + "learning_rate": 6.506178783501846e-05, + "loss": 2.5451, + "theoretical_loss": 3.398831638985883, + "tokens_seen": 2237202432 + }, + { + "epoch": 0.36, + "learning_rate": 6.505376344086021e-05, + "loss": 2.547, + "theoretical_loss": 3.398815400197785, + "tokens_seen": 2237333504 + }, + { + "epoch": 0.36, + "learning_rate": 6.504573904670198e-05, + "loss": 2.504, + "theoretical_loss": 3.3987991626273493, + "tokens_seen": 2237464576 + }, + { + "epoch": 0.36, + "learning_rate": 6.503771465254373e-05, + "loss": 2.6316, + "theoretical_loss": 3.3987829262744116, + "tokens_seen": 2237595648 + }, + { + "epoch": 0.36, + "learning_rate": 6.50296902583855e-05, + "loss": 2.5986, + "theoretical_loss": 3.3987666911388104, + "tokens_seen": 2237726720 + }, + { + "epoch": 0.36, + "learning_rate": 6.502166586422725e-05, + "loss": 2.3704, + "theoretical_loss": 3.398750457220382, + "tokens_seen": 2237857792 + }, + { + "epoch": 0.36, + "learning_rate": 6.501364147006901e-05, + "loss": 2.733, + "theoretical_loss": 3.3987342245189653, + "tokens_seen": 2237988864 + }, + { + "epoch": 0.36, + "learning_rate": 6.500561707591077e-05, + "loss": 2.6359, + "theoretical_loss": 3.398717993034397, + "tokens_seen": 2238119936 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1235138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.893874406814575, + "objective/train/theoretical_loss": 3.39870987774838, + "objective/train/tokens_used": 608644576, + "theoretical_loss": 3.39870987774838, + "tokens_seen": 2238185472 + }, + { + "epoch": 0.36, + "learning_rate": 6.499759268175253e-05, + "loss": 2.5401, + "theoretical_loss": 3.3987017627665144, + "tokens_seen": 2238251008 + }, + { + "epoch": 0.36, + "learning_rate": 6.49895682875943e-05, + "loss": 2.3885, + "theoretical_loss": 3.398685533715155, + "tokens_seen": 2238382080 + }, + { + "epoch": 0.36, + "learning_rate": 6.498154389343605e-05, + "loss": 2.6437, + "theoretical_loss": 3.3986693058801576, + "tokens_seen": 2238513152 + }, + { + "epoch": 0.36, + "learning_rate": 6.497351949927781e-05, + "loss": 2.5716, + "theoretical_loss": 3.3986530792613583, + "tokens_seen": 2238644224 + }, + { + "epoch": 0.36, + "learning_rate": 6.496549510511957e-05, + "loss": 2.6233, + "theoretical_loss": 3.3986368538585956, + "tokens_seen": 2238775296 + }, + { + "epoch": 0.36, + "learning_rate": 6.495747071096132e-05, + "loss": 2.4871, + "theoretical_loss": 3.398620629671707, + "tokens_seen": 2238906368 + }, + { + "epoch": 0.36, + "learning_rate": 6.494944631680309e-05, + "loss": 2.4061, + "theoretical_loss": 3.3986044067005303, + "tokens_seen": 2239037440 + }, + { + "epoch": 0.36, + "learning_rate": 6.494142192264484e-05, + "loss": 2.4802, + "theoretical_loss": 3.398588184944903, + "tokens_seen": 2239168512 + }, + { + "epoch": 0.36, + "learning_rate": 6.49333975284866e-05, + "loss": 2.4283, + "theoretical_loss": 3.3985719644046632, + "tokens_seen": 2239299584 + }, + { + "epoch": 0.36, + "learning_rate": 6.492537313432836e-05, + "loss": 2.5663, + "theoretical_loss": 3.3985557450796486, + "tokens_seen": 2239430656 + }, + { + "epoch": 0.36, + "learning_rate": 6.491734874017013e-05, + "loss": 2.4968, + "theoretical_loss": 3.3985395269696967, + "tokens_seen": 2239561728 + }, + { + "epoch": 0.36, + "learning_rate": 6.490932434601188e-05, + "loss": 2.6019, + "theoretical_loss": 3.398523310074646, + "tokens_seen": 2239692800 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1235870, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.631380558013916, + "objective/train/theoretical_loss": 3.3985070943943336, + "objective/train/tokens_used": 610282976, + "theoretical_loss": 3.3985070943943336, + "tokens_seen": 2239823872 + }, + { + "epoch": 0.36, + "learning_rate": 6.490129995185363e-05, + "loss": 2.4624, + "theoretical_loss": 3.3985070943943336, + "tokens_seen": 2239823872 + }, + { + "epoch": 0.36, + "learning_rate": 6.48932755576954e-05, + "loss": 2.6135, + "theoretical_loss": 3.398490879928598, + "tokens_seen": 2239954944 + }, + { + "epoch": 0.36, + "learning_rate": 6.488525116353715e-05, + "loss": 2.3966, + "theoretical_loss": 3.398474666677277, + "tokens_seen": 2240086016 + }, + { + "epoch": 0.36, + "learning_rate": 6.487722676937892e-05, + "loss": 2.5696, + "theoretical_loss": 3.398458454640209, + "tokens_seen": 2240217088 + }, + { + "epoch": 0.36, + "learning_rate": 6.486920237522067e-05, + "loss": 2.5395, + "theoretical_loss": 3.3984422438172315, + "tokens_seen": 2240348160 + }, + { + "epoch": 0.36, + "learning_rate": 6.486117798106244e-05, + "loss": 2.5404, + "theoretical_loss": 3.3984260342081822, + "tokens_seen": 2240479232 + }, + { + "epoch": 0.36, + "learning_rate": 6.485315358690419e-05, + "loss": 2.5419, + "theoretical_loss": 3.3984098258129003, + "tokens_seen": 2240610304 + }, + { + "epoch": 0.36, + "learning_rate": 6.484512919274594e-05, + "loss": 2.4847, + "theoretical_loss": 3.398393618631223, + "tokens_seen": 2240741376 + }, + { + "epoch": 0.36, + "learning_rate": 6.483710479858771e-05, + "loss": 2.6941, + "theoretical_loss": 3.398377412662989, + "tokens_seen": 2240872448 + }, + { + "epoch": 0.36, + "learning_rate": 6.482908040442946e-05, + "loss": 2.4717, + "theoretical_loss": 3.398361207908036, + "tokens_seen": 2241003520 + }, + { + "epoch": 0.36, + "learning_rate": 6.482105601027123e-05, + "loss": 2.543, + "theoretical_loss": 3.3983450043662025, + "tokens_seen": 2241134592 + }, + { + "epoch": 0.36, + "learning_rate": 6.481303161611298e-05, + "loss": 2.4651, + "theoretical_loss": 3.398328802037327, + "tokens_seen": 2241265664 + }, + { + "epoch": 0.36, + "learning_rate": 6.480500722195475e-05, + "loss": 2.4543, + "theoretical_loss": 3.398312600921247, + "tokens_seen": 2241396736 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1236973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588179588317871, + "objective/train/theoretical_loss": 3.3983045008179555, + "objective/train/tokens_used": 611921376, + "theoretical_loss": 3.3983045008179555, + "tokens_seen": 2241462272 + }, + { + "epoch": 0.36, + "learning_rate": 6.47969828277965e-05, + "loss": 2.3791, + "theoretical_loss": 3.3982964010178014, + "tokens_seen": 2241527808 + }, + { + "epoch": 0.36, + "learning_rate": 6.478895843363826e-05, + "loss": 2.6522, + "theoretical_loss": 3.398280202326829, + "tokens_seen": 2241658880 + }, + { + "epoch": 0.36, + "learning_rate": 6.478093403948002e-05, + "loss": 2.4594, + "theoretical_loss": 3.398264004848167, + "tokens_seen": 2241789952 + }, + { + "epoch": 0.36, + "learning_rate": 6.477290964532178e-05, + "loss": 2.5694, + "theoretical_loss": 3.3982478085816545, + "tokens_seen": 2241921024 + }, + { + "epoch": 0.36, + "learning_rate": 6.476488525116354e-05, + "loss": 2.6404, + "theoretical_loss": 3.3982316135271295, + "tokens_seen": 2242052096 + }, + { + "epoch": 0.36, + "learning_rate": 6.47568608570053e-05, + "loss": 2.548, + "theoretical_loss": 3.3982154196844316, + "tokens_seen": 2242183168 + }, + { + "epoch": 0.36, + "learning_rate": 6.474883646284706e-05, + "loss": 2.5171, + "theoretical_loss": 3.398199227053398, + "tokens_seen": 2242314240 + }, + { + "epoch": 0.36, + "learning_rate": 6.474081206868882e-05, + "loss": 2.5647, + "theoretical_loss": 3.3981830356338674, + "tokens_seen": 2242445312 + }, + { + "epoch": 0.36, + "learning_rate": 6.473278767453057e-05, + "loss": 2.5136, + "theoretical_loss": 3.3981668454256786, + "tokens_seen": 2242576384 + }, + { + "epoch": 0.36, + "learning_rate": 6.472476328037234e-05, + "loss": 2.4726, + "theoretical_loss": 3.3981506564286708, + "tokens_seen": 2242707456 + }, + { + "epoch": 0.36, + "learning_rate": 6.471673888621409e-05, + "loss": 2.5908, + "theoretical_loss": 3.3981344686426818, + "tokens_seen": 2242838528 + }, + { + "epoch": 0.36, + "learning_rate": 6.470871449205586e-05, + "loss": 2.4583, + "theoretical_loss": 3.3981182820675504, + "tokens_seen": 2242969600 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1238252, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0139503479003906, + "objective/train/theoretical_loss": 3.3981020967031155, + "objective/train/tokens_used": 613559776, + "theoretical_loss": 3.3981020967031155, + "tokens_seen": 2243100672 + }, + { + "epoch": 0.36, + "learning_rate": 6.470069009789761e-05, + "loss": 2.596, + "theoretical_loss": 3.3981020967031155, + "tokens_seen": 2243100672 + }, + { + "epoch": 0.36, + "learning_rate": 6.469266570373938e-05, + "loss": 2.4517, + "theoretical_loss": 3.3980859125492153, + "tokens_seen": 2243231744 + }, + { + "epoch": 0.36, + "learning_rate": 6.468464130958113e-05, + "loss": 2.5418, + "theoretical_loss": 3.398069729605689, + "tokens_seen": 2243362816 + }, + { + "epoch": 0.36, + "learning_rate": 6.46766169154229e-05, + "loss": 2.5578, + "theoretical_loss": 3.3980535478723755, + "tokens_seen": 2243493888 + }, + { + "epoch": 0.36, + "learning_rate": 6.466859252126465e-05, + "loss": 2.5241, + "theoretical_loss": 3.3980373673491133, + "tokens_seen": 2243624960 + }, + { + "epoch": 0.36, + "learning_rate": 6.46605681271064e-05, + "loss": 2.5103, + "theoretical_loss": 3.398021188035741, + "tokens_seen": 2243756032 + }, + { + "epoch": 0.36, + "learning_rate": 6.465254373294817e-05, + "loss": 2.6019, + "theoretical_loss": 3.398005009932098, + "tokens_seen": 2243887104 + }, + { + "epoch": 0.36, + "learning_rate": 6.464451933878992e-05, + "loss": 2.4528, + "theoretical_loss": 3.3979888330380232, + "tokens_seen": 2244018176 + }, + { + "epoch": 0.36, + "learning_rate": 6.463649494463169e-05, + "loss": 2.4556, + "theoretical_loss": 3.3979726573533546, + "tokens_seen": 2244149248 + }, + { + "epoch": 0.36, + "learning_rate": 6.462847055047344e-05, + "loss": 2.6524, + "theoretical_loss": 3.3979564828779325, + "tokens_seen": 2244280320 + }, + { + "epoch": 0.36, + "learning_rate": 6.462044615631521e-05, + "loss": 2.6528, + "theoretical_loss": 3.3979403096115948, + "tokens_seen": 2244411392 + }, + { + "epoch": 0.36, + "learning_rate": 6.461242176215696e-05, + "loss": 2.5461, + "theoretical_loss": 3.397924137554181, + "tokens_seen": 2244542464 + }, + { + "epoch": 0.36, + "learning_rate": 6.460439736799871e-05, + "loss": 2.6122, + "theoretical_loss": 3.3979079667055303, + "tokens_seen": 2244673536 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1238737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.071171522140503, + "objective/train/theoretical_loss": 3.397899881734441, + "objective/train/tokens_used": 615198176, + "theoretical_loss": 3.397899881734441, + "tokens_seen": 2244739072 + }, + { + "epoch": 0.36, + "learning_rate": 6.459637297384048e-05, + "loss": 2.5194, + "theoretical_loss": 3.3978917970654816, + "tokens_seen": 2244804608 + }, + { + "epoch": 0.36, + "learning_rate": 6.458834857968223e-05, + "loss": 2.5799, + "theoretical_loss": 3.397875628633874, + "tokens_seen": 2244935680 + }, + { + "epoch": 0.36, + "learning_rate": 6.4580324185524e-05, + "loss": 2.5241, + "theoretical_loss": 3.397859461410546, + "tokens_seen": 2245066752 + }, + { + "epoch": 0.36, + "learning_rate": 6.457229979136575e-05, + "loss": 2.4155, + "theoretical_loss": 3.397843295395338, + "tokens_seen": 2245197824 + }, + { + "epoch": 0.36, + "learning_rate": 6.456427539720752e-05, + "loss": 2.5593, + "theoretical_loss": 3.3978271305880883, + "tokens_seen": 2245328896 + }, + { + "epoch": 0.36, + "learning_rate": 6.455625100304927e-05, + "loss": 2.3937, + "theoretical_loss": 3.3978109669886365, + "tokens_seen": 2245459968 + }, + { + "epoch": 0.36, + "learning_rate": 6.454822660889103e-05, + "loss": 2.569, + "theoretical_loss": 3.3977948045968214, + "tokens_seen": 2245591040 + }, + { + "epoch": 0.36, + "learning_rate": 6.45402022147328e-05, + "loss": 2.6149, + "theoretical_loss": 3.397778643412483, + "tokens_seen": 2245722112 + }, + { + "epoch": 0.36, + "learning_rate": 6.453217782057455e-05, + "loss": 2.669, + "theoretical_loss": 3.3977624834354603, + "tokens_seen": 2245853184 + }, + { + "epoch": 0.36, + "learning_rate": 6.452415342641631e-05, + "loss": 2.5393, + "theoretical_loss": 3.397746324665593, + "tokens_seen": 2245984256 + }, + { + "epoch": 0.36, + "learning_rate": 6.451612903225807e-05, + "loss": 2.5196, + "theoretical_loss": 3.3977301671027194, + "tokens_seen": 2246115328 + }, + { + "epoch": 0.36, + "learning_rate": 6.450810463809983e-05, + "loss": 2.517, + "theoretical_loss": 3.39771401074668, + "tokens_seen": 2246246400 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1239806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.199644088745117, + "objective/train/theoretical_loss": 3.397697855597314, + "objective/train/tokens_used": 616836576, + "theoretical_loss": 3.397697855597314, + "tokens_seen": 2246377472 + }, + { + "epoch": 0.36, + "learning_rate": 6.450008024394159e-05, + "loss": 2.5675, + "theoretical_loss": 3.397697855597314, + "tokens_seen": 2246377472 + }, + { + "epoch": 0.36, + "learning_rate": 6.449205584978334e-05, + "loss": 2.5408, + "theoretical_loss": 3.3976817016544603, + "tokens_seen": 2246508544 + }, + { + "epoch": 0.36, + "learning_rate": 6.448403145562511e-05, + "loss": 2.555, + "theoretical_loss": 3.3976655489179595, + "tokens_seen": 2246639616 + }, + { + "epoch": 0.36, + "learning_rate": 6.447600706146686e-05, + "loss": 2.4769, + "theoretical_loss": 3.3976493973876503, + "tokens_seen": 2246770688 + }, + { + "epoch": 0.36, + "learning_rate": 6.446798266730863e-05, + "loss": 2.5246, + "theoretical_loss": 3.397633247063372, + "tokens_seen": 2246901760 + }, + { + "epoch": 0.36, + "learning_rate": 6.445995827315038e-05, + "loss": 2.4229, + "theoretical_loss": 3.3976170979449654, + "tokens_seen": 2247032832 + }, + { + "epoch": 0.36, + "learning_rate": 6.445193387899215e-05, + "loss": 2.4574, + "theoretical_loss": 3.397600950032269, + "tokens_seen": 2247163904 + }, + { + "epoch": 0.36, + "learning_rate": 6.44439094848339e-05, + "loss": 2.6299, + "theoretical_loss": 3.3975848033251226, + "tokens_seen": 2247294976 + }, + { + "epoch": 0.36, + "learning_rate": 6.443588509067565e-05, + "loss": 2.6822, + "theoretical_loss": 3.3975686578233666, + "tokens_seen": 2247426048 + }, + { + "epoch": 0.36, + "learning_rate": 6.442786069651742e-05, + "loss": 2.513, + "theoretical_loss": 3.3975525135268403, + "tokens_seen": 2247557120 + }, + { + "epoch": 0.36, + "learning_rate": 6.441983630235917e-05, + "loss": 2.6912, + "theoretical_loss": 3.3975363704353834, + "tokens_seen": 2247688192 + }, + { + "epoch": 0.36, + "learning_rate": 6.441181190820094e-05, + "loss": 2.4947, + "theoretical_loss": 3.397520228548836, + "tokens_seen": 2247819264 + }, + { + "epoch": 0.36, + "learning_rate": 6.440378751404269e-05, + "loss": 2.5929, + "theoretical_loss": 3.397504087867037, + "tokens_seen": 2247950336 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1241075, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.425755739212036, + "objective/train/theoretical_loss": 3.3974960179778684, + "objective/train/tokens_used": 618474976, + "theoretical_loss": 3.3974960179778684, + "tokens_seen": 2248015872 + }, + { + "epoch": 0.36, + "learning_rate": 6.439576311988446e-05, + "loss": 2.5835, + "theoretical_loss": 3.397487948389827, + "tokens_seen": 2248081408 + }, + { + "epoch": 0.36, + "learning_rate": 6.438773872572621e-05, + "loss": 2.6561, + "theoretical_loss": 3.3974718101170462, + "tokens_seen": 2248212480 + }, + { + "epoch": 0.36, + "learning_rate": 6.437971433156798e-05, + "loss": 2.6205, + "theoretical_loss": 3.397455673048534, + "tokens_seen": 2248343552 + }, + { + "epoch": 0.36, + "learning_rate": 6.437168993740973e-05, + "loss": 2.5423, + "theoretical_loss": 3.3974395371841304, + "tokens_seen": 2248474624 + }, + { + "epoch": 0.36, + "learning_rate": 6.436366554325149e-05, + "loss": 2.643, + "theoretical_loss": 3.397423402523675, + "tokens_seen": 2248605696 + }, + { + "epoch": 0.36, + "learning_rate": 6.435564114909325e-05, + "loss": 2.5475, + "theoretical_loss": 3.3974072690670085, + "tokens_seen": 2248736768 + }, + { + "epoch": 0.36, + "learning_rate": 6.4347616754935e-05, + "loss": 2.5904, + "theoretical_loss": 3.3973911368139706, + "tokens_seen": 2248867840 + }, + { + "epoch": 0.36, + "learning_rate": 6.433959236077677e-05, + "loss": 2.4771, + "theoretical_loss": 3.397375005764401, + "tokens_seen": 2248998912 + }, + { + "epoch": 0.36, + "learning_rate": 6.433156796661852e-05, + "loss": 2.6246, + "theoretical_loss": 3.397358875918141, + "tokens_seen": 2249129984 + }, + { + "epoch": 0.36, + "learning_rate": 6.432354357246029e-05, + "loss": 2.6396, + "theoretical_loss": 3.397342747275029, + "tokens_seen": 2249261056 + }, + { + "epoch": 0.36, + "learning_rate": 6.431551917830204e-05, + "loss": 2.6124, + "theoretical_loss": 3.3973266198349066, + "tokens_seen": 2249392128 + }, + { + "epoch": 0.36, + "learning_rate": 6.43074947841438e-05, + "loss": 2.425, + "theoretical_loss": 3.397310493597613, + "tokens_seen": 2249523200 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1241620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.306302547454834, + "objective/train/theoretical_loss": 3.397294368562989, + "objective/train/tokens_used": 620113376, + "theoretical_loss": 3.397294368562989, + "tokens_seen": 2249654272 + }, + { + "epoch": 0.36, + "learning_rate": 6.429947038998556e-05, + "loss": 2.5863, + "theoretical_loss": 3.397294368562989, + "tokens_seen": 2249654272 + }, + { + "epoch": 0.36, + "learning_rate": 6.429144599582732e-05, + "loss": 2.6603, + "theoretical_loss": 3.3972782447308747, + "tokens_seen": 2249785344 + }, + { + "epoch": 0.36, + "learning_rate": 6.428342160166908e-05, + "loss": 2.6064, + "theoretical_loss": 3.3972621221011106, + "tokens_seen": 2249916416 + }, + { + "epoch": 0.36, + "learning_rate": 6.427539720751084e-05, + "loss": 2.5384, + "theoretical_loss": 3.3972460006735363, + "tokens_seen": 2250047488 + }, + { + "epoch": 0.36, + "learning_rate": 6.42673728133526e-05, + "loss": 2.6714, + "theoretical_loss": 3.397229880447993, + "tokens_seen": 2250178560 + }, + { + "epoch": 0.36, + "learning_rate": 6.425934841919436e-05, + "loss": 2.4576, + "theoretical_loss": 3.3972137614243203, + "tokens_seen": 2250309632 + }, + { + "epoch": 0.36, + "learning_rate": 6.425132402503611e-05, + "loss": 2.6373, + "theoretical_loss": 3.397197643602359, + "tokens_seen": 2250440704 + }, + { + "epoch": 0.36, + "learning_rate": 6.424329963087788e-05, + "loss": 2.4951, + "theoretical_loss": 3.3971815269819494, + "tokens_seen": 2250571776 + }, + { + "epoch": 0.36, + "learning_rate": 6.423527523671963e-05, + "loss": 2.4305, + "theoretical_loss": 3.3971654115629324, + "tokens_seen": 2250702848 + }, + { + "epoch": 0.36, + "learning_rate": 6.42272508425614e-05, + "loss": 2.6251, + "theoretical_loss": 3.397149297345148, + "tokens_seen": 2250833920 + }, + { + "epoch": 0.36, + "learning_rate": 6.421922644840315e-05, + "loss": 2.6587, + "theoretical_loss": 3.3971331843284367, + "tokens_seen": 2250964992 + }, + { + "epoch": 0.36, + "learning_rate": 6.421120205424492e-05, + "loss": 2.6134, + "theoretical_loss": 3.397117072512639, + "tokens_seen": 2251096064 + }, + { + "epoch": 0.36, + "learning_rate": 6.420317766008667e-05, + "loss": 2.5786, + "theoretical_loss": 3.397100961897596, + "tokens_seen": 2251227136 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1242746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.16178297996521, + "objective/train/theoretical_loss": 3.3970929070403075, + "objective/train/tokens_used": 621751776, + "theoretical_loss": 3.3970929070403075, + "tokens_seen": 2251292672 + }, + { + "epoch": 0.36, + "learning_rate": 6.419515326592842e-05, + "loss": 2.4616, + "theoretical_loss": 3.3970848524831476, + "tokens_seen": 2251358208 + }, + { + "epoch": 0.36, + "learning_rate": 6.418712887177019e-05, + "loss": 2.5961, + "theoretical_loss": 3.397068744269135, + "tokens_seen": 2251489280 + }, + { + "epoch": 0.36, + "learning_rate": 6.417910447761194e-05, + "loss": 2.5784, + "theoretical_loss": 3.397052637255399, + "tokens_seen": 2251620352 + }, + { + "epoch": 0.36, + "learning_rate": 6.417108008345371e-05, + "loss": 2.4737, + "theoretical_loss": 3.3970365314417794, + "tokens_seen": 2251751424 + }, + { + "epoch": 0.36, + "learning_rate": 6.416305568929546e-05, + "loss": 2.4705, + "theoretical_loss": 3.397020426828118, + "tokens_seen": 2251882496 + }, + { + "epoch": 0.36, + "learning_rate": 6.415503129513723e-05, + "loss": 2.4693, + "theoretical_loss": 3.397004323414255, + "tokens_seen": 2252013568 + }, + { + "epoch": 0.36, + "learning_rate": 6.414700690097898e-05, + "loss": 2.5308, + "theoretical_loss": 3.396988221200031, + "tokens_seen": 2252144640 + }, + { + "epoch": 0.37, + "learning_rate": 6.413898250682074e-05, + "loss": 2.5019, + "theoretical_loss": 3.3969721201852874, + "tokens_seen": 2252275712 + }, + { + "epoch": 0.37, + "learning_rate": 6.41309581126625e-05, + "loss": 2.6541, + "theoretical_loss": 3.3969560203698643, + "tokens_seen": 2252406784 + }, + { + "epoch": 0.37, + "learning_rate": 6.412293371850426e-05, + "loss": 2.644, + "theoretical_loss": 3.3969399217536034, + "tokens_seen": 2252537856 + }, + { + "epoch": 0.37, + "learning_rate": 6.411490932434602e-05, + "loss": 2.3905, + "theoretical_loss": 3.396923824336345, + "tokens_seen": 2252668928 + }, + { + "epoch": 0.37, + "learning_rate": 6.410688493018777e-05, + "loss": 2.7238, + "theoretical_loss": 3.39690772811793, + "tokens_seen": 2252800000 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1243228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.317286491394043, + "objective/train/theoretical_loss": 3.3968916330982006, + "objective/train/tokens_used": 623390176, + "theoretical_loss": 3.3968916330982006, + "tokens_seen": 2252931072 + }, + { + "epoch": 0.37, + "learning_rate": 6.409886053602954e-05, + "loss": 2.6101, + "theoretical_loss": 3.3968916330982006, + "tokens_seen": 2252931072 + }, + { + "epoch": 0.37, + "learning_rate": 6.40908361418713e-05, + "loss": 2.6364, + "theoretical_loss": 3.396875539276996, + "tokens_seen": 2253062144 + }, + { + "epoch": 0.37, + "learning_rate": 6.408281174771305e-05, + "loss": 2.591, + "theoretical_loss": 3.3968594466541586, + "tokens_seen": 2253193216 + }, + { + "epoch": 0.37, + "learning_rate": 6.407478735355481e-05, + "loss": 2.5017, + "theoretical_loss": 3.3968433552295285, + "tokens_seen": 2253324288 + }, + { + "epoch": 0.37, + "learning_rate": 6.406676295939657e-05, + "loss": 2.5772, + "theoretical_loss": 3.3968272650029476, + "tokens_seen": 2253455360 + }, + { + "epoch": 0.37, + "learning_rate": 6.405873856523833e-05, + "loss": 2.5291, + "theoretical_loss": 3.3968111759742565, + "tokens_seen": 2253586432 + }, + { + "epoch": 0.37, + "learning_rate": 6.405071417108009e-05, + "loss": 2.4687, + "theoretical_loss": 3.3967950881432967, + "tokens_seen": 2253717504 + }, + { + "epoch": 0.37, + "learning_rate": 6.404268977692185e-05, + "loss": 2.6126, + "theoretical_loss": 3.396779001509909, + "tokens_seen": 2253848576 + }, + { + "epoch": 0.37, + "learning_rate": 6.403466538276361e-05, + "loss": 2.5522, + "theoretical_loss": 3.3967629160739348, + "tokens_seen": 2253979648 + }, + { + "epoch": 0.37, + "learning_rate": 6.402664098860537e-05, + "loss": 2.7067, + "theoretical_loss": 3.3967468318352156, + "tokens_seen": 2254110720 + }, + { + "epoch": 0.37, + "learning_rate": 6.401861659444713e-05, + "loss": 2.6029, + "theoretical_loss": 3.3967307487935923, + "tokens_seen": 2254241792 + }, + { + "epoch": 0.37, + "learning_rate": 6.401059220028888e-05, + "loss": 2.5112, + "theoretical_loss": 3.3967146669489066, + "tokens_seen": 2254372864 + }, + { + "epoch": 0.37, + "learning_rate": 6.400256780613065e-05, + "loss": 2.4022, + "theoretical_loss": 3.3966985863009995, + "tokens_seen": 2254503936 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1244407, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6968259811401367, + "objective/train/theoretical_loss": 3.396690546425788, + "objective/train/tokens_used": 625028576, + "theoretical_loss": 3.396690546425788, + "tokens_seen": 2254569472 + }, + { + "epoch": 0.37, + "learning_rate": 6.39945434119724e-05, + "loss": 2.5944, + "theoretical_loss": 3.396682506849712, + "tokens_seen": 2254635008 + }, + { + "epoch": 0.37, + "learning_rate": 6.398651901781417e-05, + "loss": 2.4521, + "theoretical_loss": 3.3966664285948864, + "tokens_seen": 2254766080 + }, + { + "epoch": 0.37, + "learning_rate": 6.397849462365592e-05, + "loss": 2.621, + "theoretical_loss": 3.3966503515363637, + "tokens_seen": 2254897152 + }, + { + "epoch": 0.37, + "learning_rate": 6.397047022949769e-05, + "loss": 2.6413, + "theoretical_loss": 3.3966342756739856, + "tokens_seen": 2255028224 + }, + { + "epoch": 0.37, + "learning_rate": 6.396244583533944e-05, + "loss": 2.6067, + "theoretical_loss": 3.396618201007593, + "tokens_seen": 2255159296 + }, + { + "epoch": 0.37, + "learning_rate": 6.395442144118119e-05, + "loss": 2.583, + "theoretical_loss": 3.3966021275370277, + "tokens_seen": 2255290368 + }, + { + "epoch": 0.37, + "learning_rate": 6.394639704702296e-05, + "loss": 2.5455, + "theoretical_loss": 3.3965860552621314, + "tokens_seen": 2255421440 + }, + { + "epoch": 0.37, + "learning_rate": 6.393837265286471e-05, + "loss": 2.5062, + "theoretical_loss": 3.3965699841827455, + "tokens_seen": 2255552512 + }, + { + "epoch": 0.37, + "learning_rate": 6.393034825870648e-05, + "loss": 2.5036, + "theoretical_loss": 3.3965539142987122, + "tokens_seen": 2255683584 + }, + { + "epoch": 0.37, + "learning_rate": 6.392232386454823e-05, + "loss": 2.5674, + "theoretical_loss": 3.3965378456098723, + "tokens_seen": 2255814656 + }, + { + "epoch": 0.37, + "learning_rate": 6.391429947039e-05, + "loss": 2.6788, + "theoretical_loss": 3.3965217781160675, + "tokens_seen": 2255945728 + }, + { + "epoch": 0.37, + "learning_rate": 6.390627507623175e-05, + "loss": 2.6405, + "theoretical_loss": 3.39650571181714, + "tokens_seen": 2256076800 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1245462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.199613094329834, + "objective/train/theoretical_loss": 3.3964896467129315, + "objective/train/tokens_used": 626666976, + "theoretical_loss": 3.3964896467129315, + "tokens_seen": 2256207872 + }, + { + "epoch": 0.37, + "learning_rate": 6.38982506820735e-05, + "loss": 2.4546, + "theoretical_loss": 3.3964896467129315, + "tokens_seen": 2256207872 + }, + { + "epoch": 0.37, + "learning_rate": 6.389022628791527e-05, + "loss": 2.4392, + "theoretical_loss": 3.3964735828032833, + "tokens_seen": 2256338944 + }, + { + "epoch": 0.37, + "learning_rate": 6.388220189375703e-05, + "loss": 2.6483, + "theoretical_loss": 3.396457520088038, + "tokens_seen": 2256470016 + }, + { + "epoch": 0.37, + "learning_rate": 6.387417749959879e-05, + "loss": 2.6039, + "theoretical_loss": 3.3964414585670366, + "tokens_seen": 2256601088 + }, + { + "epoch": 0.37, + "learning_rate": 6.386615310544055e-05, + "loss": 2.6122, + "theoretical_loss": 3.396425398240121, + "tokens_seen": 2256732160 + }, + { + "epoch": 0.37, + "learning_rate": 6.385812871128231e-05, + "loss": 2.6314, + "theoretical_loss": 3.3964093391071337, + "tokens_seen": 2256863232 + }, + { + "epoch": 0.37, + "learning_rate": 6.385010431712406e-05, + "loss": 2.5326, + "theoretical_loss": 3.3963932811679163, + "tokens_seen": 2256994304 + }, + { + "epoch": 0.37, + "learning_rate": 6.384207992296582e-05, + "loss": 2.6952, + "theoretical_loss": 3.3963772244223107, + "tokens_seen": 2257125376 + }, + { + "epoch": 0.37, + "learning_rate": 6.383405552880758e-05, + "loss": 2.5161, + "theoretical_loss": 3.3963611688701585, + "tokens_seen": 2257256448 + }, + { + "epoch": 0.37, + "learning_rate": 6.382603113464934e-05, + "loss": 2.4891, + "theoretical_loss": 3.3963451145113024, + "tokens_seen": 2257387520 + }, + { + "epoch": 0.37, + "learning_rate": 6.38180067404911e-05, + "loss": 2.7088, + "theoretical_loss": 3.396329061345584, + "tokens_seen": 2257518592 + }, + { + "epoch": 0.37, + "learning_rate": 6.380998234633286e-05, + "loss": 2.6063, + "theoretical_loss": 3.3963130093728453, + "tokens_seen": 2257649664 + }, + { + "epoch": 0.37, + "learning_rate": 6.380195795217462e-05, + "loss": 2.7143, + "theoretical_loss": 3.396296958592929, + "tokens_seen": 2257780736 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1246138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.176940679550171, + "objective/train/theoretical_loss": 3.3962889336502293, + "objective/train/tokens_used": 628305376, + "theoretical_loss": 3.3962889336502293, + "tokens_seen": 2257846272 + }, + { + "epoch": 0.37, + "learning_rate": 6.379393355801638e-05, + "loss": 2.6306, + "theoretical_loss": 3.396280909005676, + "tokens_seen": 2257911808 + }, + { + "epoch": 0.37, + "learning_rate": 6.378590916385813e-05, + "loss": 2.5263, + "theoretical_loss": 3.39626486061093, + "tokens_seen": 2258042880 + }, + { + "epoch": 0.37, + "learning_rate": 6.37778847696999e-05, + "loss": 2.6265, + "theoretical_loss": 3.396248813408532, + "tokens_seen": 2258173952 + }, + { + "epoch": 0.37, + "learning_rate": 6.376986037554165e-05, + "loss": 2.6052, + "theoretical_loss": 3.396232767398325, + "tokens_seen": 2258305024 + }, + { + "epoch": 0.37, + "learning_rate": 6.376183598138342e-05, + "loss": 2.4728, + "theoretical_loss": 3.3962167225801503, + "tokens_seen": 2258436096 + }, + { + "epoch": 0.37, + "learning_rate": 6.375381158722517e-05, + "loss": 2.5031, + "theoretical_loss": 3.3962006789538512, + "tokens_seen": 2258567168 + }, + { + "epoch": 0.37, + "learning_rate": 6.374578719306694e-05, + "loss": 2.6393, + "theoretical_loss": 3.396184636519269, + "tokens_seen": 2258698240 + }, + { + "epoch": 0.37, + "learning_rate": 6.373776279890869e-05, + "loss": 2.598, + "theoretical_loss": 3.3961685952762473, + "tokens_seen": 2258829312 + }, + { + "epoch": 0.37, + "learning_rate": 6.372973840475044e-05, + "loss": 2.5913, + "theoretical_loss": 3.3961525552246274, + "tokens_seen": 2258960384 + }, + { + "epoch": 0.37, + "learning_rate": 6.372171401059221e-05, + "loss": 2.694, + "theoretical_loss": 3.396136516364252, + "tokens_seen": 2259091456 + }, + { + "epoch": 0.37, + "learning_rate": 6.371368961643396e-05, + "loss": 2.6971, + "theoretical_loss": 3.3961204786949635, + "tokens_seen": 2259222528 + }, + { + "epoch": 0.37, + "learning_rate": 6.370566522227573e-05, + "loss": 2.4899, + "theoretical_loss": 3.3961044422166045, + "tokens_seen": 2259353600 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1246815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6428818702697754, + "objective/train/theoretical_loss": 3.3960884069290174, + "objective/train/tokens_used": 629943776, + "theoretical_loss": 3.3960884069290174, + "tokens_seen": 2259484672 + }, + { + "epoch": 0.37, + "learning_rate": 6.369764082811748e-05, + "loss": 2.5699, + "theoretical_loss": 3.3960884069290174, + "tokens_seen": 2259484672 + }, + { + "epoch": 0.37, + "learning_rate": 6.368961643395925e-05, + "loss": 2.4269, + "theoretical_loss": 3.396072372832044, + "tokens_seen": 2259615744 + }, + { + "epoch": 0.37, + "learning_rate": 6.3681592039801e-05, + "loss": 2.5024, + "theoretical_loss": 3.3960563399255284, + "tokens_seen": 2259746816 + }, + { + "epoch": 0.37, + "learning_rate": 6.367356764564276e-05, + "loss": 2.4734, + "theoretical_loss": 3.396040308209312, + "tokens_seen": 2259877888 + }, + { + "epoch": 0.37, + "learning_rate": 6.366554325148452e-05, + "loss": 2.5707, + "theoretical_loss": 3.396024277683238, + "tokens_seen": 2260008960 + }, + { + "epoch": 0.37, + "learning_rate": 6.365751885732628e-05, + "loss": 2.5011, + "theoretical_loss": 3.3960082483471483, + "tokens_seen": 2260140032 + }, + { + "epoch": 0.37, + "learning_rate": 6.364949446316804e-05, + "loss": 2.6712, + "theoretical_loss": 3.395992220200886, + "tokens_seen": 2260271104 + }, + { + "epoch": 0.37, + "learning_rate": 6.36414700690098e-05, + "loss": 2.6961, + "theoretical_loss": 3.395976193244294, + "tokens_seen": 2260402176 + }, + { + "epoch": 0.37, + "learning_rate": 6.363344567485155e-05, + "loss": 2.6244, + "theoretical_loss": 3.395960167477215, + "tokens_seen": 2260533248 + }, + { + "epoch": 0.37, + "learning_rate": 6.362542128069332e-05, + "loss": 2.5395, + "theoretical_loss": 3.395944142899491, + "tokens_seen": 2260664320 + }, + { + "epoch": 0.37, + "learning_rate": 6.361739688653507e-05, + "loss": 2.6363, + "theoretical_loss": 3.3959281195109656, + "tokens_seen": 2260795392 + }, + { + "epoch": 0.37, + "learning_rate": 6.360937249237683e-05, + "loss": 2.57, + "theoretical_loss": 3.3959120973114816, + "tokens_seen": 2260926464 + }, + { + "epoch": 0.37, + "learning_rate": 6.360134809821859e-05, + "loss": 2.4798, + "theoretical_loss": 3.395896076300881, + "tokens_seen": 2261057536 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1247928, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4681272506713867, + "objective/train/theoretical_loss": 3.3958880662413633, + "objective/train/tokens_used": 631582176, + "theoretical_loss": 3.3958880662413633, + "tokens_seen": 2261123072 + }, + { + "epoch": 0.37, + "learning_rate": 6.359332370406035e-05, + "loss": 2.5529, + "theoretical_loss": 3.3958800564790077, + "tokens_seen": 2261188608 + }, + { + "epoch": 0.37, + "learning_rate": 6.358529930990211e-05, + "loss": 2.4942, + "theoretical_loss": 3.3958640378457043, + "tokens_seen": 2261319680 + }, + { + "epoch": 0.37, + "learning_rate": 6.357727491574386e-05, + "loss": 2.5433, + "theoretical_loss": 3.3958480204008135, + "tokens_seen": 2261450752 + }, + { + "epoch": 0.37, + "learning_rate": 6.356925052158563e-05, + "loss": 2.4331, + "theoretical_loss": 3.395832004144178, + "tokens_seen": 2261581824 + }, + { + "epoch": 0.37, + "learning_rate": 6.356122612742738e-05, + "loss": 2.4636, + "theoretical_loss": 3.3958159890756416, + "tokens_seen": 2261712896 + }, + { + "epoch": 0.37, + "learning_rate": 6.355320173326915e-05, + "loss": 2.5002, + "theoretical_loss": 3.3957999751950467, + "tokens_seen": 2261843968 + }, + { + "epoch": 0.37, + "learning_rate": 6.35451773391109e-05, + "loss": 2.5064, + "theoretical_loss": 3.3957839625022364, + "tokens_seen": 2261975040 + }, + { + "epoch": 0.37, + "learning_rate": 6.353715294495265e-05, + "loss": 2.4867, + "theoretical_loss": 3.395767950997054, + "tokens_seen": 2262106112 + }, + { + "epoch": 0.37, + "learning_rate": 6.352912855079442e-05, + "loss": 2.5062, + "theoretical_loss": 3.3957519406793426, + "tokens_seen": 2262237184 + }, + { + "epoch": 0.37, + "learning_rate": 6.352110415663617e-05, + "loss": 2.6116, + "theoretical_loss": 3.3957359315489453, + "tokens_seen": 2262368256 + }, + { + "epoch": 0.37, + "learning_rate": 6.351307976247794e-05, + "loss": 2.6306, + "theoretical_loss": 3.395719923605705, + "tokens_seen": 2262499328 + }, + { + "epoch": 0.37, + "learning_rate": 6.35050553683197e-05, + "loss": 2.4499, + "theoretical_loss": 3.395703916849465, + "tokens_seen": 2262630400 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1248487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7316555976867676, + "objective/train/theoretical_loss": 3.395687911280069, + "objective/train/tokens_used": 633220576, + "theoretical_loss": 3.395687911280069, + "tokens_seen": 2262761472 + }, + { + "epoch": 0.37, + "learning_rate": 6.349703097416146e-05, + "loss": 2.6194, + "theoretical_loss": 3.395687911280069, + "tokens_seen": 2262761472 + }, + { + "epoch": 0.37, + "learning_rate": 6.348900658000321e-05, + "loss": 2.6273, + "theoretical_loss": 3.39567190689736, + "tokens_seen": 2262892544 + }, + { + "epoch": 0.37, + "learning_rate": 6.348098218584497e-05, + "loss": 2.4054, + "theoretical_loss": 3.3956559037011806, + "tokens_seen": 2263023616 + }, + { + "epoch": 0.37, + "learning_rate": 6.347295779168673e-05, + "loss": 2.6728, + "theoretical_loss": 3.395639901691375, + "tokens_seen": 2263154688 + }, + { + "epoch": 0.37, + "learning_rate": 6.346493339752849e-05, + "loss": 2.587, + "theoretical_loss": 3.395623900867786, + "tokens_seen": 2263285760 + }, + { + "epoch": 0.37, + "learning_rate": 6.345690900337025e-05, + "loss": 2.4727, + "theoretical_loss": 3.395607901230258, + "tokens_seen": 2263416832 + }, + { + "epoch": 0.37, + "learning_rate": 6.3448884609212e-05, + "loss": 2.5248, + "theoretical_loss": 3.395591902778633, + "tokens_seen": 2263547904 + }, + { + "epoch": 0.37, + "learning_rate": 6.344086021505376e-05, + "loss": 2.4696, + "theoretical_loss": 3.3955759055127555, + "tokens_seen": 2263678976 + }, + { + "epoch": 0.37, + "learning_rate": 6.343283582089553e-05, + "loss": 2.631, + "theoretical_loss": 3.3955599094324684, + "tokens_seen": 2263810048 + }, + { + "epoch": 0.37, + "learning_rate": 6.342481142673728e-05, + "loss": 2.5313, + "theoretical_loss": 3.395543914537615, + "tokens_seen": 2263941120 + }, + { + "epoch": 0.37, + "learning_rate": 6.341678703257905e-05, + "loss": 2.4899, + "theoretical_loss": 3.3955279208280396, + "tokens_seen": 2264072192 + }, + { + "epoch": 0.37, + "learning_rate": 6.34087626384208e-05, + "loss": 2.5598, + "theoretical_loss": 3.395511928303585, + "tokens_seen": 2264203264 + }, + { + "epoch": 0.37, + "learning_rate": 6.340073824426257e-05, + "loss": 2.5354, + "theoretical_loss": 3.395495936964095, + "tokens_seen": 2264334336 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1249541, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5170953273773193, + "objective/train/theoretical_loss": 3.395487941738663, + "objective/train/tokens_used": 634858976, + "theoretical_loss": 3.395487941738663, + "tokens_seen": 2264399872 + }, + { + "epoch": 0.37, + "learning_rate": 6.339271385010432e-05, + "loss": 2.5583, + "theoretical_loss": 3.395479946809414, + "tokens_seen": 2264465408 + }, + { + "epoch": 0.37, + "learning_rate": 6.338468945594607e-05, + "loss": 2.5488, + "theoretical_loss": 3.3954639578393846, + "tokens_seen": 2264596480 + }, + { + "epoch": 0.37, + "learning_rate": 6.337666506178784e-05, + "loss": 2.554, + "theoretical_loss": 3.3954479700538505, + "tokens_seen": 2264727552 + }, + { + "epoch": 0.37, + "learning_rate": 6.336864066762959e-05, + "loss": 2.5779, + "theoretical_loss": 3.3954319834526565, + "tokens_seen": 2264858624 + }, + { + "epoch": 0.37, + "learning_rate": 6.336061627347136e-05, + "loss": 2.4888, + "theoretical_loss": 3.395415998035645, + "tokens_seen": 2264989696 + }, + { + "epoch": 0.37, + "learning_rate": 6.335259187931311e-05, + "loss": 2.5179, + "theoretical_loss": 3.3954000138026603, + "tokens_seen": 2265120768 + }, + { + "epoch": 0.37, + "learning_rate": 6.334456748515486e-05, + "loss": 2.5611, + "theoretical_loss": 3.395384030753547, + "tokens_seen": 2265251840 + }, + { + "epoch": 0.37, + "learning_rate": 6.333654309099663e-05, + "loss": 2.5947, + "theoretical_loss": 3.3953680488881472, + "tokens_seen": 2265382912 + }, + { + "epoch": 0.37, + "learning_rate": 6.332851869683838e-05, + "loss": 2.6412, + "theoretical_loss": 3.3953520682063063, + "tokens_seen": 2265513984 + }, + { + "epoch": 0.37, + "learning_rate": 6.332049430268015e-05, + "loss": 2.4971, + "theoretical_loss": 3.395336088707867, + "tokens_seen": 2265645056 + }, + { + "epoch": 0.37, + "learning_rate": 6.33124699085219e-05, + "loss": 2.4861, + "theoretical_loss": 3.3953201103926745, + "tokens_seen": 2265776128 + }, + { + "epoch": 0.37, + "learning_rate": 6.330444551436366e-05, + "loss": 2.6189, + "theoretical_loss": 3.395304133260572, + "tokens_seen": 2265907200 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1250567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.461559534072876, + "objective/train/theoretical_loss": 3.3952881573114033, + "objective/train/tokens_used": 636497376, + "theoretical_loss": 3.3952881573114033, + "tokens_seen": 2266038272 + }, + { + "epoch": 0.37, + "learning_rate": 6.329642112020542e-05, + "loss": 2.4765, + "theoretical_loss": 3.3952881573114033, + "tokens_seen": 2266038272 + }, + { + "epoch": 0.37, + "learning_rate": 6.328839672604718e-05, + "loss": 2.4616, + "theoretical_loss": 3.3952721825450127, + "tokens_seen": 2266169344 + }, + { + "epoch": 0.37, + "learning_rate": 6.328037233188894e-05, + "loss": 2.6036, + "theoretical_loss": 3.395256208961244, + "tokens_seen": 2266300416 + }, + { + "epoch": 0.37, + "learning_rate": 6.32723479377307e-05, + "loss": 2.481, + "theoretical_loss": 3.3952402365599417, + "tokens_seen": 2266431488 + }, + { + "epoch": 0.37, + "learning_rate": 6.326432354357246e-05, + "loss": 2.6836, + "theoretical_loss": 3.3952242653409495, + "tokens_seen": 2266562560 + }, + { + "epoch": 0.37, + "learning_rate": 6.325629914941422e-05, + "loss": 2.5822, + "theoretical_loss": 3.3952082953041116, + "tokens_seen": 2266693632 + }, + { + "epoch": 0.37, + "learning_rate": 6.324827475525597e-05, + "loss": 2.5497, + "theoretical_loss": 3.395192326449272, + "tokens_seen": 2266824704 + }, + { + "epoch": 0.37, + "learning_rate": 6.324025036109774e-05, + "loss": 2.5206, + "theoretical_loss": 3.395176358776275, + "tokens_seen": 2266955776 + }, + { + "epoch": 0.37, + "learning_rate": 6.323222596693949e-05, + "loss": 2.5478, + "theoretical_loss": 3.3951603922849656, + "tokens_seen": 2267086848 + }, + { + "epoch": 0.37, + "learning_rate": 6.322420157278126e-05, + "loss": 2.5811, + "theoretical_loss": 3.3951444269751865, + "tokens_seen": 2267217920 + }, + { + "epoch": 0.37, + "learning_rate": 6.321617717862301e-05, + "loss": 2.5322, + "theoretical_loss": 3.3951284628467833, + "tokens_seen": 2267348992 + }, + { + "epoch": 0.37, + "learning_rate": 6.320815278446478e-05, + "loss": 2.5236, + "theoretical_loss": 3.395112499899599, + "tokens_seen": 2267480064 + }, + { + "epoch": 0.37, + "learning_rate": 6.320012839030653e-05, + "loss": 2.5783, + "theoretical_loss": 3.3950965381334797, + "tokens_seen": 2267611136 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1251582, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6694164276123047, + "objective/train/theoretical_loss": 3.39508855769327, + "objective/train/tokens_used": 638135776, + "theoretical_loss": 3.39508855769327, + "tokens_seen": 2267676672 + }, + { + "epoch": 0.37, + "learning_rate": 6.319210399614828e-05, + "loss": 2.4533, + "theoretical_loss": 3.395080577548268, + "tokens_seen": 2267742208 + }, + { + "epoch": 0.37, + "learning_rate": 6.318407960199005e-05, + "loss": 2.7162, + "theoretical_loss": 3.3950646181438096, + "tokens_seen": 2267873280 + }, + { + "epoch": 0.37, + "learning_rate": 6.31760552078318e-05, + "loss": 2.6106, + "theoretical_loss": 3.395048659919948, + "tokens_seen": 2268004352 + }, + { + "epoch": 0.37, + "learning_rate": 6.316803081367357e-05, + "loss": 2.4827, + "theoretical_loss": 3.395032702876528, + "tokens_seen": 2268135424 + }, + { + "epoch": 0.37, + "learning_rate": 6.316000641951532e-05, + "loss": 2.4875, + "theoretical_loss": 3.3950167470133943, + "tokens_seen": 2268266496 + }, + { + "epoch": 0.37, + "learning_rate": 6.315198202535709e-05, + "loss": 2.5619, + "theoretical_loss": 3.3950007923303906, + "tokens_seen": 2268397568 + }, + { + "epoch": 0.37, + "learning_rate": 6.314395763119884e-05, + "loss": 2.6272, + "theoretical_loss": 3.3949848388273627, + "tokens_seen": 2268528640 + }, + { + "epoch": 0.37, + "learning_rate": 6.31359332370406e-05, + "loss": 2.3802, + "theoretical_loss": 3.394968886504154, + "tokens_seen": 2268659712 + }, + { + "epoch": 0.38, + "learning_rate": 6.312790884288236e-05, + "loss": 2.5548, + "theoretical_loss": 3.39495293536061, + "tokens_seen": 2268790784 + }, + { + "epoch": 0.38, + "learning_rate": 6.311988444872411e-05, + "loss": 2.6083, + "theoretical_loss": 3.3949369853965745, + "tokens_seen": 2268921856 + }, + { + "epoch": 0.38, + "learning_rate": 6.311186005456588e-05, + "loss": 2.6153, + "theoretical_loss": 3.3949210366118923, + "tokens_seen": 2269052928 + }, + { + "epoch": 0.38, + "learning_rate": 6.310383566040763e-05, + "loss": 2.4796, + "theoretical_loss": 3.3949050890064085, + "tokens_seen": 2269184000 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1252189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5453248023986816, + "objective/train/theoretical_loss": 3.394889142579968, + "objective/train/tokens_used": 639774176, + "theoretical_loss": 3.394889142579968, + "tokens_seen": 2269315072 + }, + { + "epoch": 0.38, + "learning_rate": 6.30958112662494e-05, + "loss": 2.4309, + "theoretical_loss": 3.394889142579968, + "tokens_seen": 2269315072 + }, + { + "epoch": 0.38, + "learning_rate": 6.308778687209115e-05, + "loss": 2.503, + "theoretical_loss": 3.394873197332415, + "tokens_seen": 2269446144 + }, + { + "epoch": 0.38, + "learning_rate": 6.307976247793292e-05, + "loss": 2.5122, + "theoretical_loss": 3.394857253263594, + "tokens_seen": 2269577216 + }, + { + "epoch": 0.38, + "learning_rate": 6.307173808377467e-05, + "loss": 2.6993, + "theoretical_loss": 3.3948413103733506, + "tokens_seen": 2269708288 + }, + { + "epoch": 0.38, + "learning_rate": 6.306371368961643e-05, + "loss": 2.5629, + "theoretical_loss": 3.394825368661529, + "tokens_seen": 2269839360 + }, + { + "epoch": 0.38, + "learning_rate": 6.30556892954582e-05, + "loss": 2.4151, + "theoretical_loss": 3.3948094281279744, + "tokens_seen": 2269970432 + }, + { + "epoch": 0.38, + "learning_rate": 6.304766490129995e-05, + "loss": 2.6158, + "theoretical_loss": 3.3947934887725317, + "tokens_seen": 2270101504 + }, + { + "epoch": 0.38, + "learning_rate": 6.303964050714171e-05, + "loss": 2.5988, + "theoretical_loss": 3.3947775505950455, + "tokens_seen": 2270232576 + }, + { + "epoch": 0.38, + "learning_rate": 6.303161611298347e-05, + "loss": 2.4931, + "theoretical_loss": 3.394761613595361, + "tokens_seen": 2270363648 + }, + { + "epoch": 0.38, + "learning_rate": 6.302359171882523e-05, + "loss": 2.6879, + "theoretical_loss": 3.394745677773323, + "tokens_seen": 2270494720 + }, + { + "epoch": 0.38, + "learning_rate": 6.301556732466699e-05, + "loss": 2.4496, + "theoretical_loss": 3.394729743128777, + "tokens_seen": 2270625792 + }, + { + "epoch": 0.38, + "learning_rate": 6.300754293050874e-05, + "loss": 2.7436, + "theoretical_loss": 3.394713809661568, + "tokens_seen": 2270756864 + }, + { + "epoch": 0.38, + "learning_rate": 6.29995185363505e-05, + "loss": 2.5725, + "theoretical_loss": 3.3946978773715397, + "tokens_seen": 2270887936 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1253382, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.357855796813965, + "objective/train/theoretical_loss": 3.394689911667921, + "objective/train/tokens_used": 641412576, + "theoretical_loss": 3.394689911667921, + "tokens_seen": 2270953472 + }, + { + "epoch": 0.38, + "learning_rate": 6.299149414219226e-05, + "loss": 2.4205, + "theoretical_loss": 3.394681946258539, + "tokens_seen": 2271019008 + }, + { + "epoch": 0.38, + "learning_rate": 6.298346974803403e-05, + "loss": 2.5027, + "theoretical_loss": 3.39466601632241, + "tokens_seen": 2271150080 + }, + { + "epoch": 0.38, + "learning_rate": 6.297544535387578e-05, + "loss": 2.5171, + "theoretical_loss": 3.3946500875629977, + "tokens_seen": 2271281152 + }, + { + "epoch": 0.38, + "learning_rate": 6.296742095971755e-05, + "loss": 2.6532, + "theoretical_loss": 3.394634159980148, + "tokens_seen": 2271412224 + }, + { + "epoch": 0.38, + "learning_rate": 6.29593965655593e-05, + "loss": 2.6409, + "theoretical_loss": 3.3946182335737056, + "tokens_seen": 2271543296 + }, + { + "epoch": 0.38, + "learning_rate": 6.295137217140105e-05, + "loss": 2.5364, + "theoretical_loss": 3.3946023083435164, + "tokens_seen": 2271674368 + }, + { + "epoch": 0.38, + "learning_rate": 6.294334777724282e-05, + "loss": 2.5256, + "theoretical_loss": 3.3945863842894246, + "tokens_seen": 2271805440 + }, + { + "epoch": 0.38, + "learning_rate": 6.293532338308457e-05, + "loss": 2.5709, + "theoretical_loss": 3.394570461411276, + "tokens_seen": 2271936512 + }, + { + "epoch": 0.38, + "learning_rate": 6.292729898892634e-05, + "loss": 2.5675, + "theoretical_loss": 3.3945545397089165, + "tokens_seen": 2272067584 + }, + { + "epoch": 0.38, + "learning_rate": 6.291927459476809e-05, + "loss": 2.551, + "theoretical_loss": 3.3945386191821907, + "tokens_seen": 2272198656 + }, + { + "epoch": 0.38, + "learning_rate": 6.291125020060986e-05, + "loss": 2.593, + "theoretical_loss": 3.3945226998309446, + "tokens_seen": 2272329728 + }, + { + "epoch": 0.38, + "learning_rate": 6.290322580645161e-05, + "loss": 2.5302, + "theoretical_loss": 3.3945067816550223, + "tokens_seen": 2272460800 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1253927, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2348384857177734, + "objective/train/theoretical_loss": 3.394490864654271, + "objective/train/tokens_used": 643050976, + "theoretical_loss": 3.394490864654271, + "tokens_seen": 2272591872 + }, + { + "epoch": 0.38, + "learning_rate": 6.289520141229336e-05, + "loss": 2.5167, + "theoretical_loss": 3.394490864654271, + "tokens_seen": 2272591872 + }, + { + "epoch": 0.38, + "learning_rate": 6.288717701813513e-05, + "loss": 2.4235, + "theoretical_loss": 3.394474948828535, + "tokens_seen": 2272722944 + }, + { + "epoch": 0.38, + "learning_rate": 6.287915262397688e-05, + "loss": 2.3021, + "theoretical_loss": 3.39445903417766, + "tokens_seen": 2272854016 + }, + { + "epoch": 0.38, + "learning_rate": 6.287112822981865e-05, + "loss": 2.4075, + "theoretical_loss": 3.3944431207014922, + "tokens_seen": 2272985088 + }, + { + "epoch": 0.38, + "learning_rate": 6.28631038356604e-05, + "loss": 2.5877, + "theoretical_loss": 3.3944272083998763, + "tokens_seen": 2273116160 + }, + { + "epoch": 0.38, + "learning_rate": 6.285507944150217e-05, + "loss": 2.5949, + "theoretical_loss": 3.394411297272659, + "tokens_seen": 2273247232 + }, + { + "epoch": 0.38, + "learning_rate": 6.284705504734392e-05, + "loss": 2.6306, + "theoretical_loss": 3.394395387319684, + "tokens_seen": 2273378304 + }, + { + "epoch": 0.38, + "learning_rate": 6.283903065318568e-05, + "loss": 2.4229, + "theoretical_loss": 3.394379478540799, + "tokens_seen": 2273509376 + }, + { + "epoch": 0.38, + "learning_rate": 6.283100625902744e-05, + "loss": 2.5252, + "theoretical_loss": 3.394363570935848, + "tokens_seen": 2273640448 + }, + { + "epoch": 0.38, + "learning_rate": 6.28229818648692e-05, + "loss": 2.5287, + "theoretical_loss": 3.3943476645046777, + "tokens_seen": 2273771520 + }, + { + "epoch": 0.38, + "learning_rate": 6.281495747071096e-05, + "loss": 2.4188, + "theoretical_loss": 3.394331759247134, + "tokens_seen": 2273902592 + }, + { + "epoch": 0.38, + "learning_rate": 6.280693307655272e-05, + "loss": 2.4368, + "theoretical_loss": 3.394315855163062, + "tokens_seen": 2274033664 + }, + { + "epoch": 0.38, + "learning_rate": 6.279890868239448e-05, + "loss": 2.6502, + "theoretical_loss": 3.394299952252308, + "tokens_seen": 2274164736 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1255080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.468050479888916, + "objective/train/theoretical_loss": 3.3942920012368765, + "objective/train/tokens_used": 644689376, + "theoretical_loss": 3.3942920012368765, + "tokens_seen": 2274230272 + }, + { + "epoch": 0.38, + "learning_rate": 6.279088428823624e-05, + "loss": 2.672, + "theoretical_loss": 3.394284050514717, + "tokens_seen": 2274295808 + }, + { + "epoch": 0.38, + "learning_rate": 6.2782859894078e-05, + "loss": 2.4558, + "theoretical_loss": 3.394268149950136, + "tokens_seen": 2274426880 + }, + { + "epoch": 0.38, + "learning_rate": 6.277483549991976e-05, + "loss": 2.3806, + "theoretical_loss": 3.3942522505584103, + "tokens_seen": 2274557952 + }, + { + "epoch": 0.38, + "learning_rate": 6.276681110576151e-05, + "loss": 2.5326, + "theoretical_loss": 3.3942363523393855, + "tokens_seen": 2274689024 + }, + { + "epoch": 0.38, + "learning_rate": 6.275878671160328e-05, + "loss": 2.4212, + "theoretical_loss": 3.394220455292908, + "tokens_seen": 2274820096 + }, + { + "epoch": 0.38, + "learning_rate": 6.275076231744503e-05, + "loss": 2.5278, + "theoretical_loss": 3.394204559418824, + "tokens_seen": 2274951168 + }, + { + "epoch": 0.38, + "learning_rate": 6.27427379232868e-05, + "loss": 2.5986, + "theoretical_loss": 3.394188664716979, + "tokens_seen": 2275082240 + }, + { + "epoch": 0.38, + "learning_rate": 6.273471352912855e-05, + "loss": 2.511, + "theoretical_loss": 3.3941727711872187, + "tokens_seen": 2275213312 + }, + { + "epoch": 0.38, + "learning_rate": 6.272668913497032e-05, + "loss": 2.452, + "theoretical_loss": 3.3941568788293903, + "tokens_seen": 2275344384 + }, + { + "epoch": 0.38, + "learning_rate": 6.271866474081207e-05, + "loss": 2.5879, + "theoretical_loss": 3.394140987643339, + "tokens_seen": 2275475456 + }, + { + "epoch": 0.38, + "learning_rate": 6.271064034665382e-05, + "loss": 2.6386, + "theoretical_loss": 3.3941250976289106, + "tokens_seen": 2275606528 + }, + { + "epoch": 0.38, + "learning_rate": 6.270261595249559e-05, + "loss": 2.5057, + "theoretical_loss": 3.3941092087859523, + "tokens_seen": 2275737600 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1255731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5856258869171143, + "objective/train/theoretical_loss": 3.3940933211143096, + "objective/train/tokens_used": 646327776, + "theoretical_loss": 3.3940933211143096, + "tokens_seen": 2275868672 + }, + { + "epoch": 0.38, + "learning_rate": 6.269459155833734e-05, + "loss": 2.5352, + "theoretical_loss": 3.3940933211143096, + "tokens_seen": 2275868672 + }, + { + "epoch": 0.38, + "learning_rate": 6.268656716417911e-05, + "loss": 2.3659, + "theoretical_loss": 3.3940774346138287, + "tokens_seen": 2275999744 + }, + { + "epoch": 0.38, + "learning_rate": 6.267854277002086e-05, + "loss": 2.4207, + "theoretical_loss": 3.394061549284356, + "tokens_seen": 2276130816 + }, + { + "epoch": 0.38, + "learning_rate": 6.267051837586263e-05, + "loss": 2.5071, + "theoretical_loss": 3.394045665125738, + "tokens_seen": 2276261888 + }, + { + "epoch": 0.38, + "learning_rate": 6.266249398170438e-05, + "loss": 2.5107, + "theoretical_loss": 3.39402978213782, + "tokens_seen": 2276392960 + }, + { + "epoch": 0.38, + "learning_rate": 6.265446958754613e-05, + "loss": 2.4449, + "theoretical_loss": 3.3940139003204495, + "tokens_seen": 2276524032 + }, + { + "epoch": 0.38, + "learning_rate": 6.26464451933879e-05, + "loss": 2.4925, + "theoretical_loss": 3.3939980196734725, + "tokens_seen": 2276655104 + }, + { + "epoch": 0.38, + "learning_rate": 6.263842079922965e-05, + "loss": 2.4598, + "theoretical_loss": 3.3939821401967354, + "tokens_seen": 2276786176 + }, + { + "epoch": 0.38, + "learning_rate": 6.263039640507142e-05, + "loss": 2.4818, + "theoretical_loss": 3.393966261890084, + "tokens_seen": 2276917248 + }, + { + "epoch": 0.38, + "learning_rate": 6.262237201091317e-05, + "loss": 2.4662, + "theoretical_loss": 3.3939503847533654, + "tokens_seen": 2277048320 + }, + { + "epoch": 0.38, + "learning_rate": 6.261434761675494e-05, + "loss": 2.4891, + "theoretical_loss": 3.3939345087864257, + "tokens_seen": 2277179392 + }, + { + "epoch": 0.38, + "learning_rate": 6.26063232225967e-05, + "loss": 2.4176, + "theoretical_loss": 3.3939186339891116, + "tokens_seen": 2277310464 + }, + { + "epoch": 0.38, + "learning_rate": 6.259829882843845e-05, + "loss": 2.4436, + "theoretical_loss": 3.3939027603612697, + "tokens_seen": 2277441536 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1256864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.405860424041748, + "objective/train/theoretical_loss": 3.3938948239858524, + "objective/train/tokens_used": 647966176, + "theoretical_loss": 3.3938948239858524, + "tokens_seen": 2277507072 + }, + { + "epoch": 0.38, + "learning_rate": 6.259027443428021e-05, + "loss": 2.5856, + "theoretical_loss": 3.393886887902746, + "tokens_seen": 2277572608 + }, + { + "epoch": 0.38, + "learning_rate": 6.258225004012197e-05, + "loss": 2.55, + "theoretical_loss": 3.393871016613388, + "tokens_seen": 2277703680 + }, + { + "epoch": 0.38, + "learning_rate": 6.257422564596373e-05, + "loss": 2.6433, + "theoretical_loss": 3.393855146493041, + "tokens_seen": 2277834752 + }, + { + "epoch": 0.38, + "learning_rate": 6.256620125180549e-05, + "loss": 2.5091, + "theoretical_loss": 3.393839277541553, + "tokens_seen": 2277965824 + }, + { + "epoch": 0.38, + "learning_rate": 6.255817685764725e-05, + "loss": 2.5015, + "theoretical_loss": 3.3938234097587703, + "tokens_seen": 2278096896 + }, + { + "epoch": 0.38, + "learning_rate": 6.255015246348901e-05, + "loss": 2.5052, + "theoretical_loss": 3.393807543144539, + "tokens_seen": 2278227968 + }, + { + "epoch": 0.38, + "learning_rate": 6.254212806933076e-05, + "loss": 2.4866, + "theoretical_loss": 3.3937916776987063, + "tokens_seen": 2278359040 + }, + { + "epoch": 0.38, + "learning_rate": 6.253410367517253e-05, + "loss": 2.4978, + "theoretical_loss": 3.3937758134211187, + "tokens_seen": 2278490112 + }, + { + "epoch": 0.38, + "learning_rate": 6.252607928101428e-05, + "loss": 2.413, + "theoretical_loss": 3.3937599503116234, + "tokens_seen": 2278621184 + }, + { + "epoch": 0.38, + "learning_rate": 6.251805488685605e-05, + "loss": 2.4349, + "theoretical_loss": 3.3937440883700667, + "tokens_seen": 2278752256 + }, + { + "epoch": 0.38, + "learning_rate": 6.25100304926978e-05, + "loss": 2.5007, + "theoretical_loss": 3.393728227596296, + "tokens_seen": 2278883328 + }, + { + "epoch": 0.38, + "learning_rate": 6.250200609853957e-05, + "loss": 2.5965, + "theoretical_loss": 3.3937123679901573, + "tokens_seen": 2279014400 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1257410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6075518131256104, + "objective/train/theoretical_loss": 3.393696509551498, + "objective/train/tokens_used": 649604576, + "theoretical_loss": 3.393696509551498, + "tokens_seen": 2279145472 + }, + { + "epoch": 0.38, + "learning_rate": 6.249398170438132e-05, + "loss": 2.3826, + "theoretical_loss": 3.393696509551498, + "tokens_seen": 2279145472 + }, + { + "epoch": 0.38, + "learning_rate": 6.248595731022309e-05, + "loss": 2.5311, + "theoretical_loss": 3.3936806522801657, + "tokens_seen": 2279276544 + }, + { + "epoch": 0.38, + "learning_rate": 6.247793291606484e-05, + "loss": 2.5216, + "theoretical_loss": 3.3936647961760062, + "tokens_seen": 2279407616 + }, + { + "epoch": 0.38, + "learning_rate": 6.246990852190659e-05, + "loss": 2.5704, + "theoretical_loss": 3.393648941238867, + "tokens_seen": 2279538688 + }, + { + "epoch": 0.38, + "learning_rate": 6.246188412774836e-05, + "loss": 2.5134, + "theoretical_loss": 3.3936330874685954, + "tokens_seen": 2279669760 + }, + { + "epoch": 0.38, + "learning_rate": 6.245385973359011e-05, + "loss": 2.5912, + "theoretical_loss": 3.393617234865038, + "tokens_seen": 2279800832 + }, + { + "epoch": 0.38, + "learning_rate": 6.244583533943188e-05, + "loss": 2.4987, + "theoretical_loss": 3.393601383428042, + "tokens_seen": 2279931904 + }, + { + "epoch": 0.38, + "learning_rate": 6.243781094527363e-05, + "loss": 2.5887, + "theoretical_loss": 3.393585533157455, + "tokens_seen": 2280062976 + }, + { + "epoch": 0.38, + "learning_rate": 6.24297865511154e-05, + "loss": 2.4106, + "theoretical_loss": 3.393569684053123, + "tokens_seen": 2280194048 + }, + { + "epoch": 0.38, + "learning_rate": 6.242176215695715e-05, + "loss": 2.5223, + "theoretical_loss": 3.393553836114894, + "tokens_seen": 2280325120 + }, + { + "epoch": 0.38, + "learning_rate": 6.24137377627989e-05, + "loss": 2.5535, + "theoretical_loss": 3.393537989342615, + "tokens_seen": 2280456192 + }, + { + "epoch": 0.38, + "learning_rate": 6.240571336864067e-05, + "loss": 2.4859, + "theoretical_loss": 3.393522143736133, + "tokens_seen": 2280587264 + }, + { + "epoch": 0.38, + "learning_rate": 6.239768897448242e-05, + "loss": 2.3197, + "theoretical_loss": 3.3935062992952956, + "tokens_seen": 2280718336 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1258640, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.744055986404419, + "objective/train/theoretical_loss": 3.393498377511946, + "objective/train/tokens_used": 651242976, + "theoretical_loss": 3.393498377511946, + "tokens_seen": 2280783872 + }, + { + "epoch": 0.38, + "learning_rate": 6.238966458032419e-05, + "loss": 2.5835, + "theoretical_loss": 3.39349045601995, + "tokens_seen": 2280849408 + }, + { + "epoch": 0.38, + "learning_rate": 6.238164018616594e-05, + "loss": 2.4064, + "theoretical_loss": 3.393474613909943, + "tokens_seen": 2280980480 + }, + { + "epoch": 0.38, + "learning_rate": 6.237361579200771e-05, + "loss": 2.4895, + "theoretical_loss": 3.393458772965123, + "tokens_seen": 2281111552 + }, + { + "epoch": 0.38, + "learning_rate": 6.236559139784946e-05, + "loss": 2.4248, + "theoretical_loss": 3.3934429331853364, + "tokens_seen": 2281242624 + }, + { + "epoch": 0.38, + "learning_rate": 6.235756700369122e-05, + "loss": 2.419, + "theoretical_loss": 3.3934270945704306, + "tokens_seen": 2281373696 + }, + { + "epoch": 0.38, + "learning_rate": 6.234954260953298e-05, + "loss": 2.3879, + "theoretical_loss": 3.3934112571202535, + "tokens_seen": 2281504768 + }, + { + "epoch": 0.38, + "learning_rate": 6.234151821537474e-05, + "loss": 2.6913, + "theoretical_loss": 3.3933954208346524, + "tokens_seen": 2281635840 + }, + { + "epoch": 0.38, + "learning_rate": 6.23334938212165e-05, + "loss": 2.4367, + "theoretical_loss": 3.3933795857134745, + "tokens_seen": 2281766912 + }, + { + "epoch": 0.38, + "learning_rate": 6.232546942705826e-05, + "loss": 2.5412, + "theoretical_loss": 3.3933637517565676, + "tokens_seen": 2281897984 + }, + { + "epoch": 0.38, + "learning_rate": 6.231744503290002e-05, + "loss": 2.533, + "theoretical_loss": 3.393347918963779, + "tokens_seen": 2282029056 + }, + { + "epoch": 0.38, + "learning_rate": 6.230942063874178e-05, + "loss": 2.423, + "theoretical_loss": 3.3933320873349566, + "tokens_seen": 2282160128 + }, + { + "epoch": 0.38, + "learning_rate": 6.230139624458353e-05, + "loss": 2.5225, + "theoretical_loss": 3.3933162568699475, + "tokens_seen": 2282291200 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1259875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7450172901153564, + "objective/train/theoretical_loss": 3.3933004275686, + "objective/train/tokens_used": 652881376, + "theoretical_loss": 3.3933004275686, + "tokens_seen": 2282422272 + }, + { + "epoch": 0.38, + "learning_rate": 6.22933718504253e-05, + "loss": 2.6164, + "theoretical_loss": 3.3933004275686, + "tokens_seen": 2282422272 + }, + { + "epoch": 0.38, + "learning_rate": 6.228534745626705e-05, + "loss": 2.4909, + "theoretical_loss": 3.393284599430761, + "tokens_seen": 2282553344 + }, + { + "epoch": 0.38, + "learning_rate": 6.227732306210882e-05, + "loss": 2.4819, + "theoretical_loss": 3.3932687724562784, + "tokens_seen": 2282684416 + }, + { + "epoch": 0.38, + "learning_rate": 6.226929866795057e-05, + "loss": 2.4915, + "theoretical_loss": 3.3932529466450005, + "tokens_seen": 2282815488 + }, + { + "epoch": 0.38, + "learning_rate": 6.226127427379234e-05, + "loss": 2.4388, + "theoretical_loss": 3.3932371219967745, + "tokens_seen": 2282946560 + }, + { + "epoch": 0.38, + "learning_rate": 6.225324987963409e-05, + "loss": 2.5142, + "theoretical_loss": 3.393221298511448, + "tokens_seen": 2283077632 + }, + { + "epoch": 0.38, + "learning_rate": 6.224522548547584e-05, + "loss": 2.6628, + "theoretical_loss": 3.393205476188869, + "tokens_seen": 2283208704 + }, + { + "epoch": 0.38, + "learning_rate": 6.223720109131761e-05, + "loss": 2.5043, + "theoretical_loss": 3.393189655028885, + "tokens_seen": 2283339776 + }, + { + "epoch": 0.38, + "learning_rate": 6.222917669715936e-05, + "loss": 2.5316, + "theoretical_loss": 3.3931738350313445, + "tokens_seen": 2283470848 + }, + { + "epoch": 0.38, + "learning_rate": 6.222115230300113e-05, + "loss": 2.7371, + "theoretical_loss": 3.393158016196095, + "tokens_seen": 2283601920 + }, + { + "epoch": 0.38, + "learning_rate": 6.221312790884288e-05, + "loss": 2.5326, + "theoretical_loss": 3.393142198522984, + "tokens_seen": 2283732992 + }, + { + "epoch": 0.38, + "learning_rate": 6.220510351468465e-05, + "loss": 2.5107, + "theoretical_loss": 3.3931263820118605, + "tokens_seen": 2283864064 + }, + { + "epoch": 0.38, + "learning_rate": 6.21970791205264e-05, + "loss": 2.4426, + "theoretical_loss": 3.393110566662571, + "tokens_seen": 2283995136 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1260339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0385963916778564, + "objective/train/theoretical_loss": 3.393102659423567, + "objective/train/tokens_used": 654519776, + "theoretical_loss": 3.393102659423567, + "tokens_seen": 2284060672 + }, + { + "epoch": 0.38, + "learning_rate": 6.218905472636816e-05, + "loss": 2.5297, + "theoretical_loss": 3.393094752474965, + "tokens_seen": 2284126208 + }, + { + "epoch": 0.38, + "learning_rate": 6.218103033220992e-05, + "loss": 2.5378, + "theoretical_loss": 3.39307893944889, + "tokens_seen": 2284257280 + }, + { + "epoch": 0.38, + "learning_rate": 6.217300593805168e-05, + "loss": 2.6055, + "theoretical_loss": 3.393063127584193, + "tokens_seen": 2284388352 + }, + { + "epoch": 0.38, + "learning_rate": 6.216498154389344e-05, + "loss": 2.547, + "theoretical_loss": 3.3930473168807236, + "tokens_seen": 2284519424 + }, + { + "epoch": 0.38, + "learning_rate": 6.21569571497352e-05, + "loss": 2.4072, + "theoretical_loss": 3.3930315073383293, + "tokens_seen": 2284650496 + }, + { + "epoch": 0.38, + "learning_rate": 6.214893275557696e-05, + "loss": 2.4872, + "theoretical_loss": 3.393015698956858, + "tokens_seen": 2284781568 + }, + { + "epoch": 0.38, + "learning_rate": 6.214090836141871e-05, + "loss": 2.6139, + "theoretical_loss": 3.3929998917361575, + "tokens_seen": 2284912640 + }, + { + "epoch": 0.38, + "learning_rate": 6.213288396726048e-05, + "loss": 2.4484, + "theoretical_loss": 3.392984085676077, + "tokens_seen": 2285043712 + }, + { + "epoch": 0.38, + "learning_rate": 6.212485957310223e-05, + "loss": 2.4945, + "theoretical_loss": 3.3929682807764645, + "tokens_seen": 2285174784 + }, + { + "epoch": 0.39, + "learning_rate": 6.211683517894399e-05, + "loss": 2.3858, + "theoretical_loss": 3.3929524770371677, + "tokens_seen": 2285305856 + }, + { + "epoch": 0.39, + "learning_rate": 6.210881078478575e-05, + "loss": 2.5794, + "theoretical_loss": 3.392936674458035, + "tokens_seen": 2285436928 + }, + { + "epoch": 0.39, + "learning_rate": 6.210078639062751e-05, + "loss": 2.5333, + "theoretical_loss": 3.392920873038915, + "tokens_seen": 2285568000 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1261840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0126242637634277, + "objective/train/theoretical_loss": 3.392905072779656, + "objective/train/tokens_used": 656158176, + "theoretical_loss": 3.392905072779656, + "tokens_seen": 2285699072 + }, + { + "epoch": 0.39, + "learning_rate": 6.209276199646927e-05, + "loss": 2.6094, + "theoretical_loss": 3.392905072779656, + "tokens_seen": 2285699072 + }, + { + "epoch": 0.39, + "learning_rate": 6.208473760231103e-05, + "loss": 2.4284, + "theoretical_loss": 3.392889273680106, + "tokens_seen": 2285830144 + }, + { + "epoch": 0.39, + "learning_rate": 6.20767132081528e-05, + "loss": 2.6252, + "theoretical_loss": 3.3928734757401138, + "tokens_seen": 2285961216 + }, + { + "epoch": 0.39, + "learning_rate": 6.206868881399455e-05, + "loss": 2.5987, + "theoretical_loss": 3.3928576789595275, + "tokens_seen": 2286092288 + }, + { + "epoch": 0.39, + "learning_rate": 6.20606644198363e-05, + "loss": 2.5607, + "theoretical_loss": 3.3928418833381957, + "tokens_seen": 2286223360 + }, + { + "epoch": 0.39, + "learning_rate": 6.205264002567807e-05, + "loss": 2.545, + "theoretical_loss": 3.392826088875967, + "tokens_seen": 2286354432 + }, + { + "epoch": 0.39, + "learning_rate": 6.204461563151982e-05, + "loss": 2.576, + "theoretical_loss": 3.3928102955726898, + "tokens_seen": 2286485504 + }, + { + "epoch": 0.39, + "learning_rate": 6.203659123736159e-05, + "loss": 2.6459, + "theoretical_loss": 3.3927945034282123, + "tokens_seen": 2286616576 + }, + { + "epoch": 0.39, + "learning_rate": 6.202856684320334e-05, + "loss": 2.5031, + "theoretical_loss": 3.3927787124423836, + "tokens_seen": 2286747648 + }, + { + "epoch": 0.39, + "learning_rate": 6.20205424490451e-05, + "loss": 2.516, + "theoretical_loss": 3.392762922615052, + "tokens_seen": 2286878720 + }, + { + "epoch": 0.39, + "learning_rate": 6.201251805488686e-05, + "loss": 2.5389, + "theoretical_loss": 3.3927471339460666, + "tokens_seen": 2287009792 + }, + { + "epoch": 0.39, + "learning_rate": 6.200449366072861e-05, + "loss": 2.5867, + "theoretical_loss": 3.392731346435275, + "tokens_seen": 2287140864 + }, + { + "epoch": 0.39, + "learning_rate": 6.199646926657038e-05, + "loss": 2.6023, + "theoretical_loss": 3.3927155600825265, + "tokens_seen": 2287271936 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1262387, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.811039447784424, + "objective/train/theoretical_loss": 3.3927076673403715, + "objective/train/tokens_used": 657796576, + "theoretical_loss": 3.3927076673403715, + "tokens_seen": 2287337472 + }, + { + "epoch": 0.39, + "learning_rate": 6.198844487241213e-05, + "loss": 2.3361, + "theoretical_loss": 3.39269977488767, + "tokens_seen": 2287403008 + }, + { + "epoch": 0.39, + "learning_rate": 6.19804204782539e-05, + "loss": 2.3675, + "theoretical_loss": 3.392683990850554, + "tokens_seen": 2287534080 + }, + { + "epoch": 0.39, + "learning_rate": 6.197239608409565e-05, + "loss": 2.6312, + "theoretical_loss": 3.3926682079710275, + "tokens_seen": 2287665152 + }, + { + "epoch": 0.39, + "learning_rate": 6.196437168993742e-05, + "loss": 2.5818, + "theoretical_loss": 3.3926524262489384, + "tokens_seen": 2287796224 + }, + { + "epoch": 0.39, + "learning_rate": 6.195634729577917e-05, + "loss": 2.4892, + "theoretical_loss": 3.392636645684137, + "tokens_seen": 2287927296 + }, + { + "epoch": 0.39, + "learning_rate": 6.194832290162093e-05, + "loss": 2.4498, + "theoretical_loss": 3.3926208662764705, + "tokens_seen": 2288058368 + }, + { + "epoch": 0.39, + "learning_rate": 6.194029850746269e-05, + "loss": 2.5447, + "theoretical_loss": 3.392605088025789, + "tokens_seen": 2288189440 + }, + { + "epoch": 0.39, + "learning_rate": 6.193227411330445e-05, + "loss": 2.5413, + "theoretical_loss": 3.3925893109319407, + "tokens_seen": 2288320512 + }, + { + "epoch": 0.39, + "learning_rate": 6.192424971914621e-05, + "loss": 2.6062, + "theoretical_loss": 3.392573534994775, + "tokens_seen": 2288451584 + }, + { + "epoch": 0.39, + "learning_rate": 6.191622532498797e-05, + "loss": 2.5116, + "theoretical_loss": 3.3925577602141406, + "tokens_seen": 2288582656 + }, + { + "epoch": 0.39, + "learning_rate": 6.190820093082973e-05, + "loss": 2.5007, + "theoretical_loss": 3.3925419865898867, + "tokens_seen": 2288713728 + }, + { + "epoch": 0.39, + "learning_rate": 6.190017653667148e-05, + "loss": 2.4369, + "theoretical_loss": 3.392526214121862, + "tokens_seen": 2288844800 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1263579, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.539278745651245, + "objective/train/theoretical_loss": 3.392510442809916, + "objective/train/tokens_used": 659434976, + "theoretical_loss": 3.392510442809916, + "tokens_seen": 2288975872 + }, + { + "epoch": 0.39, + "learning_rate": 6.189215214251324e-05, + "loss": 2.6523, + "theoretical_loss": 3.392510442809916, + "tokens_seen": 2288975872 + }, + { + "epoch": 0.39, + "learning_rate": 6.1884127748355e-05, + "loss": 2.5045, + "theoretical_loss": 3.392494672653897, + "tokens_seen": 2289106944 + }, + { + "epoch": 0.39, + "learning_rate": 6.187610335419676e-05, + "loss": 2.5141, + "theoretical_loss": 3.392478903653655, + "tokens_seen": 2289238016 + }, + { + "epoch": 0.39, + "learning_rate": 6.186807896003852e-05, + "loss": 2.6254, + "theoretical_loss": 3.392463135809039, + "tokens_seen": 2289369088 + }, + { + "epoch": 0.39, + "learning_rate": 6.186005456588028e-05, + "loss": 2.4007, + "theoretical_loss": 3.392447369119897, + "tokens_seen": 2289500160 + }, + { + "epoch": 0.39, + "learning_rate": 6.185203017172204e-05, + "loss": 2.4746, + "theoretical_loss": 3.3924316035860795, + "tokens_seen": 2289631232 + }, + { + "epoch": 0.39, + "learning_rate": 6.18440057775638e-05, + "loss": 2.6335, + "theoretical_loss": 3.3924158392074353, + "tokens_seen": 2289762304 + }, + { + "epoch": 0.39, + "learning_rate": 6.183598138340556e-05, + "loss": 2.5504, + "theoretical_loss": 3.3924000759838133, + "tokens_seen": 2289893376 + }, + { + "epoch": 0.39, + "learning_rate": 6.182795698924732e-05, + "loss": 2.5918, + "theoretical_loss": 3.3923843139150636, + "tokens_seen": 2290024448 + }, + { + "epoch": 0.39, + "learning_rate": 6.181993259508907e-05, + "loss": 2.541, + "theoretical_loss": 3.3923685530010346, + "tokens_seen": 2290155520 + }, + { + "epoch": 0.39, + "learning_rate": 6.181190820093084e-05, + "loss": 2.5115, + "theoretical_loss": 3.3923527932415762, + "tokens_seen": 2290286592 + }, + { + "epoch": 0.39, + "learning_rate": 6.180388380677259e-05, + "loss": 2.6476, + "theoretical_loss": 3.392337034636537, + "tokens_seen": 2290417664 + }, + { + "epoch": 0.39, + "learning_rate": 6.179585941261436e-05, + "loss": 2.6693, + "theoretical_loss": 3.3923212771857676, + "tokens_seen": 2290548736 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1264107, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.165374755859375, + "objective/train/theoretical_loss": 3.3923133988931866, + "objective/train/tokens_used": 661073376, + "theoretical_loss": 3.3923133988931866, + "tokens_seen": 2290614272 + }, + { + "epoch": 0.39, + "learning_rate": 6.178783501845611e-05, + "loss": 2.6599, + "theoretical_loss": 3.392305520889116, + "tokens_seen": 2290679808 + }, + { + "epoch": 0.39, + "learning_rate": 6.177981062429788e-05, + "loss": 2.5995, + "theoretical_loss": 3.3922897657464333, + "tokens_seen": 2290810880 + }, + { + "epoch": 0.39, + "learning_rate": 6.177178623013963e-05, + "loss": 2.5903, + "theoretical_loss": 3.392274011757567, + "tokens_seen": 2290941952 + }, + { + "epoch": 0.39, + "learning_rate": 6.176376183598138e-05, + "loss": 2.6104, + "theoretical_loss": 3.3922582589223684, + "tokens_seen": 2291073024 + }, + { + "epoch": 0.39, + "learning_rate": 6.175573744182315e-05, + "loss": 2.6337, + "theoretical_loss": 3.392242507240686, + "tokens_seen": 2291204096 + }, + { + "epoch": 0.39, + "learning_rate": 6.17477130476649e-05, + "loss": 2.5884, + "theoretical_loss": 3.3922267567123696, + "tokens_seen": 2291335168 + }, + { + "epoch": 0.39, + "learning_rate": 6.173968865350667e-05, + "loss": 2.495, + "theoretical_loss": 3.3922110073372687, + "tokens_seen": 2291466240 + }, + { + "epoch": 0.39, + "learning_rate": 6.173166425934842e-05, + "loss": 2.6121, + "theoretical_loss": 3.392195259115233, + "tokens_seen": 2291597312 + }, + { + "epoch": 0.39, + "learning_rate": 6.172363986519019e-05, + "loss": 2.612, + "theoretical_loss": 3.3921795120461122, + "tokens_seen": 2291728384 + }, + { + "epoch": 0.39, + "learning_rate": 6.171561547103194e-05, + "loss": 2.5074, + "theoretical_loss": 3.392163766129756, + "tokens_seen": 2291859456 + }, + { + "epoch": 0.39, + "learning_rate": 6.17075910768737e-05, + "loss": 2.4929, + "theoretical_loss": 3.3921480213660136, + "tokens_seen": 2291990528 + }, + { + "epoch": 0.39, + "learning_rate": 6.169956668271546e-05, + "loss": 2.528, + "theoretical_loss": 3.3921322777547354, + "tokens_seen": 2292121600 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1264661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.496931314468384, + "objective/train/theoretical_loss": 3.392116535295771, + "objective/train/tokens_used": 662711776, + "theoretical_loss": 3.392116535295771, + "tokens_seen": 2292252672 + }, + { + "epoch": 0.39, + "learning_rate": 6.169154228855722e-05, + "loss": 2.5325, + "theoretical_loss": 3.392116535295771, + "tokens_seen": 2292252672 + }, + { + "epoch": 0.39, + "learning_rate": 6.168351789439898e-05, + "loss": 2.6184, + "theoretical_loss": 3.39210079398897, + "tokens_seen": 2292383744 + }, + { + "epoch": 0.39, + "learning_rate": 6.167549350024074e-05, + "loss": 2.4983, + "theoretical_loss": 3.392085053834182, + "tokens_seen": 2292514816 + }, + { + "epoch": 0.39, + "learning_rate": 6.16674691060825e-05, + "loss": 2.6247, + "theoretical_loss": 3.3920693148312573, + "tokens_seen": 2292645888 + }, + { + "epoch": 0.39, + "learning_rate": 6.165944471192425e-05, + "loss": 2.5484, + "theoretical_loss": 3.392053576980045, + "tokens_seen": 2292776960 + }, + { + "epoch": 0.39, + "learning_rate": 6.165142031776601e-05, + "loss": 2.5444, + "theoretical_loss": 3.3920378402803957, + "tokens_seen": 2292908032 + }, + { + "epoch": 0.39, + "learning_rate": 6.164339592360777e-05, + "loss": 2.4537, + "theoretical_loss": 3.3920221047321597, + "tokens_seen": 2293039104 + }, + { + "epoch": 0.39, + "learning_rate": 6.163537152944953e-05, + "loss": 2.4478, + "theoretical_loss": 3.392006370335186, + "tokens_seen": 2293170176 + }, + { + "epoch": 0.39, + "learning_rate": 6.16273471352913e-05, + "loss": 2.5498, + "theoretical_loss": 3.3919906370893247, + "tokens_seen": 2293301248 + }, + { + "epoch": 0.39, + "learning_rate": 6.161932274113305e-05, + "loss": 2.439, + "theoretical_loss": 3.3919749049944263, + "tokens_seen": 2293432320 + }, + { + "epoch": 0.39, + "learning_rate": 6.161129834697481e-05, + "loss": 2.4433, + "theoretical_loss": 3.3919591740503408, + "tokens_seen": 2293563392 + }, + { + "epoch": 0.39, + "learning_rate": 6.160327395281657e-05, + "loss": 2.5527, + "theoretical_loss": 3.391943444256918, + "tokens_seen": 2293694464 + }, + { + "epoch": 0.39, + "learning_rate": 6.159524955865832e-05, + "loss": 2.5234, + "theoretical_loss": 3.3919277156140075, + "tokens_seen": 2293825536 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1265892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.169233560562134, + "objective/train/theoretical_loss": 3.391919851723948, + "objective/train/tokens_used": 664350176, + "theoretical_loss": 3.391919851723948, + "tokens_seen": 2293891072 + }, + { + "epoch": 0.39, + "learning_rate": 6.158722516450009e-05, + "loss": 2.5547, + "theoretical_loss": 3.3919119881214606, + "tokens_seen": 2293956608 + }, + { + "epoch": 0.39, + "learning_rate": 6.157920077034184e-05, + "loss": 2.4961, + "theoretical_loss": 3.3918962617791264, + "tokens_seen": 2294087680 + }, + { + "epoch": 0.39, + "learning_rate": 6.157117637618361e-05, + "loss": 2.6464, + "theoretical_loss": 3.3918805365868554, + "tokens_seen": 2294218752 + }, + { + "epoch": 0.39, + "learning_rate": 6.156315198202536e-05, + "loss": 2.5783, + "theoretical_loss": 3.391864812544498, + "tokens_seen": 2294349824 + }, + { + "epoch": 0.39, + "learning_rate": 6.155512758786713e-05, + "loss": 2.7335, + "theoretical_loss": 3.3918490896519042, + "tokens_seen": 2294480896 + }, + { + "epoch": 0.39, + "learning_rate": 6.154710319370888e-05, + "loss": 2.5165, + "theoretical_loss": 3.3918333679089243, + "tokens_seen": 2294611968 + }, + { + "epoch": 0.39, + "learning_rate": 6.153907879955065e-05, + "loss": 2.5589, + "theoretical_loss": 3.391817647315409, + "tokens_seen": 2294743040 + }, + { + "epoch": 0.39, + "learning_rate": 6.15310544053924e-05, + "loss": 2.4536, + "theoretical_loss": 3.391801927871208, + "tokens_seen": 2294874112 + }, + { + "epoch": 0.39, + "learning_rate": 6.152303001123415e-05, + "loss": 2.4815, + "theoretical_loss": 3.3917862095761717, + "tokens_seen": 2295005184 + }, + { + "epoch": 0.39, + "learning_rate": 6.151500561707592e-05, + "loss": 2.6744, + "theoretical_loss": 3.3917704924301506, + "tokens_seen": 2295136256 + }, + { + "epoch": 0.39, + "learning_rate": 6.150698122291767e-05, + "loss": 2.4871, + "theoretical_loss": 3.3917547764329954, + "tokens_seen": 2295267328 + }, + { + "epoch": 0.39, + "learning_rate": 6.149895682875944e-05, + "loss": 2.6185, + "theoretical_loss": 3.3917390615845564, + "tokens_seen": 2295398400 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1267119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5357112884521484, + "objective/train/theoretical_loss": 3.3917233478846835, + "objective/train/tokens_used": 665988576, + "theoretical_loss": 3.3917233478846835, + "tokens_seen": 2295529472 + }, + { + "epoch": 0.39, + "learning_rate": 6.149093243460119e-05, + "loss": 2.4802, + "theoretical_loss": 3.3917233478846835, + "tokens_seen": 2295529472 + }, + { + "epoch": 0.39, + "learning_rate": 6.148290804044296e-05, + "loss": 2.473, + "theoretical_loss": 3.3917076353332276, + "tokens_seen": 2295660544 + }, + { + "epoch": 0.39, + "learning_rate": 6.147488364628471e-05, + "loss": 2.5155, + "theoretical_loss": 3.39169192393004, + "tokens_seen": 2295791616 + }, + { + "epoch": 0.39, + "learning_rate": 6.146685925212647e-05, + "loss": 2.5867, + "theoretical_loss": 3.391676213674969, + "tokens_seen": 2295922688 + }, + { + "epoch": 0.39, + "learning_rate": 6.145883485796823e-05, + "loss": 2.5223, + "theoretical_loss": 3.391660504567868, + "tokens_seen": 2296053760 + }, + { + "epoch": 0.39, + "learning_rate": 6.145081046380999e-05, + "loss": 2.5643, + "theoretical_loss": 3.391644796608585, + "tokens_seen": 2296184832 + }, + { + "epoch": 0.39, + "learning_rate": 6.144278606965175e-05, + "loss": 2.6168, + "theoretical_loss": 3.3916290897969725, + "tokens_seen": 2296315904 + }, + { + "epoch": 0.39, + "learning_rate": 6.14347616754935e-05, + "loss": 2.5584, + "theoretical_loss": 3.39161338413288, + "tokens_seen": 2296446976 + }, + { + "epoch": 0.39, + "learning_rate": 6.142673728133527e-05, + "loss": 2.5964, + "theoretical_loss": 3.3915976796161593, + "tokens_seen": 2296578048 + }, + { + "epoch": 0.39, + "learning_rate": 6.141871288717702e-05, + "loss": 2.495, + "theoretical_loss": 3.39158197624666, + "tokens_seen": 2296709120 + }, + { + "epoch": 0.39, + "learning_rate": 6.141068849301878e-05, + "loss": 2.5428, + "theoretical_loss": 3.391566274024233, + "tokens_seen": 2296840192 + }, + { + "epoch": 0.39, + "learning_rate": 6.140266409886054e-05, + "loss": 2.6739, + "theoretical_loss": 3.39155057294873, + "tokens_seen": 2296971264 + }, + { + "epoch": 0.39, + "learning_rate": 6.13946397047023e-05, + "loss": 2.479, + "theoretical_loss": 3.3915348730200003, + "tokens_seen": 2297102336 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1267727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8644251823425293, + "objective/train/theoretical_loss": 3.3915270234856294, + "objective/train/tokens_used": 667626976, + "theoretical_loss": 3.3915270234856294, + "tokens_seen": 2297167872 + }, + { + "epoch": 0.39, + "learning_rate": 6.138661531054406e-05, + "loss": 2.6772, + "theoretical_loss": 3.3915191742378963, + "tokens_seen": 2297233408 + }, + { + "epoch": 0.39, + "learning_rate": 6.137859091638582e-05, + "loss": 2.6287, + "theoretical_loss": 3.3915034766022676, + "tokens_seen": 2297364480 + }, + { + "epoch": 0.39, + "learning_rate": 6.137056652222758e-05, + "loss": 2.5341, + "theoretical_loss": 3.3914877801129655, + "tokens_seen": 2297495552 + }, + { + "epoch": 0.39, + "learning_rate": 6.136254212806934e-05, + "loss": 2.6252, + "theoretical_loss": 3.391472084769841, + "tokens_seen": 2297626624 + }, + { + "epoch": 0.39, + "learning_rate": 6.135451773391109e-05, + "loss": 2.434, + "theoretical_loss": 3.391456390572745, + "tokens_seen": 2297757696 + }, + { + "epoch": 0.39, + "learning_rate": 6.134649333975286e-05, + "loss": 2.5343, + "theoretical_loss": 3.3914406975215283, + "tokens_seen": 2297888768 + }, + { + "epoch": 0.39, + "learning_rate": 6.133846894559461e-05, + "loss": 2.6846, + "theoretical_loss": 3.391425005616042, + "tokens_seen": 2298019840 + }, + { + "epoch": 0.39, + "learning_rate": 6.133044455143638e-05, + "loss": 2.6056, + "theoretical_loss": 3.3914093148561375, + "tokens_seen": 2298150912 + }, + { + "epoch": 0.39, + "learning_rate": 6.132242015727813e-05, + "loss": 2.6407, + "theoretical_loss": 3.391393625241665, + "tokens_seen": 2298281984 + }, + { + "epoch": 0.39, + "learning_rate": 6.13143957631199e-05, + "loss": 2.563, + "theoretical_loss": 3.391377936772476, + "tokens_seen": 2298413056 + }, + { + "epoch": 0.39, + "learning_rate": 6.130637136896165e-05, + "loss": 2.4517, + "theoretical_loss": 3.391362249448422, + "tokens_seen": 2298544128 + }, + { + "epoch": 0.39, + "learning_rate": 6.12983469748034e-05, + "loss": 2.4836, + "theoretical_loss": 3.3913465632693534, + "tokens_seen": 2298675200 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1268805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6418638229370117, + "objective/train/theoretical_loss": 3.3913308782351215, + "objective/train/tokens_used": 669265376, + "theoretical_loss": 3.3913308782351215, + "tokens_seen": 2298806272 + }, + { + "epoch": 0.39, + "learning_rate": 6.129032258064517e-05, + "loss": 2.4528, + "theoretical_loss": 3.3913308782351215, + "tokens_seen": 2298806272 + }, + { + "epoch": 0.39, + "learning_rate": 6.128229818648692e-05, + "loss": 2.4908, + "theoretical_loss": 3.391315194345578, + "tokens_seen": 2298937344 + }, + { + "epoch": 0.39, + "learning_rate": 6.127427379232869e-05, + "loss": 2.5891, + "theoretical_loss": 3.3912995116005735, + "tokens_seen": 2299068416 + }, + { + "epoch": 0.39, + "learning_rate": 6.126624939817044e-05, + "loss": 2.568, + "theoretical_loss": 3.3912838299999595, + "tokens_seen": 2299199488 + }, + { + "epoch": 0.39, + "learning_rate": 6.125822500401221e-05, + "loss": 2.4487, + "theoretical_loss": 3.391268149543587, + "tokens_seen": 2299330560 + }, + { + "epoch": 0.39, + "learning_rate": 6.125020060985396e-05, + "loss": 2.4322, + "theoretical_loss": 3.391252470231308, + "tokens_seen": 2299461632 + }, + { + "epoch": 0.39, + "learning_rate": 6.124217621569572e-05, + "loss": 2.5052, + "theoretical_loss": 3.391236792062973, + "tokens_seen": 2299592704 + }, + { + "epoch": 0.39, + "learning_rate": 6.123415182153748e-05, + "loss": 2.5633, + "theoretical_loss": 3.391221115038434, + "tokens_seen": 2299723776 + }, + { + "epoch": 0.39, + "learning_rate": 6.122612742737924e-05, + "loss": 2.6071, + "theoretical_loss": 3.3912054391575417, + "tokens_seen": 2299854848 + }, + { + "epoch": 0.39, + "learning_rate": 6.1218103033221e-05, + "loss": 2.3632, + "theoretical_loss": 3.3911897644201483, + "tokens_seen": 2299985920 + }, + { + "epoch": 0.39, + "learning_rate": 6.121007863906276e-05, + "loss": 2.5011, + "theoretical_loss": 3.3911740908261043, + "tokens_seen": 2300116992 + }, + { + "epoch": 0.39, + "learning_rate": 6.120205424490452e-05, + "loss": 2.4519, + "theoretical_loss": 3.3911584183752623, + "tokens_seen": 2300248064 + }, + { + "epoch": 0.39, + "learning_rate": 6.119402985074628e-05, + "loss": 2.6335, + "theoretical_loss": 3.3911427470674727, + "tokens_seen": 2300379136 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1269425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5342440605163574, + "objective/train/theoretical_loss": 3.391134911842176, + "objective/train/tokens_used": 670903776, + "theoretical_loss": 3.391134911842176, + "tokens_seen": 2300444672 + }, + { + "epoch": 0.39, + "learning_rate": 6.118600545658804e-05, + "loss": 2.5486, + "theoretical_loss": 3.3911270769025874, + "tokens_seen": 2300510208 + }, + { + "epoch": 0.39, + "learning_rate": 6.11779810624298e-05, + "loss": 2.5324, + "theoretical_loss": 3.391111407880458, + "tokens_seen": 2300641280 + }, + { + "epoch": 0.39, + "learning_rate": 6.116995666827155e-05, + "loss": 2.4238, + "theoretical_loss": 3.391095740000936, + "tokens_seen": 2300772352 + }, + { + "epoch": 0.39, + "learning_rate": 6.116193227411331e-05, + "loss": 2.5696, + "theoretical_loss": 3.3910800732638733, + "tokens_seen": 2300903424 + }, + { + "epoch": 0.39, + "learning_rate": 6.115390787995507e-05, + "loss": 2.4227, + "theoretical_loss": 3.3910644076691208, + "tokens_seen": 2301034496 + }, + { + "epoch": 0.39, + "learning_rate": 6.114588348579683e-05, + "loss": 2.512, + "theoretical_loss": 3.391048743216531, + "tokens_seen": 2301165568 + }, + { + "epoch": 0.39, + "learning_rate": 6.113785909163859e-05, + "loss": 2.5283, + "theoretical_loss": 3.391033079905955, + "tokens_seen": 2301296640 + }, + { + "epoch": 0.39, + "learning_rate": 6.112983469748035e-05, + "loss": 2.4977, + "theoretical_loss": 3.3910174177372445, + "tokens_seen": 2301427712 + }, + { + "epoch": 0.39, + "learning_rate": 6.112181030332211e-05, + "loss": 2.4592, + "theoretical_loss": 3.391001756710252, + "tokens_seen": 2301558784 + }, + { + "epoch": 0.39, + "learning_rate": 6.111378590916386e-05, + "loss": 2.5168, + "theoretical_loss": 3.390986096824828, + "tokens_seen": 2301689856 + }, + { + "epoch": 0.4, + "learning_rate": 6.110576151500563e-05, + "loss": 2.524, + "theoretical_loss": 3.390970438080825, + "tokens_seen": 2301820928 + }, + { + "epoch": 0.4, + "learning_rate": 6.109773712084738e-05, + "loss": 2.5812, + "theoretical_loss": 3.3909547804780953, + "tokens_seen": 2301952000 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1270779, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4318535327911377, + "objective/train/theoretical_loss": 3.39093912401649, + "objective/train/tokens_used": 672542176, + "theoretical_loss": 3.39093912401649, + "tokens_seen": 2302083072 + }, + { + "epoch": 0.4, + "learning_rate": 6.108971272668915e-05, + "loss": 2.5938, + "theoretical_loss": 3.39093912401649, + "tokens_seen": 2302083072 + }, + { + "epoch": 0.4, + "learning_rate": 6.10816883325309e-05, + "loss": 2.4497, + "theoretical_loss": 3.3909234686958616, + "tokens_seen": 2302214144 + }, + { + "epoch": 0.4, + "learning_rate": 6.107366393837267e-05, + "loss": 2.3694, + "theoretical_loss": 3.390907814516061, + "tokens_seen": 2302345216 + }, + { + "epoch": 0.4, + "learning_rate": 6.106563954421442e-05, + "loss": 2.5692, + "theoretical_loss": 3.3908921614769407, + "tokens_seen": 2302476288 + }, + { + "epoch": 0.4, + "learning_rate": 6.105761515005617e-05, + "loss": 2.4478, + "theoretical_loss": 3.390876509578353, + "tokens_seen": 2302607360 + }, + { + "epoch": 0.4, + "learning_rate": 6.104959075589794e-05, + "loss": 2.4334, + "theoretical_loss": 3.390860858820149, + "tokens_seen": 2302738432 + }, + { + "epoch": 0.4, + "learning_rate": 6.104156636173969e-05, + "loss": 2.475, + "theoretical_loss": 3.390845209202182, + "tokens_seen": 2302869504 + }, + { + "epoch": 0.4, + "learning_rate": 6.103354196758145e-05, + "loss": 2.6671, + "theoretical_loss": 3.390829560724303, + "tokens_seen": 2303000576 + }, + { + "epoch": 0.4, + "learning_rate": 6.1025517573423206e-05, + "loss": 2.51, + "theoretical_loss": 3.3908139133863644, + "tokens_seen": 2303131648 + }, + { + "epoch": 0.4, + "learning_rate": 6.101749317926497e-05, + "loss": 2.5443, + "theoretical_loss": 3.390798267188218, + "tokens_seen": 2303262720 + }, + { + "epoch": 0.4, + "learning_rate": 6.1009468785106726e-05, + "loss": 2.4232, + "theoretical_loss": 3.3907826221297164, + "tokens_seen": 2303393792 + }, + { + "epoch": 0.4, + "learning_rate": 6.100144439094848e-05, + "loss": 2.5159, + "theoretical_loss": 3.3907669782107113, + "tokens_seen": 2303524864 + }, + { + "epoch": 0.4, + "learning_rate": 6.0993419996790246e-05, + "loss": 2.6357, + "theoretical_loss": 3.3907513354310557, + "tokens_seen": 2303655936 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1271371, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2652809619903564, + "objective/train/theoretical_loss": 3.3907435144684372, + "objective/train/tokens_used": 674180576, + "theoretical_loss": 3.3907435144684372, + "tokens_seen": 2303721472 + }, + { + "epoch": 0.4, + "learning_rate": 6.0985395602632e-05, + "loss": 2.4737, + "theoretical_loss": 3.3907356937906004, + "tokens_seen": 2303787008 + }, + { + "epoch": 0.4, + "learning_rate": 6.0977371208473766e-05, + "loss": 2.565, + "theoretical_loss": 3.390720053289199, + "tokens_seen": 2303918080 + }, + { + "epoch": 0.4, + "learning_rate": 6.096934681431552e-05, + "loss": 2.257, + "theoretical_loss": 3.390704413926703, + "tokens_seen": 2304049152 + }, + { + "epoch": 0.4, + "learning_rate": 6.0961322420157285e-05, + "loss": 2.5282, + "theoretical_loss": 3.390688775702965, + "tokens_seen": 2304180224 + }, + { + "epoch": 0.4, + "learning_rate": 6.095329802599904e-05, + "loss": 2.5341, + "theoretical_loss": 3.390673138617837, + "tokens_seen": 2304311296 + }, + { + "epoch": 0.4, + "learning_rate": 6.094527363184079e-05, + "loss": 2.5535, + "theoretical_loss": 3.3906575026711714, + "tokens_seen": 2304442368 + }, + { + "epoch": 0.4, + "learning_rate": 6.093724923768256e-05, + "loss": 2.5071, + "theoretical_loss": 3.3906418678628207, + "tokens_seen": 2304573440 + }, + { + "epoch": 0.4, + "learning_rate": 6.092922484352431e-05, + "loss": 2.5005, + "theoretical_loss": 3.3906262341926374, + "tokens_seen": 2304704512 + }, + { + "epoch": 0.4, + "learning_rate": 6.092120044936608e-05, + "loss": 2.4322, + "theoretical_loss": 3.390610601660474, + "tokens_seen": 2304835584 + }, + { + "epoch": 0.4, + "learning_rate": 6.091317605520783e-05, + "loss": 2.4756, + "theoretical_loss": 3.3905949702661826, + "tokens_seen": 2304966656 + }, + { + "epoch": 0.4, + "learning_rate": 6.09051516610496e-05, + "loss": 2.3177, + "theoretical_loss": 3.390579340009616, + "tokens_seen": 2305097728 + }, + { + "epoch": 0.4, + "learning_rate": 6.089712726689135e-05, + "loss": 2.4599, + "theoretical_loss": 3.390563710890626, + "tokens_seen": 2305228800 + }, + { + "debugging/Self-BLEU-5": 0.43298113666112864, + "debugging/distinct-1-grams": 0.7776999941413213, + "debugging/distinct-2-grams": 0.9535998885349362, + "debugging/entropy-1-grams": 5.6285201862472185, + "debugging/entropy-2-grams": 6.29126139825565, + "debugging/length": 545.75, + "debugging/num_segments": 8, + "debugging/score": 0.006776578879058141, + "debugging/score_std": 0.003879118421535622, + "epoch": 0.4, + "objective/train/docs_used": 1272482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.490351676940918, + "objective/train/theoretical_loss": 3.390548082909066, + "objective/train/tokens_used": 675818976, + "theoretical_loss": 3.390548082909066, + "tokens_seen": 2305359872 + }, + { + "epoch": 0.4, + "learning_rate": 6.088910287273312e-05, + "loss": 2.5077, + "theoretical_loss": 3.390548082909066, + "tokens_seen": 2305359872 + }, + { + "epoch": 0.4, + "learning_rate": 6.088107847857487e-05, + "loss": 2.4906, + "theoretical_loss": 3.3905324560647885, + "tokens_seen": 2305490944 + }, + { + "epoch": 0.4, + "learning_rate": 6.0873054084416624e-05, + "loss": 2.5881, + "theoretical_loss": 3.3905168303576456, + "tokens_seen": 2305622016 + }, + { + "epoch": 0.4, + "learning_rate": 6.086502969025839e-05, + "loss": 2.6183, + "theoretical_loss": 3.39050120578749, + "tokens_seen": 2305753088 + }, + { + "epoch": 0.4, + "learning_rate": 6.0857005296100144e-05, + "loss": 2.5572, + "theoretical_loss": 3.3904855823541746, + "tokens_seen": 2305884160 + }, + { + "epoch": 0.4, + "learning_rate": 6.084898090194191e-05, + "loss": 2.5107, + "theoretical_loss": 3.390469960057552, + "tokens_seen": 2306015232 + }, + { + "epoch": 0.4, + "learning_rate": 6.0840956507783664e-05, + "loss": 2.6371, + "theoretical_loss": 3.3904543388974746, + "tokens_seen": 2306146304 + }, + { + "epoch": 0.4, + "learning_rate": 6.083293211362543e-05, + "loss": 2.6085, + "theoretical_loss": 3.3904387188737957, + "tokens_seen": 2306277376 + }, + { + "epoch": 0.4, + "learning_rate": 6.0824907719467183e-05, + "loss": 2.4177, + "theoretical_loss": 3.390423099986368, + "tokens_seen": 2306408448 + }, + { + "epoch": 0.4, + "learning_rate": 6.0816883325308937e-05, + "loss": 2.5985, + "theoretical_loss": 3.390407482235043, + "tokens_seen": 2306539520 + }, + { + "epoch": 0.4, + "learning_rate": 6.08088589311507e-05, + "loss": 2.5685, + "theoretical_loss": 3.390391865619675, + "tokens_seen": 2306670592 + }, + { + "epoch": 0.4, + "learning_rate": 6.0800834536992456e-05, + "loss": 2.7307, + "theoretical_loss": 3.3903762501401165, + "tokens_seen": 2306801664 + }, + { + "epoch": 0.4, + "learning_rate": 6.079281014283422e-05, + "loss": 2.4756, + "theoretical_loss": 3.3903606357962204, + "tokens_seen": 2306932736 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1273129, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.518545627593994, + "objective/train/theoretical_loss": 3.3903528290500993, + "objective/train/tokens_used": 677457376, + "theoretical_loss": 3.3903528290500993, + "tokens_seen": 2306998272 + }, + { + "epoch": 0.4, + "learning_rate": 6.0784785748675976e-05, + "loss": 2.3769, + "theoretical_loss": 3.390345022587839, + "tokens_seen": 2307063808 + }, + { + "epoch": 0.4, + "learning_rate": 6.077676135451774e-05, + "loss": 2.5298, + "theoretical_loss": 3.390329410514826, + "tokens_seen": 2307194880 + }, + { + "epoch": 0.4, + "learning_rate": 6.0768736960359496e-05, + "loss": 2.4769, + "theoretical_loss": 3.3903137995770334, + "tokens_seen": 2307325952 + }, + { + "epoch": 0.4, + "learning_rate": 6.076071256620125e-05, + "loss": 2.7667, + "theoretical_loss": 3.390298189774315, + "tokens_seen": 2307457024 + }, + { + "epoch": 0.4, + "learning_rate": 6.0752688172043016e-05, + "loss": 2.5325, + "theoretical_loss": 3.390282581106524, + "tokens_seen": 2307588096 + }, + { + "epoch": 0.4, + "learning_rate": 6.074466377788477e-05, + "loss": 2.5, + "theoretical_loss": 3.3902669735735125, + "tokens_seen": 2307719168 + }, + { + "epoch": 0.4, + "learning_rate": 6.0736639383726536e-05, + "loss": 2.5798, + "theoretical_loss": 3.390251367175134, + "tokens_seen": 2307850240 + }, + { + "epoch": 0.4, + "learning_rate": 6.072861498956829e-05, + "loss": 2.5994, + "theoretical_loss": 3.3902357619112413, + "tokens_seen": 2307981312 + }, + { + "epoch": 0.4, + "learning_rate": 6.0720590595410055e-05, + "loss": 2.4868, + "theoretical_loss": 3.3902201577816884, + "tokens_seen": 2308112384 + }, + { + "epoch": 0.4, + "learning_rate": 6.071256620125181e-05, + "loss": 2.5244, + "theoretical_loss": 3.3902045547863273, + "tokens_seen": 2308243456 + }, + { + "epoch": 0.4, + "learning_rate": 6.070454180709356e-05, + "loss": 2.4328, + "theoretical_loss": 3.390188952925012, + "tokens_seen": 2308374528 + }, + { + "epoch": 0.4, + "learning_rate": 6.069651741293533e-05, + "loss": 2.6042, + "theoretical_loss": 3.3901733521975954, + "tokens_seen": 2308505600 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1273675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2433297634124756, + "objective/train/theoretical_loss": 3.3901577526039306, + "objective/train/tokens_used": 679095776, + "theoretical_loss": 3.3901577526039306, + "tokens_seen": 2308636672 + }, + { + "epoch": 0.4, + "learning_rate": 6.068849301877708e-05, + "loss": 2.455, + "theoretical_loss": 3.3901577526039306, + "tokens_seen": 2308636672 + }, + { + "epoch": 0.4, + "learning_rate": 6.068046862461885e-05, + "loss": 2.4181, + "theoretical_loss": 3.390142154143871, + "tokens_seen": 2308767744 + }, + { + "epoch": 0.4, + "learning_rate": 6.06724442304606e-05, + "loss": 2.4947, + "theoretical_loss": 3.39012655681727, + "tokens_seen": 2308898816 + }, + { + "epoch": 0.4, + "learning_rate": 6.066441983630237e-05, + "loss": 2.3941, + "theoretical_loss": 3.3901109606239808, + "tokens_seen": 2309029888 + }, + { + "epoch": 0.4, + "learning_rate": 6.065639544214412e-05, + "loss": 2.4661, + "theoretical_loss": 3.390095365563856, + "tokens_seen": 2309160960 + }, + { + "epoch": 0.4, + "learning_rate": 6.0648371047985874e-05, + "loss": 2.3421, + "theoretical_loss": 3.3900797716367506, + "tokens_seen": 2309292032 + }, + { + "epoch": 0.4, + "learning_rate": 6.064034665382764e-05, + "loss": 2.5064, + "theoretical_loss": 3.390064178842516, + "tokens_seen": 2309423104 + }, + { + "epoch": 0.4, + "learning_rate": 6.0632322259669394e-05, + "loss": 2.6014, + "theoretical_loss": 3.390048587181007, + "tokens_seen": 2309554176 + }, + { + "epoch": 0.4, + "learning_rate": 6.062429786551116e-05, + "loss": 2.5406, + "theoretical_loss": 3.390032996652077, + "tokens_seen": 2309685248 + }, + { + "epoch": 0.4, + "learning_rate": 6.0616273471352914e-05, + "loss": 2.6872, + "theoretical_loss": 3.390017407255579, + "tokens_seen": 2309816320 + }, + { + "epoch": 0.4, + "learning_rate": 6.060824907719468e-05, + "loss": 2.5322, + "theoretical_loss": 3.390001818991366, + "tokens_seen": 2309947392 + }, + { + "epoch": 0.4, + "learning_rate": 6.0600224683036434e-05, + "loss": 2.4703, + "theoretical_loss": 3.3899862318592926, + "tokens_seen": 2310078464 + }, + { + "epoch": 0.4, + "learning_rate": 6.05922002888782e-05, + "loss": 2.5021, + "theoretical_loss": 3.389970645859212, + "tokens_seen": 2310209536 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1274788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4272327423095703, + "objective/train/theoretical_loss": 3.389962853283623, + "objective/train/tokens_used": 680734176, + "theoretical_loss": 3.389962853283623, + "tokens_seen": 2310275072 + }, + { + "epoch": 0.4, + "learning_rate": 6.0584175894719954e-05, + "loss": 2.4588, + "theoretical_loss": 3.389955060990977, + "tokens_seen": 2310340608 + }, + { + "epoch": 0.4, + "learning_rate": 6.057615150056171e-05, + "loss": 2.6382, + "theoretical_loss": 3.3899394772544422, + "tokens_seen": 2310471680 + }, + { + "epoch": 0.4, + "learning_rate": 6.056812710640347e-05, + "loss": 2.4812, + "theoretical_loss": 3.3899238946494608, + "tokens_seen": 2310602752 + }, + { + "epoch": 0.4, + "learning_rate": 6.0560102712245226e-05, + "loss": 2.6145, + "theoretical_loss": 3.3899083131758867, + "tokens_seen": 2310733824 + }, + { + "epoch": 0.4, + "learning_rate": 6.055207831808699e-05, + "loss": 2.4977, + "theoretical_loss": 3.389892732833573, + "tokens_seen": 2310864896 + }, + { + "epoch": 0.4, + "learning_rate": 6.0544053923928746e-05, + "loss": 2.4079, + "theoretical_loss": 3.389877153622374, + "tokens_seen": 2310995968 + }, + { + "epoch": 0.4, + "learning_rate": 6.053602952977051e-05, + "loss": 2.6061, + "theoretical_loss": 3.3898615755421435, + "tokens_seen": 2311127040 + }, + { + "epoch": 0.4, + "learning_rate": 6.0528005135612266e-05, + "loss": 2.4405, + "theoretical_loss": 3.389845998592735, + "tokens_seen": 2311258112 + }, + { + "epoch": 0.4, + "learning_rate": 6.051998074145402e-05, + "loss": 2.5403, + "theoretical_loss": 3.3898304227740024, + "tokens_seen": 2311389184 + }, + { + "epoch": 0.4, + "learning_rate": 6.0511956347295786e-05, + "loss": 2.5089, + "theoretical_loss": 3.389814848085799, + "tokens_seen": 2311520256 + }, + { + "epoch": 0.4, + "learning_rate": 6.050393195313754e-05, + "loss": 2.525, + "theoretical_loss": 3.389799274527979, + "tokens_seen": 2311651328 + }, + { + "epoch": 0.4, + "learning_rate": 6.0495907558979306e-05, + "loss": 2.4141, + "theoretical_loss": 3.3897837021003965, + "tokens_seen": 2311782400 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1275747, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.075777053833008, + "objective/train/theoretical_loss": 3.3897681308029055, + "objective/train/tokens_used": 682372576, + "theoretical_loss": 3.3897681308029055, + "tokens_seen": 2311913472 + }, + { + "epoch": 0.4, + "learning_rate": 6.048788316482106e-05, + "loss": 2.4512, + "theoretical_loss": 3.3897681308029055, + "tokens_seen": 2311913472 + }, + { + "epoch": 0.4, + "learning_rate": 6.0479858770662826e-05, + "loss": 2.5678, + "theoretical_loss": 3.389752560635359, + "tokens_seen": 2312044544 + }, + { + "epoch": 0.4, + "learning_rate": 6.047183437650458e-05, + "loss": 2.6394, + "theoretical_loss": 3.389736991597612, + "tokens_seen": 2312175616 + }, + { + "epoch": 0.4, + "learning_rate": 6.046380998234633e-05, + "loss": 2.6075, + "theoretical_loss": 3.3897214236895183, + "tokens_seen": 2312306688 + }, + { + "epoch": 0.4, + "learning_rate": 6.04557855881881e-05, + "loss": 2.518, + "theoretical_loss": 3.3897058569109317, + "tokens_seen": 2312437760 + }, + { + "epoch": 0.4, + "learning_rate": 6.044776119402985e-05, + "loss": 2.4914, + "theoretical_loss": 3.3896902912617057, + "tokens_seen": 2312568832 + }, + { + "epoch": 0.4, + "learning_rate": 6.043973679987162e-05, + "loss": 2.6291, + "theoretical_loss": 3.3896747267416956, + "tokens_seen": 2312699904 + }, + { + "epoch": 0.4, + "learning_rate": 6.043171240571337e-05, + "loss": 2.5727, + "theoretical_loss": 3.3896591633507542, + "tokens_seen": 2312830976 + }, + { + "epoch": 0.4, + "learning_rate": 6.042368801155514e-05, + "loss": 2.4357, + "theoretical_loss": 3.3896436010887365, + "tokens_seen": 2312962048 + }, + { + "epoch": 0.4, + "learning_rate": 6.041566361739689e-05, + "loss": 2.6064, + "theoretical_loss": 3.3896280399554963, + "tokens_seen": 2313093120 + }, + { + "epoch": 0.4, + "learning_rate": 6.0407639223238644e-05, + "loss": 2.4785, + "theoretical_loss": 3.3896124799508875, + "tokens_seen": 2313224192 + }, + { + "epoch": 0.4, + "learning_rate": 6.039961482908041e-05, + "loss": 2.5568, + "theoretical_loss": 3.389596921074765, + "tokens_seen": 2313355264 + }, + { + "epoch": 0.4, + "learning_rate": 6.0391590434922164e-05, + "loss": 2.6242, + "theoretical_loss": 3.3895813633269825, + "tokens_seen": 2313486336 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1276293, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2193737030029297, + "objective/train/theoretical_loss": 3.389573584876173, + "objective/train/tokens_used": 684010976, + "theoretical_loss": 3.389573584876173, + "tokens_seen": 2313551872 + }, + { + "epoch": 0.4, + "learning_rate": 6.038356604076393e-05, + "loss": 2.4727, + "theoretical_loss": 3.3895658067073944, + "tokens_seen": 2313617408 + }, + { + "epoch": 0.4, + "learning_rate": 6.0375541646605684e-05, + "loss": 2.6057, + "theoretical_loss": 3.389550251215855, + "tokens_seen": 2313748480 + }, + { + "epoch": 0.4, + "learning_rate": 6.036751725244745e-05, + "loss": 2.5193, + "theoretical_loss": 3.389534696852219, + "tokens_seen": 2313879552 + }, + { + "epoch": 0.4, + "learning_rate": 6.0359492858289204e-05, + "loss": 2.7326, + "theoretical_loss": 3.3895191436163397, + "tokens_seen": 2314010624 + }, + { + "epoch": 0.4, + "learning_rate": 6.035146846413096e-05, + "loss": 2.7271, + "theoretical_loss": 3.3895035915080722, + "tokens_seen": 2314141696 + }, + { + "epoch": 0.4, + "learning_rate": 6.0343444069972724e-05, + "loss": 2.7267, + "theoretical_loss": 3.389488040527271, + "tokens_seen": 2314272768 + }, + { + "epoch": 0.4, + "learning_rate": 6.033541967581448e-05, + "loss": 2.6486, + "theoretical_loss": 3.38947249067379, + "tokens_seen": 2314403840 + }, + { + "epoch": 0.4, + "learning_rate": 6.0327395281656243e-05, + "loss": 2.4973, + "theoretical_loss": 3.3894569419474836, + "tokens_seen": 2314534912 + }, + { + "epoch": 0.4, + "learning_rate": 6.0319370887497997e-05, + "loss": 2.4632, + "theoretical_loss": 3.389441394348207, + "tokens_seen": 2314665984 + }, + { + "epoch": 0.4, + "learning_rate": 6.0311346493339756e-05, + "loss": 2.6751, + "theoretical_loss": 3.3894258478758146, + "tokens_seen": 2314797056 + }, + { + "epoch": 0.4, + "learning_rate": 6.0303322099181516e-05, + "loss": 2.436, + "theoretical_loss": 3.3894103025301603, + "tokens_seen": 2314928128 + }, + { + "epoch": 0.4, + "learning_rate": 6.029529770502327e-05, + "loss": 2.5694, + "theoretical_loss": 3.3893947583110986, + "tokens_seen": 2315059200 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1277268, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.424570322036743, + "objective/train/theoretical_loss": 3.3893792152184847, + "objective/train/tokens_used": 685649376, + "theoretical_loss": 3.3893792152184847, + "tokens_seen": 2315190272 + }, + { + "epoch": 0.4, + "learning_rate": 6.0287273310865036e-05, + "loss": 2.5064, + "theoretical_loss": 3.3893792152184847, + "tokens_seen": 2315190272 + }, + { + "epoch": 0.4, + "learning_rate": 6.027924891670679e-05, + "loss": 2.6438, + "theoretical_loss": 3.3893636732521735, + "tokens_seen": 2315321344 + }, + { + "epoch": 0.4, + "learning_rate": 6.0271224522548556e-05, + "loss": 2.7128, + "theoretical_loss": 3.3893481324120183, + "tokens_seen": 2315452416 + }, + { + "epoch": 0.4, + "learning_rate": 6.026320012839031e-05, + "loss": 2.7182, + "theoretical_loss": 3.389332592697875, + "tokens_seen": 2315583488 + }, + { + "epoch": 0.4, + "learning_rate": 6.025517573423207e-05, + "loss": 2.5202, + "theoretical_loss": 3.3893170541095974, + "tokens_seen": 2315714560 + }, + { + "epoch": 0.4, + "learning_rate": 6.024715134007383e-05, + "loss": 2.644, + "theoretical_loss": 3.389301516647041, + "tokens_seen": 2315845632 + }, + { + "epoch": 0.4, + "learning_rate": 6.023912694591559e-05, + "loss": 2.4893, + "theoretical_loss": 3.38928598031006, + "tokens_seen": 2315976704 + }, + { + "epoch": 0.4, + "learning_rate": 6.023110255175735e-05, + "loss": 2.4382, + "theoretical_loss": 3.389270445098509, + "tokens_seen": 2316107776 + }, + { + "epoch": 0.4, + "learning_rate": 6.02230781575991e-05, + "loss": 2.5753, + "theoretical_loss": 3.3892549110122436, + "tokens_seen": 2316238848 + }, + { + "epoch": 0.4, + "learning_rate": 6.021505376344086e-05, + "loss": 2.4659, + "theoretical_loss": 3.3892393780511174, + "tokens_seen": 2316369920 + }, + { + "epoch": 0.4, + "learning_rate": 6.020702936928262e-05, + "loss": 2.6214, + "theoretical_loss": 3.389223846214987, + "tokens_seen": 2316500992 + }, + { + "epoch": 0.4, + "learning_rate": 6.019900497512438e-05, + "loss": 2.6186, + "theoretical_loss": 3.389208315503706, + "tokens_seen": 2316632064 + }, + { + "epoch": 0.4, + "learning_rate": 6.019098058096614e-05, + "loss": 2.5551, + "theoretical_loss": 3.3891927859171287, + "tokens_seen": 2316763136 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1277883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.049539804458618, + "objective/train/theoretical_loss": 3.3891850215455595, + "objective/train/tokens_used": 687287776, + "theoretical_loss": 3.3891850215455595, + "tokens_seen": 2316828672 + }, + { + "epoch": 0.4, + "learning_rate": 6.01829561868079e-05, + "loss": 2.7491, + "theoretical_loss": 3.3891772574551116, + "tokens_seen": 2316894208 + }, + { + "epoch": 0.4, + "learning_rate": 6.017493179264966e-05, + "loss": 2.587, + "theoretical_loss": 3.389161730117509, + "tokens_seen": 2317025280 + }, + { + "epoch": 0.4, + "learning_rate": 6.0166907398491414e-05, + "loss": 2.7186, + "theoretical_loss": 3.3891462039041755, + "tokens_seen": 2317156352 + }, + { + "epoch": 0.4, + "learning_rate": 6.0158883004333174e-05, + "loss": 2.5124, + "theoretical_loss": 3.3891306788149667, + "tokens_seen": 2317287424 + }, + { + "epoch": 0.4, + "learning_rate": 6.0150858610174934e-05, + "loss": 2.5957, + "theoretical_loss": 3.389115154849737, + "tokens_seen": 2317418496 + }, + { + "epoch": 0.4, + "learning_rate": 6.0142834216016694e-05, + "loss": 2.7088, + "theoretical_loss": 3.3890996320083424, + "tokens_seen": 2317549568 + }, + { + "epoch": 0.4, + "learning_rate": 6.0134809821858454e-05, + "loss": 2.7321, + "theoretical_loss": 3.3890841102906366, + "tokens_seen": 2317680640 + }, + { + "epoch": 0.4, + "learning_rate": 6.0126785427700214e-05, + "loss": 2.5725, + "theoretical_loss": 3.3890685896964765, + "tokens_seen": 2317811712 + }, + { + "epoch": 0.4, + "learning_rate": 6.011876103354197e-05, + "loss": 2.5568, + "theoretical_loss": 3.3890530702257156, + "tokens_seen": 2317942784 + }, + { + "epoch": 0.4, + "learning_rate": 6.011073663938373e-05, + "loss": 2.5002, + "theoretical_loss": 3.38903755187821, + "tokens_seen": 2318073856 + }, + { + "epoch": 0.4, + "learning_rate": 6.010271224522549e-05, + "loss": 2.5189, + "theoretical_loss": 3.3890220346538142, + "tokens_seen": 2318204928 + }, + { + "epoch": 0.41, + "learning_rate": 6.009468785106725e-05, + "loss": 2.5884, + "theoretical_loss": 3.3890065185523843, + "tokens_seen": 2318336000 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1279166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8162686824798584, + "objective/train/theoretical_loss": 3.388991003573775, + "objective/train/tokens_used": 688926176, + "theoretical_loss": 3.388991003573775, + "tokens_seen": 2318467072 + }, + { + "epoch": 0.41, + "learning_rate": 6.008666345690901e-05, + "loss": 2.434, + "theoretical_loss": 3.388991003573775, + "tokens_seen": 2318467072 + }, + { + "epoch": 0.41, + "learning_rate": 6.007863906275077e-05, + "loss": 2.5111, + "theoretical_loss": 3.3889754897178417, + "tokens_seen": 2318598144 + }, + { + "epoch": 0.41, + "learning_rate": 6.0070614668592527e-05, + "loss": 2.6265, + "theoretical_loss": 3.38895997698444, + "tokens_seen": 2318729216 + }, + { + "epoch": 0.41, + "learning_rate": 6.006259027443428e-05, + "loss": 2.4992, + "theoretical_loss": 3.388944465373424, + "tokens_seen": 2318860288 + }, + { + "epoch": 0.41, + "learning_rate": 6.005456588027604e-05, + "loss": 2.5315, + "theoretical_loss": 3.388928954884651, + "tokens_seen": 2318991360 + }, + { + "epoch": 0.41, + "learning_rate": 6.00465414861178e-05, + "loss": 2.5491, + "theoretical_loss": 3.3889134455179746, + "tokens_seen": 2319122432 + }, + { + "epoch": 0.41, + "learning_rate": 6.003851709195956e-05, + "loss": 2.6084, + "theoretical_loss": 3.388897937273251, + "tokens_seen": 2319253504 + }, + { + "epoch": 0.41, + "learning_rate": 6.003049269780132e-05, + "loss": 2.5187, + "theoretical_loss": 3.388882430150336, + "tokens_seen": 2319384576 + }, + { + "epoch": 0.41, + "learning_rate": 6.002246830364307e-05, + "loss": 2.6039, + "theoretical_loss": 3.3888669241490845, + "tokens_seen": 2319515648 + }, + { + "epoch": 0.41, + "learning_rate": 6.001444390948484e-05, + "loss": 2.5828, + "theoretical_loss": 3.3888514192693524, + "tokens_seen": 2319646720 + }, + { + "epoch": 0.41, + "learning_rate": 6.000641951532659e-05, + "loss": 2.4489, + "theoretical_loss": 3.388835915510995, + "tokens_seen": 2319777792 + }, + { + "epoch": 0.41, + "learning_rate": 5.999839512116835e-05, + "loss": 2.533, + "theoretical_loss": 3.3888204128738675, + "tokens_seen": 2319908864 + }, + { + "epoch": 0.41, + "learning_rate": 5.999037072701011e-05, + "loss": 2.6468, + "theoretical_loss": 3.388804911357826, + "tokens_seen": 2320039936 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1279806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8292062282562256, + "objective/train/theoretical_loss": 3.3887971610201673, + "objective/train/tokens_used": 690564576, + "theoretical_loss": 3.3887971610201673, + "tokens_seen": 2320105472 + }, + { + "epoch": 0.41, + "learning_rate": 5.998234633285187e-05, + "loss": 2.5787, + "theoretical_loss": 3.3887894109627257, + "tokens_seen": 2320171008 + }, + { + "epoch": 0.41, + "learning_rate": 5.997432193869363e-05, + "loss": 2.7014, + "theoretical_loss": 3.388773911688422, + "tokens_seen": 2320302080 + }, + { + "epoch": 0.41, + "learning_rate": 5.9966297544535385e-05, + "loss": 2.5897, + "theoretical_loss": 3.388758413534772, + "tokens_seen": 2320433152 + }, + { + "epoch": 0.41, + "learning_rate": 5.995827315037715e-05, + "loss": 2.7845, + "theoretical_loss": 3.38874291650163, + "tokens_seen": 2320564224 + }, + { + "epoch": 0.41, + "learning_rate": 5.9950248756218905e-05, + "loss": 2.5022, + "theoretical_loss": 3.3887274205888516, + "tokens_seen": 2320695296 + }, + { + "epoch": 0.41, + "learning_rate": 5.994222436206067e-05, + "loss": 2.4866, + "theoretical_loss": 3.388711925796293, + "tokens_seen": 2320826368 + }, + { + "epoch": 0.41, + "learning_rate": 5.9934199967902425e-05, + "loss": 2.4957, + "theoretical_loss": 3.3886964321238104, + "tokens_seen": 2320957440 + }, + { + "epoch": 0.41, + "learning_rate": 5.992617557374418e-05, + "loss": 2.6397, + "theoretical_loss": 3.388680939571259, + "tokens_seen": 2321088512 + }, + { + "epoch": 0.41, + "learning_rate": 5.9918151179585944e-05, + "loss": 2.5842, + "theoretical_loss": 3.388665448138495, + "tokens_seen": 2321219584 + }, + { + "epoch": 0.41, + "learning_rate": 5.99101267854277e-05, + "loss": 2.6929, + "theoretical_loss": 3.3886499578253737, + "tokens_seen": 2321350656 + }, + { + "epoch": 0.41, + "learning_rate": 5.9902102391269464e-05, + "loss": 2.5263, + "theoretical_loss": 3.388634468631751, + "tokens_seen": 2321481728 + }, + { + "epoch": 0.41, + "learning_rate": 5.989407799711122e-05, + "loss": 2.5219, + "theoretical_loss": 3.3886189805574833, + "tokens_seen": 2321612800 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1281164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.399672269821167, + "objective/train/theoretical_loss": 3.3886034936024263, + "objective/train/tokens_used": 692202976, + "theoretical_loss": 3.3886034936024263, + "tokens_seen": 2321743872 + }, + { + "epoch": 0.41, + "learning_rate": 5.9886053602952984e-05, + "loss": 2.6074, + "theoretical_loss": 3.3886034936024263, + "tokens_seen": 2321743872 + }, + { + "epoch": 0.41, + "learning_rate": 5.987802920879474e-05, + "loss": 2.5763, + "theoretical_loss": 3.388588007766436, + "tokens_seen": 2321874944 + }, + { + "epoch": 0.41, + "learning_rate": 5.987000481463649e-05, + "loss": 2.4471, + "theoretical_loss": 3.3885725230493677, + "tokens_seen": 2322006016 + }, + { + "epoch": 0.41, + "learning_rate": 5.986198042047826e-05, + "loss": 2.5937, + "theoretical_loss": 3.3885570394510784, + "tokens_seen": 2322137088 + }, + { + "epoch": 0.41, + "learning_rate": 5.985395602632001e-05, + "loss": 2.6915, + "theoretical_loss": 3.3885415569714237, + "tokens_seen": 2322268160 + }, + { + "epoch": 0.41, + "learning_rate": 5.984593163216178e-05, + "loss": 2.4943, + "theoretical_loss": 3.3885260756102595, + "tokens_seen": 2322399232 + }, + { + "epoch": 0.41, + "learning_rate": 5.983790723800353e-05, + "loss": 2.6171, + "theoretical_loss": 3.388510595367442, + "tokens_seen": 2322530304 + }, + { + "epoch": 0.41, + "learning_rate": 5.98298828438453e-05, + "loss": 2.5476, + "theoretical_loss": 3.3884951162428276, + "tokens_seen": 2322661376 + }, + { + "epoch": 0.41, + "learning_rate": 5.982185844968705e-05, + "loss": 2.6127, + "theoretical_loss": 3.388479638236272, + "tokens_seen": 2322792448 + }, + { + "epoch": 0.41, + "learning_rate": 5.98138340555288e-05, + "loss": 2.611, + "theoretical_loss": 3.388464161347632, + "tokens_seen": 2322923520 + }, + { + "epoch": 0.41, + "learning_rate": 5.980580966137057e-05, + "loss": 2.6217, + "theoretical_loss": 3.388448685576763, + "tokens_seen": 2323054592 + }, + { + "epoch": 0.41, + "learning_rate": 5.979778526721232e-05, + "loss": 2.6289, + "theoretical_loss": 3.3884332109235213, + "tokens_seen": 2323185664 + }, + { + "epoch": 0.41, + "learning_rate": 5.978976087305409e-05, + "loss": 2.552, + "theoretical_loss": 3.3884177373877633, + "tokens_seen": 2323316736 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1281765, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3108794689178467, + "objective/train/theoretical_loss": 3.388410001038896, + "objective/train/tokens_used": 693841376, + "theoretical_loss": 3.388410001038896, + "tokens_seen": 2323382272 + }, + { + "epoch": 0.41, + "learning_rate": 5.978173647889584e-05, + "loss": 2.5039, + "theoretical_loss": 3.3884022649693453, + "tokens_seen": 2323447808 + }, + { + "epoch": 0.41, + "learning_rate": 5.977371208473761e-05, + "loss": 2.577, + "theoretical_loss": 3.3883867936681242, + "tokens_seen": 2323578880 + }, + { + "epoch": 0.41, + "learning_rate": 5.976568769057936e-05, + "loss": 2.6303, + "theoretical_loss": 3.3883713234839554, + "tokens_seen": 2323709952 + }, + { + "epoch": 0.41, + "learning_rate": 5.9757663296421115e-05, + "loss": 2.555, + "theoretical_loss": 3.3883558544166954, + "tokens_seen": 2323841024 + }, + { + "epoch": 0.41, + "learning_rate": 5.974963890226288e-05, + "loss": 2.6508, + "theoretical_loss": 3.388340386466201, + "tokens_seen": 2323972096 + }, + { + "epoch": 0.41, + "learning_rate": 5.9741614508104635e-05, + "loss": 2.4878, + "theoretical_loss": 3.3883249196323284, + "tokens_seen": 2324103168 + }, + { + "epoch": 0.41, + "learning_rate": 5.97335901139464e-05, + "loss": 2.3204, + "theoretical_loss": 3.3883094539149337, + "tokens_seen": 2324234240 + }, + { + "epoch": 0.41, + "learning_rate": 5.9725565719788155e-05, + "loss": 2.4961, + "theoretical_loss": 3.388293989313874, + "tokens_seen": 2324365312 + }, + { + "epoch": 0.41, + "learning_rate": 5.971754132562992e-05, + "loss": 2.6842, + "theoretical_loss": 3.388278525829005, + "tokens_seen": 2324496384 + }, + { + "epoch": 0.41, + "learning_rate": 5.9709516931471675e-05, + "loss": 2.6049, + "theoretical_loss": 3.388263063460184, + "tokens_seen": 2324627456 + }, + { + "epoch": 0.41, + "learning_rate": 5.970149253731343e-05, + "loss": 2.5084, + "theoretical_loss": 3.388247602207267, + "tokens_seen": 2324758528 + }, + { + "epoch": 0.41, + "learning_rate": 5.9693468143155195e-05, + "loss": 2.577, + "theoretical_loss": 3.3882321420701103, + "tokens_seen": 2324889600 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1282973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2994158267974854, + "objective/train/theoretical_loss": 3.3882166830485714, + "objective/train/tokens_used": 695479776, + "theoretical_loss": 3.3882166830485714, + "tokens_seen": 2325020672 + }, + { + "epoch": 0.41, + "learning_rate": 5.968544374899695e-05, + "loss": 2.524, + "theoretical_loss": 3.3882166830485714, + "tokens_seen": 2325020672 + }, + { + "epoch": 0.41, + "learning_rate": 5.9677419354838715e-05, + "loss": 2.5124, + "theoretical_loss": 3.388201225142506, + "tokens_seen": 2325151744 + }, + { + "epoch": 0.41, + "learning_rate": 5.966939496068047e-05, + "loss": 2.4966, + "theoretical_loss": 3.3881857683517715, + "tokens_seen": 2325282816 + }, + { + "epoch": 0.41, + "learning_rate": 5.9661370566522234e-05, + "loss": 2.4446, + "theoretical_loss": 3.388170312676224, + "tokens_seen": 2325413888 + }, + { + "epoch": 0.41, + "learning_rate": 5.965334617236399e-05, + "loss": 2.5587, + "theoretical_loss": 3.3881548581157204, + "tokens_seen": 2325544960 + }, + { + "epoch": 0.41, + "learning_rate": 5.9645321778205754e-05, + "loss": 2.4818, + "theoretical_loss": 3.3881394046701176, + "tokens_seen": 2325676032 + }, + { + "epoch": 0.41, + "learning_rate": 5.963729738404751e-05, + "loss": 2.6061, + "theoretical_loss": 3.3881239523392717, + "tokens_seen": 2325807104 + }, + { + "epoch": 0.41, + "learning_rate": 5.962927298988926e-05, + "loss": 2.6662, + "theoretical_loss": 3.3881085011230403, + "tokens_seen": 2325938176 + }, + { + "epoch": 0.41, + "learning_rate": 5.962124859573103e-05, + "loss": 2.4937, + "theoretical_loss": 3.3880930510212797, + "tokens_seen": 2326069248 + }, + { + "epoch": 0.41, + "learning_rate": 5.961322420157278e-05, + "loss": 2.6491, + "theoretical_loss": 3.388077602033847, + "tokens_seen": 2326200320 + }, + { + "epoch": 0.41, + "learning_rate": 5.960519980741455e-05, + "loss": 2.508, + "theoretical_loss": 3.3880621541605986, + "tokens_seen": 2326331392 + }, + { + "epoch": 0.41, + "learning_rate": 5.95971754132563e-05, + "loss": 2.3725, + "theoretical_loss": 3.388046707401392, + "tokens_seen": 2326462464 + }, + { + "epoch": 0.41, + "learning_rate": 5.958915101909807e-05, + "loss": 2.4341, + "theoretical_loss": 3.3880312617560833, + "tokens_seen": 2326593536 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1283390, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1181111335754395, + "objective/train/theoretical_loss": 3.3880235393510962, + "objective/train/tokens_used": 697118176, + "theoretical_loss": 3.3880235393510962, + "tokens_seen": 2326659072 + }, + { + "epoch": 0.41, + "learning_rate": 5.958112662493982e-05, + "loss": 2.5477, + "theoretical_loss": 3.38801581722453, + "tokens_seen": 2326724608 + }, + { + "epoch": 0.41, + "learning_rate": 5.957310223078157e-05, + "loss": 2.4414, + "theoretical_loss": 3.388000373806589, + "tokens_seen": 2326855680 + }, + { + "epoch": 0.41, + "learning_rate": 5.956507783662334e-05, + "loss": 2.527, + "theoretical_loss": 3.3879849315021175, + "tokens_seen": 2326986752 + }, + { + "epoch": 0.41, + "learning_rate": 5.955705344246509e-05, + "loss": 2.3674, + "theoretical_loss": 3.387969490310972, + "tokens_seen": 2327117824 + }, + { + "epoch": 0.41, + "learning_rate": 5.954902904830686e-05, + "loss": 2.7315, + "theoretical_loss": 3.3879540502330094, + "tokens_seen": 2327248896 + }, + { + "epoch": 0.41, + "learning_rate": 5.954100465414861e-05, + "loss": 2.6365, + "theoretical_loss": 3.387938611268088, + "tokens_seen": 2327379968 + }, + { + "epoch": 0.41, + "learning_rate": 5.953298025999038e-05, + "loss": 2.5415, + "theoretical_loss": 3.387923173416063, + "tokens_seen": 2327511040 + }, + { + "epoch": 0.41, + "learning_rate": 5.952495586583213e-05, + "loss": 2.5746, + "theoretical_loss": 3.387907736676793, + "tokens_seen": 2327642112 + }, + { + "epoch": 0.41, + "learning_rate": 5.9516931471673886e-05, + "loss": 2.5115, + "theoretical_loss": 3.387892301050135, + "tokens_seen": 2327773184 + }, + { + "epoch": 0.41, + "learning_rate": 5.950890707751565e-05, + "loss": 2.6609, + "theoretical_loss": 3.3878768665359456, + "tokens_seen": 2327904256 + }, + { + "epoch": 0.41, + "learning_rate": 5.9500882683357405e-05, + "loss": 2.5681, + "theoretical_loss": 3.387861433134082, + "tokens_seen": 2328035328 + }, + { + "epoch": 0.41, + "learning_rate": 5.949285828919917e-05, + "loss": 2.4529, + "theoretical_loss": 3.3878460008444016, + "tokens_seen": 2328166400 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1284624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.565575122833252, + "objective/train/theoretical_loss": 3.387830569666762, + "objective/train/tokens_used": 698756576, + "theoretical_loss": 3.387830569666762, + "tokens_seen": 2328297472 + }, + { + "epoch": 0.41, + "learning_rate": 5.9484833895040925e-05, + "loss": 2.5543, + "theoretical_loss": 3.387830569666762, + "tokens_seen": 2328297472 + }, + { + "epoch": 0.41, + "learning_rate": 5.947680950088269e-05, + "loss": 2.5037, + "theoretical_loss": 3.38781513960102, + "tokens_seen": 2328428544 + }, + { + "epoch": 0.41, + "learning_rate": 5.9468785106724445e-05, + "loss": 2.5511, + "theoretical_loss": 3.387799710647033, + "tokens_seen": 2328559616 + }, + { + "epoch": 0.41, + "learning_rate": 5.94607607125662e-05, + "loss": 2.4309, + "theoretical_loss": 3.3877842828046587, + "tokens_seen": 2328690688 + }, + { + "epoch": 0.41, + "learning_rate": 5.9452736318407965e-05, + "loss": 2.4482, + "theoretical_loss": 3.3877688560737536, + "tokens_seen": 2328821760 + }, + { + "epoch": 0.41, + "learning_rate": 5.944471192424972e-05, + "loss": 2.5951, + "theoretical_loss": 3.387753430454176, + "tokens_seen": 2328952832 + }, + { + "epoch": 0.41, + "learning_rate": 5.9436687530091485e-05, + "loss": 2.5926, + "theoretical_loss": 3.3877380059457827, + "tokens_seen": 2329083904 + }, + { + "epoch": 0.41, + "learning_rate": 5.942866313593324e-05, + "loss": 2.61, + "theoretical_loss": 3.3877225825484314, + "tokens_seen": 2329214976 + }, + { + "epoch": 0.41, + "learning_rate": 5.9420638741775004e-05, + "loss": 2.6907, + "theoretical_loss": 3.3877071602619795, + "tokens_seen": 2329346048 + }, + { + "epoch": 0.41, + "learning_rate": 5.941261434761676e-05, + "loss": 2.4477, + "theoretical_loss": 3.3876917390862844, + "tokens_seen": 2329477120 + }, + { + "epoch": 0.41, + "learning_rate": 5.940458995345851e-05, + "loss": 2.3911, + "theoretical_loss": 3.387676319021204, + "tokens_seen": 2329608192 + }, + { + "epoch": 0.41, + "learning_rate": 5.939656555930028e-05, + "loss": 2.5841, + "theoretical_loss": 3.387660900066595, + "tokens_seen": 2329739264 + }, + { + "epoch": 0.41, + "learning_rate": 5.938854116514203e-05, + "loss": 2.5772, + "theoretical_loss": 3.387645482222316, + "tokens_seen": 2329870336 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1285083, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9569989442825317, + "objective/train/theoretical_loss": 3.3876377737165053, + "objective/train/tokens_used": 700394976, + "theoretical_loss": 3.3876377737165053, + "tokens_seen": 2329935872 + }, + { + "epoch": 0.41, + "learning_rate": 5.93805167709838e-05, + "loss": 2.332, + "theoretical_loss": 3.387630065488224, + "tokens_seen": 2330001408 + }, + { + "epoch": 0.41, + "learning_rate": 5.937249237682555e-05, + "loss": 2.6089, + "theoretical_loss": 3.387614649864176, + "tokens_seen": 2330132480 + }, + { + "epoch": 0.41, + "learning_rate": 5.936446798266732e-05, + "loss": 2.6178, + "theoretical_loss": 3.387599235350031, + "tokens_seen": 2330263552 + }, + { + "epoch": 0.41, + "learning_rate": 5.935644358850907e-05, + "loss": 2.5833, + "theoretical_loss": 3.387583821945646, + "tokens_seen": 2330394624 + }, + { + "epoch": 0.41, + "learning_rate": 5.934841919435082e-05, + "loss": 2.5008, + "theoretical_loss": 3.3875684096508785, + "tokens_seen": 2330525696 + }, + { + "epoch": 0.41, + "learning_rate": 5.934039480019259e-05, + "loss": 2.417, + "theoretical_loss": 3.387552998465586, + "tokens_seen": 2330656768 + }, + { + "epoch": 0.41, + "learning_rate": 5.933237040603434e-05, + "loss": 2.4472, + "theoretical_loss": 3.3875375883896273, + "tokens_seen": 2330787840 + }, + { + "epoch": 0.41, + "learning_rate": 5.932434601187611e-05, + "loss": 2.6171, + "theoretical_loss": 3.3875221794228594, + "tokens_seen": 2330918912 + }, + { + "epoch": 0.41, + "learning_rate": 5.931632161771786e-05, + "loss": 2.5458, + "theoretical_loss": 3.38750677156514, + "tokens_seen": 2331049984 + }, + { + "epoch": 0.41, + "learning_rate": 5.930829722355963e-05, + "loss": 2.6473, + "theoretical_loss": 3.3874913648163267, + "tokens_seen": 2331181056 + }, + { + "epoch": 0.41, + "learning_rate": 5.930027282940138e-05, + "loss": 2.5647, + "theoretical_loss": 3.387475959176278, + "tokens_seen": 2331312128 + }, + { + "epoch": 0.41, + "learning_rate": 5.929224843524315e-05, + "loss": 2.5548, + "theoretical_loss": 3.387460554644852, + "tokens_seen": 2331443200 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1286380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.882779836654663, + "objective/train/theoretical_loss": 3.387445151221906, + "objective/train/tokens_used": 702033376, + "theoretical_loss": 3.387445151221906, + "tokens_seen": 2331574272 + }, + { + "epoch": 0.41, + "learning_rate": 5.92842240410849e-05, + "loss": 2.5951, + "theoretical_loss": 3.387445151221906, + "tokens_seen": 2331574272 + }, + { + "epoch": 0.41, + "learning_rate": 5.9276199646926656e-05, + "loss": 2.5724, + "theoretical_loss": 3.387429748907298, + "tokens_seen": 2331705344 + }, + { + "epoch": 0.41, + "learning_rate": 5.926817525276842e-05, + "loss": 2.5311, + "theoretical_loss": 3.3874143477008865, + "tokens_seen": 2331836416 + }, + { + "epoch": 0.41, + "learning_rate": 5.9260150858610175e-05, + "loss": 2.446, + "theoretical_loss": 3.387398947602528, + "tokens_seen": 2331967488 + }, + { + "epoch": 0.41, + "learning_rate": 5.925212646445194e-05, + "loss": 2.4216, + "theoretical_loss": 3.3873835486120822, + "tokens_seen": 2332098560 + }, + { + "epoch": 0.41, + "learning_rate": 5.9244102070293695e-05, + "loss": 2.4455, + "theoretical_loss": 3.3873681507294067, + "tokens_seen": 2332229632 + }, + { + "epoch": 0.41, + "learning_rate": 5.923607767613546e-05, + "loss": 2.4227, + "theoretical_loss": 3.387352753954359, + "tokens_seen": 2332360704 + }, + { + "epoch": 0.41, + "learning_rate": 5.9228053281977215e-05, + "loss": 2.4882, + "theoretical_loss": 3.387337358286797, + "tokens_seen": 2332491776 + }, + { + "epoch": 0.41, + "learning_rate": 5.922002888781897e-05, + "loss": 2.4296, + "theoretical_loss": 3.3873219637265803, + "tokens_seen": 2332622848 + }, + { + "epoch": 0.41, + "learning_rate": 5.9212004493660735e-05, + "loss": 2.5089, + "theoretical_loss": 3.3873065702735654, + "tokens_seen": 2332753920 + }, + { + "epoch": 0.41, + "learning_rate": 5.920398009950249e-05, + "loss": 2.5498, + "theoretical_loss": 3.3872911779276116, + "tokens_seen": 2332884992 + }, + { + "epoch": 0.41, + "learning_rate": 5.9195955705344255e-05, + "loss": 2.6462, + "theoretical_loss": 3.3872757866885763, + "tokens_seen": 2333016064 + }, + { + "epoch": 0.41, + "learning_rate": 5.918793131118601e-05, + "loss": 2.6157, + "theoretical_loss": 3.387260396556318, + "tokens_seen": 2333147136 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1286803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8880960941314697, + "objective/train/theoretical_loss": 3.3872527019051857, + "objective/train/tokens_used": 703671776, + "theoretical_loss": 3.3872527019051857, + "tokens_seen": 2333212672 + }, + { + "epoch": 0.41, + "learning_rate": 5.9179906917027774e-05, + "loss": 2.6258, + "theoretical_loss": 3.387245007530695, + "tokens_seen": 2333278208 + }, + { + "epoch": 0.41, + "learning_rate": 5.917188252286953e-05, + "loss": 2.638, + "theoretical_loss": 3.387229619611565, + "tokens_seen": 2333409280 + }, + { + "epoch": 0.41, + "learning_rate": 5.916385812871128e-05, + "loss": 2.6047, + "theoretical_loss": 3.3872142327987875, + "tokens_seen": 2333540352 + }, + { + "epoch": 0.41, + "learning_rate": 5.915583373455305e-05, + "loss": 2.5849, + "theoretical_loss": 3.38719884709222, + "tokens_seen": 2333671424 + }, + { + "epoch": 0.41, + "learning_rate": 5.91478093403948e-05, + "loss": 2.4521, + "theoretical_loss": 3.387183462491721, + "tokens_seen": 2333802496 + }, + { + "epoch": 0.41, + "learning_rate": 5.913978494623657e-05, + "loss": 2.657, + "theoretical_loss": 3.3871680789971492, + "tokens_seen": 2333933568 + }, + { + "epoch": 0.41, + "learning_rate": 5.913176055207832e-05, + "loss": 2.4622, + "theoretical_loss": 3.3871526966083625, + "tokens_seen": 2334064640 + }, + { + "epoch": 0.41, + "learning_rate": 5.912373615792009e-05, + "loss": 2.4988, + "theoretical_loss": 3.3871373153252193, + "tokens_seen": 2334195712 + }, + { + "epoch": 0.41, + "learning_rate": 5.911571176376184e-05, + "loss": 2.5101, + "theoretical_loss": 3.387121935147578, + "tokens_seen": 2334326784 + }, + { + "epoch": 0.41, + "learning_rate": 5.910768736960359e-05, + "loss": 2.4301, + "theoretical_loss": 3.3871065560752975, + "tokens_seen": 2334457856 + }, + { + "epoch": 0.41, + "learning_rate": 5.909966297544536e-05, + "loss": 2.6256, + "theoretical_loss": 3.3870911781082365, + "tokens_seen": 2334588928 + }, + { + "epoch": 0.41, + "learning_rate": 5.909163858128711e-05, + "loss": 2.524, + "theoretical_loss": 3.3870758012462527, + "tokens_seen": 2334720000 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1288265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5533502101898193, + "objective/train/theoretical_loss": 3.387060425489205, + "objective/train/tokens_used": 705310176, + "theoretical_loss": 3.387060425489205, + "tokens_seen": 2334851072 + }, + { + "epoch": 0.42, + "learning_rate": 5.908361418712888e-05, + "loss": 2.5422, + "theoretical_loss": 3.387060425489205, + "tokens_seen": 2334851072 + }, + { + "epoch": 0.42, + "learning_rate": 5.907558979297063e-05, + "loss": 2.7267, + "theoretical_loss": 3.3870450508369525, + "tokens_seen": 2334982144 + }, + { + "epoch": 0.42, + "learning_rate": 5.90675653988124e-05, + "loss": 2.5964, + "theoretical_loss": 3.3870296772893536, + "tokens_seen": 2335113216 + }, + { + "epoch": 0.42, + "learning_rate": 5.905954100465415e-05, + "loss": 2.5346, + "theoretical_loss": 3.387014304846266, + "tokens_seen": 2335244288 + }, + { + "epoch": 0.42, + "learning_rate": 5.9051516610495906e-05, + "loss": 2.4027, + "theoretical_loss": 3.3869989335075497, + "tokens_seen": 2335375360 + }, + { + "epoch": 0.42, + "learning_rate": 5.904349221633767e-05, + "loss": 2.465, + "theoretical_loss": 3.3869835632730627, + "tokens_seen": 2335506432 + }, + { + "epoch": 0.42, + "learning_rate": 5.9035467822179426e-05, + "loss": 2.563, + "theoretical_loss": 3.3869681941426633, + "tokens_seen": 2335637504 + }, + { + "epoch": 0.42, + "learning_rate": 5.902744342802119e-05, + "loss": 2.4427, + "theoretical_loss": 3.386952826116211, + "tokens_seen": 2335768576 + }, + { + "epoch": 0.42, + "learning_rate": 5.9019419033862945e-05, + "loss": 2.6736, + "theoretical_loss": 3.3869374591935646, + "tokens_seen": 2335899648 + }, + { + "epoch": 0.42, + "learning_rate": 5.901139463970471e-05, + "loss": 2.6175, + "theoretical_loss": 3.386922093374582, + "tokens_seen": 2336030720 + }, + { + "epoch": 0.42, + "learning_rate": 5.9003370245546465e-05, + "loss": 2.6195, + "theoretical_loss": 3.386906728659123, + "tokens_seen": 2336161792 + }, + { + "epoch": 0.42, + "learning_rate": 5.8995345851388225e-05, + "loss": 2.6358, + "theoretical_loss": 3.3868913650470462, + "tokens_seen": 2336292864 + }, + { + "epoch": 0.42, + "learning_rate": 5.8987321457229985e-05, + "loss": 2.6297, + "theoretical_loss": 3.3868760025382096, + "tokens_seen": 2336423936 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1288802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3036727905273438, + "objective/train/theoretical_loss": 3.386868321697463, + "objective/train/tokens_used": 706948576, + "theoretical_loss": 3.386868321697463, + "tokens_seen": 2336489472 + }, + { + "epoch": 0.42, + "learning_rate": 5.897929706307174e-05, + "loss": 2.4331, + "theoretical_loss": 3.3868606411324738, + "tokens_seen": 2336555008 + }, + { + "epoch": 0.42, + "learning_rate": 5.8971272668913505e-05, + "loss": 2.4585, + "theoretical_loss": 3.386845280829696, + "tokens_seen": 2336686080 + }, + { + "epoch": 0.42, + "learning_rate": 5.896324827475526e-05, + "loss": 2.5723, + "theoretical_loss": 3.3868299216297357, + "tokens_seen": 2336817152 + }, + { + "epoch": 0.42, + "learning_rate": 5.8955223880597025e-05, + "loss": 2.661, + "theoretical_loss": 3.3868145635324525, + "tokens_seen": 2336948224 + }, + { + "epoch": 0.42, + "learning_rate": 5.894719948643878e-05, + "loss": 2.6349, + "theoretical_loss": 3.3867992065377046, + "tokens_seen": 2337079296 + }, + { + "epoch": 0.42, + "learning_rate": 5.893917509228054e-05, + "loss": 2.4407, + "theoretical_loss": 3.386783850645352, + "tokens_seen": 2337210368 + }, + { + "epoch": 0.42, + "learning_rate": 5.89311506981223e-05, + "loss": 2.4847, + "theoretical_loss": 3.386768495855252, + "tokens_seen": 2337341440 + }, + { + "epoch": 0.42, + "learning_rate": 5.892312630396405e-05, + "loss": 2.599, + "theoretical_loss": 3.3867531421672656, + "tokens_seen": 2337472512 + }, + { + "epoch": 0.42, + "learning_rate": 5.891510190980582e-05, + "loss": 2.486, + "theoretical_loss": 3.3867377895812507, + "tokens_seen": 2337603584 + }, + { + "epoch": 0.42, + "learning_rate": 5.890707751564757e-05, + "loss": 2.4696, + "theoretical_loss": 3.386722438097067, + "tokens_seen": 2337734656 + }, + { + "epoch": 0.42, + "learning_rate": 5.889905312148933e-05, + "loss": 2.5101, + "theoretical_loss": 3.3867070877145733, + "tokens_seen": 2337865728 + }, + { + "epoch": 0.42, + "learning_rate": 5.889102872733109e-05, + "loss": 2.6604, + "theoretical_loss": 3.3866917384336293, + "tokens_seen": 2337996800 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1289730, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0127034187316895, + "objective/train/theoretical_loss": 3.3866763902540935, + "objective/train/tokens_used": 708586976, + "theoretical_loss": 3.3866763902540935, + "tokens_seen": 2338127872 + }, + { + "epoch": 0.42, + "learning_rate": 5.888300433317285e-05, + "loss": 2.5908, + "theoretical_loss": 3.3866763902540935, + "tokens_seen": 2338127872 + }, + { + "epoch": 0.42, + "learning_rate": 5.887497993901461e-05, + "loss": 2.6727, + "theoretical_loss": 3.3866610431758253, + "tokens_seen": 2338258944 + }, + { + "epoch": 0.42, + "learning_rate": 5.886695554485636e-05, + "loss": 2.6052, + "theoretical_loss": 3.3866456971986842, + "tokens_seen": 2338390016 + }, + { + "epoch": 0.42, + "learning_rate": 5.885893115069813e-05, + "loss": 2.5066, + "theoretical_loss": 3.3866303523225296, + "tokens_seen": 2338521088 + }, + { + "epoch": 0.42, + "learning_rate": 5.885090675653988e-05, + "loss": 2.5048, + "theoretical_loss": 3.3866150085472206, + "tokens_seen": 2338652160 + }, + { + "epoch": 0.42, + "learning_rate": 5.884288236238164e-05, + "loss": 2.5927, + "theoretical_loss": 3.3865996658726165, + "tokens_seen": 2338783232 + }, + { + "epoch": 0.42, + "learning_rate": 5.88348579682234e-05, + "loss": 2.5956, + "theoretical_loss": 3.3865843242985765, + "tokens_seen": 2338914304 + }, + { + "epoch": 0.42, + "learning_rate": 5.882683357406516e-05, + "loss": 2.5094, + "theoretical_loss": 3.3865689838249606, + "tokens_seen": 2339045376 + }, + { + "epoch": 0.42, + "learning_rate": 5.881880917990692e-05, + "loss": 2.5411, + "theoretical_loss": 3.3865536444516273, + "tokens_seen": 2339176448 + }, + { + "epoch": 0.42, + "learning_rate": 5.8810784785748676e-05, + "loss": 2.4394, + "theoretical_loss": 3.386538306178437, + "tokens_seen": 2339307520 + }, + { + "epoch": 0.42, + "learning_rate": 5.8802760391590436e-05, + "loss": 2.3762, + "theoretical_loss": 3.3865229690052487, + "tokens_seen": 2339438592 + }, + { + "epoch": 0.42, + "learning_rate": 5.8794735997432196e-05, + "loss": 2.5426, + "theoretical_loss": 3.3865076329319215, + "tokens_seen": 2339569664 + }, + { + "epoch": 0.42, + "learning_rate": 5.8786711603273956e-05, + "loss": 2.681, + "theoretical_loss": 3.3864922979583154, + "tokens_seen": 2339700736 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1290793, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1198654174804688, + "objective/train/theoretical_loss": 3.386484630883864, + "objective/train/tokens_used": 710225376, + "theoretical_loss": 3.386484630883864, + "tokens_seen": 2339766272 + }, + { + "epoch": 0.42, + "learning_rate": 5.8778687209115716e-05, + "loss": 2.5137, + "theoretical_loss": 3.3864769640842898, + "tokens_seen": 2339831808 + }, + { + "epoch": 0.42, + "learning_rate": 5.8770662814957475e-05, + "loss": 2.5042, + "theoretical_loss": 3.3864616313097047, + "tokens_seen": 2339962880 + }, + { + "epoch": 0.42, + "learning_rate": 5.8762638420799235e-05, + "loss": 2.3994, + "theoretical_loss": 3.386446299634419, + "tokens_seen": 2340093952 + }, + { + "epoch": 0.42, + "learning_rate": 5.875461402664099e-05, + "loss": 2.5447, + "theoretical_loss": 3.386430969058293, + "tokens_seen": 2340225024 + }, + { + "epoch": 0.42, + "learning_rate": 5.874658963248275e-05, + "loss": 2.3838, + "theoretical_loss": 3.3864156395811853, + "tokens_seen": 2340356096 + }, + { + "epoch": 0.42, + "learning_rate": 5.873856523832451e-05, + "loss": 2.5524, + "theoretical_loss": 3.386400311202957, + "tokens_seen": 2340487168 + }, + { + "epoch": 0.42, + "learning_rate": 5.873054084416627e-05, + "loss": 2.5065, + "theoretical_loss": 3.3863849839234663, + "tokens_seen": 2340618240 + }, + { + "epoch": 0.42, + "learning_rate": 5.872251645000803e-05, + "loss": 2.4252, + "theoretical_loss": 3.386369657742574, + "tokens_seen": 2340749312 + }, + { + "epoch": 0.42, + "learning_rate": 5.871449205584979e-05, + "loss": 2.5425, + "theoretical_loss": 3.3863543326601393, + "tokens_seen": 2340880384 + }, + { + "epoch": 0.42, + "learning_rate": 5.870646766169154e-05, + "loss": 2.5635, + "theoretical_loss": 3.386339008676022, + "tokens_seen": 2341011456 + }, + { + "epoch": 0.42, + "learning_rate": 5.869844326753331e-05, + "loss": 2.6626, + "theoretical_loss": 3.3863236857900825, + "tokens_seen": 2341142528 + }, + { + "epoch": 0.42, + "learning_rate": 5.869041887337506e-05, + "loss": 2.3922, + "theoretical_loss": 3.38630836400218, + "tokens_seen": 2341273600 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1291340, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.626932382583618, + "objective/train/theoretical_loss": 3.386293043312175, + "objective/train/tokens_used": 711863776, + "theoretical_loss": 3.386293043312175, + "tokens_seen": 2341404672 + }, + { + "epoch": 0.42, + "learning_rate": 5.868239447921682e-05, + "loss": 2.6208, + "theoretical_loss": 3.386293043312175, + "tokens_seen": 2341404672 + }, + { + "epoch": 0.42, + "learning_rate": 5.867437008505858e-05, + "loss": 2.4149, + "theoretical_loss": 3.3862777237199264, + "tokens_seen": 2341535744 + }, + { + "epoch": 0.42, + "learning_rate": 5.8666345690900334e-05, + "loss": 2.5403, + "theoretical_loss": 3.3862624052252945, + "tokens_seen": 2341666816 + }, + { + "epoch": 0.42, + "learning_rate": 5.86583212967421e-05, + "loss": 2.5613, + "theoretical_loss": 3.38624708782814, + "tokens_seen": 2341797888 + }, + { + "epoch": 0.42, + "learning_rate": 5.8650296902583854e-05, + "loss": 2.4958, + "theoretical_loss": 3.3862317715283217, + "tokens_seen": 2341928960 + }, + { + "epoch": 0.42, + "learning_rate": 5.864227250842562e-05, + "loss": 2.5425, + "theoretical_loss": 3.3862164563257, + "tokens_seen": 2342060032 + }, + { + "epoch": 0.42, + "learning_rate": 5.8634248114267374e-05, + "loss": 2.6168, + "theoretical_loss": 3.386201142220135, + "tokens_seen": 2342191104 + }, + { + "epoch": 0.42, + "learning_rate": 5.8626223720109133e-05, + "loss": 2.4386, + "theoretical_loss": 3.386185829211487, + "tokens_seen": 2342322176 + }, + { + "epoch": 0.42, + "learning_rate": 5.861819932595089e-05, + "loss": 2.4753, + "theoretical_loss": 3.3861705172996155, + "tokens_seen": 2342453248 + }, + { + "epoch": 0.42, + "learning_rate": 5.8610174931792646e-05, + "loss": 2.4767, + "theoretical_loss": 3.386155206484381, + "tokens_seen": 2342584320 + }, + { + "epoch": 0.42, + "learning_rate": 5.860215053763441e-05, + "loss": 2.5488, + "theoretical_loss": 3.3861398967656435, + "tokens_seen": 2342715392 + }, + { + "epoch": 0.42, + "learning_rate": 5.8594126143476166e-05, + "loss": 2.393, + "theoretical_loss": 3.3861245881432627, + "tokens_seen": 2342846464 + }, + { + "epoch": 0.42, + "learning_rate": 5.858610174931793e-05, + "loss": 2.5483, + "theoretical_loss": 3.3861092806170996, + "tokens_seen": 2342977536 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1292270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6421544551849365, + "objective/train/theoretical_loss": 3.386101627265056, + "objective/train/tokens_used": 713502176, + "theoretical_loss": 3.386101627265056, + "tokens_seen": 2343043072 + }, + { + "epoch": 0.42, + "learning_rate": 5.8578077355159686e-05, + "loss": 2.5855, + "theoretical_loss": 3.3860939741870135, + "tokens_seen": 2343108608 + }, + { + "epoch": 0.42, + "learning_rate": 5.857005296100144e-05, + "loss": 2.3921, + "theoretical_loss": 3.3860786688528655, + "tokens_seen": 2343239680 + }, + { + "epoch": 0.42, + "learning_rate": 5.8562028566843206e-05, + "loss": 2.587, + "theoretical_loss": 3.386063364614515, + "tokens_seen": 2343370752 + }, + { + "epoch": 0.42, + "learning_rate": 5.855400417268496e-05, + "loss": 2.6345, + "theoretical_loss": 3.386048061471823, + "tokens_seen": 2343501824 + }, + { + "epoch": 0.42, + "learning_rate": 5.8545979778526726e-05, + "loss": 2.5607, + "theoretical_loss": 3.386032759424649, + "tokens_seen": 2343632896 + }, + { + "epoch": 0.42, + "learning_rate": 5.853795538436848e-05, + "loss": 2.3979, + "theoretical_loss": 3.386017458472854, + "tokens_seen": 2343763968 + }, + { + "epoch": 0.42, + "learning_rate": 5.8529930990210246e-05, + "loss": 2.6633, + "theoretical_loss": 3.386002158616298, + "tokens_seen": 2343895040 + }, + { + "epoch": 0.42, + "learning_rate": 5.8521906596052e-05, + "loss": 2.489, + "theoretical_loss": 3.3859868598548415, + "tokens_seen": 2344026112 + }, + { + "epoch": 0.42, + "learning_rate": 5.851388220189375e-05, + "loss": 2.4694, + "theoretical_loss": 3.3859715621883444, + "tokens_seen": 2344157184 + }, + { + "epoch": 0.42, + "learning_rate": 5.850585780773552e-05, + "loss": 2.5831, + "theoretical_loss": 3.385956265616668, + "tokens_seen": 2344288256 + }, + { + "epoch": 0.42, + "learning_rate": 5.849783341357727e-05, + "loss": 2.6124, + "theoretical_loss": 3.3859409701396723, + "tokens_seen": 2344419328 + }, + { + "epoch": 0.42, + "learning_rate": 5.848980901941904e-05, + "loss": 2.6426, + "theoretical_loss": 3.3859256757572176, + "tokens_seen": 2344550400 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1292697, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3175153732299805, + "objective/train/theoretical_loss": 3.3859103824691643, + "objective/train/tokens_used": 715140576, + "theoretical_loss": 3.3859103824691643, + "tokens_seen": 2344681472 + }, + { + "epoch": 0.42, + "learning_rate": 5.848178462526079e-05, + "loss": 2.6733, + "theoretical_loss": 3.3859103824691643, + "tokens_seen": 2344681472 + }, + { + "epoch": 0.42, + "learning_rate": 5.847376023110256e-05, + "loss": 2.5185, + "theoretical_loss": 3.3858950902753735, + "tokens_seen": 2344812544 + }, + { + "epoch": 0.42, + "learning_rate": 5.846573583694431e-05, + "loss": 2.6166, + "theoretical_loss": 3.3858797991757053, + "tokens_seen": 2344943616 + }, + { + "epoch": 0.42, + "learning_rate": 5.8457711442786064e-05, + "loss": 2.5241, + "theoretical_loss": 3.38586450917002, + "tokens_seen": 2345074688 + }, + { + "epoch": 0.42, + "learning_rate": 5.844968704862783e-05, + "loss": 2.5079, + "theoretical_loss": 3.385849220258179, + "tokens_seen": 2345205760 + }, + { + "epoch": 0.42, + "learning_rate": 5.8441662654469584e-05, + "loss": 2.4976, + "theoretical_loss": 3.3858339324400424, + "tokens_seen": 2345336832 + }, + { + "epoch": 0.42, + "learning_rate": 5.843363826031135e-05, + "loss": 2.5434, + "theoretical_loss": 3.3858186457154704, + "tokens_seen": 2345467904 + }, + { + "epoch": 0.42, + "learning_rate": 5.8425613866153104e-05, + "loss": 2.5645, + "theoretical_loss": 3.3858033600843247, + "tokens_seen": 2345598976 + }, + { + "epoch": 0.42, + "learning_rate": 5.841758947199487e-05, + "loss": 2.6008, + "theoretical_loss": 3.3857880755464658, + "tokens_seen": 2345730048 + }, + { + "epoch": 0.42, + "learning_rate": 5.8409565077836624e-05, + "loss": 2.608, + "theoretical_loss": 3.3857727921017533, + "tokens_seen": 2345861120 + }, + { + "epoch": 0.42, + "learning_rate": 5.840154068367838e-05, + "loss": 2.5562, + "theoretical_loss": 3.385757509750049, + "tokens_seen": 2345992192 + }, + { + "epoch": 0.42, + "learning_rate": 5.8393516289520144e-05, + "loss": 2.6444, + "theoretical_loss": 3.385742228491214, + "tokens_seen": 2346123264 + }, + { + "epoch": 0.42, + "learning_rate": 5.83854918953619e-05, + "loss": 2.6761, + "theoretical_loss": 3.385726948325108, + "tokens_seen": 2346254336 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1293722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2620153427124023, + "objective/train/theoretical_loss": 3.385719308651785, + "objective/train/tokens_used": 716778976, + "theoretical_loss": 3.385719308651785, + "tokens_seen": 2346319872 + }, + { + "epoch": 0.42, + "learning_rate": 5.8377467501203663e-05, + "loss": 2.4176, + "theoretical_loss": 3.3857116692515925, + "tokens_seen": 2346385408 + }, + { + "epoch": 0.42, + "learning_rate": 5.8369443107045417e-05, + "loss": 2.6602, + "theoretical_loss": 3.385696391270528, + "tokens_seen": 2346516480 + }, + { + "epoch": 0.42, + "learning_rate": 5.836141871288718e-05, + "loss": 2.535, + "theoretical_loss": 3.3856811143817755, + "tokens_seen": 2346647552 + }, + { + "epoch": 0.42, + "learning_rate": 5.8353394318728936e-05, + "loss": 2.5504, + "theoretical_loss": 3.385665838585196, + "tokens_seen": 2346778624 + }, + { + "epoch": 0.42, + "learning_rate": 5.83453699245707e-05, + "loss": 2.6594, + "theoretical_loss": 3.3856505638806507, + "tokens_seen": 2346909696 + }, + { + "epoch": 0.42, + "learning_rate": 5.8337345530412456e-05, + "loss": 2.616, + "theoretical_loss": 3.3856352902679996, + "tokens_seen": 2347040768 + }, + { + "epoch": 0.42, + "learning_rate": 5.832932113625421e-05, + "loss": 2.6099, + "theoretical_loss": 3.385620017747105, + "tokens_seen": 2347171840 + }, + { + "epoch": 0.42, + "learning_rate": 5.8321296742095976e-05, + "loss": 2.6064, + "theoretical_loss": 3.385604746317827, + "tokens_seen": 2347302912 + }, + { + "epoch": 0.42, + "learning_rate": 5.831327234793773e-05, + "loss": 2.5122, + "theoretical_loss": 3.3855894759800265, + "tokens_seen": 2347433984 + }, + { + "epoch": 0.42, + "learning_rate": 5.8305247953779496e-05, + "loss": 2.3559, + "theoretical_loss": 3.3855742067335655, + "tokens_seen": 2347565056 + }, + { + "epoch": 0.42, + "learning_rate": 5.829722355962125e-05, + "loss": 2.5931, + "theoretical_loss": 3.385558938578304, + "tokens_seen": 2347696128 + }, + { + "epoch": 0.42, + "learning_rate": 5.8289199165463016e-05, + "loss": 2.6417, + "theoretical_loss": 3.385543671514104, + "tokens_seen": 2347827200 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1294165, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2179014682769775, + "objective/train/theoretical_loss": 3.3855284055408257, + "objective/train/tokens_used": 718417376, + "theoretical_loss": 3.3855284055408257, + "tokens_seen": 2347958272 + }, + { + "epoch": 0.42, + "learning_rate": 5.828117477130477e-05, + "loss": 2.4197, + "theoretical_loss": 3.3855284055408257, + "tokens_seen": 2347958272 + }, + { + "epoch": 0.42, + "learning_rate": 5.827315037714652e-05, + "loss": 2.5461, + "theoretical_loss": 3.385513140658331, + "tokens_seen": 2348089344 + }, + { + "epoch": 0.42, + "learning_rate": 5.826512598298829e-05, + "loss": 2.6113, + "theoretical_loss": 3.385497876866481, + "tokens_seen": 2348220416 + }, + { + "epoch": 0.42, + "learning_rate": 5.825710158883004e-05, + "loss": 2.6432, + "theoretical_loss": 3.3854826141651366, + "tokens_seen": 2348351488 + }, + { + "epoch": 0.42, + "learning_rate": 5.824907719467181e-05, + "loss": 2.5889, + "theoretical_loss": 3.385467352554159, + "tokens_seen": 2348482560 + }, + { + "epoch": 0.42, + "learning_rate": 5.824105280051356e-05, + "loss": 2.5039, + "theoretical_loss": 3.38545209203341, + "tokens_seen": 2348613632 + }, + { + "epoch": 0.42, + "learning_rate": 5.823302840635533e-05, + "loss": 2.6764, + "theoretical_loss": 3.38543683260275, + "tokens_seen": 2348744704 + }, + { + "epoch": 0.42, + "learning_rate": 5.822500401219708e-05, + "loss": 2.6093, + "theoretical_loss": 3.3854215742620415, + "tokens_seen": 2348875776 + }, + { + "epoch": 0.42, + "learning_rate": 5.8216979618038834e-05, + "loss": 2.4995, + "theoretical_loss": 3.3854063170111446, + "tokens_seen": 2349006848 + }, + { + "epoch": 0.42, + "learning_rate": 5.82089552238806e-05, + "loss": 2.7344, + "theoretical_loss": 3.3853910608499214, + "tokens_seen": 2349137920 + }, + { + "epoch": 0.42, + "learning_rate": 5.8200930829722354e-05, + "loss": 2.7268, + "theoretical_loss": 3.3853758057782333, + "tokens_seen": 2349268992 + }, + { + "epoch": 0.42, + "learning_rate": 5.819290643556412e-05, + "loss": 2.4094, + "theoretical_loss": 3.385360551795941, + "tokens_seen": 2349400064 + }, + { + "epoch": 0.42, + "learning_rate": 5.8184882041405874e-05, + "loss": 2.6371, + "theoretical_loss": 3.385345298902907, + "tokens_seen": 2349531136 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1295470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5095014572143555, + "objective/train/theoretical_loss": 3.385337672864818, + "objective/train/tokens_used": 720055776, + "theoretical_loss": 3.385337672864818, + "tokens_seen": 2349596672 + }, + { + "epoch": 0.42, + "learning_rate": 5.817685764724764e-05, + "loss": 2.7028, + "theoretical_loss": 3.385330047098992, + "tokens_seen": 2349662208 + }, + { + "epoch": 0.42, + "learning_rate": 5.8168833253089394e-05, + "loss": 2.3716, + "theoretical_loss": 3.3853147963840575, + "tokens_seen": 2349793280 + }, + { + "epoch": 0.42, + "learning_rate": 5.816080885893115e-05, + "loss": 2.5393, + "theoretical_loss": 3.385299546757965, + "tokens_seen": 2349924352 + }, + { + "epoch": 0.42, + "learning_rate": 5.8152784464772914e-05, + "loss": 2.6145, + "theoretical_loss": 3.3852842982205766, + "tokens_seen": 2350055424 + }, + { + "epoch": 0.42, + "learning_rate": 5.814476007061467e-05, + "loss": 2.5938, + "theoretical_loss": 3.385269050771753, + "tokens_seen": 2350186496 + }, + { + "epoch": 0.42, + "learning_rate": 5.8136735676456433e-05, + "loss": 2.6389, + "theoretical_loss": 3.3852538044113567, + "tokens_seen": 2350317568 + }, + { + "epoch": 0.42, + "learning_rate": 5.812871128229819e-05, + "loss": 2.5708, + "theoretical_loss": 3.385238559139249, + "tokens_seen": 2350448640 + }, + { + "epoch": 0.42, + "learning_rate": 5.812068688813995e-05, + "loss": 2.6075, + "theoretical_loss": 3.385223314955291, + "tokens_seen": 2350579712 + }, + { + "epoch": 0.42, + "learning_rate": 5.8112662493981706e-05, + "loss": 2.5073, + "theoretical_loss": 3.385208071859345, + "tokens_seen": 2350710784 + }, + { + "epoch": 0.42, + "learning_rate": 5.810463809982346e-05, + "loss": 2.4663, + "theoretical_loss": 3.3851928298512726, + "tokens_seen": 2350841856 + }, + { + "epoch": 0.42, + "learning_rate": 5.8096613705665226e-05, + "loss": 2.573, + "theoretical_loss": 3.3851775889309352, + "tokens_seen": 2350972928 + }, + { + "epoch": 0.42, + "learning_rate": 5.808858931150698e-05, + "loss": 2.533, + "theoretical_loss": 3.385162349098195, + "tokens_seen": 2351104000 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1295872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.463392734527588, + "objective/train/theoretical_loss": 3.385147110352913, + "objective/train/tokens_used": 721694176, + "theoretical_loss": 3.385147110352913, + "tokens_seen": 2351235072 + }, + { + "epoch": 0.43, + "learning_rate": 5.8080564917348746e-05, + "loss": 2.4912, + "theoretical_loss": 3.385147110352913, + "tokens_seen": 2351235072 + }, + { + "epoch": 0.43, + "learning_rate": 5.80725405231905e-05, + "loss": 2.6252, + "theoretical_loss": 3.3851318726949513, + "tokens_seen": 2351366144 + }, + { + "epoch": 0.43, + "learning_rate": 5.8064516129032266e-05, + "loss": 2.4905, + "theoretical_loss": 3.3851166361241725, + "tokens_seen": 2351497216 + }, + { + "epoch": 0.43, + "learning_rate": 5.805649173487402e-05, + "loss": 2.5806, + "theoretical_loss": 3.385101400640438, + "tokens_seen": 2351628288 + }, + { + "epoch": 0.43, + "learning_rate": 5.8048467340715786e-05, + "loss": 2.649, + "theoretical_loss": 3.3850861662436085, + "tokens_seen": 2351759360 + }, + { + "epoch": 0.43, + "learning_rate": 5.804044294655754e-05, + "loss": 2.514, + "theoretical_loss": 3.3850709329335475, + "tokens_seen": 2351890432 + }, + { + "epoch": 0.43, + "learning_rate": 5.803241855239929e-05, + "loss": 2.4705, + "theoretical_loss": 3.385055700710116, + "tokens_seen": 2352021504 + }, + { + "epoch": 0.43, + "learning_rate": 5.802439415824106e-05, + "loss": 2.5074, + "theoretical_loss": 3.3850404695731764, + "tokens_seen": 2352152576 + }, + { + "epoch": 0.43, + "learning_rate": 5.801636976408281e-05, + "loss": 2.5268, + "theoretical_loss": 3.385025239522591, + "tokens_seen": 2352283648 + }, + { + "epoch": 0.43, + "learning_rate": 5.800834536992458e-05, + "loss": 2.5474, + "theoretical_loss": 3.3850100105582204, + "tokens_seen": 2352414720 + }, + { + "epoch": 0.43, + "learning_rate": 5.800032097576633e-05, + "loss": 2.5474, + "theoretical_loss": 3.384994782679928, + "tokens_seen": 2352545792 + }, + { + "epoch": 0.43, + "learning_rate": 5.79922965816081e-05, + "loss": 2.6361, + "theoretical_loss": 3.384979555887575, + "tokens_seen": 2352676864 + }, + { + "epoch": 0.43, + "learning_rate": 5.798427218744985e-05, + "loss": 2.5841, + "theoretical_loss": 3.384964330181024, + "tokens_seen": 2352807936 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5529563426971436, + "objective/train/theoretical_loss": 3.3849567177348807, + "objective/train/tokens_used": 723332576, + "theoretical_loss": 3.3849567177348807, + "tokens_seen": 2352873472 + }, + { + "epoch": 0.43, + "learning_rate": 5.7976247793291605e-05, + "loss": 2.6446, + "theoretical_loss": 3.3849491055601364, + "tokens_seen": 2352939008 + }, + { + "epoch": 0.43, + "learning_rate": 5.796822339913337e-05, + "loss": 2.4279, + "theoretical_loss": 3.3849338820247756, + "tokens_seen": 2353070080 + }, + { + "epoch": 0.43, + "learning_rate": 5.7960199004975124e-05, + "loss": 2.6022, + "theoretical_loss": 3.3849186595748026, + "tokens_seen": 2353201152 + }, + { + "epoch": 0.43, + "learning_rate": 5.795217461081689e-05, + "loss": 2.6832, + "theoretical_loss": 3.3849034382100798, + "tokens_seen": 2353332224 + }, + { + "epoch": 0.43, + "learning_rate": 5.7944150216658644e-05, + "loss": 2.6591, + "theoretical_loss": 3.3848882179304693, + "tokens_seen": 2353463296 + }, + { + "epoch": 0.43, + "learning_rate": 5.793612582250041e-05, + "loss": 2.6853, + "theoretical_loss": 3.384872998735834, + "tokens_seen": 2353594368 + }, + { + "epoch": 0.43, + "learning_rate": 5.7928101428342164e-05, + "loss": 2.6657, + "theoretical_loss": 3.3848577806260356, + "tokens_seen": 2353725440 + }, + { + "epoch": 0.43, + "learning_rate": 5.792007703418392e-05, + "loss": 2.6369, + "theoretical_loss": 3.3848425636009365, + "tokens_seen": 2353856512 + }, + { + "epoch": 0.43, + "learning_rate": 5.7912052640025684e-05, + "loss": 2.5669, + "theoretical_loss": 3.3848273476603987, + "tokens_seen": 2353987584 + }, + { + "epoch": 0.43, + "learning_rate": 5.790402824586744e-05, + "loss": 2.6338, + "theoretical_loss": 3.3848121328042846, + "tokens_seen": 2354118656 + }, + { + "epoch": 0.43, + "learning_rate": 5.7896003851709204e-05, + "loss": 2.6512, + "theoretical_loss": 3.384796919032457, + "tokens_seen": 2354249728 + }, + { + "epoch": 0.43, + "learning_rate": 5.788797945755096e-05, + "loss": 2.6298, + "theoretical_loss": 3.3847817063447776, + "tokens_seen": 2354380800 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4254140853881836, + "objective/train/theoretical_loss": 3.3847664947411094, + "objective/train/tokens_used": 724970976, + "theoretical_loss": 3.3847664947411094, + "tokens_seen": 2354511872 + }, + { + "epoch": 0.43, + "learning_rate": 5.787995506339272e-05, + "loss": 2.586, + "theoretical_loss": 3.3847664947411094, + "tokens_seen": 2354511872 + }, + { + "epoch": 0.43, + "learning_rate": 5.7871930669234477e-05, + "loss": 2.6616, + "theoretical_loss": 3.3847512842213145, + "tokens_seen": 2354642944 + }, + { + "epoch": 0.43, + "learning_rate": 5.786390627507623e-05, + "loss": 2.7663, + "theoretical_loss": 3.384736074785255, + "tokens_seen": 2354774016 + }, + { + "epoch": 0.43, + "learning_rate": 5.7855881880917996e-05, + "loss": 2.7011, + "theoretical_loss": 3.384720866432794, + "tokens_seen": 2354905088 + }, + { + "epoch": 0.43, + "learning_rate": 5.784785748675975e-05, + "loss": 2.8609, + "theoretical_loss": 3.384705659163794, + "tokens_seen": 2355036160 + }, + { + "epoch": 0.43, + "learning_rate": 5.7839833092601516e-05, + "loss": 2.6335, + "theoretical_loss": 3.384690452978117, + "tokens_seen": 2355167232 + }, + { + "epoch": 0.43, + "learning_rate": 5.783180869844327e-05, + "loss": 2.7139, + "theoretical_loss": 3.3846752478756263, + "tokens_seen": 2355298304 + }, + { + "epoch": 0.43, + "learning_rate": 5.7823784304285036e-05, + "loss": 2.5951, + "theoretical_loss": 3.384660043856184, + "tokens_seen": 2355429376 + }, + { + "epoch": 0.43, + "learning_rate": 5.781575991012679e-05, + "loss": 2.7079, + "theoretical_loss": 3.384644840919652, + "tokens_seen": 2355560448 + }, + { + "epoch": 0.43, + "learning_rate": 5.780773551596854e-05, + "loss": 2.6649, + "theoretical_loss": 3.384629639065894, + "tokens_seen": 2355691520 + }, + { + "epoch": 0.43, + "learning_rate": 5.779971112181031e-05, + "loss": 2.8242, + "theoretical_loss": 3.384614438294772, + "tokens_seen": 2355822592 + }, + { + "epoch": 0.43, + "learning_rate": 5.779168672765206e-05, + "loss": 2.6926, + "theoretical_loss": 3.384599238606149, + "tokens_seen": 2355953664 + }, + { + "epoch": 0.43, + "learning_rate": 5.778366233349383e-05, + "loss": 2.7872, + "theoretical_loss": 3.384584039999888, + "tokens_seen": 2356084736 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6371724605560303, + "objective/train/theoretical_loss": 3.3845764411026, + "objective/train/tokens_used": 726609376, + "theoretical_loss": 3.3845764411026, + "tokens_seen": 2356150272 + }, + { + "epoch": 0.43, + "learning_rate": 5.777563793933558e-05, + "loss": 2.6989, + "theoretical_loss": 3.384568842475851, + "tokens_seen": 2356215808 + }, + { + "epoch": 0.43, + "learning_rate": 5.776761354517735e-05, + "loss": 2.7725, + "theoretical_loss": 3.3845536460339014, + "tokens_seen": 2356346880 + }, + { + "epoch": 0.43, + "learning_rate": 5.77595891510191e-05, + "loss": 2.8416, + "theoretical_loss": 3.384538450673901, + "tokens_seen": 2356477952 + }, + { + "epoch": 0.43, + "learning_rate": 5.775156475686087e-05, + "loss": 2.7238, + "theoretical_loss": 3.3845232563957137, + "tokens_seen": 2356609024 + }, + { + "epoch": 0.43, + "learning_rate": 5.774354036270262e-05, + "loss": 2.6543, + "theoretical_loss": 3.3845080631992017, + "tokens_seen": 2356740096 + }, + { + "epoch": 0.43, + "learning_rate": 5.7735515968544375e-05, + "loss": 2.7049, + "theoretical_loss": 3.384492871084228, + "tokens_seen": 2356871168 + }, + { + "epoch": 0.43, + "learning_rate": 5.772749157438614e-05, + "loss": 2.7103, + "theoretical_loss": 3.3844776800506557, + "tokens_seen": 2357002240 + }, + { + "epoch": 0.43, + "learning_rate": 5.7719467180227894e-05, + "loss": 2.6335, + "theoretical_loss": 3.3844624900983473, + "tokens_seen": 2357133312 + }, + { + "epoch": 0.43, + "learning_rate": 5.771144278606966e-05, + "loss": 2.5968, + "theoretical_loss": 3.3844473012271656, + "tokens_seen": 2357264384 + }, + { + "epoch": 0.43, + "learning_rate": 5.7703418391911414e-05, + "loss": 2.8002, + "theoretical_loss": 3.3844321134369744, + "tokens_seen": 2357395456 + }, + { + "epoch": 0.43, + "learning_rate": 5.769539399775318e-05, + "loss": 2.7501, + "theoretical_loss": 3.384416926727636, + "tokens_seen": 2357526528 + }, + { + "epoch": 0.43, + "learning_rate": 5.7687369603594934e-05, + "loss": 2.7608, + "theoretical_loss": 3.384401741099013, + "tokens_seen": 2357657600 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.529184103012085, + "objective/train/theoretical_loss": 3.384386556550969, + "objective/train/tokens_used": 728247776, + "theoretical_loss": 3.384386556550969, + "tokens_seen": 2357788672 + }, + { + "epoch": 0.43, + "learning_rate": 5.767934520943669e-05, + "loss": 2.7595, + "theoretical_loss": 3.384386556550969, + "tokens_seen": 2357788672 + }, + { + "epoch": 0.43, + "learning_rate": 5.7671320815278454e-05, + "loss": 2.7837, + "theoretical_loss": 3.384371373083367, + "tokens_seen": 2357919744 + }, + { + "epoch": 0.43, + "learning_rate": 5.766329642112021e-05, + "loss": 2.7426, + "theoretical_loss": 3.3843561906960704, + "tokens_seen": 2358050816 + }, + { + "epoch": 0.43, + "learning_rate": 5.7655272026961974e-05, + "loss": 2.8049, + "theoretical_loss": 3.3843410093889417, + "tokens_seen": 2358181888 + }, + { + "epoch": 0.43, + "learning_rate": 5.764724763280373e-05, + "loss": 2.836, + "theoretical_loss": 3.384325829161844, + "tokens_seen": 2358312960 + }, + { + "epoch": 0.43, + "learning_rate": 5.763922323864549e-05, + "loss": 2.7711, + "theoretical_loss": 3.384310650014641, + "tokens_seen": 2358444032 + }, + { + "epoch": 0.43, + "learning_rate": 5.7631198844487247e-05, + "loss": 2.8929, + "theoretical_loss": 3.3842954719471954, + "tokens_seen": 2358575104 + }, + { + "epoch": 0.43, + "learning_rate": 5.7623174450329e-05, + "loss": 2.7552, + "theoretical_loss": 3.3842802949593707, + "tokens_seen": 2358706176 + }, + { + "epoch": 0.43, + "learning_rate": 5.7615150056170766e-05, + "loss": 2.6488, + "theoretical_loss": 3.3842651190510296, + "tokens_seen": 2358837248 + }, + { + "epoch": 0.43, + "learning_rate": 5.760712566201252e-05, + "loss": 2.7407, + "theoretical_loss": 3.384249944222036, + "tokens_seen": 2358968320 + }, + { + "epoch": 0.43, + "learning_rate": 5.7599101267854286e-05, + "loss": 2.8939, + "theoretical_loss": 3.384234770472253, + "tokens_seen": 2359099392 + }, + { + "epoch": 0.43, + "learning_rate": 5.759107687369604e-05, + "loss": 2.8017, + "theoretical_loss": 3.3842195978015432, + "tokens_seen": 2359230464 + }, + { + "epoch": 0.43, + "learning_rate": 5.75830524795378e-05, + "loss": 2.7878, + "theoretical_loss": 3.384204426209771, + "tokens_seen": 2359361536 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9862067699432373, + "objective/train/theoretical_loss": 3.3841968408184435, + "objective/train/tokens_used": 729886176, + "theoretical_loss": 3.3841968408184435, + "tokens_seen": 2359427072 + }, + { + "epoch": 0.43, + "learning_rate": 5.757502808537956e-05, + "loss": 2.6802, + "theoretical_loss": 3.384189255696799, + "tokens_seen": 2359492608 + }, + { + "epoch": 0.43, + "learning_rate": 5.756700369122131e-05, + "loss": 2.6695, + "theoretical_loss": 3.3841740862624907, + "tokens_seen": 2359623680 + }, + { + "epoch": 0.43, + "learning_rate": 5.755897929706308e-05, + "loss": 2.6718, + "theoretical_loss": 3.38415891790671, + "tokens_seen": 2359754752 + }, + { + "epoch": 0.43, + "learning_rate": 5.755095490290483e-05, + "loss": 2.9466, + "theoretical_loss": 3.3841437506293195, + "tokens_seen": 2359885824 + }, + { + "epoch": 0.43, + "learning_rate": 5.754293050874659e-05, + "loss": 2.8172, + "theoretical_loss": 3.384128584430183, + "tokens_seen": 2360016896 + }, + { + "epoch": 0.43, + "learning_rate": 5.753490611458835e-05, + "loss": 2.8624, + "theoretical_loss": 3.3841134193091644, + "tokens_seen": 2360147968 + }, + { + "epoch": 0.43, + "learning_rate": 5.752688172043011e-05, + "loss": 2.8597, + "theoretical_loss": 3.3840982552661263, + "tokens_seen": 2360279040 + }, + { + "epoch": 0.43, + "learning_rate": 5.751885732627187e-05, + "loss": 2.8153, + "theoretical_loss": 3.3840830923009335, + "tokens_seen": 2360410112 + }, + { + "epoch": 0.43, + "learning_rate": 5.7510832932113625e-05, + "loss": 2.8208, + "theoretical_loss": 3.384067930413448, + "tokens_seen": 2360541184 + }, + { + "epoch": 0.43, + "learning_rate": 5.750280853795539e-05, + "loss": 2.8885, + "theoretical_loss": 3.3840527696035343, + "tokens_seen": 2360672256 + }, + { + "epoch": 0.43, + "learning_rate": 5.7494784143797145e-05, + "loss": 2.9305, + "theoretical_loss": 3.384037609871056, + "tokens_seen": 2360803328 + }, + { + "epoch": 0.43, + "learning_rate": 5.7486759749638905e-05, + "loss": 2.7255, + "theoretical_loss": 3.3840224512158765, + "tokens_seen": 2360934400 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.547431468963623, + "objective/train/theoretical_loss": 3.3840072936378593, + "objective/train/tokens_used": 731524576, + "theoretical_loss": 3.3840072936378593, + "tokens_seen": 2361065472 + }, + { + "epoch": 0.43, + "learning_rate": 5.7478735355480664e-05, + "loss": 2.6954, + "theoretical_loss": 3.3840072936378593, + "tokens_seen": 2361065472 + }, + { + "epoch": 0.43, + "learning_rate": 5.7470710961322424e-05, + "loss": 2.9092, + "theoretical_loss": 3.3839921371368686, + "tokens_seen": 2361196544 + }, + { + "epoch": 0.43, + "learning_rate": 5.7462686567164184e-05, + "loss": 2.8024, + "theoretical_loss": 3.3839769817127676, + "tokens_seen": 2361327616 + }, + { + "epoch": 0.43, + "learning_rate": 5.745466217300594e-05, + "loss": 2.7412, + "theoretical_loss": 3.38396182736542, + "tokens_seen": 2361458688 + }, + { + "epoch": 0.43, + "learning_rate": 5.74466377788477e-05, + "loss": 2.9804, + "theoretical_loss": 3.38394667409469, + "tokens_seen": 2361589760 + }, + { + "epoch": 0.43, + "learning_rate": 5.743861338468946e-05, + "loss": 2.8076, + "theoretical_loss": 3.3839315219004407, + "tokens_seen": 2361720832 + }, + { + "epoch": 0.43, + "learning_rate": 5.743058899053122e-05, + "loss": 2.873, + "theoretical_loss": 3.3839163707825364, + "tokens_seen": 2361851904 + }, + { + "epoch": 0.43, + "learning_rate": 5.742256459637298e-05, + "loss": 2.8507, + "theoretical_loss": 3.383901220740841, + "tokens_seen": 2361982976 + }, + { + "epoch": 0.43, + "learning_rate": 5.741454020221474e-05, + "loss": 2.8076, + "theoretical_loss": 3.383886071775218, + "tokens_seen": 2362114048 + }, + { + "epoch": 0.43, + "learning_rate": 5.74065158080565e-05, + "loss": 2.9191, + "theoretical_loss": 3.383870923885531, + "tokens_seen": 2362245120 + }, + { + "epoch": 0.43, + "learning_rate": 5.739849141389826e-05, + "loss": 2.8189, + "theoretical_loss": 3.383855777071645, + "tokens_seen": 2362376192 + }, + { + "epoch": 0.43, + "learning_rate": 5.739046701974001e-05, + "loss": 2.7455, + "theoretical_loss": 3.3838406313334226, + "tokens_seen": 2362507264 + }, + { + "epoch": 0.43, + "learning_rate": 5.738244262558177e-05, + "loss": 2.7955, + "theoretical_loss": 3.3838254866707285, + "tokens_seen": 2362638336 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1296897, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8064303398132324, + "objective/train/theoretical_loss": 3.3838179147426617, + "objective/train/tokens_used": 733162976, + "theoretical_loss": 3.3838179147426617, + "tokens_seen": 2362703872 + }, + { + "epoch": 0.43, + "learning_rate": 5.737441823142353e-05, + "loss": 2.7695, + "theoretical_loss": 3.383810343083426, + "tokens_seen": 2362769408 + }, + { + "epoch": 0.43, + "learning_rate": 5.736639383726529e-05, + "loss": 2.8729, + "theoretical_loss": 3.3837952005713805, + "tokens_seen": 2362900480 + }, + { + "epoch": 0.43, + "learning_rate": 5.735836944310705e-05, + "loss": 2.8079, + "theoretical_loss": 3.3837800591344545, + "tokens_seen": 2363031552 + }, + { + "epoch": 0.43, + "learning_rate": 5.73503450489488e-05, + "loss": 2.8238, + "theoretical_loss": 3.3837649187725125, + "tokens_seen": 2363162624 + }, + { + "epoch": 0.43, + "learning_rate": 5.734232065479057e-05, + "loss": 2.8355, + "theoretical_loss": 3.383749779485419, + "tokens_seen": 2363293696 + }, + { + "epoch": 0.43, + "learning_rate": 5.733429626063232e-05, + "loss": 2.707, + "theoretical_loss": 3.383734641273038, + "tokens_seen": 2363424768 + }, + { + "epoch": 0.43, + "learning_rate": 5.732627186647408e-05, + "loss": 2.8741, + "theoretical_loss": 3.383719504135233, + "tokens_seen": 2363555840 + }, + { + "epoch": 0.43, + "learning_rate": 5.731824747231584e-05, + "loss": 2.6753, + "theoretical_loss": 3.383704368071869, + "tokens_seen": 2363686912 + }, + { + "epoch": 0.43, + "learning_rate": 5.73102230781576e-05, + "loss": 2.7772, + "theoretical_loss": 3.383689233082809, + "tokens_seen": 2363817984 + }, + { + "epoch": 0.43, + "learning_rate": 5.730219868399936e-05, + "loss": 2.6709, + "theoretical_loss": 3.383674099167918, + "tokens_seen": 2363949056 + }, + { + "epoch": 0.43, + "learning_rate": 5.7294174289841115e-05, + "loss": 2.8573, + "theoretical_loss": 3.38365896632706, + "tokens_seen": 2364080128 + }, + { + "epoch": 0.43, + "learning_rate": 5.728614989568288e-05, + "loss": 2.8733, + "theoretical_loss": 3.3836438345601, + "tokens_seen": 2364211200 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1297792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8505661487579346, + "objective/train/theoretical_loss": 3.383628703866901, + "objective/train/tokens_used": 734801376, + "theoretical_loss": 3.383628703866901, + "tokens_seen": 2364342272 + }, + { + "epoch": 0.43, + "learning_rate": 5.7278125501524635e-05, + "loss": 2.7348, + "theoretical_loss": 3.383628703866901, + "tokens_seen": 2364342272 + }, + { + "epoch": 0.43, + "learning_rate": 5.7270101107366395e-05, + "loss": 2.7654, + "theoretical_loss": 3.3836135742473283, + "tokens_seen": 2364473344 + }, + { + "epoch": 0.43, + "learning_rate": 5.7262076713208155e-05, + "loss": 2.7622, + "theoretical_loss": 3.3835984457012453, + "tokens_seen": 2364604416 + }, + { + "epoch": 0.43, + "learning_rate": 5.725405231904991e-05, + "loss": 2.7652, + "theoretical_loss": 3.383583318228517, + "tokens_seen": 2364735488 + }, + { + "epoch": 0.43, + "learning_rate": 5.7246027924891675e-05, + "loss": 2.7463, + "theoretical_loss": 3.3835681918290077, + "tokens_seen": 2364866560 + }, + { + "epoch": 0.43, + "learning_rate": 5.723800353073343e-05, + "loss": 2.7084, + "theoretical_loss": 3.3835530665025813, + "tokens_seen": 2364997632 + }, + { + "epoch": 0.43, + "learning_rate": 5.7229979136575194e-05, + "loss": 2.8003, + "theoretical_loss": 3.3835379422491028, + "tokens_seen": 2365128704 + }, + { + "epoch": 0.43, + "learning_rate": 5.722195474241695e-05, + "loss": 2.7724, + "theoretical_loss": 3.3835228190684363, + "tokens_seen": 2365259776 + }, + { + "epoch": 0.43, + "learning_rate": 5.721393034825871e-05, + "loss": 2.8615, + "theoretical_loss": 3.3835076969604465, + "tokens_seen": 2365390848 + }, + { + "epoch": 0.43, + "learning_rate": 5.720590595410047e-05, + "loss": 2.7432, + "theoretical_loss": 3.383492575924998, + "tokens_seen": 2365521920 + }, + { + "epoch": 0.43, + "learning_rate": 5.719788155994222e-05, + "loss": 2.8073, + "theoretical_loss": 3.3834774559619545, + "tokens_seen": 2365652992 + }, + { + "epoch": 0.43, + "learning_rate": 5.718985716578399e-05, + "loss": 2.8, + "theoretical_loss": 3.3834623370711814, + "tokens_seen": 2365784064 + }, + { + "epoch": 0.43, + "learning_rate": 5.718183277162574e-05, + "loss": 2.8586, + "theoretical_loss": 3.3834472192525427, + "tokens_seen": 2365915136 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1298994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.880993604660034, + "objective/train/theoretical_loss": 3.3834396607452315, + "objective/train/tokens_used": 736439776, + "theoretical_loss": 3.3834396607452315, + "tokens_seen": 2365980672 + }, + { + "epoch": 0.43, + "learning_rate": 5.717380837746751e-05, + "loss": 2.748, + "theoretical_loss": 3.3834321025059033, + "tokens_seen": 2366046208 + }, + { + "epoch": 0.43, + "learning_rate": 5.716578398330926e-05, + "loss": 2.7962, + "theoretical_loss": 3.383416986831128, + "tokens_seen": 2366177280 + }, + { + "epoch": 0.43, + "learning_rate": 5.715775958915101e-05, + "loss": 2.9175, + "theoretical_loss": 3.383401872228081, + "tokens_seen": 2366308352 + }, + { + "epoch": 0.43, + "learning_rate": 5.714973519499278e-05, + "loss": 2.7201, + "theoretical_loss": 3.3833867586966266, + "tokens_seen": 2366439424 + }, + { + "epoch": 0.43, + "learning_rate": 5.714171080083453e-05, + "loss": 2.8739, + "theoretical_loss": 3.383371646236631, + "tokens_seen": 2366570496 + }, + { + "epoch": 0.43, + "learning_rate": 5.71336864066763e-05, + "loss": 2.7995, + "theoretical_loss": 3.383356534847957, + "tokens_seen": 2366701568 + }, + { + "epoch": 0.43, + "learning_rate": 5.712566201251805e-05, + "loss": 2.7746, + "theoretical_loss": 3.3833414245304705, + "tokens_seen": 2366832640 + }, + { + "epoch": 0.43, + "learning_rate": 5.711763761835982e-05, + "loss": 2.7344, + "theoretical_loss": 3.3833263152840356, + "tokens_seen": 2366963712 + }, + { + "epoch": 0.43, + "learning_rate": 5.710961322420157e-05, + "loss": 2.722, + "theoretical_loss": 3.383311207108518, + "tokens_seen": 2367094784 + }, + { + "epoch": 0.43, + "learning_rate": 5.710158883004334e-05, + "loss": 2.5951, + "theoretical_loss": 3.383296100003782, + "tokens_seen": 2367225856 + }, + { + "epoch": 0.43, + "learning_rate": 5.709356443588509e-05, + "loss": 2.7145, + "theoretical_loss": 3.383280993969692, + "tokens_seen": 2367356928 + }, + { + "epoch": 0.43, + "learning_rate": 5.7085540041726846e-05, + "loss": 2.7715, + "theoretical_loss": 3.3832658890061134, + "tokens_seen": 2367488000 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1299472, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0345065593719482, + "objective/train/theoretical_loss": 3.383250785112911, + "objective/train/tokens_used": 738078176, + "theoretical_loss": 3.383250785112911, + "tokens_seen": 2367619072 + }, + { + "epoch": 0.43, + "learning_rate": 5.707751564756861e-05, + "loss": 2.784, + "theoretical_loss": 3.383250785112911, + "tokens_seen": 2367619072 + }, + { + "epoch": 0.44, + "learning_rate": 5.7069491253410365e-05, + "loss": 2.6148, + "theoretical_loss": 3.383235682289949, + "tokens_seen": 2367750144 + }, + { + "epoch": 0.44, + "learning_rate": 5.706146685925213e-05, + "loss": 2.7107, + "theoretical_loss": 3.3832205805370936, + "tokens_seen": 2367881216 + }, + { + "epoch": 0.44, + "learning_rate": 5.7053442465093885e-05, + "loss": 2.8164, + "theoretical_loss": 3.383205479854209, + "tokens_seen": 2368012288 + }, + { + "epoch": 0.44, + "learning_rate": 5.704541807093565e-05, + "loss": 2.7386, + "theoretical_loss": 3.3831903802411603, + "tokens_seen": 2368143360 + }, + { + "epoch": 0.44, + "learning_rate": 5.7037393676777405e-05, + "loss": 2.8022, + "theoretical_loss": 3.3831752816978122, + "tokens_seen": 2368274432 + }, + { + "epoch": 0.44, + "learning_rate": 5.702936928261916e-05, + "loss": 2.6544, + "theoretical_loss": 3.38316018422403, + "tokens_seen": 2368405504 + }, + { + "epoch": 0.44, + "learning_rate": 5.7021344888460925e-05, + "loss": 2.7692, + "theoretical_loss": 3.3831450878196785, + "tokens_seen": 2368536576 + }, + { + "epoch": 0.44, + "learning_rate": 5.701332049430268e-05, + "loss": 2.8365, + "theoretical_loss": 3.383129992484623, + "tokens_seen": 2368667648 + }, + { + "epoch": 0.44, + "learning_rate": 5.7005296100144445e-05, + "loss": 2.803, + "theoretical_loss": 3.383114898218729, + "tokens_seen": 2368798720 + }, + { + "epoch": 0.44, + "learning_rate": 5.69972717059862e-05, + "loss": 2.807, + "theoretical_loss": 3.3830998050218613, + "tokens_seen": 2368929792 + }, + { + "epoch": 0.44, + "learning_rate": 5.6989247311827965e-05, + "loss": 2.9609, + "theoretical_loss": 3.3830847128938846, + "tokens_seen": 2369060864 + }, + { + "epoch": 0.44, + "learning_rate": 5.698122291766972e-05, + "loss": 2.7113, + "theoretical_loss": 3.3830696218346645, + "tokens_seen": 2369191936 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1300592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.462731122970581, + "objective/train/theoretical_loss": 3.383062076705796, + "objective/train/tokens_used": 739716576, + "theoretical_loss": 3.383062076705796, + "tokens_seen": 2369257472 + }, + { + "epoch": 0.44, + "learning_rate": 5.697319852351147e-05, + "loss": 2.6522, + "theoretical_loss": 3.383054531844066, + "tokens_seen": 2369323008 + }, + { + "epoch": 0.44, + "learning_rate": 5.696517412935324e-05, + "loss": 2.8167, + "theoretical_loss": 3.383039442921955, + "tokens_seen": 2369454080 + }, + { + "epoch": 0.44, + "learning_rate": 5.695714973519499e-05, + "loss": 2.8221, + "theoretical_loss": 3.3830243550681955, + "tokens_seen": 2369585152 + }, + { + "epoch": 0.44, + "learning_rate": 5.694912534103676e-05, + "loss": 2.7618, + "theoretical_loss": 3.383009268282654, + "tokens_seen": 2369716224 + }, + { + "epoch": 0.44, + "learning_rate": 5.694110094687851e-05, + "loss": 2.6241, + "theoretical_loss": 3.382994182565195, + "tokens_seen": 2369847296 + }, + { + "epoch": 0.44, + "learning_rate": 5.693307655272028e-05, + "loss": 2.8554, + "theoretical_loss": 3.3829790979156837, + "tokens_seen": 2369978368 + }, + { + "epoch": 0.44, + "learning_rate": 5.692505215856203e-05, + "loss": 2.6757, + "theoretical_loss": 3.3829640143339863, + "tokens_seen": 2370109440 + }, + { + "epoch": 0.44, + "learning_rate": 5.691702776440378e-05, + "loss": 2.6053, + "theoretical_loss": 3.3829489318199673, + "tokens_seen": 2370240512 + }, + { + "epoch": 0.44, + "learning_rate": 5.690900337024555e-05, + "loss": 2.7019, + "theoretical_loss": 3.382933850373493, + "tokens_seen": 2370371584 + }, + { + "epoch": 0.44, + "learning_rate": 5.69009789760873e-05, + "loss": 2.7461, + "theoretical_loss": 3.3829187699944274, + "tokens_seen": 2370502656 + }, + { + "epoch": 0.44, + "learning_rate": 5.689295458192907e-05, + "loss": 2.795, + "theoretical_loss": 3.3829036906826375, + "tokens_seen": 2370633728 + }, + { + "epoch": 0.44, + "learning_rate": 5.688493018777082e-05, + "loss": 2.7016, + "theoretical_loss": 3.3828886124379878, + "tokens_seen": 2370764800 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1301282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8106579780578613, + "objective/train/theoretical_loss": 3.3828735352603445, + "objective/train/tokens_used": 741354976, + "theoretical_loss": 3.3828735352603445, + "tokens_seen": 2370895872 + }, + { + "epoch": 0.44, + "learning_rate": 5.687690579361259e-05, + "loss": 2.6631, + "theoretical_loss": 3.3828735352603445, + "tokens_seen": 2370895872 + }, + { + "epoch": 0.44, + "learning_rate": 5.686888139945434e-05, + "loss": 2.7346, + "theoretical_loss": 3.3828584591495723, + "tokens_seen": 2371026944 + }, + { + "epoch": 0.44, + "learning_rate": 5.6860857005296096e-05, + "loss": 2.7316, + "theoretical_loss": 3.382843384105537, + "tokens_seen": 2371158016 + }, + { + "epoch": 0.44, + "learning_rate": 5.685283261113786e-05, + "loss": 2.646, + "theoretical_loss": 3.3828283101281045, + "tokens_seen": 2371289088 + }, + { + "epoch": 0.44, + "learning_rate": 5.6844808216979616e-05, + "loss": 2.6804, + "theoretical_loss": 3.38281323721714, + "tokens_seen": 2371420160 + }, + { + "epoch": 0.44, + "learning_rate": 5.683678382282138e-05, + "loss": 2.7237, + "theoretical_loss": 3.382798165372509, + "tokens_seen": 2371551232 + }, + { + "epoch": 0.44, + "learning_rate": 5.6828759428663136e-05, + "loss": 2.7476, + "theoretical_loss": 3.382783094594078, + "tokens_seen": 2371682304 + }, + { + "epoch": 0.44, + "learning_rate": 5.68207350345049e-05, + "loss": 2.8014, + "theoretical_loss": 3.382768024881712, + "tokens_seen": 2371813376 + }, + { + "epoch": 0.44, + "learning_rate": 5.6812710640346655e-05, + "loss": 2.8276, + "theoretical_loss": 3.3827529562352763, + "tokens_seen": 2371944448 + }, + { + "epoch": 0.44, + "learning_rate": 5.680468624618841e-05, + "loss": 2.8166, + "theoretical_loss": 3.382737888654637, + "tokens_seen": 2372075520 + }, + { + "epoch": 0.44, + "learning_rate": 5.6796661852030175e-05, + "loss": 2.68, + "theoretical_loss": 3.38272282213966, + "tokens_seen": 2372206592 + }, + { + "epoch": 0.44, + "learning_rate": 5.678863745787193e-05, + "loss": 2.7083, + "theoretical_loss": 3.3827077566902113, + "tokens_seen": 2372337664 + }, + { + "epoch": 0.44, + "learning_rate": 5.6780613063713695e-05, + "loss": 2.7768, + "theoretical_loss": 3.382692692306156, + "tokens_seen": 2372468736 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1302157, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.031487226486206, + "objective/train/theoretical_loss": 3.382685160513609, + "objective/train/tokens_used": 742993376, + "theoretical_loss": 3.382685160513609, + "tokens_seen": 2372534272 + }, + { + "epoch": 0.44, + "learning_rate": 5.677258866955545e-05, + "loss": 2.7334, + "theoretical_loss": 3.38267762898736, + "tokens_seen": 2372599808 + }, + { + "epoch": 0.44, + "learning_rate": 5.6764564275397215e-05, + "loss": 2.6291, + "theoretical_loss": 3.38266256673369, + "tokens_seen": 2372730880 + }, + { + "epoch": 0.44, + "learning_rate": 5.675653988123897e-05, + "loss": 2.7014, + "theoretical_loss": 3.3826475055450107, + "tokens_seen": 2372861952 + }, + { + "epoch": 0.44, + "learning_rate": 5.6748515487080735e-05, + "loss": 2.7827, + "theoretical_loss": 3.3826324454211885, + "tokens_seen": 2372993024 + }, + { + "epoch": 0.44, + "learning_rate": 5.674049109292249e-05, + "loss": 2.7505, + "theoretical_loss": 3.3826173863620896, + "tokens_seen": 2373124096 + }, + { + "epoch": 0.44, + "learning_rate": 5.673246669876424e-05, + "loss": 2.6665, + "theoretical_loss": 3.3826023283675792, + "tokens_seen": 2373255168 + }, + { + "epoch": 0.44, + "learning_rate": 5.672444230460601e-05, + "loss": 2.6076, + "theoretical_loss": 3.382587271437524, + "tokens_seen": 2373386240 + }, + { + "epoch": 0.44, + "learning_rate": 5.671641791044776e-05, + "loss": 2.8538, + "theoretical_loss": 3.3825722155717894, + "tokens_seen": 2373517312 + }, + { + "epoch": 0.44, + "learning_rate": 5.670839351628953e-05, + "loss": 2.6044, + "theoretical_loss": 3.3825571607702414, + "tokens_seen": 2373648384 + }, + { + "epoch": 0.44, + "learning_rate": 5.670036912213128e-05, + "loss": 2.7014, + "theoretical_loss": 3.3825421070327466, + "tokens_seen": 2373779456 + }, + { + "epoch": 0.44, + "learning_rate": 5.669234472797305e-05, + "loss": 2.6982, + "theoretical_loss": 3.3825270543591706, + "tokens_seen": 2373910528 + }, + { + "epoch": 0.44, + "learning_rate": 5.66843203338148e-05, + "loss": 2.7067, + "theoretical_loss": 3.3825120027493796, + "tokens_seen": 2374041600 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1302854, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0018539428710938, + "objective/train/theoretical_loss": 3.3824969522032395, + "objective/train/tokens_used": 744631776, + "theoretical_loss": 3.3824969522032395, + "tokens_seen": 2374172672 + }, + { + "epoch": 0.44, + "learning_rate": 5.6676295939656553e-05, + "loss": 2.8434, + "theoretical_loss": 3.3824969522032395, + "tokens_seen": 2374172672 + }, + { + "epoch": 0.44, + "learning_rate": 5.666827154549832e-05, + "loss": 2.6915, + "theoretical_loss": 3.3824819027206168, + "tokens_seen": 2374303744 + }, + { + "epoch": 0.44, + "learning_rate": 5.666024715134007e-05, + "loss": 2.7146, + "theoretical_loss": 3.382466854301377, + "tokens_seen": 2374434816 + }, + { + "epoch": 0.44, + "learning_rate": 5.665222275718184e-05, + "loss": 2.7207, + "theoretical_loss": 3.382451806945387, + "tokens_seen": 2374565888 + }, + { + "epoch": 0.44, + "learning_rate": 5.664419836302359e-05, + "loss": 2.6916, + "theoretical_loss": 3.3824367606525128, + "tokens_seen": 2374696960 + }, + { + "epoch": 0.44, + "learning_rate": 5.663617396886536e-05, + "loss": 2.5943, + "theoretical_loss": 3.3824217154226206, + "tokens_seen": 2374828032 + }, + { + "epoch": 0.44, + "learning_rate": 5.662814957470711e-05, + "loss": 2.7399, + "theoretical_loss": 3.382406671255576, + "tokens_seen": 2374959104 + }, + { + "epoch": 0.44, + "learning_rate": 5.6620125180548866e-05, + "loss": 2.7384, + "theoretical_loss": 3.3823916281512463, + "tokens_seen": 2375090176 + }, + { + "epoch": 0.44, + "learning_rate": 5.661210078639063e-05, + "loss": 2.6197, + "theoretical_loss": 3.382376586109497, + "tokens_seen": 2375221248 + }, + { + "epoch": 0.44, + "learning_rate": 5.6604076392232386e-05, + "loss": 2.7029, + "theoretical_loss": 3.3823615451301947, + "tokens_seen": 2375352320 + }, + { + "epoch": 0.44, + "learning_rate": 5.659605199807415e-05, + "loss": 2.7693, + "theoretical_loss": 3.382346505213206, + "tokens_seen": 2375483392 + }, + { + "epoch": 0.44, + "learning_rate": 5.6588027603915906e-05, + "loss": 2.5935, + "theoretical_loss": 3.382331466358397, + "tokens_seen": 2375614464 + }, + { + "epoch": 0.44, + "learning_rate": 5.658000320975767e-05, + "loss": 2.7039, + "theoretical_loss": 3.382316428565634, + "tokens_seen": 2375745536 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1303970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.590346336364746, + "objective/train/theoretical_loss": 3.382308910067478, + "objective/train/tokens_used": 746270176, + "theoretical_loss": 3.382308910067478, + "tokens_seen": 2375811072 + }, + { + "epoch": 0.44, + "learning_rate": 5.6571978815599425e-05, + "loss": 2.6357, + "theoretical_loss": 3.3823013918347833, + "tokens_seen": 2375876608 + }, + { + "epoch": 0.44, + "learning_rate": 5.656395442144118e-05, + "loss": 2.6932, + "theoretical_loss": 3.3822863561657117, + "tokens_seen": 2376007680 + }, + { + "epoch": 0.44, + "learning_rate": 5.6555930027282945e-05, + "loss": 2.716, + "theoretical_loss": 3.382271321558286, + "tokens_seen": 2376138752 + }, + { + "epoch": 0.44, + "learning_rate": 5.65479056331247e-05, + "loss": 2.6002, + "theoretical_loss": 3.3822562880123717, + "tokens_seen": 2376269824 + }, + { + "epoch": 0.44, + "learning_rate": 5.6539881238966465e-05, + "loss": 2.6741, + "theoretical_loss": 3.382241255527836, + "tokens_seen": 2376400896 + }, + { + "epoch": 0.44, + "learning_rate": 5.653185684480822e-05, + "loss": 2.8283, + "theoretical_loss": 3.3822262241045444, + "tokens_seen": 2376531968 + }, + { + "epoch": 0.44, + "learning_rate": 5.6523832450649985e-05, + "loss": 2.7868, + "theoretical_loss": 3.382211193742365, + "tokens_seen": 2376663040 + }, + { + "epoch": 0.44, + "learning_rate": 5.651580805649174e-05, + "loss": 2.7779, + "theoretical_loss": 3.3821961644411633, + "tokens_seen": 2376794112 + }, + { + "epoch": 0.44, + "learning_rate": 5.650778366233349e-05, + "loss": 2.6668, + "theoretical_loss": 3.3821811362008067, + "tokens_seen": 2376925184 + }, + { + "epoch": 0.44, + "learning_rate": 5.649975926817526e-05, + "loss": 2.8336, + "theoretical_loss": 3.382166109021161, + "tokens_seen": 2377056256 + }, + { + "epoch": 0.44, + "learning_rate": 5.649173487401701e-05, + "loss": 2.7752, + "theoretical_loss": 3.3821510829020935, + "tokens_seen": 2377187328 + }, + { + "epoch": 0.44, + "learning_rate": 5.648371047985878e-05, + "loss": 2.5915, + "theoretical_loss": 3.3821360578434705, + "tokens_seen": 2377318400 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1304587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7148685455322266, + "objective/train/theoretical_loss": 3.3821210338451593, + "objective/train/tokens_used": 747908576, + "theoretical_loss": 3.3821210338451593, + "tokens_seen": 2377449472 + }, + { + "epoch": 0.44, + "learning_rate": 5.647568608570053e-05, + "loss": 2.7112, + "theoretical_loss": 3.3821210338451593, + "tokens_seen": 2377449472 + }, + { + "epoch": 0.44, + "learning_rate": 5.64676616915423e-05, + "loss": 2.5038, + "theoretical_loss": 3.382106010907026, + "tokens_seen": 2377580544 + }, + { + "epoch": 0.44, + "learning_rate": 5.645963729738405e-05, + "loss": 2.5681, + "theoretical_loss": 3.382090989028937, + "tokens_seen": 2377711616 + }, + { + "epoch": 0.44, + "learning_rate": 5.645161290322582e-05, + "loss": 2.6103, + "theoretical_loss": 3.3820759682107595, + "tokens_seen": 2377842688 + }, + { + "epoch": 0.44, + "learning_rate": 5.644358850906757e-05, + "loss": 2.6683, + "theoretical_loss": 3.3820609484523603, + "tokens_seen": 2377973760 + }, + { + "epoch": 0.44, + "learning_rate": 5.6435564114909324e-05, + "loss": 2.7189, + "theoretical_loss": 3.3820459297536067, + "tokens_seen": 2378104832 + }, + { + "epoch": 0.44, + "learning_rate": 5.642753972075109e-05, + "loss": 2.8319, + "theoretical_loss": 3.3820309121143652, + "tokens_seen": 2378235904 + }, + { + "epoch": 0.44, + "learning_rate": 5.641951532659284e-05, + "loss": 2.578, + "theoretical_loss": 3.3820158955345025, + "tokens_seen": 2378366976 + }, + { + "epoch": 0.44, + "learning_rate": 5.641149093243461e-05, + "loss": 2.5791, + "theoretical_loss": 3.3820008800138854, + "tokens_seen": 2378498048 + }, + { + "epoch": 0.44, + "learning_rate": 5.640346653827636e-05, + "loss": 2.7067, + "theoretical_loss": 3.381985865552381, + "tokens_seen": 2378629120 + }, + { + "epoch": 0.44, + "learning_rate": 5.639544214411813e-05, + "loss": 2.6642, + "theoretical_loss": 3.3819708521498564, + "tokens_seen": 2378760192 + }, + { + "epoch": 0.44, + "learning_rate": 5.638741774995988e-05, + "loss": 2.4669, + "theoretical_loss": 3.381955839806178, + "tokens_seen": 2378891264 + }, + { + "epoch": 0.44, + "learning_rate": 5.6379393355801636e-05, + "loss": 2.6951, + "theoretical_loss": 3.3819408285212136, + "tokens_seen": 2379022336 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1305970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3961341381073, + "objective/train/theoretical_loss": 3.3819333232757076, + "objective/train/tokens_used": 749546976, + "theoretical_loss": 3.3819333232757076, + "tokens_seen": 2379087872 + }, + { + "epoch": 0.44, + "learning_rate": 5.63713689616434e-05, + "loss": 2.5991, + "theoretical_loss": 3.3819258182948295, + "tokens_seen": 2379153408 + }, + { + "epoch": 0.44, + "learning_rate": 5.6363344567485156e-05, + "loss": 2.516, + "theoretical_loss": 3.3819108091268935, + "tokens_seen": 2379284480 + }, + { + "epoch": 0.44, + "learning_rate": 5.635532017332692e-05, + "loss": 2.6829, + "theoretical_loss": 3.3818958010172717, + "tokens_seen": 2379415552 + }, + { + "epoch": 0.44, + "learning_rate": 5.6347295779168676e-05, + "loss": 2.7288, + "theoretical_loss": 3.381880793965832, + "tokens_seen": 2379546624 + }, + { + "epoch": 0.44, + "learning_rate": 5.633927138501044e-05, + "loss": 2.5515, + "theoretical_loss": 3.3818657879724414, + "tokens_seen": 2379677696 + }, + { + "epoch": 0.44, + "learning_rate": 5.6331246990852196e-05, + "loss": 2.617, + "theoretical_loss": 3.3818507830369664, + "tokens_seen": 2379808768 + }, + { + "epoch": 0.44, + "learning_rate": 5.632322259669395e-05, + "loss": 2.7448, + "theoretical_loss": 3.3818357791592746, + "tokens_seen": 2379939840 + }, + { + "epoch": 0.44, + "learning_rate": 5.6315198202535715e-05, + "loss": 2.7401, + "theoretical_loss": 3.381820776339233, + "tokens_seen": 2380070912 + }, + { + "epoch": 0.44, + "learning_rate": 5.630717380837747e-05, + "loss": 2.5827, + "theoretical_loss": 3.38180577457671, + "tokens_seen": 2380201984 + }, + { + "epoch": 0.44, + "learning_rate": 5.6299149414219235e-05, + "loss": 2.735, + "theoretical_loss": 3.381790773871571, + "tokens_seen": 2380333056 + }, + { + "epoch": 0.44, + "learning_rate": 5.629112502006099e-05, + "loss": 2.6706, + "theoretical_loss": 3.381775774223684, + "tokens_seen": 2380464128 + }, + { + "epoch": 0.44, + "learning_rate": 5.6283100625902755e-05, + "loss": 2.7461, + "theoretical_loss": 3.3817607756329164, + "tokens_seen": 2380595200 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1307290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9881303310394287, + "objective/train/theoretical_loss": 3.3817457780991353, + "objective/train/tokens_used": 751185376, + "theoretical_loss": 3.3817457780991353, + "tokens_seen": 2380726272 + }, + { + "epoch": 0.44, + "learning_rate": 5.627507623174451e-05, + "loss": 2.6021, + "theoretical_loss": 3.3817457780991353, + "tokens_seen": 2380726272 + }, + { + "epoch": 0.44, + "learning_rate": 5.626705183758626e-05, + "loss": 2.6797, + "theoretical_loss": 3.3817307816222084, + "tokens_seen": 2380857344 + }, + { + "epoch": 0.44, + "learning_rate": 5.625902744342803e-05, + "loss": 2.6479, + "theoretical_loss": 3.381715786202003, + "tokens_seen": 2380988416 + }, + { + "epoch": 0.44, + "learning_rate": 5.625100304926978e-05, + "loss": 2.54, + "theoretical_loss": 3.381700791838386, + "tokens_seen": 2381119488 + }, + { + "epoch": 0.44, + "learning_rate": 5.624297865511155e-05, + "loss": 2.5575, + "theoretical_loss": 3.381685798531225, + "tokens_seen": 2381250560 + }, + { + "epoch": 0.44, + "learning_rate": 5.62349542609533e-05, + "loss": 2.6102, + "theoretical_loss": 3.3816708062803875, + "tokens_seen": 2381381632 + }, + { + "epoch": 0.44, + "learning_rate": 5.622692986679506e-05, + "loss": 2.6952, + "theoretical_loss": 3.3816558150857414, + "tokens_seen": 2381512704 + }, + { + "epoch": 0.44, + "learning_rate": 5.621890547263682e-05, + "loss": 2.497, + "theoretical_loss": 3.3816408249471532, + "tokens_seen": 2381643776 + }, + { + "epoch": 0.44, + "learning_rate": 5.6210881078478574e-05, + "loss": 2.6587, + "theoretical_loss": 3.381625835864491, + "tokens_seen": 2381774848 + }, + { + "epoch": 0.44, + "learning_rate": 5.620285668432034e-05, + "loss": 2.6085, + "theoretical_loss": 3.3816108478376226, + "tokens_seen": 2381905920 + }, + { + "epoch": 0.44, + "learning_rate": 5.6194832290162094e-05, + "loss": 2.6701, + "theoretical_loss": 3.3815958608664145, + "tokens_seen": 2382036992 + }, + { + "epoch": 0.44, + "learning_rate": 5.6186807896003853e-05, + "loss": 2.6523, + "theoretical_loss": 3.3815808749507354, + "tokens_seen": 2382168064 + }, + { + "epoch": 0.44, + "learning_rate": 5.6178783501845613e-05, + "loss": 2.6501, + "theoretical_loss": 3.381565890090452, + "tokens_seen": 2382299136 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1307945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9193787574768066, + "objective/train/theoretical_loss": 3.381558398056043, + "objective/train/tokens_used": 752823776, + "theoretical_loss": 3.381558398056043, + "tokens_seen": 2382364672 + }, + { + "epoch": 0.44, + "learning_rate": 5.617075910768737e-05, + "loss": 2.7432, + "theoretical_loss": 3.3815509062854328, + "tokens_seen": 2382430208 + }, + { + "epoch": 0.44, + "learning_rate": 5.616273471352913e-05, + "loss": 2.5401, + "theoretical_loss": 3.381535923535545, + "tokens_seen": 2382561280 + }, + { + "epoch": 0.44, + "learning_rate": 5.615471031937089e-05, + "loss": 2.6953, + "theoretical_loss": 3.3815209418406558, + "tokens_seen": 2382692352 + }, + { + "epoch": 0.44, + "learning_rate": 5.614668592521265e-05, + "loss": 2.5979, + "theoretical_loss": 3.3815059612006335, + "tokens_seen": 2382823424 + }, + { + "epoch": 0.44, + "learning_rate": 5.6138661531054406e-05, + "loss": 2.6828, + "theoretical_loss": 3.381490981615346, + "tokens_seen": 2382954496 + }, + { + "epoch": 0.44, + "learning_rate": 5.6130637136896166e-05, + "loss": 2.7251, + "theoretical_loss": 3.38147600308466, + "tokens_seen": 2383085568 + }, + { + "epoch": 0.44, + "learning_rate": 5.6122612742737926e-05, + "loss": 2.4586, + "theoretical_loss": 3.381461025608444, + "tokens_seen": 2383216640 + }, + { + "epoch": 0.44, + "learning_rate": 5.6114588348579686e-05, + "loss": 2.5998, + "theoretical_loss": 3.381446049186566, + "tokens_seen": 2383347712 + }, + { + "epoch": 0.44, + "learning_rate": 5.6106563954421446e-05, + "loss": 2.6438, + "theoretical_loss": 3.3814310738188933, + "tokens_seen": 2383478784 + }, + { + "epoch": 0.44, + "learning_rate": 5.6098539560263206e-05, + "loss": 2.6546, + "theoretical_loss": 3.3814160995052935, + "tokens_seen": 2383609856 + }, + { + "epoch": 0.44, + "learning_rate": 5.609051516610496e-05, + "loss": 2.6753, + "theoretical_loss": 3.3814011262456356, + "tokens_seen": 2383740928 + }, + { + "epoch": 0.44, + "learning_rate": 5.608249077194672e-05, + "loss": 2.5646, + "theoretical_loss": 3.3813861540397863, + "tokens_seen": 2383872000 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1309221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7739500999450684, + "objective/train/theoretical_loss": 3.381371182887614, + "objective/train/tokens_used": 754462176, + "theoretical_loss": 3.381371182887614, + "tokens_seen": 2384003072 + }, + { + "epoch": 0.44, + "learning_rate": 5.607446637778848e-05, + "loss": 2.7357, + "theoretical_loss": 3.381371182887614, + "tokens_seen": 2384003072 + }, + { + "epoch": 0.44, + "learning_rate": 5.606644198363024e-05, + "loss": 2.6646, + "theoretical_loss": 3.3813562127889867, + "tokens_seen": 2384134144 + }, + { + "epoch": 0.45, + "learning_rate": 5.6058417589472e-05, + "loss": 2.5678, + "theoretical_loss": 3.3813412437437718, + "tokens_seen": 2384265216 + }, + { + "epoch": 0.45, + "learning_rate": 5.605039319531376e-05, + "loss": 2.5437, + "theoretical_loss": 3.381326275751838, + "tokens_seen": 2384396288 + }, + { + "epoch": 0.45, + "learning_rate": 5.604236880115552e-05, + "loss": 2.7734, + "theoretical_loss": 3.3813113088130526, + "tokens_seen": 2384527360 + }, + { + "epoch": 0.45, + "learning_rate": 5.603434440699727e-05, + "loss": 2.6464, + "theoretical_loss": 3.3812963429272846, + "tokens_seen": 2384658432 + }, + { + "epoch": 0.45, + "learning_rate": 5.602632001283903e-05, + "loss": 2.514, + "theoretical_loss": 3.381281378094401, + "tokens_seen": 2384789504 + }, + { + "epoch": 0.45, + "learning_rate": 5.601829561868079e-05, + "loss": 2.7409, + "theoretical_loss": 3.38126641431427, + "tokens_seen": 2384920576 + }, + { + "epoch": 0.45, + "learning_rate": 5.601027122452255e-05, + "loss": 2.7185, + "theoretical_loss": 3.3812514515867607, + "tokens_seen": 2385051648 + }, + { + "epoch": 0.45, + "learning_rate": 5.600224683036431e-05, + "loss": 2.5876, + "theoretical_loss": 3.38123648991174, + "tokens_seen": 2385182720 + }, + { + "epoch": 0.45, + "learning_rate": 5.5994222436206064e-05, + "loss": 2.5251, + "theoretical_loss": 3.3812215292890766, + "tokens_seen": 2385313792 + }, + { + "epoch": 0.45, + "learning_rate": 5.598619804204783e-05, + "loss": 2.5594, + "theoretical_loss": 3.3812065697186386, + "tokens_seen": 2385444864 + }, + { + "epoch": 0.45, + "learning_rate": 5.5978173647889584e-05, + "loss": 2.6084, + "theoretical_loss": 3.381191611200294, + "tokens_seen": 2385575936 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1309756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.809346914291382, + "objective/train/theoretical_loss": 3.381184132335616, + "objective/train/tokens_used": 756100576, + "theoretical_loss": 3.381184132335616, + "tokens_seen": 2385641472 + }, + { + "epoch": 0.45, + "learning_rate": 5.5970149253731344e-05, + "loss": 2.6377, + "theoretical_loss": 3.381176653733912, + "tokens_seen": 2385707008 + }, + { + "epoch": 0.45, + "learning_rate": 5.5962124859573104e-05, + "loss": 2.6447, + "theoretical_loss": 3.3811616973193592, + "tokens_seen": 2385838080 + }, + { + "epoch": 0.45, + "learning_rate": 5.5954100465414864e-05, + "loss": 2.6085, + "theoretical_loss": 3.3811467419565044, + "tokens_seen": 2385969152 + }, + { + "epoch": 0.45, + "learning_rate": 5.5946076071256624e-05, + "loss": 2.5808, + "theoretical_loss": 3.381131787645217, + "tokens_seen": 2386100224 + }, + { + "epoch": 0.45, + "learning_rate": 5.593805167709838e-05, + "loss": 2.6756, + "theoretical_loss": 3.381116834385364, + "tokens_seen": 2386231296 + }, + { + "epoch": 0.45, + "learning_rate": 5.593002728294014e-05, + "loss": 2.4824, + "theoretical_loss": 3.381101882176814, + "tokens_seen": 2386362368 + }, + { + "epoch": 0.45, + "learning_rate": 5.5922002888781897e-05, + "loss": 2.7075, + "theoretical_loss": 3.3810869310194356, + "tokens_seen": 2386493440 + }, + { + "epoch": 0.45, + "learning_rate": 5.5913978494623656e-05, + "loss": 2.6192, + "theoretical_loss": 3.3810719809130974, + "tokens_seen": 2386624512 + }, + { + "epoch": 0.45, + "learning_rate": 5.5905954100465416e-05, + "loss": 2.4395, + "theoretical_loss": 3.381057031857667, + "tokens_seen": 2386755584 + }, + { + "epoch": 0.45, + "learning_rate": 5.589792970630717e-05, + "loss": 2.5051, + "theoretical_loss": 3.3810420838530137, + "tokens_seen": 2386886656 + }, + { + "epoch": 0.45, + "learning_rate": 5.5889905312148936e-05, + "loss": 2.5776, + "theoretical_loss": 3.3810271368990055, + "tokens_seen": 2387017728 + }, + { + "epoch": 0.45, + "learning_rate": 5.588188091799069e-05, + "loss": 2.6031, + "theoretical_loss": 3.3810121909955106, + "tokens_seen": 2387148800 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1311026, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2848219871520996, + "objective/train/theoretical_loss": 3.3809972461423983, + "objective/train/tokens_used": 757738976, + "theoretical_loss": 3.3809972461423983, + "tokens_seen": 2387279872 + }, + { + "epoch": 0.45, + "learning_rate": 5.5873856523832456e-05, + "loss": 2.6379, + "theoretical_loss": 3.3809972461423983, + "tokens_seen": 2387279872 + }, + { + "epoch": 0.45, + "learning_rate": 5.586583212967421e-05, + "loss": 2.5815, + "theoretical_loss": 3.3809823023395364, + "tokens_seen": 2387410944 + }, + { + "epoch": 0.45, + "learning_rate": 5.585780773551597e-05, + "loss": 2.5957, + "theoretical_loss": 3.3809673595867933, + "tokens_seen": 2387542016 + }, + { + "epoch": 0.45, + "learning_rate": 5.584978334135773e-05, + "loss": 2.4613, + "theoretical_loss": 3.3809524178840387, + "tokens_seen": 2387673088 + }, + { + "epoch": 0.45, + "learning_rate": 5.584175894719948e-05, + "loss": 2.6596, + "theoretical_loss": 3.38093747723114, + "tokens_seen": 2387804160 + }, + { + "epoch": 0.45, + "learning_rate": 5.583373455304125e-05, + "loss": 2.6319, + "theoretical_loss": 3.380922537627966, + "tokens_seen": 2387935232 + }, + { + "epoch": 0.45, + "learning_rate": 5.5825710158883e-05, + "loss": 2.4635, + "theoretical_loss": 3.3809075990743858, + "tokens_seen": 2388066304 + }, + { + "epoch": 0.45, + "learning_rate": 5.581768576472477e-05, + "loss": 2.5646, + "theoretical_loss": 3.3808926615702677, + "tokens_seen": 2388197376 + }, + { + "epoch": 0.45, + "learning_rate": 5.580966137056652e-05, + "loss": 2.5483, + "theoretical_loss": 3.380877725115481, + "tokens_seen": 2388328448 + }, + { + "epoch": 0.45, + "learning_rate": 5.580163697640829e-05, + "loss": 2.5691, + "theoretical_loss": 3.380862789709893, + "tokens_seen": 2388459520 + }, + { + "epoch": 0.45, + "learning_rate": 5.579361258225004e-05, + "loss": 2.6872, + "theoretical_loss": 3.380847855353374, + "tokens_seen": 2388590592 + }, + { + "epoch": 0.45, + "learning_rate": 5.5785588188091795e-05, + "loss": 2.6173, + "theoretical_loss": 3.380832922045792, + "tokens_seen": 2388721664 + }, + { + "epoch": 0.45, + "learning_rate": 5.577756379393356e-05, + "loss": 2.5909, + "theoretical_loss": 3.380817989787016, + "tokens_seen": 2388852736 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1311624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7213802337646484, + "objective/train/theoretical_loss": 3.3808105240508888, + "objective/train/tokens_used": 759377376, + "theoretical_loss": 3.3808105240508888, + "tokens_seen": 2388918272 + }, + { + "epoch": 0.45, + "learning_rate": 5.5769539399775314e-05, + "loss": 2.4739, + "theoretical_loss": 3.380803058576914, + "tokens_seen": 2388983808 + }, + { + "epoch": 0.45, + "learning_rate": 5.576151500561708e-05, + "loss": 2.6132, + "theoretical_loss": 3.3807881284153565, + "tokens_seen": 2389114880 + }, + { + "epoch": 0.45, + "learning_rate": 5.5753490611458834e-05, + "loss": 2.701, + "theoretical_loss": 3.3807731993022108, + "tokens_seen": 2389245952 + }, + { + "epoch": 0.45, + "learning_rate": 5.57454662173006e-05, + "loss": 2.6447, + "theoretical_loss": 3.380758271237346, + "tokens_seen": 2389377024 + }, + { + "epoch": 0.45, + "learning_rate": 5.5737441823142354e-05, + "loss": 2.6681, + "theoretical_loss": 3.3807433442206314, + "tokens_seen": 2389508096 + }, + { + "epoch": 0.45, + "learning_rate": 5.572941742898411e-05, + "loss": 2.7169, + "theoretical_loss": 3.380728418251936, + "tokens_seen": 2389639168 + }, + { + "epoch": 0.45, + "learning_rate": 5.5721393034825874e-05, + "loss": 2.6528, + "theoretical_loss": 3.380713493331129, + "tokens_seen": 2389770240 + }, + { + "epoch": 0.45, + "learning_rate": 5.571336864066763e-05, + "loss": 2.7172, + "theoretical_loss": 3.3806985694580782, + "tokens_seen": 2389901312 + }, + { + "epoch": 0.45, + "learning_rate": 5.5705344246509394e-05, + "loss": 2.6488, + "theoretical_loss": 3.3806836466326535, + "tokens_seen": 2390032384 + }, + { + "epoch": 0.45, + "learning_rate": 5.569731985235115e-05, + "loss": 2.6782, + "theoretical_loss": 3.3806687248547242, + "tokens_seen": 2390163456 + }, + { + "epoch": 0.45, + "learning_rate": 5.5689295458192913e-05, + "loss": 2.5, + "theoretical_loss": 3.380653804124158, + "tokens_seen": 2390294528 + }, + { + "epoch": 0.45, + "learning_rate": 5.5681271064034667e-05, + "loss": 2.6104, + "theoretical_loss": 3.3806388844408257, + "tokens_seen": 2390425600 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1312746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.828706979751587, + "objective/train/theoretical_loss": 3.380623965804595, + "objective/train/tokens_used": 761015776, + "theoretical_loss": 3.380623965804595, + "tokens_seen": 2390556672 + }, + { + "epoch": 0.45, + "learning_rate": 5.567324666987642e-05, + "loss": 2.5251, + "theoretical_loss": 3.380623965804595, + "tokens_seen": 2390556672 + }, + { + "epoch": 0.45, + "learning_rate": 5.5665222275718186e-05, + "loss": 2.5664, + "theoretical_loss": 3.3806090482153355, + "tokens_seen": 2390687744 + }, + { + "epoch": 0.45, + "learning_rate": 5.565719788155994e-05, + "loss": 2.5467, + "theoretical_loss": 3.3805941316729164, + "tokens_seen": 2390818816 + }, + { + "epoch": 0.45, + "learning_rate": 5.5649173487401706e-05, + "loss": 2.5371, + "theoretical_loss": 3.380579216177207, + "tokens_seen": 2390949888 + }, + { + "epoch": 0.45, + "learning_rate": 5.564114909324346e-05, + "loss": 2.6228, + "theoretical_loss": 3.380564301728076, + "tokens_seen": 2391080960 + }, + { + "epoch": 0.45, + "learning_rate": 5.5633124699085226e-05, + "loss": 2.6312, + "theoretical_loss": 3.380549388325393, + "tokens_seen": 2391212032 + }, + { + "epoch": 0.45, + "learning_rate": 5.562510030492698e-05, + "loss": 2.5165, + "theoretical_loss": 3.3805344759690272, + "tokens_seen": 2391343104 + }, + { + "epoch": 0.45, + "learning_rate": 5.561707591076873e-05, + "loss": 2.5614, + "theoretical_loss": 3.3805195646588473, + "tokens_seen": 2391474176 + }, + { + "epoch": 0.45, + "learning_rate": 5.56090515166105e-05, + "loss": 2.6058, + "theoretical_loss": 3.3805046543947235, + "tokens_seen": 2391605248 + }, + { + "epoch": 0.45, + "learning_rate": 5.560102712245225e-05, + "loss": 2.5771, + "theoretical_loss": 3.3804897451765243, + "tokens_seen": 2391736320 + }, + { + "epoch": 0.45, + "learning_rate": 5.559300272829402e-05, + "loss": 2.573, + "theoretical_loss": 3.3804748370041193, + "tokens_seen": 2391867392 + }, + { + "epoch": 0.45, + "learning_rate": 5.558497833413577e-05, + "loss": 2.6682, + "theoretical_loss": 3.380459929877378, + "tokens_seen": 2391998464 + }, + { + "epoch": 0.45, + "learning_rate": 5.557695393997754e-05, + "loss": 2.5988, + "theoretical_loss": 3.3804450237961694, + "tokens_seen": 2392129536 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1313476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7581470012664795, + "objective/train/theoretical_loss": 3.3804375711475996, + "objective/train/tokens_used": 762654176, + "theoretical_loss": 3.3804375711475996, + "tokens_seen": 2392195072 + }, + { + "epoch": 0.45, + "learning_rate": 5.556892954581929e-05, + "loss": 2.7053, + "theoretical_loss": 3.380430118760364, + "tokens_seen": 2392260608 + }, + { + "epoch": 0.45, + "learning_rate": 5.5560905151661045e-05, + "loss": 2.6768, + "theoretical_loss": 3.380415214769829, + "tokens_seen": 2392391680 + }, + { + "epoch": 0.45, + "learning_rate": 5.555288075750281e-05, + "loss": 2.6145, + "theoretical_loss": 3.380400311824436, + "tokens_seen": 2392522752 + }, + { + "epoch": 0.45, + "learning_rate": 5.5544856363344565e-05, + "loss": 2.4482, + "theoretical_loss": 3.3803854099240533, + "tokens_seen": 2392653824 + }, + { + "epoch": 0.45, + "learning_rate": 5.553683196918633e-05, + "loss": 2.6114, + "theoretical_loss": 3.380370509068551, + "tokens_seen": 2392784896 + }, + { + "epoch": 0.45, + "learning_rate": 5.5528807575028084e-05, + "loss": 2.494, + "theoretical_loss": 3.3803556092577978, + "tokens_seen": 2392915968 + }, + { + "epoch": 0.45, + "learning_rate": 5.552078318086985e-05, + "loss": 2.5101, + "theoretical_loss": 3.380340710491664, + "tokens_seen": 2393047040 + }, + { + "epoch": 0.45, + "learning_rate": 5.5512758786711604e-05, + "loss": 2.5249, + "theoretical_loss": 3.3803258127700193, + "tokens_seen": 2393178112 + }, + { + "epoch": 0.45, + "learning_rate": 5.550473439255337e-05, + "loss": 2.7193, + "theoretical_loss": 3.380310916092732, + "tokens_seen": 2393309184 + }, + { + "epoch": 0.45, + "learning_rate": 5.5496709998395124e-05, + "loss": 2.6344, + "theoretical_loss": 3.3802960204596735, + "tokens_seen": 2393440256 + }, + { + "epoch": 0.45, + "learning_rate": 5.548868560423688e-05, + "loss": 2.6615, + "theoretical_loss": 3.380281125870712, + "tokens_seen": 2393571328 + }, + { + "epoch": 0.45, + "learning_rate": 5.5480661210078644e-05, + "loss": 2.5943, + "theoretical_loss": 3.3802662323257175, + "tokens_seen": 2393702400 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1314763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.905195951461792, + "objective/train/theoretical_loss": 3.38025133982456, + "objective/train/tokens_used": 764292576, + "theoretical_loss": 3.38025133982456, + "tokens_seen": 2393833472 + }, + { + "epoch": 0.45, + "learning_rate": 5.54726368159204e-05, + "loss": 2.6042, + "theoretical_loss": 3.38025133982456, + "tokens_seen": 2393833472 + }, + { + "epoch": 0.45, + "learning_rate": 5.5464612421762164e-05, + "loss": 2.5882, + "theoretical_loss": 3.380236448367109, + "tokens_seen": 2393964544 + }, + { + "epoch": 0.45, + "learning_rate": 5.545658802760392e-05, + "loss": 2.44, + "theoretical_loss": 3.380221557953234, + "tokens_seen": 2394095616 + }, + { + "epoch": 0.45, + "learning_rate": 5.5448563633445684e-05, + "loss": 2.714, + "theoretical_loss": 3.3802066685828054, + "tokens_seen": 2394226688 + }, + { + "epoch": 0.45, + "learning_rate": 5.544053923928744e-05, + "loss": 2.4515, + "theoretical_loss": 3.380191780255692, + "tokens_seen": 2394357760 + }, + { + "epoch": 0.45, + "learning_rate": 5.543251484512919e-05, + "loss": 2.6101, + "theoretical_loss": 3.380176892971764, + "tokens_seen": 2394488832 + }, + { + "epoch": 0.45, + "learning_rate": 5.5424490450970956e-05, + "loss": 2.5848, + "theoretical_loss": 3.3801620067308917, + "tokens_seen": 2394619904 + }, + { + "epoch": 0.45, + "learning_rate": 5.541646605681271e-05, + "loss": 2.5574, + "theoretical_loss": 3.3801471215329446, + "tokens_seen": 2394750976 + }, + { + "epoch": 0.45, + "learning_rate": 5.5408441662654476e-05, + "loss": 2.5288, + "theoretical_loss": 3.3801322373777927, + "tokens_seen": 2394882048 + }, + { + "epoch": 0.45, + "learning_rate": 5.540041726849623e-05, + "loss": 2.5644, + "theoretical_loss": 3.3801173542653054, + "tokens_seen": 2395013120 + }, + { + "epoch": 0.45, + "learning_rate": 5.5392392874337996e-05, + "loss": 2.5046, + "theoretical_loss": 3.3801024721953525, + "tokens_seen": 2395144192 + }, + { + "epoch": 0.45, + "learning_rate": 5.538436848017975e-05, + "loss": 2.4909, + "theoretical_loss": 3.3800875911678046, + "tokens_seen": 2395275264 + }, + { + "epoch": 0.45, + "learning_rate": 5.53763440860215e-05, + "loss": 2.4923, + "theoretical_loss": 3.3800727111825317, + "tokens_seen": 2395406336 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1315396, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6342225074768066, + "objective/train/theoretical_loss": 3.3800652715807074, + "objective/train/tokens_used": 765930976, + "theoretical_loss": 3.3800652715807074, + "tokens_seen": 2395471872 + }, + { + "epoch": 0.45, + "learning_rate": 5.536831969186327e-05, + "loss": 2.4363, + "theoretical_loss": 3.3800578322394026, + "tokens_seen": 2395537408 + }, + { + "epoch": 0.45, + "learning_rate": 5.536029529770502e-05, + "loss": 2.5553, + "theoretical_loss": 3.380042954338289, + "tokens_seen": 2395668480 + }, + { + "epoch": 0.45, + "learning_rate": 5.535227090354679e-05, + "loss": 2.6008, + "theoretical_loss": 3.3800280774790594, + "tokens_seen": 2395799552 + }, + { + "epoch": 0.45, + "learning_rate": 5.534424650938854e-05, + "loss": 2.6577, + "theoretical_loss": 3.380013201661585, + "tokens_seen": 2395930624 + }, + { + "epoch": 0.45, + "learning_rate": 5.533622211523031e-05, + "loss": 2.4719, + "theoretical_loss": 3.379998326885735, + "tokens_seen": 2396061696 + }, + { + "epoch": 0.45, + "learning_rate": 5.532819772107206e-05, + "loss": 2.4111, + "theoretical_loss": 3.3799834531513797, + "tokens_seen": 2396192768 + }, + { + "epoch": 0.45, + "learning_rate": 5.5320173326913815e-05, + "loss": 2.5345, + "theoretical_loss": 3.37996858045839, + "tokens_seen": 2396323840 + }, + { + "epoch": 0.45, + "learning_rate": 5.531214893275558e-05, + "loss": 2.4507, + "theoretical_loss": 3.379953708806635, + "tokens_seen": 2396454912 + }, + { + "epoch": 0.45, + "learning_rate": 5.5304124538597335e-05, + "loss": 2.5533, + "theoretical_loss": 3.379938838195985, + "tokens_seen": 2396585984 + }, + { + "epoch": 0.45, + "learning_rate": 5.52961001444391e-05, + "loss": 2.6565, + "theoretical_loss": 3.379923968626311, + "tokens_seen": 2396717056 + }, + { + "epoch": 0.45, + "learning_rate": 5.5288075750280855e-05, + "loss": 2.5447, + "theoretical_loss": 3.3799091000974824, + "tokens_seen": 2396848128 + }, + { + "epoch": 0.45, + "learning_rate": 5.528005135612262e-05, + "loss": 2.5323, + "theoretical_loss": 3.3798942326093697, + "tokens_seen": 2396979200 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1316445, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.639875650405884, + "objective/train/theoretical_loss": 3.379879366161843, + "objective/train/tokens_used": 767569376, + "theoretical_loss": 3.379879366161843, + "tokens_seen": 2397110272 + }, + { + "epoch": 0.45, + "learning_rate": 5.5272026961964374e-05, + "loss": 2.5263, + "theoretical_loss": 3.379879366161843, + "tokens_seen": 2397110272 + }, + { + "epoch": 0.45, + "learning_rate": 5.526400256780613e-05, + "loss": 2.569, + "theoretical_loss": 3.3798645007547727, + "tokens_seen": 2397241344 + }, + { + "epoch": 0.45, + "learning_rate": 5.5255978173647894e-05, + "loss": 2.5214, + "theoretical_loss": 3.3798496363880295, + "tokens_seen": 2397372416 + }, + { + "epoch": 0.45, + "learning_rate": 5.524795377948965e-05, + "loss": 2.3692, + "theoretical_loss": 3.3798347730614826, + "tokens_seen": 2397503488 + }, + { + "epoch": 0.45, + "learning_rate": 5.5239929385331414e-05, + "loss": 2.5421, + "theoretical_loss": 3.3798199107750038, + "tokens_seen": 2397634560 + }, + { + "epoch": 0.45, + "learning_rate": 5.523190499117317e-05, + "loss": 2.5866, + "theoretical_loss": 3.3798050495284624, + "tokens_seen": 2397765632 + }, + { + "epoch": 0.45, + "learning_rate": 5.5223880597014934e-05, + "loss": 2.4599, + "theoretical_loss": 3.379790189321729, + "tokens_seen": 2397896704 + }, + { + "epoch": 0.45, + "learning_rate": 5.521585620285669e-05, + "loss": 2.4592, + "theoretical_loss": 3.3797753301546742, + "tokens_seen": 2398027776 + }, + { + "epoch": 0.45, + "learning_rate": 5.5207831808698454e-05, + "loss": 2.4998, + "theoretical_loss": 3.3797604720271686, + "tokens_seen": 2398158848 + }, + { + "epoch": 0.45, + "learning_rate": 5.519980741454021e-05, + "loss": 2.4942, + "theoretical_loss": 3.3797456149390825, + "tokens_seen": 2398289920 + }, + { + "epoch": 0.45, + "learning_rate": 5.519178302038196e-05, + "loss": 2.6444, + "theoretical_loss": 3.379730758890286, + "tokens_seen": 2398420992 + }, + { + "epoch": 0.45, + "learning_rate": 5.5183758626223727e-05, + "loss": 2.4578, + "theoretical_loss": 3.3797159038806504, + "tokens_seen": 2398552064 + }, + { + "epoch": 0.45, + "learning_rate": 5.517573423206548e-05, + "loss": 2.6303, + "theoretical_loss": 3.3797010499100453, + "tokens_seen": 2398683136 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1317092, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3850042819976807, + "objective/train/theoretical_loss": 3.379693623314339, + "objective/train/tokens_used": 769207776, + "theoretical_loss": 3.379693623314339, + "tokens_seen": 2398748672 + }, + { + "epoch": 0.45, + "learning_rate": 5.5167709837907246e-05, + "loss": 2.4892, + "theoretical_loss": 3.379686196978342, + "tokens_seen": 2398814208 + }, + { + "epoch": 0.45, + "learning_rate": 5.5159685443749e-05, + "loss": 2.6844, + "theoretical_loss": 3.3796713450854106, + "tokens_seen": 2398945280 + }, + { + "epoch": 0.45, + "learning_rate": 5.5151661049590766e-05, + "loss": 2.5773, + "theoretical_loss": 3.379656494231122, + "tokens_seen": 2399076352 + }, + { + "epoch": 0.45, + "learning_rate": 5.514363665543252e-05, + "loss": 2.5601, + "theoretical_loss": 3.379641644415347, + "tokens_seen": 2399207424 + }, + { + "epoch": 0.45, + "learning_rate": 5.513561226127427e-05, + "loss": 2.4204, + "theoretical_loss": 3.3796267956379555, + "tokens_seen": 2399338496 + }, + { + "epoch": 0.45, + "learning_rate": 5.512758786711604e-05, + "loss": 2.4937, + "theoretical_loss": 3.379611947898819, + "tokens_seen": 2399469568 + }, + { + "epoch": 0.45, + "learning_rate": 5.511956347295779e-05, + "loss": 2.5614, + "theoretical_loss": 3.379597101197808, + "tokens_seen": 2399600640 + }, + { + "epoch": 0.45, + "learning_rate": 5.511153907879956e-05, + "loss": 2.4579, + "theoretical_loss": 3.3795822555347925, + "tokens_seen": 2399731712 + }, + { + "epoch": 0.45, + "learning_rate": 5.510351468464131e-05, + "loss": 2.4595, + "theoretical_loss": 3.3795674109096443, + "tokens_seen": 2399862784 + }, + { + "epoch": 0.45, + "learning_rate": 5.509549029048308e-05, + "loss": 2.5017, + "theoretical_loss": 3.3795525673222335, + "tokens_seen": 2399993856 + }, + { + "epoch": 0.45, + "learning_rate": 5.508746589632483e-05, + "loss": 2.5535, + "theoretical_loss": 3.3795377247724314, + "tokens_seen": 2400124928 + }, + { + "epoch": 0.45, + "learning_rate": 5.5079441502166585e-05, + "loss": 2.5604, + "theoretical_loss": 3.379522883260108, + "tokens_seen": 2400256000 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1318422, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.598881959915161, + "objective/train/theoretical_loss": 3.379508042785135, + "objective/train/tokens_used": 770846176, + "theoretical_loss": 3.379508042785135, + "tokens_seen": 2400387072 + }, + { + "epoch": 0.45, + "learning_rate": 5.507141710800835e-05, + "loss": 2.5778, + "theoretical_loss": 3.379508042785135, + "tokens_seen": 2400387072 + }, + { + "epoch": 0.45, + "learning_rate": 5.5063392713850105e-05, + "loss": 2.4412, + "theoretical_loss": 3.3794932033473826, + "tokens_seen": 2400518144 + }, + { + "epoch": 0.45, + "learning_rate": 5.505536831969187e-05, + "loss": 2.5274, + "theoretical_loss": 3.3794783649467215, + "tokens_seen": 2400649216 + }, + { + "epoch": 0.46, + "learning_rate": 5.5047343925533625e-05, + "loss": 2.5056, + "theoretical_loss": 3.379463527583024, + "tokens_seen": 2400780288 + }, + { + "epoch": 0.46, + "learning_rate": 5.503931953137539e-05, + "loss": 2.5522, + "theoretical_loss": 3.379448691256159, + "tokens_seen": 2400911360 + }, + { + "epoch": 0.46, + "learning_rate": 5.5031295137217144e-05, + "loss": 2.5819, + "theoretical_loss": 3.379433855965999, + "tokens_seen": 2401042432 + }, + { + "epoch": 0.46, + "learning_rate": 5.50232707430589e-05, + "loss": 2.529, + "theoretical_loss": 3.379419021712415, + "tokens_seen": 2401173504 + }, + { + "epoch": 0.46, + "learning_rate": 5.5015246348900664e-05, + "loss": 2.3629, + "theoretical_loss": 3.379404188495277, + "tokens_seen": 2401304576 + }, + { + "epoch": 0.46, + "learning_rate": 5.500722195474242e-05, + "loss": 2.5947, + "theoretical_loss": 3.3793893563144564, + "tokens_seen": 2401435648 + }, + { + "epoch": 0.46, + "learning_rate": 5.4999197560584184e-05, + "loss": 2.5389, + "theoretical_loss": 3.3793745251698244, + "tokens_seen": 2401566720 + }, + { + "epoch": 0.46, + "learning_rate": 5.499117316642594e-05, + "loss": 2.4569, + "theoretical_loss": 3.379359695061252, + "tokens_seen": 2401697792 + }, + { + "epoch": 0.46, + "learning_rate": 5.4983148772267704e-05, + "loss": 2.5019, + "theoretical_loss": 3.3793448659886103, + "tokens_seen": 2401828864 + }, + { + "epoch": 0.46, + "learning_rate": 5.497512437810946e-05, + "loss": 2.567, + "theoretical_loss": 3.3793300379517706, + "tokens_seen": 2401959936 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1319687, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7546088695526123, + "objective/train/theoretical_loss": 3.379322624321736, + "objective/train/tokens_used": 772484576, + "theoretical_loss": 3.379322624321736, + "tokens_seen": 2402025472 + }, + { + "epoch": 0.46, + "learning_rate": 5.496709998395121e-05, + "loss": 2.4976, + "theoretical_loss": 3.3793152109506037, + "tokens_seen": 2402091008 + }, + { + "epoch": 0.46, + "learning_rate": 5.495907558979298e-05, + "loss": 2.5438, + "theoretical_loss": 3.3793003849849805, + "tokens_seen": 2402222080 + }, + { + "epoch": 0.46, + "learning_rate": 5.495105119563473e-05, + "loss": 2.4978, + "theoretical_loss": 3.379285560054773, + "tokens_seen": 2402353152 + }, + { + "epoch": 0.46, + "learning_rate": 5.49430268014765e-05, + "loss": 2.5106, + "theoretical_loss": 3.379270736159852, + "tokens_seen": 2402484224 + }, + { + "epoch": 0.46, + "learning_rate": 5.493500240731825e-05, + "loss": 2.5004, + "theoretical_loss": 3.3792559133000886, + "tokens_seen": 2402615296 + }, + { + "epoch": 0.46, + "learning_rate": 5.4926978013160016e-05, + "loss": 2.6882, + "theoretical_loss": 3.3792410914753543, + "tokens_seen": 2402746368 + }, + { + "epoch": 0.46, + "learning_rate": 5.491895361900177e-05, + "loss": 2.4467, + "theoretical_loss": 3.37922627068552, + "tokens_seen": 2402877440 + }, + { + "epoch": 0.46, + "learning_rate": 5.491092922484352e-05, + "loss": 2.5897, + "theoretical_loss": 3.379211450930457, + "tokens_seen": 2403008512 + }, + { + "epoch": 0.46, + "learning_rate": 5.490290483068529e-05, + "loss": 2.5128, + "theoretical_loss": 3.3791966322100375, + "tokens_seen": 2403139584 + }, + { + "epoch": 0.46, + "learning_rate": 5.489488043652704e-05, + "loss": 2.5733, + "theoretical_loss": 3.3791818145241317, + "tokens_seen": 2403270656 + }, + { + "epoch": 0.46, + "learning_rate": 5.488685604236881e-05, + "loss": 2.7445, + "theoretical_loss": 3.3791669978726113, + "tokens_seen": 2403401728 + }, + { + "epoch": 0.46, + "learning_rate": 5.487883164821056e-05, + "loss": 2.4769, + "theoretical_loss": 3.379152182255348, + "tokens_seen": 2403532800 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1320162, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.617546796798706, + "objective/train/theoretical_loss": 3.379137367672213, + "objective/train/tokens_used": 774122976, + "theoretical_loss": 3.379137367672213, + "tokens_seen": 2403663872 + }, + { + "epoch": 0.46, + "learning_rate": 5.487080725405232e-05, + "loss": 2.3921, + "theoretical_loss": 3.379137367672213, + "tokens_seen": 2403663872 + }, + { + "epoch": 0.46, + "learning_rate": 5.486278285989408e-05, + "loss": 2.5964, + "theoretical_loss": 3.3791225541230783, + "tokens_seen": 2403794944 + }, + { + "epoch": 0.46, + "learning_rate": 5.485475846573584e-05, + "loss": 2.4279, + "theoretical_loss": 3.379107741607814, + "tokens_seen": 2403926016 + }, + { + "epoch": 0.46, + "learning_rate": 5.48467340715776e-05, + "loss": 2.5559, + "theoretical_loss": 3.379092930126293, + "tokens_seen": 2404057088 + }, + { + "epoch": 0.46, + "learning_rate": 5.4838709677419355e-05, + "loss": 2.4422, + "theoretical_loss": 3.379078119678386, + "tokens_seen": 2404188160 + }, + { + "epoch": 0.46, + "learning_rate": 5.483068528326112e-05, + "loss": 2.4967, + "theoretical_loss": 3.3790633102639647, + "tokens_seen": 2404319232 + }, + { + "epoch": 0.46, + "learning_rate": 5.4822660889102875e-05, + "loss": 2.5944, + "theoretical_loss": 3.3790485018829006, + "tokens_seen": 2404450304 + }, + { + "epoch": 0.46, + "learning_rate": 5.4814636494944635e-05, + "loss": 2.6128, + "theoretical_loss": 3.379033694535065, + "tokens_seen": 2404581376 + }, + { + "epoch": 0.46, + "learning_rate": 5.4806612100786395e-05, + "loss": 2.5823, + "theoretical_loss": 3.3790188882203305, + "tokens_seen": 2404712448 + }, + { + "epoch": 0.46, + "learning_rate": 5.4798587706628155e-05, + "loss": 2.4524, + "theoretical_loss": 3.3790040829385677, + "tokens_seen": 2404843520 + }, + { + "epoch": 0.46, + "learning_rate": 5.4790563312469914e-05, + "loss": 2.5909, + "theoretical_loss": 3.378989278689649, + "tokens_seen": 2404974592 + }, + { + "epoch": 0.46, + "learning_rate": 5.478253891831167e-05, + "loss": 2.513, + "theoretical_loss": 3.378974475473445, + "tokens_seen": 2405105664 + }, + { + "epoch": 0.46, + "learning_rate": 5.477451452415343e-05, + "loss": 2.5035, + "theoretical_loss": 3.3789596732898284, + "tokens_seen": 2405236736 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1321303, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4370169639587402, + "objective/train/theoretical_loss": 3.3789522725852, + "objective/train/tokens_used": 775761376, + "theoretical_loss": 3.3789522725852, + "tokens_seen": 2405302272 + }, + { + "epoch": 0.46, + "learning_rate": 5.476649012999519e-05, + "loss": 2.4061, + "theoretical_loss": 3.3789448721386703, + "tokens_seen": 2405367808 + }, + { + "epoch": 0.46, + "learning_rate": 5.475846573583695e-05, + "loss": 2.5969, + "theoretical_loss": 3.378930072019843, + "tokens_seen": 2405498880 + }, + { + "epoch": 0.46, + "learning_rate": 5.475044134167871e-05, + "loss": 2.5381, + "theoretical_loss": 3.378915272933218, + "tokens_seen": 2405629952 + }, + { + "epoch": 0.46, + "learning_rate": 5.474241694752047e-05, + "loss": 2.405, + "theoretical_loss": 3.378900474878667, + "tokens_seen": 2405761024 + }, + { + "epoch": 0.46, + "learning_rate": 5.473439255336223e-05, + "loss": 2.4685, + "theoretical_loss": 3.3788856778560614, + "tokens_seen": 2405892096 + }, + { + "epoch": 0.46, + "learning_rate": 5.472636815920398e-05, + "loss": 2.5649, + "theoretical_loss": 3.3788708818652737, + "tokens_seen": 2406023168 + }, + { + "epoch": 0.46, + "learning_rate": 5.471834376504574e-05, + "loss": 2.5203, + "theoretical_loss": 3.3788560869061754, + "tokens_seen": 2406154240 + }, + { + "epoch": 0.46, + "learning_rate": 5.47103193708875e-05, + "loss": 2.6004, + "theoretical_loss": 3.3788412929786382, + "tokens_seen": 2406285312 + }, + { + "epoch": 0.46, + "learning_rate": 5.470229497672926e-05, + "loss": 2.5926, + "theoretical_loss": 3.3788265000825346, + "tokens_seen": 2406416384 + }, + { + "epoch": 0.46, + "learning_rate": 5.469427058257102e-05, + "loss": 2.6392, + "theoretical_loss": 3.3788117082177362, + "tokens_seen": 2406547456 + }, + { + "epoch": 0.46, + "learning_rate": 5.468624618841278e-05, + "loss": 2.5661, + "theoretical_loss": 3.378796917384115, + "tokens_seen": 2406678528 + }, + { + "epoch": 0.46, + "learning_rate": 5.467822179425453e-05, + "loss": 2.5163, + "theoretical_loss": 3.3787821275815424, + "tokens_seen": 2406809600 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1321925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.268547296524048, + "objective/train/theoretical_loss": 3.378767338809891, + "objective/train/tokens_used": 777399776, + "theoretical_loss": 3.378767338809891, + "tokens_seen": 2406940672 + }, + { + "epoch": 0.46, + "learning_rate": 5.467019740009629e-05, + "loss": 2.435, + "theoretical_loss": 3.378767338809891, + "tokens_seen": 2406940672 + }, + { + "epoch": 0.46, + "learning_rate": 5.466217300593805e-05, + "loss": 2.5572, + "theoretical_loss": 3.378752551069033, + "tokens_seen": 2407071744 + }, + { + "epoch": 0.46, + "learning_rate": 5.465414861177981e-05, + "loss": 2.6102, + "theoretical_loss": 3.37873776435884, + "tokens_seen": 2407202816 + }, + { + "epoch": 0.46, + "learning_rate": 5.464612421762157e-05, + "loss": 2.704, + "theoretical_loss": 3.3787229786791837, + "tokens_seen": 2407333888 + }, + { + "epoch": 0.46, + "learning_rate": 5.463809982346333e-05, + "loss": 2.401, + "theoretical_loss": 3.3787081940299366, + "tokens_seen": 2407464960 + }, + { + "epoch": 0.46, + "learning_rate": 5.463007542930509e-05, + "loss": 2.4335, + "theoretical_loss": 3.378693410410971, + "tokens_seen": 2407596032 + }, + { + "epoch": 0.46, + "learning_rate": 5.4622051035146845e-05, + "loss": 2.628, + "theoretical_loss": 3.378678627822159, + "tokens_seen": 2407727104 + }, + { + "epoch": 0.46, + "learning_rate": 5.4614026640988605e-05, + "loss": 2.5319, + "theoretical_loss": 3.3786638462633722, + "tokens_seen": 2407858176 + }, + { + "epoch": 0.46, + "learning_rate": 5.4606002246830365e-05, + "loss": 2.4947, + "theoretical_loss": 3.3786490657344834, + "tokens_seen": 2407989248 + }, + { + "epoch": 0.46, + "learning_rate": 5.4597977852672125e-05, + "loss": 2.468, + "theoretical_loss": 3.378634286235364, + "tokens_seen": 2408120320 + }, + { + "epoch": 0.46, + "learning_rate": 5.4589953458513885e-05, + "loss": 2.516, + "theoretical_loss": 3.3786195077658876, + "tokens_seen": 2408251392 + }, + { + "epoch": 0.46, + "learning_rate": 5.458192906435564e-05, + "loss": 2.4975, + "theoretical_loss": 3.3786047303259252, + "tokens_seen": 2408382464 + }, + { + "epoch": 0.46, + "learning_rate": 5.4573904670197405e-05, + "loss": 2.597, + "theoretical_loss": 3.378589953915349, + "tokens_seen": 2408513536 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1323064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5613491535186768, + "objective/train/theoretical_loss": 3.3785825660960414, + "objective/train/tokens_used": 779038176, + "theoretical_loss": 3.3785825660960414, + "tokens_seen": 2408579072 + }, + { + "epoch": 0.46, + "learning_rate": 5.456588027603916e-05, + "loss": 2.6614, + "theoretical_loss": 3.378575178534032, + "tokens_seen": 2408644608 + }, + { + "epoch": 0.46, + "learning_rate": 5.4557855881880925e-05, + "loss": 2.5185, + "theoretical_loss": 3.3785604041818464, + "tokens_seen": 2408775680 + }, + { + "epoch": 0.46, + "learning_rate": 5.454983148772268e-05, + "loss": 2.5571, + "theoretical_loss": 3.3785456308586643, + "tokens_seen": 2408906752 + }, + { + "epoch": 0.46, + "learning_rate": 5.454180709356443e-05, + "loss": 2.5294, + "theoretical_loss": 3.378530858564358, + "tokens_seen": 2409037824 + }, + { + "epoch": 0.46, + "learning_rate": 5.45337826994062e-05, + "loss": 2.5203, + "theoretical_loss": 3.3785160872987996, + "tokens_seen": 2409168896 + }, + { + "epoch": 0.46, + "learning_rate": 5.452575830524795e-05, + "loss": 2.6502, + "theoretical_loss": 3.378501317061862, + "tokens_seen": 2409299968 + }, + { + "epoch": 0.46, + "learning_rate": 5.451773391108972e-05, + "loss": 2.5972, + "theoretical_loss": 3.3784865478534174, + "tokens_seen": 2409431040 + }, + { + "epoch": 0.46, + "learning_rate": 5.450970951693147e-05, + "loss": 2.5656, + "theoretical_loss": 3.3784717796733386, + "tokens_seen": 2409562112 + }, + { + "epoch": 0.46, + "learning_rate": 5.450168512277324e-05, + "loss": 2.5192, + "theoretical_loss": 3.3784570125214977, + "tokens_seen": 2409693184 + }, + { + "epoch": 0.46, + "learning_rate": 5.449366072861499e-05, + "loss": 2.5197, + "theoretical_loss": 3.378442246397767, + "tokens_seen": 2409824256 + }, + { + "epoch": 0.46, + "learning_rate": 5.4485636334456743e-05, + "loss": 2.428, + "theoretical_loss": 3.3784274813020194, + "tokens_seen": 2409955328 + }, + { + "epoch": 0.46, + "learning_rate": 5.447761194029851e-05, + "loss": 2.5974, + "theoretical_loss": 3.378412717234127, + "tokens_seen": 2410086400 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1323777, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4356136322021484, + "objective/train/theoretical_loss": 3.378397954193963, + "objective/train/tokens_used": 780676576, + "theoretical_loss": 3.378397954193963, + "tokens_seen": 2410217472 + }, + { + "epoch": 0.46, + "learning_rate": 5.446958754614026e-05, + "loss": 2.5048, + "theoretical_loss": 3.378397954193963, + "tokens_seen": 2410217472 + }, + { + "epoch": 0.46, + "learning_rate": 5.446156315198203e-05, + "loss": 2.5731, + "theoretical_loss": 3.3783831921813996, + "tokens_seen": 2410348544 + }, + { + "epoch": 0.46, + "learning_rate": 5.445353875782378e-05, + "loss": 2.3714, + "theoretical_loss": 3.378368431196309, + "tokens_seen": 2410479616 + }, + { + "epoch": 0.46, + "learning_rate": 5.444551436366555e-05, + "loss": 2.4744, + "theoretical_loss": 3.3783536712385644, + "tokens_seen": 2410610688 + }, + { + "epoch": 0.46, + "learning_rate": 5.44374899695073e-05, + "loss": 2.7018, + "theoretical_loss": 3.3783389123080387, + "tokens_seen": 2410741760 + }, + { + "epoch": 0.46, + "learning_rate": 5.4429465575349056e-05, + "loss": 2.5208, + "theoretical_loss": 3.378324154404604, + "tokens_seen": 2410872832 + }, + { + "epoch": 0.46, + "learning_rate": 5.442144118119082e-05, + "loss": 2.5483, + "theoretical_loss": 3.378309397528133, + "tokens_seen": 2411003904 + }, + { + "epoch": 0.46, + "learning_rate": 5.4413416787032576e-05, + "loss": 2.4754, + "theoretical_loss": 3.3782946416784982, + "tokens_seen": 2411134976 + }, + { + "epoch": 0.46, + "learning_rate": 5.440539239287434e-05, + "loss": 2.5531, + "theoretical_loss": 3.378279886855573, + "tokens_seen": 2411266048 + }, + { + "epoch": 0.46, + "learning_rate": 5.4397367998716096e-05, + "loss": 2.5559, + "theoretical_loss": 3.3782651330592297, + "tokens_seen": 2411397120 + }, + { + "epoch": 0.46, + "learning_rate": 5.438934360455786e-05, + "loss": 2.3269, + "theoretical_loss": 3.3782503802893413, + "tokens_seen": 2411528192 + }, + { + "epoch": 0.46, + "learning_rate": 5.4381319210399615e-05, + "loss": 2.5977, + "theoretical_loss": 3.3782356285457804, + "tokens_seen": 2411659264 + }, + { + "epoch": 0.46, + "learning_rate": 5.437329481624137e-05, + "loss": 2.4914, + "theoretical_loss": 3.37822087782842, + "tokens_seen": 2411790336 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1325032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5119264125823975, + "objective/train/theoretical_loss": 3.3782135028545253, + "objective/train/tokens_used": 782314976, + "theoretical_loss": 3.3782135028545253, + "tokens_seen": 2411855872 + }, + { + "epoch": 0.46, + "learning_rate": 5.4365270422083135e-05, + "loss": 2.5239, + "theoretical_loss": 3.378206128137133, + "tokens_seen": 2411921408 + }, + { + "epoch": 0.46, + "learning_rate": 5.435724602792489e-05, + "loss": 2.569, + "theoretical_loss": 3.378191379471792, + "tokens_seen": 2412052480 + }, + { + "epoch": 0.46, + "learning_rate": 5.4349221633766655e-05, + "loss": 2.5473, + "theoretical_loss": 3.3781766318322703, + "tokens_seen": 2412183552 + }, + { + "epoch": 0.46, + "learning_rate": 5.434119723960841e-05, + "loss": 2.6283, + "theoretical_loss": 3.3781618852184403, + "tokens_seen": 2412314624 + }, + { + "epoch": 0.46, + "learning_rate": 5.4333172845450175e-05, + "loss": 2.442, + "theoretical_loss": 3.378147139630175, + "tokens_seen": 2412445696 + }, + { + "epoch": 0.46, + "learning_rate": 5.432514845129193e-05, + "loss": 2.5497, + "theoretical_loss": 3.3781323950673485, + "tokens_seen": 2412576768 + }, + { + "epoch": 0.46, + "learning_rate": 5.431712405713368e-05, + "loss": 2.587, + "theoretical_loss": 3.378117651529832, + "tokens_seen": 2412707840 + }, + { + "epoch": 0.46, + "learning_rate": 5.430909966297545e-05, + "loss": 2.7423, + "theoretical_loss": 3.3781029090174997, + "tokens_seen": 2412838912 + }, + { + "epoch": 0.46, + "learning_rate": 5.43010752688172e-05, + "loss": 2.6118, + "theoretical_loss": 3.3780881675302243, + "tokens_seen": 2412969984 + }, + { + "epoch": 0.46, + "learning_rate": 5.429305087465897e-05, + "loss": 2.324, + "theoretical_loss": 3.3780734270678785, + "tokens_seen": 2413101056 + }, + { + "epoch": 0.46, + "learning_rate": 5.428502648050072e-05, + "loss": 2.5258, + "theoretical_loss": 3.378058687630336, + "tokens_seen": 2413232128 + }, + { + "epoch": 0.46, + "learning_rate": 5.427700208634249e-05, + "loss": 2.4511, + "theoretical_loss": 3.3780439492174694, + "tokens_seen": 2413363200 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1325688, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5455198287963867, + "objective/train/theoretical_loss": 3.378029211829152, + "objective/train/tokens_used": 783953376, + "theoretical_loss": 3.378029211829152, + "tokens_seen": 2413494272 + }, + { + "epoch": 0.46, + "learning_rate": 5.426897769218424e-05, + "loss": 2.595, + "theoretical_loss": 3.378029211829152, + "tokens_seen": 2413494272 + }, + { + "epoch": 0.46, + "learning_rate": 5.426095329802601e-05, + "loss": 2.5726, + "theoretical_loss": 3.3780144754652577, + "tokens_seen": 2413625344 + }, + { + "epoch": 0.46, + "learning_rate": 5.425292890386776e-05, + "loss": 2.4572, + "theoretical_loss": 3.3779997401256585, + "tokens_seen": 2413756416 + }, + { + "epoch": 0.46, + "learning_rate": 5.4244904509709514e-05, + "loss": 2.5213, + "theoretical_loss": 3.3779850058102276, + "tokens_seen": 2413887488 + }, + { + "epoch": 0.46, + "learning_rate": 5.423688011555128e-05, + "loss": 2.6122, + "theoretical_loss": 3.377970272518839, + "tokens_seen": 2414018560 + }, + { + "epoch": 0.46, + "learning_rate": 5.422885572139303e-05, + "loss": 2.4838, + "theoretical_loss": 3.3779555402513655, + "tokens_seen": 2414149632 + }, + { + "epoch": 0.46, + "learning_rate": 5.42208313272348e-05, + "loss": 2.4408, + "theoretical_loss": 3.3779408090076806, + "tokens_seen": 2414280704 + }, + { + "epoch": 0.46, + "learning_rate": 5.421280693307655e-05, + "loss": 2.5687, + "theoretical_loss": 3.3779260787876573, + "tokens_seen": 2414411776 + }, + { + "epoch": 0.46, + "learning_rate": 5.420478253891832e-05, + "loss": 2.5043, + "theoretical_loss": 3.377911349591169, + "tokens_seen": 2414542848 + }, + { + "epoch": 0.46, + "learning_rate": 5.419675814476007e-05, + "loss": 2.4312, + "theoretical_loss": 3.377896621418089, + "tokens_seen": 2414673920 + }, + { + "epoch": 0.46, + "learning_rate": 5.4188733750601826e-05, + "loss": 2.487, + "theoretical_loss": 3.377881894268291, + "tokens_seen": 2414804992 + }, + { + "epoch": 0.46, + "learning_rate": 5.418070935644359e-05, + "loss": 2.4378, + "theoretical_loss": 3.3778671681416474, + "tokens_seen": 2414936064 + }, + { + "epoch": 0.46, + "learning_rate": 5.4172684962285346e-05, + "loss": 2.5642, + "theoretical_loss": 3.377852443038033, + "tokens_seen": 2415067136 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1326614, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4216179847717285, + "objective/train/theoretical_loss": 3.3778450808698213, + "objective/train/tokens_used": 785591776, + "theoretical_loss": 3.3778450808698213, + "tokens_seen": 2415132672 + }, + { + "epoch": 0.46, + "learning_rate": 5.416466056812711e-05, + "loss": 2.5758, + "theoretical_loss": 3.37783771895732, + "tokens_seen": 2415198208 + }, + { + "epoch": 0.46, + "learning_rate": 5.4156636173968866e-05, + "loss": 2.5595, + "theoretical_loss": 3.377822995899382, + "tokens_seen": 2415329280 + }, + { + "epoch": 0.46, + "learning_rate": 5.414861177981063e-05, + "loss": 2.4036, + "theoretical_loss": 3.377808273864093, + "tokens_seen": 2415460352 + }, + { + "epoch": 0.46, + "learning_rate": 5.4140587385652386e-05, + "loss": 2.6588, + "theoretical_loss": 3.377793552851326, + "tokens_seen": 2415591424 + }, + { + "epoch": 0.46, + "learning_rate": 5.413256299149414e-05, + "loss": 2.5935, + "theoretical_loss": 3.3777788328609555, + "tokens_seen": 2415722496 + }, + { + "epoch": 0.46, + "learning_rate": 5.4124538597335905e-05, + "loss": 2.5657, + "theoretical_loss": 3.3777641138928534, + "tokens_seen": 2415853568 + }, + { + "epoch": 0.46, + "learning_rate": 5.411651420317766e-05, + "loss": 2.4348, + "theoretical_loss": 3.3777493959468945, + "tokens_seen": 2415984640 + }, + { + "epoch": 0.46, + "learning_rate": 5.4108489809019425e-05, + "loss": 2.6172, + "theoretical_loss": 3.377734679022952, + "tokens_seen": 2416115712 + }, + { + "epoch": 0.46, + "learning_rate": 5.410046541486118e-05, + "loss": 2.4744, + "theoretical_loss": 3.3777199631208994, + "tokens_seen": 2416246784 + }, + { + "epoch": 0.46, + "learning_rate": 5.4092441020702945e-05, + "loss": 2.4383, + "theoretical_loss": 3.3777052482406105, + "tokens_seen": 2416377856 + }, + { + "epoch": 0.46, + "learning_rate": 5.40844166265447e-05, + "loss": 2.4488, + "theoretical_loss": 3.3776905343819585, + "tokens_seen": 2416508928 + }, + { + "epoch": 0.46, + "learning_rate": 5.407639223238645e-05, + "loss": 2.5904, + "theoretical_loss": 3.377675821544818, + "tokens_seen": 2416640000 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1327216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4877939224243164, + "objective/train/theoretical_loss": 3.377661109729061, + "objective/train/tokens_used": 787230176, + "theoretical_loss": 3.377661109729061, + "tokens_seen": 2416771072 + }, + { + "epoch": 0.46, + "learning_rate": 5.406836783822822e-05, + "loss": 2.3429, + "theoretical_loss": 3.377661109729061, + "tokens_seen": 2416771072 + }, + { + "epoch": 0.46, + "learning_rate": 5.406034344406997e-05, + "loss": 2.419, + "theoretical_loss": 3.377646398934563, + "tokens_seen": 2416902144 + }, + { + "epoch": 0.46, + "learning_rate": 5.405231904991174e-05, + "loss": 2.6701, + "theoretical_loss": 3.377631689161197, + "tokens_seen": 2417033216 + }, + { + "epoch": 0.46, + "learning_rate": 5.404429465575349e-05, + "loss": 2.5824, + "theoretical_loss": 3.3776169804088365, + "tokens_seen": 2417164288 + }, + { + "epoch": 0.47, + "learning_rate": 5.403627026159526e-05, + "loss": 2.6254, + "theoretical_loss": 3.377602272677356, + "tokens_seen": 2417295360 + }, + { + "epoch": 0.47, + "learning_rate": 5.402824586743701e-05, + "loss": 2.4818, + "theoretical_loss": 3.3775875659666283, + "tokens_seen": 2417426432 + }, + { + "epoch": 0.47, + "learning_rate": 5.4020221473278764e-05, + "loss": 2.6733, + "theoretical_loss": 3.377572860276528, + "tokens_seen": 2417557504 + }, + { + "epoch": 0.47, + "learning_rate": 5.401219707912053e-05, + "loss": 2.6771, + "theoretical_loss": 3.3775581556069283, + "tokens_seen": 2417688576 + }, + { + "epoch": 0.47, + "learning_rate": 5.4004172684962284e-05, + "loss": 2.6009, + "theoretical_loss": 3.377543451957704, + "tokens_seen": 2417819648 + }, + { + "epoch": 0.47, + "learning_rate": 5.399614829080405e-05, + "loss": 2.5779, + "theoretical_loss": 3.377528749328728, + "tokens_seen": 2417950720 + }, + { + "epoch": 0.47, + "learning_rate": 5.3988123896645803e-05, + "loss": 2.5507, + "theoretical_loss": 3.377514047719875, + "tokens_seen": 2418081792 + }, + { + "epoch": 0.47, + "learning_rate": 5.398009950248757e-05, + "loss": 2.5918, + "theoretical_loss": 3.377499347131018, + "tokens_seen": 2418212864 + }, + { + "epoch": 0.47, + "learning_rate": 5.397207510832932e-05, + "loss": 2.5341, + "theoretical_loss": 3.377484647562032, + "tokens_seen": 2418343936 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1328499, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.503131151199341, + "objective/train/theoretical_loss": 3.377477298159951, + "objective/train/tokens_used": 788868576, + "theoretical_loss": 3.377477298159951, + "tokens_seen": 2418409472 + }, + { + "epoch": 0.47, + "learning_rate": 5.3964050714171076e-05, + "loss": 2.5173, + "theoretical_loss": 3.37746994901279, + "tokens_seen": 2418475008 + }, + { + "epoch": 0.47, + "learning_rate": 5.395602632001284e-05, + "loss": 2.5265, + "theoretical_loss": 3.377455251483167, + "tokens_seen": 2418606080 + }, + { + "epoch": 0.47, + "learning_rate": 5.3948001925854596e-05, + "loss": 2.6263, + "theoretical_loss": 3.3774405549730364, + "tokens_seen": 2418737152 + }, + { + "epoch": 0.47, + "learning_rate": 5.393997753169636e-05, + "loss": 2.3933, + "theoretical_loss": 3.377425859482272, + "tokens_seen": 2418868224 + }, + { + "epoch": 0.47, + "learning_rate": 5.3931953137538116e-05, + "loss": 2.4687, + "theoretical_loss": 3.3774111650107486, + "tokens_seen": 2418999296 + }, + { + "epoch": 0.47, + "learning_rate": 5.392392874337988e-05, + "loss": 2.5433, + "theoretical_loss": 3.37739647155834, + "tokens_seen": 2419130368 + }, + { + "epoch": 0.47, + "learning_rate": 5.3915904349221636e-05, + "loss": 2.6307, + "theoretical_loss": 3.37738177912492, + "tokens_seen": 2419261440 + }, + { + "epoch": 0.47, + "learning_rate": 5.39078799550634e-05, + "loss": 2.7126, + "theoretical_loss": 3.377367087710363, + "tokens_seen": 2419392512 + }, + { + "epoch": 0.47, + "learning_rate": 5.3899855560905156e-05, + "loss": 2.6034, + "theoretical_loss": 3.3773523973145427, + "tokens_seen": 2419523584 + }, + { + "epoch": 0.47, + "learning_rate": 5.389183116674691e-05, + "loss": 2.5136, + "theoretical_loss": 3.377337707937334, + "tokens_seen": 2419654656 + }, + { + "epoch": 0.47, + "learning_rate": 5.3883806772588675e-05, + "loss": 2.5819, + "theoretical_loss": 3.377323019578611, + "tokens_seen": 2419785728 + }, + { + "epoch": 0.47, + "learning_rate": 5.387578237843043e-05, + "loss": 2.4908, + "theoretical_loss": 3.3773083322382473, + "tokens_seen": 2419916800 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1329844, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.938718557357788, + "objective/train/theoretical_loss": 3.377293645916118, + "objective/train/tokens_used": 790506976, + "theoretical_loss": 3.377293645916118, + "tokens_seen": 2420047872 + }, + { + "epoch": 0.47, + "learning_rate": 5.3867757984272195e-05, + "loss": 2.5834, + "theoretical_loss": 3.377293645916118, + "tokens_seen": 2420047872 + }, + { + "epoch": 0.47, + "learning_rate": 5.385973359011395e-05, + "loss": 2.514, + "theoretical_loss": 3.377278960612096, + "tokens_seen": 2420178944 + }, + { + "epoch": 0.47, + "learning_rate": 5.3851709195955715e-05, + "loss": 2.5113, + "theoretical_loss": 3.3772642763260574, + "tokens_seen": 2420310016 + }, + { + "epoch": 0.47, + "learning_rate": 5.384368480179747e-05, + "loss": 2.5993, + "theoretical_loss": 3.377249593057875, + "tokens_seen": 2420441088 + }, + { + "epoch": 0.47, + "learning_rate": 5.383566040763922e-05, + "loss": 2.4977, + "theoretical_loss": 3.377234910807424, + "tokens_seen": 2420572160 + }, + { + "epoch": 0.47, + "learning_rate": 5.382763601348099e-05, + "loss": 2.6783, + "theoretical_loss": 3.3772202295745783, + "tokens_seen": 2420703232 + }, + { + "epoch": 0.47, + "learning_rate": 5.381961161932274e-05, + "loss": 2.4175, + "theoretical_loss": 3.377205549359213, + "tokens_seen": 2420834304 + }, + { + "epoch": 0.47, + "learning_rate": 5.381158722516451e-05, + "loss": 2.3895, + "theoretical_loss": 3.3771908701612015, + "tokens_seen": 2420965376 + }, + { + "epoch": 0.47, + "learning_rate": 5.380356283100626e-05, + "loss": 2.3957, + "theoretical_loss": 3.3771761919804186, + "tokens_seen": 2421096448 + }, + { + "epoch": 0.47, + "learning_rate": 5.379553843684803e-05, + "loss": 2.2962, + "theoretical_loss": 3.377161514816739, + "tokens_seen": 2421227520 + }, + { + "epoch": 0.47, + "learning_rate": 5.378751404268978e-05, + "loss": 2.6406, + "theoretical_loss": 3.377146838670037, + "tokens_seen": 2421358592 + }, + { + "epoch": 0.47, + "learning_rate": 5.3779489648531534e-05, + "loss": 2.4753, + "theoretical_loss": 3.377132163540187, + "tokens_seen": 2421489664 + }, + { + "epoch": 0.47, + "learning_rate": 5.37714652543733e-05, + "loss": 2.5184, + "theoretical_loss": 3.377117489427064, + "tokens_seen": 2421620736 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1330351, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.552644968032837, + "objective/train/theoretical_loss": 3.3771101527517358, + "objective/train/tokens_used": 792145376, + "theoretical_loss": 3.3771101527517358, + "tokens_seen": 2421686272 + }, + { + "epoch": 0.47, + "learning_rate": 5.3763440860215054e-05, + "loss": 2.6453, + "theoretical_loss": 3.377102816330542, + "tokens_seen": 2421751808 + }, + { + "epoch": 0.47, + "learning_rate": 5.375541646605682e-05, + "loss": 2.49, + "theoretical_loss": 3.3770881442504956, + "tokens_seen": 2421882880 + }, + { + "epoch": 0.47, + "learning_rate": 5.3747392071898574e-05, + "loss": 2.4389, + "theoretical_loss": 3.3770734731867993, + "tokens_seen": 2422013952 + }, + { + "epoch": 0.47, + "learning_rate": 5.373936767774034e-05, + "loss": 2.4152, + "theoretical_loss": 3.377058803139328, + "tokens_seen": 2422145024 + }, + { + "epoch": 0.47, + "learning_rate": 5.373134328358209e-05, + "loss": 2.529, + "theoretical_loss": 3.377044134107956, + "tokens_seen": 2422276096 + }, + { + "epoch": 0.47, + "learning_rate": 5.3723318889423846e-05, + "loss": 2.4047, + "theoretical_loss": 3.377029466092558, + "tokens_seen": 2422407168 + }, + { + "epoch": 0.47, + "learning_rate": 5.371529449526561e-05, + "loss": 2.5738, + "theoretical_loss": 3.3770147990930095, + "tokens_seen": 2422538240 + }, + { + "epoch": 0.47, + "learning_rate": 5.3707270101107366e-05, + "loss": 2.6262, + "theoretical_loss": 3.377000133109184, + "tokens_seen": 2422669312 + }, + { + "epoch": 0.47, + "learning_rate": 5.369924570694913e-05, + "loss": 2.5234, + "theoretical_loss": 3.3769854681409566, + "tokens_seen": 2422800384 + }, + { + "epoch": 0.47, + "learning_rate": 5.3691221312790886e-05, + "loss": 2.4704, + "theoretical_loss": 3.3769708041882023, + "tokens_seen": 2422931456 + }, + { + "epoch": 0.47, + "learning_rate": 5.368319691863265e-05, + "loss": 2.4136, + "theoretical_loss": 3.376956141250796, + "tokens_seen": 2423062528 + }, + { + "epoch": 0.47, + "learning_rate": 5.3675172524474406e-05, + "loss": 2.3786, + "theoretical_loss": 3.3769414793286114, + "tokens_seen": 2423193600 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1331600, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.512211799621582, + "objective/train/theoretical_loss": 3.3769268184215244, + "objective/train/tokens_used": 793783776, + "theoretical_loss": 3.3769268184215244, + "tokens_seen": 2423324672 + }, + { + "epoch": 0.47, + "learning_rate": 5.366714813031616e-05, + "loss": 2.4773, + "theoretical_loss": 3.3769268184215244, + "tokens_seen": 2423324672 + }, + { + "epoch": 0.47, + "learning_rate": 5.3659123736157926e-05, + "loss": 2.5779, + "theoretical_loss": 3.37691215852941, + "tokens_seen": 2423455744 + }, + { + "epoch": 0.47, + "learning_rate": 5.365109934199968e-05, + "loss": 2.3919, + "theoretical_loss": 3.376897499652142, + "tokens_seen": 2423586816 + }, + { + "epoch": 0.47, + "learning_rate": 5.3643074947841446e-05, + "loss": 2.5697, + "theoretical_loss": 3.376882841789596, + "tokens_seen": 2423717888 + }, + { + "epoch": 0.47, + "learning_rate": 5.36350505536832e-05, + "loss": 2.4966, + "theoretical_loss": 3.3768681849416464, + "tokens_seen": 2423848960 + }, + { + "epoch": 0.47, + "learning_rate": 5.3627026159524965e-05, + "loss": 2.5197, + "theoretical_loss": 3.3768535291081685, + "tokens_seen": 2423980032 + }, + { + "epoch": 0.47, + "learning_rate": 5.361900176536672e-05, + "loss": 2.5537, + "theoretical_loss": 3.3768388742890374, + "tokens_seen": 2424111104 + }, + { + "epoch": 0.47, + "learning_rate": 5.361097737120848e-05, + "loss": 2.4198, + "theoretical_loss": 3.376824220484128, + "tokens_seen": 2424242176 + }, + { + "epoch": 0.47, + "learning_rate": 5.360295297705024e-05, + "loss": 2.4866, + "theoretical_loss": 3.3768095676933143, + "tokens_seen": 2424373248 + }, + { + "epoch": 0.47, + "learning_rate": 5.359492858289199e-05, + "loss": 2.6334, + "theoretical_loss": 3.3767949159164727, + "tokens_seen": 2424504320 + }, + { + "epoch": 0.47, + "learning_rate": 5.358690418873376e-05, + "loss": 2.6085, + "theoretical_loss": 3.3767802651534775, + "tokens_seen": 2424635392 + }, + { + "epoch": 0.47, + "learning_rate": 5.357887979457551e-05, + "loss": 2.4126, + "theoretical_loss": 3.376765615404204, + "tokens_seen": 2424766464 + }, + { + "epoch": 0.47, + "learning_rate": 5.357085540041728e-05, + "loss": 2.3968, + "theoretical_loss": 3.3767509666685265, + "tokens_seen": 2424897536 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1332182, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9074771404266357, + "objective/train/theoretical_loss": 3.3767436426807476, + "objective/train/tokens_used": 795422176, + "theoretical_loss": 3.3767436426807476, + "tokens_seen": 2424963072 + }, + { + "epoch": 0.47, + "learning_rate": 5.356283100625903e-05, + "loss": 2.3835, + "theoretical_loss": 3.376736318946321, + "tokens_seen": 2425028608 + }, + { + "epoch": 0.47, + "learning_rate": 5.355480661210079e-05, + "loss": 2.3949, + "theoretical_loss": 3.376721672237462, + "tokens_seen": 2425159680 + }, + { + "epoch": 0.47, + "learning_rate": 5.354678221794255e-05, + "loss": 2.6908, + "theoretical_loss": 3.3767070265418253, + "tokens_seen": 2425290752 + }, + { + "epoch": 0.47, + "learning_rate": 5.3538757823784304e-05, + "loss": 2.4389, + "theoretical_loss": 3.3766923818592858, + "tokens_seen": 2425421824 + }, + { + "epoch": 0.47, + "learning_rate": 5.353073342962607e-05, + "loss": 2.6217, + "theoretical_loss": 3.3766777381897186, + "tokens_seen": 2425552896 + }, + { + "epoch": 0.47, + "learning_rate": 5.3522709035467824e-05, + "loss": 2.5104, + "theoretical_loss": 3.3766630955329986, + "tokens_seen": 2425683968 + }, + { + "epoch": 0.47, + "learning_rate": 5.3514684641309584e-05, + "loss": 2.5317, + "theoretical_loss": 3.3766484538890014, + "tokens_seen": 2425815040 + }, + { + "epoch": 0.47, + "learning_rate": 5.3506660247151344e-05, + "loss": 2.7119, + "theoretical_loss": 3.376633813257602, + "tokens_seen": 2425946112 + }, + { + "epoch": 0.47, + "learning_rate": 5.3498635852993104e-05, + "loss": 2.4551, + "theoretical_loss": 3.376619173638676, + "tokens_seen": 2426077184 + }, + { + "epoch": 0.47, + "learning_rate": 5.3490611458834863e-05, + "loss": 2.4489, + "theoretical_loss": 3.3766045350320986, + "tokens_seen": 2426208256 + }, + { + "epoch": 0.47, + "learning_rate": 5.3482587064676617e-05, + "loss": 2.4335, + "theoretical_loss": 3.3765898974377446, + "tokens_seen": 2426339328 + }, + { + "epoch": 0.47, + "learning_rate": 5.347456267051838e-05, + "loss": 2.5277, + "theoretical_loss": 3.37657526085549, + "tokens_seen": 2426470400 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1333317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.659700393676758, + "objective/train/theoretical_loss": 3.37656062528521, + "objective/train/tokens_used": 797060576, + "theoretical_loss": 3.37656062528521, + "tokens_seen": 2426601472 + }, + { + "epoch": 0.47, + "learning_rate": 5.3466538276360136e-05, + "loss": 2.5915, + "theoretical_loss": 3.37656062528521, + "tokens_seen": 2426601472 + }, + { + "epoch": 0.47, + "learning_rate": 5.3458513882201896e-05, + "loss": 2.4816, + "theoretical_loss": 3.3765459907267794, + "tokens_seen": 2426732544 + }, + { + "epoch": 0.47, + "learning_rate": 5.3450489488043656e-05, + "loss": 2.4914, + "theoretical_loss": 3.3765313571800744, + "tokens_seen": 2426863616 + }, + { + "epoch": 0.47, + "learning_rate": 5.3442465093885416e-05, + "loss": 2.5979, + "theoretical_loss": 3.37651672464497, + "tokens_seen": 2426994688 + }, + { + "epoch": 0.47, + "learning_rate": 5.3434440699727176e-05, + "loss": 2.6746, + "theoretical_loss": 3.3765020931213416, + "tokens_seen": 2427125760 + }, + { + "epoch": 0.47, + "learning_rate": 5.342641630556893e-05, + "loss": 2.379, + "theoretical_loss": 3.3764874626090653, + "tokens_seen": 2427256832 + }, + { + "epoch": 0.47, + "learning_rate": 5.341839191141069e-05, + "loss": 2.562, + "theoretical_loss": 3.3764728331080156, + "tokens_seen": 2427387904 + }, + { + "epoch": 0.47, + "learning_rate": 5.341036751725245e-05, + "loss": 2.3364, + "theoretical_loss": 3.3764582046180687, + "tokens_seen": 2427518976 + }, + { + "epoch": 0.47, + "learning_rate": 5.340234312309421e-05, + "loss": 2.4422, + "theoretical_loss": 3.3764435771390997, + "tokens_seen": 2427650048 + }, + { + "epoch": 0.47, + "learning_rate": 5.339431872893597e-05, + "loss": 2.389, + "theoretical_loss": 3.3764289506709844, + "tokens_seen": 2427781120 + }, + { + "epoch": 0.47, + "learning_rate": 5.338629433477773e-05, + "loss": 2.5068, + "theoretical_loss": 3.3764143252135983, + "tokens_seen": 2427912192 + }, + { + "epoch": 0.47, + "learning_rate": 5.337826994061949e-05, + "loss": 2.532, + "theoretical_loss": 3.376399700766817, + "tokens_seen": 2428043264 + }, + { + "epoch": 0.47, + "learning_rate": 5.337024554646124e-05, + "loss": 2.4549, + "theoretical_loss": 3.376385077330516, + "tokens_seen": 2428174336 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1333920, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.964534044265747, + "objective/train/theoretical_loss": 3.3763777659912573, + "objective/train/tokens_used": 798698976, + "theoretical_loss": 3.3763777659912573, + "tokens_seen": 2428239872 + }, + { + "epoch": 0.47, + "learning_rate": 5.3362221152303e-05, + "loss": 2.5831, + "theoretical_loss": 3.376370454904572, + "tokens_seen": 2428305408 + }, + { + "epoch": 0.47, + "learning_rate": 5.335419675814476e-05, + "loss": 2.5318, + "theoretical_loss": 3.376355833488859, + "tokens_seen": 2428436480 + }, + { + "epoch": 0.47, + "learning_rate": 5.334617236398652e-05, + "loss": 2.6454, + "theoretical_loss": 3.3763412130832537, + "tokens_seen": 2428567552 + }, + { + "epoch": 0.47, + "learning_rate": 5.333814796982828e-05, + "loss": 2.4917, + "theoretical_loss": 3.376326593687631, + "tokens_seen": 2428698624 + }, + { + "epoch": 0.47, + "learning_rate": 5.333012357567004e-05, + "loss": 2.4694, + "theoretical_loss": 3.376311975301868, + "tokens_seen": 2428829696 + }, + { + "epoch": 0.47, + "learning_rate": 5.3322099181511794e-05, + "loss": 2.6382, + "theoretical_loss": 3.3762973579258393, + "tokens_seen": 2428960768 + }, + { + "epoch": 0.47, + "learning_rate": 5.331407478735356e-05, + "loss": 2.5724, + "theoretical_loss": 3.3762827415594208, + "tokens_seen": 2429091840 + }, + { + "epoch": 0.47, + "learning_rate": 5.3306050393195314e-05, + "loss": 2.5351, + "theoretical_loss": 3.3762681262024885, + "tokens_seen": 2429222912 + }, + { + "epoch": 0.47, + "learning_rate": 5.3298025999037074e-05, + "loss": 2.5725, + "theoretical_loss": 3.3762535118549186, + "tokens_seen": 2429353984 + }, + { + "epoch": 0.47, + "learning_rate": 5.3290001604878834e-05, + "loss": 2.453, + "theoretical_loss": 3.3762388985165863, + "tokens_seen": 2429485056 + }, + { + "epoch": 0.47, + "learning_rate": 5.3281977210720594e-05, + "loss": 2.6346, + "theoretical_loss": 3.3762242861873677, + "tokens_seen": 2429616128 + }, + { + "epoch": 0.47, + "learning_rate": 5.3273952816562354e-05, + "loss": 2.5161, + "theoretical_loss": 3.3762096748671384, + "tokens_seen": 2429747200 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1334728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1242477893829346, + "objective/train/theoretical_loss": 3.376195064555775, + "objective/train/tokens_used": 800337376, + "theoretical_loss": 3.376195064555775, + "tokens_seen": 2429878272 + }, + { + "epoch": 0.47, + "learning_rate": 5.326592842240411e-05, + "loss": 2.5918, + "theoretical_loss": 3.376195064555775, + "tokens_seen": 2429878272 + }, + { + "epoch": 0.47, + "learning_rate": 5.3257904028245874e-05, + "loss": 2.305, + "theoretical_loss": 3.376180455253153, + "tokens_seen": 2430009344 + }, + { + "epoch": 0.47, + "learning_rate": 5.324987963408763e-05, + "loss": 2.5245, + "theoretical_loss": 3.3761658469591485, + "tokens_seen": 2430140416 + }, + { + "epoch": 0.47, + "learning_rate": 5.324185523992939e-05, + "loss": 2.5522, + "theoretical_loss": 3.376151239673637, + "tokens_seen": 2430271488 + }, + { + "epoch": 0.47, + "learning_rate": 5.3233830845771147e-05, + "loss": 2.5314, + "theoretical_loss": 3.376136633396495, + "tokens_seen": 2430402560 + }, + { + "epoch": 0.47, + "learning_rate": 5.32258064516129e-05, + "loss": 2.5198, + "theoretical_loss": 3.3761220281275985, + "tokens_seen": 2430533632 + }, + { + "epoch": 0.47, + "learning_rate": 5.3217782057454666e-05, + "loss": 2.6067, + "theoretical_loss": 3.376107423866823, + "tokens_seen": 2430664704 + }, + { + "epoch": 0.47, + "learning_rate": 5.320975766329642e-05, + "loss": 2.5802, + "theoretical_loss": 3.376092820614045, + "tokens_seen": 2430795776 + }, + { + "epoch": 0.47, + "learning_rate": 5.3201733269138186e-05, + "loss": 2.4849, + "theoretical_loss": 3.376078218369141, + "tokens_seen": 2430926848 + }, + { + "epoch": 0.47, + "learning_rate": 5.319370887497994e-05, + "loss": 2.461, + "theoretical_loss": 3.3760636171319867, + "tokens_seen": 2431057920 + }, + { + "epoch": 0.47, + "learning_rate": 5.31856844808217e-05, + "loss": 2.5747, + "theoretical_loss": 3.3760490169024573, + "tokens_seen": 2431188992 + }, + { + "epoch": 0.47, + "learning_rate": 5.317766008666346e-05, + "loss": 2.5342, + "theoretical_loss": 3.3760344176804304, + "tokens_seen": 2431320064 + }, + { + "epoch": 0.47, + "learning_rate": 5.316963569250521e-05, + "loss": 2.5691, + "theoretical_loss": 3.376019819465782, + "tokens_seen": 2431451136 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1335789, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4593210220336914, + "objective/train/theoretical_loss": 3.376012520736185, + "objective/train/tokens_used": 801975776, + "theoretical_loss": 3.376012520736185, + "tokens_seen": 2431516672 + }, + { + "epoch": 0.47, + "learning_rate": 5.316161129834698e-05, + "loss": 2.578, + "theoretical_loss": 3.376005222258387, + "tokens_seen": 2431582208 + }, + { + "epoch": 0.47, + "learning_rate": 5.315358690418873e-05, + "loss": 2.4974, + "theoretical_loss": 3.3759906260581225, + "tokens_seen": 2431713280 + }, + { + "epoch": 0.47, + "learning_rate": 5.31455625100305e-05, + "loss": 2.4921, + "theoretical_loss": 3.375976030864865, + "tokens_seen": 2431844352 + }, + { + "epoch": 0.47, + "learning_rate": 5.313753811587225e-05, + "loss": 2.5215, + "theoretical_loss": 3.375961436678491, + "tokens_seen": 2431975424 + }, + { + "epoch": 0.47, + "learning_rate": 5.3129513721714005e-05, + "loss": 2.5972, + "theoretical_loss": 3.3759468434988755, + "tokens_seen": 2432106496 + }, + { + "epoch": 0.47, + "learning_rate": 5.312148932755577e-05, + "loss": 2.458, + "theoretical_loss": 3.375932251325896, + "tokens_seen": 2432237568 + }, + { + "epoch": 0.47, + "learning_rate": 5.3113464933397525e-05, + "loss": 2.5595, + "theoretical_loss": 3.375917660159428, + "tokens_seen": 2432368640 + }, + { + "epoch": 0.47, + "learning_rate": 5.310544053923929e-05, + "loss": 2.4036, + "theoretical_loss": 3.375903069999348, + "tokens_seen": 2432499712 + }, + { + "epoch": 0.47, + "learning_rate": 5.3097416145081045e-05, + "loss": 2.5762, + "theoretical_loss": 3.375888480845533, + "tokens_seen": 2432630784 + }, + { + "epoch": 0.47, + "learning_rate": 5.308939175092281e-05, + "loss": 2.57, + "theoretical_loss": 3.3758738926978586, + "tokens_seen": 2432761856 + }, + { + "epoch": 0.47, + "learning_rate": 5.3081367356764564e-05, + "loss": 2.4884, + "theoretical_loss": 3.375859305556202, + "tokens_seen": 2432892928 + }, + { + "epoch": 0.47, + "learning_rate": 5.307334296260632e-05, + "loss": 2.6351, + "theoretical_loss": 3.375844719420439, + "tokens_seen": 2433024000 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1336369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7755563259124756, + "objective/train/theoretical_loss": 3.375830134290446, + "objective/train/tokens_used": 803614176, + "theoretical_loss": 3.375830134290446, + "tokens_seen": 2433155072 + }, + { + "epoch": 0.47, + "learning_rate": 5.3065318568448084e-05, + "loss": 2.4941, + "theoretical_loss": 3.375830134290446, + "tokens_seen": 2433155072 + }, + { + "epoch": 0.47, + "learning_rate": 5.305729417428984e-05, + "loss": 2.5475, + "theoretical_loss": 3.3758155501661, + "tokens_seen": 2433286144 + }, + { + "epoch": 0.47, + "learning_rate": 5.3049269780131604e-05, + "loss": 2.6054, + "theoretical_loss": 3.3758009670472773, + "tokens_seen": 2433417216 + }, + { + "epoch": 0.47, + "learning_rate": 5.304124538597336e-05, + "loss": 2.7243, + "theoretical_loss": 3.375786384933854, + "tokens_seen": 2433548288 + }, + { + "epoch": 0.47, + "learning_rate": 5.3033220991815124e-05, + "loss": 2.7134, + "theoretical_loss": 3.375771803825707, + "tokens_seen": 2433679360 + }, + { + "epoch": 0.48, + "learning_rate": 5.302519659765688e-05, + "loss": 2.63, + "theoretical_loss": 3.375757223722713, + "tokens_seen": 2433810432 + }, + { + "epoch": 0.48, + "learning_rate": 5.301717220349863e-05, + "loss": 2.4773, + "theoretical_loss": 3.3757426446247485, + "tokens_seen": 2433941504 + }, + { + "epoch": 0.48, + "learning_rate": 5.30091478093404e-05, + "loss": 2.6269, + "theoretical_loss": 3.3757280665316896, + "tokens_seen": 2434072576 + }, + { + "epoch": 0.48, + "learning_rate": 5.300112341518215e-05, + "loss": 2.5212, + "theoretical_loss": 3.375713489443414, + "tokens_seen": 2434203648 + }, + { + "epoch": 0.48, + "learning_rate": 5.2993099021023917e-05, + "loss": 2.4834, + "theoretical_loss": 3.375698913359797, + "tokens_seen": 2434334720 + }, + { + "epoch": 0.48, + "learning_rate": 5.298507462686567e-05, + "loss": 2.5918, + "theoretical_loss": 3.375684338280716, + "tokens_seen": 2434465792 + }, + { + "epoch": 0.48, + "learning_rate": 5.2977050232707436e-05, + "loss": 2.4533, + "theoretical_loss": 3.375669764206048, + "tokens_seen": 2434596864 + }, + { + "epoch": 0.48, + "learning_rate": 5.296902583854919e-05, + "loss": 2.5103, + "theoretical_loss": 3.3756551911356696, + "tokens_seen": 2434727936 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1337478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.477384567260742, + "objective/train/theoretical_loss": 3.37564790497705, + "objective/train/tokens_used": 805252576, + "theoretical_loss": 3.37564790497705, + "tokens_seen": 2434793472 + }, + { + "epoch": 0.48, + "learning_rate": 5.2961001444390956e-05, + "loss": 2.6785, + "theoretical_loss": 3.375640619069457, + "tokens_seen": 2434859008 + }, + { + "epoch": 0.48, + "learning_rate": 5.295297705023271e-05, + "loss": 2.5723, + "theoretical_loss": 3.375626048007287, + "tokens_seen": 2434990080 + }, + { + "epoch": 0.48, + "learning_rate": 5.294495265607446e-05, + "loss": 2.5132, + "theoretical_loss": 3.375611477949037, + "tokens_seen": 2435121152 + }, + { + "epoch": 0.48, + "learning_rate": 5.293692826191623e-05, + "loss": 2.525, + "theoretical_loss": 3.3755969088945834, + "tokens_seen": 2435252224 + }, + { + "epoch": 0.48, + "learning_rate": 5.292890386775798e-05, + "loss": 2.6585, + "theoretical_loss": 3.375582340843803, + "tokens_seen": 2435383296 + }, + { + "epoch": 0.48, + "learning_rate": 5.292087947359975e-05, + "loss": 2.547, + "theoretical_loss": 3.375567773796573, + "tokens_seen": 2435514368 + }, + { + "epoch": 0.48, + "learning_rate": 5.29128550794415e-05, + "loss": 2.5946, + "theoretical_loss": 3.3755532077527692, + "tokens_seen": 2435645440 + }, + { + "epoch": 0.48, + "learning_rate": 5.290483068528327e-05, + "loss": 2.4455, + "theoretical_loss": 3.3755386427122698, + "tokens_seen": 2435776512 + }, + { + "epoch": 0.48, + "learning_rate": 5.289680629112502e-05, + "loss": 2.5117, + "theoretical_loss": 3.375524078674951, + "tokens_seen": 2435907584 + }, + { + "epoch": 0.48, + "learning_rate": 5.2888781896966775e-05, + "loss": 2.6521, + "theoretical_loss": 3.3755095156406902, + "tokens_seen": 2436038656 + }, + { + "epoch": 0.48, + "learning_rate": 5.288075750280854e-05, + "loss": 2.5638, + "theoretical_loss": 3.3754949536093637, + "tokens_seen": 2436169728 + }, + { + "epoch": 0.48, + "learning_rate": 5.2872733108650295e-05, + "loss": 2.4175, + "theoretical_loss": 3.375480392580849, + "tokens_seen": 2436300800 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1338213, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.685955047607422, + "objective/train/theoretical_loss": 3.375465832555023, + "objective/train/tokens_used": 806890976, + "theoretical_loss": 3.375465832555023, + "tokens_seen": 2436431872 + }, + { + "epoch": 0.48, + "learning_rate": 5.286470871449206e-05, + "loss": 2.7174, + "theoretical_loss": 3.375465832555023, + "tokens_seen": 2436431872 + }, + { + "epoch": 0.48, + "learning_rate": 5.2856684320333815e-05, + "loss": 2.5334, + "theoretical_loss": 3.3754512735317626, + "tokens_seen": 2436562944 + }, + { + "epoch": 0.48, + "learning_rate": 5.284865992617558e-05, + "loss": 2.485, + "theoretical_loss": 3.3754367155109444, + "tokens_seen": 2436694016 + }, + { + "epoch": 0.48, + "learning_rate": 5.2840635532017334e-05, + "loss": 2.6672, + "theoretical_loss": 3.3754221584924466, + "tokens_seen": 2436825088 + }, + { + "epoch": 0.48, + "learning_rate": 5.283261113785909e-05, + "loss": 2.6874, + "theoretical_loss": 3.3754076024761455, + "tokens_seen": 2436956160 + }, + { + "epoch": 0.48, + "learning_rate": 5.2824586743700854e-05, + "loss": 2.5222, + "theoretical_loss": 3.375393047461918, + "tokens_seen": 2437087232 + }, + { + "epoch": 0.48, + "learning_rate": 5.281656234954261e-05, + "loss": 2.6489, + "theoretical_loss": 3.3753784934496416, + "tokens_seen": 2437218304 + }, + { + "epoch": 0.48, + "learning_rate": 5.2808537955384374e-05, + "loss": 2.5962, + "theoretical_loss": 3.3753639404391937, + "tokens_seen": 2437349376 + }, + { + "epoch": 0.48, + "learning_rate": 5.280051356122613e-05, + "loss": 2.5768, + "theoretical_loss": 3.3753493884304513, + "tokens_seen": 2437480448 + }, + { + "epoch": 0.48, + "learning_rate": 5.2792489167067894e-05, + "loss": 2.5981, + "theoretical_loss": 3.375334837423291, + "tokens_seen": 2437611520 + }, + { + "epoch": 0.48, + "learning_rate": 5.278446477290965e-05, + "loss": 2.492, + "theoretical_loss": 3.3753202874175905, + "tokens_seen": 2437742592 + }, + { + "epoch": 0.48, + "learning_rate": 5.27764403787514e-05, + "loss": 2.4221, + "theoretical_loss": 3.3753057384132275, + "tokens_seen": 2437873664 + }, + { + "epoch": 0.48, + "learning_rate": 5.276841598459317e-05, + "loss": 2.5098, + "theoretical_loss": 3.3752911904100786, + "tokens_seen": 2438004736 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1339345, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3139004707336426, + "objective/train/theoretical_loss": 3.375283916783921, + "objective/train/tokens_used": 808529376, + "theoretical_loss": 3.375283916783921, + "tokens_seen": 2438070272 + }, + { + "epoch": 0.48, + "learning_rate": 5.276039159043492e-05, + "loss": 2.4769, + "theoretical_loss": 3.375276643408021, + "tokens_seen": 2438135808 + }, + { + "epoch": 0.48, + "learning_rate": 5.275236719627669e-05, + "loss": 2.6318, + "theoretical_loss": 3.3752620974069325, + "tokens_seen": 2438266880 + }, + { + "epoch": 0.48, + "learning_rate": 5.274434280211844e-05, + "loss": 2.6198, + "theoretical_loss": 3.37524755240669, + "tokens_seen": 2438397952 + }, + { + "epoch": 0.48, + "learning_rate": 5.2736318407960206e-05, + "loss": 2.4233, + "theoretical_loss": 3.3752330084071707, + "tokens_seen": 2438529024 + }, + { + "epoch": 0.48, + "learning_rate": 5.272829401380196e-05, + "loss": 2.7931, + "theoretical_loss": 3.3752184654082527, + "tokens_seen": 2438660096 + }, + { + "epoch": 0.48, + "learning_rate": 5.272026961964371e-05, + "loss": 2.4301, + "theoretical_loss": 3.375203923409813, + "tokens_seen": 2438791168 + }, + { + "epoch": 0.48, + "learning_rate": 5.271224522548548e-05, + "loss": 2.5547, + "theoretical_loss": 3.3751893824117287, + "tokens_seen": 2438922240 + }, + { + "epoch": 0.48, + "learning_rate": 5.270422083132723e-05, + "loss": 2.467, + "theoretical_loss": 3.3751748424138777, + "tokens_seen": 2439053312 + }, + { + "epoch": 0.48, + "learning_rate": 5.2696196437169e-05, + "loss": 2.4588, + "theoretical_loss": 3.3751603034161373, + "tokens_seen": 2439184384 + }, + { + "epoch": 0.48, + "learning_rate": 5.268817204301075e-05, + "loss": 2.5843, + "theoretical_loss": 3.375145765418385, + "tokens_seen": 2439315456 + }, + { + "epoch": 0.48, + "learning_rate": 5.268014764885252e-05, + "loss": 2.4959, + "theoretical_loss": 3.3751312284204977, + "tokens_seen": 2439446528 + }, + { + "epoch": 0.48, + "learning_rate": 5.267212325469427e-05, + "loss": 2.6227, + "theoretical_loss": 3.375116692422354, + "tokens_seen": 2439577600 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1339900, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3154780864715576, + "objective/train/theoretical_loss": 3.3751021574238305, + "objective/train/tokens_used": 810167776, + "theoretical_loss": 3.3751021574238305, + "tokens_seen": 2439708672 + }, + { + "epoch": 0.48, + "learning_rate": 5.266409886053604e-05, + "loss": 2.5463, + "theoretical_loss": 3.3751021574238305, + "tokens_seen": 2439708672 + }, + { + "epoch": 0.48, + "learning_rate": 5.265607446637779e-05, + "loss": 2.6162, + "theoretical_loss": 3.3750876234248053, + "tokens_seen": 2439839744 + }, + { + "epoch": 0.48, + "learning_rate": 5.2648050072219545e-05, + "loss": 2.4569, + "theoretical_loss": 3.375073090425156, + "tokens_seen": 2439970816 + }, + { + "epoch": 0.48, + "learning_rate": 5.264002567806131e-05, + "loss": 2.7436, + "theoretical_loss": 3.37505855842476, + "tokens_seen": 2440101888 + }, + { + "epoch": 0.48, + "learning_rate": 5.2632001283903065e-05, + "loss": 2.5632, + "theoretical_loss": 3.3750440274234945, + "tokens_seen": 2440232960 + }, + { + "epoch": 0.48, + "learning_rate": 5.262397688974483e-05, + "loss": 2.5628, + "theoretical_loss": 3.375029497421238, + "tokens_seen": 2440364032 + }, + { + "epoch": 0.48, + "learning_rate": 5.2615952495586585e-05, + "loss": 2.5599, + "theoretical_loss": 3.3750149684178674, + "tokens_seen": 2440495104 + }, + { + "epoch": 0.48, + "learning_rate": 5.260792810142835e-05, + "loss": 2.456, + "theoretical_loss": 3.375000440413261, + "tokens_seen": 2440626176 + }, + { + "epoch": 0.48, + "learning_rate": 5.2599903707270105e-05, + "loss": 2.4967, + "theoretical_loss": 3.374985913407296, + "tokens_seen": 2440757248 + }, + { + "epoch": 0.48, + "learning_rate": 5.259187931311186e-05, + "loss": 2.4109, + "theoretical_loss": 3.374971387399851, + "tokens_seen": 2440888320 + }, + { + "epoch": 0.48, + "learning_rate": 5.2583854918953624e-05, + "loss": 2.4852, + "theoretical_loss": 3.3749568623908024, + "tokens_seen": 2441019392 + }, + { + "epoch": 0.48, + "learning_rate": 5.257583052479538e-05, + "loss": 2.5694, + "theoretical_loss": 3.3749423383800288, + "tokens_seen": 2441150464 + }, + { + "epoch": 0.48, + "learning_rate": 5.2567806130637144e-05, + "loss": 2.4533, + "theoretical_loss": 3.374927815367408, + "tokens_seen": 2441281536 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1340505, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.749099016189575, + "objective/train/theoretical_loss": 3.374920554235367, + "objective/train/tokens_used": 811806176, + "theoretical_loss": 3.374920554235367, + "tokens_seen": 2441347072 + }, + { + "epoch": 0.48, + "learning_rate": 5.25597817364789e-05, + "loss": 2.4812, + "theoretical_loss": 3.374913293352818, + "tokens_seen": 2441412608 + }, + { + "epoch": 0.48, + "learning_rate": 5.2551757342320664e-05, + "loss": 2.735, + "theoretical_loss": 3.3748987723361363, + "tokens_seen": 2441543680 + }, + { + "epoch": 0.48, + "learning_rate": 5.254373294816242e-05, + "loss": 2.6154, + "theoretical_loss": 3.3748842523172407, + "tokens_seen": 2441674752 + }, + { + "epoch": 0.48, + "learning_rate": 5.253570855400417e-05, + "loss": 2.5746, + "theoretical_loss": 3.3748697332960087, + "tokens_seen": 2441805824 + }, + { + "epoch": 0.48, + "learning_rate": 5.252768415984594e-05, + "loss": 2.4751, + "theoretical_loss": 3.3748552152723192, + "tokens_seen": 2441936896 + }, + { + "epoch": 0.48, + "learning_rate": 5.251965976568769e-05, + "loss": 2.4473, + "theoretical_loss": 3.3748406982460497, + "tokens_seen": 2442067968 + }, + { + "epoch": 0.48, + "learning_rate": 5.251163537152946e-05, + "loss": 2.6456, + "theoretical_loss": 3.374826182217078, + "tokens_seen": 2442199040 + }, + { + "epoch": 0.48, + "learning_rate": 5.250361097737121e-05, + "loss": 2.5668, + "theoretical_loss": 3.3748116671852824, + "tokens_seen": 2442330112 + }, + { + "epoch": 0.48, + "learning_rate": 5.2495586583212977e-05, + "loss": 2.4327, + "theoretical_loss": 3.3747971531505403, + "tokens_seen": 2442461184 + }, + { + "epoch": 0.48, + "learning_rate": 5.248756218905473e-05, + "loss": 2.6369, + "theoretical_loss": 3.3747826401127297, + "tokens_seen": 2442592256 + }, + { + "epoch": 0.48, + "learning_rate": 5.247953779489648e-05, + "loss": 2.5001, + "theoretical_loss": 3.3747681280717297, + "tokens_seen": 2442723328 + }, + { + "epoch": 0.48, + "learning_rate": 5.247151340073825e-05, + "loss": 2.4715, + "theoretical_loss": 3.374753617027417, + "tokens_seen": 2442854400 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1341788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.345665454864502, + "objective/train/theoretical_loss": 3.374739106979671, + "objective/train/tokens_used": 813444576, + "theoretical_loss": 3.374739106979671, + "tokens_seen": 2442985472 + }, + { + "epoch": 0.48, + "learning_rate": 5.246348900658e-05, + "loss": 2.5201, + "theoretical_loss": 3.374739106979671, + "tokens_seen": 2442985472 + }, + { + "epoch": 0.48, + "learning_rate": 5.245546461242177e-05, + "loss": 2.4456, + "theoretical_loss": 3.3747245979283687, + "tokens_seen": 2443116544 + }, + { + "epoch": 0.48, + "learning_rate": 5.244744021826352e-05, + "loss": 2.5736, + "theoretical_loss": 3.374710089873388, + "tokens_seen": 2443247616 + }, + { + "epoch": 0.48, + "learning_rate": 5.243941582410529e-05, + "loss": 2.5839, + "theoretical_loss": 3.3746955828146086, + "tokens_seen": 2443378688 + }, + { + "epoch": 0.48, + "learning_rate": 5.243139142994704e-05, + "loss": 2.5527, + "theoretical_loss": 3.3746810767519073, + "tokens_seen": 2443509760 + }, + { + "epoch": 0.48, + "learning_rate": 5.2423367035788795e-05, + "loss": 2.6065, + "theoretical_loss": 3.374666571685163, + "tokens_seen": 2443640832 + }, + { + "epoch": 0.48, + "learning_rate": 5.241534264163056e-05, + "loss": 2.63, + "theoretical_loss": 3.3746520676142535, + "tokens_seen": 2443771904 + }, + { + "epoch": 0.48, + "learning_rate": 5.2407318247472315e-05, + "loss": 2.6752, + "theoretical_loss": 3.374637564539057, + "tokens_seen": 2443902976 + }, + { + "epoch": 0.48, + "learning_rate": 5.239929385331408e-05, + "loss": 2.5175, + "theoretical_loss": 3.374623062459452, + "tokens_seen": 2444034048 + }, + { + "epoch": 0.48, + "learning_rate": 5.2391269459155835e-05, + "loss": 2.5524, + "theoretical_loss": 3.3746085613753163, + "tokens_seen": 2444165120 + }, + { + "epoch": 0.48, + "learning_rate": 5.23832450649976e-05, + "loss": 2.5963, + "theoretical_loss": 3.374594061286529, + "tokens_seen": 2444296192 + }, + { + "epoch": 0.48, + "learning_rate": 5.2375220670839355e-05, + "loss": 2.5523, + "theoretical_loss": 3.3745795621929675, + "tokens_seen": 2444427264 + }, + { + "epoch": 0.48, + "learning_rate": 5.236719627668112e-05, + "loss": 2.5234, + "theoretical_loss": 3.374565064094511, + "tokens_seen": 2444558336 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1342804, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.517542839050293, + "objective/train/theoretical_loss": 3.374557815418409, + "objective/train/tokens_used": 815082976, + "theoretical_loss": 3.374557815418409, + "tokens_seen": 2444623872 + }, + { + "epoch": 0.48, + "learning_rate": 5.2359171882522875e-05, + "loss": 2.6831, + "theoretical_loss": 3.3745505669910374, + "tokens_seen": 2444689408 + }, + { + "epoch": 0.48, + "learning_rate": 5.235114748836463e-05, + "loss": 2.6545, + "theoretical_loss": 3.374536070882425, + "tokens_seen": 2444820480 + }, + { + "epoch": 0.48, + "learning_rate": 5.2343123094206394e-05, + "loss": 2.7343, + "theoretical_loss": 3.3745215757685525, + "tokens_seen": 2444951552 + }, + { + "epoch": 0.48, + "learning_rate": 5.233509870004815e-05, + "loss": 2.5024, + "theoretical_loss": 3.3745070816492975, + "tokens_seen": 2445082624 + }, + { + "epoch": 0.48, + "learning_rate": 5.2327074305889914e-05, + "loss": 2.6012, + "theoretical_loss": 3.37449258852454, + "tokens_seen": 2445213696 + }, + { + "epoch": 0.48, + "learning_rate": 5.231904991173167e-05, + "loss": 2.5065, + "theoretical_loss": 3.374478096394157, + "tokens_seen": 2445344768 + }, + { + "epoch": 0.48, + "learning_rate": 5.2311025517573434e-05, + "loss": 2.7124, + "theoretical_loss": 3.3744636052580272, + "tokens_seen": 2445475840 + }, + { + "epoch": 0.48, + "learning_rate": 5.230300112341519e-05, + "loss": 2.4797, + "theoretical_loss": 3.37444911511603, + "tokens_seen": 2445606912 + }, + { + "epoch": 0.48, + "learning_rate": 5.229497672925694e-05, + "loss": 2.624, + "theoretical_loss": 3.3744346259680427, + "tokens_seen": 2445737984 + }, + { + "epoch": 0.48, + "learning_rate": 5.228695233509871e-05, + "loss": 2.6437, + "theoretical_loss": 3.374420137813945, + "tokens_seen": 2445869056 + }, + { + "epoch": 0.48, + "learning_rate": 5.227892794094046e-05, + "loss": 2.591, + "theoretical_loss": 3.374405650653615, + "tokens_seen": 2446000128 + }, + { + "epoch": 0.48, + "learning_rate": 5.227090354678223e-05, + "loss": 2.6138, + "theoretical_loss": 3.374391164486931, + "tokens_seen": 2446131200 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1343168, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7890498638153076, + "objective/train/theoretical_loss": 3.3743766793137717, + "objective/train/tokens_used": 816721376, + "theoretical_loss": 3.3743766793137717, + "tokens_seen": 2446262272 + }, + { + "epoch": 0.48, + "learning_rate": 5.226287915262398e-05, + "loss": 2.6727, + "theoretical_loss": 3.3743766793137717, + "tokens_seen": 2446262272 + }, + { + "epoch": 0.48, + "learning_rate": 5.225485475846575e-05, + "loss": 2.5655, + "theoretical_loss": 3.374362195134016, + "tokens_seen": 2446393344 + }, + { + "epoch": 0.48, + "learning_rate": 5.22468303643075e-05, + "loss": 2.6566, + "theoretical_loss": 3.374347711947543, + "tokens_seen": 2446524416 + }, + { + "epoch": 0.48, + "learning_rate": 5.223880597014925e-05, + "loss": 2.4497, + "theoretical_loss": 3.37433322975423, + "tokens_seen": 2446655488 + }, + { + "epoch": 0.48, + "learning_rate": 5.223078157599102e-05, + "loss": 2.5973, + "theoretical_loss": 3.3743187485539567, + "tokens_seen": 2446786560 + }, + { + "epoch": 0.48, + "learning_rate": 5.222275718183277e-05, + "loss": 2.5613, + "theoretical_loss": 3.374304268346602, + "tokens_seen": 2446917632 + }, + { + "epoch": 0.48, + "learning_rate": 5.221473278767454e-05, + "loss": 2.5539, + "theoretical_loss": 3.374289789132044, + "tokens_seen": 2447048704 + }, + { + "epoch": 0.48, + "learning_rate": 5.220670839351629e-05, + "loss": 2.4292, + "theoretical_loss": 3.3742753109101615, + "tokens_seen": 2447179776 + }, + { + "epoch": 0.48, + "learning_rate": 5.219868399935805e-05, + "loss": 2.3909, + "theoretical_loss": 3.3742608336808337, + "tokens_seen": 2447310848 + }, + { + "epoch": 0.48, + "learning_rate": 5.219065960519981e-05, + "loss": 2.5961, + "theoretical_loss": 3.3742463574439396, + "tokens_seen": 2447441920 + }, + { + "epoch": 0.48, + "learning_rate": 5.2182635211041565e-05, + "loss": 2.5862, + "theoretical_loss": 3.374231882199357, + "tokens_seen": 2447572992 + }, + { + "epoch": 0.48, + "learning_rate": 5.217461081688333e-05, + "loss": 2.5475, + "theoretical_loss": 3.374217407946966, + "tokens_seen": 2447704064 + }, + { + "epoch": 0.48, + "learning_rate": 5.2166586422725085e-05, + "loss": 2.6481, + "theoretical_loss": 3.374202934686644, + "tokens_seen": 2447835136 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1344177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.358229398727417, + "objective/train/theoretical_loss": 3.374195698428472, + "objective/train/tokens_used": 818359776, + "theoretical_loss": 3.374195698428472, + "tokens_seen": 2447900672 + }, + { + "epoch": 0.48, + "learning_rate": 5.215856202856685e-05, + "loss": 2.5777, + "theoretical_loss": 3.3741884624182714, + "tokens_seen": 2447966208 + }, + { + "epoch": 0.48, + "learning_rate": 5.2150537634408605e-05, + "loss": 2.6033, + "theoretical_loss": 3.3741739911417263, + "tokens_seen": 2448097280 + }, + { + "epoch": 0.48, + "learning_rate": 5.2142513240250365e-05, + "loss": 2.6933, + "theoretical_loss": 3.3741595208568875, + "tokens_seen": 2448228352 + }, + { + "epoch": 0.48, + "learning_rate": 5.2134488846092125e-05, + "loss": 2.4876, + "theoretical_loss": 3.3741450515636346, + "tokens_seen": 2448359424 + }, + { + "epoch": 0.48, + "learning_rate": 5.212646445193388e-05, + "loss": 2.5319, + "theoretical_loss": 3.374130583261846, + "tokens_seen": 2448490496 + }, + { + "epoch": 0.48, + "learning_rate": 5.2118440057775645e-05, + "loss": 2.6603, + "theoretical_loss": 3.374116115951401, + "tokens_seen": 2448621568 + }, + { + "epoch": 0.48, + "learning_rate": 5.21104156636174e-05, + "loss": 2.7219, + "theoretical_loss": 3.374101649632178, + "tokens_seen": 2448752640 + }, + { + "epoch": 0.48, + "learning_rate": 5.210239126945916e-05, + "loss": 2.4685, + "theoretical_loss": 3.3740871843040567, + "tokens_seen": 2448883712 + }, + { + "epoch": 0.48, + "learning_rate": 5.209436687530092e-05, + "loss": 2.6344, + "theoretical_loss": 3.3740727199669163, + "tokens_seen": 2449014784 + }, + { + "epoch": 0.48, + "learning_rate": 5.208634248114268e-05, + "loss": 2.7244, + "theoretical_loss": 3.3740582566206356, + "tokens_seen": 2449145856 + }, + { + "epoch": 0.48, + "learning_rate": 5.207831808698444e-05, + "loss": 2.5339, + "theoretical_loss": 3.374043794265093, + "tokens_seen": 2449276928 + }, + { + "epoch": 0.48, + "learning_rate": 5.207029369282619e-05, + "loss": 2.4953, + "theoretical_loss": 3.374029332900169, + "tokens_seen": 2449408000 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1344624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.644493818283081, + "objective/train/theoretical_loss": 3.3740148725257417, + "objective/train/tokens_used": 819998176, + "theoretical_loss": 3.3740148725257417, + "tokens_seen": 2449539072 + }, + { + "epoch": 0.48, + "learning_rate": 5.206226929866795e-05, + "loss": 2.6183, + "theoretical_loss": 3.3740148725257417, + "tokens_seen": 2449539072 + }, + { + "epoch": 0.48, + "learning_rate": 5.205424490450971e-05, + "loss": 2.7257, + "theoretical_loss": 3.3740004131416907, + "tokens_seen": 2449670144 + }, + { + "epoch": 0.48, + "learning_rate": 5.204622051035147e-05, + "loss": 2.6306, + "theoretical_loss": 3.373985954747895, + "tokens_seen": 2449801216 + }, + { + "epoch": 0.48, + "learning_rate": 5.203819611619323e-05, + "loss": 2.7159, + "theoretical_loss": 3.3739714973442343, + "tokens_seen": 2449932288 + }, + { + "epoch": 0.48, + "learning_rate": 5.203017172203499e-05, + "loss": 2.5434, + "theoretical_loss": 3.373957040930587, + "tokens_seen": 2450063360 + }, + { + "epoch": 0.48, + "learning_rate": 5.202214732787675e-05, + "loss": 2.5135, + "theoretical_loss": 3.373942585506833, + "tokens_seen": 2450194432 + }, + { + "epoch": 0.49, + "learning_rate": 5.201412293371851e-05, + "loss": 2.6426, + "theoretical_loss": 3.3739281310728506, + "tokens_seen": 2450325504 + }, + { + "epoch": 0.49, + "learning_rate": 5.200609853956026e-05, + "loss": 2.5788, + "theoretical_loss": 3.3739136776285203, + "tokens_seen": 2450456576 + }, + { + "epoch": 0.49, + "learning_rate": 5.199807414540202e-05, + "loss": 2.5713, + "theoretical_loss": 3.373899225173721, + "tokens_seen": 2450587648 + }, + { + "epoch": 0.49, + "learning_rate": 5.199004975124378e-05, + "loss": 2.6554, + "theoretical_loss": 3.3738847737083324, + "tokens_seen": 2450718720 + }, + { + "epoch": 0.49, + "learning_rate": 5.198202535708554e-05, + "loss": 2.6084, + "theoretical_loss": 3.3738703232322327, + "tokens_seen": 2450849792 + }, + { + "epoch": 0.49, + "learning_rate": 5.19740009629273e-05, + "loss": 2.5222, + "theoretical_loss": 3.373855873745302, + "tokens_seen": 2450980864 + }, + { + "epoch": 0.49, + "learning_rate": 5.1965976568769056e-05, + "loss": 2.5954, + "theoretical_loss": 3.37384142524742, + "tokens_seen": 2451111936 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1344624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7756404876708984, + "objective/train/theoretical_loss": 3.3738342013693345, + "objective/train/tokens_used": 821636576, + "theoretical_loss": 3.3738342013693345, + "tokens_seen": 2451177472 + }, + { + "epoch": 0.49, + "learning_rate": 5.195795217461082e-05, + "loss": 2.524, + "theoretical_loss": 3.373826977738466, + "tokens_seen": 2451243008 + }, + { + "epoch": 0.49, + "learning_rate": 5.1949927780452576e-05, + "loss": 2.6562, + "theoretical_loss": 3.3738125312183187, + "tokens_seen": 2451374080 + }, + { + "epoch": 0.49, + "learning_rate": 5.1941903386294336e-05, + "loss": 2.6662, + "theoretical_loss": 3.3737980856868584, + "tokens_seen": 2451505152 + }, + { + "epoch": 0.49, + "learning_rate": 5.1933878992136095e-05, + "loss": 2.8164, + "theoretical_loss": 3.3737836411439646, + "tokens_seen": 2451636224 + }, + { + "epoch": 0.49, + "learning_rate": 5.1925854597977855e-05, + "loss": 2.7042, + "theoretical_loss": 3.3737691975895157, + "tokens_seen": 2451767296 + }, + { + "epoch": 0.49, + "learning_rate": 5.1917830203819615e-05, + "loss": 2.764, + "theoretical_loss": 3.3737547550233926, + "tokens_seen": 2451898368 + }, + { + "epoch": 0.49, + "learning_rate": 5.190980580966137e-05, + "loss": 2.8597, + "theoretical_loss": 3.373740313445474, + "tokens_seen": 2452029440 + }, + { + "epoch": 0.49, + "learning_rate": 5.1901781415503135e-05, + "loss": 2.7401, + "theoretical_loss": 3.37372587285564, + "tokens_seen": 2452160512 + }, + { + "epoch": 0.49, + "learning_rate": 5.189375702134489e-05, + "loss": 2.8246, + "theoretical_loss": 3.37371143325377, + "tokens_seen": 2452291584 + }, + { + "epoch": 0.49, + "learning_rate": 5.188573262718665e-05, + "loss": 2.8532, + "theoretical_loss": 3.373696994639743, + "tokens_seen": 2452422656 + }, + { + "epoch": 0.49, + "learning_rate": 5.187770823302841e-05, + "loss": 2.7613, + "theoretical_loss": 3.3736825570134394, + "tokens_seen": 2452553728 + }, + { + "epoch": 0.49, + "learning_rate": 5.186968383887016e-05, + "loss": 2.7873, + "theoretical_loss": 3.3736681203747385, + "tokens_seen": 2452684800 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1345207, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.519474983215332, + "objective/train/theoretical_loss": 3.37365368472352, + "objective/train/tokens_used": 823274976, + "theoretical_loss": 3.37365368472352, + "tokens_seen": 2452815872 + }, + { + "epoch": 0.49, + "learning_rate": 5.186165944471193e-05, + "loss": 2.7948, + "theoretical_loss": 3.37365368472352, + "tokens_seen": 2452815872 + }, + { + "epoch": 0.49, + "learning_rate": 5.185363505055368e-05, + "loss": 2.6965, + "theoretical_loss": 3.373639250059664, + "tokens_seen": 2452946944 + }, + { + "epoch": 0.49, + "learning_rate": 5.184561065639545e-05, + "loss": 2.8823, + "theoretical_loss": 3.3736248163830496, + "tokens_seen": 2453078016 + }, + { + "epoch": 0.49, + "learning_rate": 5.18375862622372e-05, + "loss": 2.8049, + "theoretical_loss": 3.3736103836935567, + "tokens_seen": 2453209088 + }, + { + "epoch": 0.49, + "learning_rate": 5.182956186807896e-05, + "loss": 2.7712, + "theoretical_loss": 3.3735959519910654, + "tokens_seen": 2453340160 + }, + { + "epoch": 0.49, + "learning_rate": 5.182153747392072e-05, + "loss": 2.7904, + "theoretical_loss": 3.373581521275455, + "tokens_seen": 2453471232 + }, + { + "epoch": 0.49, + "learning_rate": 5.1813513079762474e-05, + "loss": 2.7418, + "theoretical_loss": 3.3735670915466054, + "tokens_seen": 2453602304 + }, + { + "epoch": 0.49, + "learning_rate": 5.180548868560424e-05, + "loss": 2.7428, + "theoretical_loss": 3.3735526628043973, + "tokens_seen": 2453733376 + }, + { + "epoch": 0.49, + "learning_rate": 5.1797464291445994e-05, + "loss": 2.6637, + "theoretical_loss": 3.3735382350487093, + "tokens_seen": 2453864448 + }, + { + "epoch": 0.49, + "learning_rate": 5.178943989728776e-05, + "loss": 2.7932, + "theoretical_loss": 3.3735238082794217, + "tokens_seen": 2453995520 + }, + { + "epoch": 0.49, + "learning_rate": 5.178141550312951e-05, + "loss": 2.7252, + "theoretical_loss": 3.3735093824964144, + "tokens_seen": 2454126592 + }, + { + "epoch": 0.49, + "learning_rate": 5.1773391108971266e-05, + "loss": 2.7983, + "theoretical_loss": 3.373494957699567, + "tokens_seen": 2454257664 + }, + { + "epoch": 0.49, + "learning_rate": 5.176536671481303e-05, + "loss": 2.5646, + "theoretical_loss": 3.3734805338887606, + "tokens_seen": 2454388736 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1345665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9347848892211914, + "objective/train/theoretical_loss": 3.3734733223530844, + "objective/train/tokens_used": 824913376, + "theoretical_loss": 3.3734733223530844, + "tokens_seen": 2454454272 + }, + { + "epoch": 0.49, + "learning_rate": 5.1757342320654786e-05, + "loss": 2.6224, + "theoretical_loss": 3.3734661110638737, + "tokens_seen": 2454519808 + }, + { + "epoch": 0.49, + "learning_rate": 5.174931792649655e-05, + "loss": 2.7118, + "theoretical_loss": 3.3734516892247868, + "tokens_seen": 2454650880 + }, + { + "epoch": 0.49, + "learning_rate": 5.1741293532338306e-05, + "loss": 2.6218, + "theoretical_loss": 3.3734372683713802, + "tokens_seen": 2454781952 + }, + { + "epoch": 0.49, + "learning_rate": 5.173326913818007e-05, + "loss": 2.6646, + "theoretical_loss": 3.373422848503534, + "tokens_seen": 2454913024 + }, + { + "epoch": 0.49, + "learning_rate": 5.1725244744021826e-05, + "loss": 2.5645, + "theoretical_loss": 3.3734084296211275, + "tokens_seen": 2455044096 + }, + { + "epoch": 0.49, + "learning_rate": 5.171722034986359e-05, + "loss": 2.5378, + "theoretical_loss": 3.373394011724041, + "tokens_seen": 2455175168 + }, + { + "epoch": 0.49, + "learning_rate": 5.1709195955705346e-05, + "loss": 2.9382, + "theoretical_loss": 3.373379594812155, + "tokens_seen": 2455306240 + }, + { + "epoch": 0.49, + "learning_rate": 5.17011715615471e-05, + "loss": 2.6852, + "theoretical_loss": 3.373365178885349, + "tokens_seen": 2455437312 + }, + { + "epoch": 0.49, + "learning_rate": 5.1693147167388866e-05, + "loss": 2.7406, + "theoretical_loss": 3.373350763943504, + "tokens_seen": 2455568384 + }, + { + "epoch": 0.49, + "learning_rate": 5.168512277323062e-05, + "loss": 2.6722, + "theoretical_loss": 3.373336349986499, + "tokens_seen": 2455699456 + }, + { + "epoch": 0.49, + "learning_rate": 5.1677098379072385e-05, + "loss": 2.7807, + "theoretical_loss": 3.373321937014215, + "tokens_seen": 2455830528 + }, + { + "epoch": 0.49, + "learning_rate": 5.166907398491414e-05, + "loss": 2.6833, + "theoretical_loss": 3.3733075250265316, + "tokens_seen": 2455961600 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1346863, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5424647331237793, + "objective/train/theoretical_loss": 3.3732931140233293, + "objective/train/tokens_used": 826551776, + "theoretical_loss": 3.3732931140233293, + "tokens_seen": 2456092672 + }, + { + "epoch": 0.49, + "learning_rate": 5.1661049590755905e-05, + "loss": 2.6451, + "theoretical_loss": 3.3732931140233293, + "tokens_seen": 2456092672 + }, + { + "epoch": 0.49, + "learning_rate": 5.165302519659766e-05, + "loss": 2.662, + "theoretical_loss": 3.3732787040044885, + "tokens_seen": 2456223744 + }, + { + "epoch": 0.49, + "learning_rate": 5.164500080243941e-05, + "loss": 2.6761, + "theoretical_loss": 3.373264294969889, + "tokens_seen": 2456354816 + }, + { + "epoch": 0.49, + "learning_rate": 5.163697640828118e-05, + "loss": 2.7512, + "theoretical_loss": 3.3732498869194116, + "tokens_seen": 2456485888 + }, + { + "epoch": 0.49, + "learning_rate": 5.162895201412293e-05, + "loss": 2.6402, + "theoretical_loss": 3.373235479852936, + "tokens_seen": 2456616960 + }, + { + "epoch": 0.49, + "learning_rate": 5.16209276199647e-05, + "loss": 2.7339, + "theoretical_loss": 3.3732210737703427, + "tokens_seen": 2456748032 + }, + { + "epoch": 0.49, + "learning_rate": 5.161290322580645e-05, + "loss": 2.5177, + "theoretical_loss": 3.3732066686715125, + "tokens_seen": 2456879104 + }, + { + "epoch": 0.49, + "learning_rate": 5.160487883164822e-05, + "loss": 2.7342, + "theoretical_loss": 3.373192264556325, + "tokens_seen": 2457010176 + }, + { + "epoch": 0.49, + "learning_rate": 5.159685443748997e-05, + "loss": 2.7059, + "theoretical_loss": 3.373177861424661, + "tokens_seen": 2457141248 + }, + { + "epoch": 0.49, + "learning_rate": 5.1588830043331724e-05, + "loss": 2.7184, + "theoretical_loss": 3.373163459276401, + "tokens_seen": 2457272320 + }, + { + "epoch": 0.49, + "learning_rate": 5.158080564917349e-05, + "loss": 2.7066, + "theoretical_loss": 3.3731490581114247, + "tokens_seen": 2457403392 + }, + { + "epoch": 0.49, + "learning_rate": 5.1572781255015244e-05, + "loss": 2.6591, + "theoretical_loss": 3.3731346579296133, + "tokens_seen": 2457534464 + }, + { + "epoch": 0.49, + "learning_rate": 5.156475686085701e-05, + "loss": 2.8136, + "theoretical_loss": 3.3731202587308475, + "tokens_seen": 2457665536 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1347964, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6252315044403076, + "objective/train/theoretical_loss": 3.373113059500069, + "objective/train/tokens_used": 828190176, + "theoretical_loss": 3.373113059500069, + "tokens_seen": 2457731072 + }, + { + "epoch": 0.49, + "learning_rate": 5.1556732466698764e-05, + "loss": 2.7172, + "theoretical_loss": 3.3731058605150066, + "tokens_seen": 2457796608 + }, + { + "epoch": 0.49, + "learning_rate": 5.154870807254053e-05, + "loss": 2.7197, + "theoretical_loss": 3.3730914632819715, + "tokens_seen": 2457927680 + }, + { + "epoch": 0.49, + "learning_rate": 5.1540683678382283e-05, + "loss": 2.7076, + "theoretical_loss": 3.3730770670316237, + "tokens_seen": 2458058752 + }, + { + "epoch": 0.49, + "learning_rate": 5.1532659284224037e-05, + "loss": 2.5872, + "theoretical_loss": 3.3730626717638423, + "tokens_seen": 2458189824 + }, + { + "epoch": 0.49, + "learning_rate": 5.15246348900658e-05, + "loss": 2.5952, + "theoretical_loss": 3.373048277478509, + "tokens_seen": 2458320896 + }, + { + "epoch": 0.49, + "learning_rate": 5.1516610495907556e-05, + "loss": 2.6224, + "theoretical_loss": 3.373033884175504, + "tokens_seen": 2458451968 + }, + { + "epoch": 0.49, + "learning_rate": 5.150858610174932e-05, + "loss": 2.8066, + "theoretical_loss": 3.3730194918547074, + "tokens_seen": 2458583040 + }, + { + "epoch": 0.49, + "learning_rate": 5.1500561707591076e-05, + "loss": 2.6329, + "theoretical_loss": 3.373005100516, + "tokens_seen": 2458714112 + }, + { + "epoch": 0.49, + "learning_rate": 5.149253731343284e-05, + "loss": 2.6037, + "theoretical_loss": 3.372990710159263, + "tokens_seen": 2458845184 + }, + { + "epoch": 0.49, + "learning_rate": 5.1484512919274596e-05, + "loss": 2.7772, + "theoretical_loss": 3.372976320784377, + "tokens_seen": 2458976256 + }, + { + "epoch": 0.49, + "learning_rate": 5.147648852511635e-05, + "loss": 2.5924, + "theoretical_loss": 3.3729619323912217, + "tokens_seen": 2459107328 + }, + { + "epoch": 0.49, + "learning_rate": 5.1468464130958116e-05, + "loss": 2.5872, + "theoretical_loss": 3.372947544979679, + "tokens_seen": 2459238400 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1348513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.667315721511841, + "objective/train/theoretical_loss": 3.372933158549629, + "objective/train/tokens_used": 829828576, + "theoretical_loss": 3.372933158549629, + "tokens_seen": 2459369472 + }, + { + "epoch": 0.49, + "learning_rate": 5.146043973679987e-05, + "loss": 2.55, + "theoretical_loss": 3.372933158549629, + "tokens_seen": 2459369472 + }, + { + "epoch": 0.49, + "learning_rate": 5.1452415342641636e-05, + "loss": 2.8019, + "theoretical_loss": 3.3729187731009524, + "tokens_seen": 2459500544 + }, + { + "epoch": 0.49, + "learning_rate": 5.144439094848339e-05, + "loss": 2.6017, + "theoretical_loss": 3.37290438863353, + "tokens_seen": 2459631616 + }, + { + "epoch": 0.49, + "learning_rate": 5.1436366554325155e-05, + "loss": 2.7036, + "theoretical_loss": 3.372890005147243, + "tokens_seen": 2459762688 + }, + { + "epoch": 0.49, + "learning_rate": 5.142834216016691e-05, + "loss": 2.6547, + "theoretical_loss": 3.372875622641972, + "tokens_seen": 2459893760 + }, + { + "epoch": 0.49, + "learning_rate": 5.1420317766008675e-05, + "loss": 2.5111, + "theoretical_loss": 3.3728612411175973, + "tokens_seen": 2460024832 + }, + { + "epoch": 0.49, + "learning_rate": 5.141229337185043e-05, + "loss": 2.7321, + "theoretical_loss": 3.372846860574, + "tokens_seen": 2460155904 + }, + { + "epoch": 0.49, + "learning_rate": 5.140426897769218e-05, + "loss": 2.6469, + "theoretical_loss": 3.3728324810110615, + "tokens_seen": 2460286976 + }, + { + "epoch": 0.49, + "learning_rate": 5.139624458353395e-05, + "loss": 2.6475, + "theoretical_loss": 3.3728181024286625, + "tokens_seen": 2460418048 + }, + { + "epoch": 0.49, + "learning_rate": 5.13882201893757e-05, + "loss": 2.7767, + "theoretical_loss": 3.3728037248266833, + "tokens_seen": 2460549120 + }, + { + "epoch": 0.49, + "learning_rate": 5.138019579521747e-05, + "loss": 2.7516, + "theoretical_loss": 3.3727893482050053, + "tokens_seen": 2460680192 + }, + { + "epoch": 0.49, + "learning_rate": 5.137217140105922e-05, + "loss": 2.7426, + "theoretical_loss": 3.3727749725635094, + "tokens_seen": 2460811264 + }, + { + "epoch": 0.49, + "learning_rate": 5.136414700690099e-05, + "loss": 2.5846, + "theoretical_loss": 3.3727605979020767, + "tokens_seen": 2460942336 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1349898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7990500926971436, + "objective/train/theoretical_loss": 3.372753410938847, + "objective/train/tokens_used": 831466976, + "theoretical_loss": 3.372753410938847, + "tokens_seen": 2461007872 + }, + { + "epoch": 0.49, + "learning_rate": 5.135612261274274e-05, + "loss": 2.608, + "theoretical_loss": 3.372746224220588, + "tokens_seen": 2461073408 + }, + { + "epoch": 0.49, + "learning_rate": 5.1348098218584494e-05, + "loss": 2.6354, + "theoretical_loss": 3.372731851518924, + "tokens_seen": 2461204480 + }, + { + "epoch": 0.49, + "learning_rate": 5.134007382442626e-05, + "loss": 2.7203, + "theoretical_loss": 3.3727174797969663, + "tokens_seen": 2461335552 + }, + { + "epoch": 0.49, + "learning_rate": 5.1332049430268014e-05, + "loss": 2.4329, + "theoretical_loss": 3.372703109054596, + "tokens_seen": 2461466624 + }, + { + "epoch": 0.49, + "learning_rate": 5.132402503610978e-05, + "loss": 2.7455, + "theoretical_loss": 3.3726887392916938, + "tokens_seen": 2461597696 + }, + { + "epoch": 0.49, + "learning_rate": 5.1316000641951534e-05, + "loss": 2.5633, + "theoretical_loss": 3.372674370508141, + "tokens_seen": 2461728768 + }, + { + "epoch": 0.49, + "learning_rate": 5.13079762477933e-05, + "loss": 2.7246, + "theoretical_loss": 3.3726600027038183, + "tokens_seen": 2461859840 + }, + { + "epoch": 0.49, + "learning_rate": 5.1299951853635053e-05, + "loss": 2.6832, + "theoretical_loss": 3.3726456358786074, + "tokens_seen": 2461990912 + }, + { + "epoch": 0.49, + "learning_rate": 5.1291927459476807e-05, + "loss": 2.6317, + "theoretical_loss": 3.3726312700323895, + "tokens_seen": 2462121984 + }, + { + "epoch": 0.49, + "learning_rate": 5.128390306531857e-05, + "loss": 2.6313, + "theoretical_loss": 3.372616905165045, + "tokens_seen": 2462253056 + }, + { + "epoch": 0.49, + "learning_rate": 5.1275878671160326e-05, + "loss": 2.7807, + "theoretical_loss": 3.3726025412764558, + "tokens_seen": 2462384128 + }, + { + "epoch": 0.49, + "learning_rate": 5.126785427700209e-05, + "loss": 2.6791, + "theoretical_loss": 3.372588178366503, + "tokens_seen": 2462515200 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1350590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.718848466873169, + "objective/train/theoretical_loss": 3.3725738164350676, + "objective/train/tokens_used": 833105376, + "theoretical_loss": 3.3725738164350676, + "tokens_seen": 2462646272 + }, + { + "epoch": 0.49, + "learning_rate": 5.1259829882843846e-05, + "loss": 2.5802, + "theoretical_loss": 3.3725738164350676, + "tokens_seen": 2462646272 + }, + { + "epoch": 0.49, + "learning_rate": 5.125180548868561e-05, + "loss": 2.6466, + "theoretical_loss": 3.372559455482031, + "tokens_seen": 2462777344 + }, + { + "epoch": 0.49, + "learning_rate": 5.1243781094527366e-05, + "loss": 2.6131, + "theoretical_loss": 3.3725450955072747, + "tokens_seen": 2462908416 + }, + { + "epoch": 0.49, + "learning_rate": 5.123575670036912e-05, + "loss": 2.4635, + "theoretical_loss": 3.3725307365106794, + "tokens_seen": 2463039488 + }, + { + "epoch": 0.49, + "learning_rate": 5.1227732306210886e-05, + "loss": 2.6381, + "theoretical_loss": 3.3725163784921275, + "tokens_seen": 2463170560 + }, + { + "epoch": 0.49, + "learning_rate": 5.121970791205264e-05, + "loss": 2.6174, + "theoretical_loss": 3.372502021451499, + "tokens_seen": 2463301632 + }, + { + "epoch": 0.49, + "learning_rate": 5.1211683517894406e-05, + "loss": 2.6745, + "theoretical_loss": 3.372487665388676, + "tokens_seen": 2463432704 + }, + { + "epoch": 0.49, + "learning_rate": 5.120365912373616e-05, + "loss": 2.5964, + "theoretical_loss": 3.37247331030354, + "tokens_seen": 2463563776 + }, + { + "epoch": 0.49, + "learning_rate": 5.1195634729577925e-05, + "loss": 2.7596, + "theoretical_loss": 3.372458956195972, + "tokens_seen": 2463694848 + }, + { + "epoch": 0.49, + "learning_rate": 5.118761033541968e-05, + "loss": 2.7028, + "theoretical_loss": 3.372444603065854, + "tokens_seen": 2463825920 + }, + { + "epoch": 0.49, + "learning_rate": 5.117958594126143e-05, + "loss": 2.637, + "theoretical_loss": 3.372430250913067, + "tokens_seen": 2463956992 + }, + { + "epoch": 0.49, + "learning_rate": 5.11715615471032e-05, + "loss": 2.6179, + "theoretical_loss": 3.3724158997374922, + "tokens_seen": 2464088064 + }, + { + "epoch": 0.49, + "learning_rate": 5.116353715294495e-05, + "loss": 2.6899, + "theoretical_loss": 3.3724015495390116, + "tokens_seen": 2464219136 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1351948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8566057682037354, + "objective/train/theoretical_loss": 3.3723943748061447, + "objective/train/tokens_used": 834743776, + "theoretical_loss": 3.3723943748061447, + "tokens_seen": 2464284672 + }, + { + "epoch": 0.49, + "learning_rate": 5.115551275878672e-05, + "loss": 2.5858, + "theoretical_loss": 3.372387200317507, + "tokens_seen": 2464350208 + }, + { + "epoch": 0.49, + "learning_rate": 5.114748836462847e-05, + "loss": 2.6211, + "theoretical_loss": 3.372372852072859, + "tokens_seen": 2464481280 + }, + { + "epoch": 0.49, + "learning_rate": 5.113946397047024e-05, + "loss": 2.7022, + "theoretical_loss": 3.3723585048049496, + "tokens_seen": 2464612352 + }, + { + "epoch": 0.49, + "learning_rate": 5.113143957631199e-05, + "loss": 2.6312, + "theoretical_loss": 3.3723441585136604, + "tokens_seen": 2464743424 + }, + { + "epoch": 0.49, + "learning_rate": 5.1123415182153744e-05, + "loss": 2.5399, + "theoretical_loss": 3.3723298131988733, + "tokens_seen": 2464874496 + }, + { + "epoch": 0.49, + "learning_rate": 5.111539078799551e-05, + "loss": 2.6586, + "theoretical_loss": 3.3723154688604695, + "tokens_seen": 2465005568 + }, + { + "epoch": 0.49, + "learning_rate": 5.1107366393837264e-05, + "loss": 2.5115, + "theoretical_loss": 3.3723011254983306, + "tokens_seen": 2465136640 + }, + { + "epoch": 0.49, + "learning_rate": 5.109934199967903e-05, + "loss": 2.615, + "theoretical_loss": 3.3722867831123384, + "tokens_seen": 2465267712 + }, + { + "epoch": 0.49, + "learning_rate": 5.1091317605520784e-05, + "loss": 2.7011, + "theoretical_loss": 3.372272441702375, + "tokens_seen": 2465398784 + }, + { + "epoch": 0.49, + "learning_rate": 5.108329321136255e-05, + "loss": 2.4709, + "theoretical_loss": 3.372258101268321, + "tokens_seen": 2465529856 + }, + { + "epoch": 0.49, + "learning_rate": 5.1075268817204304e-05, + "loss": 2.7755, + "theoretical_loss": 3.3722437618100587, + "tokens_seen": 2465660928 + }, + { + "epoch": 0.49, + "learning_rate": 5.106724442304607e-05, + "loss": 2.5329, + "theoretical_loss": 3.3722294233274703, + "tokens_seen": 2465792000 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1352648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8698949813842773, + "objective/train/theoretical_loss": 3.372215085820437, + "objective/train/tokens_used": 836382176, + "theoretical_loss": 3.372215085820437, + "tokens_seen": 2465923072 + }, + { + "epoch": 0.49, + "learning_rate": 5.1059220028887824e-05, + "loss": 2.5483, + "theoretical_loss": 3.372215085820437, + "tokens_seen": 2465923072 + }, + { + "epoch": 0.49, + "learning_rate": 5.105119563472958e-05, + "loss": 2.7256, + "theoretical_loss": 3.372200749288841, + "tokens_seen": 2466054144 + }, + { + "epoch": 0.49, + "learning_rate": 5.104317124057134e-05, + "loss": 2.5776, + "theoretical_loss": 3.372186413732564, + "tokens_seen": 2466185216 + }, + { + "epoch": 0.49, + "learning_rate": 5.1035146846413096e-05, + "loss": 2.6369, + "theoretical_loss": 3.372172079151487, + "tokens_seen": 2466316288 + }, + { + "epoch": 0.49, + "learning_rate": 5.102712245225486e-05, + "loss": 2.7825, + "theoretical_loss": 3.3721577455454925, + "tokens_seen": 2466447360 + }, + { + "epoch": 0.49, + "learning_rate": 5.1019098058096616e-05, + "loss": 2.7237, + "theoretical_loss": 3.372143412914463, + "tokens_seen": 2466578432 + }, + { + "epoch": 0.49, + "learning_rate": 5.101107366393838e-05, + "loss": 2.6892, + "theoretical_loss": 3.3721290812582794, + "tokens_seen": 2466709504 + }, + { + "epoch": 0.5, + "learning_rate": 5.1003049269780136e-05, + "loss": 2.639, + "theoretical_loss": 3.3721147505768236, + "tokens_seen": 2466840576 + }, + { + "epoch": 0.5, + "learning_rate": 5.099502487562189e-05, + "loss": 2.5936, + "theoretical_loss": 3.3721004208699785, + "tokens_seen": 2466971648 + }, + { + "epoch": 0.5, + "learning_rate": 5.0987000481463656e-05, + "loss": 2.5451, + "theoretical_loss": 3.372086092137625, + "tokens_seen": 2467102720 + }, + { + "epoch": 0.5, + "learning_rate": 5.097897608730541e-05, + "loss": 2.6, + "theoretical_loss": 3.3720717643796454, + "tokens_seen": 2467233792 + }, + { + "epoch": 0.5, + "learning_rate": 5.0970951693147176e-05, + "loss": 2.64, + "theoretical_loss": 3.372057437595922, + "tokens_seen": 2467364864 + }, + { + "epoch": 0.5, + "learning_rate": 5.096292729898893e-05, + "loss": 2.5725, + "theoretical_loss": 3.3720431117863363, + "tokens_seen": 2467495936 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1353854, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4819204807281494, + "objective/train/theoretical_loss": 3.3720359492468086, + "objective/train/tokens_used": 838020576, + "theoretical_loss": 3.3720359492468086, + "tokens_seen": 2467561472 + }, + { + "epoch": 0.5, + "learning_rate": 5.0954902904830696e-05, + "loss": 2.598, + "theoretical_loss": 3.3720287869507706, + "tokens_seen": 2467627008 + }, + { + "epoch": 0.5, + "learning_rate": 5.094687851067245e-05, + "loss": 2.5879, + "theoretical_loss": 3.3720144630891076, + "tokens_seen": 2467758080 + }, + { + "epoch": 0.5, + "learning_rate": 5.09388541165142e-05, + "loss": 2.6088, + "theoretical_loss": 3.372000140201228, + "tokens_seen": 2467889152 + }, + { + "epoch": 0.5, + "learning_rate": 5.093082972235597e-05, + "loss": 2.7607, + "theoretical_loss": 3.3719858182870146, + "tokens_seen": 2468020224 + }, + { + "epoch": 0.5, + "learning_rate": 5.092280532819772e-05, + "loss": 2.5092, + "theoretical_loss": 3.3719714973463493, + "tokens_seen": 2468151296 + }, + { + "epoch": 0.5, + "learning_rate": 5.091478093403949e-05, + "loss": 2.7098, + "theoretical_loss": 3.3719571773791146, + "tokens_seen": 2468282368 + }, + { + "epoch": 0.5, + "learning_rate": 5.090675653988124e-05, + "loss": 2.4803, + "theoretical_loss": 3.3719428583851925, + "tokens_seen": 2468413440 + }, + { + "epoch": 0.5, + "learning_rate": 5.089873214572301e-05, + "loss": 2.5423, + "theoretical_loss": 3.3719285403644648, + "tokens_seen": 2468544512 + }, + { + "epoch": 0.5, + "learning_rate": 5.089070775156476e-05, + "loss": 2.5517, + "theoretical_loss": 3.3719142233168142, + "tokens_seen": 2468675584 + }, + { + "epoch": 0.5, + "learning_rate": 5.0882683357406514e-05, + "loss": 2.6932, + "theoretical_loss": 3.371899907242123, + "tokens_seen": 2468806656 + }, + { + "epoch": 0.5, + "learning_rate": 5.087465896324828e-05, + "loss": 2.5645, + "theoretical_loss": 3.371885592140273, + "tokens_seen": 2468937728 + }, + { + "epoch": 0.5, + "learning_rate": 5.0866634569090034e-05, + "loss": 2.5671, + "theoretical_loss": 3.371871278011146, + "tokens_seen": 2469068800 + }, + { + "debugging/Self-BLEU-5": 0.42407658532325987, + "debugging/distinct-1-grams": 0.7177325581395348, + "debugging/distinct-2-grams": 0.8906535491482647, + "debugging/entropy-1-grams": 5.181652846266418, + "debugging/entropy-2-grams": 5.834764426176351, + "debugging/length": 560.8, + "debugging/num_segments": 5, + "debugging/score": 0.0042737600834809476, + "debugging/score_std": 0.004323690737910133, + "epoch": 0.5, + "objective/train/docs_used": 1354527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7488014698028564, + "objective/train/theoretical_loss": 3.3718569648546257, + "objective/train/tokens_used": 839658976, + "theoretical_loss": 3.3718569648546257, + "tokens_seen": 2469199872 + }, + { + "epoch": 0.5, + "learning_rate": 5.08586101749318e-05, + "loss": 2.6643, + "theoretical_loss": 3.3718569648546257, + "tokens_seen": 2469199872 + }, + { + "epoch": 0.5, + "learning_rate": 5.0850585780773554e-05, + "loss": 2.5209, + "theoretical_loss": 3.371842652670593, + "tokens_seen": 2469330944 + }, + { + "epoch": 0.5, + "learning_rate": 5.0842561386615314e-05, + "loss": 2.6081, + "theoretical_loss": 3.371828341458931, + "tokens_seen": 2469462016 + }, + { + "epoch": 0.5, + "learning_rate": 5.0834536992457074e-05, + "loss": 2.7511, + "theoretical_loss": 3.371814031219522, + "tokens_seen": 2469593088 + }, + { + "epoch": 0.5, + "learning_rate": 5.082651259829883e-05, + "loss": 2.635, + "theoretical_loss": 3.371799721952248, + "tokens_seen": 2469724160 + }, + { + "epoch": 0.5, + "learning_rate": 5.0818488204140594e-05, + "loss": 2.5922, + "theoretical_loss": 3.3717854136569914, + "tokens_seen": 2469855232 + }, + { + "epoch": 0.5, + "learning_rate": 5.081046380998235e-05, + "loss": 2.7167, + "theoretical_loss": 3.3717711063336346, + "tokens_seen": 2469986304 + }, + { + "epoch": 0.5, + "learning_rate": 5.0802439415824113e-05, + "loss": 2.4931, + "theoretical_loss": 3.37175679998206, + "tokens_seen": 2470117376 + }, + { + "epoch": 0.5, + "learning_rate": 5.0794415021665867e-05, + "loss": 2.5988, + "theoretical_loss": 3.3717424946021506, + "tokens_seen": 2470248448 + }, + { + "epoch": 0.5, + "learning_rate": 5.0786390627507626e-05, + "loss": 2.4903, + "theoretical_loss": 3.371728190193788, + "tokens_seen": 2470379520 + }, + { + "epoch": 0.5, + "learning_rate": 5.0778366233349386e-05, + "loss": 2.5597, + "theoretical_loss": 3.3717138867568552, + "tokens_seen": 2470510592 + }, + { + "epoch": 0.5, + "learning_rate": 5.0770341839191146e-05, + "loss": 2.3949, + "theoretical_loss": 3.371699584291235, + "tokens_seen": 2470641664 + }, + { + "epoch": 0.5, + "learning_rate": 5.0762317445032906e-05, + "loss": 2.6598, + "theoretical_loss": 3.371685282796809, + "tokens_seen": 2470772736 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1355685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7235753536224365, + "objective/train/theoretical_loss": 3.3716781324137575, + "objective/train/tokens_used": 841297376, + "theoretical_loss": 3.3716781324137575, + "tokens_seen": 2470838272 + }, + { + "epoch": 0.5, + "learning_rate": 5.075429305087466e-05, + "loss": 2.6547, + "theoretical_loss": 3.3716709822734603, + "tokens_seen": 2470903808 + }, + { + "epoch": 0.5, + "learning_rate": 5.074626865671642e-05, + "loss": 2.5513, + "theoretical_loss": 3.3716566827210714, + "tokens_seen": 2471034880 + }, + { + "epoch": 0.5, + "learning_rate": 5.073824426255818e-05, + "loss": 2.4664, + "theoretical_loss": 3.371642384139525, + "tokens_seen": 2471165952 + }, + { + "epoch": 0.5, + "learning_rate": 5.073021986839994e-05, + "loss": 2.4618, + "theoretical_loss": 3.3716280865287036, + "tokens_seen": 2471297024 + }, + { + "epoch": 0.5, + "learning_rate": 5.07221954742417e-05, + "loss": 2.7009, + "theoretical_loss": 3.3716137898884897, + "tokens_seen": 2471428096 + }, + { + "epoch": 0.5, + "learning_rate": 5.071417108008346e-05, + "loss": 2.6315, + "theoretical_loss": 3.371599494218766, + "tokens_seen": 2471559168 + }, + { + "epoch": 0.5, + "learning_rate": 5.070614668592522e-05, + "loss": 2.4532, + "theoretical_loss": 3.3715851995194153, + "tokens_seen": 2471690240 + }, + { + "epoch": 0.5, + "learning_rate": 5.069812229176697e-05, + "loss": 2.6446, + "theoretical_loss": 3.3715709057903203, + "tokens_seen": 2471821312 + }, + { + "epoch": 0.5, + "learning_rate": 5.069009789760873e-05, + "loss": 2.6, + "theoretical_loss": 3.371556613031363, + "tokens_seen": 2471952384 + }, + { + "epoch": 0.5, + "learning_rate": 5.068207350345049e-05, + "loss": 2.4892, + "theoretical_loss": 3.371542321242427, + "tokens_seen": 2472083456 + }, + { + "epoch": 0.5, + "learning_rate": 5.067404910929225e-05, + "loss": 2.608, + "theoretical_loss": 3.371528030423395, + "tokens_seen": 2472214528 + }, + { + "epoch": 0.5, + "learning_rate": 5.066602471513401e-05, + "loss": 2.5077, + "theoretical_loss": 3.371513740574149, + "tokens_seen": 2472345600 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1356735, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5951905250549316, + "objective/train/theoretical_loss": 3.3714994516945724, + "objective/train/tokens_used": 842935776, + "theoretical_loss": 3.3714994516945724, + "tokens_seen": 2472476672 + }, + { + "epoch": 0.5, + "learning_rate": 5.065800032097577e-05, + "loss": 2.6154, + "theoretical_loss": 3.3714994516945724, + "tokens_seen": 2472476672 + }, + { + "epoch": 0.5, + "learning_rate": 5.0649975926817525e-05, + "loss": 2.5361, + "theoretical_loss": 3.371485163784548, + "tokens_seen": 2472607744 + }, + { + "epoch": 0.5, + "learning_rate": 5.0641951532659284e-05, + "loss": 2.5984, + "theoretical_loss": 3.3714708768439583, + "tokens_seen": 2472738816 + }, + { + "epoch": 0.5, + "learning_rate": 5.0633927138501044e-05, + "loss": 2.6947, + "theoretical_loss": 3.3714565908726866, + "tokens_seen": 2472869888 + }, + { + "epoch": 0.5, + "learning_rate": 5.0625902744342804e-05, + "loss": 2.6804, + "theoretical_loss": 3.371442305870615, + "tokens_seen": 2473000960 + }, + { + "epoch": 0.5, + "learning_rate": 5.0617878350184564e-05, + "loss": 2.5793, + "theoretical_loss": 3.3714280218376276, + "tokens_seen": 2473132032 + }, + { + "epoch": 0.5, + "learning_rate": 5.0609853956026324e-05, + "loss": 2.7422, + "theoretical_loss": 3.371413738773606, + "tokens_seen": 2473263104 + }, + { + "epoch": 0.5, + "learning_rate": 5.0601829561868084e-05, + "loss": 2.7063, + "theoretical_loss": 3.371399456678434, + "tokens_seen": 2473394176 + }, + { + "epoch": 0.5, + "learning_rate": 5.059380516770984e-05, + "loss": 2.6895, + "theoretical_loss": 3.371385175551994, + "tokens_seen": 2473525248 + }, + { + "epoch": 0.5, + "learning_rate": 5.05857807735516e-05, + "loss": 2.7181, + "theoretical_loss": 3.3713708953941692, + "tokens_seen": 2473656320 + }, + { + "epoch": 0.5, + "learning_rate": 5.057775637939336e-05, + "loss": 2.4096, + "theoretical_loss": 3.3713566162048427, + "tokens_seen": 2473787392 + }, + { + "epoch": 0.5, + "learning_rate": 5.056973198523512e-05, + "loss": 2.6133, + "theoretical_loss": 3.3713423379838976, + "tokens_seen": 2473918464 + }, + { + "epoch": 0.5, + "learning_rate": 5.056170759107688e-05, + "loss": 2.6098, + "theoretical_loss": 3.3713280607312166, + "tokens_seen": 2474049536 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1357343, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.478666067123413, + "objective/train/theoretical_loss": 3.3713209224679384, + "objective/train/tokens_used": 844574176, + "theoretical_loss": 3.3713209224679384, + "tokens_seen": 2474115072 + }, + { + "epoch": 0.5, + "learning_rate": 5.055368319691863e-05, + "loss": 2.6532, + "theoretical_loss": 3.3713137844466825, + "tokens_seen": 2474180608 + }, + { + "epoch": 0.5, + "learning_rate": 5.0545658802760397e-05, + "loss": 2.6792, + "theoretical_loss": 3.3712995091301794, + "tokens_seen": 2474311680 + }, + { + "epoch": 0.5, + "learning_rate": 5.053763440860215e-05, + "loss": 2.7248, + "theoretical_loss": 3.371285234781589, + "tokens_seen": 2474442752 + }, + { + "epoch": 0.5, + "learning_rate": 5.052961001444391e-05, + "loss": 2.5951, + "theoretical_loss": 3.371270961400796, + "tokens_seen": 2474573824 + }, + { + "epoch": 0.5, + "learning_rate": 5.052158562028567e-05, + "loss": 2.6659, + "theoretical_loss": 3.371256688987682, + "tokens_seen": 2474704896 + }, + { + "epoch": 0.5, + "learning_rate": 5.051356122612743e-05, + "loss": 2.6503, + "theoretical_loss": 3.371242417542131, + "tokens_seen": 2474835968 + }, + { + "epoch": 0.5, + "learning_rate": 5.050553683196919e-05, + "loss": 2.7322, + "theoretical_loss": 3.3712281470640257, + "tokens_seen": 2474967040 + }, + { + "epoch": 0.5, + "learning_rate": 5.049751243781094e-05, + "loss": 2.7879, + "theoretical_loss": 3.37121387755325, + "tokens_seen": 2475098112 + }, + { + "epoch": 0.5, + "learning_rate": 5.048948804365271e-05, + "loss": 2.6329, + "theoretical_loss": 3.3711996090096865, + "tokens_seen": 2475229184 + }, + { + "epoch": 0.5, + "learning_rate": 5.048146364949446e-05, + "loss": 2.4507, + "theoretical_loss": 3.3711853414332187, + "tokens_seen": 2475360256 + }, + { + "epoch": 0.5, + "learning_rate": 5.047343925533623e-05, + "loss": 2.6312, + "theoretical_loss": 3.3711710748237294, + "tokens_seen": 2475491328 + }, + { + "epoch": 0.5, + "learning_rate": 5.046541486117798e-05, + "loss": 2.7004, + "theoretical_loss": 3.3711568091811026, + "tokens_seen": 2475622400 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1358753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6069936752319336, + "objective/train/theoretical_loss": 3.371142544505221, + "objective/train/tokens_used": 846212576, + "theoretical_loss": 3.371142544505221, + "tokens_seen": 2475753472 + }, + { + "epoch": 0.5, + "learning_rate": 5.0457390467019735e-05, + "loss": 2.4947, + "theoretical_loss": 3.371142544505221, + "tokens_seen": 2475753472 + }, + { + "epoch": 0.5, + "learning_rate": 5.04493660728615e-05, + "loss": 2.4908, + "theoretical_loss": 3.3711282807959684, + "tokens_seen": 2475884544 + }, + { + "epoch": 0.5, + "learning_rate": 5.0441341678703255e-05, + "loss": 2.5781, + "theoretical_loss": 3.3711140180532273, + "tokens_seen": 2476015616 + }, + { + "epoch": 0.5, + "learning_rate": 5.043331728454502e-05, + "loss": 2.5672, + "theoretical_loss": 3.371099756276882, + "tokens_seen": 2476146688 + }, + { + "epoch": 0.5, + "learning_rate": 5.0425292890386775e-05, + "loss": 2.4306, + "theoretical_loss": 3.3710854954668155, + "tokens_seen": 2476277760 + }, + { + "epoch": 0.5, + "learning_rate": 5.041726849622854e-05, + "loss": 2.6745, + "theoretical_loss": 3.371071235622911, + "tokens_seen": 2476408832 + }, + { + "epoch": 0.5, + "learning_rate": 5.0409244102070295e-05, + "loss": 2.6364, + "theoretical_loss": 3.3710569767450522, + "tokens_seen": 2476539904 + }, + { + "epoch": 0.5, + "learning_rate": 5.040121970791205e-05, + "loss": 2.6201, + "theoretical_loss": 3.371042718833122, + "tokens_seen": 2476670976 + }, + { + "epoch": 0.5, + "learning_rate": 5.0393195313753814e-05, + "loss": 2.7649, + "theoretical_loss": 3.3710284618870046, + "tokens_seen": 2476802048 + }, + { + "epoch": 0.5, + "learning_rate": 5.038517091959557e-05, + "loss": 2.4941, + "theoretical_loss": 3.371014205906583, + "tokens_seen": 2476933120 + }, + { + "epoch": 0.5, + "learning_rate": 5.0377146525437334e-05, + "loss": 2.5364, + "theoretical_loss": 3.3709999508917408, + "tokens_seen": 2477064192 + }, + { + "epoch": 0.5, + "learning_rate": 5.036912213127909e-05, + "loss": 2.7514, + "theoretical_loss": 3.3709856968423617, + "tokens_seen": 2477195264 + }, + { + "epoch": 0.5, + "learning_rate": 5.0361097737120854e-05, + "loss": 2.6732, + "theoretical_loss": 3.370971443758329, + "tokens_seen": 2477326336 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1359369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6083905696868896, + "objective/train/theoretical_loss": 3.3709643175782813, + "objective/train/tokens_used": 847850976, + "theoretical_loss": 3.3709643175782813, + "tokens_seen": 2477391872 + }, + { + "epoch": 0.5, + "learning_rate": 5.035307334296261e-05, + "loss": 2.5511, + "theoretical_loss": 3.3709571916395262, + "tokens_seen": 2477457408 + }, + { + "epoch": 0.5, + "learning_rate": 5.034504894880436e-05, + "loss": 2.6522, + "theoretical_loss": 3.370942940485837, + "tokens_seen": 2477588480 + }, + { + "epoch": 0.5, + "learning_rate": 5.033702455464613e-05, + "loss": 2.6202, + "theoretical_loss": 3.3709286902971454, + "tokens_seen": 2477719552 + }, + { + "epoch": 0.5, + "learning_rate": 5.032900016048788e-05, + "loss": 2.3975, + "theoretical_loss": 3.3709144410733343, + "tokens_seen": 2477850624 + }, + { + "epoch": 0.5, + "learning_rate": 5.032097576632965e-05, + "loss": 2.4792, + "theoretical_loss": 3.3709001928142874, + "tokens_seen": 2477981696 + }, + { + "epoch": 0.5, + "learning_rate": 5.03129513721714e-05, + "loss": 2.5769, + "theoretical_loss": 3.3708859455198885, + "tokens_seen": 2478112768 + }, + { + "epoch": 0.5, + "learning_rate": 5.030492697801317e-05, + "loss": 2.6005, + "theoretical_loss": 3.3708716991900216, + "tokens_seen": 2478243840 + }, + { + "epoch": 0.5, + "learning_rate": 5.029690258385492e-05, + "loss": 2.6088, + "theoretical_loss": 3.3708574538245704, + "tokens_seen": 2478374912 + }, + { + "epoch": 0.5, + "learning_rate": 5.028887818969667e-05, + "loss": 2.366, + "theoretical_loss": 3.370843209423418, + "tokens_seen": 2478505984 + }, + { + "epoch": 0.5, + "learning_rate": 5.028085379553844e-05, + "loss": 2.6869, + "theoretical_loss": 3.3708289659864485, + "tokens_seen": 2478637056 + }, + { + "epoch": 0.5, + "learning_rate": 5.027282940138019e-05, + "loss": 2.5781, + "theoretical_loss": 3.3708147235135457, + "tokens_seen": 2478768128 + }, + { + "epoch": 0.5, + "learning_rate": 5.026480500722196e-05, + "loss": 2.5796, + "theoretical_loss": 3.3708004820045936, + "tokens_seen": 2478899200 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1360741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.478170871734619, + "objective/train/theoretical_loss": 3.3707862414594754, + "objective/train/tokens_used": 849489376, + "theoretical_loss": 3.3707862414594754, + "tokens_seen": 2479030272 + }, + { + "epoch": 0.5, + "learning_rate": 5.025678061306371e-05, + "loss": 2.6031, + "theoretical_loss": 3.3707862414594754, + "tokens_seen": 2479030272 + }, + { + "epoch": 0.5, + "learning_rate": 5.024875621890548e-05, + "loss": 2.7964, + "theoretical_loss": 3.370772001878075, + "tokens_seen": 2479161344 + }, + { + "epoch": 0.5, + "learning_rate": 5.024073182474723e-05, + "loss": 2.5007, + "theoretical_loss": 3.370757763260277, + "tokens_seen": 2479292416 + }, + { + "epoch": 0.5, + "learning_rate": 5.0232707430588985e-05, + "loss": 2.5323, + "theoretical_loss": 3.3707435256059646, + "tokens_seen": 2479423488 + }, + { + "epoch": 0.5, + "learning_rate": 5.022468303643075e-05, + "loss": 2.6202, + "theoretical_loss": 3.3707292889150215, + "tokens_seen": 2479554560 + }, + { + "epoch": 0.5, + "learning_rate": 5.0216658642272505e-05, + "loss": 2.6986, + "theoretical_loss": 3.370715053187332, + "tokens_seen": 2479685632 + }, + { + "epoch": 0.5, + "learning_rate": 5.020863424811427e-05, + "loss": 2.7548, + "theoretical_loss": 3.37070081842278, + "tokens_seen": 2479816704 + }, + { + "epoch": 0.5, + "learning_rate": 5.0200609853956025e-05, + "loss": 2.5603, + "theoretical_loss": 3.3706865846212493, + "tokens_seen": 2479947776 + }, + { + "epoch": 0.5, + "learning_rate": 5.019258545979779e-05, + "loss": 2.6339, + "theoretical_loss": 3.3706723517826243, + "tokens_seen": 2480078848 + }, + { + "epoch": 0.5, + "learning_rate": 5.0184561065639545e-05, + "loss": 2.6331, + "theoretical_loss": 3.3706581199067878, + "tokens_seen": 2480209920 + }, + { + "epoch": 0.5, + "learning_rate": 5.01765366714813e-05, + "loss": 2.4755, + "theoretical_loss": 3.370643888993625, + "tokens_seen": 2480340992 + }, + { + "epoch": 0.5, + "learning_rate": 5.0168512277323065e-05, + "loss": 2.5282, + "theoretical_loss": 3.3706296590430194, + "tokens_seen": 2480472064 + }, + { + "epoch": 0.5, + "learning_rate": 5.016048788316482e-05, + "loss": 2.5725, + "theoretical_loss": 3.370615430054855, + "tokens_seen": 2480603136 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1361264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.49733829498291, + "objective/train/theoretical_loss": 3.370608315921652, + "objective/train/tokens_used": 851127776, + "theoretical_loss": 3.370608315921652, + "tokens_seen": 2480668672 + }, + { + "epoch": 0.5, + "learning_rate": 5.0152463489006585e-05, + "loss": 2.5991, + "theoretical_loss": 3.3706012020290164, + "tokens_seen": 2480734208 + }, + { + "epoch": 0.5, + "learning_rate": 5.014443909484834e-05, + "loss": 2.4878, + "theoretical_loss": 3.370586974965387, + "tokens_seen": 2480865280 + }, + { + "epoch": 0.5, + "learning_rate": 5.0136414700690104e-05, + "loss": 2.5512, + "theoretical_loss": 3.370572748863851, + "tokens_seen": 2480996352 + }, + { + "epoch": 0.5, + "learning_rate": 5.012839030653186e-05, + "loss": 2.4598, + "theoretical_loss": 3.370558523724293, + "tokens_seen": 2481127424 + }, + { + "epoch": 0.5, + "learning_rate": 5.0120365912373624e-05, + "loss": 2.5487, + "theoretical_loss": 3.3705442995465966, + "tokens_seen": 2481258496 + }, + { + "epoch": 0.5, + "learning_rate": 5.011234151821538e-05, + "loss": 2.5929, + "theoretical_loss": 3.3705300763306463, + "tokens_seen": 2481389568 + }, + { + "epoch": 0.5, + "learning_rate": 5.010431712405713e-05, + "loss": 2.6349, + "theoretical_loss": 3.3705158540763263, + "tokens_seen": 2481520640 + }, + { + "epoch": 0.5, + "learning_rate": 5.00962927298989e-05, + "loss": 2.5983, + "theoretical_loss": 3.3705016327835198, + "tokens_seen": 2481651712 + }, + { + "epoch": 0.5, + "learning_rate": 5.008826833574065e-05, + "loss": 2.4785, + "theoretical_loss": 3.3704874124521127, + "tokens_seen": 2481782784 + }, + { + "epoch": 0.5, + "learning_rate": 5.008024394158242e-05, + "loss": 2.5442, + "theoretical_loss": 3.3704731930819882, + "tokens_seen": 2481913856 + }, + { + "epoch": 0.5, + "learning_rate": 5.007221954742417e-05, + "loss": 2.6366, + "theoretical_loss": 3.3704589746730305, + "tokens_seen": 2482044928 + }, + { + "epoch": 0.5, + "learning_rate": 5.006419515326594e-05, + "loss": 2.5095, + "theoretical_loss": 3.3704447572251244, + "tokens_seen": 2482176000 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1362376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.822953462600708, + "objective/train/theoretical_loss": 3.3704305407381536, + "objective/train/tokens_used": 852766176, + "theoretical_loss": 3.3704305407381536, + "tokens_seen": 2482307072 + }, + { + "epoch": 0.5, + "learning_rate": 5.005617075910769e-05, + "loss": 2.5868, + "theoretical_loss": 3.3704305407381536, + "tokens_seen": 2482307072 + }, + { + "epoch": 0.5, + "learning_rate": 5.004814636494944e-05, + "loss": 2.5643, + "theoretical_loss": 3.370416325212003, + "tokens_seen": 2482438144 + }, + { + "epoch": 0.5, + "learning_rate": 5.004012197079121e-05, + "loss": 2.6704, + "theoretical_loss": 3.3704021106465567, + "tokens_seen": 2482569216 + }, + { + "epoch": 0.5, + "learning_rate": 5.003209757663296e-05, + "loss": 2.4693, + "theoretical_loss": 3.370387897041699, + "tokens_seen": 2482700288 + }, + { + "epoch": 0.5, + "learning_rate": 5.002407318247473e-05, + "loss": 2.4997, + "theoretical_loss": 3.370373684397314, + "tokens_seen": 2482831360 + }, + { + "epoch": 0.5, + "learning_rate": 5.001604878831648e-05, + "loss": 2.5091, + "theoretical_loss": 3.370359472713287, + "tokens_seen": 2482962432 + }, + { + "epoch": 0.5, + "learning_rate": 5.000802439415825e-05, + "loss": 2.5122, + "theoretical_loss": 3.3703452619895016, + "tokens_seen": 2483093504 + }, + { + "epoch": 0.51, + "learning_rate": 5e-05, + "loss": 2.6438, + "theoretical_loss": 3.370331052225842, + "tokens_seen": 2483224576 + }, + { + "epoch": 0.51, + "learning_rate": 4.999197560584176e-05, + "loss": 2.5141, + "theoretical_loss": 3.3703168434221937, + "tokens_seen": 2483355648 + }, + { + "epoch": 0.51, + "learning_rate": 4.998395121168352e-05, + "loss": 2.5419, + "theoretical_loss": 3.3703026355784402, + "tokens_seen": 2483486720 + }, + { + "epoch": 0.51, + "learning_rate": 4.997592681752528e-05, + "loss": 2.556, + "theoretical_loss": 3.370288428694467, + "tokens_seen": 2483617792 + }, + { + "epoch": 0.51, + "learning_rate": 4.9967902423367035e-05, + "loss": 2.6535, + "theoretical_loss": 3.370274222770157, + "tokens_seen": 2483748864 + }, + { + "epoch": 0.51, + "learning_rate": 4.9959878029208795e-05, + "loss": 2.5, + "theoretical_loss": 3.3702600178053967, + "tokens_seen": 2483879936 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1362743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5020596981048584, + "objective/train/theoretical_loss": 3.370252915682811, + "objective/train/tokens_used": 854404576, + "theoretical_loss": 3.370252915682811, + "tokens_seen": 2483945472 + }, + { + "epoch": 0.51, + "learning_rate": 4.9951853635050555e-05, + "loss": 2.6361, + "theoretical_loss": 3.3702458138000693, + "tokens_seen": 2484011008 + }, + { + "epoch": 0.51, + "learning_rate": 4.9943829240892315e-05, + "loss": 2.5663, + "theoretical_loss": 3.3702316107540597, + "tokens_seen": 2484142080 + }, + { + "epoch": 0.51, + "learning_rate": 4.9935804846734075e-05, + "loss": 2.427, + "theoretical_loss": 3.3702174086672527, + "tokens_seen": 2484273152 + }, + { + "epoch": 0.51, + "learning_rate": 4.9927780452575835e-05, + "loss": 2.6123, + "theoretical_loss": 3.3702032075395327, + "tokens_seen": 2484404224 + }, + { + "epoch": 0.51, + "learning_rate": 4.9919756058417595e-05, + "loss": 2.5062, + "theoretical_loss": 3.3701890073707848, + "tokens_seen": 2484535296 + }, + { + "epoch": 0.51, + "learning_rate": 4.9911731664259355e-05, + "loss": 2.5672, + "theoretical_loss": 3.370174808160893, + "tokens_seen": 2484666368 + }, + { + "epoch": 0.51, + "learning_rate": 4.990370727010111e-05, + "loss": 2.4534, + "theoretical_loss": 3.370160609909742, + "tokens_seen": 2484797440 + }, + { + "epoch": 0.51, + "learning_rate": 4.989568287594287e-05, + "loss": 2.5751, + "theoretical_loss": 3.3701464126172174, + "tokens_seen": 2484928512 + }, + { + "epoch": 0.51, + "learning_rate": 4.988765848178463e-05, + "loss": 2.4986, + "theoretical_loss": 3.370132216283203, + "tokens_seen": 2485059584 + }, + { + "epoch": 0.51, + "learning_rate": 4.987963408762639e-05, + "loss": 2.4899, + "theoretical_loss": 3.370118020907584, + "tokens_seen": 2485190656 + }, + { + "epoch": 0.51, + "learning_rate": 4.987160969346815e-05, + "loss": 2.5682, + "theoretical_loss": 3.3701038264902445, + "tokens_seen": 2485321728 + }, + { + "epoch": 0.51, + "learning_rate": 4.986358529930991e-05, + "loss": 2.556, + "theoretical_loss": 3.37008963303107, + "tokens_seen": 2485452800 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1363961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8076930046081543, + "objective/train/theoretical_loss": 3.370075440529945, + "objective/train/tokens_used": 856042976, + "theoretical_loss": 3.370075440529945, + "tokens_seen": 2485583872 + }, + { + "epoch": 0.51, + "learning_rate": 4.985556090515167e-05, + "loss": 2.649, + "theoretical_loss": 3.370075440529945, + "tokens_seen": 2485583872 + }, + { + "epoch": 0.51, + "learning_rate": 4.984753651099342e-05, + "loss": 2.4928, + "theoretical_loss": 3.370061248986754, + "tokens_seen": 2485714944 + }, + { + "epoch": 0.51, + "learning_rate": 4.983951211683518e-05, + "loss": 2.5232, + "theoretical_loss": 3.370047058401383, + "tokens_seen": 2485846016 + }, + { + "epoch": 0.51, + "learning_rate": 4.983148772267694e-05, + "loss": 2.6823, + "theoretical_loss": 3.3700328687737153, + "tokens_seen": 2485977088 + }, + { + "epoch": 0.51, + "learning_rate": 4.98234633285187e-05, + "loss": 2.4137, + "theoretical_loss": 3.3700186801036374, + "tokens_seen": 2486108160 + }, + { + "epoch": 0.51, + "learning_rate": 4.981543893436046e-05, + "loss": 2.5207, + "theoretical_loss": 3.3700044923910326, + "tokens_seen": 2486239232 + }, + { + "epoch": 0.51, + "learning_rate": 4.980741454020222e-05, + "loss": 2.5952, + "theoretical_loss": 3.3699903056357865, + "tokens_seen": 2486370304 + }, + { + "epoch": 0.51, + "learning_rate": 4.979939014604398e-05, + "loss": 2.5019, + "theoretical_loss": 3.3699761198377844, + "tokens_seen": 2486501376 + }, + { + "epoch": 0.51, + "learning_rate": 4.979136575188573e-05, + "loss": 2.6931, + "theoretical_loss": 3.369961934996911, + "tokens_seen": 2486632448 + }, + { + "epoch": 0.51, + "learning_rate": 4.978334135772749e-05, + "loss": 2.6033, + "theoretical_loss": 3.3699477511130516, + "tokens_seen": 2486763520 + }, + { + "epoch": 0.51, + "learning_rate": 4.977531696356925e-05, + "loss": 2.5466, + "theoretical_loss": 3.36993356818609, + "tokens_seen": 2486894592 + }, + { + "epoch": 0.51, + "learning_rate": 4.976729256941101e-05, + "loss": 2.4649, + "theoretical_loss": 3.3699193862159125, + "tokens_seen": 2487025664 + }, + { + "epoch": 0.51, + "learning_rate": 4.975926817525277e-05, + "loss": 2.491, + "theoretical_loss": 3.3699052052024037, + "tokens_seen": 2487156736 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1365358, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3158061504364014, + "objective/train/theoretical_loss": 3.369898115054364, + "objective/train/tokens_used": 857681376, + "theoretical_loss": 3.369898115054364, + "tokens_seen": 2487222272 + }, + { + "epoch": 0.51, + "learning_rate": 4.975124378109453e-05, + "loss": 2.5351, + "theoretical_loss": 3.3698910251454484, + "tokens_seen": 2487287808 + }, + { + "epoch": 0.51, + "learning_rate": 4.974321938693629e-05, + "loss": 2.5235, + "theoretical_loss": 3.369876846044932, + "tokens_seen": 2487418880 + }, + { + "epoch": 0.51, + "learning_rate": 4.973519499277805e-05, + "loss": 2.5791, + "theoretical_loss": 3.3698626679007395, + "tokens_seen": 2487549952 + }, + { + "epoch": 0.51, + "learning_rate": 4.9727170598619805e-05, + "loss": 2.5045, + "theoretical_loss": 3.369848490712756, + "tokens_seen": 2487681024 + }, + { + "epoch": 0.51, + "learning_rate": 4.9719146204461565e-05, + "loss": 2.5234, + "theoretical_loss": 3.3698343144808662, + "tokens_seen": 2487812096 + }, + { + "epoch": 0.51, + "learning_rate": 4.9711121810303325e-05, + "loss": 2.6729, + "theoretical_loss": 3.3698201392049563, + "tokens_seen": 2487943168 + }, + { + "epoch": 0.51, + "learning_rate": 4.9703097416145085e-05, + "loss": 2.6268, + "theoretical_loss": 3.3698059648849106, + "tokens_seen": 2488074240 + }, + { + "epoch": 0.51, + "learning_rate": 4.9695073021986845e-05, + "loss": 2.5327, + "theoretical_loss": 3.3697917915206146, + "tokens_seen": 2488205312 + }, + { + "epoch": 0.51, + "learning_rate": 4.9687048627828605e-05, + "loss": 2.6413, + "theoretical_loss": 3.3697776191119537, + "tokens_seen": 2488336384 + }, + { + "epoch": 0.51, + "learning_rate": 4.9679024233670365e-05, + "loss": 2.6266, + "theoretical_loss": 3.3697634476588125, + "tokens_seen": 2488467456 + }, + { + "epoch": 0.51, + "learning_rate": 4.967099983951212e-05, + "loss": 2.4853, + "theoretical_loss": 3.369749277161077, + "tokens_seen": 2488598528 + }, + { + "epoch": 0.51, + "learning_rate": 4.966297544535388e-05, + "loss": 2.5342, + "theoretical_loss": 3.3697351076186317, + "tokens_seen": 2488729600 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1366123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4803953170776367, + "objective/train/theoretical_loss": 3.369720939031363, + "objective/train/tokens_used": 859319776, + "theoretical_loss": 3.369720939031363, + "tokens_seen": 2488860672 + }, + { + "epoch": 0.51, + "learning_rate": 4.965495105119564e-05, + "loss": 2.5624, + "theoretical_loss": 3.369720939031363, + "tokens_seen": 2488860672 + }, + { + "epoch": 0.51, + "learning_rate": 4.96469266570374e-05, + "loss": 2.7138, + "theoretical_loss": 3.369706771399155, + "tokens_seen": 2488991744 + }, + { + "epoch": 0.51, + "learning_rate": 4.963890226287916e-05, + "loss": 2.4183, + "theoretical_loss": 3.3696926047218936, + "tokens_seen": 2489122816 + }, + { + "epoch": 0.51, + "learning_rate": 4.963087786872092e-05, + "loss": 2.5153, + "theoretical_loss": 3.3696784389994643, + "tokens_seen": 2489253888 + }, + { + "epoch": 0.51, + "learning_rate": 4.962285347456268e-05, + "loss": 2.5115, + "theoretical_loss": 3.369664274231752, + "tokens_seen": 2489384960 + }, + { + "epoch": 0.51, + "learning_rate": 4.961482908040443e-05, + "loss": 2.6792, + "theoretical_loss": 3.369650110418643, + "tokens_seen": 2489516032 + }, + { + "epoch": 0.51, + "learning_rate": 4.960680468624619e-05, + "loss": 2.614, + "theoretical_loss": 3.369635947560022, + "tokens_seen": 2489647104 + }, + { + "epoch": 0.51, + "learning_rate": 4.959878029208795e-05, + "loss": 2.6546, + "theoretical_loss": 3.369621785655774, + "tokens_seen": 2489778176 + }, + { + "epoch": 0.51, + "learning_rate": 4.959075589792971e-05, + "loss": 2.5551, + "theoretical_loss": 3.3696076247057856, + "tokens_seen": 2489909248 + }, + { + "epoch": 0.51, + "learning_rate": 4.958273150377147e-05, + "loss": 2.4587, + "theoretical_loss": 3.3695934647099413, + "tokens_seen": 2490040320 + }, + { + "epoch": 0.51, + "learning_rate": 4.957470710961323e-05, + "loss": 2.4979, + "theoretical_loss": 3.369579305668127, + "tokens_seen": 2490171392 + }, + { + "epoch": 0.51, + "learning_rate": 4.956668271545499e-05, + "loss": 2.5172, + "theoretical_loss": 3.3695651475802286, + "tokens_seen": 2490302464 + }, + { + "epoch": 0.51, + "learning_rate": 4.955865832129675e-05, + "loss": 2.6105, + "theoretical_loss": 3.3695509904461307, + "tokens_seen": 2490433536 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1367242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.245980978012085, + "objective/train/theoretical_loss": 3.3695439122367215, + "objective/train/tokens_used": 860958176, + "theoretical_loss": 3.3695439122367215, + "tokens_seen": 2490499072 + }, + { + "epoch": 0.51, + "learning_rate": 4.95506339271385e-05, + "loss": 2.4907, + "theoretical_loss": 3.3695368342657197, + "tokens_seen": 2490564608 + }, + { + "epoch": 0.51, + "learning_rate": 4.954260953298026e-05, + "loss": 2.5153, + "theoretical_loss": 3.36952267903888, + "tokens_seen": 2490695680 + }, + { + "epoch": 0.51, + "learning_rate": 4.953458513882202e-05, + "loss": 2.6679, + "theoretical_loss": 3.3695085247654992, + "tokens_seen": 2490826752 + }, + { + "epoch": 0.51, + "learning_rate": 4.952656074466378e-05, + "loss": 2.5825, + "theoretical_loss": 3.369494371445461, + "tokens_seen": 2490957824 + }, + { + "epoch": 0.51, + "learning_rate": 4.951853635050554e-05, + "loss": 2.6121, + "theoretical_loss": 3.369480219078652, + "tokens_seen": 2491088896 + }, + { + "epoch": 0.51, + "learning_rate": 4.95105119563473e-05, + "loss": 2.6157, + "theoretical_loss": 3.3694660676649573, + "tokens_seen": 2491219968 + }, + { + "epoch": 0.51, + "learning_rate": 4.950248756218906e-05, + "loss": 2.5463, + "theoretical_loss": 3.369451917204263, + "tokens_seen": 2491351040 + }, + { + "epoch": 0.51, + "learning_rate": 4.9494463168030815e-05, + "loss": 2.6447, + "theoretical_loss": 3.369437767696455, + "tokens_seen": 2491482112 + }, + { + "epoch": 0.51, + "learning_rate": 4.9486438773872575e-05, + "loss": 2.5978, + "theoretical_loss": 3.3694236191414184, + "tokens_seen": 2491613184 + }, + { + "epoch": 0.51, + "learning_rate": 4.9478414379714335e-05, + "loss": 2.6127, + "theoretical_loss": 3.369409471539039, + "tokens_seen": 2491744256 + }, + { + "epoch": 0.51, + "learning_rate": 4.9470389985556095e-05, + "loss": 2.6995, + "theoretical_loss": 3.369395324889203, + "tokens_seen": 2491875328 + }, + { + "epoch": 0.51, + "learning_rate": 4.9462365591397855e-05, + "loss": 2.3666, + "theoretical_loss": 3.369381179191796, + "tokens_seen": 2492006400 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1367729, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2709412574768066, + "objective/train/theoretical_loss": 3.3693670344467033, + "objective/train/tokens_used": 862596576, + "theoretical_loss": 3.3693670344467033, + "tokens_seen": 2492137472 + }, + { + "epoch": 0.51, + "learning_rate": 4.9454341197239615e-05, + "loss": 2.4568, + "theoretical_loss": 3.3693670344467033, + "tokens_seen": 2492137472 + }, + { + "epoch": 0.51, + "learning_rate": 4.9446316803081375e-05, + "loss": 2.6108, + "theoretical_loss": 3.3693528906538113, + "tokens_seen": 2492268544 + }, + { + "epoch": 0.51, + "learning_rate": 4.943829240892313e-05, + "loss": 2.7159, + "theoretical_loss": 3.3693387478130057, + "tokens_seen": 2492399616 + }, + { + "epoch": 0.51, + "learning_rate": 4.943026801476489e-05, + "loss": 2.3573, + "theoretical_loss": 3.3693246059241724, + "tokens_seen": 2492530688 + }, + { + "epoch": 0.51, + "learning_rate": 4.942224362060665e-05, + "loss": 2.4629, + "theoretical_loss": 3.3693104649871968, + "tokens_seen": 2492661760 + }, + { + "epoch": 0.51, + "learning_rate": 4.941421922644841e-05, + "loss": 2.4896, + "theoretical_loss": 3.369296325001965, + "tokens_seen": 2492792832 + }, + { + "epoch": 0.51, + "learning_rate": 4.940619483229017e-05, + "loss": 2.6108, + "theoretical_loss": 3.369282185968364, + "tokens_seen": 2492923904 + }, + { + "epoch": 0.51, + "learning_rate": 4.939817043813193e-05, + "loss": 2.4589, + "theoretical_loss": 3.369268047886278, + "tokens_seen": 2493054976 + }, + { + "epoch": 0.51, + "learning_rate": 4.939014604397368e-05, + "loss": 2.439, + "theoretical_loss": 3.3692539107555937, + "tokens_seen": 2493186048 + }, + { + "epoch": 0.51, + "learning_rate": 4.938212164981544e-05, + "loss": 2.7177, + "theoretical_loss": 3.3692397745761973, + "tokens_seen": 2493317120 + }, + { + "epoch": 0.51, + "learning_rate": 4.93740972556572e-05, + "loss": 2.6196, + "theoretical_loss": 3.3692256393479747, + "tokens_seen": 2493448192 + }, + { + "epoch": 0.51, + "learning_rate": 4.936607286149896e-05, + "loss": 2.5573, + "theoretical_loss": 3.369211505070811, + "tokens_seen": 2493579264 + }, + { + "epoch": 0.51, + "learning_rate": 4.935804846734072e-05, + "loss": 2.6251, + "theoretical_loss": 3.3691973717445935, + "tokens_seen": 2493710336 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1368772, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5175697803497314, + "objective/train/theoretical_loss": 3.369190305438054, + "objective/train/tokens_used": 864234976, + "theoretical_loss": 3.369190305438054, + "tokens_seen": 2493775872 + }, + { + "epoch": 0.51, + "learning_rate": 4.935002407318248e-05, + "loss": 2.5824, + "theoretical_loss": 3.369183239369208, + "tokens_seen": 2493841408 + }, + { + "epoch": 0.51, + "learning_rate": 4.934199967902423e-05, + "loss": 2.4809, + "theoretical_loss": 3.36916910794454, + "tokens_seen": 2493972480 + }, + { + "epoch": 0.51, + "learning_rate": 4.933397528486599e-05, + "loss": 2.5429, + "theoretical_loss": 3.369154977470476, + "tokens_seen": 2494103552 + }, + { + "epoch": 0.51, + "learning_rate": 4.932595089070775e-05, + "loss": 2.5156, + "theoretical_loss": 3.369140847946902, + "tokens_seen": 2494234624 + }, + { + "epoch": 0.51, + "learning_rate": 4.931792649654951e-05, + "loss": 2.5766, + "theoretical_loss": 3.369126719373704, + "tokens_seen": 2494365696 + }, + { + "epoch": 0.51, + "learning_rate": 4.930990210239127e-05, + "loss": 2.4916, + "theoretical_loss": 3.369112591750768, + "tokens_seen": 2494496768 + }, + { + "epoch": 0.51, + "learning_rate": 4.930187770823303e-05, + "loss": 2.6154, + "theoretical_loss": 3.3690984650779807, + "tokens_seen": 2494627840 + }, + { + "epoch": 0.51, + "learning_rate": 4.9293853314074786e-05, + "loss": 2.4216, + "theoretical_loss": 3.3690843393552274, + "tokens_seen": 2494758912 + }, + { + "epoch": 0.51, + "learning_rate": 4.9285828919916546e-05, + "loss": 2.5462, + "theoretical_loss": 3.3690702145823956, + "tokens_seen": 2494889984 + }, + { + "epoch": 0.51, + "learning_rate": 4.9277804525758306e-05, + "loss": 2.5539, + "theoretical_loss": 3.3690560907593707, + "tokens_seen": 2495021056 + }, + { + "epoch": 0.51, + "learning_rate": 4.9269780131600066e-05, + "loss": 2.6297, + "theoretical_loss": 3.369041967886039, + "tokens_seen": 2495152128 + }, + { + "epoch": 0.51, + "learning_rate": 4.9261755737441826e-05, + "loss": 2.3965, + "theoretical_loss": 3.3690278459622864, + "tokens_seen": 2495283200 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1369331, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3660659790039062, + "objective/train/theoretical_loss": 3.369013724988, + "objective/train/tokens_used": 865873376, + "theoretical_loss": 3.369013724988, + "tokens_seen": 2495414272 + }, + { + "epoch": 0.51, + "learning_rate": 4.9253731343283586e-05, + "loss": 2.6804, + "theoretical_loss": 3.369013724988, + "tokens_seen": 2495414272 + }, + { + "epoch": 0.51, + "learning_rate": 4.924570694912534e-05, + "loss": 2.5133, + "theoretical_loss": 3.368999604963066, + "tokens_seen": 2495545344 + }, + { + "epoch": 0.51, + "learning_rate": 4.92376825549671e-05, + "loss": 2.7273, + "theoretical_loss": 3.36898548588737, + "tokens_seen": 2495676416 + }, + { + "epoch": 0.51, + "learning_rate": 4.922965816080886e-05, + "loss": 2.5223, + "theoretical_loss": 3.3689713677607984, + "tokens_seen": 2495807488 + }, + { + "epoch": 0.51, + "learning_rate": 4.922163376665062e-05, + "loss": 2.6208, + "theoretical_loss": 3.368957250583238, + "tokens_seen": 2495938560 + }, + { + "epoch": 0.51, + "learning_rate": 4.921360937249238e-05, + "loss": 2.5849, + "theoretical_loss": 3.3689431343545753, + "tokens_seen": 2496069632 + }, + { + "epoch": 0.51, + "learning_rate": 4.920558497833414e-05, + "loss": 2.4412, + "theoretical_loss": 3.3689290190746966, + "tokens_seen": 2496200704 + }, + { + "epoch": 0.51, + "learning_rate": 4.919756058417589e-05, + "loss": 2.5758, + "theoretical_loss": 3.368914904743488, + "tokens_seen": 2496331776 + }, + { + "epoch": 0.51, + "learning_rate": 4.918953619001765e-05, + "loss": 2.4951, + "theoretical_loss": 3.368900791360836, + "tokens_seen": 2496462848 + }, + { + "epoch": 0.51, + "learning_rate": 4.918151179585941e-05, + "loss": 2.608, + "theoretical_loss": 3.368886678926627, + "tokens_seen": 2496593920 + }, + { + "epoch": 0.51, + "learning_rate": 4.917348740170117e-05, + "loss": 2.4136, + "theoretical_loss": 3.3688725674407483, + "tokens_seen": 2496724992 + }, + { + "epoch": 0.51, + "learning_rate": 4.916546300754293e-05, + "loss": 2.5014, + "theoretical_loss": 3.3688584569030855, + "tokens_seen": 2496856064 + }, + { + "epoch": 0.51, + "learning_rate": 4.915743861338469e-05, + "loss": 2.3839, + "theoretical_loss": 3.368844347313525, + "tokens_seen": 2496987136 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1369989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0298242568969727, + "objective/train/theoretical_loss": 3.3688372928742476, + "objective/train/tokens_used": 867511776, + "theoretical_loss": 3.3688372928742476, + "tokens_seen": 2497052672 + }, + { + "epoch": 0.51, + "learning_rate": 4.914941421922645e-05, + "loss": 2.5819, + "theoretical_loss": 3.3688302386719537, + "tokens_seen": 2497118208 + }, + { + "epoch": 0.51, + "learning_rate": 4.9141389825068204e-05, + "loss": 2.5843, + "theoretical_loss": 3.3688161309782583, + "tokens_seen": 2497249280 + }, + { + "epoch": 0.51, + "learning_rate": 4.9133365430909964e-05, + "loss": 2.507, + "theoretical_loss": 3.368802024232325, + "tokens_seen": 2497380352 + }, + { + "epoch": 0.51, + "learning_rate": 4.9125341036751724e-05, + "loss": 2.5364, + "theoretical_loss": 3.368787918434041, + "tokens_seen": 2497511424 + }, + { + "epoch": 0.51, + "learning_rate": 4.9117316642593484e-05, + "loss": 2.3699, + "theoretical_loss": 3.3687738135832923, + "tokens_seen": 2497642496 + }, + { + "epoch": 0.51, + "learning_rate": 4.9109292248435244e-05, + "loss": 2.4503, + "theoretical_loss": 3.368759709679966, + "tokens_seen": 2497773568 + }, + { + "epoch": 0.51, + "learning_rate": 4.9101267854277003e-05, + "loss": 2.5373, + "theoretical_loss": 3.368745606723948, + "tokens_seen": 2497904640 + }, + { + "epoch": 0.51, + "learning_rate": 4.909324346011876e-05, + "loss": 2.7115, + "theoretical_loss": 3.368731504715126, + "tokens_seen": 2498035712 + }, + { + "epoch": 0.51, + "learning_rate": 4.908521906596052e-05, + "loss": 2.6725, + "theoretical_loss": 3.3687174036533856, + "tokens_seen": 2498166784 + }, + { + "epoch": 0.51, + "learning_rate": 4.9077194671802276e-05, + "loss": 2.5665, + "theoretical_loss": 3.3687033035386147, + "tokens_seen": 2498297856 + }, + { + "epoch": 0.51, + "learning_rate": 4.9069170277644036e-05, + "loss": 2.4985, + "theoretical_loss": 3.368689204370699, + "tokens_seen": 2498428928 + }, + { + "epoch": 0.51, + "learning_rate": 4.9061145883485796e-05, + "loss": 2.5538, + "theoretical_loss": 3.3686751061495253, + "tokens_seen": 2498560000 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1371286, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.572655200958252, + "objective/train/theoretical_loss": 3.368661008874981, + "objective/train/tokens_used": 869150176, + "theoretical_loss": 3.368661008874981, + "tokens_seen": 2498691072 + }, + { + "epoch": 0.51, + "learning_rate": 4.9053121489327556e-05, + "loss": 2.5129, + "theoretical_loss": 3.368661008874981, + "tokens_seen": 2498691072 + }, + { + "epoch": 0.51, + "learning_rate": 4.9045097095169316e-05, + "loss": 2.5709, + "theoretical_loss": 3.3686469125469527, + "tokens_seen": 2498822144 + }, + { + "epoch": 0.51, + "learning_rate": 4.9037072701011076e-05, + "loss": 2.6015, + "theoretical_loss": 3.3686328171653273, + "tokens_seen": 2498953216 + }, + { + "epoch": 0.51, + "learning_rate": 4.9029048306852836e-05, + "loss": 2.4381, + "theoretical_loss": 3.368618722729991, + "tokens_seen": 2499084288 + }, + { + "epoch": 0.51, + "learning_rate": 4.902102391269459e-05, + "loss": 2.3638, + "theoretical_loss": 3.3686046292408314, + "tokens_seen": 2499215360 + }, + { + "epoch": 0.51, + "learning_rate": 4.901299951853635e-05, + "loss": 2.5115, + "theoretical_loss": 3.368590536697735, + "tokens_seen": 2499346432 + }, + { + "epoch": 0.51, + "learning_rate": 4.900497512437811e-05, + "loss": 2.5566, + "theoretical_loss": 3.3685764451005884, + "tokens_seen": 2499477504 + }, + { + "epoch": 0.51, + "learning_rate": 4.899695073021987e-05, + "loss": 2.5392, + "theoretical_loss": 3.368562354449279, + "tokens_seen": 2499608576 + }, + { + "epoch": 0.52, + "learning_rate": 4.898892633606163e-05, + "loss": 2.4501, + "theoretical_loss": 3.3685482647436937, + "tokens_seen": 2499739648 + }, + { + "epoch": 0.52, + "learning_rate": 4.898090194190339e-05, + "loss": 2.6578, + "theoretical_loss": 3.368534175983719, + "tokens_seen": 2499870720 + }, + { + "epoch": 0.52, + "learning_rate": 4.897287754774515e-05, + "loss": 2.5015, + "theoretical_loss": 3.3685200881692423, + "tokens_seen": 2500001792 + }, + { + "epoch": 0.52, + "learning_rate": 4.896485315358691e-05, + "loss": 2.5423, + "theoretical_loss": 3.36850600130015, + "tokens_seen": 2500132864 + }, + { + "epoch": 0.52, + "learning_rate": 4.895682875942866e-05, + "loss": 2.6089, + "theoretical_loss": 3.3684919153763304, + "tokens_seen": 2500263936 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1371802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.660773277282715, + "objective/train/theoretical_loss": 3.368484872768862, + "objective/train/tokens_used": 870788576, + "theoretical_loss": 3.368484872768862, + "tokens_seen": 2500329472 + }, + { + "epoch": 0.52, + "learning_rate": 4.894880436527042e-05, + "loss": 2.5976, + "theoretical_loss": 3.3684778303976692, + "tokens_seen": 2500395008 + }, + { + "epoch": 0.52, + "learning_rate": 4.894077997111218e-05, + "loss": 2.5346, + "theoretical_loss": 3.3684637463640534, + "tokens_seen": 2500526080 + }, + { + "epoch": 0.52, + "learning_rate": 4.893275557695394e-05, + "loss": 2.532, + "theoretical_loss": 3.3684496632753715, + "tokens_seen": 2500657152 + }, + { + "epoch": 0.52, + "learning_rate": 4.89247311827957e-05, + "loss": 2.6926, + "theoretical_loss": 3.368435581131509, + "tokens_seen": 2500788224 + }, + { + "epoch": 0.52, + "learning_rate": 4.891670678863746e-05, + "loss": 2.7204, + "theoretical_loss": 3.368421499932354, + "tokens_seen": 2500919296 + }, + { + "epoch": 0.52, + "learning_rate": 4.890868239447922e-05, + "loss": 2.5059, + "theoretical_loss": 3.3684074196777924, + "tokens_seen": 2501050368 + }, + { + "epoch": 0.52, + "learning_rate": 4.8900658000320974e-05, + "loss": 2.6287, + "theoretical_loss": 3.368393340367713, + "tokens_seen": 2501181440 + }, + { + "epoch": 0.52, + "learning_rate": 4.8892633606162734e-05, + "loss": 2.6112, + "theoretical_loss": 3.368379262002002, + "tokens_seen": 2501312512 + }, + { + "epoch": 0.52, + "learning_rate": 4.8884609212004494e-05, + "loss": 2.6782, + "theoretical_loss": 3.3683651845805467, + "tokens_seen": 2501443584 + }, + { + "epoch": 0.52, + "learning_rate": 4.8876584817846254e-05, + "loss": 2.6049, + "theoretical_loss": 3.368351108103234, + "tokens_seen": 2501574656 + }, + { + "epoch": 0.52, + "learning_rate": 4.8868560423688014e-05, + "loss": 2.5635, + "theoretical_loss": 3.3683370325699515, + "tokens_seen": 2501705728 + }, + { + "epoch": 0.52, + "learning_rate": 4.8860536029529774e-05, + "loss": 2.571, + "theoretical_loss": 3.3683229579805865, + "tokens_seen": 2501836800 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1373004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.812018871307373, + "objective/train/theoretical_loss": 3.3683088843350264, + "objective/train/tokens_used": 872426976, + "theoretical_loss": 3.3683088843350264, + "tokens_seen": 2501967872 + }, + { + "epoch": 0.52, + "learning_rate": 4.8852511635371533e-05, + "loss": 2.5624, + "theoretical_loss": 3.3683088843350264, + "tokens_seen": 2501967872 + }, + { + "epoch": 0.52, + "learning_rate": 4.8844487241213287e-05, + "loss": 2.5664, + "theoretical_loss": 3.3682948116331577, + "tokens_seen": 2502098944 + }, + { + "epoch": 0.52, + "learning_rate": 4.8836462847055046e-05, + "loss": 2.5155, + "theoretical_loss": 3.3682807398748684, + "tokens_seen": 2502230016 + }, + { + "epoch": 0.52, + "learning_rate": 4.8828438452896806e-05, + "loss": 2.6867, + "theoretical_loss": 3.3682666690600453, + "tokens_seen": 2502361088 + }, + { + "epoch": 0.52, + "learning_rate": 4.8820414058738566e-05, + "loss": 2.6165, + "theoretical_loss": 3.368252599188576, + "tokens_seen": 2502492160 + }, + { + "epoch": 0.52, + "learning_rate": 4.8812389664580326e-05, + "loss": 2.638, + "theoretical_loss": 3.3682385302603484, + "tokens_seen": 2502623232 + }, + { + "epoch": 0.52, + "learning_rate": 4.8804365270422086e-05, + "loss": 2.5934, + "theoretical_loss": 3.368224462275249, + "tokens_seen": 2502754304 + }, + { + "epoch": 0.52, + "learning_rate": 4.8796340876263846e-05, + "loss": 2.518, + "theoretical_loss": 3.3682103952331652, + "tokens_seen": 2502885376 + }, + { + "epoch": 0.52, + "learning_rate": 4.8788316482105606e-05, + "loss": 2.4324, + "theoretical_loss": 3.368196329133985, + "tokens_seen": 2503016448 + }, + { + "epoch": 0.52, + "learning_rate": 4.878029208794736e-05, + "loss": 2.5402, + "theoretical_loss": 3.368182263977596, + "tokens_seen": 2503147520 + }, + { + "epoch": 0.52, + "learning_rate": 4.877226769378912e-05, + "loss": 2.557, + "theoretical_loss": 3.3681681997638844, + "tokens_seen": 2503278592 + }, + { + "epoch": 0.52, + "learning_rate": 4.876424329963088e-05, + "loss": 2.4608, + "theoretical_loss": 3.368154136492739, + "tokens_seen": 2503409664 + }, + { + "epoch": 0.52, + "learning_rate": 4.875621890547264e-05, + "loss": 2.6075, + "theoretical_loss": 3.368140074164047, + "tokens_seen": 2503540736 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1374296, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.103992462158203, + "objective/train/theoretical_loss": 3.3681330433530854, + "objective/train/tokens_used": 874065376, + "theoretical_loss": 3.3681330433530854, + "tokens_seen": 2503606272 + }, + { + "epoch": 0.52, + "learning_rate": 4.87481945113144e-05, + "loss": 2.4213, + "theoretical_loss": 3.3681260127776946, + "tokens_seen": 2503671808 + }, + { + "epoch": 0.52, + "learning_rate": 4.874017011715616e-05, + "loss": 2.4916, + "theoretical_loss": 3.3681119523335714, + "tokens_seen": 2503802880 + }, + { + "epoch": 0.52, + "learning_rate": 4.873214572299792e-05, + "loss": 2.5939, + "theoretical_loss": 3.3680978928315635, + "tokens_seen": 2503933952 + }, + { + "epoch": 0.52, + "learning_rate": 4.872412132883967e-05, + "loss": 2.573, + "theoretical_loss": 3.368083834271559, + "tokens_seen": 2504065024 + }, + { + "epoch": 0.52, + "learning_rate": 4.871609693468143e-05, + "loss": 2.6522, + "theoretical_loss": 3.3680697766534453, + "tokens_seen": 2504196096 + }, + { + "epoch": 0.52, + "learning_rate": 4.870807254052319e-05, + "loss": 2.5498, + "theoretical_loss": 3.36805571997711, + "tokens_seen": 2504327168 + }, + { + "epoch": 0.52, + "learning_rate": 4.870004814636495e-05, + "loss": 2.4721, + "theoretical_loss": 3.368041664242441, + "tokens_seen": 2504458240 + }, + { + "epoch": 0.52, + "learning_rate": 4.869202375220671e-05, + "loss": 2.4782, + "theoretical_loss": 3.3680276094493253, + "tokens_seen": 2504589312 + }, + { + "epoch": 0.52, + "learning_rate": 4.868399935804847e-05, + "loss": 2.6176, + "theoretical_loss": 3.3680135555976514, + "tokens_seen": 2504720384 + }, + { + "epoch": 0.52, + "learning_rate": 4.867597496389023e-05, + "loss": 2.464, + "theoretical_loss": 3.367999502687306, + "tokens_seen": 2504851456 + }, + { + "epoch": 0.52, + "learning_rate": 4.8667950569731984e-05, + "loss": 2.5257, + "theoretical_loss": 3.367985450718178, + "tokens_seen": 2504982528 + }, + { + "epoch": 0.52, + "learning_rate": 4.8659926175573744e-05, + "loss": 2.6085, + "theoretical_loss": 3.367971399690154, + "tokens_seen": 2505113600 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1374892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3439693450927734, + "objective/train/theoretical_loss": 3.367957349603123, + "objective/train/tokens_used": 875703776, + "theoretical_loss": 3.367957349603123, + "tokens_seen": 2505244672 + }, + { + "epoch": 0.52, + "learning_rate": 4.8651901781415504e-05, + "loss": 2.5946, + "theoretical_loss": 3.367957349603123, + "tokens_seen": 2505244672 + }, + { + "epoch": 0.52, + "learning_rate": 4.8643877387257264e-05, + "loss": 2.6487, + "theoretical_loss": 3.367943300456971, + "tokens_seen": 2505375744 + }, + { + "epoch": 0.52, + "learning_rate": 4.8635852993099024e-05, + "loss": 2.5105, + "theoretical_loss": 3.3679292522515873, + "tokens_seen": 2505506816 + }, + { + "epoch": 0.52, + "learning_rate": 4.8627828598940784e-05, + "loss": 2.5514, + "theoretical_loss": 3.367915204986859, + "tokens_seen": 2505637888 + }, + { + "epoch": 0.52, + "learning_rate": 4.8619804204782544e-05, + "loss": 2.5829, + "theoretical_loss": 3.367901158662674, + "tokens_seen": 2505768960 + }, + { + "epoch": 0.52, + "learning_rate": 4.8611779810624304e-05, + "loss": 2.3615, + "theoretical_loss": 3.36788711327892, + "tokens_seen": 2505900032 + }, + { + "epoch": 0.52, + "learning_rate": 4.860375541646606e-05, + "loss": 2.5354, + "theoretical_loss": 3.367873068835485, + "tokens_seen": 2506031104 + }, + { + "epoch": 0.52, + "learning_rate": 4.8595731022307817e-05, + "loss": 2.4848, + "theoretical_loss": 3.367859025332258, + "tokens_seen": 2506162176 + }, + { + "epoch": 0.52, + "learning_rate": 4.8587706628149576e-05, + "loss": 2.5954, + "theoretical_loss": 3.3678449827691246, + "tokens_seen": 2506293248 + }, + { + "epoch": 0.52, + "learning_rate": 4.8579682233991336e-05, + "loss": 2.5928, + "theoretical_loss": 3.367830941145974, + "tokens_seen": 2506424320 + }, + { + "epoch": 0.52, + "learning_rate": 4.8571657839833096e-05, + "loss": 2.5398, + "theoretical_loss": 3.3678169004626946, + "tokens_seen": 2506555392 + }, + { + "epoch": 0.52, + "learning_rate": 4.8563633445674856e-05, + "loss": 2.5305, + "theoretical_loss": 3.3678028607191735, + "tokens_seen": 2506686464 + }, + { + "epoch": 0.52, + "learning_rate": 4.8555609051516616e-05, + "loss": 2.6508, + "theoretical_loss": 3.3677888219152985, + "tokens_seen": 2506817536 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1376192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.142772674560547, + "objective/train/theoretical_loss": 3.367781802865694, + "objective/train/tokens_used": 877342176, + "theoretical_loss": 3.367781802865694, + "tokens_seen": 2506883072 + }, + { + "epoch": 0.52, + "learning_rate": 4.854758465735837e-05, + "loss": 2.5421, + "theoretical_loss": 3.3677747840509586, + "tokens_seen": 2506948608 + }, + { + "epoch": 0.52, + "learning_rate": 4.853956026320013e-05, + "loss": 2.517, + "theoretical_loss": 3.367760747126041, + "tokens_seen": 2507079680 + }, + { + "epoch": 0.52, + "learning_rate": 4.853153586904189e-05, + "loss": 2.48, + "theoretical_loss": 3.367746711140434, + "tokens_seen": 2507210752 + }, + { + "epoch": 0.52, + "learning_rate": 4.852351147488365e-05, + "loss": 2.5435, + "theoretical_loss": 3.367732676094026, + "tokens_seen": 2507341824 + }, + { + "epoch": 0.52, + "learning_rate": 4.851548708072541e-05, + "loss": 2.43, + "theoretical_loss": 3.3677186419867042, + "tokens_seen": 2507472896 + }, + { + "epoch": 0.52, + "learning_rate": 4.850746268656717e-05, + "loss": 2.4617, + "theoretical_loss": 3.3677046088183573, + "tokens_seen": 2507603968 + }, + { + "epoch": 0.52, + "learning_rate": 4.849943829240893e-05, + "loss": 2.5015, + "theoretical_loss": 3.367690576588873, + "tokens_seen": 2507735040 + }, + { + "epoch": 0.52, + "learning_rate": 4.849141389825069e-05, + "loss": 2.4922, + "theoretical_loss": 3.36767654529814, + "tokens_seen": 2507866112 + }, + { + "epoch": 0.52, + "learning_rate": 4.848338950409244e-05, + "loss": 2.5681, + "theoretical_loss": 3.367662514946046, + "tokens_seen": 2507997184 + }, + { + "epoch": 0.52, + "learning_rate": 4.84753651099342e-05, + "loss": 2.6439, + "theoretical_loss": 3.367648485532479, + "tokens_seen": 2508128256 + }, + { + "epoch": 0.52, + "learning_rate": 4.846734071577596e-05, + "loss": 2.6526, + "theoretical_loss": 3.367634457057328, + "tokens_seen": 2508259328 + }, + { + "epoch": 0.52, + "learning_rate": 4.845931632161772e-05, + "loss": 2.5571, + "theoretical_loss": 3.36762042952048, + "tokens_seen": 2508390400 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1376516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5870392322540283, + "objective/train/theoretical_loss": 3.3676064029218242, + "objective/train/tokens_used": 878980576, + "theoretical_loss": 3.3676064029218242, + "tokens_seen": 2508521472 + }, + { + "epoch": 0.52, + "learning_rate": 4.845129192745948e-05, + "loss": 2.6055, + "theoretical_loss": 3.3676064029218242, + "tokens_seen": 2508521472 + }, + { + "epoch": 0.52, + "learning_rate": 4.844326753330124e-05, + "loss": 2.5119, + "theoretical_loss": 3.3675923772612486, + "tokens_seen": 2508652544 + }, + { + "epoch": 0.52, + "learning_rate": 4.8435243139143e-05, + "loss": 2.5264, + "theoretical_loss": 3.3675783525386414, + "tokens_seen": 2508783616 + }, + { + "epoch": 0.52, + "learning_rate": 4.8427218744984754e-05, + "loss": 2.5302, + "theoretical_loss": 3.3675643287538906, + "tokens_seen": 2508914688 + }, + { + "epoch": 0.52, + "learning_rate": 4.8419194350826514e-05, + "loss": 2.72, + "theoretical_loss": 3.3675503059068848, + "tokens_seen": 2509045760 + }, + { + "epoch": 0.52, + "learning_rate": 4.8411169956668274e-05, + "loss": 2.4637, + "theoretical_loss": 3.367536283997512, + "tokens_seen": 2509176832 + }, + { + "epoch": 0.52, + "learning_rate": 4.8403145562510034e-05, + "loss": 2.571, + "theoretical_loss": 3.367522263025661, + "tokens_seen": 2509307904 + }, + { + "epoch": 0.52, + "learning_rate": 4.8395121168351794e-05, + "loss": 2.4807, + "theoretical_loss": 3.36750824299122, + "tokens_seen": 2509438976 + }, + { + "epoch": 0.52, + "learning_rate": 4.8387096774193554e-05, + "loss": 2.5401, + "theoretical_loss": 3.3674942238940773, + "tokens_seen": 2509570048 + }, + { + "epoch": 0.52, + "learning_rate": 4.8379072380035314e-05, + "loss": 2.4567, + "theoretical_loss": 3.367480205734121, + "tokens_seen": 2509701120 + }, + { + "epoch": 0.52, + "learning_rate": 4.837104798587707e-05, + "loss": 2.6387, + "theoretical_loss": 3.36746618851124, + "tokens_seen": 2509832192 + }, + { + "epoch": 0.52, + "learning_rate": 4.836302359171883e-05, + "loss": 2.6431, + "theoretical_loss": 3.367452172225322, + "tokens_seen": 2509963264 + }, + { + "epoch": 0.52, + "learning_rate": 4.835499919756059e-05, + "loss": 2.6223, + "theoretical_loss": 3.367438156876257, + "tokens_seen": 2510094336 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1376516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8122689723968506, + "objective/train/theoretical_loss": 3.3674311495530085, + "objective/train/tokens_used": 880618976, + "theoretical_loss": 3.3674311495530085, + "tokens_seen": 2510159872 + }, + { + "epoch": 0.52, + "learning_rate": 4.8346974803402347e-05, + "loss": 2.5765, + "theoretical_loss": 3.3674241424639315, + "tokens_seen": 2510225408 + }, + { + "epoch": 0.52, + "learning_rate": 4.8338950409244106e-05, + "loss": 2.6367, + "theoretical_loss": 3.367410128988235, + "tokens_seen": 2510356480 + }, + { + "epoch": 0.52, + "learning_rate": 4.8330926015085866e-05, + "loss": 2.6433, + "theoretical_loss": 3.367396116449056, + "tokens_seen": 2510487552 + }, + { + "epoch": 0.52, + "learning_rate": 4.8322901620927626e-05, + "loss": 2.6982, + "theoretical_loss": 3.3673821048462833, + "tokens_seen": 2510618624 + }, + { + "epoch": 0.52, + "learning_rate": 4.8314877226769386e-05, + "loss": 2.7463, + "theoretical_loss": 3.3673680941798048, + "tokens_seen": 2510749696 + }, + { + "epoch": 0.52, + "learning_rate": 4.830685283261114e-05, + "loss": 2.6165, + "theoretical_loss": 3.3673540844495093, + "tokens_seen": 2510880768 + }, + { + "epoch": 0.52, + "learning_rate": 4.82988284384529e-05, + "loss": 2.7569, + "theoretical_loss": 3.367340075655285, + "tokens_seen": 2511011840 + }, + { + "epoch": 0.52, + "learning_rate": 4.829080404429466e-05, + "loss": 2.5475, + "theoretical_loss": 3.3673260677970216, + "tokens_seen": 2511142912 + }, + { + "epoch": 0.52, + "learning_rate": 4.828277965013642e-05, + "loss": 2.746, + "theoretical_loss": 3.3673120608746068, + "tokens_seen": 2511273984 + }, + { + "epoch": 0.52, + "learning_rate": 4.827475525597818e-05, + "loss": 2.7353, + "theoretical_loss": 3.367298054887929, + "tokens_seen": 2511405056 + }, + { + "epoch": 0.52, + "learning_rate": 4.826673086181994e-05, + "loss": 2.8524, + "theoretical_loss": 3.367284049836878, + "tokens_seen": 2511536128 + }, + { + "epoch": 0.52, + "learning_rate": 4.82587064676617e-05, + "loss": 2.5463, + "theoretical_loss": 3.3672700457213414, + "tokens_seen": 2511667200 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1377425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6701934337615967, + "objective/train/theoretical_loss": 3.367256042541208, + "objective/train/tokens_used": 882257376, + "theoretical_loss": 3.367256042541208, + "tokens_seen": 2511798272 + }, + { + "epoch": 0.52, + "learning_rate": 4.825068207350345e-05, + "loss": 2.6103, + "theoretical_loss": 3.367256042541208, + "tokens_seen": 2511798272 + }, + { + "epoch": 0.52, + "learning_rate": 4.824265767934521e-05, + "loss": 2.4983, + "theoretical_loss": 3.3672420402963676, + "tokens_seen": 2511929344 + }, + { + "epoch": 0.52, + "learning_rate": 4.823463328518697e-05, + "loss": 2.7509, + "theoretical_loss": 3.3672280389867075, + "tokens_seen": 2512060416 + }, + { + "epoch": 0.52, + "learning_rate": 4.822660889102873e-05, + "loss": 2.8019, + "theoretical_loss": 3.3672140386121177, + "tokens_seen": 2512191488 + }, + { + "epoch": 0.52, + "learning_rate": 4.821858449687049e-05, + "loss": 2.614, + "theoretical_loss": 3.3672000391724857, + "tokens_seen": 2512322560 + }, + { + "epoch": 0.52, + "learning_rate": 4.821056010271225e-05, + "loss": 2.6892, + "theoretical_loss": 3.367186040667701, + "tokens_seen": 2512453632 + }, + { + "epoch": 0.52, + "learning_rate": 4.820253570855401e-05, + "loss": 2.5277, + "theoretical_loss": 3.3671720430976526, + "tokens_seen": 2512584704 + }, + { + "epoch": 0.52, + "learning_rate": 4.8194511314395764e-05, + "loss": 2.757, + "theoretical_loss": 3.367158046462229, + "tokens_seen": 2512715776 + }, + { + "epoch": 0.52, + "learning_rate": 4.8186486920237524e-05, + "loss": 2.6371, + "theoretical_loss": 3.367144050761319, + "tokens_seen": 2512846848 + }, + { + "epoch": 0.52, + "learning_rate": 4.8178462526079284e-05, + "loss": 2.6286, + "theoretical_loss": 3.367130055994812, + "tokens_seen": 2512977920 + }, + { + "epoch": 0.52, + "learning_rate": 4.8170438131921044e-05, + "loss": 2.6977, + "theoretical_loss": 3.3671160621625957, + "tokens_seen": 2513108992 + }, + { + "epoch": 0.52, + "learning_rate": 4.8162413737762804e-05, + "loss": 2.5703, + "theoretical_loss": 3.3671020692645603, + "tokens_seen": 2513240064 + }, + { + "epoch": 0.52, + "learning_rate": 4.8154389343604564e-05, + "loss": 2.4807, + "theoretical_loss": 3.3670880773005942, + "tokens_seen": 2513371136 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1378070, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3798375129699707, + "objective/train/theoretical_loss": 3.3670810816688523, + "objective/train/tokens_used": 883895776, + "theoretical_loss": 3.3670810816688523, + "tokens_seen": 2513436672 + }, + { + "epoch": 0.52, + "learning_rate": 4.8146364949446324e-05, + "loss": 2.6042, + "theoretical_loss": 3.367074086270586, + "tokens_seen": 2513502208 + }, + { + "epoch": 0.52, + "learning_rate": 4.8138340555288084e-05, + "loss": 2.4991, + "theoretical_loss": 3.3670600961744253, + "tokens_seen": 2513633280 + }, + { + "epoch": 0.52, + "learning_rate": 4.813031616112984e-05, + "loss": 2.6307, + "theoretical_loss": 3.367046107012001, + "tokens_seen": 2513764352 + }, + { + "epoch": 0.52, + "learning_rate": 4.81222917669716e-05, + "loss": 2.5312, + "theoretical_loss": 3.3670321187832015, + "tokens_seen": 2513895424 + }, + { + "epoch": 0.52, + "learning_rate": 4.811426737281336e-05, + "loss": 2.5389, + "theoretical_loss": 3.3670181314879164, + "tokens_seen": 2514026496 + }, + { + "epoch": 0.52, + "learning_rate": 4.8106242978655117e-05, + "loss": 2.6553, + "theoretical_loss": 3.3670041451260344, + "tokens_seen": 2514157568 + }, + { + "epoch": 0.52, + "learning_rate": 4.8098218584496877e-05, + "loss": 2.5299, + "theoretical_loss": 3.3669901596974445, + "tokens_seen": 2514288640 + }, + { + "epoch": 0.52, + "learning_rate": 4.8090194190338636e-05, + "loss": 2.5993, + "theoretical_loss": 3.366976175202036, + "tokens_seen": 2514419712 + }, + { + "epoch": 0.52, + "learning_rate": 4.8082169796180396e-05, + "loss": 2.5919, + "theoretical_loss": 3.3669621916396983, + "tokens_seen": 2514550784 + }, + { + "epoch": 0.52, + "learning_rate": 4.807414540202215e-05, + "loss": 2.6763, + "theoretical_loss": 3.36694820901032, + "tokens_seen": 2514681856 + }, + { + "epoch": 0.52, + "learning_rate": 4.806612100786391e-05, + "loss": 2.5359, + "theoretical_loss": 3.36693422731379, + "tokens_seen": 2514812928 + }, + { + "epoch": 0.52, + "learning_rate": 4.805809661370567e-05, + "loss": 2.6382, + "theoretical_loss": 3.3669202465499986, + "tokens_seen": 2514944000 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1379318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.843836545944214, + "objective/train/theoretical_loss": 3.366906266718834, + "objective/train/tokens_used": 885534176, + "theoretical_loss": 3.366906266718834, + "tokens_seen": 2515075072 + }, + { + "epoch": 0.52, + "learning_rate": 4.805007221954743e-05, + "loss": 2.7247, + "theoretical_loss": 3.366906266718834, + "tokens_seen": 2515075072 + }, + { + "epoch": 0.52, + "learning_rate": 4.804204782538919e-05, + "loss": 2.6126, + "theoretical_loss": 3.366892287820185, + "tokens_seen": 2515206144 + }, + { + "epoch": 0.52, + "learning_rate": 4.803402343123095e-05, + "loss": 2.5857, + "theoretical_loss": 3.366878309853942, + "tokens_seen": 2515337216 + }, + { + "epoch": 0.52, + "learning_rate": 4.80259990370727e-05, + "loss": 2.6734, + "theoretical_loss": 3.3668643328199934, + "tokens_seen": 2515468288 + }, + { + "epoch": 0.52, + "learning_rate": 4.801797464291446e-05, + "loss": 2.6715, + "theoretical_loss": 3.3668503567182286, + "tokens_seen": 2515599360 + }, + { + "epoch": 0.52, + "learning_rate": 4.800995024875622e-05, + "loss": 2.7025, + "theoretical_loss": 3.366836381548537, + "tokens_seen": 2515730432 + }, + { + "epoch": 0.52, + "learning_rate": 4.800192585459798e-05, + "loss": 2.6618, + "theoretical_loss": 3.3668224073108077, + "tokens_seen": 2515861504 + }, + { + "epoch": 0.52, + "learning_rate": 4.799390146043974e-05, + "loss": 2.6227, + "theoretical_loss": 3.3668084340049305, + "tokens_seen": 2515992576 + }, + { + "epoch": 0.52, + "learning_rate": 4.79858770662815e-05, + "loss": 2.6876, + "theoretical_loss": 3.366794461630794, + "tokens_seen": 2516123648 + }, + { + "epoch": 0.53, + "learning_rate": 4.7977852672123255e-05, + "loss": 2.6487, + "theoretical_loss": 3.366780490188288, + "tokens_seen": 2516254720 + }, + { + "epoch": 0.53, + "learning_rate": 4.7969828277965015e-05, + "loss": 2.6247, + "theoretical_loss": 3.366766519677302, + "tokens_seen": 2516385792 + }, + { + "epoch": 0.53, + "learning_rate": 4.7961803883806775e-05, + "loss": 2.6998, + "theoretical_loss": 3.3667525500977247, + "tokens_seen": 2516516864 + }, + { + "epoch": 0.53, + "learning_rate": 4.7953779489648534e-05, + "loss": 2.6981, + "theoretical_loss": 3.366738581449446, + "tokens_seen": 2516647936 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1379956, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.391172409057617, + "objective/train/theoretical_loss": 3.3667315974745096, + "objective/train/tokens_used": 887172576, + "theoretical_loss": 3.3667315974745096, + "tokens_seen": 2516713472 + }, + { + "epoch": 0.53, + "learning_rate": 4.7945755095490294e-05, + "loss": 2.4727, + "theoretical_loss": 3.3667246137323557, + "tokens_seen": 2516779008 + }, + { + "epoch": 0.53, + "learning_rate": 4.7937730701332054e-05, + "loss": 2.6718, + "theoretical_loss": 3.366710646946342, + "tokens_seen": 2516910080 + }, + { + "epoch": 0.53, + "learning_rate": 4.792970630717381e-05, + "loss": 2.679, + "theoretical_loss": 3.366696681091296, + "tokens_seen": 2517041152 + }, + { + "epoch": 0.53, + "learning_rate": 4.792168191301557e-05, + "loss": 2.6756, + "theoretical_loss": 3.3666827161671056, + "tokens_seen": 2517172224 + }, + { + "epoch": 0.53, + "learning_rate": 4.791365751885733e-05, + "loss": 2.6579, + "theoretical_loss": 3.366668752173661, + "tokens_seen": 2517303296 + }, + { + "epoch": 0.53, + "learning_rate": 4.790563312469909e-05, + "loss": 2.7533, + "theoretical_loss": 3.366654789110852, + "tokens_seen": 2517434368 + }, + { + "epoch": 0.53, + "learning_rate": 4.789760873054085e-05, + "loss": 2.4818, + "theoretical_loss": 3.3666408269785677, + "tokens_seen": 2517565440 + }, + { + "epoch": 0.53, + "learning_rate": 4.78895843363826e-05, + "loss": 2.4124, + "theoretical_loss": 3.3666268657766976, + "tokens_seen": 2517696512 + }, + { + "epoch": 0.53, + "learning_rate": 4.788155994222436e-05, + "loss": 2.6119, + "theoretical_loss": 3.3666129055051317, + "tokens_seen": 2517827584 + }, + { + "epoch": 0.53, + "learning_rate": 4.787353554806612e-05, + "loss": 2.576, + "theoretical_loss": 3.366598946163759, + "tokens_seen": 2517958656 + }, + { + "epoch": 0.53, + "learning_rate": 4.786551115390788e-05, + "loss": 2.5626, + "theoretical_loss": 3.3665849877524696, + "tokens_seen": 2518089728 + }, + { + "epoch": 0.53, + "learning_rate": 4.785748675974964e-05, + "loss": 2.6858, + "theoretical_loss": 3.366571030271153, + "tokens_seen": 2518220800 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1381079, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.684238910675049, + "objective/train/theoretical_loss": 3.366557073719699, + "objective/train/tokens_used": 888810976, + "theoretical_loss": 3.366557073719699, + "tokens_seen": 2518351872 + }, + { + "epoch": 0.53, + "learning_rate": 4.78494623655914e-05, + "loss": 2.6355, + "theoretical_loss": 3.366557073719699, + "tokens_seen": 2518351872 + }, + { + "epoch": 0.53, + "learning_rate": 4.784143797143316e-05, + "loss": 2.6237, + "theoretical_loss": 3.366543118097997, + "tokens_seen": 2518482944 + }, + { + "epoch": 0.53, + "learning_rate": 4.783341357727491e-05, + "loss": 2.4905, + "theoretical_loss": 3.366529163405936, + "tokens_seen": 2518614016 + }, + { + "epoch": 0.53, + "learning_rate": 4.782538918311667e-05, + "loss": 2.5384, + "theoretical_loss": 3.3665152096434072, + "tokens_seen": 2518745088 + }, + { + "epoch": 0.53, + "learning_rate": 4.781736478895843e-05, + "loss": 2.5284, + "theoretical_loss": 3.366501256810299, + "tokens_seen": 2518876160 + }, + { + "epoch": 0.53, + "learning_rate": 4.780934039480019e-05, + "loss": 2.7665, + "theoretical_loss": 3.366487304906502, + "tokens_seen": 2519007232 + }, + { + "epoch": 0.53, + "learning_rate": 4.780131600064195e-05, + "loss": 2.3966, + "theoretical_loss": 3.366473353931905, + "tokens_seen": 2519138304 + }, + { + "epoch": 0.53, + "learning_rate": 4.779329160648371e-05, + "loss": 2.5817, + "theoretical_loss": 3.3664594038863993, + "tokens_seen": 2519269376 + }, + { + "epoch": 0.53, + "learning_rate": 4.778526721232547e-05, + "loss": 2.8254, + "theoretical_loss": 3.3664454547698734, + "tokens_seen": 2519400448 + }, + { + "epoch": 0.53, + "learning_rate": 4.7777242818167225e-05, + "loss": 2.492, + "theoretical_loss": 3.3664315065822175, + "tokens_seen": 2519531520 + }, + { + "epoch": 0.53, + "learning_rate": 4.7769218424008985e-05, + "loss": 2.5963, + "theoretical_loss": 3.3664175593233217, + "tokens_seen": 2519662592 + }, + { + "epoch": 0.53, + "learning_rate": 4.7761194029850745e-05, + "loss": 2.6447, + "theoretical_loss": 3.366403612993075, + "tokens_seen": 2519793664 + }, + { + "epoch": 0.53, + "learning_rate": 4.7753169635692505e-05, + "loss": 2.6689, + "theoretical_loss": 3.3663896675913683, + "tokens_seen": 2519924736 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1381616, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9621894359588623, + "objective/train/theoretical_loss": 3.366382695238683, + "objective/train/tokens_used": 890449376, + "theoretical_loss": 3.366382695238683, + "tokens_seen": 2519990272 + }, + { + "epoch": 0.53, + "learning_rate": 4.7745145241534265e-05, + "loss": 2.556, + "theoretical_loss": 3.366375723118091, + "tokens_seen": 2520055808 + }, + { + "epoch": 0.53, + "learning_rate": 4.7737120847376025e-05, + "loss": 2.6218, + "theoretical_loss": 3.3663617795731327, + "tokens_seen": 2520186880 + }, + { + "epoch": 0.53, + "learning_rate": 4.7729096453217785e-05, + "loss": 2.559, + "theoretical_loss": 3.366347836956384, + "tokens_seen": 2520317952 + }, + { + "epoch": 0.53, + "learning_rate": 4.772107205905954e-05, + "loss": 2.5452, + "theoretical_loss": 3.3663338952677346, + "tokens_seen": 2520449024 + }, + { + "epoch": 0.53, + "learning_rate": 4.77130476649013e-05, + "loss": 2.5247, + "theoretical_loss": 3.366319954507074, + "tokens_seen": 2520580096 + }, + { + "epoch": 0.53, + "learning_rate": 4.770502327074306e-05, + "loss": 2.5345, + "theoretical_loss": 3.3663060146742927, + "tokens_seen": 2520711168 + }, + { + "epoch": 0.53, + "learning_rate": 4.769699887658482e-05, + "loss": 2.4603, + "theoretical_loss": 3.3662920757692802, + "tokens_seen": 2520842240 + }, + { + "epoch": 0.53, + "learning_rate": 4.768897448242658e-05, + "loss": 2.7226, + "theoretical_loss": 3.3662781377919275, + "tokens_seen": 2520973312 + }, + { + "epoch": 0.53, + "learning_rate": 4.768095008826834e-05, + "loss": 2.5504, + "theoretical_loss": 3.3662642007421235, + "tokens_seen": 2521104384 + }, + { + "epoch": 0.53, + "learning_rate": 4.76729256941101e-05, + "loss": 2.5523, + "theoretical_loss": 3.366250264619759, + "tokens_seen": 2521235456 + }, + { + "epoch": 0.53, + "learning_rate": 4.766490129995186e-05, + "loss": 2.5654, + "theoretical_loss": 3.3662363294247237, + "tokens_seen": 2521366528 + }, + { + "epoch": 0.53, + "learning_rate": 4.765687690579361e-05, + "loss": 2.6242, + "theoretical_loss": 3.3662223951569077, + "tokens_seen": 2521497600 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1382692, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.633779764175415, + "objective/train/theoretical_loss": 3.3662084618162016, + "objective/train/tokens_used": 892087776, + "theoretical_loss": 3.3662084618162016, + "tokens_seen": 2521628672 + }, + { + "epoch": 0.53, + "learning_rate": 4.764885251163537e-05, + "loss": 2.4958, + "theoretical_loss": 3.3662084618162016, + "tokens_seen": 2521628672 + }, + { + "epoch": 0.53, + "learning_rate": 4.764082811747713e-05, + "loss": 2.5144, + "theoretical_loss": 3.366194529402495, + "tokens_seen": 2521759744 + }, + { + "epoch": 0.53, + "learning_rate": 4.763280372331889e-05, + "loss": 2.5313, + "theoretical_loss": 3.366180597915678, + "tokens_seen": 2521890816 + }, + { + "epoch": 0.53, + "learning_rate": 4.762477932916065e-05, + "loss": 2.7044, + "theoretical_loss": 3.366166667355641, + "tokens_seen": 2522021888 + }, + { + "epoch": 0.53, + "learning_rate": 4.761675493500241e-05, + "loss": 2.6387, + "theoretical_loss": 3.366152737722274, + "tokens_seen": 2522152960 + }, + { + "epoch": 0.53, + "learning_rate": 4.760873054084417e-05, + "loss": 2.6255, + "theoretical_loss": 3.366138809015468, + "tokens_seen": 2522284032 + }, + { + "epoch": 0.53, + "learning_rate": 4.760070614668592e-05, + "loss": 2.6242, + "theoretical_loss": 3.366124881235112, + "tokens_seen": 2522415104 + }, + { + "epoch": 0.53, + "learning_rate": 4.759268175252768e-05, + "loss": 2.6112, + "theoretical_loss": 3.3661109543810968, + "tokens_seen": 2522546176 + }, + { + "epoch": 0.53, + "learning_rate": 4.758465735836944e-05, + "loss": 2.7586, + "theoretical_loss": 3.3660970284533134, + "tokens_seen": 2522677248 + }, + { + "epoch": 0.53, + "learning_rate": 4.75766329642112e-05, + "loss": 2.604, + "theoretical_loss": 3.3660831034516505, + "tokens_seen": 2522808320 + }, + { + "epoch": 0.53, + "learning_rate": 4.756860857005296e-05, + "loss": 2.5642, + "theoretical_loss": 3.366069179376, + "tokens_seen": 2522939392 + }, + { + "epoch": 0.53, + "learning_rate": 4.756058417589472e-05, + "loss": 2.5398, + "theoretical_loss": 3.366055256226251, + "tokens_seen": 2523070464 + }, + { + "epoch": 0.53, + "learning_rate": 4.755255978173648e-05, + "loss": 2.6155, + "theoretical_loss": 3.3660413340022943, + "tokens_seen": 2523201536 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1383183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.505521297454834, + "objective/train/theoretical_loss": 3.366034373237454, + "objective/train/tokens_used": 893726176, + "theoretical_loss": 3.366034373237454, + "tokens_seen": 2523267072 + }, + { + "epoch": 0.53, + "learning_rate": 4.754453538757824e-05, + "loss": 2.6113, + "theoretical_loss": 3.36602741270402, + "tokens_seen": 2523332608 + }, + { + "epoch": 0.53, + "learning_rate": 4.7536510993419995e-05, + "loss": 2.7356, + "theoretical_loss": 3.3660134923313194, + "tokens_seen": 2523463680 + }, + { + "epoch": 0.53, + "learning_rate": 4.7528486599261755e-05, + "loss": 2.5268, + "theoretical_loss": 3.365999572884082, + "tokens_seen": 2523594752 + }, + { + "epoch": 0.53, + "learning_rate": 4.7520462205103515e-05, + "loss": 2.7341, + "theoretical_loss": 3.3659856543621984, + "tokens_seen": 2523725824 + }, + { + "epoch": 0.53, + "learning_rate": 4.7512437810945275e-05, + "loss": 2.6552, + "theoretical_loss": 3.3659717367655593, + "tokens_seen": 2523856896 + }, + { + "epoch": 0.53, + "learning_rate": 4.7504413416787035e-05, + "loss": 2.5299, + "theoretical_loss": 3.3659578200940548, + "tokens_seen": 2523987968 + }, + { + "epoch": 0.53, + "learning_rate": 4.7496389022628795e-05, + "loss": 2.5459, + "theoretical_loss": 3.365943904347575, + "tokens_seen": 2524119040 + }, + { + "epoch": 0.53, + "learning_rate": 4.7488364628470555e-05, + "loss": 2.5231, + "theoretical_loss": 3.3659299895260113, + "tokens_seen": 2524250112 + }, + { + "epoch": 0.53, + "learning_rate": 4.748034023431231e-05, + "loss": 2.6234, + "theoretical_loss": 3.365916075629254, + "tokens_seen": 2524381184 + }, + { + "epoch": 0.53, + "learning_rate": 4.747231584015407e-05, + "loss": 2.544, + "theoretical_loss": 3.3659021626571928, + "tokens_seen": 2524512256 + }, + { + "epoch": 0.53, + "learning_rate": 4.746429144599583e-05, + "loss": 2.4326, + "theoretical_loss": 3.3658882506097196, + "tokens_seen": 2524643328 + }, + { + "epoch": 0.53, + "learning_rate": 4.745626705183759e-05, + "loss": 2.5907, + "theoretical_loss": 3.365874339486724, + "tokens_seen": 2524774400 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1384240, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3049447536468506, + "objective/train/theoretical_loss": 3.3658604292880963, + "objective/train/tokens_used": 895364576, + "theoretical_loss": 3.3658604292880963, + "tokens_seen": 2524905472 + }, + { + "epoch": 0.53, + "learning_rate": 4.744824265767935e-05, + "loss": 2.4003, + "theoretical_loss": 3.3658604292880963, + "tokens_seen": 2524905472 + }, + { + "epoch": 0.53, + "learning_rate": 4.744021826352111e-05, + "loss": 2.6488, + "theoretical_loss": 3.3658465200137275, + "tokens_seen": 2525036544 + }, + { + "epoch": 0.53, + "learning_rate": 4.743219386936287e-05, + "loss": 2.4416, + "theoretical_loss": 3.365832611663509, + "tokens_seen": 2525167616 + }, + { + "epoch": 0.53, + "learning_rate": 4.742416947520462e-05, + "loss": 2.5324, + "theoretical_loss": 3.36581870423733, + "tokens_seen": 2525298688 + }, + { + "epoch": 0.53, + "learning_rate": 4.741614508104638e-05, + "loss": 2.6176, + "theoretical_loss": 3.3658047977350822, + "tokens_seen": 2525429760 + }, + { + "epoch": 0.53, + "learning_rate": 4.740812068688814e-05, + "loss": 2.5067, + "theoretical_loss": 3.365790892156656, + "tokens_seen": 2525560832 + }, + { + "epoch": 0.53, + "learning_rate": 4.74000962927299e-05, + "loss": 2.7227, + "theoretical_loss": 3.365776987501942, + "tokens_seen": 2525691904 + }, + { + "epoch": 0.53, + "learning_rate": 4.739207189857166e-05, + "loss": 2.5388, + "theoretical_loss": 3.365763083770831, + "tokens_seen": 2525822976 + }, + { + "epoch": 0.53, + "learning_rate": 4.738404750441342e-05, + "loss": 2.5103, + "theoretical_loss": 3.3657491809632134, + "tokens_seen": 2525954048 + }, + { + "epoch": 0.53, + "learning_rate": 4.737602311025518e-05, + "loss": 2.5524, + "theoretical_loss": 3.3657352790789807, + "tokens_seen": 2526085120 + }, + { + "epoch": 0.53, + "learning_rate": 4.736799871609694e-05, + "loss": 2.6606, + "theoretical_loss": 3.365721378118023, + "tokens_seen": 2526216192 + }, + { + "epoch": 0.53, + "learning_rate": 4.735997432193869e-05, + "loss": 2.4384, + "theoretical_loss": 3.365707478080231, + "tokens_seen": 2526347264 + }, + { + "epoch": 0.53, + "learning_rate": 4.735194992778045e-05, + "loss": 2.5257, + "theoretical_loss": 3.3656935789654963, + "tokens_seen": 2526478336 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1385479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.296886444091797, + "objective/train/theoretical_loss": 3.3656866297542405, + "objective/train/tokens_used": 897002976, + "theoretical_loss": 3.3656866297542405, + "tokens_seen": 2526543872 + }, + { + "epoch": 0.53, + "learning_rate": 4.734392553362221e-05, + "loss": 2.6146, + "theoretical_loss": 3.3656796807737086, + "tokens_seen": 2526609408 + }, + { + "epoch": 0.53, + "learning_rate": 4.733590113946397e-05, + "loss": 2.6998, + "theoretical_loss": 3.3656657835047596, + "tokens_seen": 2526740480 + }, + { + "epoch": 0.53, + "learning_rate": 4.732787674530573e-05, + "loss": 2.6655, + "theoretical_loss": 3.3656518871585397, + "tokens_seen": 2526871552 + }, + { + "epoch": 0.53, + "learning_rate": 4.731985235114749e-05, + "loss": 2.4147, + "theoretical_loss": 3.36563799173494, + "tokens_seen": 2527002624 + }, + { + "epoch": 0.53, + "learning_rate": 4.731182795698925e-05, + "loss": 2.3529, + "theoretical_loss": 3.3656240972338516, + "tokens_seen": 2527133696 + }, + { + "epoch": 0.53, + "learning_rate": 4.7303803562831006e-05, + "loss": 2.6953, + "theoretical_loss": 3.365610203655165, + "tokens_seen": 2527264768 + }, + { + "epoch": 0.53, + "learning_rate": 4.7295779168672765e-05, + "loss": 2.5245, + "theoretical_loss": 3.365596310998771, + "tokens_seen": 2527395840 + }, + { + "epoch": 0.53, + "learning_rate": 4.7287754774514525e-05, + "loss": 2.4851, + "theoretical_loss": 3.3655824192645616, + "tokens_seen": 2527526912 + }, + { + "epoch": 0.53, + "learning_rate": 4.7279730380356285e-05, + "loss": 2.5641, + "theoretical_loss": 3.3655685284524264, + "tokens_seen": 2527657984 + }, + { + "epoch": 0.53, + "learning_rate": 4.7271705986198045e-05, + "loss": 2.6068, + "theoretical_loss": 3.365554638562257, + "tokens_seen": 2527789056 + }, + { + "epoch": 0.53, + "learning_rate": 4.7263681592039805e-05, + "loss": 2.6198, + "theoretical_loss": 3.3655407495939444, + "tokens_seen": 2527920128 + }, + { + "epoch": 0.53, + "learning_rate": 4.7255657197881565e-05, + "loss": 2.5761, + "theoretical_loss": 3.36552686154738, + "tokens_seen": 2528051200 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1385945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3946783542633057, + "objective/train/theoretical_loss": 3.365512974422454, + "objective/train/tokens_used": 898641376, + "theoretical_loss": 3.365512974422454, + "tokens_seen": 2528182272 + }, + { + "epoch": 0.53, + "learning_rate": 4.724763280372332e-05, + "loss": 2.6438, + "theoretical_loss": 3.365512974422454, + "tokens_seen": 2528182272 + }, + { + "epoch": 0.53, + "learning_rate": 4.723960840956508e-05, + "loss": 2.593, + "theoretical_loss": 3.3654990882190585, + "tokens_seen": 2528313344 + }, + { + "epoch": 0.53, + "learning_rate": 4.723158401540684e-05, + "loss": 2.7042, + "theoretical_loss": 3.3654852029370836, + "tokens_seen": 2528444416 + }, + { + "epoch": 0.53, + "learning_rate": 4.72235596212486e-05, + "loss": 2.6143, + "theoretical_loss": 3.365471318576421, + "tokens_seen": 2528575488 + }, + { + "epoch": 0.53, + "learning_rate": 4.721553522709036e-05, + "loss": 2.5997, + "theoretical_loss": 3.3654574351369617, + "tokens_seen": 2528706560 + }, + { + "epoch": 0.53, + "learning_rate": 4.720751083293212e-05, + "loss": 2.4652, + "theoretical_loss": 3.365443552618596, + "tokens_seen": 2528837632 + }, + { + "epoch": 0.53, + "learning_rate": 4.719948643877388e-05, + "loss": 2.5552, + "theoretical_loss": 3.365429671021217, + "tokens_seen": 2528968704 + }, + { + "epoch": 0.53, + "learning_rate": 4.719146204461564e-05, + "loss": 2.6855, + "theoretical_loss": 3.365415790344714, + "tokens_seen": 2529099776 + }, + { + "epoch": 0.53, + "learning_rate": 4.718343765045739e-05, + "loss": 2.6556, + "theoretical_loss": 3.365401910588979, + "tokens_seen": 2529230848 + }, + { + "epoch": 0.53, + "learning_rate": 4.717541325629915e-05, + "loss": 2.4773, + "theoretical_loss": 3.365388031753903, + "tokens_seen": 2529361920 + }, + { + "epoch": 0.53, + "learning_rate": 4.716738886214091e-05, + "loss": 2.7238, + "theoretical_loss": 3.3653741538393778, + "tokens_seen": 2529492992 + }, + { + "epoch": 0.53, + "learning_rate": 4.715936446798267e-05, + "loss": 2.6008, + "theoretical_loss": 3.3653602768452937, + "tokens_seen": 2529624064 + }, + { + "epoch": 0.53, + "learning_rate": 4.715134007382443e-05, + "loss": 2.5201, + "theoretical_loss": 3.3653464007715423, + "tokens_seen": 2529755136 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1387323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5731987953186035, + "objective/train/theoretical_loss": 3.3653394630797577, + "objective/train/tokens_used": 900279776, + "theoretical_loss": 3.3653394630797577, + "tokens_seen": 2529820672 + }, + { + "epoch": 0.53, + "learning_rate": 4.714331567966619e-05, + "loss": 2.6717, + "theoretical_loss": 3.3653325256180153, + "tokens_seen": 2529886208 + }, + { + "epoch": 0.53, + "learning_rate": 4.713529128550795e-05, + "loss": 2.4919, + "theoretical_loss": 3.3653186513846034, + "tokens_seen": 2530017280 + }, + { + "epoch": 0.53, + "learning_rate": 4.71272668913497e-05, + "loss": 2.6858, + "theoretical_loss": 3.3653047780711987, + "tokens_seen": 2530148352 + }, + { + "epoch": 0.53, + "learning_rate": 4.711924249719146e-05, + "loss": 2.5927, + "theoretical_loss": 3.365290905677692, + "tokens_seen": 2530279424 + }, + { + "epoch": 0.53, + "learning_rate": 4.711121810303322e-05, + "loss": 2.4124, + "theoretical_loss": 3.365277034203974, + "tokens_seen": 2530410496 + }, + { + "epoch": 0.53, + "learning_rate": 4.710319370887498e-05, + "loss": 2.5957, + "theoretical_loss": 3.3652631636499377, + "tokens_seen": 2530541568 + }, + { + "epoch": 0.53, + "learning_rate": 4.709516931471674e-05, + "loss": 2.4652, + "theoretical_loss": 3.3652492940154732, + "tokens_seen": 2530672640 + }, + { + "epoch": 0.53, + "learning_rate": 4.70871449205585e-05, + "loss": 2.722, + "theoretical_loss": 3.3652354253004724, + "tokens_seen": 2530803712 + }, + { + "epoch": 0.53, + "learning_rate": 4.707912052640026e-05, + "loss": 2.7066, + "theoretical_loss": 3.3652215575048268, + "tokens_seen": 2530934784 + }, + { + "epoch": 0.53, + "learning_rate": 4.707109613224202e-05, + "loss": 2.5831, + "theoretical_loss": 3.365207690628427, + "tokens_seen": 2531065856 + }, + { + "epoch": 0.53, + "learning_rate": 4.7063071738083776e-05, + "loss": 2.5865, + "theoretical_loss": 3.365193824671166, + "tokens_seen": 2531196928 + }, + { + "epoch": 0.53, + "learning_rate": 4.7055047343925536e-05, + "loss": 2.7067, + "theoretical_loss": 3.3651799596329344, + "tokens_seen": 2531328000 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1387695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5774648189544678, + "objective/train/theoretical_loss": 3.365166095513623, + "objective/train/tokens_used": 901918176, + "theoretical_loss": 3.365166095513623, + "tokens_seen": 2531459072 + }, + { + "epoch": 0.53, + "learning_rate": 4.7047022949767295e-05, + "loss": 2.4986, + "theoretical_loss": 3.365166095513623, + "tokens_seen": 2531459072 + }, + { + "epoch": 0.53, + "learning_rate": 4.7038998555609055e-05, + "loss": 2.5257, + "theoretical_loss": 3.3651522323131244, + "tokens_seen": 2531590144 + }, + { + "epoch": 0.53, + "learning_rate": 4.7030974161450815e-05, + "loss": 2.597, + "theoretical_loss": 3.36513837003133, + "tokens_seen": 2531721216 + }, + { + "epoch": 0.53, + "learning_rate": 4.7022949767292575e-05, + "loss": 2.7001, + "theoretical_loss": 3.3651245086681314, + "tokens_seen": 2531852288 + }, + { + "epoch": 0.53, + "learning_rate": 4.7014925373134335e-05, + "loss": 2.5374, + "theoretical_loss": 3.3651106482234194, + "tokens_seen": 2531983360 + }, + { + "epoch": 0.53, + "learning_rate": 4.700690097897609e-05, + "loss": 2.5377, + "theoretical_loss": 3.3650967886970866, + "tokens_seen": 2532114432 + }, + { + "epoch": 0.53, + "learning_rate": 4.699887658481785e-05, + "loss": 2.5266, + "theoretical_loss": 3.365082930089024, + "tokens_seen": 2532245504 + }, + { + "epoch": 0.53, + "learning_rate": 4.699085219065961e-05, + "loss": 2.6136, + "theoretical_loss": 3.3650690723991232, + "tokens_seen": 2532376576 + }, + { + "epoch": 0.53, + "learning_rate": 4.698282779650137e-05, + "loss": 2.4857, + "theoretical_loss": 3.3650552156272764, + "tokens_seen": 2532507648 + }, + { + "epoch": 0.53, + "learning_rate": 4.697480340234313e-05, + "loss": 2.7073, + "theoretical_loss": 3.365041359773375, + "tokens_seen": 2532638720 + }, + { + "epoch": 0.54, + "learning_rate": 4.696677900818489e-05, + "loss": 2.5181, + "theoretical_loss": 3.36502750483731, + "tokens_seen": 2532769792 + }, + { + "epoch": 0.54, + "learning_rate": 4.695875461402665e-05, + "loss": 2.4032, + "theoretical_loss": 3.365013650818974, + "tokens_seen": 2532900864 + }, + { + "epoch": 0.54, + "learning_rate": 4.69507302198684e-05, + "loss": 2.5306, + "theoretical_loss": 3.3649997977182586, + "tokens_seen": 2533031936 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1388584, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8600099086761475, + "objective/train/theoretical_loss": 3.3649928715119746, + "objective/train/tokens_used": 903556576, + "theoretical_loss": 3.3649928715119746, + "tokens_seen": 2533097472 + }, + { + "epoch": 0.54, + "learning_rate": 4.694270582571016e-05, + "loss": 2.7793, + "theoretical_loss": 3.3649859455350555, + "tokens_seen": 2533163008 + }, + { + "epoch": 0.54, + "learning_rate": 4.693468143155192e-05, + "loss": 2.5522, + "theoretical_loss": 3.3649720942692563, + "tokens_seen": 2533294080 + }, + { + "epoch": 0.54, + "learning_rate": 4.692665703739368e-05, + "loss": 2.6216, + "theoretical_loss": 3.3649582439207526, + "tokens_seen": 2533425152 + }, + { + "epoch": 0.54, + "learning_rate": 4.691863264323544e-05, + "loss": 2.5232, + "theoretical_loss": 3.364944394489436, + "tokens_seen": 2533556224 + }, + { + "epoch": 0.54, + "learning_rate": 4.69106082490772e-05, + "loss": 2.5141, + "theoretical_loss": 3.3649305459751995, + "tokens_seen": 2533687296 + }, + { + "epoch": 0.54, + "learning_rate": 4.690258385491896e-05, + "loss": 2.6482, + "theoretical_loss": 3.364916698377934, + "tokens_seen": 2533818368 + }, + { + "epoch": 0.54, + "learning_rate": 4.689455946076072e-05, + "loss": 2.6283, + "theoretical_loss": 3.3649028516975314, + "tokens_seen": 2533949440 + }, + { + "epoch": 0.54, + "learning_rate": 4.688653506660247e-05, + "loss": 2.6347, + "theoretical_loss": 3.364889005933884, + "tokens_seen": 2534080512 + }, + { + "epoch": 0.54, + "learning_rate": 4.687851067244423e-05, + "loss": 2.5845, + "theoretical_loss": 3.364875161086883, + "tokens_seen": 2534211584 + }, + { + "epoch": 0.54, + "learning_rate": 4.687048627828599e-05, + "loss": 2.5044, + "theoretical_loss": 3.3648613171564206, + "tokens_seen": 2534342656 + }, + { + "epoch": 0.54, + "learning_rate": 4.686246188412775e-05, + "loss": 2.5999, + "theoretical_loss": 3.3648474741423895, + "tokens_seen": 2534473728 + }, + { + "epoch": 0.54, + "learning_rate": 4.685443748996951e-05, + "loss": 2.4732, + "theoretical_loss": 3.36483363204468, + "tokens_seen": 2534604800 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1389704, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.414130449295044, + "objective/train/theoretical_loss": 3.364819790863186, + "objective/train/tokens_used": 905194976, + "theoretical_loss": 3.364819790863186, + "tokens_seen": 2534735872 + }, + { + "epoch": 0.54, + "learning_rate": 4.684641309581127e-05, + "loss": 2.4403, + "theoretical_loss": 3.364819790863186, + "tokens_seen": 2534735872 + }, + { + "epoch": 0.54, + "learning_rate": 4.683838870165303e-05, + "loss": 2.5613, + "theoretical_loss": 3.364805950597798, + "tokens_seen": 2534866944 + }, + { + "epoch": 0.54, + "learning_rate": 4.6830364307494786e-05, + "loss": 2.6744, + "theoretical_loss": 3.3647921112484087, + "tokens_seen": 2534998016 + }, + { + "epoch": 0.54, + "learning_rate": 4.6822339913336546e-05, + "loss": 2.6223, + "theoretical_loss": 3.36477827281491, + "tokens_seen": 2535129088 + }, + { + "epoch": 0.54, + "learning_rate": 4.6814315519178306e-05, + "loss": 2.5623, + "theoretical_loss": 3.364764435297194, + "tokens_seen": 2535260160 + }, + { + "epoch": 0.54, + "learning_rate": 4.6806291125020066e-05, + "loss": 2.5392, + "theoretical_loss": 3.364750598695152, + "tokens_seen": 2535391232 + }, + { + "epoch": 0.54, + "learning_rate": 4.6798266730861825e-05, + "loss": 2.4866, + "theoretical_loss": 3.3647367630086773, + "tokens_seen": 2535522304 + }, + { + "epoch": 0.54, + "learning_rate": 4.6790242336703585e-05, + "loss": 2.5354, + "theoretical_loss": 3.364722928237661, + "tokens_seen": 2535653376 + }, + { + "epoch": 0.54, + "learning_rate": 4.6782217942545345e-05, + "loss": 2.5069, + "theoretical_loss": 3.364709094381996, + "tokens_seen": 2535784448 + }, + { + "epoch": 0.54, + "learning_rate": 4.67741935483871e-05, + "loss": 2.5704, + "theoretical_loss": 3.364695261441574, + "tokens_seen": 2535915520 + }, + { + "epoch": 0.54, + "learning_rate": 4.676616915422886e-05, + "loss": 2.5766, + "theoretical_loss": 3.364681429416287, + "tokens_seen": 2536046592 + }, + { + "epoch": 0.54, + "learning_rate": 4.675814476007062e-05, + "loss": 2.6808, + "theoretical_loss": 3.3646675983060272, + "tokens_seen": 2536177664 + }, + { + "epoch": 0.54, + "learning_rate": 4.675012036591238e-05, + "loss": 2.5677, + "theoretical_loss": 3.364653768110687, + "tokens_seen": 2536308736 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1390171, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4923295974731445, + "objective/train/theoretical_loss": 3.3646468533560787, + "objective/train/tokens_used": 906833376, + "theoretical_loss": 3.3646468533560787, + "tokens_seen": 2536374272 + }, + { + "epoch": 0.54, + "learning_rate": 4.674209597175414e-05, + "loss": 2.5076, + "theoretical_loss": 3.364639938830159, + "tokens_seen": 2536439808 + }, + { + "epoch": 0.54, + "learning_rate": 4.67340715775959e-05, + "loss": 2.5555, + "theoretical_loss": 3.364626110464335, + "tokens_seen": 2536570880 + }, + { + "epoch": 0.54, + "learning_rate": 4.672604718343766e-05, + "loss": 2.5306, + "theoretical_loss": 3.3646122830131064, + "tokens_seen": 2536701952 + }, + { + "epoch": 0.54, + "learning_rate": 4.671802278927941e-05, + "loss": 2.6071, + "theoretical_loss": 3.364598456476367, + "tokens_seen": 2536833024 + }, + { + "epoch": 0.54, + "learning_rate": 4.670999839512117e-05, + "loss": 2.7333, + "theoretical_loss": 3.364584630854008, + "tokens_seen": 2536964096 + }, + { + "epoch": 0.54, + "learning_rate": 4.670197400096293e-05, + "loss": 2.4534, + "theoretical_loss": 3.364570806145922, + "tokens_seen": 2537095168 + }, + { + "epoch": 0.54, + "learning_rate": 4.669394960680469e-05, + "loss": 2.6157, + "theoretical_loss": 3.3645569823520014, + "tokens_seen": 2537226240 + }, + { + "epoch": 0.54, + "learning_rate": 4.668592521264645e-05, + "loss": 2.6054, + "theoretical_loss": 3.3645431594721384, + "tokens_seen": 2537357312 + }, + { + "epoch": 0.54, + "learning_rate": 4.667790081848821e-05, + "loss": 2.6694, + "theoretical_loss": 3.3645293375062253, + "tokens_seen": 2537488384 + }, + { + "epoch": 0.54, + "learning_rate": 4.6669876424329964e-05, + "loss": 2.7058, + "theoretical_loss": 3.364515516454155, + "tokens_seen": 2537619456 + }, + { + "epoch": 0.54, + "learning_rate": 4.6661852030171723e-05, + "loss": 2.6628, + "theoretical_loss": 3.364501696315819, + "tokens_seen": 2537750528 + }, + { + "epoch": 0.54, + "learning_rate": 4.6653827636013483e-05, + "loss": 2.5902, + "theoretical_loss": 3.3644878770911104, + "tokens_seen": 2537881600 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1391196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.740980863571167, + "objective/train/theoretical_loss": 3.364474058779922, + "objective/train/tokens_used": 908471776, + "theoretical_loss": 3.364474058779922, + "tokens_seen": 2538012672 + }, + { + "epoch": 0.54, + "learning_rate": 4.664580324185524e-05, + "loss": 2.6695, + "theoretical_loss": 3.364474058779922, + "tokens_seen": 2538012672 + }, + { + "epoch": 0.54, + "learning_rate": 4.6637778847697e-05, + "loss": 2.6864, + "theoretical_loss": 3.3644602413821447, + "tokens_seen": 2538143744 + }, + { + "epoch": 0.54, + "learning_rate": 4.662975445353876e-05, + "loss": 2.5227, + "theoretical_loss": 3.3644464248976718, + "tokens_seen": 2538274816 + }, + { + "epoch": 0.54, + "learning_rate": 4.6621730059380516e-05, + "loss": 2.672, + "theoretical_loss": 3.3644326093263963, + "tokens_seen": 2538405888 + }, + { + "epoch": 0.54, + "learning_rate": 4.6613705665222276e-05, + "loss": 2.7277, + "theoretical_loss": 3.36441879466821, + "tokens_seen": 2538536960 + }, + { + "epoch": 0.54, + "learning_rate": 4.6605681271064036e-05, + "loss": 2.5514, + "theoretical_loss": 3.364404980923006, + "tokens_seen": 2538668032 + }, + { + "epoch": 0.54, + "learning_rate": 4.6597656876905796e-05, + "loss": 2.6118, + "theoretical_loss": 3.3643911680906764, + "tokens_seen": 2538799104 + }, + { + "epoch": 0.54, + "learning_rate": 4.6589632482747556e-05, + "loss": 2.5991, + "theoretical_loss": 3.3643773561711137, + "tokens_seen": 2538930176 + }, + { + "epoch": 0.54, + "learning_rate": 4.6581608088589316e-05, + "loss": 2.5461, + "theoretical_loss": 3.3643635451642107, + "tokens_seen": 2539061248 + }, + { + "epoch": 0.54, + "learning_rate": 4.657358369443107e-05, + "loss": 2.5967, + "theoretical_loss": 3.36434973506986, + "tokens_seen": 2539192320 + }, + { + "epoch": 0.54, + "learning_rate": 4.656555930027283e-05, + "loss": 2.4883, + "theoretical_loss": 3.364335925887954, + "tokens_seen": 2539323392 + }, + { + "epoch": 0.54, + "learning_rate": 4.655753490611459e-05, + "loss": 2.6744, + "theoretical_loss": 3.3643221176183853, + "tokens_seen": 2539454464 + }, + { + "epoch": 0.54, + "learning_rate": 4.654951051195635e-05, + "loss": 2.6774, + "theoretical_loss": 3.364308310261047, + "tokens_seen": 2539585536 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1391838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.731227159500122, + "objective/train/theoretical_loss": 3.3643014069244304, + "objective/train/tokens_used": 910110176, + "theoretical_loss": 3.3643014069244304, + "tokens_seen": 2539651072 + }, + { + "epoch": 0.54, + "learning_rate": 4.654148611779811e-05, + "loss": 2.7158, + "theoretical_loss": 3.364294503815831, + "tokens_seen": 2539716608 + }, + { + "epoch": 0.54, + "learning_rate": 4.653346172363987e-05, + "loss": 2.706, + "theoretical_loss": 3.3642806982826308, + "tokens_seen": 2539847680 + }, + { + "epoch": 0.54, + "learning_rate": 4.652543732948162e-05, + "loss": 2.513, + "theoretical_loss": 3.364266893661338, + "tokens_seen": 2539978752 + }, + { + "epoch": 0.54, + "learning_rate": 4.651741293532338e-05, + "loss": 2.7028, + "theoretical_loss": 3.3642530899518466, + "tokens_seen": 2540109824 + }, + { + "epoch": 0.54, + "learning_rate": 4.650938854116514e-05, + "loss": 2.5307, + "theoretical_loss": 3.3642392871540485, + "tokens_seen": 2540240896 + }, + { + "epoch": 0.54, + "learning_rate": 4.65013641470069e-05, + "loss": 2.4628, + "theoretical_loss": 3.364225485267837, + "tokens_seen": 2540371968 + }, + { + "epoch": 0.54, + "learning_rate": 4.649333975284866e-05, + "loss": 2.5531, + "theoretical_loss": 3.364211684293104, + "tokens_seen": 2540503040 + }, + { + "epoch": 0.54, + "learning_rate": 4.648531535869042e-05, + "loss": 2.5976, + "theoretical_loss": 3.364197884229743, + "tokens_seen": 2540634112 + }, + { + "epoch": 0.54, + "learning_rate": 4.6477290964532174e-05, + "loss": 2.6477, + "theoretical_loss": 3.364184085077647, + "tokens_seen": 2540765184 + }, + { + "epoch": 0.54, + "learning_rate": 4.6469266570373934e-05, + "loss": 2.387, + "theoretical_loss": 3.364170286836708, + "tokens_seen": 2540896256 + }, + { + "epoch": 0.54, + "learning_rate": 4.6461242176215694e-05, + "loss": 2.5084, + "theoretical_loss": 3.3641564895068194, + "tokens_seen": 2541027328 + }, + { + "epoch": 0.54, + "learning_rate": 4.6453217782057454e-05, + "loss": 2.5988, + "theoretical_loss": 3.3641426930878744, + "tokens_seen": 2541158400 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1392953, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.693942070007324, + "objective/train/theoretical_loss": 3.3641288975797647, + "objective/train/tokens_used": 911748576, + "theoretical_loss": 3.3641288975797647, + "tokens_seen": 2541289472 + }, + { + "epoch": 0.54, + "learning_rate": 4.6445193387899214e-05, + "loss": 2.4564, + "theoretical_loss": 3.3641288975797647, + "tokens_seen": 2541289472 + }, + { + "epoch": 0.54, + "learning_rate": 4.6437168993740974e-05, + "loss": 2.6742, + "theoretical_loss": 3.3641151029823844, + "tokens_seen": 2541420544 + }, + { + "epoch": 0.54, + "learning_rate": 4.6429144599582734e-05, + "loss": 2.6573, + "theoretical_loss": 3.3641013092956253, + "tokens_seen": 2541551616 + }, + { + "epoch": 0.54, + "learning_rate": 4.6421120205424494e-05, + "loss": 2.612, + "theoretical_loss": 3.3640875165193815, + "tokens_seen": 2541682688 + }, + { + "epoch": 0.54, + "learning_rate": 4.641309581126625e-05, + "loss": 2.5469, + "theoretical_loss": 3.3640737246535455, + "tokens_seen": 2541813760 + }, + { + "epoch": 0.54, + "learning_rate": 4.6405071417108007e-05, + "loss": 2.6249, + "theoretical_loss": 3.36405993369801, + "tokens_seen": 2541944832 + }, + { + "epoch": 0.54, + "learning_rate": 4.6397047022949767e-05, + "loss": 2.5959, + "theoretical_loss": 3.364046143652668, + "tokens_seen": 2542075904 + }, + { + "epoch": 0.54, + "learning_rate": 4.6389022628791526e-05, + "loss": 2.6936, + "theoretical_loss": 3.3640323545174127, + "tokens_seen": 2542206976 + }, + { + "epoch": 0.54, + "learning_rate": 4.6380998234633286e-05, + "loss": 2.7513, + "theoretical_loss": 3.3640185662921374, + "tokens_seen": 2542338048 + }, + { + "epoch": 0.54, + "learning_rate": 4.6372973840475046e-05, + "loss": 2.5647, + "theoretical_loss": 3.3640047789767342, + "tokens_seen": 2542469120 + }, + { + "epoch": 0.54, + "learning_rate": 4.6364949446316806e-05, + "loss": 2.5039, + "theoretical_loss": 3.3639909925710976, + "tokens_seen": 2542600192 + }, + { + "epoch": 0.54, + "learning_rate": 4.635692505215856e-05, + "loss": 2.5581, + "theoretical_loss": 3.363977207075119, + "tokens_seen": 2542731264 + }, + { + "epoch": 0.54, + "learning_rate": 4.634890065800032e-05, + "loss": 2.6156, + "theoretical_loss": 3.363963422488693, + "tokens_seen": 2542862336 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1393377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.728156805038452, + "objective/train/theoretical_loss": 3.3639565305365284, + "objective/train/tokens_used": 913386976, + "theoretical_loss": 3.3639565305365284, + "tokens_seen": 2542927872 + }, + { + "epoch": 0.54, + "learning_rate": 4.634087626384208e-05, + "loss": 2.6067, + "theoretical_loss": 3.363949638811712, + "tokens_seen": 2542993408 + }, + { + "epoch": 0.54, + "learning_rate": 4.633285186968384e-05, + "loss": 2.6934, + "theoretical_loss": 3.3639358560440686, + "tokens_seen": 2543124480 + }, + { + "epoch": 0.54, + "learning_rate": 4.63248274755256e-05, + "loss": 2.6777, + "theoretical_loss": 3.363922074185657, + "tokens_seen": 2543255552 + }, + { + "epoch": 0.54, + "learning_rate": 4.631680308136736e-05, + "loss": 2.6308, + "theoretical_loss": 3.3639082932363698, + "tokens_seen": 2543386624 + }, + { + "epoch": 0.54, + "learning_rate": 4.630877868720912e-05, + "loss": 2.7084, + "theoretical_loss": 3.3638945131961, + "tokens_seen": 2543517696 + }, + { + "epoch": 0.54, + "learning_rate": 4.630075429305087e-05, + "loss": 2.5315, + "theoretical_loss": 3.3638807340647414, + "tokens_seen": 2543648768 + }, + { + "epoch": 0.54, + "learning_rate": 4.629272989889263e-05, + "loss": 2.5836, + "theoretical_loss": 3.3638669558421865, + "tokens_seen": 2543779840 + }, + { + "epoch": 0.54, + "learning_rate": 4.628470550473439e-05, + "loss": 2.5142, + "theoretical_loss": 3.363853178528329, + "tokens_seen": 2543910912 + }, + { + "epoch": 0.54, + "learning_rate": 4.627668111057615e-05, + "loss": 2.656, + "theoretical_loss": 3.3638394021230624, + "tokens_seen": 2544041984 + }, + { + "epoch": 0.54, + "learning_rate": 4.626865671641791e-05, + "loss": 2.5832, + "theoretical_loss": 3.3638256266262796, + "tokens_seen": 2544173056 + }, + { + "epoch": 0.54, + "learning_rate": 4.626063232225967e-05, + "loss": 2.5888, + "theoretical_loss": 3.363811852037874, + "tokens_seen": 2544304128 + }, + { + "epoch": 0.54, + "learning_rate": 4.625260792810143e-05, + "loss": 2.5318, + "theoretical_loss": 3.3637980783577386, + "tokens_seen": 2544435200 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1394146, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2629482746124268, + "objective/train/theoretical_loss": 3.3637843055857672, + "objective/train/tokens_used": 915025376, + "theoretical_loss": 3.3637843055857672, + "tokens_seen": 2544566272 + }, + { + "epoch": 0.54, + "learning_rate": 4.624458353394319e-05, + "loss": 2.6897, + "theoretical_loss": 3.3637843055857672, + "tokens_seen": 2544566272 + }, + { + "epoch": 0.54, + "learning_rate": 4.6236559139784944e-05, + "loss": 2.5242, + "theoretical_loss": 3.3637705337218526, + "tokens_seen": 2544697344 + }, + { + "epoch": 0.54, + "learning_rate": 4.6228534745626704e-05, + "loss": 2.6539, + "theoretical_loss": 3.3637567627658886, + "tokens_seen": 2544828416 + }, + { + "epoch": 0.54, + "learning_rate": 4.6220510351468464e-05, + "loss": 2.5891, + "theoretical_loss": 3.3637429927177687, + "tokens_seen": 2544959488 + }, + { + "epoch": 0.54, + "learning_rate": 4.6212485957310224e-05, + "loss": 2.5884, + "theoretical_loss": 3.363729223577386, + "tokens_seen": 2545090560 + }, + { + "epoch": 0.54, + "learning_rate": 4.6204461563151984e-05, + "loss": 2.5462, + "theoretical_loss": 3.363715455344634, + "tokens_seen": 2545221632 + }, + { + "epoch": 0.54, + "learning_rate": 4.6196437168993744e-05, + "loss": 2.7389, + "theoretical_loss": 3.3637016880194057, + "tokens_seen": 2545352704 + }, + { + "epoch": 0.54, + "learning_rate": 4.6188412774835504e-05, + "loss": 2.6143, + "theoretical_loss": 3.3636879216015956, + "tokens_seen": 2545483776 + }, + { + "epoch": 0.54, + "learning_rate": 4.618038838067726e-05, + "loss": 2.8025, + "theoretical_loss": 3.363674156091096, + "tokens_seen": 2545614848 + }, + { + "epoch": 0.54, + "learning_rate": 4.617236398651902e-05, + "loss": 2.5315, + "theoretical_loss": 3.363660391487801, + "tokens_seen": 2545745920 + }, + { + "epoch": 0.54, + "learning_rate": 4.616433959236078e-05, + "loss": 2.5752, + "theoretical_loss": 3.3636466277916046, + "tokens_seen": 2545876992 + }, + { + "epoch": 0.54, + "learning_rate": 4.6156315198202537e-05, + "loss": 2.5858, + "theoretical_loss": 3.3636328650023994, + "tokens_seen": 2546008064 + }, + { + "epoch": 0.54, + "learning_rate": 4.6148290804044296e-05, + "loss": 2.6586, + "theoretical_loss": 3.363619103120079, + "tokens_seen": 2546139136 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1395124, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2976861000061035, + "objective/train/theoretical_loss": 3.3636122225189675, + "objective/train/tokens_used": 916663776, + "theoretical_loss": 3.3636122225189675, + "tokens_seen": 2546204672 + }, + { + "epoch": 0.54, + "learning_rate": 4.6140266409886056e-05, + "loss": 2.5335, + "theoretical_loss": 3.3636053421445378, + "tokens_seen": 2546270208 + }, + { + "epoch": 0.54, + "learning_rate": 4.6132242015727816e-05, + "loss": 2.5371, + "theoretical_loss": 3.3635915820756686, + "tokens_seen": 2546401280 + }, + { + "epoch": 0.54, + "learning_rate": 4.612421762156957e-05, + "loss": 2.6153, + "theoretical_loss": 3.3635778229133653, + "tokens_seen": 2546532352 + }, + { + "epoch": 0.54, + "learning_rate": 4.611619322741133e-05, + "loss": 2.6593, + "theoretical_loss": 3.3635640646575213, + "tokens_seen": 2546663424 + }, + { + "epoch": 0.54, + "learning_rate": 4.610816883325309e-05, + "loss": 2.552, + "theoretical_loss": 3.3635503073080306, + "tokens_seen": 2546794496 + }, + { + "epoch": 0.54, + "learning_rate": 4.610014443909485e-05, + "loss": 2.5235, + "theoretical_loss": 3.3635365508647865, + "tokens_seen": 2546925568 + }, + { + "epoch": 0.54, + "learning_rate": 4.609212004493661e-05, + "loss": 2.6038, + "theoretical_loss": 3.363522795327683, + "tokens_seen": 2547056640 + }, + { + "epoch": 0.54, + "learning_rate": 4.608409565077837e-05, + "loss": 2.529, + "theoretical_loss": 3.3635090406966133, + "tokens_seen": 2547187712 + }, + { + "epoch": 0.54, + "learning_rate": 4.607607125662013e-05, + "loss": 2.6483, + "theoretical_loss": 3.363495286971472, + "tokens_seen": 2547318784 + }, + { + "epoch": 0.54, + "learning_rate": 4.606804686246189e-05, + "loss": 2.4785, + "theoretical_loss": 3.3634815341521516, + "tokens_seen": 2547449856 + }, + { + "epoch": 0.54, + "learning_rate": 4.606002246830364e-05, + "loss": 2.5247, + "theoretical_loss": 3.3634677822385464, + "tokens_seen": 2547580928 + }, + { + "epoch": 0.54, + "learning_rate": 4.60519980741454e-05, + "loss": 2.6911, + "theoretical_loss": 3.3634540312305505, + "tokens_seen": 2547712000 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1395711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6512081623077393, + "objective/train/theoretical_loss": 3.3634402811280575, + "objective/train/tokens_used": 918302176, + "theoretical_loss": 3.3634402811280575, + "tokens_seen": 2547843072 + }, + { + "epoch": 0.54, + "learning_rate": 4.604397367998716e-05, + "loss": 2.5705, + "theoretical_loss": 3.3634402811280575, + "tokens_seen": 2547843072 + }, + { + "epoch": 0.54, + "learning_rate": 4.603594928582892e-05, + "loss": 2.6375, + "theoretical_loss": 3.363426531930961, + "tokens_seen": 2547974144 + }, + { + "epoch": 0.54, + "learning_rate": 4.602792489167068e-05, + "loss": 2.6553, + "theoretical_loss": 3.3634127836391547, + "tokens_seen": 2548105216 + }, + { + "epoch": 0.54, + "learning_rate": 4.601990049751244e-05, + "loss": 2.6957, + "theoretical_loss": 3.363399036252533, + "tokens_seen": 2548236288 + }, + { + "epoch": 0.54, + "learning_rate": 4.60118761033542e-05, + "loss": 2.5482, + "theoretical_loss": 3.363385289770989, + "tokens_seen": 2548367360 + }, + { + "epoch": 0.54, + "learning_rate": 4.6003851709195954e-05, + "loss": 2.5083, + "theoretical_loss": 3.3633715441944174, + "tokens_seen": 2548498432 + }, + { + "epoch": 0.54, + "learning_rate": 4.5995827315037714e-05, + "loss": 2.4978, + "theoretical_loss": 3.3633577995227113, + "tokens_seen": 2548629504 + }, + { + "epoch": 0.54, + "learning_rate": 4.5987802920879474e-05, + "loss": 2.5004, + "theoretical_loss": 3.363344055755765, + "tokens_seen": 2548760576 + }, + { + "epoch": 0.54, + "learning_rate": 4.5979778526721234e-05, + "loss": 2.5514, + "theoretical_loss": 3.3633303128934724, + "tokens_seen": 2548891648 + }, + { + "epoch": 0.54, + "learning_rate": 4.5971754132562994e-05, + "loss": 2.5959, + "theoretical_loss": 3.3633165709357273, + "tokens_seen": 2549022720 + }, + { + "epoch": 0.54, + "learning_rate": 4.5963729738404754e-05, + "loss": 2.6378, + "theoretical_loss": 3.363302829882424, + "tokens_seen": 2549153792 + }, + { + "epoch": 0.55, + "learning_rate": 4.5955705344246514e-05, + "loss": 2.6907, + "theoretical_loss": 3.363289089733456, + "tokens_seen": 2549284864 + }, + { + "epoch": 0.55, + "learning_rate": 4.5947680950088274e-05, + "loss": 2.6071, + "theoretical_loss": 3.363275350488718, + "tokens_seen": 2549415936 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1397015, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.686042308807373, + "objective/train/theoretical_loss": 3.3632684812054015, + "objective/train/tokens_used": 919940576, + "theoretical_loss": 3.3632684812054015, + "tokens_seen": 2549481472 + }, + { + "epoch": 0.55, + "learning_rate": 4.593965655593003e-05, + "loss": 2.6513, + "theoretical_loss": 3.363261612148103, + "tokens_seen": 2549547008 + }, + { + "epoch": 0.55, + "learning_rate": 4.593163216177179e-05, + "loss": 2.634, + "theoretical_loss": 3.3632478747115053, + "tokens_seen": 2549678080 + }, + { + "epoch": 0.55, + "learning_rate": 4.592360776761355e-05, + "loss": 2.5412, + "theoretical_loss": 3.3632341381788198, + "tokens_seen": 2549809152 + }, + { + "epoch": 0.55, + "learning_rate": 4.591558337345531e-05, + "loss": 2.6273, + "theoretical_loss": 3.3632204025499397, + "tokens_seen": 2549940224 + }, + { + "epoch": 0.55, + "learning_rate": 4.5907558979297067e-05, + "loss": 2.496, + "theoretical_loss": 3.3632066678247594, + "tokens_seen": 2550071296 + }, + { + "epoch": 0.55, + "learning_rate": 4.5899534585138826e-05, + "loss": 2.6088, + "theoretical_loss": 3.3631929340031728, + "tokens_seen": 2550202368 + }, + { + "epoch": 0.55, + "learning_rate": 4.5891510190980586e-05, + "loss": 2.6715, + "theoretical_loss": 3.3631792010850745, + "tokens_seen": 2550333440 + }, + { + "epoch": 0.55, + "learning_rate": 4.588348579682234e-05, + "loss": 2.4074, + "theoretical_loss": 3.363165469070358, + "tokens_seen": 2550464512 + }, + { + "epoch": 0.55, + "learning_rate": 4.58754614026641e-05, + "loss": 2.6167, + "theoretical_loss": 3.3631517379589178, + "tokens_seen": 2550595584 + }, + { + "epoch": 0.55, + "learning_rate": 4.586743700850586e-05, + "loss": 2.5815, + "theoretical_loss": 3.363138007750648, + "tokens_seen": 2550726656 + }, + { + "epoch": 0.55, + "learning_rate": 4.585941261434762e-05, + "loss": 2.5765, + "theoretical_loss": 3.3631242784454427, + "tokens_seen": 2550857728 + }, + { + "epoch": 0.55, + "learning_rate": 4.585138822018938e-05, + "loss": 2.5372, + "theoretical_loss": 3.3631105500431966, + "tokens_seen": 2550988800 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1397571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.595946788787842, + "objective/train/theoretical_loss": 3.363096822543803, + "objective/train/tokens_used": 921578976, + "theoretical_loss": 3.363096822543803, + "tokens_seen": 2551119872 + }, + { + "epoch": 0.55, + "learning_rate": 4.584336382603114e-05, + "loss": 2.5753, + "theoretical_loss": 3.363096822543803, + "tokens_seen": 2551119872 + }, + { + "epoch": 0.55, + "learning_rate": 4.58353394318729e-05, + "loss": 2.4736, + "theoretical_loss": 3.363083095947157, + "tokens_seen": 2551250944 + }, + { + "epoch": 0.55, + "learning_rate": 4.582731503771465e-05, + "loss": 2.5798, + "theoretical_loss": 3.3630693702531524, + "tokens_seen": 2551382016 + }, + { + "epoch": 0.55, + "learning_rate": 4.581929064355641e-05, + "loss": 2.6599, + "theoretical_loss": 3.3630556454616833, + "tokens_seen": 2551513088 + }, + { + "epoch": 0.55, + "learning_rate": 4.581126624939817e-05, + "loss": 2.5256, + "theoretical_loss": 3.363041921572645, + "tokens_seen": 2551644160 + }, + { + "epoch": 0.55, + "learning_rate": 4.580324185523993e-05, + "loss": 2.6855, + "theoretical_loss": 3.3630281985859307, + "tokens_seen": 2551775232 + }, + { + "epoch": 0.55, + "learning_rate": 4.579521746108169e-05, + "loss": 2.5716, + "theoretical_loss": 3.3630144765014354, + "tokens_seen": 2551906304 + }, + { + "epoch": 0.55, + "learning_rate": 4.578719306692345e-05, + "loss": 2.5468, + "theoretical_loss": 3.363000755319053, + "tokens_seen": 2552037376 + }, + { + "epoch": 0.55, + "learning_rate": 4.577916867276521e-05, + "loss": 2.5389, + "theoretical_loss": 3.3629870350386777, + "tokens_seen": 2552168448 + }, + { + "epoch": 0.55, + "learning_rate": 4.577114427860697e-05, + "loss": 2.4739, + "theoretical_loss": 3.362973315660205, + "tokens_seen": 2552299520 + }, + { + "epoch": 0.55, + "learning_rate": 4.5763119884448725e-05, + "loss": 2.6023, + "theoretical_loss": 3.362959597183528, + "tokens_seen": 2552430592 + }, + { + "epoch": 0.55, + "learning_rate": 4.5755095490290484e-05, + "loss": 2.6172, + "theoretical_loss": 3.3629458796085414, + "tokens_seen": 2552561664 + }, + { + "epoch": 0.55, + "learning_rate": 4.5747071096132244e-05, + "loss": 2.5386, + "theoretical_loss": 3.3629321629351407, + "tokens_seen": 2552692736 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1398421, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1858367919921875, + "objective/train/theoretical_loss": 3.3629253049365015, + "objective/train/tokens_used": 923217376, + "theoretical_loss": 3.3629253049365015, + "tokens_seen": 2552758272 + }, + { + "epoch": 0.55, + "learning_rate": 4.5739046701974004e-05, + "loss": 2.5921, + "theoretical_loss": 3.362918447163219, + "tokens_seen": 2552823808 + }, + { + "epoch": 0.55, + "learning_rate": 4.5731022307815764e-05, + "loss": 2.5813, + "theoretical_loss": 3.3629047322926713, + "tokens_seen": 2552954880 + }, + { + "epoch": 0.55, + "learning_rate": 4.5722997913657524e-05, + "loss": 2.4869, + "theoretical_loss": 3.3628910183233924, + "tokens_seen": 2553085952 + }, + { + "epoch": 0.55, + "learning_rate": 4.5714973519499284e-05, + "loss": 2.7103, + "theoretical_loss": 3.3628773052552763, + "tokens_seen": 2553217024 + }, + { + "epoch": 0.55, + "learning_rate": 4.570694912534104e-05, + "loss": 2.5565, + "theoretical_loss": 3.362863593088218, + "tokens_seen": 2553348096 + }, + { + "epoch": 0.55, + "learning_rate": 4.56989247311828e-05, + "loss": 2.5588, + "theoretical_loss": 3.3628498818221115, + "tokens_seen": 2553479168 + }, + { + "epoch": 0.55, + "learning_rate": 4.569090033702456e-05, + "loss": 2.6041, + "theoretical_loss": 3.362836171456852, + "tokens_seen": 2553610240 + }, + { + "epoch": 0.55, + "learning_rate": 4.568287594286632e-05, + "loss": 2.5203, + "theoretical_loss": 3.3628224619923333, + "tokens_seen": 2553741312 + }, + { + "epoch": 0.55, + "learning_rate": 4.567485154870808e-05, + "loss": 2.5707, + "theoretical_loss": 3.3628087534284505, + "tokens_seen": 2553872384 + }, + { + "epoch": 0.55, + "learning_rate": 4.566682715454984e-05, + "loss": 2.5169, + "theoretical_loss": 3.3627950457650986, + "tokens_seen": 2554003456 + }, + { + "epoch": 0.55, + "learning_rate": 4.5658802760391597e-05, + "loss": 2.5348, + "theoretical_loss": 3.3627813390021712, + "tokens_seen": 2554134528 + }, + { + "epoch": 0.55, + "learning_rate": 4.565077836623335e-05, + "loss": 2.5583, + "theoretical_loss": 3.3627676331395637, + "tokens_seen": 2554265600 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1399303, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.303102493286133, + "objective/train/theoretical_loss": 3.3627539281771712, + "objective/train/tokens_used": 924855776, + "theoretical_loss": 3.3627539281771712, + "tokens_seen": 2554396672 + }, + { + "epoch": 0.55, + "learning_rate": 4.564275397207511e-05, + "loss": 2.3812, + "theoretical_loss": 3.3627539281771712, + "tokens_seen": 2554396672 + }, + { + "epoch": 0.55, + "learning_rate": 4.563472957791687e-05, + "loss": 2.6224, + "theoretical_loss": 3.362740224114887, + "tokens_seen": 2554527744 + }, + { + "epoch": 0.55, + "learning_rate": 4.562670518375863e-05, + "loss": 2.487, + "theoretical_loss": 3.3627265209526067, + "tokens_seen": 2554658816 + }, + { + "epoch": 0.55, + "learning_rate": 4.561868078960039e-05, + "loss": 2.4577, + "theoretical_loss": 3.362712818690225, + "tokens_seen": 2554789888 + }, + { + "epoch": 0.55, + "learning_rate": 4.561065639544215e-05, + "loss": 2.5577, + "theoretical_loss": 3.362699117327637, + "tokens_seen": 2554920960 + }, + { + "epoch": 0.55, + "learning_rate": 4.560263200128391e-05, + "loss": 2.4576, + "theoretical_loss": 3.3626854168647364, + "tokens_seen": 2555052032 + }, + { + "epoch": 0.55, + "learning_rate": 4.559460760712567e-05, + "loss": 2.6145, + "theoretical_loss": 3.362671717301419, + "tokens_seen": 2555183104 + }, + { + "epoch": 0.55, + "learning_rate": 4.558658321296742e-05, + "loss": 2.5011, + "theoretical_loss": 3.3626580186375787, + "tokens_seen": 2555314176 + }, + { + "epoch": 0.55, + "learning_rate": 4.557855881880918e-05, + "loss": 2.4978, + "theoretical_loss": 3.362644320873111, + "tokens_seen": 2555445248 + }, + { + "epoch": 0.55, + "learning_rate": 4.557053442465094e-05, + "loss": 2.6062, + "theoretical_loss": 3.3626306240079105, + "tokens_seen": 2555576320 + }, + { + "epoch": 0.55, + "learning_rate": 4.55625100304927e-05, + "loss": 2.614, + "theoretical_loss": 3.3626169280418723, + "tokens_seen": 2555707392 + }, + { + "epoch": 0.55, + "learning_rate": 4.555448563633446e-05, + "loss": 2.4572, + "theoretical_loss": 3.3626032329748905, + "tokens_seen": 2555838464 + }, + { + "epoch": 0.55, + "learning_rate": 4.554646124217622e-05, + "loss": 2.5684, + "theoretical_loss": 3.362589538806861, + "tokens_seen": 2555969536 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1399979, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.480012893676758, + "objective/train/theoretical_loss": 3.36258269205992, + "objective/train/tokens_used": 926494176, + "theoretical_loss": 3.36258269205992, + "tokens_seen": 2556035072 + }, + { + "epoch": 0.55, + "learning_rate": 4.553843684801798e-05, + "loss": 2.5772, + "theoretical_loss": 3.3625758455376777, + "tokens_seen": 2556100608 + }, + { + "epoch": 0.55, + "learning_rate": 4.5530412453859735e-05, + "loss": 2.5696, + "theoretical_loss": 3.3625621531672367, + "tokens_seen": 2556231680 + }, + { + "epoch": 0.55, + "learning_rate": 4.5522388059701495e-05, + "loss": 2.5449, + "theoretical_loss": 3.3625484616954315, + "tokens_seen": 2556362752 + }, + { + "epoch": 0.55, + "learning_rate": 4.5514363665543255e-05, + "loss": 2.3782, + "theoretical_loss": 3.3625347711221583, + "tokens_seen": 2556493824 + }, + { + "epoch": 0.55, + "learning_rate": 4.5506339271385014e-05, + "loss": 2.6605, + "theoretical_loss": 3.362521081447311, + "tokens_seen": 2556624896 + }, + { + "epoch": 0.55, + "learning_rate": 4.5498314877226774e-05, + "loss": 2.6123, + "theoretical_loss": 3.3625073926707856, + "tokens_seen": 2556755968 + }, + { + "epoch": 0.55, + "learning_rate": 4.5490290483068534e-05, + "loss": 2.5033, + "theoretical_loss": 3.3624937047924766, + "tokens_seen": 2556887040 + }, + { + "epoch": 0.55, + "learning_rate": 4.5482266088910294e-05, + "loss": 2.5175, + "theoretical_loss": 3.362480017812279, + "tokens_seen": 2557018112 + }, + { + "epoch": 0.55, + "learning_rate": 4.5474241694752054e-05, + "loss": 2.5411, + "theoretical_loss": 3.362466331730088, + "tokens_seen": 2557149184 + }, + { + "epoch": 0.55, + "learning_rate": 4.546621730059381e-05, + "loss": 2.6008, + "theoretical_loss": 3.3624526465457985, + "tokens_seen": 2557280256 + }, + { + "epoch": 0.55, + "learning_rate": 4.545819290643557e-05, + "loss": 2.6209, + "theoretical_loss": 3.362438962259305, + "tokens_seen": 2557411328 + }, + { + "epoch": 0.55, + "learning_rate": 4.545016851227733e-05, + "loss": 2.4899, + "theoretical_loss": 3.362425278870504, + "tokens_seen": 2557542400 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1401394, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3670003414154053, + "objective/train/theoretical_loss": 3.3624115963792898, + "objective/train/tokens_used": 928132576, + "theoretical_loss": 3.3624115963792898, + "tokens_seen": 2557673472 + }, + { + "epoch": 0.55, + "learning_rate": 4.544214411811909e-05, + "loss": 2.5437, + "theoretical_loss": 3.3624115963792898, + "tokens_seen": 2557673472 + }, + { + "epoch": 0.55, + "learning_rate": 4.543411972396085e-05, + "loss": 2.5497, + "theoretical_loss": 3.362397914785557, + "tokens_seen": 2557804544 + }, + { + "epoch": 0.55, + "learning_rate": 4.542609532980261e-05, + "loss": 2.4735, + "theoretical_loss": 3.3623842340892023, + "tokens_seen": 2557935616 + }, + { + "epoch": 0.55, + "learning_rate": 4.541807093564437e-05, + "loss": 2.6774, + "theoretical_loss": 3.362370554290119, + "tokens_seen": 2558066688 + }, + { + "epoch": 0.55, + "learning_rate": 4.541004654148612e-05, + "loss": 2.5931, + "theoretical_loss": 3.362356875388204, + "tokens_seen": 2558197760 + }, + { + "epoch": 0.55, + "learning_rate": 4.540202214732788e-05, + "loss": 2.6006, + "theoretical_loss": 3.3623431973833506, + "tokens_seen": 2558328832 + }, + { + "epoch": 0.55, + "learning_rate": 4.539399775316964e-05, + "loss": 2.5531, + "theoretical_loss": 3.362329520275456, + "tokens_seen": 2558459904 + }, + { + "epoch": 0.55, + "learning_rate": 4.53859733590114e-05, + "loss": 2.5637, + "theoretical_loss": 3.3623158440644136, + "tokens_seen": 2558590976 + }, + { + "epoch": 0.55, + "learning_rate": 4.537794896485316e-05, + "loss": 2.545, + "theoretical_loss": 3.3623021687501202, + "tokens_seen": 2558722048 + }, + { + "epoch": 0.55, + "learning_rate": 4.536992457069492e-05, + "loss": 2.7055, + "theoretical_loss": 3.3622884943324705, + "tokens_seen": 2558853120 + }, + { + "epoch": 0.55, + "learning_rate": 4.536190017653667e-05, + "loss": 2.6548, + "theoretical_loss": 3.3622748208113595, + "tokens_seen": 2558984192 + }, + { + "epoch": 0.55, + "learning_rate": 4.535387578237843e-05, + "loss": 2.4317, + "theoretical_loss": 3.3622611481866826, + "tokens_seen": 2559115264 + }, + { + "epoch": 0.55, + "learning_rate": 4.534585138822019e-05, + "loss": 2.4451, + "theoretical_loss": 3.3622474764583354, + "tokens_seen": 2559246336 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1402751, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5578465461730957, + "objective/train/theoretical_loss": 3.3622406409302528, + "objective/train/tokens_used": 929770976, + "theoretical_loss": 3.3622406409302528, + "tokens_seen": 2559311872 + }, + { + "epoch": 0.55, + "learning_rate": 4.533782699406195e-05, + "loss": 2.7087, + "theoretical_loss": 3.3622338056262127, + "tokens_seen": 2559377408 + }, + { + "epoch": 0.55, + "learning_rate": 4.532980259990371e-05, + "loss": 2.5805, + "theoretical_loss": 3.362220135690211, + "tokens_seen": 2559508480 + }, + { + "epoch": 0.55, + "learning_rate": 4.532177820574547e-05, + "loss": 2.5726, + "theoretical_loss": 3.362206466650224, + "tokens_seen": 2559639552 + }, + { + "epoch": 0.55, + "learning_rate": 4.5313753811587225e-05, + "loss": 2.6407, + "theoretical_loss": 3.3621927985061486, + "tokens_seen": 2559770624 + }, + { + "epoch": 0.55, + "learning_rate": 4.5305729417428985e-05, + "loss": 2.4324, + "theoretical_loss": 3.3621791312578795, + "tokens_seen": 2559901696 + }, + { + "epoch": 0.55, + "learning_rate": 4.5297705023270745e-05, + "loss": 2.5759, + "theoretical_loss": 3.362165464905312, + "tokens_seen": 2560032768 + }, + { + "epoch": 0.55, + "learning_rate": 4.5289680629112505e-05, + "loss": 2.3916, + "theoretical_loss": 3.362151799448342, + "tokens_seen": 2560163840 + }, + { + "epoch": 0.55, + "learning_rate": 4.5281656234954265e-05, + "loss": 2.663, + "theoretical_loss": 3.3621381348868646, + "tokens_seen": 2560294912 + }, + { + "epoch": 0.55, + "learning_rate": 4.5273631840796025e-05, + "loss": 2.5597, + "theoretical_loss": 3.3621244712207754, + "tokens_seen": 2560425984 + }, + { + "epoch": 0.55, + "learning_rate": 4.526560744663778e-05, + "loss": 2.4081, + "theoretical_loss": 3.36211080844997, + "tokens_seen": 2560557056 + }, + { + "epoch": 0.55, + "learning_rate": 4.525758305247954e-05, + "loss": 2.6559, + "theoretical_loss": 3.3620971465743437, + "tokens_seen": 2560688128 + }, + { + "epoch": 0.55, + "learning_rate": 4.52495586583213e-05, + "loss": 2.606, + "theoretical_loss": 3.3620834855937924, + "tokens_seen": 2560819200 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1403315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7906315326690674, + "objective/train/theoretical_loss": 3.362069825508211, + "objective/train/tokens_used": 931409376, + "theoretical_loss": 3.362069825508211, + "tokens_seen": 2560950272 + }, + { + "epoch": 0.55, + "learning_rate": 4.524153426416306e-05, + "loss": 2.6635, + "theoretical_loss": 3.362069825508211, + "tokens_seen": 2560950272 + }, + { + "epoch": 0.55, + "learning_rate": 4.523350987000482e-05, + "loss": 2.4842, + "theoretical_loss": 3.3620561663174957, + "tokens_seen": 2561081344 + }, + { + "epoch": 0.55, + "learning_rate": 4.522548547584658e-05, + "loss": 2.5513, + "theoretical_loss": 3.362042508021542, + "tokens_seen": 2561212416 + }, + { + "epoch": 0.55, + "learning_rate": 4.521746108168833e-05, + "loss": 2.6291, + "theoretical_loss": 3.3620288506202454, + "tokens_seen": 2561343488 + }, + { + "epoch": 0.55, + "learning_rate": 4.520943668753009e-05, + "loss": 2.6285, + "theoretical_loss": 3.3620151941135012, + "tokens_seen": 2561474560 + }, + { + "epoch": 0.55, + "learning_rate": 4.520141229337185e-05, + "loss": 2.5342, + "theoretical_loss": 3.3620015385012056, + "tokens_seen": 2561605632 + }, + { + "epoch": 0.55, + "learning_rate": 4.519338789921361e-05, + "loss": 2.6027, + "theoretical_loss": 3.3619878837832537, + "tokens_seen": 2561736704 + }, + { + "epoch": 0.55, + "learning_rate": 4.518536350505537e-05, + "loss": 2.6192, + "theoretical_loss": 3.361974229959541, + "tokens_seen": 2561867776 + }, + { + "epoch": 0.55, + "learning_rate": 4.517733911089713e-05, + "loss": 2.5033, + "theoretical_loss": 3.3619605770299645, + "tokens_seen": 2561998848 + }, + { + "epoch": 0.55, + "learning_rate": 4.516931471673888e-05, + "loss": 2.5271, + "theoretical_loss": 3.361946924994419, + "tokens_seen": 2562129920 + }, + { + "epoch": 0.55, + "learning_rate": 4.516129032258064e-05, + "loss": 2.3692, + "theoretical_loss": 3.3619332738527996, + "tokens_seen": 2562260992 + }, + { + "epoch": 0.55, + "learning_rate": 4.51532659284224e-05, + "loss": 2.5969, + "theoretical_loss": 3.3619196236050035, + "tokens_seen": 2562392064 + }, + { + "epoch": 0.55, + "learning_rate": 4.514524153426416e-05, + "loss": 2.5732, + "theoretical_loss": 3.361905974250925, + "tokens_seen": 2562523136 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1404572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8795578479766846, + "objective/train/theoretical_loss": 3.3618991499089974, + "objective/train/tokens_used": 933047776, + "theoretical_loss": 3.3618991499089974, + "tokens_seen": 2562588672 + }, + { + "epoch": 0.55, + "learning_rate": 4.513721714010592e-05, + "loss": 2.6146, + "theoretical_loss": 3.3618923257904605, + "tokens_seen": 2562654208 + }, + { + "epoch": 0.55, + "learning_rate": 4.512919274594768e-05, + "loss": 2.5252, + "theoretical_loss": 3.3618786782235066, + "tokens_seen": 2562785280 + }, + { + "epoch": 0.55, + "learning_rate": 4.512116835178944e-05, + "loss": 2.5647, + "theoretical_loss": 3.3618650315499576, + "tokens_seen": 2562916352 + }, + { + "epoch": 0.55, + "learning_rate": 4.5113143957631196e-05, + "loss": 2.5046, + "theoretical_loss": 3.3618513857697105, + "tokens_seen": 2563047424 + }, + { + "epoch": 0.55, + "learning_rate": 4.5105119563472956e-05, + "loss": 2.6008, + "theoretical_loss": 3.3618377408826605, + "tokens_seen": 2563178496 + }, + { + "epoch": 0.55, + "learning_rate": 4.5097095169314715e-05, + "loss": 2.4981, + "theoretical_loss": 3.3618240968887037, + "tokens_seen": 2563309568 + }, + { + "epoch": 0.55, + "learning_rate": 4.5089070775156475e-05, + "loss": 2.5218, + "theoretical_loss": 3.361810453787736, + "tokens_seen": 2563440640 + }, + { + "epoch": 0.55, + "learning_rate": 4.5081046380998235e-05, + "loss": 2.4183, + "theoretical_loss": 3.3617968115796533, + "tokens_seen": 2563571712 + }, + { + "epoch": 0.55, + "learning_rate": 4.5073021986839995e-05, + "loss": 2.5733, + "theoretical_loss": 3.3617831702643515, + "tokens_seen": 2563702784 + }, + { + "epoch": 0.55, + "learning_rate": 4.5064997592681755e-05, + "loss": 2.507, + "theoretical_loss": 3.3617695298417267, + "tokens_seen": 2563833856 + }, + { + "epoch": 0.55, + "learning_rate": 4.505697319852351e-05, + "loss": 2.6407, + "theoretical_loss": 3.361755890311674, + "tokens_seen": 2563964928 + }, + { + "epoch": 0.55, + "learning_rate": 4.504894880436527e-05, + "loss": 2.7645, + "theoretical_loss": 3.361742251674091, + "tokens_seen": 2564096000 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1405290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.719271659851074, + "objective/train/theoretical_loss": 3.361728613928872, + "objective/train/tokens_used": 934686176, + "theoretical_loss": 3.361728613928872, + "tokens_seen": 2564227072 + }, + { + "epoch": 0.55, + "learning_rate": 4.504092441020703e-05, + "loss": 2.5601, + "theoretical_loss": 3.361728613928872, + "tokens_seen": 2564227072 + }, + { + "epoch": 0.55, + "learning_rate": 4.503290001604879e-05, + "loss": 2.4802, + "theoretical_loss": 3.361714977075914, + "tokens_seen": 2564358144 + }, + { + "epoch": 0.55, + "learning_rate": 4.502487562189055e-05, + "loss": 2.6267, + "theoretical_loss": 3.3617013411151127, + "tokens_seen": 2564489216 + }, + { + "epoch": 0.55, + "learning_rate": 4.501685122773231e-05, + "loss": 2.431, + "theoretical_loss": 3.3616877060463644, + "tokens_seen": 2564620288 + }, + { + "epoch": 0.55, + "learning_rate": 4.500882683357407e-05, + "loss": 2.2944, + "theoretical_loss": 3.3616740718695644, + "tokens_seen": 2564751360 + }, + { + "epoch": 0.55, + "learning_rate": 4.500080243941583e-05, + "loss": 2.3951, + "theoretical_loss": 3.36166043858461, + "tokens_seen": 2564882432 + }, + { + "epoch": 0.55, + "learning_rate": 4.499277804525758e-05, + "loss": 2.5859, + "theoretical_loss": 3.361646806191396, + "tokens_seen": 2565013504 + }, + { + "epoch": 0.55, + "learning_rate": 4.498475365109934e-05, + "loss": 2.4016, + "theoretical_loss": 3.36163317468982, + "tokens_seen": 2565144576 + }, + { + "epoch": 0.55, + "learning_rate": 4.49767292569411e-05, + "loss": 2.3147, + "theoretical_loss": 3.3616195440797765, + "tokens_seen": 2565275648 + }, + { + "epoch": 0.55, + "learning_rate": 4.496870486278286e-05, + "loss": 2.4702, + "theoretical_loss": 3.3616059143611623, + "tokens_seen": 2565406720 + }, + { + "epoch": 0.55, + "learning_rate": 4.496068046862462e-05, + "loss": 2.4628, + "theoretical_loss": 3.3615922855338742, + "tokens_seen": 2565537792 + }, + { + "epoch": 0.55, + "learning_rate": 4.495265607446638e-05, + "loss": 2.5872, + "theoretical_loss": 3.3615786575978075, + "tokens_seen": 2565668864 + }, + { + "epoch": 0.56, + "learning_rate": 4.494463168030814e-05, + "loss": 2.4588, + "theoretical_loss": 3.3615650305528586, + "tokens_seen": 2565799936 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1405994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2271580696105957, + "objective/train/theoretical_loss": 3.361558217364521, + "objective/train/tokens_used": 936324576, + "theoretical_loss": 3.361558217364521, + "tokens_seen": 2565865472 + }, + { + "epoch": 0.56, + "learning_rate": 4.493660728614989e-05, + "loss": 2.3935, + "theoretical_loss": 3.361551404398924, + "tokens_seen": 2565931008 + }, + { + "epoch": 0.56, + "learning_rate": 4.492858289199165e-05, + "loss": 2.6389, + "theoretical_loss": 3.3615377791359, + "tokens_seen": 2566062080 + }, + { + "epoch": 0.56, + "learning_rate": 4.492055849783341e-05, + "loss": 2.5881, + "theoretical_loss": 3.361524154763682, + "tokens_seen": 2566193152 + }, + { + "epoch": 0.56, + "learning_rate": 4.491253410367517e-05, + "loss": 2.589, + "theoretical_loss": 3.3615105312821676, + "tokens_seen": 2566324224 + }, + { + "epoch": 0.56, + "learning_rate": 4.490450970951693e-05, + "loss": 2.5712, + "theoretical_loss": 3.3614969086912523, + "tokens_seen": 2566455296 + }, + { + "epoch": 0.56, + "learning_rate": 4.489648531535869e-05, + "loss": 2.7065, + "theoretical_loss": 3.361483286990832, + "tokens_seen": 2566586368 + }, + { + "epoch": 0.56, + "learning_rate": 4.488846092120045e-05, + "loss": 2.3785, + "theoretical_loss": 3.361469666180804, + "tokens_seen": 2566717440 + }, + { + "epoch": 0.56, + "learning_rate": 4.4880436527042206e-05, + "loss": 2.61, + "theoretical_loss": 3.361456046261064, + "tokens_seen": 2566848512 + }, + { + "epoch": 0.56, + "learning_rate": 4.4872412132883966e-05, + "loss": 2.6727, + "theoretical_loss": 3.3614424272315078, + "tokens_seen": 2566979584 + }, + { + "epoch": 0.56, + "learning_rate": 4.4864387738725726e-05, + "loss": 2.6467, + "theoretical_loss": 3.361428809092033, + "tokens_seen": 2567110656 + }, + { + "epoch": 0.56, + "learning_rate": 4.4856363344567486e-05, + "loss": 2.4898, + "theoretical_loss": 3.3614151918425357, + "tokens_seen": 2567241728 + }, + { + "epoch": 0.56, + "learning_rate": 4.4848338950409245e-05, + "loss": 2.5376, + "theoretical_loss": 3.3614015754829114, + "tokens_seen": 2567372800 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1406689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3093464374542236, + "objective/train/theoretical_loss": 3.361387960013057, + "objective/train/tokens_used": 937962976, + "theoretical_loss": 3.361387960013057, + "tokens_seen": 2567503872 + }, + { + "epoch": 0.56, + "learning_rate": 4.4840314556251005e-05, + "loss": 2.5131, + "theoretical_loss": 3.361387960013057, + "tokens_seen": 2567503872 + }, + { + "epoch": 0.56, + "learning_rate": 4.4832290162092765e-05, + "loss": 2.371, + "theoretical_loss": 3.36137434543287, + "tokens_seen": 2567634944 + }, + { + "epoch": 0.56, + "learning_rate": 4.4824265767934525e-05, + "loss": 2.5093, + "theoretical_loss": 3.3613607317422454, + "tokens_seen": 2567766016 + }, + { + "epoch": 0.56, + "learning_rate": 4.481624137377628e-05, + "loss": 2.4777, + "theoretical_loss": 3.3613471189410804, + "tokens_seen": 2567897088 + }, + { + "epoch": 0.56, + "learning_rate": 4.480821697961804e-05, + "loss": 2.6384, + "theoretical_loss": 3.361333507029271, + "tokens_seen": 2568028160 + }, + { + "epoch": 0.56, + "learning_rate": 4.48001925854598e-05, + "loss": 2.5472, + "theoretical_loss": 3.361319896006714, + "tokens_seen": 2568159232 + }, + { + "epoch": 0.56, + "learning_rate": 4.479216819130156e-05, + "loss": 2.4633, + "theoretical_loss": 3.3613062858733063, + "tokens_seen": 2568290304 + }, + { + "epoch": 0.56, + "learning_rate": 4.478414379714332e-05, + "loss": 2.5846, + "theoretical_loss": 3.361292676628944, + "tokens_seen": 2568421376 + }, + { + "epoch": 0.56, + "learning_rate": 4.477611940298508e-05, + "loss": 2.4566, + "theoretical_loss": 3.361279068273524, + "tokens_seen": 2568552448 + }, + { + "epoch": 0.56, + "learning_rate": 4.476809500882684e-05, + "loss": 2.4543, + "theoretical_loss": 3.3612654608069423, + "tokens_seen": 2568683520 + }, + { + "epoch": 0.56, + "learning_rate": 4.476007061466859e-05, + "loss": 2.4805, + "theoretical_loss": 3.3612518542290957, + "tokens_seen": 2568814592 + }, + { + "epoch": 0.56, + "learning_rate": 4.475204622051035e-05, + "loss": 2.5021, + "theoretical_loss": 3.361238248539881, + "tokens_seen": 2568945664 + }, + { + "epoch": 0.56, + "learning_rate": 4.474402182635211e-05, + "loss": 2.53, + "theoretical_loss": 3.361224643739195, + "tokens_seen": 2569076736 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1408046, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.762916088104248, + "objective/train/theoretical_loss": 3.3612178416720178, + "objective/train/tokens_used": 939601376, + "theoretical_loss": 3.3612178416720178, + "tokens_seen": 2569142272 + }, + { + "epoch": 0.56, + "learning_rate": 4.473599743219387e-05, + "loss": 2.4548, + "theoretical_loss": 3.3612110398269337, + "tokens_seen": 2569207808 + }, + { + "epoch": 0.56, + "learning_rate": 4.472797303803563e-05, + "loss": 2.5672, + "theoretical_loss": 3.3611974368029944, + "tokens_seen": 2569338880 + }, + { + "epoch": 0.56, + "learning_rate": 4.471994864387739e-05, + "loss": 2.548, + "theoretical_loss": 3.3611838346672736, + "tokens_seen": 2569469952 + }, + { + "epoch": 0.56, + "learning_rate": 4.471192424971915e-05, + "loss": 2.5908, + "theoretical_loss": 3.361170233419668, + "tokens_seen": 2569601024 + }, + { + "epoch": 0.56, + "learning_rate": 4.47038998555609e-05, + "loss": 2.4423, + "theoretical_loss": 3.361156633060074, + "tokens_seen": 2569732096 + }, + { + "epoch": 0.56, + "learning_rate": 4.469587546140266e-05, + "loss": 2.5423, + "theoretical_loss": 3.361143033588389, + "tokens_seen": 2569863168 + }, + { + "epoch": 0.56, + "learning_rate": 4.468785106724442e-05, + "loss": 2.552, + "theoretical_loss": 3.361129435004509, + "tokens_seen": 2569994240 + }, + { + "epoch": 0.56, + "learning_rate": 4.467982667308618e-05, + "loss": 2.5089, + "theoretical_loss": 3.3611158373083314, + "tokens_seen": 2570125312 + }, + { + "epoch": 0.56, + "learning_rate": 4.467180227892794e-05, + "loss": 2.4429, + "theoretical_loss": 3.3611022404997524, + "tokens_seen": 2570256384 + }, + { + "epoch": 0.56, + "learning_rate": 4.46637778847697e-05, + "loss": 2.5397, + "theoretical_loss": 3.361088644578669, + "tokens_seen": 2570387456 + }, + { + "epoch": 0.56, + "learning_rate": 4.465575349061146e-05, + "loss": 2.4347, + "theoretical_loss": 3.3610750495449784, + "tokens_seen": 2570518528 + }, + { + "epoch": 0.56, + "learning_rate": 4.464772909645322e-05, + "loss": 2.5479, + "theoretical_loss": 3.3610614553985774, + "tokens_seen": 2570649600 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1408629, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9207708835601807, + "objective/train/theoretical_loss": 3.3610478621393622, + "objective/train/tokens_used": 941239776, + "theoretical_loss": 3.3610478621393622, + "tokens_seen": 2570780672 + }, + { + "epoch": 0.56, + "learning_rate": 4.4639704702294976e-05, + "loss": 2.7436, + "theoretical_loss": 3.3610478621393622, + "tokens_seen": 2570780672 + }, + { + "epoch": 0.56, + "learning_rate": 4.4631680308136736e-05, + "loss": 2.4666, + "theoretical_loss": 3.3610342697672304, + "tokens_seen": 2570911744 + }, + { + "epoch": 0.56, + "learning_rate": 4.4623655913978496e-05, + "loss": 2.4001, + "theoretical_loss": 3.3610206782820784, + "tokens_seen": 2571042816 + }, + { + "epoch": 0.56, + "learning_rate": 4.4615631519820256e-05, + "loss": 2.5874, + "theoretical_loss": 3.361007087683803, + "tokens_seen": 2571173888 + }, + { + "epoch": 0.56, + "learning_rate": 4.4607607125662015e-05, + "loss": 2.4978, + "theoretical_loss": 3.3609934979723017, + "tokens_seen": 2571304960 + }, + { + "epoch": 0.56, + "learning_rate": 4.4599582731503775e-05, + "loss": 2.5588, + "theoretical_loss": 3.360979909147471, + "tokens_seen": 2571436032 + }, + { + "epoch": 0.56, + "learning_rate": 4.4591558337345535e-05, + "loss": 2.5659, + "theoretical_loss": 3.3609663212092085, + "tokens_seen": 2571567104 + }, + { + "epoch": 0.56, + "learning_rate": 4.458353394318729e-05, + "loss": 2.4862, + "theoretical_loss": 3.36095273415741, + "tokens_seen": 2571698176 + }, + { + "epoch": 0.56, + "learning_rate": 4.457550954902905e-05, + "loss": 2.5765, + "theoretical_loss": 3.3609391479919735, + "tokens_seen": 2571829248 + }, + { + "epoch": 0.56, + "learning_rate": 4.456748515487081e-05, + "loss": 2.4857, + "theoretical_loss": 3.360925562712796, + "tokens_seen": 2571960320 + }, + { + "epoch": 0.56, + "learning_rate": 4.455946076071257e-05, + "loss": 2.4668, + "theoretical_loss": 3.3609119783197734, + "tokens_seen": 2572091392 + }, + { + "epoch": 0.56, + "learning_rate": 4.455143636655433e-05, + "loss": 2.5625, + "theoretical_loss": 3.360898394812804, + "tokens_seen": 2572222464 + }, + { + "epoch": 0.56, + "learning_rate": 4.454341197239609e-05, + "loss": 2.5671, + "theoretical_loss": 3.3608848121917845, + "tokens_seen": 2572353536 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1409650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6852149963378906, + "objective/train/theoretical_loss": 3.3608780212134737, + "objective/train/tokens_used": 942878176, + "theoretical_loss": 3.3608780212134737, + "tokens_seen": 2572419072 + }, + { + "epoch": 0.56, + "learning_rate": 4.453538757823785e-05, + "loss": 2.667, + "theoretical_loss": 3.3608712304566115, + "tokens_seen": 2572484608 + }, + { + "epoch": 0.56, + "learning_rate": 4.452736318407961e-05, + "loss": 2.4249, + "theoretical_loss": 3.360857649607183, + "tokens_seen": 2572615680 + }, + { + "epoch": 0.56, + "learning_rate": 4.451933878992136e-05, + "loss": 2.5772, + "theoretical_loss": 3.360844069643395, + "tokens_seen": 2572746752 + }, + { + "epoch": 0.56, + "learning_rate": 4.451131439576312e-05, + "loss": 2.7505, + "theoretical_loss": 3.360830490565146, + "tokens_seen": 2572877824 + }, + { + "epoch": 0.56, + "learning_rate": 4.450329000160488e-05, + "loss": 2.5656, + "theoretical_loss": 3.3608169123723317, + "tokens_seen": 2573008896 + }, + { + "epoch": 0.56, + "learning_rate": 4.449526560744664e-05, + "loss": 2.5588, + "theoretical_loss": 3.3608033350648503, + "tokens_seen": 2573139968 + }, + { + "epoch": 0.56, + "learning_rate": 4.44872412132884e-05, + "loss": 2.4582, + "theoretical_loss": 3.360789758642598, + "tokens_seen": 2573271040 + }, + { + "epoch": 0.56, + "learning_rate": 4.447921681913016e-05, + "loss": 2.5461, + "theoretical_loss": 3.3607761831054734, + "tokens_seen": 2573402112 + }, + { + "epoch": 0.56, + "learning_rate": 4.447119242497192e-05, + "loss": 2.5004, + "theoretical_loss": 3.3607626084533724, + "tokens_seen": 2573533184 + }, + { + "epoch": 0.56, + "learning_rate": 4.4463168030813673e-05, + "loss": 2.5159, + "theoretical_loss": 3.360749034686193, + "tokens_seen": 2573664256 + }, + { + "epoch": 0.56, + "learning_rate": 4.445514363665543e-05, + "loss": 2.577, + "theoretical_loss": 3.360735461803832, + "tokens_seen": 2573795328 + }, + { + "epoch": 0.56, + "learning_rate": 4.444711924249719e-05, + "loss": 2.6183, + "theoretical_loss": 3.360721889806187, + "tokens_seen": 2573926400 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1410556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3823792934417725, + "objective/train/theoretical_loss": 3.3607083186931552, + "objective/train/tokens_used": 944516576, + "theoretical_loss": 3.3607083186931552, + "tokens_seen": 2574057472 + }, + { + "epoch": 0.56, + "learning_rate": 4.443909484833895e-05, + "loss": 2.5098, + "theoretical_loss": 3.3607083186931552, + "tokens_seen": 2574057472 + }, + { + "epoch": 0.56, + "learning_rate": 4.443107045418071e-05, + "loss": 2.5404, + "theoretical_loss": 3.360694748464634, + "tokens_seen": 2574188544 + }, + { + "epoch": 0.56, + "learning_rate": 4.442304606002247e-05, + "loss": 2.4381, + "theoretical_loss": 3.3606811791205202, + "tokens_seen": 2574319616 + }, + { + "epoch": 0.56, + "learning_rate": 4.441502166586423e-05, + "loss": 2.5789, + "theoretical_loss": 3.3606676106607116, + "tokens_seen": 2574450688 + }, + { + "epoch": 0.56, + "learning_rate": 4.4406997271705986e-05, + "loss": 2.6221, + "theoretical_loss": 3.3606540430851055, + "tokens_seen": 2574581760 + }, + { + "epoch": 0.56, + "learning_rate": 4.4398972877547746e-05, + "loss": 2.4964, + "theoretical_loss": 3.3606404763935993, + "tokens_seen": 2574712832 + }, + { + "epoch": 0.56, + "learning_rate": 4.4390948483389506e-05, + "loss": 2.4961, + "theoretical_loss": 3.3606269105860904, + "tokens_seen": 2574843904 + }, + { + "epoch": 0.56, + "learning_rate": 4.4382924089231266e-05, + "loss": 2.295, + "theoretical_loss": 3.3606133456624763, + "tokens_seen": 2574974976 + }, + { + "epoch": 0.56, + "learning_rate": 4.4374899695073026e-05, + "loss": 2.5428, + "theoretical_loss": 3.3605997816226534, + "tokens_seen": 2575106048 + }, + { + "epoch": 0.56, + "learning_rate": 4.4366875300914786e-05, + "loss": 2.6581, + "theoretical_loss": 3.360586218466521, + "tokens_seen": 2575237120 + }, + { + "epoch": 0.56, + "learning_rate": 4.4358850906756545e-05, + "loss": 2.4116, + "theoretical_loss": 3.360572656193975, + "tokens_seen": 2575368192 + }, + { + "epoch": 0.56, + "learning_rate": 4.4350826512598305e-05, + "loss": 2.4559, + "theoretical_loss": 3.3605590948049135, + "tokens_seen": 2575499264 + }, + { + "epoch": 0.56, + "learning_rate": 4.434280211844006e-05, + "loss": 2.4946, + "theoretical_loss": 3.360545534299234, + "tokens_seen": 2575630336 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1411184, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4256932735443115, + "objective/train/theoretical_loss": 3.36053875437763, + "objective/train/tokens_used": 946154976, + "theoretical_loss": 3.36053875437763, + "tokens_seen": 2575695872 + }, + { + "epoch": 0.56, + "learning_rate": 4.433477772428182e-05, + "loss": 2.5491, + "theoretical_loss": 3.3605319746768334, + "tokens_seen": 2575761408 + }, + { + "epoch": 0.56, + "learning_rate": 4.432675333012358e-05, + "loss": 2.6144, + "theoretical_loss": 3.3605184159376105, + "tokens_seen": 2575892480 + }, + { + "epoch": 0.56, + "learning_rate": 4.431872893596534e-05, + "loss": 2.5346, + "theoretical_loss": 3.3605048580814616, + "tokens_seen": 2576023552 + }, + { + "epoch": 0.56, + "learning_rate": 4.43107045418071e-05, + "loss": 2.5056, + "theoretical_loss": 3.3604913011082846, + "tokens_seen": 2576154624 + }, + { + "epoch": 0.56, + "learning_rate": 4.430268014764886e-05, + "loss": 2.5873, + "theoretical_loss": 3.3604777450179775, + "tokens_seen": 2576285696 + }, + { + "epoch": 0.56, + "learning_rate": 4.429465575349062e-05, + "loss": 2.536, + "theoretical_loss": 3.360464189810438, + "tokens_seen": 2576416768 + }, + { + "epoch": 0.56, + "learning_rate": 4.428663135933237e-05, + "loss": 2.5434, + "theoretical_loss": 3.3604506354855626, + "tokens_seen": 2576547840 + }, + { + "epoch": 0.56, + "learning_rate": 4.427860696517413e-05, + "loss": 2.6844, + "theoretical_loss": 3.3604370820432496, + "tokens_seen": 2576678912 + }, + { + "epoch": 0.56, + "learning_rate": 4.427058257101589e-05, + "loss": 2.748, + "theoretical_loss": 3.360423529483397, + "tokens_seen": 2576809984 + }, + { + "epoch": 0.56, + "learning_rate": 4.426255817685765e-05, + "loss": 2.6688, + "theoretical_loss": 3.3604099778059022, + "tokens_seen": 2576941056 + }, + { + "epoch": 0.56, + "learning_rate": 4.425453378269941e-05, + "loss": 2.4215, + "theoretical_loss": 3.3603964270106625, + "tokens_seen": 2577072128 + }, + { + "epoch": 0.56, + "learning_rate": 4.424650938854117e-05, + "loss": 2.6003, + "theoretical_loss": 3.3603828770975763, + "tokens_seen": 2577203200 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1412403, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7557308673858643, + "objective/train/theoretical_loss": 3.3603693280665405, + "objective/train/tokens_used": 947793376, + "theoretical_loss": 3.3603693280665405, + "tokens_seen": 2577334272 + }, + { + "epoch": 0.56, + "learning_rate": 4.423848499438293e-05, + "loss": 2.5315, + "theoretical_loss": 3.3603693280665405, + "tokens_seen": 2577334272 + }, + { + "epoch": 0.56, + "learning_rate": 4.4230460600224684e-05, + "loss": 2.4856, + "theoretical_loss": 3.3603557799174535, + "tokens_seen": 2577465344 + }, + { + "epoch": 0.56, + "learning_rate": 4.4222436206066444e-05, + "loss": 2.4088, + "theoretical_loss": 3.360342232650213, + "tokens_seen": 2577596416 + }, + { + "epoch": 0.56, + "learning_rate": 4.4214411811908203e-05, + "loss": 2.5578, + "theoretical_loss": 3.360328686264716, + "tokens_seen": 2577727488 + }, + { + "epoch": 0.56, + "learning_rate": 4.420638741774996e-05, + "loss": 2.5137, + "theoretical_loss": 3.3603151407608616, + "tokens_seen": 2577858560 + }, + { + "epoch": 0.56, + "learning_rate": 4.419836302359172e-05, + "loss": 2.6998, + "theoretical_loss": 3.3603015961385463, + "tokens_seen": 2577989632 + }, + { + "epoch": 0.56, + "learning_rate": 4.419033862943348e-05, + "loss": 2.4891, + "theoretical_loss": 3.3602880523976686, + "tokens_seen": 2578120704 + }, + { + "epoch": 0.56, + "learning_rate": 4.418231423527524e-05, + "loss": 2.71, + "theoretical_loss": 3.360274509538126, + "tokens_seen": 2578251776 + }, + { + "epoch": 0.56, + "learning_rate": 4.4174289841117e-05, + "loss": 2.6158, + "theoretical_loss": 3.3602609675598165, + "tokens_seen": 2578382848 + }, + { + "epoch": 0.56, + "learning_rate": 4.4166265446958756e-05, + "loss": 2.5242, + "theoretical_loss": 3.3602474264626383, + "tokens_seen": 2578513920 + }, + { + "epoch": 0.56, + "learning_rate": 4.4158241052800516e-05, + "loss": 2.4932, + "theoretical_loss": 3.3602338862464887, + "tokens_seen": 2578644992 + }, + { + "epoch": 0.56, + "learning_rate": 4.4150216658642276e-05, + "loss": 2.6617, + "theoretical_loss": 3.360220346911266, + "tokens_seen": 2578776064 + }, + { + "epoch": 0.56, + "learning_rate": 4.4142192264484036e-05, + "loss": 2.4997, + "theoretical_loss": 3.3602068084568684, + "tokens_seen": 2578907136 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1412775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.234215259552002, + "objective/train/theoretical_loss": 3.3602000395599463, + "objective/train/tokens_used": 949431776, + "theoretical_loss": 3.3602000395599463, + "tokens_seen": 2578972672 + }, + { + "epoch": 0.56, + "learning_rate": 4.4134167870325796e-05, + "loss": 2.5342, + "theoretical_loss": 3.360193270883193, + "tokens_seen": 2579038208 + }, + { + "epoch": 0.56, + "learning_rate": 4.4126143476167556e-05, + "loss": 2.564, + "theoretical_loss": 3.360179734190138, + "tokens_seen": 2579169280 + }, + { + "epoch": 0.56, + "learning_rate": 4.4118119082009316e-05, + "loss": 2.6898, + "theoretical_loss": 3.360166198377602, + "tokens_seen": 2579300352 + }, + { + "epoch": 0.56, + "learning_rate": 4.411009468785107e-05, + "loss": 2.4675, + "theoretical_loss": 3.360152663445482, + "tokens_seen": 2579431424 + }, + { + "epoch": 0.56, + "learning_rate": 4.410207029369283e-05, + "loss": 2.5835, + "theoretical_loss": 3.3601391293936764, + "tokens_seen": 2579562496 + }, + { + "epoch": 0.56, + "learning_rate": 4.409404589953459e-05, + "loss": 2.5584, + "theoretical_loss": 3.360125596222084, + "tokens_seen": 2579693568 + }, + { + "epoch": 0.56, + "learning_rate": 4.408602150537635e-05, + "loss": 2.5646, + "theoretical_loss": 3.360112063930602, + "tokens_seen": 2579824640 + }, + { + "epoch": 0.56, + "learning_rate": 4.407799711121811e-05, + "loss": 2.6259, + "theoretical_loss": 3.3600985325191286, + "tokens_seen": 2579955712 + }, + { + "epoch": 0.56, + "learning_rate": 4.406997271705987e-05, + "loss": 2.5303, + "theoretical_loss": 3.360085001987562, + "tokens_seen": 2580086784 + }, + { + "epoch": 0.56, + "learning_rate": 4.406194832290163e-05, + "loss": 2.7238, + "theoretical_loss": 3.3600714723358, + "tokens_seen": 2580217856 + }, + { + "epoch": 0.56, + "learning_rate": 4.405392392874339e-05, + "loss": 2.5707, + "theoretical_loss": 3.3600579435637408, + "tokens_seen": 2580348928 + }, + { + "epoch": 0.56, + "learning_rate": 4.404589953458514e-05, + "loss": 2.7141, + "theoretical_loss": 3.3600444156712825, + "tokens_seen": 2580480000 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1413702, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.833010673522949, + "objective/train/theoretical_loss": 3.3600308886583234, + "objective/train/tokens_used": 951070176, + "theoretical_loss": 3.3600308886583234, + "tokens_seen": 2580611072 + }, + { + "epoch": 0.56, + "learning_rate": 4.40378751404269e-05, + "loss": 2.6625, + "theoretical_loss": 3.3600308886583234, + "tokens_seen": 2580611072 + }, + { + "epoch": 0.56, + "learning_rate": 4.402985074626866e-05, + "loss": 2.4369, + "theoretical_loss": 3.3600173625247622, + "tokens_seen": 2580742144 + }, + { + "epoch": 0.56, + "learning_rate": 4.402182635211042e-05, + "loss": 2.6426, + "theoretical_loss": 3.360003837270496, + "tokens_seen": 2580873216 + }, + { + "epoch": 0.56, + "learning_rate": 4.401380195795218e-05, + "loss": 2.4289, + "theoretical_loss": 3.359990312895423, + "tokens_seen": 2581004288 + }, + { + "epoch": 0.56, + "learning_rate": 4.400577756379394e-05, + "loss": 2.5973, + "theoretical_loss": 3.3599767893994428, + "tokens_seen": 2581135360 + }, + { + "epoch": 0.56, + "learning_rate": 4.3997753169635694e-05, + "loss": 2.4835, + "theoretical_loss": 3.359963266782452, + "tokens_seen": 2581266432 + }, + { + "epoch": 0.56, + "learning_rate": 4.3989728775477454e-05, + "loss": 2.5595, + "theoretical_loss": 3.35994974504435, + "tokens_seen": 2581397504 + }, + { + "epoch": 0.56, + "learning_rate": 4.3981704381319214e-05, + "loss": 2.6413, + "theoretical_loss": 3.3599362241850343, + "tokens_seen": 2581528576 + }, + { + "epoch": 0.56, + "learning_rate": 4.3973679987160974e-05, + "loss": 2.5503, + "theoretical_loss": 3.3599227042044033, + "tokens_seen": 2581659648 + }, + { + "epoch": 0.56, + "learning_rate": 4.3965655593002733e-05, + "loss": 2.5183, + "theoretical_loss": 3.3599091851023557, + "tokens_seen": 2581790720 + }, + { + "epoch": 0.56, + "learning_rate": 4.395763119884449e-05, + "loss": 2.4992, + "theoretical_loss": 3.35989566687879, + "tokens_seen": 2581921792 + }, + { + "epoch": 0.56, + "learning_rate": 4.3949606804686246e-05, + "loss": 2.5265, + "theoretical_loss": 3.359882149533603, + "tokens_seen": 2582052864 + }, + { + "epoch": 0.56, + "learning_rate": 4.3941582410528006e-05, + "loss": 2.6401, + "theoretical_loss": 3.359868633066695, + "tokens_seen": 2582183936 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1414384, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.684948682785034, + "objective/train/theoretical_loss": 3.3598618751625633, + "objective/train/tokens_used": 952708576, + "theoretical_loss": 3.3598618751625633, + "tokens_seen": 2582249472 + }, + { + "epoch": 0.57, + "learning_rate": 4.3933558016369766e-05, + "loss": 2.6089, + "theoretical_loss": 3.359855117477963, + "tokens_seen": 2582315008 + }, + { + "epoch": 0.57, + "learning_rate": 4.3925533622211526e-05, + "loss": 2.366, + "theoretical_loss": 3.359841602767306, + "tokens_seen": 2582446080 + }, + { + "epoch": 0.57, + "learning_rate": 4.3917509228053286e-05, + "loss": 2.4965, + "theoretical_loss": 3.3598280889346226, + "tokens_seen": 2582577152 + }, + { + "epoch": 0.57, + "learning_rate": 4.3909484833895046e-05, + "loss": 2.5821, + "theoretical_loss": 3.3598145759798106, + "tokens_seen": 2582708224 + }, + { + "epoch": 0.57, + "learning_rate": 4.39014604397368e-05, + "loss": 2.6362, + "theoretical_loss": 3.3598010639027684, + "tokens_seen": 2582839296 + }, + { + "epoch": 0.57, + "learning_rate": 4.389343604557856e-05, + "loss": 2.6019, + "theoretical_loss": 3.359787552703395, + "tokens_seen": 2582970368 + }, + { + "epoch": 0.57, + "learning_rate": 4.388541165142032e-05, + "loss": 2.5663, + "theoretical_loss": 3.3597740423815883, + "tokens_seen": 2583101440 + }, + { + "epoch": 0.57, + "learning_rate": 4.387738725726208e-05, + "loss": 2.5881, + "theoretical_loss": 3.3597605329372473, + "tokens_seen": 2583232512 + }, + { + "epoch": 0.57, + "learning_rate": 4.386936286310384e-05, + "loss": 2.6147, + "theoretical_loss": 3.3597470243702703, + "tokens_seen": 2583363584 + }, + { + "epoch": 0.57, + "learning_rate": 4.38613384689456e-05, + "loss": 2.4081, + "theoretical_loss": 3.3597335166805555, + "tokens_seen": 2583494656 + }, + { + "epoch": 0.57, + "learning_rate": 4.385331407478735e-05, + "loss": 2.5283, + "theoretical_loss": 3.359720009868002, + "tokens_seen": 2583625728 + }, + { + "epoch": 0.57, + "learning_rate": 4.384528968062911e-05, + "loss": 2.4299, + "theoretical_loss": 3.3597065039325074, + "tokens_seen": 2583756800 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1415698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.223771810531616, + "objective/train/theoretical_loss": 3.3596929988739714, + "objective/train/tokens_used": 954346976, + "theoretical_loss": 3.3596929988739714, + "tokens_seen": 2583887872 + }, + { + "epoch": 0.57, + "learning_rate": 4.383726528647087e-05, + "loss": 2.4919, + "theoretical_loss": 3.3596929988739714, + "tokens_seen": 2583887872 + }, + { + "epoch": 0.57, + "learning_rate": 4.382924089231263e-05, + "loss": 2.638, + "theoretical_loss": 3.3596794946922923, + "tokens_seen": 2584018944 + }, + { + "epoch": 0.57, + "learning_rate": 4.382121649815439e-05, + "loss": 2.598, + "theoretical_loss": 3.359665991387368, + "tokens_seen": 2584150016 + }, + { + "epoch": 0.57, + "learning_rate": 4.381319210399615e-05, + "loss": 2.5097, + "theoretical_loss": 3.3596524889590977, + "tokens_seen": 2584281088 + }, + { + "epoch": 0.57, + "learning_rate": 4.3805167709837904e-05, + "loss": 2.5365, + "theoretical_loss": 3.3596389874073798, + "tokens_seen": 2584412160 + }, + { + "epoch": 0.57, + "learning_rate": 4.3797143315679664e-05, + "loss": 2.4864, + "theoretical_loss": 3.359625486732113, + "tokens_seen": 2584543232 + }, + { + "epoch": 0.57, + "learning_rate": 4.3789118921521424e-05, + "loss": 2.51, + "theoretical_loss": 3.359611986933196, + "tokens_seen": 2584674304 + }, + { + "epoch": 0.57, + "learning_rate": 4.3781094527363184e-05, + "loss": 2.5867, + "theoretical_loss": 3.3595984880105276, + "tokens_seen": 2584805376 + }, + { + "epoch": 0.57, + "learning_rate": 4.3773070133204944e-05, + "loss": 2.4277, + "theoretical_loss": 3.3595849899640067, + "tokens_seen": 2584936448 + }, + { + "epoch": 0.57, + "learning_rate": 4.3765045739046704e-05, + "loss": 2.6637, + "theoretical_loss": 3.359571492793531, + "tokens_seen": 2585067520 + }, + { + "epoch": 0.57, + "learning_rate": 4.375702134488846e-05, + "loss": 2.4325, + "theoretical_loss": 3.3595579964990003, + "tokens_seen": 2585198592 + }, + { + "epoch": 0.57, + "learning_rate": 4.374899695073022e-05, + "loss": 2.6014, + "theoretical_loss": 3.359544501080313, + "tokens_seen": 2585329664 + }, + { + "epoch": 0.57, + "learning_rate": 4.374097255657198e-05, + "loss": 2.5262, + "theoretical_loss": 3.3595310065373676, + "tokens_seen": 2585460736 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1416170, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.412297010421753, + "objective/train/theoretical_loss": 3.3595242595942665, + "objective/train/tokens_used": 955985376, + "theoretical_loss": 3.3595242595942665, + "tokens_seen": 2585526272 + }, + { + "epoch": 0.57, + "learning_rate": 4.373294816241374e-05, + "loss": 2.5099, + "theoretical_loss": 3.359517512870063, + "tokens_seen": 2585591808 + }, + { + "epoch": 0.57, + "learning_rate": 4.37249237682555e-05, + "loss": 2.5199, + "theoretical_loss": 3.3595040200782984, + "tokens_seen": 2585722880 + }, + { + "epoch": 0.57, + "learning_rate": 4.371689937409726e-05, + "loss": 2.4225, + "theoretical_loss": 3.3594905281619725, + "tokens_seen": 2585853952 + }, + { + "epoch": 0.57, + "learning_rate": 4.3708874979939017e-05, + "loss": 2.3979, + "theoretical_loss": 3.3594770371209837, + "tokens_seen": 2585985024 + }, + { + "epoch": 0.57, + "learning_rate": 4.3700850585780776e-05, + "loss": 2.5816, + "theoretical_loss": 3.359463546955231, + "tokens_seen": 2586116096 + }, + { + "epoch": 0.57, + "learning_rate": 4.369282619162253e-05, + "loss": 2.4065, + "theoretical_loss": 3.3594500576646134, + "tokens_seen": 2586247168 + }, + { + "epoch": 0.57, + "learning_rate": 4.368480179746429e-05, + "loss": 2.4668, + "theoretical_loss": 3.35943656924903, + "tokens_seen": 2586378240 + }, + { + "epoch": 0.57, + "learning_rate": 4.367677740330605e-05, + "loss": 2.5211, + "theoretical_loss": 3.3594230817083788, + "tokens_seen": 2586509312 + }, + { + "epoch": 0.57, + "learning_rate": 4.366875300914781e-05, + "loss": 2.5312, + "theoretical_loss": 3.3594095950425595, + "tokens_seen": 2586640384 + }, + { + "epoch": 0.57, + "learning_rate": 4.366072861498957e-05, + "loss": 2.3878, + "theoretical_loss": 3.3593961092514713, + "tokens_seen": 2586771456 + }, + { + "epoch": 0.57, + "learning_rate": 4.365270422083133e-05, + "loss": 2.4908, + "theoretical_loss": 3.3593826243350127, + "tokens_seen": 2586902528 + }, + { + "epoch": 0.57, + "learning_rate": 4.364467982667309e-05, + "loss": 2.6214, + "theoretical_loss": 3.359369140293082, + "tokens_seen": 2587033600 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1417329, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5255184173583984, + "objective/train/theoretical_loss": 3.3593556571255796, + "objective/train/tokens_used": 957623776, + "theoretical_loss": 3.3593556571255796, + "tokens_seen": 2587164672 + }, + { + "epoch": 0.57, + "learning_rate": 4.363665543251484e-05, + "loss": 2.6045, + "theoretical_loss": 3.3593556571255796, + "tokens_seen": 2587164672 + }, + { + "epoch": 0.57, + "learning_rate": 4.36286310383566e-05, + "loss": 2.4828, + "theoretical_loss": 3.3593421748324035, + "tokens_seen": 2587295744 + }, + { + "epoch": 0.57, + "learning_rate": 4.362060664419836e-05, + "loss": 2.5751, + "theoretical_loss": 3.359328693413453, + "tokens_seen": 2587426816 + }, + { + "epoch": 0.57, + "learning_rate": 4.361258225004012e-05, + "loss": 2.4211, + "theoretical_loss": 3.359315212868627, + "tokens_seen": 2587557888 + }, + { + "epoch": 0.57, + "learning_rate": 4.360455785588188e-05, + "loss": 2.6507, + "theoretical_loss": 3.359301733197825, + "tokens_seen": 2587688960 + }, + { + "epoch": 0.57, + "learning_rate": 4.359653346172364e-05, + "loss": 2.5545, + "theoretical_loss": 3.3592882544009455, + "tokens_seen": 2587820032 + }, + { + "epoch": 0.57, + "learning_rate": 4.35885090675654e-05, + "loss": 2.5015, + "theoretical_loss": 3.3592747764778874, + "tokens_seen": 2587951104 + }, + { + "epoch": 0.57, + "learning_rate": 4.358048467340716e-05, + "loss": 2.5675, + "theoretical_loss": 3.3592612994285505, + "tokens_seen": 2588082176 + }, + { + "epoch": 0.57, + "learning_rate": 4.3572460279248915e-05, + "loss": 2.5841, + "theoretical_loss": 3.3592478232528338, + "tokens_seen": 2588213248 + }, + { + "epoch": 0.57, + "learning_rate": 4.3564435885090675e-05, + "loss": 2.5749, + "theoretical_loss": 3.359234347950636, + "tokens_seen": 2588344320 + }, + { + "epoch": 0.57, + "learning_rate": 4.3556411490932434e-05, + "loss": 2.5033, + "theoretical_loss": 3.3592208735218567, + "tokens_seen": 2588475392 + }, + { + "epoch": 0.57, + "learning_rate": 4.3548387096774194e-05, + "loss": 2.4545, + "theoretical_loss": 3.3592073999663947, + "tokens_seen": 2588606464 + }, + { + "epoch": 0.57, + "learning_rate": 4.3540362702615954e-05, + "loss": 2.6047, + "theoretical_loss": 3.3591939272841493, + "tokens_seen": 2588737536 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1417937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3127598762512207, + "objective/train/theoretical_loss": 3.359187191270452, + "objective/train/tokens_used": 959262176, + "theoretical_loss": 3.359187191270452, + "tokens_seen": 2588803072 + }, + { + "epoch": 0.57, + "learning_rate": 4.3532338308457714e-05, + "loss": 2.472, + "theoretical_loss": 3.35918045547502, + "tokens_seen": 2588868608 + }, + { + "epoch": 0.57, + "learning_rate": 4.3524313914299474e-05, + "loss": 2.6133, + "theoretical_loss": 3.359166984538906, + "tokens_seen": 2588999680 + }, + { + "epoch": 0.57, + "learning_rate": 4.351628952014123e-05, + "loss": 2.2738, + "theoretical_loss": 3.3591535144757056, + "tokens_seen": 2589130752 + }, + { + "epoch": 0.57, + "learning_rate": 4.350826512598299e-05, + "loss": 2.6542, + "theoretical_loss": 3.3591400452853195, + "tokens_seen": 2589261824 + }, + { + "epoch": 0.57, + "learning_rate": 4.350024073182475e-05, + "loss": 2.4462, + "theoretical_loss": 3.3591265769676455, + "tokens_seen": 2589392896 + }, + { + "epoch": 0.57, + "learning_rate": 4.349221633766651e-05, + "loss": 2.5233, + "theoretical_loss": 3.359113109522584, + "tokens_seen": 2589523968 + }, + { + "epoch": 0.57, + "learning_rate": 4.348419194350827e-05, + "loss": 2.5961, + "theoretical_loss": 3.3590996429500337, + "tokens_seen": 2589655040 + }, + { + "epoch": 0.57, + "learning_rate": 4.347616754935003e-05, + "loss": 2.3866, + "theoretical_loss": 3.3590861772498943, + "tokens_seen": 2589786112 + }, + { + "epoch": 0.57, + "learning_rate": 4.3468143155191787e-05, + "loss": 2.6735, + "theoretical_loss": 3.3590727124220647, + "tokens_seen": 2589917184 + }, + { + "epoch": 0.57, + "learning_rate": 4.346011876103354e-05, + "loss": 2.5495, + "theoretical_loss": 3.3590592484664445, + "tokens_seen": 2590048256 + }, + { + "epoch": 0.57, + "learning_rate": 4.34520943668753e-05, + "loss": 2.5255, + "theoretical_loss": 3.359045785382933, + "tokens_seen": 2590179328 + }, + { + "epoch": 0.57, + "learning_rate": 4.344406997271706e-05, + "loss": 2.5271, + "theoretical_loss": 3.35903232317143, + "tokens_seen": 2590310400 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1418406, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.977363109588623, + "objective/train/theoretical_loss": 3.359018861831834, + "objective/train/tokens_used": 960900576, + "theoretical_loss": 3.359018861831834, + "tokens_seen": 2590441472 + }, + { + "epoch": 0.57, + "learning_rate": 4.343604557855882e-05, + "loss": 2.6362, + "theoretical_loss": 3.359018861831834, + "tokens_seen": 2590441472 + }, + { + "epoch": 0.57, + "learning_rate": 4.342802118440058e-05, + "loss": 2.4466, + "theoretical_loss": 3.359005401364046, + "tokens_seen": 2590572544 + }, + { + "epoch": 0.57, + "learning_rate": 4.341999679024234e-05, + "loss": 2.5624, + "theoretical_loss": 3.3589919417679632, + "tokens_seen": 2590703616 + }, + { + "epoch": 0.57, + "learning_rate": 4.34119723960841e-05, + "loss": 2.4344, + "theoretical_loss": 3.3589784830434866, + "tokens_seen": 2590834688 + }, + { + "epoch": 0.57, + "learning_rate": 4.340394800192586e-05, + "loss": 2.6116, + "theoretical_loss": 3.3589650251905154, + "tokens_seen": 2590965760 + }, + { + "epoch": 0.57, + "learning_rate": 4.339592360776761e-05, + "loss": 2.5694, + "theoretical_loss": 3.358951568208949, + "tokens_seen": 2591096832 + }, + { + "epoch": 0.57, + "learning_rate": 4.338789921360937e-05, + "loss": 2.5525, + "theoretical_loss": 3.358938112098687, + "tokens_seen": 2591227904 + }, + { + "epoch": 0.57, + "learning_rate": 4.337987481945113e-05, + "loss": 2.5562, + "theoretical_loss": 3.358924656859629, + "tokens_seen": 2591358976 + }, + { + "epoch": 0.57, + "learning_rate": 4.337185042529289e-05, + "loss": 2.3901, + "theoretical_loss": 3.3589112024916736, + "tokens_seen": 2591490048 + }, + { + "epoch": 0.57, + "learning_rate": 4.336382603113465e-05, + "loss": 2.3919, + "theoretical_loss": 3.3588977489947216, + "tokens_seen": 2591621120 + }, + { + "epoch": 0.57, + "learning_rate": 4.335580163697641e-05, + "loss": 2.5913, + "theoretical_loss": 3.358884296368672, + "tokens_seen": 2591752192 + }, + { + "epoch": 0.57, + "learning_rate": 4.334777724281817e-05, + "loss": 2.5002, + "theoretical_loss": 3.358870844613424, + "tokens_seen": 2591883264 + }, + { + "epoch": 0.57, + "learning_rate": 4.3339752848659925e-05, + "loss": 2.4822, + "theoretical_loss": 3.3588573937288784, + "tokens_seen": 2592014336 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1419620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6387507915496826, + "objective/train/theoretical_loss": 3.358850668613087, + "objective/train/tokens_used": 962538976, + "theoretical_loss": 3.358850668613087, + "tokens_seen": 2592079872 + }, + { + "epoch": 0.57, + "learning_rate": 4.3331728454501685e-05, + "loss": 2.5499, + "theoretical_loss": 3.3588439437149336, + "tokens_seen": 2592145408 + }, + { + "epoch": 0.57, + "learning_rate": 4.3323704060343445e-05, + "loss": 2.481, + "theoretical_loss": 3.3588304945714897, + "tokens_seen": 2592276480 + }, + { + "epoch": 0.57, + "learning_rate": 4.3315679666185204e-05, + "loss": 2.4989, + "theoretical_loss": 3.3588170462984466, + "tokens_seen": 2592407552 + }, + { + "epoch": 0.57, + "learning_rate": 4.3307655272026964e-05, + "loss": 2.61, + "theoretical_loss": 3.3588035988957032, + "tokens_seen": 2592538624 + }, + { + "epoch": 0.57, + "learning_rate": 4.3299630877868724e-05, + "loss": 2.4469, + "theoretical_loss": 3.3587901523631603, + "tokens_seen": 2592669696 + }, + { + "epoch": 0.57, + "learning_rate": 4.3291606483710484e-05, + "loss": 2.5985, + "theoretical_loss": 3.3587767067007164, + "tokens_seen": 2592800768 + }, + { + "epoch": 0.57, + "learning_rate": 4.328358208955224e-05, + "loss": 2.4818, + "theoretical_loss": 3.358763261908272, + "tokens_seen": 2592931840 + }, + { + "epoch": 0.57, + "learning_rate": 4.3275557695394e-05, + "loss": 2.5826, + "theoretical_loss": 3.3587498179857267, + "tokens_seen": 2593062912 + }, + { + "epoch": 0.57, + "learning_rate": 4.326753330123576e-05, + "loss": 2.4678, + "theoretical_loss": 3.35873637493298, + "tokens_seen": 2593193984 + }, + { + "epoch": 0.57, + "learning_rate": 4.325950890707752e-05, + "loss": 2.5931, + "theoretical_loss": 3.3587229327499326, + "tokens_seen": 2593325056 + }, + { + "epoch": 0.57, + "learning_rate": 4.325148451291928e-05, + "loss": 2.5536, + "theoretical_loss": 3.358709491436483, + "tokens_seen": 2593456128 + }, + { + "epoch": 0.57, + "learning_rate": 4.324346011876104e-05, + "loss": 2.4584, + "theoretical_loss": 3.358696050992531, + "tokens_seen": 2593587200 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1420064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8486673831939697, + "objective/train/theoretical_loss": 3.3586826114179775, + "objective/train/tokens_used": 964177376, + "theoretical_loss": 3.3586826114179775, + "tokens_seen": 2593718272 + }, + { + "epoch": 0.57, + "learning_rate": 4.32354357246028e-05, + "loss": 2.4882, + "theoretical_loss": 3.3586826114179775, + "tokens_seen": 2593718272 + }, + { + "epoch": 0.57, + "learning_rate": 4.322741133044456e-05, + "loss": 2.4858, + "theoretical_loss": 3.358669172712722, + "tokens_seen": 2593849344 + }, + { + "epoch": 0.57, + "learning_rate": 4.321938693628631e-05, + "loss": 2.4799, + "theoretical_loss": 3.3586557348766637, + "tokens_seen": 2593980416 + }, + { + "epoch": 0.57, + "learning_rate": 4.321136254212807e-05, + "loss": 2.4642, + "theoretical_loss": 3.358642297909703, + "tokens_seen": 2594111488 + }, + { + "epoch": 0.57, + "learning_rate": 4.320333814796983e-05, + "loss": 2.3856, + "theoretical_loss": 3.3586288618117397, + "tokens_seen": 2594242560 + }, + { + "epoch": 0.57, + "learning_rate": 4.319531375381159e-05, + "loss": 2.554, + "theoretical_loss": 3.3586154265826735, + "tokens_seen": 2594373632 + }, + { + "epoch": 0.57, + "learning_rate": 4.318728935965335e-05, + "loss": 2.3107, + "theoretical_loss": 3.358601992222405, + "tokens_seen": 2594504704 + }, + { + "epoch": 0.57, + "learning_rate": 4.317926496549511e-05, + "loss": 2.5941, + "theoretical_loss": 3.3585885587308333, + "tokens_seen": 2594635776 + }, + { + "epoch": 0.57, + "learning_rate": 4.317124057133687e-05, + "loss": 2.4145, + "theoretical_loss": 3.3585751261078585, + "tokens_seen": 2594766848 + }, + { + "epoch": 0.57, + "learning_rate": 4.316321617717862e-05, + "loss": 2.5545, + "theoretical_loss": 3.358561694353381, + "tokens_seen": 2594897920 + }, + { + "epoch": 0.57, + "learning_rate": 4.315519178302038e-05, + "loss": 2.4887, + "theoretical_loss": 3.3585482634673003, + "tokens_seen": 2595028992 + }, + { + "epoch": 0.57, + "learning_rate": 4.314716738886214e-05, + "loss": 2.8372, + "theoretical_loss": 3.3585348334495166, + "tokens_seen": 2595160064 + }, + { + "epoch": 0.57, + "learning_rate": 4.31391429947039e-05, + "loss": 2.4657, + "theoretical_loss": 3.35852140429993, + "tokens_seen": 2595291136 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1420982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5949935913085938, + "objective/train/theoretical_loss": 3.3585146900506793, + "objective/train/tokens_used": 965815776, + "theoretical_loss": 3.3585146900506793, + "tokens_seen": 2595356672 + }, + { + "epoch": 0.57, + "learning_rate": 4.313111860054566e-05, + "loss": 2.6028, + "theoretical_loss": 3.3585079760184406, + "tokens_seen": 2595422208 + }, + { + "epoch": 0.57, + "learning_rate": 4.312309420638742e-05, + "loss": 2.5677, + "theoretical_loss": 3.3584945486049484, + "tokens_seen": 2595553280 + }, + { + "epoch": 0.57, + "learning_rate": 4.311506981222918e-05, + "loss": 2.5317, + "theoretical_loss": 3.358481122059353, + "tokens_seen": 2595684352 + }, + { + "epoch": 0.57, + "learning_rate": 4.310704541807094e-05, + "loss": 2.55, + "theoretical_loss": 3.358467696381555, + "tokens_seen": 2595815424 + }, + { + "epoch": 0.57, + "learning_rate": 4.3099021023912695e-05, + "loss": 2.5328, + "theoretical_loss": 3.3584542715714543, + "tokens_seen": 2595946496 + }, + { + "epoch": 0.57, + "learning_rate": 4.3090996629754455e-05, + "loss": 2.4475, + "theoretical_loss": 3.3584408476289513, + "tokens_seen": 2596077568 + }, + { + "epoch": 0.57, + "learning_rate": 4.3082972235596215e-05, + "loss": 2.4731, + "theoretical_loss": 3.358427424553946, + "tokens_seen": 2596208640 + }, + { + "epoch": 0.57, + "learning_rate": 4.3074947841437975e-05, + "loss": 2.5151, + "theoretical_loss": 3.358414002346338, + "tokens_seen": 2596339712 + }, + { + "epoch": 0.57, + "learning_rate": 4.3066923447279734e-05, + "loss": 2.5764, + "theoretical_loss": 3.3584005810060282, + "tokens_seen": 2596470784 + }, + { + "epoch": 0.57, + "learning_rate": 4.3058899053121494e-05, + "loss": 2.5897, + "theoretical_loss": 3.3583871605329163, + "tokens_seen": 2596601856 + }, + { + "epoch": 0.57, + "learning_rate": 4.3050874658963254e-05, + "loss": 2.5039, + "theoretical_loss": 3.358373740926903, + "tokens_seen": 2596732928 + }, + { + "epoch": 0.57, + "learning_rate": 4.304285026480501e-05, + "loss": 2.5007, + "theoretical_loss": 3.358360322187888, + "tokens_seen": 2596864000 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1421571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.679337978363037, + "objective/train/theoretical_loss": 3.3583469043157717, + "objective/train/tokens_used": 967454176, + "theoretical_loss": 3.3583469043157717, + "tokens_seen": 2596995072 + }, + { + "epoch": 0.57, + "learning_rate": 4.303482587064677e-05, + "loss": 2.5579, + "theoretical_loss": 3.3583469043157717, + "tokens_seen": 2596995072 + }, + { + "epoch": 0.57, + "learning_rate": 4.302680147648853e-05, + "loss": 2.4033, + "theoretical_loss": 3.3583334873104547, + "tokens_seen": 2597126144 + }, + { + "epoch": 0.57, + "learning_rate": 4.301877708233029e-05, + "loss": 2.65, + "theoretical_loss": 3.3583200711718364, + "tokens_seen": 2597257216 + }, + { + "epoch": 0.57, + "learning_rate": 4.301075268817205e-05, + "loss": 2.4555, + "theoretical_loss": 3.358306655899818, + "tokens_seen": 2597388288 + }, + { + "epoch": 0.57, + "learning_rate": 4.300272829401381e-05, + "loss": 2.3422, + "theoretical_loss": 3.3582932414942994, + "tokens_seen": 2597519360 + }, + { + "epoch": 0.57, + "learning_rate": 4.299470389985557e-05, + "loss": 2.4833, + "theoretical_loss": 3.358279827955181, + "tokens_seen": 2597650432 + }, + { + "epoch": 0.57, + "learning_rate": 4.298667950569732e-05, + "loss": 2.432, + "theoretical_loss": 3.3582664152823627, + "tokens_seen": 2597781504 + }, + { + "epoch": 0.57, + "learning_rate": 4.297865511153908e-05, + "loss": 2.5715, + "theoretical_loss": 3.3582530034757454, + "tokens_seen": 2597912576 + }, + { + "epoch": 0.57, + "learning_rate": 4.297063071738084e-05, + "loss": 2.4453, + "theoretical_loss": 3.3582395925352295, + "tokens_seen": 2598043648 + }, + { + "epoch": 0.57, + "learning_rate": 4.29626063232226e-05, + "loss": 2.5452, + "theoretical_loss": 3.3582261824607147, + "tokens_seen": 2598174720 + }, + { + "epoch": 0.57, + "learning_rate": 4.295458192906436e-05, + "loss": 2.5757, + "theoretical_loss": 3.3582127732521023, + "tokens_seen": 2598305792 + }, + { + "epoch": 0.57, + "learning_rate": 4.294655753490612e-05, + "loss": 2.4751, + "theoretical_loss": 3.358199364909292, + "tokens_seen": 2598436864 + }, + { + "epoch": 0.57, + "learning_rate": 4.293853314074788e-05, + "loss": 2.3885, + "theoretical_loss": 3.3581859574321844, + "tokens_seen": 2598567936 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1422608, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.032858371734619, + "objective/train/theoretical_loss": 3.3581792540182382, + "objective/train/tokens_used": 969092576, + "theoretical_loss": 3.3581792540182382, + "tokens_seen": 2598633472 + }, + { + "epoch": 0.57, + "learning_rate": 4.293050874658964e-05, + "loss": 2.425, + "theoretical_loss": 3.35817255082068, + "tokens_seen": 2598699008 + }, + { + "epoch": 0.58, + "learning_rate": 4.292248435243139e-05, + "loss": 2.3444, + "theoretical_loss": 3.3581591450746795, + "tokens_seen": 2598830080 + }, + { + "epoch": 0.58, + "learning_rate": 4.291445995827315e-05, + "loss": 2.442, + "theoretical_loss": 3.358145740194083, + "tokens_seen": 2598961152 + }, + { + "epoch": 0.58, + "learning_rate": 4.290643556411491e-05, + "loss": 2.5509, + "theoretical_loss": 3.3581323361787914, + "tokens_seen": 2599092224 + }, + { + "epoch": 0.58, + "learning_rate": 4.289841116995667e-05, + "loss": 2.4825, + "theoretical_loss": 3.3581189330287047, + "tokens_seen": 2599223296 + }, + { + "epoch": 0.58, + "learning_rate": 4.289038677579843e-05, + "loss": 2.6839, + "theoretical_loss": 3.3581055307437238, + "tokens_seen": 2599354368 + }, + { + "epoch": 0.58, + "learning_rate": 4.288236238164019e-05, + "loss": 2.681, + "theoretical_loss": 3.358092129323749, + "tokens_seen": 2599485440 + }, + { + "epoch": 0.58, + "learning_rate": 4.287433798748195e-05, + "loss": 2.578, + "theoretical_loss": 3.358078728768681, + "tokens_seen": 2599616512 + }, + { + "epoch": 0.58, + "learning_rate": 4.2866313593323705e-05, + "loss": 2.5397, + "theoretical_loss": 3.358065329078421, + "tokens_seen": 2599747584 + }, + { + "epoch": 0.58, + "learning_rate": 4.2858289199165465e-05, + "loss": 2.7148, + "theoretical_loss": 3.358051930252868, + "tokens_seen": 2599878656 + }, + { + "epoch": 0.58, + "learning_rate": 4.2850264805007225e-05, + "loss": 2.5636, + "theoretical_loss": 3.358038532291924, + "tokens_seen": 2600009728 + }, + { + "epoch": 0.58, + "learning_rate": 4.2842240410848985e-05, + "loss": 2.4291, + "theoretical_loss": 3.358025135195489, + "tokens_seen": 2600140800 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1423091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.546130657196045, + "objective/train/theoretical_loss": 3.3580117389634645, + "objective/train/tokens_used": 970730976, + "theoretical_loss": 3.3580117389634645, + "tokens_seen": 2600271872 + }, + { + "epoch": 0.58, + "learning_rate": 4.2834216016690745e-05, + "loss": 2.5889, + "theoretical_loss": 3.3580117389634645, + "tokens_seen": 2600271872 + }, + { + "epoch": 0.58, + "learning_rate": 4.2826191622532505e-05, + "loss": 2.4526, + "theoretical_loss": 3.35799834359575, + "tokens_seen": 2600402944 + }, + { + "epoch": 0.58, + "learning_rate": 4.2818167228374264e-05, + "loss": 2.5043, + "theoretical_loss": 3.3579849490922467, + "tokens_seen": 2600534016 + }, + { + "epoch": 0.58, + "learning_rate": 4.281014283421602e-05, + "loss": 2.4791, + "theoretical_loss": 3.3579715554528553, + "tokens_seen": 2600665088 + }, + { + "epoch": 0.58, + "learning_rate": 4.280211844005778e-05, + "loss": 2.5038, + "theoretical_loss": 3.3579581626774764, + "tokens_seen": 2600796160 + }, + { + "epoch": 0.58, + "learning_rate": 4.279409404589954e-05, + "loss": 2.4278, + "theoretical_loss": 3.357944770766011, + "tokens_seen": 2600927232 + }, + { + "epoch": 0.58, + "learning_rate": 4.27860696517413e-05, + "loss": 2.5213, + "theoretical_loss": 3.3579313797183596, + "tokens_seen": 2601058304 + }, + { + "epoch": 0.58, + "learning_rate": 4.277804525758306e-05, + "loss": 2.5781, + "theoretical_loss": 3.357917989534423, + "tokens_seen": 2601189376 + }, + { + "epoch": 0.58, + "learning_rate": 4.277002086342482e-05, + "loss": 2.515, + "theoretical_loss": 3.3579046002141015, + "tokens_seen": 2601320448 + }, + { + "epoch": 0.58, + "learning_rate": 4.276199646926658e-05, + "loss": 2.5409, + "theoretical_loss": 3.3578912117572965, + "tokens_seen": 2601451520 + }, + { + "epoch": 0.58, + "learning_rate": 4.275397207510834e-05, + "loss": 2.4656, + "theoretical_loss": 3.3578778241639093, + "tokens_seen": 2601582592 + }, + { + "epoch": 0.58, + "learning_rate": 4.274594768095009e-05, + "loss": 2.5142, + "theoretical_loss": 3.3578644374338396, + "tokens_seen": 2601713664 + }, + { + "epoch": 0.58, + "learning_rate": 4.273792328679185e-05, + "loss": 2.6067, + "theoretical_loss": 3.357851051566989, + "tokens_seen": 2601844736 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1424292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5811870098114014, + "objective/train/theoretical_loss": 3.3578443589572395, + "objective/train/tokens_used": 972369376, + "theoretical_loss": 3.3578443589572395, + "tokens_seen": 2601910272 + }, + { + "epoch": 0.58, + "learning_rate": 4.272989889263361e-05, + "loss": 2.5605, + "theoretical_loss": 3.357837666563258, + "tokens_seen": 2601975808 + }, + { + "epoch": 0.58, + "learning_rate": 4.272187449847537e-05, + "loss": 2.4923, + "theoretical_loss": 3.357824282422547, + "tokens_seen": 2602106880 + }, + { + "epoch": 0.58, + "learning_rate": 4.271385010431713e-05, + "loss": 2.449, + "theoretical_loss": 3.357810899144758, + "tokens_seen": 2602237952 + }, + { + "epoch": 0.58, + "learning_rate": 4.270582571015889e-05, + "loss": 2.6668, + "theoretical_loss": 3.357797516729791, + "tokens_seen": 2602369024 + }, + { + "epoch": 0.58, + "learning_rate": 4.269780131600065e-05, + "loss": 2.5838, + "theoretical_loss": 3.357784135177548, + "tokens_seen": 2602500096 + }, + { + "epoch": 0.58, + "learning_rate": 4.26897769218424e-05, + "loss": 2.7115, + "theoretical_loss": 3.3577707544879285, + "tokens_seen": 2602631168 + }, + { + "epoch": 0.58, + "learning_rate": 4.268175252768416e-05, + "loss": 2.5673, + "theoretical_loss": 3.3577573746608342, + "tokens_seen": 2602762240 + }, + { + "epoch": 0.58, + "learning_rate": 4.267372813352592e-05, + "loss": 2.5383, + "theoretical_loss": 3.357743995696166, + "tokens_seen": 2602893312 + }, + { + "epoch": 0.58, + "learning_rate": 4.266570373936768e-05, + "loss": 2.5499, + "theoretical_loss": 3.357730617593825, + "tokens_seen": 2603024384 + }, + { + "epoch": 0.58, + "learning_rate": 4.265767934520944e-05, + "loss": 2.5093, + "theoretical_loss": 3.357717240353712, + "tokens_seen": 2603155456 + }, + { + "epoch": 0.58, + "learning_rate": 4.26496549510512e-05, + "loss": 2.663, + "theoretical_loss": 3.3577038639757286, + "tokens_seen": 2603286528 + }, + { + "epoch": 0.58, + "learning_rate": 4.2641630556892955e-05, + "loss": 2.7217, + "theoretical_loss": 3.357690488459775, + "tokens_seen": 2603417600 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1424790, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.476832866668701, + "objective/train/theoretical_loss": 3.3576771138057526, + "objective/train/tokens_used": 974007776, + "theoretical_loss": 3.3576771138057526, + "tokens_seen": 2603548672 + }, + { + "epoch": 0.58, + "learning_rate": 4.2633606162734715e-05, + "loss": 2.5643, + "theoretical_loss": 3.3576771138057526, + "tokens_seen": 2603548672 + }, + { + "epoch": 0.58, + "learning_rate": 4.2625581768576475e-05, + "loss": 2.4756, + "theoretical_loss": 3.357663740013563, + "tokens_seen": 2603679744 + }, + { + "epoch": 0.58, + "learning_rate": 4.2617557374418235e-05, + "loss": 2.5981, + "theoretical_loss": 3.3576503670831057, + "tokens_seen": 2603810816 + }, + { + "epoch": 0.58, + "learning_rate": 4.2609532980259995e-05, + "loss": 2.5278, + "theoretical_loss": 3.3576369950142837, + "tokens_seen": 2603941888 + }, + { + "epoch": 0.58, + "learning_rate": 4.2601508586101755e-05, + "loss": 2.5158, + "theoretical_loss": 3.3576236238069974, + "tokens_seen": 2604072960 + }, + { + "epoch": 0.58, + "learning_rate": 4.259348419194351e-05, + "loss": 2.3904, + "theoretical_loss": 3.3576102534611474, + "tokens_seen": 2604204032 + }, + { + "epoch": 0.58, + "learning_rate": 4.258545979778527e-05, + "loss": 2.6679, + "theoretical_loss": 3.357596883976635, + "tokens_seen": 2604335104 + }, + { + "epoch": 0.58, + "learning_rate": 4.257743540362703e-05, + "loss": 2.5455, + "theoretical_loss": 3.357583515353362, + "tokens_seen": 2604466176 + }, + { + "epoch": 0.58, + "learning_rate": 4.256941100946879e-05, + "loss": 2.585, + "theoretical_loss": 3.3575701475912294, + "tokens_seen": 2604597248 + }, + { + "epoch": 0.58, + "learning_rate": 4.256138661531055e-05, + "loss": 2.5585, + "theoretical_loss": 3.357556780690138, + "tokens_seen": 2604728320 + }, + { + "epoch": 0.58, + "learning_rate": 4.255336222115231e-05, + "loss": 2.5108, + "theoretical_loss": 3.357543414649989, + "tokens_seen": 2604859392 + }, + { + "epoch": 0.58, + "learning_rate": 4.254533782699406e-05, + "loss": 2.4726, + "theoretical_loss": 3.3575300494706846, + "tokens_seen": 2604990464 + }, + { + "epoch": 0.58, + "learning_rate": 4.253731343283582e-05, + "loss": 2.4168, + "theoretical_loss": 3.3575166851521248, + "tokens_seen": 2605121536 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1425919, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.612056016921997, + "objective/train/theoretical_loss": 3.3575100033155936, + "objective/train/tokens_used": 975646176, + "theoretical_loss": 3.3575100033155936, + "tokens_seen": 2605187072 + }, + { + "epoch": 0.58, + "learning_rate": 4.252928903867758e-05, + "loss": 2.439, + "theoretical_loss": 3.357503321694211, + "tokens_seen": 2605252608 + }, + { + "epoch": 0.58, + "learning_rate": 4.252126464451934e-05, + "loss": 2.6102, + "theoretical_loss": 3.3574899590968457, + "tokens_seen": 2605383680 + }, + { + "epoch": 0.58, + "learning_rate": 4.25132402503611e-05, + "loss": 2.5469, + "theoretical_loss": 3.3574765973599288, + "tokens_seen": 2605514752 + }, + { + "epoch": 0.58, + "learning_rate": 4.250521585620286e-05, + "loss": 2.5334, + "theoretical_loss": 3.3574632364833623, + "tokens_seen": 2605645824 + }, + { + "epoch": 0.58, + "learning_rate": 4.249719146204461e-05, + "loss": 2.5219, + "theoretical_loss": 3.3574498764670473, + "tokens_seen": 2605776896 + }, + { + "epoch": 0.58, + "learning_rate": 4.248916706788637e-05, + "loss": 2.4258, + "theoretical_loss": 3.3574365173108855, + "tokens_seen": 2605907968 + }, + { + "epoch": 0.58, + "learning_rate": 4.248114267372813e-05, + "loss": 2.4869, + "theoretical_loss": 3.3574231590147776, + "tokens_seen": 2606039040 + }, + { + "epoch": 0.58, + "learning_rate": 4.247311827956989e-05, + "loss": 2.5574, + "theoretical_loss": 3.3574098015786253, + "tokens_seen": 2606170112 + }, + { + "epoch": 0.58, + "learning_rate": 4.246509388541165e-05, + "loss": 2.6704, + "theoretical_loss": 3.3573964450023306, + "tokens_seen": 2606301184 + }, + { + "epoch": 0.58, + "learning_rate": 4.245706949125341e-05, + "loss": 2.5527, + "theoretical_loss": 3.3573830892857943, + "tokens_seen": 2606432256 + }, + { + "epoch": 0.58, + "learning_rate": 4.2449045097095166e-05, + "loss": 2.5624, + "theoretical_loss": 3.3573697344289175, + "tokens_seen": 2606563328 + }, + { + "epoch": 0.58, + "learning_rate": 4.2441020702936926e-05, + "loss": 2.6472, + "theoretical_loss": 3.357356380431602, + "tokens_seen": 2606694400 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1426919, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6040198802948, + "objective/train/theoretical_loss": 3.3573430272937497, + "objective/train/tokens_used": 977284576, + "theoretical_loss": 3.3573430272937497, + "tokens_seen": 2606825472 + }, + { + "epoch": 0.58, + "learning_rate": 4.2432996308778686e-05, + "loss": 2.3225, + "theoretical_loss": 3.3573430272937497, + "tokens_seen": 2606825472 + }, + { + "epoch": 0.58, + "learning_rate": 4.2424971914620446e-05, + "loss": 2.4597, + "theoretical_loss": 3.3573296750152615, + "tokens_seen": 2606956544 + }, + { + "epoch": 0.58, + "learning_rate": 4.2416947520462206e-05, + "loss": 2.4333, + "theoretical_loss": 3.3573163235960393, + "tokens_seen": 2607087616 + }, + { + "epoch": 0.58, + "learning_rate": 4.2408923126303965e-05, + "loss": 2.5935, + "theoretical_loss": 3.357302973035984, + "tokens_seen": 2607218688 + }, + { + "epoch": 0.58, + "learning_rate": 4.2400898732145725e-05, + "loss": 2.572, + "theoretical_loss": 3.3572896233349976, + "tokens_seen": 2607349760 + }, + { + "epoch": 0.58, + "learning_rate": 4.239287433798748e-05, + "loss": 2.5383, + "theoretical_loss": 3.357276274492982, + "tokens_seen": 2607480832 + }, + { + "epoch": 0.58, + "learning_rate": 4.238484994382924e-05, + "loss": 2.6162, + "theoretical_loss": 3.3572629265098377, + "tokens_seen": 2607611904 + }, + { + "epoch": 0.58, + "learning_rate": 4.2376825549671e-05, + "loss": 2.4549, + "theoretical_loss": 3.357249579385467, + "tokens_seen": 2607742976 + }, + { + "epoch": 0.58, + "learning_rate": 4.236880115551276e-05, + "loss": 2.4683, + "theoretical_loss": 3.3572362331197714, + "tokens_seen": 2607874048 + }, + { + "epoch": 0.58, + "learning_rate": 4.236077676135452e-05, + "loss": 2.5949, + "theoretical_loss": 3.3572228877126524, + "tokens_seen": 2608005120 + }, + { + "epoch": 0.58, + "learning_rate": 4.235275236719628e-05, + "loss": 2.577, + "theoretical_loss": 3.3572095431640117, + "tokens_seen": 2608136192 + }, + { + "epoch": 0.58, + "learning_rate": 4.234472797303804e-05, + "loss": 2.5982, + "theoretical_loss": 3.3571961994737514, + "tokens_seen": 2608267264 + }, + { + "epoch": 0.58, + "learning_rate": 4.233670357887979e-05, + "loss": 2.6835, + "theoretical_loss": 3.357182856641772, + "tokens_seen": 2608398336 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1427579, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4858877658843994, + "objective/train/theoretical_loss": 3.3571761855476074, + "objective/train/tokens_used": 978922976, + "theoretical_loss": 3.3571761855476074, + "tokens_seen": 2608463872 + }, + { + "epoch": 0.58, + "learning_rate": 4.232867918472155e-05, + "loss": 2.5525, + "theoretical_loss": 3.357169514667976, + "tokens_seen": 2608529408 + }, + { + "epoch": 0.58, + "learning_rate": 4.232065479056331e-05, + "loss": 2.5435, + "theoretical_loss": 3.3571561735522653, + "tokens_seen": 2608660480 + }, + { + "epoch": 0.58, + "learning_rate": 4.231263039640507e-05, + "loss": 2.7508, + "theoretical_loss": 3.357142833294541, + "tokens_seen": 2608791552 + }, + { + "epoch": 0.58, + "learning_rate": 4.230460600224683e-05, + "loss": 2.5727, + "theoretical_loss": 3.357129493894705, + "tokens_seen": 2608922624 + }, + { + "epoch": 0.58, + "learning_rate": 4.229658160808859e-05, + "loss": 2.4236, + "theoretical_loss": 3.357116155352659, + "tokens_seen": 2609053696 + }, + { + "epoch": 0.58, + "learning_rate": 4.228855721393035e-05, + "loss": 2.5318, + "theoretical_loss": 3.3571028176683053, + "tokens_seen": 2609184768 + }, + { + "epoch": 0.58, + "learning_rate": 4.228053281977211e-05, + "loss": 2.5775, + "theoretical_loss": 3.3570894808415446, + "tokens_seen": 2609315840 + }, + { + "epoch": 0.58, + "learning_rate": 4.2272508425613864e-05, + "loss": 2.4795, + "theoretical_loss": 3.3570761448722797, + "tokens_seen": 2609446912 + }, + { + "epoch": 0.58, + "learning_rate": 4.2264484031455623e-05, + "loss": 2.6324, + "theoretical_loss": 3.357062809760412, + "tokens_seen": 2609577984 + }, + { + "epoch": 0.58, + "learning_rate": 4.225645963729738e-05, + "loss": 2.4808, + "theoretical_loss": 3.357049475505843, + "tokens_seen": 2609709056 + }, + { + "epoch": 0.58, + "learning_rate": 4.224843524313914e-05, + "loss": 2.5637, + "theoretical_loss": 3.357036142108475, + "tokens_seen": 2609840128 + }, + { + "epoch": 0.58, + "learning_rate": 4.22404108489809e-05, + "loss": 2.5647, + "theoretical_loss": 3.35702280956821, + "tokens_seen": 2609971200 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1428936, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6875152587890625, + "objective/train/theoretical_loss": 3.357009477884949, + "objective/train/tokens_used": 980561376, + "theoretical_loss": 3.357009477884949, + "tokens_seen": 2610102272 + }, + { + "epoch": 0.58, + "learning_rate": 4.223238645482266e-05, + "loss": 2.5981, + "theoretical_loss": 3.357009477884949, + "tokens_seen": 2610102272 + }, + { + "epoch": 0.58, + "learning_rate": 4.222436206066442e-05, + "loss": 2.4475, + "theoretical_loss": 3.3569961470585947, + "tokens_seen": 2610233344 + }, + { + "epoch": 0.58, + "learning_rate": 4.2216337666506176e-05, + "loss": 2.6229, + "theoretical_loss": 3.3569828170890488, + "tokens_seen": 2610364416 + }, + { + "epoch": 0.58, + "learning_rate": 4.2208313272347936e-05, + "loss": 2.4604, + "theoretical_loss": 3.356969487976213, + "tokens_seen": 2610495488 + }, + { + "epoch": 0.58, + "learning_rate": 4.2200288878189696e-05, + "loss": 2.4634, + "theoretical_loss": 3.3569561597199895, + "tokens_seen": 2610626560 + }, + { + "epoch": 0.58, + "learning_rate": 4.2192264484031456e-05, + "loss": 2.3858, + "theoretical_loss": 3.35694283232028, + "tokens_seen": 2610757632 + }, + { + "epoch": 0.58, + "learning_rate": 4.2184240089873216e-05, + "loss": 2.5749, + "theoretical_loss": 3.3569295057769866, + "tokens_seen": 2610888704 + }, + { + "epoch": 0.58, + "learning_rate": 4.2176215695714976e-05, + "loss": 2.5125, + "theoretical_loss": 3.3569161800900114, + "tokens_seen": 2611019776 + }, + { + "epoch": 0.58, + "learning_rate": 4.2168191301556736e-05, + "loss": 2.5489, + "theoretical_loss": 3.3569028552592557, + "tokens_seen": 2611150848 + }, + { + "epoch": 0.58, + "learning_rate": 4.2160166907398495e-05, + "loss": 2.5466, + "theoretical_loss": 3.3568895312846223, + "tokens_seen": 2611281920 + }, + { + "epoch": 0.58, + "learning_rate": 4.215214251324025e-05, + "loss": 2.5337, + "theoretical_loss": 3.3568762081660135, + "tokens_seen": 2611412992 + }, + { + "epoch": 0.58, + "learning_rate": 4.214411811908201e-05, + "loss": 2.4732, + "theoretical_loss": 3.3568628859033303, + "tokens_seen": 2611544064 + }, + { + "epoch": 0.58, + "learning_rate": 4.213609372492377e-05, + "loss": 2.5811, + "theoretical_loss": 3.3568495644964753, + "tokens_seen": 2611675136 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1429548, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.299931764602661, + "objective/train/theoretical_loss": 3.356842904113953, + "objective/train/tokens_used": 982199776, + "theoretical_loss": 3.356842904113953, + "tokens_seen": 2611740672 + }, + { + "epoch": 0.58, + "learning_rate": 4.212806933076553e-05, + "loss": 2.6047, + "theoretical_loss": 3.3568362439453505, + "tokens_seen": 2611806208 + }, + { + "epoch": 0.58, + "learning_rate": 4.212004493660729e-05, + "loss": 2.543, + "theoretical_loss": 3.356822924249858, + "tokens_seen": 2611937280 + }, + { + "epoch": 0.58, + "learning_rate": 4.211202054244905e-05, + "loss": 2.6862, + "theoretical_loss": 3.3568096054099006, + "tokens_seen": 2612068352 + }, + { + "epoch": 0.58, + "learning_rate": 4.210399614829081e-05, + "loss": 2.6026, + "theoretical_loss": 3.356796287425379, + "tokens_seen": 2612199424 + }, + { + "epoch": 0.58, + "learning_rate": 4.209597175413256e-05, + "loss": 2.5394, + "theoretical_loss": 3.3567829702961967, + "tokens_seen": 2612330496 + }, + { + "epoch": 0.58, + "learning_rate": 4.208794735997432e-05, + "loss": 2.5377, + "theoretical_loss": 3.356769654022255, + "tokens_seen": 2612461568 + }, + { + "epoch": 0.58, + "learning_rate": 4.207992296581608e-05, + "loss": 2.6312, + "theoretical_loss": 3.3567563386034562, + "tokens_seen": 2612592640 + }, + { + "epoch": 0.58, + "learning_rate": 4.207189857165784e-05, + "loss": 2.5456, + "theoretical_loss": 3.356743024039703, + "tokens_seen": 2612723712 + }, + { + "epoch": 0.58, + "learning_rate": 4.20638741774996e-05, + "loss": 2.5076, + "theoretical_loss": 3.356729710330897, + "tokens_seen": 2612854784 + }, + { + "epoch": 0.58, + "learning_rate": 4.205584978334136e-05, + "loss": 2.6368, + "theoretical_loss": 3.3567163974769403, + "tokens_seen": 2612985856 + }, + { + "epoch": 0.58, + "learning_rate": 4.204782538918312e-05, + "loss": 2.5797, + "theoretical_loss": 3.356703085477736, + "tokens_seen": 2613116928 + }, + { + "epoch": 0.58, + "learning_rate": 4.2039800995024874e-05, + "loss": 2.6153, + "theoretical_loss": 3.3566897743331854, + "tokens_seen": 2613248000 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1430749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7607247829437256, + "objective/train/theoretical_loss": 3.3566764640431916, + "objective/train/tokens_used": 983838176, + "theoretical_loss": 3.3566764640431916, + "tokens_seen": 2613379072 + }, + { + "epoch": 0.58, + "learning_rate": 4.2031776600866634e-05, + "loss": 2.5442, + "theoretical_loss": 3.3566764640431916, + "tokens_seen": 2613379072 + }, + { + "epoch": 0.58, + "learning_rate": 4.2023752206708394e-05, + "loss": 2.4808, + "theoretical_loss": 3.356663154607656, + "tokens_seen": 2613510144 + }, + { + "epoch": 0.58, + "learning_rate": 4.2015727812550153e-05, + "loss": 2.669, + "theoretical_loss": 3.3566498460264818, + "tokens_seen": 2613641216 + }, + { + "epoch": 0.58, + "learning_rate": 4.200770341839191e-05, + "loss": 2.4666, + "theoretical_loss": 3.3566365382995706, + "tokens_seen": 2613772288 + }, + { + "epoch": 0.58, + "learning_rate": 4.199967902423367e-05, + "loss": 2.4668, + "theoretical_loss": 3.356623231426825, + "tokens_seen": 2613903360 + }, + { + "epoch": 0.58, + "learning_rate": 4.199165463007543e-05, + "loss": 2.5468, + "theoretical_loss": 3.3566099254081476, + "tokens_seen": 2614034432 + }, + { + "epoch": 0.58, + "learning_rate": 4.198363023591719e-05, + "loss": 2.604, + "theoretical_loss": 3.3565966202434407, + "tokens_seen": 2614165504 + }, + { + "epoch": 0.58, + "learning_rate": 4.1975605841758946e-05, + "loss": 2.8184, + "theoretical_loss": 3.356583315932606, + "tokens_seen": 2614296576 + }, + { + "epoch": 0.58, + "learning_rate": 4.1967581447600706e-05, + "loss": 2.6276, + "theoretical_loss": 3.356570012475547, + "tokens_seen": 2614427648 + }, + { + "epoch": 0.58, + "learning_rate": 4.1959557053442466e-05, + "loss": 2.5807, + "theoretical_loss": 3.356556709872165, + "tokens_seen": 2614558720 + }, + { + "epoch": 0.58, + "learning_rate": 4.1951532659284226e-05, + "loss": 2.4354, + "theoretical_loss": 3.3565434081223633, + "tokens_seen": 2614689792 + }, + { + "epoch": 0.58, + "learning_rate": 4.1943508265125986e-05, + "loss": 2.5711, + "theoretical_loss": 3.356530107226044, + "tokens_seen": 2614820864 + }, + { + "epoch": 0.58, + "learning_rate": 4.1935483870967746e-05, + "loss": 2.411, + "theoretical_loss": 3.3565168071831093, + "tokens_seen": 2614951936 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1431756, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9996166229248047, + "objective/train/theoretical_loss": 3.356510157481631, + "objective/train/tokens_used": 985476576, + "theoretical_loss": 3.356510157481631, + "tokens_seen": 2615017472 + }, + { + "epoch": 0.58, + "learning_rate": 4.1927459476809506e-05, + "loss": 2.5563, + "theoretical_loss": 3.356503507993462, + "tokens_seen": 2615083008 + }, + { + "epoch": 0.59, + "learning_rate": 4.191943508265126e-05, + "loss": 2.5503, + "theoretical_loss": 3.3564902096570046, + "tokens_seen": 2615214080 + }, + { + "epoch": 0.59, + "learning_rate": 4.191141068849302e-05, + "loss": 2.4169, + "theoretical_loss": 3.3564769121736395, + "tokens_seen": 2615345152 + }, + { + "epoch": 0.59, + "learning_rate": 4.190338629433478e-05, + "loss": 2.5409, + "theoretical_loss": 3.3564636155432694, + "tokens_seen": 2615476224 + }, + { + "epoch": 0.59, + "learning_rate": 4.189536190017654e-05, + "loss": 2.6269, + "theoretical_loss": 3.356450319765796, + "tokens_seen": 2615607296 + }, + { + "epoch": 0.59, + "learning_rate": 4.18873375060183e-05, + "loss": 2.5099, + "theoretical_loss": 3.3564370248411235, + "tokens_seen": 2615738368 + }, + { + "epoch": 0.59, + "learning_rate": 4.187931311186006e-05, + "loss": 2.3438, + "theoretical_loss": 3.356423730769153, + "tokens_seen": 2615869440 + }, + { + "epoch": 0.59, + "learning_rate": 4.187128871770182e-05, + "loss": 2.5988, + "theoretical_loss": 3.356410437549788, + "tokens_seen": 2616000512 + }, + { + "epoch": 0.59, + "learning_rate": 4.186326432354357e-05, + "loss": 2.517, + "theoretical_loss": 3.35639714518293, + "tokens_seen": 2616131584 + }, + { + "epoch": 0.59, + "learning_rate": 4.185523992938533e-05, + "loss": 2.7114, + "theoretical_loss": 3.3563838536684827, + "tokens_seen": 2616262656 + }, + { + "epoch": 0.59, + "learning_rate": 4.184721553522709e-05, + "loss": 2.7067, + "theoretical_loss": 3.3563705630063487, + "tokens_seen": 2616393728 + }, + { + "epoch": 0.59, + "learning_rate": 4.183919114106885e-05, + "loss": 2.5441, + "theoretical_loss": 3.35635727319643, + "tokens_seen": 2616524800 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1432292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3771212100982666, + "objective/train/theoretical_loss": 3.3563439842386296, + "objective/train/tokens_used": 987114976, + "theoretical_loss": 3.3563439842386296, + "tokens_seen": 2616655872 + }, + { + "epoch": 0.59, + "learning_rate": 4.183116674691061e-05, + "loss": 2.5881, + "theoretical_loss": 3.3563439842386296, + "tokens_seen": 2616655872 + }, + { + "epoch": 0.59, + "learning_rate": 4.182314235275237e-05, + "loss": 2.5632, + "theoretical_loss": 3.3563306961328503, + "tokens_seen": 2616786944 + }, + { + "epoch": 0.59, + "learning_rate": 4.181511795859413e-05, + "loss": 2.4448, + "theoretical_loss": 3.3563174088789944, + "tokens_seen": 2616918016 + }, + { + "epoch": 0.59, + "learning_rate": 4.180709356443589e-05, + "loss": 2.5447, + "theoretical_loss": 3.356304122476965, + "tokens_seen": 2617049088 + }, + { + "epoch": 0.59, + "learning_rate": 4.1799069170277644e-05, + "loss": 2.5341, + "theoretical_loss": 3.356290836926665, + "tokens_seen": 2617180160 + }, + { + "epoch": 0.59, + "learning_rate": 4.1791044776119404e-05, + "loss": 2.6423, + "theoretical_loss": 3.3562775522279966, + "tokens_seen": 2617311232 + }, + { + "epoch": 0.59, + "learning_rate": 4.1783020381961164e-05, + "loss": 2.6015, + "theoretical_loss": 3.356264268380863, + "tokens_seen": 2617442304 + }, + { + "epoch": 0.59, + "learning_rate": 4.1774995987802923e-05, + "loss": 2.4337, + "theoretical_loss": 3.3562509853851665, + "tokens_seen": 2617573376 + }, + { + "epoch": 0.59, + "learning_rate": 4.1766971593644683e-05, + "loss": 2.4881, + "theoretical_loss": 3.3562377032408106, + "tokens_seen": 2617704448 + }, + { + "epoch": 0.59, + "learning_rate": 4.175894719948644e-05, + "loss": 2.6313, + "theoretical_loss": 3.3562244219476973, + "tokens_seen": 2617835520 + }, + { + "epoch": 0.59, + "learning_rate": 4.17509228053282e-05, + "loss": 2.529, + "theoretical_loss": 3.3562111415057303, + "tokens_seen": 2617966592 + }, + { + "epoch": 0.59, + "learning_rate": 4.1742898411169956e-05, + "loss": 2.6077, + "theoretical_loss": 3.356197861914812, + "tokens_seen": 2618097664 + }, + { + "epoch": 0.59, + "learning_rate": 4.1734874017011716e-05, + "loss": 2.5128, + "theoretical_loss": 3.356184583174845, + "tokens_seen": 2618228736 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1433404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3382227420806885, + "objective/train/theoretical_loss": 3.356177944123938, + "objective/train/tokens_used": 988753376, + "theoretical_loss": 3.356177944123938, + "tokens_seen": 2618294272 + }, + { + "epoch": 0.59, + "learning_rate": 4.1726849622853476e-05, + "loss": 2.502, + "theoretical_loss": 3.3561713052857325, + "tokens_seen": 2618359808 + }, + { + "epoch": 0.59, + "learning_rate": 4.1718825228695236e-05, + "loss": 2.5651, + "theoretical_loss": 3.356158028247377, + "tokens_seen": 2618490880 + }, + { + "epoch": 0.59, + "learning_rate": 4.1710800834536996e-05, + "loss": 2.4837, + "theoretical_loss": 3.356144752059682, + "tokens_seen": 2618621952 + }, + { + "epoch": 0.59, + "learning_rate": 4.1702776440378756e-05, + "loss": 2.4654, + "theoretical_loss": 3.3561314767225503, + "tokens_seen": 2618753024 + }, + { + "epoch": 0.59, + "learning_rate": 4.1694752046220516e-05, + "loss": 2.4497, + "theoretical_loss": 3.3561182022358844, + "tokens_seen": 2618884096 + }, + { + "epoch": 0.59, + "learning_rate": 4.1686727652062276e-05, + "loss": 2.6605, + "theoretical_loss": 3.3561049285995876, + "tokens_seen": 2619015168 + }, + { + "epoch": 0.59, + "learning_rate": 4.167870325790403e-05, + "loss": 2.4812, + "theoretical_loss": 3.356091655813563, + "tokens_seen": 2619146240 + }, + { + "epoch": 0.59, + "learning_rate": 4.167067886374579e-05, + "loss": 2.4392, + "theoretical_loss": 3.3560783838777137, + "tokens_seen": 2619277312 + }, + { + "epoch": 0.59, + "learning_rate": 4.166265446958755e-05, + "loss": 2.5739, + "theoretical_loss": 3.3560651127919416, + "tokens_seen": 2619408384 + }, + { + "epoch": 0.59, + "learning_rate": 4.165463007542931e-05, + "loss": 2.6302, + "theoretical_loss": 3.356051842556151, + "tokens_seen": 2619539456 + }, + { + "epoch": 0.59, + "learning_rate": 4.164660568127107e-05, + "loss": 2.5662, + "theoretical_loss": 3.3560385731702445, + "tokens_seen": 2619670528 + }, + { + "epoch": 0.59, + "learning_rate": 4.163858128711283e-05, + "loss": 2.5703, + "theoretical_loss": 3.356025304634125, + "tokens_seen": 2619801600 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1434105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3893702030181885, + "objective/train/theoretical_loss": 3.356012036947696, + "objective/train/tokens_used": 990391776, + "theoretical_loss": 3.356012036947696, + "tokens_seen": 2619932672 + }, + { + "epoch": 0.59, + "learning_rate": 4.163055689295459e-05, + "loss": 2.493, + "theoretical_loss": 3.356012036947696, + "tokens_seen": 2619932672 + }, + { + "epoch": 0.59, + "learning_rate": 4.162253249879634e-05, + "loss": 2.4581, + "theoretical_loss": 3.3559987701108596, + "tokens_seen": 2620063744 + }, + { + "epoch": 0.59, + "learning_rate": 4.16145081046381e-05, + "loss": 2.384, + "theoretical_loss": 3.3559855041235203, + "tokens_seen": 2620194816 + }, + { + "epoch": 0.59, + "learning_rate": 4.160648371047986e-05, + "loss": 2.5755, + "theoretical_loss": 3.3559722389855797, + "tokens_seen": 2620325888 + }, + { + "epoch": 0.59, + "learning_rate": 4.159845931632162e-05, + "loss": 2.6417, + "theoretical_loss": 3.355958974696942, + "tokens_seen": 2620456960 + }, + { + "epoch": 0.59, + "learning_rate": 4.159043492216338e-05, + "loss": 2.6124, + "theoretical_loss": 3.3559457112575104, + "tokens_seen": 2620588032 + }, + { + "epoch": 0.59, + "learning_rate": 4.158241052800514e-05, + "loss": 2.5474, + "theoretical_loss": 3.355932448667187, + "tokens_seen": 2620719104 + }, + { + "epoch": 0.59, + "learning_rate": 4.15743861338469e-05, + "loss": 2.3975, + "theoretical_loss": 3.3559191869258767, + "tokens_seen": 2620850176 + }, + { + "epoch": 0.59, + "learning_rate": 4.1566361739688654e-05, + "loss": 2.5871, + "theoretical_loss": 3.355905926033481, + "tokens_seen": 2620981248 + }, + { + "epoch": 0.59, + "learning_rate": 4.1558337345530414e-05, + "loss": 2.5691, + "theoretical_loss": 3.355892665989904, + "tokens_seen": 2621112320 + }, + { + "epoch": 0.59, + "learning_rate": 4.1550312951372174e-05, + "loss": 2.5794, + "theoretical_loss": 3.3558794067950486, + "tokens_seen": 2621243392 + }, + { + "epoch": 0.59, + "learning_rate": 4.1542288557213934e-05, + "loss": 2.5398, + "theoretical_loss": 3.3558661484488184, + "tokens_seen": 2621374464 + }, + { + "epoch": 0.59, + "learning_rate": 4.1534264163055694e-05, + "loss": 2.4916, + "theoretical_loss": 3.355852890951116, + "tokens_seen": 2621505536 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1435307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7079474925994873, + "objective/train/theoretical_loss": 3.3558462625204326, + "objective/train/tokens_used": 992030176, + "theoretical_loss": 3.3558462625204326, + "tokens_seen": 2621571072 + }, + { + "epoch": 0.59, + "learning_rate": 4.1526239768897453e-05, + "loss": 2.422, + "theoretical_loss": 3.3558396343018453, + "tokens_seen": 2621636608 + }, + { + "epoch": 0.59, + "learning_rate": 4.151821537473921e-05, + "loss": 2.5195, + "theoretical_loss": 3.3558263785009093, + "tokens_seen": 2621767680 + }, + { + "epoch": 0.59, + "learning_rate": 4.151019098058097e-05, + "loss": 2.4978, + "theoretical_loss": 3.3558131235482116, + "tokens_seen": 2621898752 + }, + { + "epoch": 0.59, + "learning_rate": 4.1502166586422726e-05, + "loss": 2.6146, + "theoretical_loss": 3.355799869443655, + "tokens_seen": 2622029824 + }, + { + "epoch": 0.59, + "learning_rate": 4.1494142192264486e-05, + "loss": 2.6506, + "theoretical_loss": 3.3557866161871432, + "tokens_seen": 2622160896 + }, + { + "epoch": 0.59, + "learning_rate": 4.1486117798106246e-05, + "loss": 2.552, + "theoretical_loss": 3.35577336377858, + "tokens_seen": 2622291968 + }, + { + "epoch": 0.59, + "learning_rate": 4.1478093403948006e-05, + "loss": 2.5995, + "theoretical_loss": 3.355760112217868, + "tokens_seen": 2622423040 + }, + { + "epoch": 0.59, + "learning_rate": 4.1470069009789766e-05, + "loss": 2.5109, + "theoretical_loss": 3.3557468615049104, + "tokens_seen": 2622554112 + }, + { + "epoch": 0.59, + "learning_rate": 4.1462044615631526e-05, + "loss": 2.5897, + "theoretical_loss": 3.3557336116396117, + "tokens_seen": 2622685184 + }, + { + "epoch": 0.59, + "learning_rate": 4.1454020221473286e-05, + "loss": 2.4984, + "theoretical_loss": 3.355720362621874, + "tokens_seen": 2622816256 + }, + { + "epoch": 0.59, + "learning_rate": 4.144599582731504e-05, + "loss": 2.5924, + "theoretical_loss": 3.3557071144516017, + "tokens_seen": 2622947328 + }, + { + "epoch": 0.59, + "learning_rate": 4.14379714331568e-05, + "loss": 2.5467, + "theoretical_loss": 3.355693867128698, + "tokens_seen": 2623078400 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1436471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7493784427642822, + "objective/train/theoretical_loss": 3.3556806206530667, + "objective/train/tokens_used": 993668576, + "theoretical_loss": 3.3556806206530667, + "tokens_seen": 2623209472 + }, + { + "epoch": 0.59, + "learning_rate": 4.142994703899856e-05, + "loss": 2.5576, + "theoretical_loss": 3.3556806206530667, + "tokens_seen": 2623209472 + }, + { + "epoch": 0.59, + "learning_rate": 4.142192264484032e-05, + "loss": 2.6013, + "theoretical_loss": 3.3556673750246104, + "tokens_seen": 2623340544 + }, + { + "epoch": 0.59, + "learning_rate": 4.141389825068208e-05, + "loss": 2.4832, + "theoretical_loss": 3.3556541302432334, + "tokens_seen": 2623471616 + }, + { + "epoch": 0.59, + "learning_rate": 4.140587385652384e-05, + "loss": 2.4789, + "theoretical_loss": 3.3556408863088385, + "tokens_seen": 2623602688 + }, + { + "epoch": 0.59, + "learning_rate": 4.13978494623656e-05, + "loss": 2.597, + "theoretical_loss": 3.3556276432213297, + "tokens_seen": 2623733760 + }, + { + "epoch": 0.59, + "learning_rate": 4.138982506820735e-05, + "loss": 2.4543, + "theoretical_loss": 3.3556144009806106, + "tokens_seen": 2623864832 + }, + { + "epoch": 0.59, + "learning_rate": 4.138180067404911e-05, + "loss": 2.6191, + "theoretical_loss": 3.355601159586585, + "tokens_seen": 2623995904 + }, + { + "epoch": 0.59, + "learning_rate": 4.137377627989087e-05, + "loss": 2.5612, + "theoretical_loss": 3.3555879190391553, + "tokens_seen": 2624126976 + }, + { + "epoch": 0.59, + "learning_rate": 4.136575188573263e-05, + "loss": 2.4725, + "theoretical_loss": 3.3555746793382264, + "tokens_seen": 2624258048 + }, + { + "epoch": 0.59, + "learning_rate": 4.135772749157439e-05, + "loss": 2.2705, + "theoretical_loss": 3.3555614404837013, + "tokens_seen": 2624389120 + }, + { + "epoch": 0.59, + "learning_rate": 4.134970309741615e-05, + "loss": 2.4562, + "theoretical_loss": 3.3555482024754832, + "tokens_seen": 2624520192 + }, + { + "epoch": 0.59, + "learning_rate": 4.134167870325791e-05, + "loss": 2.498, + "theoretical_loss": 3.355534965313477, + "tokens_seen": 2624651264 + }, + { + "epoch": 0.59, + "learning_rate": 4.133365430909967e-05, + "loss": 2.5114, + "theoretical_loss": 3.3555217289975854, + "tokens_seen": 2624782336 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1436981, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3855788707733154, + "objective/train/theoretical_loss": 3.3555151111569024, + "objective/train/tokens_used": 995306976, + "theoretical_loss": 3.3555151111569024, + "tokens_seen": 2624847872 + }, + { + "epoch": 0.59, + "learning_rate": 4.1325629914941424e-05, + "loss": 2.5581, + "theoretical_loss": 3.3555084935277124, + "tokens_seen": 2624913408 + }, + { + "epoch": 0.59, + "learning_rate": 4.1317605520783184e-05, + "loss": 2.5813, + "theoretical_loss": 3.355495258903761, + "tokens_seen": 2625044480 + }, + { + "epoch": 0.59, + "learning_rate": 4.1309581126624944e-05, + "loss": 2.6089, + "theoretical_loss": 3.355482025125636, + "tokens_seen": 2625175552 + }, + { + "epoch": 0.59, + "learning_rate": 4.1301556732466704e-05, + "loss": 2.3194, + "theoretical_loss": 3.35546879219324, + "tokens_seen": 2625306624 + }, + { + "epoch": 0.59, + "learning_rate": 4.1293532338308464e-05, + "loss": 2.3851, + "theoretical_loss": 3.3554555601064777, + "tokens_seen": 2625437696 + }, + { + "epoch": 0.59, + "learning_rate": 4.1285507944150224e-05, + "loss": 2.5219, + "theoretical_loss": 3.3554423288652524, + "tokens_seen": 2625568768 + }, + { + "epoch": 0.59, + "learning_rate": 4.127748354999198e-05, + "loss": 2.5527, + "theoretical_loss": 3.355429098469468, + "tokens_seen": 2625699840 + }, + { + "epoch": 0.59, + "learning_rate": 4.1269459155833737e-05, + "loss": 2.4622, + "theoretical_loss": 3.3554158689190285, + "tokens_seen": 2625830912 + }, + { + "epoch": 0.59, + "learning_rate": 4.1261434761675496e-05, + "loss": 2.3983, + "theoretical_loss": 3.355402640213837, + "tokens_seen": 2625961984 + }, + { + "epoch": 0.59, + "learning_rate": 4.1253410367517256e-05, + "loss": 2.4699, + "theoretical_loss": 3.355389412353798, + "tokens_seen": 2626093056 + }, + { + "epoch": 0.59, + "learning_rate": 4.1245385973359016e-05, + "loss": 2.4312, + "theoretical_loss": 3.355376185338814, + "tokens_seen": 2626224128 + }, + { + "epoch": 0.59, + "learning_rate": 4.123736157920077e-05, + "loss": 2.5769, + "theoretical_loss": 3.355362959168791, + "tokens_seen": 2626355200 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1438359, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8393893241882324, + "objective/train/theoretical_loss": 3.3553497338436316, + "objective/train/tokens_used": 996945376, + "theoretical_loss": 3.3553497338436316, + "tokens_seen": 2626486272 + }, + { + "epoch": 0.59, + "learning_rate": 4.122933718504253e-05, + "loss": 2.5207, + "theoretical_loss": 3.3553497338436316, + "tokens_seen": 2626486272 + }, + { + "epoch": 0.59, + "learning_rate": 4.122131279088429e-05, + "loss": 2.4945, + "theoretical_loss": 3.3553365093632395, + "tokens_seen": 2626617344 + }, + { + "epoch": 0.59, + "learning_rate": 4.121328839672605e-05, + "loss": 2.54, + "theoretical_loss": 3.355323285727519, + "tokens_seen": 2626748416 + }, + { + "epoch": 0.59, + "learning_rate": 4.120526400256781e-05, + "loss": 2.5266, + "theoretical_loss": 3.3553100629363737, + "tokens_seen": 2626879488 + }, + { + "epoch": 0.59, + "learning_rate": 4.119723960840957e-05, + "loss": 2.5322, + "theoretical_loss": 3.3552968409897077, + "tokens_seen": 2627010560 + }, + { + "epoch": 0.59, + "learning_rate": 4.118921521425132e-05, + "loss": 2.5066, + "theoretical_loss": 3.355283619887425, + "tokens_seen": 2627141632 + }, + { + "epoch": 0.59, + "learning_rate": 4.118119082009308e-05, + "loss": 2.5955, + "theoretical_loss": 3.35527039962943, + "tokens_seen": 2627272704 + }, + { + "epoch": 0.59, + "learning_rate": 4.117316642593484e-05, + "loss": 2.5375, + "theoretical_loss": 3.3552571802156255, + "tokens_seen": 2627403776 + }, + { + "epoch": 0.59, + "learning_rate": 4.11651420317766e-05, + "loss": 2.4408, + "theoretical_loss": 3.3552439616459164, + "tokens_seen": 2627534848 + }, + { + "epoch": 0.59, + "learning_rate": 4.115711763761836e-05, + "loss": 2.5316, + "theoretical_loss": 3.355230743920207, + "tokens_seen": 2627665920 + }, + { + "epoch": 0.59, + "learning_rate": 4.114909324346012e-05, + "loss": 2.5714, + "theoretical_loss": 3.3552175270383997, + "tokens_seen": 2627796992 + }, + { + "epoch": 0.59, + "learning_rate": 4.1141068849301875e-05, + "loss": 2.5292, + "theoretical_loss": 3.3552043110004, + "tokens_seen": 2627928064 + }, + { + "epoch": 0.59, + "learning_rate": 4.1133044455143635e-05, + "loss": 2.482, + "theoretical_loss": 3.355191095806112, + "tokens_seen": 2628059136 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1438908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2825491428375244, + "objective/train/theoretical_loss": 3.35518448852533, + "objective/train/tokens_used": 998583776, + "theoretical_loss": 3.35518448852533, + "tokens_seen": 2628124672 + }, + { + "epoch": 0.59, + "learning_rate": 4.1125020060985395e-05, + "loss": 2.4398, + "theoretical_loss": 3.355177881455439, + "tokens_seen": 2628190208 + }, + { + "epoch": 0.59, + "learning_rate": 4.1116995666827154e-05, + "loss": 2.5495, + "theoretical_loss": 3.3551646679482854, + "tokens_seen": 2628321280 + }, + { + "epoch": 0.59, + "learning_rate": 4.1108971272668914e-05, + "loss": 2.4558, + "theoretical_loss": 3.355151455284555, + "tokens_seen": 2628452352 + }, + { + "epoch": 0.59, + "learning_rate": 4.1100946878510674e-05, + "loss": 2.4889, + "theoretical_loss": 3.355138243464153, + "tokens_seen": 2628583424 + }, + { + "epoch": 0.59, + "learning_rate": 4.109292248435243e-05, + "loss": 2.6328, + "theoretical_loss": 3.355125032486982, + "tokens_seen": 2628714496 + }, + { + "epoch": 0.59, + "learning_rate": 4.108489809019419e-05, + "loss": 2.6831, + "theoretical_loss": 3.355111822352947, + "tokens_seen": 2628845568 + }, + { + "epoch": 0.59, + "learning_rate": 4.107687369603595e-05, + "loss": 2.5482, + "theoretical_loss": 3.3550986130619522, + "tokens_seen": 2628976640 + }, + { + "epoch": 0.59, + "learning_rate": 4.106884930187771e-05, + "loss": 2.3317, + "theoretical_loss": 3.3550854046139014, + "tokens_seen": 2629107712 + }, + { + "epoch": 0.59, + "learning_rate": 4.106082490771947e-05, + "loss": 2.5663, + "theoretical_loss": 3.355072197008699, + "tokens_seen": 2629238784 + }, + { + "epoch": 0.59, + "learning_rate": 4.105280051356123e-05, + "loss": 2.4101, + "theoretical_loss": 3.3550589902462495, + "tokens_seen": 2629369856 + }, + { + "epoch": 0.59, + "learning_rate": 4.104477611940299e-05, + "loss": 2.6263, + "theoretical_loss": 3.3550457843264563, + "tokens_seen": 2629500928 + }, + { + "epoch": 0.59, + "learning_rate": 4.103675172524475e-05, + "loss": 2.4799, + "theoretical_loss": 3.355032579249224, + "tokens_seen": 2629632000 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1440219, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.028750419616699, + "objective/train/theoretical_loss": 3.3550193750144577, + "objective/train/tokens_used": 1000222176, + "theoretical_loss": 3.3550193750144577, + "tokens_seen": 2629763072 + }, + { + "epoch": 0.59, + "learning_rate": 4.10287273310865e-05, + "loss": 2.4726, + "theoretical_loss": 3.3550193750144577, + "tokens_seen": 2629763072 + }, + { + "epoch": 0.59, + "learning_rate": 4.102070293692826e-05, + "loss": 2.6196, + "theoretical_loss": 3.3550061716220605, + "tokens_seen": 2629894144 + }, + { + "epoch": 0.59, + "learning_rate": 4.101267854277002e-05, + "loss": 2.6574, + "theoretical_loss": 3.3549929690719376, + "tokens_seen": 2630025216 + }, + { + "epoch": 0.59, + "learning_rate": 4.100465414861178e-05, + "loss": 2.3955, + "theoretical_loss": 3.354979767363992, + "tokens_seen": 2630156288 + }, + { + "epoch": 0.59, + "learning_rate": 4.099662975445354e-05, + "loss": 2.4432, + "theoretical_loss": 3.3549665664981294, + "tokens_seen": 2630287360 + }, + { + "epoch": 0.59, + "learning_rate": 4.09886053602953e-05, + "loss": 2.3674, + "theoretical_loss": 3.3549533664742532, + "tokens_seen": 2630418432 + }, + { + "epoch": 0.59, + "learning_rate": 4.098058096613706e-05, + "loss": 2.3879, + "theoretical_loss": 3.3549401672922685, + "tokens_seen": 2630549504 + }, + { + "epoch": 0.59, + "learning_rate": 4.097255657197881e-05, + "loss": 2.5678, + "theoretical_loss": 3.3549269689520793, + "tokens_seen": 2630680576 + }, + { + "epoch": 0.59, + "learning_rate": 4.096453217782057e-05, + "loss": 2.5468, + "theoretical_loss": 3.3549137714535897, + "tokens_seen": 2630811648 + }, + { + "epoch": 0.59, + "learning_rate": 4.095650778366233e-05, + "loss": 2.4121, + "theoretical_loss": 3.3549005747967042, + "tokens_seen": 2630942720 + }, + { + "epoch": 0.59, + "learning_rate": 4.094848338950409e-05, + "loss": 2.3743, + "theoretical_loss": 3.3548873789813274, + "tokens_seen": 2631073792 + }, + { + "epoch": 0.59, + "learning_rate": 4.094045899534585e-05, + "loss": 2.471, + "theoretical_loss": 3.3548741840073637, + "tokens_seen": 2631204864 + }, + { + "epoch": 0.59, + "learning_rate": 4.093243460118761e-05, + "loss": 2.5331, + "theoretical_loss": 3.3548609898747177, + "tokens_seen": 2631335936 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1440922, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5051844120025635, + "objective/train/theoretical_loss": 3.3548543931238592, + "objective/train/tokens_used": 1001860576, + "theoretical_loss": 3.3548543931238592, + "tokens_seen": 2631401472 + }, + { + "epoch": 0.59, + "learning_rate": 4.092441020702937e-05, + "loss": 2.5506, + "theoretical_loss": 3.354847796583294, + "tokens_seen": 2631467008 + }, + { + "epoch": 0.59, + "learning_rate": 4.0916385812871125e-05, + "loss": 2.6356, + "theoretical_loss": 3.354834604132996, + "tokens_seen": 2631598080 + }, + { + "epoch": 0.6, + "learning_rate": 4.0908361418712885e-05, + "loss": 2.566, + "theoretical_loss": 3.3548214125237292, + "tokens_seen": 2631729152 + }, + { + "epoch": 0.6, + "learning_rate": 4.0900337024554645e-05, + "loss": 2.4861, + "theoretical_loss": 3.354808221755398, + "tokens_seen": 2631860224 + }, + { + "epoch": 0.6, + "learning_rate": 4.0892312630396405e-05, + "loss": 2.4938, + "theoretical_loss": 3.3547950318279063, + "tokens_seen": 2631991296 + }, + { + "epoch": 0.6, + "learning_rate": 4.0884288236238165e-05, + "loss": 2.3832, + "theoretical_loss": 3.3547818427411595, + "tokens_seen": 2632122368 + }, + { + "epoch": 0.6, + "learning_rate": 4.0876263842079925e-05, + "loss": 2.5016, + "theoretical_loss": 3.3547686544950612, + "tokens_seen": 2632253440 + }, + { + "epoch": 0.6, + "learning_rate": 4.0868239447921684e-05, + "loss": 2.5274, + "theoretical_loss": 3.3547554670895168, + "tokens_seen": 2632384512 + }, + { + "epoch": 0.6, + "learning_rate": 4.0860215053763444e-05, + "loss": 2.475, + "theoretical_loss": 3.3547422805244307, + "tokens_seen": 2632515584 + }, + { + "epoch": 0.6, + "learning_rate": 4.08521906596052e-05, + "loss": 2.4622, + "theoretical_loss": 3.354729094799707, + "tokens_seen": 2632646656 + }, + { + "epoch": 0.6, + "learning_rate": 4.084416626544696e-05, + "loss": 2.4274, + "theoretical_loss": 3.354715909915251, + "tokens_seen": 2632777728 + }, + { + "epoch": 0.6, + "learning_rate": 4.083614187128872e-05, + "loss": 2.5316, + "theoretical_loss": 3.3547027258709665, + "tokens_seen": 2632908800 + }, + { + "debugging/Self-BLEU-5": 0.4786452797172566, + "debugging/distinct-1-grams": 0.8009825579116673, + "debugging/distinct-2-grams": 0.976898537192655, + "debugging/entropy-1-grams": 5.427968226823284, + "debugging/entropy-2-grams": 6.077705974463511, + "debugging/length": 586.5, + "debugging/num_segments": 6, + "debugging/score": 0.005238309542870183, + "debugging/score_std": 0.0022859140799648295, + "epoch": 0.6, + "objective/train/docs_used": 1442386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5365583896636963, + "objective/train/theoretical_loss": 3.354689542666759, + "objective/train/tokens_used": 1003498976, + "theoretical_loss": 3.354689542666759, + "tokens_seen": 2633039872 + }, + { + "epoch": 0.6, + "learning_rate": 4.082811747713048e-05, + "loss": 2.5043, + "theoretical_loss": 3.354689542666759, + "tokens_seen": 2633039872 + }, + { + "epoch": 0.6, + "learning_rate": 4.082009308297224e-05, + "loss": 2.53, + "theoretical_loss": 3.3546763603025327, + "tokens_seen": 2633170944 + }, + { + "epoch": 0.6, + "learning_rate": 4.0812068688814e-05, + "loss": 2.5464, + "theoretical_loss": 3.354663178778192, + "tokens_seen": 2633302016 + }, + { + "epoch": 0.6, + "learning_rate": 4.080404429465576e-05, + "loss": 2.5042, + "theoretical_loss": 3.3546499980936426, + "tokens_seen": 2633433088 + }, + { + "epoch": 0.6, + "learning_rate": 4.079601990049751e-05, + "loss": 2.5303, + "theoretical_loss": 3.354636818248788, + "tokens_seen": 2633564160 + }, + { + "epoch": 0.6, + "learning_rate": 4.078799550633927e-05, + "loss": 2.4588, + "theoretical_loss": 3.354623639243534, + "tokens_seen": 2633695232 + }, + { + "epoch": 0.6, + "learning_rate": 4.077997111218103e-05, + "loss": 2.4629, + "theoretical_loss": 3.354610461077784, + "tokens_seen": 2633826304 + }, + { + "epoch": 0.6, + "learning_rate": 4.077194671802279e-05, + "loss": 2.5625, + "theoretical_loss": 3.354597283751444, + "tokens_seen": 2633957376 + }, + { + "epoch": 0.6, + "learning_rate": 4.076392232386455e-05, + "loss": 2.4216, + "theoretical_loss": 3.3545841072644187, + "tokens_seen": 2634088448 + }, + { + "epoch": 0.6, + "learning_rate": 4.075589792970631e-05, + "loss": 2.5635, + "theoretical_loss": 3.3545709316166117, + "tokens_seen": 2634219520 + }, + { + "epoch": 0.6, + "learning_rate": 4.074787353554807e-05, + "loss": 2.4422, + "theoretical_loss": 3.354557756807929, + "tokens_seen": 2634350592 + }, + { + "epoch": 0.6, + "learning_rate": 4.073984914138983e-05, + "loss": 2.5191, + "theoretical_loss": 3.354544582838275, + "tokens_seen": 2634481664 + }, + { + "epoch": 0.6, + "learning_rate": 4.073182474723158e-05, + "loss": 2.3669, + "theoretical_loss": 3.3545314097075547, + "tokens_seen": 2634612736 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1443077, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.969416856765747, + "objective/train/theoretical_loss": 3.3545248234567646, + "objective/train/tokens_used": 1005137376, + "theoretical_loss": 3.3545248234567646, + "tokens_seen": 2634678272 + }, + { + "epoch": 0.6, + "learning_rate": 4.072380035307334e-05, + "loss": 2.5501, + "theoretical_loss": 3.354518237415672, + "tokens_seen": 2634743808 + }, + { + "epoch": 0.6, + "learning_rate": 4.07157759589151e-05, + "loss": 2.62, + "theoretical_loss": 3.3545050659625333, + "tokens_seen": 2634874880 + }, + { + "epoch": 0.6, + "learning_rate": 4.070775156475686e-05, + "loss": 2.6452, + "theoretical_loss": 3.3544918953480423, + "tokens_seen": 2635005952 + }, + { + "epoch": 0.6, + "learning_rate": 4.069972717059862e-05, + "loss": 2.5859, + "theoretical_loss": 3.3544787255721045, + "tokens_seen": 2635137024 + }, + { + "epoch": 0.6, + "learning_rate": 4.069170277644038e-05, + "loss": 2.6151, + "theoretical_loss": 3.3544655566346244, + "tokens_seen": 2635268096 + }, + { + "epoch": 0.6, + "learning_rate": 4.068367838228214e-05, + "loss": 2.3741, + "theoretical_loss": 3.354452388535507, + "tokens_seen": 2635399168 + }, + { + "epoch": 0.6, + "learning_rate": 4.0675653988123895e-05, + "loss": 2.5628, + "theoretical_loss": 3.3544392212746574, + "tokens_seen": 2635530240 + }, + { + "epoch": 0.6, + "learning_rate": 4.0667629593965655e-05, + "loss": 2.5942, + "theoretical_loss": 3.354426054851981, + "tokens_seen": 2635661312 + }, + { + "epoch": 0.6, + "learning_rate": 4.0659605199807415e-05, + "loss": 2.5861, + "theoretical_loss": 3.354412889267382, + "tokens_seen": 2635792384 + }, + { + "epoch": 0.6, + "learning_rate": 4.0651580805649175e-05, + "loss": 2.4053, + "theoretical_loss": 3.354399724520765, + "tokens_seen": 2635923456 + }, + { + "epoch": 0.6, + "learning_rate": 4.0643556411490935e-05, + "loss": 2.5477, + "theoretical_loss": 3.3543865606120367, + "tokens_seen": 2636054528 + }, + { + "epoch": 0.6, + "learning_rate": 4.0635532017332695e-05, + "loss": 2.5193, + "theoretical_loss": 3.3543733975411003, + "tokens_seen": 2636185600 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1444286, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.795438528060913, + "objective/train/theoretical_loss": 3.354360235307862, + "objective/train/tokens_used": 1006775776, + "theoretical_loss": 3.354360235307862, + "tokens_seen": 2636316672 + }, + { + "epoch": 0.6, + "learning_rate": 4.0627507623174455e-05, + "loss": 2.5678, + "theoretical_loss": 3.354360235307862, + "tokens_seen": 2636316672 + }, + { + "epoch": 0.6, + "learning_rate": 4.061948322901621e-05, + "loss": 2.424, + "theoretical_loss": 3.3543470739122263, + "tokens_seen": 2636447744 + }, + { + "epoch": 0.6, + "learning_rate": 4.061145883485797e-05, + "loss": 2.4229, + "theoretical_loss": 3.354333913354098, + "tokens_seen": 2636578816 + }, + { + "epoch": 0.6, + "learning_rate": 4.060343444069973e-05, + "loss": 2.538, + "theoretical_loss": 3.354320753633383, + "tokens_seen": 2636709888 + }, + { + "epoch": 0.6, + "learning_rate": 4.059541004654149e-05, + "loss": 2.3964, + "theoretical_loss": 3.354307594749986, + "tokens_seen": 2636840960 + }, + { + "epoch": 0.6, + "learning_rate": 4.058738565238325e-05, + "loss": 2.386, + "theoretical_loss": 3.3542944367038117, + "tokens_seen": 2636972032 + }, + { + "epoch": 0.6, + "learning_rate": 4.057936125822501e-05, + "loss": 2.5858, + "theoretical_loss": 3.354281279494766, + "tokens_seen": 2637103104 + }, + { + "epoch": 0.6, + "learning_rate": 4.057133686406677e-05, + "loss": 2.5051, + "theoretical_loss": 3.354268123122753, + "tokens_seen": 2637234176 + }, + { + "epoch": 0.6, + "learning_rate": 4.056331246990853e-05, + "loss": 2.523, + "theoretical_loss": 3.3542549675876794, + "tokens_seen": 2637365248 + }, + { + "epoch": 0.6, + "learning_rate": 4.055528807575028e-05, + "loss": 2.4498, + "theoretical_loss": 3.3542418128894487, + "tokens_seen": 2637496320 + }, + { + "epoch": 0.6, + "learning_rate": 4.054726368159204e-05, + "loss": 2.4684, + "theoretical_loss": 3.3542286590279673, + "tokens_seen": 2637627392 + }, + { + "epoch": 0.6, + "learning_rate": 4.05392392874338e-05, + "loss": 2.5872, + "theoretical_loss": 3.3542155060031393, + "tokens_seen": 2637758464 + }, + { + "epoch": 0.6, + "learning_rate": 4.053121489327556e-05, + "loss": 2.5297, + "theoretical_loss": 3.354202353814871, + "tokens_seen": 2637889536 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1445507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5415756702423096, + "objective/train/theoretical_loss": 3.354195778034417, + "objective/train/tokens_used": 1008414176, + "theoretical_loss": 3.354195778034417, + "tokens_seen": 2637955072 + }, + { + "epoch": 0.6, + "learning_rate": 4.052319049911732e-05, + "loss": 2.5232, + "theoretical_loss": 3.354189202463067, + "tokens_seen": 2638020608 + }, + { + "epoch": 0.6, + "learning_rate": 4.051516610495908e-05, + "loss": 2.5729, + "theoretical_loss": 3.3541760519476327, + "tokens_seen": 2638151680 + }, + { + "epoch": 0.6, + "learning_rate": 4.050714171080084e-05, + "loss": 2.4598, + "theoretical_loss": 3.354162902268474, + "tokens_seen": 2638282752 + }, + { + "epoch": 0.6, + "learning_rate": 4.049911731664259e-05, + "loss": 2.6146, + "theoretical_loss": 3.354149753425495, + "tokens_seen": 2638413824 + }, + { + "epoch": 0.6, + "learning_rate": 4.049109292248435e-05, + "loss": 2.466, + "theoretical_loss": 3.354136605418601, + "tokens_seen": 2638544896 + }, + { + "epoch": 0.6, + "learning_rate": 4.048306852832611e-05, + "loss": 2.3942, + "theoretical_loss": 3.354123458247699, + "tokens_seen": 2638675968 + }, + { + "epoch": 0.6, + "learning_rate": 4.047504413416787e-05, + "loss": 2.5936, + "theoretical_loss": 3.3541103119126925, + "tokens_seen": 2638807040 + }, + { + "epoch": 0.6, + "learning_rate": 4.046701974000963e-05, + "loss": 2.5739, + "theoretical_loss": 3.3540971664134873, + "tokens_seen": 2638938112 + }, + { + "epoch": 0.6, + "learning_rate": 4.045899534585139e-05, + "loss": 2.467, + "theoretical_loss": 3.3540840217499897, + "tokens_seen": 2639069184 + }, + { + "epoch": 0.6, + "learning_rate": 4.045097095169315e-05, + "loss": 2.5329, + "theoretical_loss": 3.3540708779221036, + "tokens_seen": 2639200256 + }, + { + "epoch": 0.6, + "learning_rate": 4.0442946557534905e-05, + "loss": 2.5885, + "theoretical_loss": 3.3540577349297354, + "tokens_seen": 2639331328 + }, + { + "epoch": 0.6, + "learning_rate": 4.0434922163376665e-05, + "loss": 2.4019, + "theoretical_loss": 3.35404459277279, + "tokens_seen": 2639462400 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1446254, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5814332962036133, + "objective/train/theoretical_loss": 3.3540314514511733, + "objective/train/tokens_used": 1010052576, + "theoretical_loss": 3.3540314514511733, + "tokens_seen": 2639593472 + }, + { + "epoch": 0.6, + "learning_rate": 4.0426897769218425e-05, + "loss": 2.4546, + "theoretical_loss": 3.3540314514511733, + "tokens_seen": 2639593472 + }, + { + "epoch": 0.6, + "learning_rate": 4.0418873375060185e-05, + "loss": 2.5064, + "theoretical_loss": 3.3540183109647908, + "tokens_seen": 2639724544 + }, + { + "epoch": 0.6, + "learning_rate": 4.0410848980901945e-05, + "loss": 2.482, + "theoretical_loss": 3.3540051713135473, + "tokens_seen": 2639855616 + }, + { + "epoch": 0.6, + "learning_rate": 4.0402824586743705e-05, + "loss": 2.5414, + "theoretical_loss": 3.3539920324973482, + "tokens_seen": 2639986688 + }, + { + "epoch": 0.6, + "learning_rate": 4.0394800192585465e-05, + "loss": 2.5489, + "theoretical_loss": 3.3539788945160995, + "tokens_seen": 2640117760 + }, + { + "epoch": 0.6, + "learning_rate": 4.0386775798427225e-05, + "loss": 2.5457, + "theoretical_loss": 3.3539657573697066, + "tokens_seen": 2640248832 + }, + { + "epoch": 0.6, + "learning_rate": 4.037875140426898e-05, + "loss": 2.4012, + "theoretical_loss": 3.3539526210580752, + "tokens_seen": 2640379904 + }, + { + "epoch": 0.6, + "learning_rate": 4.037072701011074e-05, + "loss": 2.4671, + "theoretical_loss": 3.3539394855811104, + "tokens_seen": 2640510976 + }, + { + "epoch": 0.6, + "learning_rate": 4.03627026159525e-05, + "loss": 2.6483, + "theoretical_loss": 3.3539263509387176, + "tokens_seen": 2640642048 + }, + { + "epoch": 0.6, + "learning_rate": 4.035467822179426e-05, + "loss": 2.5163, + "theoretical_loss": 3.3539132171308035, + "tokens_seen": 2640773120 + }, + { + "epoch": 0.6, + "learning_rate": 4.034665382763602e-05, + "loss": 2.439, + "theoretical_loss": 3.353900084157272, + "tokens_seen": 2640904192 + }, + { + "epoch": 0.6, + "learning_rate": 4.033862943347778e-05, + "loss": 2.4362, + "theoretical_loss": 3.35388695201803, + "tokens_seen": 2641035264 + }, + { + "epoch": 0.6, + "learning_rate": 4.033060503931954e-05, + "loss": 2.3732, + "theoretical_loss": 3.353873820712982, + "tokens_seen": 2641166336 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1447304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.505378246307373, + "objective/train/theoretical_loss": 3.3538672553732516, + "objective/train/tokens_used": 1011690976, + "theoretical_loss": 3.3538672553732516, + "tokens_seen": 2641231872 + }, + { + "epoch": 0.6, + "learning_rate": 4.032258064516129e-05, + "loss": 2.6594, + "theoretical_loss": 3.3538606902420347, + "tokens_seen": 2641297408 + }, + { + "epoch": 0.6, + "learning_rate": 4.031455625100305e-05, + "loss": 2.5894, + "theoretical_loss": 3.353847560605093, + "tokens_seen": 2641428480 + }, + { + "epoch": 0.6, + "learning_rate": 4.030653185684481e-05, + "loss": 2.4465, + "theoretical_loss": 3.3538344318020634, + "tokens_seen": 2641559552 + }, + { + "epoch": 0.6, + "learning_rate": 4.029850746268657e-05, + "loss": 2.4532, + "theoretical_loss": 3.3538213038328504, + "tokens_seen": 2641690624 + }, + { + "epoch": 0.6, + "learning_rate": 4.029048306852833e-05, + "loss": 2.439, + "theoretical_loss": 3.3538081766973606, + "tokens_seen": 2641821696 + }, + { + "epoch": 0.6, + "learning_rate": 4.028245867437009e-05, + "loss": 2.3288, + "theoretical_loss": 3.353795050395499, + "tokens_seen": 2641952768 + }, + { + "epoch": 0.6, + "learning_rate": 4.027443428021185e-05, + "loss": 2.4636, + "theoretical_loss": 3.353781924927172, + "tokens_seen": 2642083840 + }, + { + "epoch": 0.6, + "learning_rate": 4.026640988605361e-05, + "loss": 2.6964, + "theoretical_loss": 3.3537688002922845, + "tokens_seen": 2642214912 + }, + { + "epoch": 0.6, + "learning_rate": 4.025838549189536e-05, + "loss": 2.3704, + "theoretical_loss": 3.353755676490743, + "tokens_seen": 2642345984 + }, + { + "epoch": 0.6, + "learning_rate": 4.025036109773712e-05, + "loss": 2.3929, + "theoretical_loss": 3.353742553522453, + "tokens_seen": 2642477056 + }, + { + "epoch": 0.6, + "learning_rate": 4.024233670357888e-05, + "loss": 2.5517, + "theoretical_loss": 3.35372943138732, + "tokens_seen": 2642608128 + }, + { + "epoch": 0.6, + "learning_rate": 4.023431230942064e-05, + "loss": 2.3499, + "theoretical_loss": 3.3537163100852503, + "tokens_seen": 2642739200 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1447854, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.72699236869812, + "objective/train/theoretical_loss": 3.353703189616149, + "objective/train/tokens_used": 1013329376, + "theoretical_loss": 3.353703189616149, + "tokens_seen": 2642870272 + }, + { + "epoch": 0.6, + "learning_rate": 4.02262879152624e-05, + "loss": 2.6275, + "theoretical_loss": 3.353703189616149, + "tokens_seen": 2642870272 + }, + { + "epoch": 0.6, + "learning_rate": 4.021826352110416e-05, + "loss": 2.3837, + "theoretical_loss": 3.3536900699799226, + "tokens_seen": 2643001344 + }, + { + "epoch": 0.6, + "learning_rate": 4.021023912694592e-05, + "loss": 2.5025, + "theoretical_loss": 3.353676951176477, + "tokens_seen": 2643132416 + }, + { + "epoch": 0.6, + "learning_rate": 4.0202214732787675e-05, + "loss": 2.5403, + "theoretical_loss": 3.353663833205717, + "tokens_seen": 2643263488 + }, + { + "epoch": 0.6, + "learning_rate": 4.0194190338629435e-05, + "loss": 2.4246, + "theoretical_loss": 3.3536507160675493, + "tokens_seen": 2643394560 + }, + { + "epoch": 0.6, + "learning_rate": 4.0186165944471195e-05, + "loss": 2.4235, + "theoretical_loss": 3.35363759976188, + "tokens_seen": 2643525632 + }, + { + "epoch": 0.6, + "learning_rate": 4.0178141550312955e-05, + "loss": 2.6227, + "theoretical_loss": 3.3536244842886145, + "tokens_seen": 2643656704 + }, + { + "epoch": 0.6, + "learning_rate": 4.0170117156154715e-05, + "loss": 2.4804, + "theoretical_loss": 3.3536113696476586, + "tokens_seen": 2643787776 + }, + { + "epoch": 0.6, + "learning_rate": 4.0162092761996475e-05, + "loss": 2.5207, + "theoretical_loss": 3.353598255838919, + "tokens_seen": 2643918848 + }, + { + "epoch": 0.6, + "learning_rate": 4.0154068367838235e-05, + "loss": 2.4021, + "theoretical_loss": 3.3535851428623005, + "tokens_seen": 2644049920 + }, + { + "epoch": 0.6, + "learning_rate": 4.014604397367999e-05, + "loss": 2.4553, + "theoretical_loss": 3.3535720307177095, + "tokens_seen": 2644180992 + }, + { + "epoch": 0.6, + "learning_rate": 4.013801957952175e-05, + "loss": 2.4578, + "theoretical_loss": 3.3535589194050526, + "tokens_seen": 2644312064 + }, + { + "epoch": 0.6, + "learning_rate": 4.012999518536351e-05, + "loss": 2.3346, + "theoretical_loss": 3.353545808924235, + "tokens_seen": 2644443136 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1449222, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.46644926071167, + "objective/train/theoretical_loss": 3.353539253995737, + "objective/train/tokens_used": 1014967776, + "theoretical_loss": 3.353539253995737, + "tokens_seen": 2644508672 + }, + { + "epoch": 0.6, + "learning_rate": 4.012197079120527e-05, + "loss": 2.4816, + "theoretical_loss": 3.353532699275163, + "tokens_seen": 2644574208 + }, + { + "epoch": 0.6, + "learning_rate": 4.011394639704703e-05, + "loss": 2.5373, + "theoretical_loss": 3.353519590457743, + "tokens_seen": 2644705280 + }, + { + "epoch": 0.6, + "learning_rate": 4.010592200288879e-05, + "loss": 2.4724, + "theoretical_loss": 3.3535064824718805, + "tokens_seen": 2644836352 + }, + { + "epoch": 0.6, + "learning_rate": 4.009789760873055e-05, + "loss": 2.4983, + "theoretical_loss": 3.3534933753174814, + "tokens_seen": 2644967424 + }, + { + "epoch": 0.6, + "learning_rate": 4.008987321457231e-05, + "loss": 2.4354, + "theoretical_loss": 3.353480268994452, + "tokens_seen": 2645098496 + }, + { + "epoch": 0.6, + "learning_rate": 4.008184882041406e-05, + "loss": 2.4113, + "theoretical_loss": 3.3534671635026987, + "tokens_seen": 2645229568 + }, + { + "epoch": 0.6, + "learning_rate": 4.007382442625582e-05, + "loss": 2.529, + "theoretical_loss": 3.353454058842127, + "tokens_seen": 2645360640 + }, + { + "epoch": 0.6, + "learning_rate": 4.006580003209758e-05, + "loss": 2.5502, + "theoretical_loss": 3.3534409550126436, + "tokens_seen": 2645491712 + }, + { + "epoch": 0.6, + "learning_rate": 4.005777563793934e-05, + "loss": 2.484, + "theoretical_loss": 3.3534278520141543, + "tokens_seen": 2645622784 + }, + { + "epoch": 0.6, + "learning_rate": 4.00497512437811e-05, + "loss": 2.584, + "theoretical_loss": 3.353414749846565, + "tokens_seen": 2645753856 + }, + { + "epoch": 0.6, + "learning_rate": 4.004172684962286e-05, + "loss": 2.3622, + "theoretical_loss": 3.353401648509782, + "tokens_seen": 2645884928 + }, + { + "epoch": 0.6, + "learning_rate": 4.003370245546462e-05, + "loss": 2.3651, + "theoretical_loss": 3.353388548003712, + "tokens_seen": 2646016000 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1449871, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.359598398208618, + "objective/train/theoretical_loss": 3.3533754483282605, + "objective/train/tokens_used": 1016606176, + "theoretical_loss": 3.3533754483282605, + "tokens_seen": 2646147072 + }, + { + "epoch": 0.6, + "learning_rate": 4.002567806130637e-05, + "loss": 2.4631, + "theoretical_loss": 3.3533754483282605, + "tokens_seen": 2646147072 + }, + { + "epoch": 0.6, + "learning_rate": 4.001765366714813e-05, + "loss": 2.4783, + "theoretical_loss": 3.3533623494833344, + "tokens_seen": 2646278144 + }, + { + "epoch": 0.6, + "learning_rate": 4.000962927298989e-05, + "loss": 2.406, + "theoretical_loss": 3.353349251468839, + "tokens_seen": 2646409216 + }, + { + "epoch": 0.6, + "learning_rate": 4.000160487883165e-05, + "loss": 2.5588, + "theoretical_loss": 3.353336154284681, + "tokens_seen": 2646540288 + }, + { + "epoch": 0.6, + "learning_rate": 3.999358048467341e-05, + "loss": 2.4229, + "theoretical_loss": 3.353323057930767, + "tokens_seen": 2646671360 + }, + { + "epoch": 0.6, + "learning_rate": 3.998555609051517e-05, + "loss": 2.5586, + "theoretical_loss": 3.3533099624070024, + "tokens_seen": 2646802432 + }, + { + "epoch": 0.6, + "learning_rate": 3.997753169635693e-05, + "loss": 2.3579, + "theoretical_loss": 3.3532968677132944, + "tokens_seen": 2646933504 + }, + { + "epoch": 0.6, + "learning_rate": 3.9969507302198685e-05, + "loss": 2.5687, + "theoretical_loss": 3.3532837738495487, + "tokens_seen": 2647064576 + }, + { + "epoch": 0.6, + "learning_rate": 3.9961482908040445e-05, + "loss": 2.6219, + "theoretical_loss": 3.3532706808156716, + "tokens_seen": 2647195648 + }, + { + "epoch": 0.6, + "learning_rate": 3.9953458513882205e-05, + "loss": 2.595, + "theoretical_loss": 3.3532575886115694, + "tokens_seen": 2647326720 + }, + { + "epoch": 0.6, + "learning_rate": 3.9945434119723965e-05, + "loss": 2.5016, + "theoretical_loss": 3.3532444972371493, + "tokens_seen": 2647457792 + }, + { + "epoch": 0.6, + "learning_rate": 3.9937409725565725e-05, + "loss": 2.5753, + "theoretical_loss": 3.3532314066923163, + "tokens_seen": 2647588864 + }, + { + "epoch": 0.6, + "learning_rate": 3.9929385331407485e-05, + "loss": 2.3992, + "theoretical_loss": 3.3532183169769776, + "tokens_seen": 2647719936 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1450926, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4421584606170654, + "objective/train/theoretical_loss": 3.3532117724303396, + "objective/train/tokens_used": 1018244576, + "theoretical_loss": 3.3532117724303396, + "tokens_seen": 2647785472 + }, + { + "epoch": 0.6, + "learning_rate": 3.992136093724924e-05, + "loss": 2.5085, + "theoretical_loss": 3.35320522809104, + "tokens_seen": 2647851008 + }, + { + "epoch": 0.6, + "learning_rate": 3.9913336543091e-05, + "loss": 2.4124, + "theoretical_loss": 3.3531921400344085, + "tokens_seen": 2647982080 + }, + { + "epoch": 0.6, + "learning_rate": 3.990531214893276e-05, + "loss": 2.4521, + "theoretical_loss": 3.3531790528069907, + "tokens_seen": 2648113152 + }, + { + "epoch": 0.61, + "learning_rate": 3.989728775477452e-05, + "loss": 2.5839, + "theoretical_loss": 3.3531659664086924, + "tokens_seen": 2648244224 + }, + { + "epoch": 0.61, + "learning_rate": 3.988926336061628e-05, + "loss": 2.4879, + "theoretical_loss": 3.3531528808394206, + "tokens_seen": 2648375296 + }, + { + "epoch": 0.61, + "learning_rate": 3.988123896645804e-05, + "loss": 2.446, + "theoretical_loss": 3.353139796099081, + "tokens_seen": 2648506368 + }, + { + "epoch": 0.61, + "learning_rate": 3.987321457229979e-05, + "loss": 2.376, + "theoretical_loss": 3.3531267121875805, + "tokens_seen": 2648637440 + }, + { + "epoch": 0.61, + "learning_rate": 3.986519017814155e-05, + "loss": 2.5896, + "theoretical_loss": 3.353113629104826, + "tokens_seen": 2648768512 + }, + { + "epoch": 0.61, + "learning_rate": 3.985716578398331e-05, + "loss": 2.2848, + "theoretical_loss": 3.3531005468507233, + "tokens_seen": 2648899584 + }, + { + "epoch": 0.61, + "learning_rate": 3.984914138982507e-05, + "loss": 2.5185, + "theoretical_loss": 3.353087465425179, + "tokens_seen": 2649030656 + }, + { + "epoch": 0.61, + "learning_rate": 3.984111699566683e-05, + "loss": 2.4667, + "theoretical_loss": 3.3530743848281004, + "tokens_seen": 2649161728 + }, + { + "epoch": 0.61, + "learning_rate": 3.983309260150859e-05, + "loss": 2.5637, + "theoretical_loss": 3.353061305059393, + "tokens_seen": 2649292800 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1451446, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.511380672454834, + "objective/train/theoretical_loss": 3.3530482261189642, + "objective/train/tokens_used": 1019882976, + "theoretical_loss": 3.3530482261189642, + "tokens_seen": 2649423872 + }, + { + "epoch": 0.61, + "learning_rate": 3.9825068207350343e-05, + "loss": 2.4012, + "theoretical_loss": 3.3530482261189642, + "tokens_seen": 2649423872 + }, + { + "epoch": 0.61, + "learning_rate": 3.98170438131921e-05, + "loss": 2.4531, + "theoretical_loss": 3.35303514800672, + "tokens_seen": 2649554944 + }, + { + "epoch": 0.61, + "learning_rate": 3.980901941903386e-05, + "loss": 2.3522, + "theoretical_loss": 3.3530220707225675, + "tokens_seen": 2649686016 + }, + { + "epoch": 0.61, + "learning_rate": 3.980099502487562e-05, + "loss": 2.431, + "theoretical_loss": 3.3530089942664123, + "tokens_seen": 2649817088 + }, + { + "epoch": 0.61, + "learning_rate": 3.979297063071738e-05, + "loss": 2.5108, + "theoretical_loss": 3.3529959186381624, + "tokens_seen": 2649948160 + }, + { + "epoch": 0.61, + "learning_rate": 3.978494623655914e-05, + "loss": 2.5979, + "theoretical_loss": 3.3529828438377236, + "tokens_seen": 2650079232 + }, + { + "epoch": 0.61, + "learning_rate": 3.9776921842400896e-05, + "loss": 2.433, + "theoretical_loss": 3.3529697698650027, + "tokens_seen": 2650210304 + }, + { + "epoch": 0.61, + "learning_rate": 3.9768897448242656e-05, + "loss": 2.3614, + "theoretical_loss": 3.352956696719906, + "tokens_seen": 2650341376 + }, + { + "epoch": 0.61, + "learning_rate": 3.9760873054084416e-05, + "loss": 2.5456, + "theoretical_loss": 3.352943624402341, + "tokens_seen": 2650472448 + }, + { + "epoch": 0.61, + "learning_rate": 3.9752848659926176e-05, + "loss": 2.3646, + "theoretical_loss": 3.352930552912214, + "tokens_seen": 2650603520 + }, + { + "epoch": 0.61, + "learning_rate": 3.9744824265767936e-05, + "loss": 2.5729, + "theoretical_loss": 3.3529174822494316, + "tokens_seen": 2650734592 + }, + { + "epoch": 0.61, + "learning_rate": 3.9736799871609696e-05, + "loss": 2.3373, + "theoretical_loss": 3.3529044124139005, + "tokens_seen": 2650865664 + }, + { + "epoch": 0.61, + "learning_rate": 3.972877547745145e-05, + "loss": 2.5708, + "theoretical_loss": 3.3528913434055276, + "tokens_seen": 2650996736 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1452778, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3329825401306152, + "objective/train/theoretical_loss": 3.3528848092114965, + "objective/train/tokens_used": 1021521376, + "theoretical_loss": 3.3528848092114965, + "tokens_seen": 2651062272 + }, + { + "epoch": 0.61, + "learning_rate": 3.972075108329321e-05, + "loss": 2.6755, + "theoretical_loss": 3.35287827522422, + "tokens_seen": 2651127808 + }, + { + "epoch": 0.61, + "learning_rate": 3.971272668913497e-05, + "loss": 2.5231, + "theoretical_loss": 3.3528652078698835, + "tokens_seen": 2651258880 + }, + { + "epoch": 0.61, + "learning_rate": 3.970470229497673e-05, + "loss": 2.5008, + "theoretical_loss": 3.3528521413424257, + "tokens_seen": 2651389952 + }, + { + "epoch": 0.61, + "learning_rate": 3.969667790081849e-05, + "loss": 2.6155, + "theoretical_loss": 3.352839075641753, + "tokens_seen": 2651521024 + }, + { + "epoch": 0.61, + "learning_rate": 3.968865350666025e-05, + "loss": 2.5128, + "theoretical_loss": 3.3528260107677728, + "tokens_seen": 2651652096 + }, + { + "epoch": 0.61, + "learning_rate": 3.968062911250201e-05, + "loss": 2.5155, + "theoretical_loss": 3.3528129467203915, + "tokens_seen": 2651783168 + }, + { + "epoch": 0.61, + "learning_rate": 3.967260471834376e-05, + "loss": 2.5497, + "theoretical_loss": 3.352799883499516, + "tokens_seen": 2651914240 + }, + { + "epoch": 0.61, + "learning_rate": 3.966458032418552e-05, + "loss": 2.469, + "theoretical_loss": 3.3527868211050524, + "tokens_seen": 2652045312 + }, + { + "epoch": 0.61, + "learning_rate": 3.965655593002728e-05, + "loss": 2.5772, + "theoretical_loss": 3.352773759536909, + "tokens_seen": 2652176384 + }, + { + "epoch": 0.61, + "learning_rate": 3.964853153586904e-05, + "loss": 2.4707, + "theoretical_loss": 3.352760698794992, + "tokens_seen": 2652307456 + }, + { + "epoch": 0.61, + "learning_rate": 3.96405071417108e-05, + "loss": 2.4533, + "theoretical_loss": 3.352747638879208, + "tokens_seen": 2652438528 + }, + { + "epoch": 0.61, + "learning_rate": 3.963248274755256e-05, + "loss": 2.3952, + "theoretical_loss": 3.3527345797894648, + "tokens_seen": 2652569600 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1453384, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5713400840759277, + "objective/train/theoretical_loss": 3.3527215215256687, + "objective/train/tokens_used": 1023159776, + "theoretical_loss": 3.3527215215256687, + "tokens_seen": 2652700672 + }, + { + "epoch": 0.61, + "learning_rate": 3.962445835339432e-05, + "loss": 2.3672, + "theoretical_loss": 3.3527215215256687, + "tokens_seen": 2652700672 + }, + { + "epoch": 0.61, + "learning_rate": 3.961643395923608e-05, + "loss": 2.4453, + "theoretical_loss": 3.3527084640877263, + "tokens_seen": 2652831744 + }, + { + "epoch": 0.61, + "learning_rate": 3.9608409565077834e-05, + "loss": 2.5298, + "theoretical_loss": 3.352695407475545, + "tokens_seen": 2652962816 + }, + { + "epoch": 0.61, + "learning_rate": 3.9600385170919594e-05, + "loss": 2.4367, + "theoretical_loss": 3.3526823516890323, + "tokens_seen": 2653093888 + }, + { + "epoch": 0.61, + "learning_rate": 3.9592360776761354e-05, + "loss": 2.3877, + "theoretical_loss": 3.352669296728094, + "tokens_seen": 2653224960 + }, + { + "epoch": 0.61, + "learning_rate": 3.9584336382603114e-05, + "loss": 2.4565, + "theoretical_loss": 3.3526562425926385, + "tokens_seen": 2653356032 + }, + { + "epoch": 0.61, + "learning_rate": 3.9576311988444873e-05, + "loss": 2.4306, + "theoretical_loss": 3.352643189282572, + "tokens_seen": 2653487104 + }, + { + "epoch": 0.61, + "learning_rate": 3.956828759428663e-05, + "loss": 2.4806, + "theoretical_loss": 3.3526301367978015, + "tokens_seen": 2653618176 + }, + { + "epoch": 0.61, + "learning_rate": 3.956026320012839e-05, + "loss": 2.4515, + "theoretical_loss": 3.352617085138234, + "tokens_seen": 2653749248 + }, + { + "epoch": 0.61, + "learning_rate": 3.9552238805970146e-05, + "loss": 2.5095, + "theoretical_loss": 3.3526040343037775, + "tokens_seen": 2653880320 + }, + { + "epoch": 0.61, + "learning_rate": 3.9544214411811906e-05, + "loss": 2.6099, + "theoretical_loss": 3.352590984294338, + "tokens_seen": 2654011392 + }, + { + "epoch": 0.61, + "learning_rate": 3.9536190017653666e-05, + "loss": 2.5294, + "theoretical_loss": 3.352577935109823, + "tokens_seen": 2654142464 + }, + { + "epoch": 0.61, + "learning_rate": 3.9528165623495426e-05, + "loss": 2.5502, + "theoretical_loss": 3.35256488675014, + "tokens_seen": 2654273536 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1454467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.468717336654663, + "objective/train/theoretical_loss": 3.3525583628795808, + "objective/train/tokens_used": 1024798176, + "theoretical_loss": 3.3525583628795808, + "tokens_seen": 2654339072 + }, + { + "epoch": 0.61, + "learning_rate": 3.9520141229337186e-05, + "loss": 2.5473, + "theoretical_loss": 3.352551839215195, + "tokens_seen": 2654404608 + }, + { + "epoch": 0.61, + "learning_rate": 3.9512116835178946e-05, + "loss": 2.4289, + "theoretical_loss": 3.352538792504897, + "tokens_seen": 2654535680 + }, + { + "epoch": 0.61, + "learning_rate": 3.9504092441020706e-05, + "loss": 2.4877, + "theoretical_loss": 3.3525257466191514, + "tokens_seen": 2654666752 + }, + { + "epoch": 0.61, + "learning_rate": 3.949606804686246e-05, + "loss": 2.5818, + "theoretical_loss": 3.352512701557866, + "tokens_seen": 2654797824 + }, + { + "epoch": 0.61, + "learning_rate": 3.948804365270422e-05, + "loss": 2.4682, + "theoretical_loss": 3.3524996573209482, + "tokens_seen": 2654928896 + }, + { + "epoch": 0.61, + "learning_rate": 3.948001925854598e-05, + "loss": 2.6306, + "theoretical_loss": 3.3524866139083054, + "tokens_seen": 2655059968 + }, + { + "epoch": 0.61, + "learning_rate": 3.947199486438774e-05, + "loss": 2.4517, + "theoretical_loss": 3.352473571319844, + "tokens_seen": 2655191040 + }, + { + "epoch": 0.61, + "learning_rate": 3.94639704702295e-05, + "loss": 2.5559, + "theoretical_loss": 3.352460529555472, + "tokens_seen": 2655322112 + }, + { + "epoch": 0.61, + "learning_rate": 3.945594607607126e-05, + "loss": 2.4411, + "theoretical_loss": 3.3524474886150966, + "tokens_seen": 2655453184 + }, + { + "epoch": 0.61, + "learning_rate": 3.944792168191302e-05, + "loss": 2.3976, + "theoretical_loss": 3.3524344484986246, + "tokens_seen": 2655584256 + }, + { + "epoch": 0.61, + "learning_rate": 3.943989728775478e-05, + "loss": 2.46, + "theoretical_loss": 3.3524214092059634, + "tokens_seen": 2655715328 + }, + { + "epoch": 0.61, + "learning_rate": 3.943187289359653e-05, + "loss": 2.4244, + "theoretical_loss": 3.3524083707370207, + "tokens_seen": 2655846400 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1455118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5718674659729004, + "objective/train/theoretical_loss": 3.3523953330917036, + "objective/train/tokens_used": 1026436576, + "theoretical_loss": 3.3523953330917036, + "tokens_seen": 2655977472 + }, + { + "epoch": 0.61, + "learning_rate": 3.942384849943829e-05, + "loss": 2.4623, + "theoretical_loss": 3.3523953330917036, + "tokens_seen": 2655977472 + }, + { + "epoch": 0.61, + "learning_rate": 3.941582410528005e-05, + "loss": 2.5067, + "theoretical_loss": 3.3523822962699192, + "tokens_seen": 2656108544 + }, + { + "epoch": 0.61, + "learning_rate": 3.940779971112181e-05, + "loss": 2.3734, + "theoretical_loss": 3.3523692602715753, + "tokens_seen": 2656239616 + }, + { + "epoch": 0.61, + "learning_rate": 3.939977531696357e-05, + "loss": 2.5545, + "theoretical_loss": 3.3523562250965786, + "tokens_seen": 2656370688 + }, + { + "epoch": 0.61, + "learning_rate": 3.939175092280533e-05, + "loss": 2.4945, + "theoretical_loss": 3.352343190744837, + "tokens_seen": 2656501760 + }, + { + "epoch": 0.61, + "learning_rate": 3.938372652864709e-05, + "loss": 2.5997, + "theoretical_loss": 3.352330157216258, + "tokens_seen": 2656632832 + }, + { + "epoch": 0.61, + "learning_rate": 3.9375702134488844e-05, + "loss": 2.5918, + "theoretical_loss": 3.352317124510749, + "tokens_seen": 2656763904 + }, + { + "epoch": 0.61, + "learning_rate": 3.9367677740330604e-05, + "loss": 2.5371, + "theoretical_loss": 3.352304092628217, + "tokens_seen": 2656894976 + }, + { + "epoch": 0.61, + "learning_rate": 3.9359653346172364e-05, + "loss": 2.3534, + "theoretical_loss": 3.3522910615685695, + "tokens_seen": 2657026048 + }, + { + "epoch": 0.61, + "learning_rate": 3.9351628952014124e-05, + "loss": 2.6058, + "theoretical_loss": 3.352278031331714, + "tokens_seen": 2657157120 + }, + { + "epoch": 0.61, + "learning_rate": 3.9343604557855884e-05, + "loss": 2.4858, + "theoretical_loss": 3.352265001917558, + "tokens_seen": 2657288192 + }, + { + "epoch": 0.61, + "learning_rate": 3.9335580163697644e-05, + "loss": 2.4256, + "theoretical_loss": 3.3522519733260094, + "tokens_seen": 2657419264 + }, + { + "epoch": 0.61, + "learning_rate": 3.9327555769539403e-05, + "loss": 2.4492, + "theoretical_loss": 3.352238945556975, + "tokens_seen": 2657550336 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1456249, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9987263679504395, + "objective/train/theoretical_loss": 3.3522324319808723, + "objective/train/tokens_used": 1028074976, + "theoretical_loss": 3.3522324319808723, + "tokens_seen": 2657615872 + }, + { + "epoch": 0.61, + "learning_rate": 3.931953137538116e-05, + "loss": 2.6716, + "theoretical_loss": 3.352225918610363, + "tokens_seen": 2657681408 + }, + { + "epoch": 0.61, + "learning_rate": 3.9311506981222916e-05, + "loss": 2.6613, + "theoretical_loss": 3.3522128924860803, + "tokens_seen": 2657812480 + }, + { + "epoch": 0.61, + "learning_rate": 3.9303482587064676e-05, + "loss": 2.4842, + "theoretical_loss": 3.3521998671840345, + "tokens_seen": 2657943552 + }, + { + "epoch": 0.61, + "learning_rate": 3.9295458192906436e-05, + "loss": 2.6776, + "theoretical_loss": 3.3521868427041337, + "tokens_seen": 2658074624 + }, + { + "epoch": 0.61, + "learning_rate": 3.9287433798748196e-05, + "loss": 2.5349, + "theoretical_loss": 3.3521738190462855, + "tokens_seen": 2658205696 + }, + { + "epoch": 0.61, + "learning_rate": 3.9279409404589956e-05, + "loss": 2.4215, + "theoretical_loss": 3.3521607962103963, + "tokens_seen": 2658336768 + }, + { + "epoch": 0.61, + "learning_rate": 3.9271385010431716e-05, + "loss": 2.4675, + "theoretical_loss": 3.352147774196375, + "tokens_seen": 2658467840 + }, + { + "epoch": 0.61, + "learning_rate": 3.9263360616273476e-05, + "loss": 2.5428, + "theoretical_loss": 3.3521347530041288, + "tokens_seen": 2658598912 + }, + { + "epoch": 0.61, + "learning_rate": 3.925533622211523e-05, + "loss": 2.5632, + "theoretical_loss": 3.352121732633565, + "tokens_seen": 2658729984 + }, + { + "epoch": 0.61, + "learning_rate": 3.924731182795699e-05, + "loss": 2.4234, + "theoretical_loss": 3.3521087130845912, + "tokens_seen": 2658861056 + }, + { + "epoch": 0.61, + "learning_rate": 3.923928743379875e-05, + "loss": 2.4522, + "theoretical_loss": 3.352095694357116, + "tokens_seen": 2658992128 + }, + { + "epoch": 0.61, + "learning_rate": 3.923126303964051e-05, + "loss": 2.4758, + "theoretical_loss": 3.352082676451046, + "tokens_seen": 2659123200 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1456946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.147951602935791, + "objective/train/theoretical_loss": 3.3520696593662898, + "objective/train/tokens_used": 1029713376, + "theoretical_loss": 3.3520696593662898, + "tokens_seen": 2659254272 + }, + { + "epoch": 0.61, + "learning_rate": 3.922323864548227e-05, + "loss": 2.4148, + "theoretical_loss": 3.3520696593662898, + "tokens_seen": 2659254272 + }, + { + "epoch": 0.61, + "learning_rate": 3.921521425132403e-05, + "loss": 2.4731, + "theoretical_loss": 3.352056643102754, + "tokens_seen": 2659385344 + }, + { + "epoch": 0.61, + "learning_rate": 3.920718985716579e-05, + "loss": 2.5643, + "theoretical_loss": 3.3520436276603474, + "tokens_seen": 2659516416 + }, + { + "epoch": 0.61, + "learning_rate": 3.919916546300754e-05, + "loss": 2.5322, + "theoretical_loss": 3.352030613038977, + "tokens_seen": 2659647488 + }, + { + "epoch": 0.61, + "learning_rate": 3.91911410688493e-05, + "loss": 2.4041, + "theoretical_loss": 3.3520175992385512, + "tokens_seen": 2659778560 + }, + { + "epoch": 0.61, + "learning_rate": 3.918311667469106e-05, + "loss": 2.3726, + "theoretical_loss": 3.352004586258977, + "tokens_seen": 2659909632 + }, + { + "epoch": 0.61, + "learning_rate": 3.917509228053282e-05, + "loss": 2.5229, + "theoretical_loss": 3.351991574100163, + "tokens_seen": 2660040704 + }, + { + "epoch": 0.61, + "learning_rate": 3.916706788637458e-05, + "loss": 2.3643, + "theoretical_loss": 3.351978562762016, + "tokens_seen": 2660171776 + }, + { + "epoch": 0.61, + "learning_rate": 3.915904349221634e-05, + "loss": 2.675, + "theoretical_loss": 3.3519655522444447, + "tokens_seen": 2660302848 + }, + { + "epoch": 0.61, + "learning_rate": 3.91510190980581e-05, + "loss": 2.4142, + "theoretical_loss": 3.3519525425473566, + "tokens_seen": 2660433920 + }, + { + "epoch": 0.61, + "learning_rate": 3.914299470389986e-05, + "loss": 2.4774, + "theoretical_loss": 3.35193953367066, + "tokens_seen": 2660564992 + }, + { + "epoch": 0.61, + "learning_rate": 3.9134970309741614e-05, + "loss": 2.5835, + "theoretical_loss": 3.3519265256142616, + "tokens_seen": 2660696064 + }, + { + "epoch": 0.61, + "learning_rate": 3.9126945915583374e-05, + "loss": 2.4976, + "theoretical_loss": 3.35191351837807, + "tokens_seen": 2660827136 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1458124, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9016952514648438, + "objective/train/theoretical_loss": 3.351907015067523, + "objective/train/tokens_used": 1031351776, + "theoretical_loss": 3.351907015067523, + "tokens_seen": 2660892672 + }, + { + "epoch": 0.61, + "learning_rate": 3.9118921521425134e-05, + "loss": 2.4925, + "theoretical_loss": 3.3519005119619933, + "tokens_seen": 2660958208 + }, + { + "epoch": 0.61, + "learning_rate": 3.9110897127266894e-05, + "loss": 2.5723, + "theoretical_loss": 3.3518875063659395, + "tokens_seen": 2661089280 + }, + { + "epoch": 0.61, + "learning_rate": 3.9102872733108654e-05, + "loss": 2.5826, + "theoretical_loss": 3.3518745015898155, + "tokens_seen": 2661220352 + }, + { + "epoch": 0.61, + "learning_rate": 3.9094848338950414e-05, + "loss": 2.5688, + "theoretical_loss": 3.3518614976335304, + "tokens_seen": 2661351424 + }, + { + "epoch": 0.61, + "learning_rate": 3.9086823944792174e-05, + "loss": 2.6345, + "theoretical_loss": 3.3518484944969913, + "tokens_seen": 2661482496 + }, + { + "epoch": 0.61, + "learning_rate": 3.907879955063393e-05, + "loss": 2.4002, + "theoretical_loss": 3.351835492180107, + "tokens_seen": 2661613568 + }, + { + "epoch": 0.61, + "learning_rate": 3.9070775156475687e-05, + "loss": 2.6174, + "theoretical_loss": 3.351822490682784, + "tokens_seen": 2661744640 + }, + { + "epoch": 0.61, + "learning_rate": 3.9062750762317446e-05, + "loss": 2.376, + "theoretical_loss": 3.351809490004932, + "tokens_seen": 2661875712 + }, + { + "epoch": 0.61, + "learning_rate": 3.9054726368159206e-05, + "loss": 2.4946, + "theoretical_loss": 3.3517964901464583, + "tokens_seen": 2662006784 + }, + { + "epoch": 0.61, + "learning_rate": 3.9046701974000966e-05, + "loss": 2.5802, + "theoretical_loss": 3.3517834911072706, + "tokens_seen": 2662137856 + }, + { + "epoch": 0.61, + "learning_rate": 3.9038677579842726e-05, + "loss": 2.6047, + "theoretical_loss": 3.351770492887277, + "tokens_seen": 2662268928 + }, + { + "epoch": 0.61, + "learning_rate": 3.9030653185684486e-05, + "loss": 2.6237, + "theoretical_loss": 3.3517574954863862, + "tokens_seen": 2662400000 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1458909, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4784724712371826, + "objective/train/theoretical_loss": 3.3517444989045053, + "objective/train/tokens_used": 1032990176, + "theoretical_loss": 3.3517444989045053, + "tokens_seen": 2662531072 + }, + { + "epoch": 0.61, + "learning_rate": 3.902262879152624e-05, + "loss": 2.4346, + "theoretical_loss": 3.3517444989045053, + "tokens_seen": 2662531072 + }, + { + "epoch": 0.61, + "learning_rate": 3.9014604397368e-05, + "loss": 2.46, + "theoretical_loss": 3.3517315031415436, + "tokens_seen": 2662662144 + }, + { + "epoch": 0.61, + "learning_rate": 3.900658000320976e-05, + "loss": 2.5694, + "theoretical_loss": 3.3517185081974077, + "tokens_seen": 2662793216 + }, + { + "epoch": 0.61, + "learning_rate": 3.899855560905152e-05, + "loss": 2.551, + "theoretical_loss": 3.351705514072007, + "tokens_seen": 2662924288 + }, + { + "epoch": 0.61, + "learning_rate": 3.899053121489328e-05, + "loss": 2.5337, + "theoretical_loss": 3.351692520765249, + "tokens_seen": 2663055360 + }, + { + "epoch": 0.61, + "learning_rate": 3.898250682073504e-05, + "loss": 2.3735, + "theoretical_loss": 3.351679528277042, + "tokens_seen": 2663186432 + }, + { + "epoch": 0.61, + "learning_rate": 3.89744824265768e-05, + "loss": 2.5325, + "theoretical_loss": 3.351666536607294, + "tokens_seen": 2663317504 + }, + { + "epoch": 0.61, + "learning_rate": 3.896645803241856e-05, + "loss": 2.646, + "theoretical_loss": 3.351653545755913, + "tokens_seen": 2663448576 + }, + { + "epoch": 0.61, + "learning_rate": 3.895843363826031e-05, + "loss": 2.5246, + "theoretical_loss": 3.351640555722808, + "tokens_seen": 2663579648 + }, + { + "epoch": 0.61, + "learning_rate": 3.895040924410207e-05, + "loss": 2.5514, + "theoretical_loss": 3.3516275665078865, + "tokens_seen": 2663710720 + }, + { + "epoch": 0.61, + "learning_rate": 3.894238484994383e-05, + "loss": 2.5213, + "theoretical_loss": 3.3516145781110565, + "tokens_seen": 2663841792 + }, + { + "epoch": 0.61, + "learning_rate": 3.893436045578559e-05, + "loss": 2.5373, + "theoretical_loss": 3.3516015905322267, + "tokens_seen": 2663972864 + }, + { + "epoch": 0.61, + "learning_rate": 3.892633606162735e-05, + "loss": 2.5211, + "theoretical_loss": 3.3515886037713054, + "tokens_seen": 2664103936 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1459694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3988447189331055, + "objective/train/theoretical_loss": 3.3515821106975316, + "objective/train/tokens_used": 1034628576, + "theoretical_loss": 3.3515821106975316, + "tokens_seen": 2664169472 + }, + { + "epoch": 0.61, + "learning_rate": 3.891831166746911e-05, + "loss": 2.5125, + "theoretical_loss": 3.3515756178282006, + "tokens_seen": 2664235008 + }, + { + "epoch": 0.61, + "learning_rate": 3.891028727331087e-05, + "loss": 2.3801, + "theoretical_loss": 3.3515626327028203, + "tokens_seen": 2664366080 + }, + { + "epoch": 0.61, + "learning_rate": 3.8902262879152624e-05, + "loss": 2.4082, + "theoretical_loss": 3.3515496483950735, + "tokens_seen": 2664497152 + }, + { + "epoch": 0.61, + "learning_rate": 3.8894238484994384e-05, + "loss": 2.5052, + "theoretical_loss": 3.351536664904868, + "tokens_seen": 2664628224 + }, + { + "epoch": 0.62, + "learning_rate": 3.8886214090836144e-05, + "loss": 2.4906, + "theoretical_loss": 3.351523682232112, + "tokens_seen": 2664759296 + }, + { + "epoch": 0.62, + "learning_rate": 3.8878189696677904e-05, + "loss": 2.4831, + "theoretical_loss": 3.3515107003767146, + "tokens_seen": 2664890368 + }, + { + "epoch": 0.62, + "learning_rate": 3.8870165302519664e-05, + "loss": 2.4661, + "theoretical_loss": 3.351497719338583, + "tokens_seen": 2665021440 + }, + { + "epoch": 0.62, + "learning_rate": 3.8862140908361424e-05, + "loss": 2.3772, + "theoretical_loss": 3.351484739117627, + "tokens_seen": 2665152512 + }, + { + "epoch": 0.62, + "learning_rate": 3.8854116514203184e-05, + "loss": 2.4558, + "theoretical_loss": 3.3514717597137533, + "tokens_seen": 2665283584 + }, + { + "epoch": 0.62, + "learning_rate": 3.884609212004494e-05, + "loss": 2.535, + "theoretical_loss": 3.3514587811268712, + "tokens_seen": 2665414656 + }, + { + "epoch": 0.62, + "learning_rate": 3.88380677258867e-05, + "loss": 2.5548, + "theoretical_loss": 3.3514458033568895, + "tokens_seen": 2665545728 + }, + { + "epoch": 0.62, + "learning_rate": 3.883004333172846e-05, + "loss": 2.5152, + "theoretical_loss": 3.3514328264037156, + "tokens_seen": 2665676800 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1460898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5448198318481445, + "objective/train/theoretical_loss": 3.3514198502672587, + "objective/train/tokens_used": 1036266976, + "theoretical_loss": 3.3514198502672587, + "tokens_seen": 2665807872 + }, + { + "epoch": 0.62, + "learning_rate": 3.8822018937570217e-05, + "loss": 2.5367, + "theoretical_loss": 3.3514198502672587, + "tokens_seen": 2665807872 + }, + { + "epoch": 0.62, + "learning_rate": 3.8813994543411976e-05, + "loss": 2.3971, + "theoretical_loss": 3.3514068749474273, + "tokens_seen": 2665938944 + }, + { + "epoch": 0.62, + "learning_rate": 3.8805970149253736e-05, + "loss": 2.6633, + "theoretical_loss": 3.3513939004441293, + "tokens_seen": 2666070016 + }, + { + "epoch": 0.62, + "learning_rate": 3.8797945755095496e-05, + "loss": 2.5633, + "theoretical_loss": 3.3513809267572734, + "tokens_seen": 2666201088 + }, + { + "epoch": 0.62, + "learning_rate": 3.8789921360937256e-05, + "loss": 2.6491, + "theoretical_loss": 3.3513679538867684, + "tokens_seen": 2666332160 + }, + { + "epoch": 0.62, + "learning_rate": 3.878189696677901e-05, + "loss": 2.5699, + "theoretical_loss": 3.3513549818325226, + "tokens_seen": 2666463232 + }, + { + "epoch": 0.62, + "learning_rate": 3.877387257262077e-05, + "loss": 2.5223, + "theoretical_loss": 3.3513420105944443, + "tokens_seen": 2666594304 + }, + { + "epoch": 0.62, + "learning_rate": 3.876584817846253e-05, + "loss": 2.4517, + "theoretical_loss": 3.351329040172442, + "tokens_seen": 2666725376 + }, + { + "epoch": 0.62, + "learning_rate": 3.875782378430429e-05, + "loss": 2.668, + "theoretical_loss": 3.3513160705664244, + "tokens_seen": 2666856448 + }, + { + "epoch": 0.62, + "learning_rate": 3.874979939014605e-05, + "loss": 2.5086, + "theoretical_loss": 3.3513031017763004, + "tokens_seen": 2666987520 + }, + { + "epoch": 0.62, + "learning_rate": 3.874177499598781e-05, + "loss": 2.4486, + "theoretical_loss": 3.3512901338019785, + "tokens_seen": 2667118592 + }, + { + "epoch": 0.62, + "learning_rate": 3.873375060182957e-05, + "loss": 2.4563, + "theoretical_loss": 3.3512771666433667, + "tokens_seen": 2667249664 + }, + { + "epoch": 0.62, + "learning_rate": 3.872572620767132e-05, + "loss": 2.5294, + "theoretical_loss": 3.351264200300374, + "tokens_seen": 2667380736 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1461683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.473564863204956, + "objective/train/theoretical_loss": 3.3512577174347067, + "objective/train/tokens_used": 1037905376, + "theoretical_loss": 3.3512577174347067, + "tokens_seen": 2667446272 + }, + { + "epoch": 0.62, + "learning_rate": 3.871770181351308e-05, + "loss": 2.5194, + "theoretical_loss": 3.3512512347729095, + "tokens_seen": 2667511808 + }, + { + "epoch": 0.62, + "learning_rate": 3.870967741935484e-05, + "loss": 2.4256, + "theoretical_loss": 3.351238270060881, + "tokens_seen": 2667642880 + }, + { + "epoch": 0.62, + "learning_rate": 3.87016530251966e-05, + "loss": 2.5957, + "theoretical_loss": 3.351225306164198, + "tokens_seen": 2667773952 + }, + { + "epoch": 0.62, + "learning_rate": 3.869362863103836e-05, + "loss": 2.469, + "theoretical_loss": 3.351212343082768, + "tokens_seen": 2667905024 + }, + { + "epoch": 0.62, + "learning_rate": 3.868560423688012e-05, + "loss": 2.4719, + "theoretical_loss": 3.351199380816501, + "tokens_seen": 2668036096 + }, + { + "epoch": 0.62, + "learning_rate": 3.867757984272188e-05, + "loss": 2.3978, + "theoretical_loss": 3.3511864193653045, + "tokens_seen": 2668167168 + }, + { + "epoch": 0.62, + "learning_rate": 3.866955544856364e-05, + "loss": 2.5279, + "theoretical_loss": 3.351173458729088, + "tokens_seen": 2668298240 + }, + { + "epoch": 0.62, + "learning_rate": 3.8661531054405394e-05, + "loss": 2.5219, + "theoretical_loss": 3.35116049890776, + "tokens_seen": 2668429312 + }, + { + "epoch": 0.62, + "learning_rate": 3.8653506660247154e-05, + "loss": 2.3042, + "theoretical_loss": 3.3511475399012296, + "tokens_seen": 2668560384 + }, + { + "epoch": 0.62, + "learning_rate": 3.8645482266088914e-05, + "loss": 2.6142, + "theoretical_loss": 3.3511345817094047, + "tokens_seen": 2668691456 + }, + { + "epoch": 0.62, + "learning_rate": 3.8637457871930674e-05, + "loss": 2.4822, + "theoretical_loss": 3.351121624332195, + "tokens_seen": 2668822528 + }, + { + "epoch": 0.62, + "learning_rate": 3.8629433477772434e-05, + "loss": 2.5254, + "theoretical_loss": 3.351108667769508, + "tokens_seen": 2668953600 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1462895, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.393538475036621, + "objective/train/theoretical_loss": 3.3510957120212543, + "objective/train/tokens_used": 1039543776, + "theoretical_loss": 3.3510957120212543, + "tokens_seen": 2669084672 + }, + { + "epoch": 0.62, + "learning_rate": 3.8621409083614194e-05, + "loss": 2.5752, + "theoretical_loss": 3.3510957120212543, + "tokens_seen": 2669084672 + }, + { + "epoch": 0.62, + "learning_rate": 3.861338468945595e-05, + "loss": 2.4482, + "theoretical_loss": 3.3510827570873416, + "tokens_seen": 2669215744 + }, + { + "epoch": 0.62, + "learning_rate": 3.860536029529771e-05, + "loss": 2.4756, + "theoretical_loss": 3.351069802967679, + "tokens_seen": 2669346816 + }, + { + "epoch": 0.62, + "learning_rate": 3.859733590113947e-05, + "loss": 2.4695, + "theoretical_loss": 3.3510568496621747, + "tokens_seen": 2669477888 + }, + { + "epoch": 0.62, + "learning_rate": 3.858931150698123e-05, + "loss": 2.4925, + "theoretical_loss": 3.3510438971707384, + "tokens_seen": 2669608960 + }, + { + "epoch": 0.62, + "learning_rate": 3.8581287112822987e-05, + "loss": 2.5088, + "theoretical_loss": 3.351030945493279, + "tokens_seen": 2669740032 + }, + { + "epoch": 0.62, + "learning_rate": 3.8573262718664747e-05, + "loss": 2.5767, + "theoretical_loss": 3.3510179946297045, + "tokens_seen": 2669871104 + }, + { + "epoch": 0.62, + "learning_rate": 3.85652383245065e-05, + "loss": 2.5396, + "theoretical_loss": 3.351005044579925, + "tokens_seen": 2670002176 + }, + { + "epoch": 0.62, + "learning_rate": 3.855721393034826e-05, + "loss": 2.6375, + "theoretical_loss": 3.350992095343848, + "tokens_seen": 2670133248 + }, + { + "epoch": 0.62, + "learning_rate": 3.854918953619002e-05, + "loss": 2.5116, + "theoretical_loss": 3.350979146921384, + "tokens_seen": 2670264320 + }, + { + "epoch": 0.62, + "learning_rate": 3.854116514203178e-05, + "loss": 2.5606, + "theoretical_loss": 3.350966199312441, + "tokens_seen": 2670395392 + }, + { + "epoch": 0.62, + "learning_rate": 3.853314074787354e-05, + "loss": 2.5939, + "theoretical_loss": 3.350953252516928, + "tokens_seen": 2670526464 + }, + { + "epoch": 0.62, + "learning_rate": 3.85251163537153e-05, + "loss": 2.5011, + "theoretical_loss": 3.350940306534754, + "tokens_seen": 2670657536 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1463527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.557771921157837, + "objective/train/theoretical_loss": 3.3509338338486407, + "objective/train/tokens_used": 1041182176, + "theoretical_loss": 3.3509338338486407, + "tokens_seen": 2670723072 + }, + { + "epoch": 0.62, + "learning_rate": 3.851709195955705e-05, + "loss": 2.644, + "theoretical_loss": 3.3509273613658284, + "tokens_seen": 2670788608 + }, + { + "epoch": 0.62, + "learning_rate": 3.850906756539881e-05, + "loss": 2.6817, + "theoretical_loss": 3.3509144170100598, + "tokens_seen": 2670919680 + }, + { + "epoch": 0.62, + "learning_rate": 3.850104317124057e-05, + "loss": 2.6022, + "theoretical_loss": 3.3509014734673572, + "tokens_seen": 2671050752 + }, + { + "epoch": 0.62, + "learning_rate": 3.849301877708233e-05, + "loss": 2.5494, + "theoretical_loss": 3.3508885307376297, + "tokens_seen": 2671181824 + }, + { + "epoch": 0.62, + "learning_rate": 3.848499438292409e-05, + "loss": 2.3845, + "theoretical_loss": 3.3508755888207866, + "tokens_seen": 2671312896 + }, + { + "epoch": 0.62, + "learning_rate": 3.847696998876585e-05, + "loss": 2.6698, + "theoretical_loss": 3.350862647716737, + "tokens_seen": 2671443968 + }, + { + "epoch": 0.62, + "learning_rate": 3.8468945594607605e-05, + "loss": 2.4087, + "theoretical_loss": 3.350849707425389, + "tokens_seen": 2671575040 + }, + { + "epoch": 0.62, + "learning_rate": 3.8460921200449365e-05, + "loss": 2.5956, + "theoretical_loss": 3.350836767946653, + "tokens_seen": 2671706112 + }, + { + "epoch": 0.62, + "learning_rate": 3.8452896806291125e-05, + "loss": 2.4426, + "theoretical_loss": 3.3508238292804378, + "tokens_seen": 2671837184 + }, + { + "epoch": 0.62, + "learning_rate": 3.8444872412132885e-05, + "loss": 2.4834, + "theoretical_loss": 3.3508108914266517, + "tokens_seen": 2671968256 + }, + { + "epoch": 0.62, + "learning_rate": 3.8436848017974645e-05, + "loss": 2.4885, + "theoretical_loss": 3.3507979543852047, + "tokens_seen": 2672099328 + }, + { + "epoch": 0.62, + "learning_rate": 3.8428823623816404e-05, + "loss": 2.4893, + "theoretical_loss": 3.3507850181560057, + "tokens_seen": 2672230400 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1464892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1921284198760986, + "objective/train/theoretical_loss": 3.3507720827389638, + "objective/train/tokens_used": 1042820576, + "theoretical_loss": 3.3507720827389638, + "tokens_seen": 2672361472 + }, + { + "epoch": 0.62, + "learning_rate": 3.842079922965816e-05, + "loss": 2.5049, + "theoretical_loss": 3.3507720827389638, + "tokens_seen": 2672361472 + }, + { + "epoch": 0.62, + "learning_rate": 3.841277483549992e-05, + "loss": 2.4215, + "theoretical_loss": 3.3507591481339882, + "tokens_seen": 2672492544 + }, + { + "epoch": 0.62, + "learning_rate": 3.840475044134168e-05, + "loss": 2.5319, + "theoretical_loss": 3.350746214340988, + "tokens_seen": 2672623616 + }, + { + "epoch": 0.62, + "learning_rate": 3.839672604718344e-05, + "loss": 2.4235, + "theoretical_loss": 3.3507332813598727, + "tokens_seen": 2672754688 + }, + { + "epoch": 0.62, + "learning_rate": 3.83887016530252e-05, + "loss": 2.4715, + "theoretical_loss": 3.350720349190551, + "tokens_seen": 2672885760 + }, + { + "epoch": 0.62, + "learning_rate": 3.838067725886696e-05, + "loss": 2.5423, + "theoretical_loss": 3.350707417832933, + "tokens_seen": 2673016832 + }, + { + "epoch": 0.62, + "learning_rate": 3.837265286470871e-05, + "loss": 2.483, + "theoretical_loss": 3.350694487286927, + "tokens_seen": 2673147904 + }, + { + "epoch": 0.62, + "learning_rate": 3.836462847055047e-05, + "loss": 2.6288, + "theoretical_loss": 3.350681557552443, + "tokens_seen": 2673278976 + }, + { + "epoch": 0.62, + "learning_rate": 3.835660407639223e-05, + "loss": 2.5473, + "theoretical_loss": 3.35066862862939, + "tokens_seen": 2673410048 + }, + { + "epoch": 0.62, + "learning_rate": 3.834857968223399e-05, + "loss": 2.4836, + "theoretical_loss": 3.3506557005176774, + "tokens_seen": 2673541120 + }, + { + "epoch": 0.62, + "learning_rate": 3.834055528807575e-05, + "loss": 2.565, + "theoretical_loss": 3.3506427732172144, + "tokens_seen": 2673672192 + }, + { + "epoch": 0.62, + "learning_rate": 3.833253089391751e-05, + "loss": 2.48, + "theoretical_loss": 3.35062984672791, + "tokens_seen": 2673803264 + }, + { + "epoch": 0.62, + "learning_rate": 3.832450649975927e-05, + "loss": 2.4047, + "theoretical_loss": 3.350616921049674, + "tokens_seen": 2673934336 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1465582, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.440080404281616, + "objective/train/theoretical_loss": 3.3506104585146783, + "objective/train/tokens_used": 1044458976, + "theoretical_loss": 3.3506104585146783, + "tokens_seen": 2673999872 + }, + { + "epoch": 0.62, + "learning_rate": 3.831648210560103e-05, + "loss": 2.4817, + "theoretical_loss": 3.3506039961824157, + "tokens_seen": 2674065408 + }, + { + "epoch": 0.62, + "learning_rate": 3.830845771144278e-05, + "loss": 2.4898, + "theoretical_loss": 3.3505910721260443, + "tokens_seen": 2674196480 + }, + { + "epoch": 0.62, + "learning_rate": 3.830043331728454e-05, + "loss": 2.5149, + "theoretical_loss": 3.3505781488804693, + "tokens_seen": 2674327552 + }, + { + "epoch": 0.62, + "learning_rate": 3.82924089231263e-05, + "loss": 2.6743, + "theoretical_loss": 3.3505652264456, + "tokens_seen": 2674458624 + }, + { + "epoch": 0.62, + "learning_rate": 3.828438452896806e-05, + "loss": 2.5963, + "theoretical_loss": 3.350552304821346, + "tokens_seen": 2674589696 + }, + { + "epoch": 0.62, + "learning_rate": 3.827636013480982e-05, + "loss": 2.5162, + "theoretical_loss": 3.3505393840076167, + "tokens_seen": 2674720768 + }, + { + "epoch": 0.62, + "learning_rate": 3.826833574065158e-05, + "loss": 2.6144, + "theoretical_loss": 3.3505264640043215, + "tokens_seen": 2674851840 + }, + { + "epoch": 0.62, + "learning_rate": 3.826031134649334e-05, + "loss": 2.5161, + "theoretical_loss": 3.3505135448113696, + "tokens_seen": 2674982912 + }, + { + "epoch": 0.62, + "learning_rate": 3.8252286952335095e-05, + "loss": 2.6351, + "theoretical_loss": 3.350500626428671, + "tokens_seen": 2675113984 + }, + { + "epoch": 0.62, + "learning_rate": 3.8244262558176855e-05, + "loss": 2.4764, + "theoretical_loss": 3.350487708856135, + "tokens_seen": 2675245056 + }, + { + "epoch": 0.62, + "learning_rate": 3.8236238164018615e-05, + "loss": 2.5263, + "theoretical_loss": 3.3504747920936704, + "tokens_seen": 2675376128 + }, + { + "epoch": 0.62, + "learning_rate": 3.8228213769860375e-05, + "loss": 2.499, + "theoretical_loss": 3.3504618761411877, + "tokens_seen": 2675507200 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1466685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.550262212753296, + "objective/train/theoretical_loss": 3.350448960998596, + "objective/train/tokens_used": 1046097376, + "theoretical_loss": 3.350448960998596, + "tokens_seen": 2675638272 + }, + { + "epoch": 0.62, + "learning_rate": 3.8220189375702135e-05, + "loss": 2.4133, + "theoretical_loss": 3.350448960998596, + "tokens_seen": 2675638272 + }, + { + "epoch": 0.62, + "learning_rate": 3.8212164981543895e-05, + "loss": 2.5227, + "theoretical_loss": 3.3504360466658047, + "tokens_seen": 2675769344 + }, + { + "epoch": 0.62, + "learning_rate": 3.8204140587385655e-05, + "loss": 2.4508, + "theoretical_loss": 3.3504231331427237, + "tokens_seen": 2675900416 + }, + { + "epoch": 0.62, + "learning_rate": 3.8196116193227415e-05, + "loss": 2.4439, + "theoretical_loss": 3.3504102204292625, + "tokens_seen": 2676031488 + }, + { + "epoch": 0.62, + "learning_rate": 3.818809179906917e-05, + "loss": 2.6036, + "theoretical_loss": 3.3503973085253307, + "tokens_seen": 2676162560 + }, + { + "epoch": 0.62, + "learning_rate": 3.818006740491093e-05, + "loss": 2.4291, + "theoretical_loss": 3.3503843974308376, + "tokens_seen": 2676293632 + }, + { + "epoch": 0.62, + "learning_rate": 3.817204301075269e-05, + "loss": 2.4807, + "theoretical_loss": 3.350371487145693, + "tokens_seen": 2676424704 + }, + { + "epoch": 0.62, + "learning_rate": 3.816401861659445e-05, + "loss": 2.5975, + "theoretical_loss": 3.3503585776698066, + "tokens_seen": 2676555776 + }, + { + "epoch": 0.62, + "learning_rate": 3.815599422243621e-05, + "loss": 2.4455, + "theoretical_loss": 3.3503456690030884, + "tokens_seen": 2676686848 + }, + { + "epoch": 0.62, + "learning_rate": 3.814796982827797e-05, + "loss": 2.4829, + "theoretical_loss": 3.350332761145447, + "tokens_seen": 2676817920 + }, + { + "epoch": 0.62, + "learning_rate": 3.813994543411973e-05, + "loss": 2.5382, + "theoretical_loss": 3.350319854096793, + "tokens_seen": 2676948992 + }, + { + "epoch": 0.62, + "learning_rate": 3.813192103996148e-05, + "loss": 2.5434, + "theoretical_loss": 3.350306947857036, + "tokens_seen": 2677080064 + }, + { + "epoch": 0.62, + "learning_rate": 3.812389664580324e-05, + "loss": 2.5872, + "theoretical_loss": 3.3502940424260856, + "tokens_seen": 2677211136 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1468128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8551368713378906, + "objective/train/theoretical_loss": 3.3502875900138847, + "objective/train/tokens_used": 1047735776, + "theoretical_loss": 3.3502875900138847, + "tokens_seen": 2677276672 + }, + { + "epoch": 0.62, + "learning_rate": 3.8115872251645e-05, + "loss": 2.5667, + "theoretical_loss": 3.3502811378038513, + "tokens_seen": 2677342208 + }, + { + "epoch": 0.62, + "learning_rate": 3.810784785748676e-05, + "loss": 2.4491, + "theoretical_loss": 3.350268233990243, + "tokens_seen": 2677473280 + }, + { + "epoch": 0.62, + "learning_rate": 3.809982346332852e-05, + "loss": 2.4046, + "theoretical_loss": 3.3502553309851706, + "tokens_seen": 2677604352 + }, + { + "epoch": 0.62, + "learning_rate": 3.809179906917028e-05, + "loss": 2.4727, + "theoretical_loss": 3.3502424287885435, + "tokens_seen": 2677735424 + }, + { + "epoch": 0.62, + "learning_rate": 3.808377467501204e-05, + "loss": 2.4968, + "theoretical_loss": 3.3502295274002716, + "tokens_seen": 2677866496 + }, + { + "epoch": 0.62, + "learning_rate": 3.807575028085379e-05, + "loss": 2.5641, + "theoretical_loss": 3.350216626820265, + "tokens_seen": 2677997568 + }, + { + "epoch": 0.62, + "learning_rate": 3.806772588669555e-05, + "loss": 2.5439, + "theoretical_loss": 3.3502037270484335, + "tokens_seen": 2678128640 + }, + { + "epoch": 0.62, + "learning_rate": 3.805970149253731e-05, + "loss": 2.4781, + "theoretical_loss": 3.350190828084686, + "tokens_seen": 2678259712 + }, + { + "epoch": 0.62, + "learning_rate": 3.805167709837907e-05, + "loss": 2.5082, + "theoretical_loss": 3.350177929928934, + "tokens_seen": 2678390784 + }, + { + "epoch": 0.62, + "learning_rate": 3.804365270422083e-05, + "loss": 2.4939, + "theoretical_loss": 3.3501650325810854, + "tokens_seen": 2678521856 + }, + { + "epoch": 0.62, + "learning_rate": 3.803562831006259e-05, + "loss": 2.4338, + "theoretical_loss": 3.3501521360410518, + "tokens_seen": 2678652928 + }, + { + "epoch": 0.62, + "learning_rate": 3.802760391590435e-05, + "loss": 2.5142, + "theoretical_loss": 3.350139240308742, + "tokens_seen": 2678784000 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1468807, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4871883392333984, + "objective/train/theoretical_loss": 3.350126345384066, + "objective/train/tokens_used": 1049374176, + "theoretical_loss": 3.350126345384066, + "tokens_seen": 2678915072 + }, + { + "epoch": 0.62, + "learning_rate": 3.801957952174611e-05, + "loss": 2.6267, + "theoretical_loss": 3.350126345384066, + "tokens_seen": 2678915072 + }, + { + "epoch": 0.62, + "learning_rate": 3.8011555127587865e-05, + "loss": 2.5409, + "theoretical_loss": 3.3501134512669344, + "tokens_seen": 2679046144 + }, + { + "epoch": 0.62, + "learning_rate": 3.8003530733429625e-05, + "loss": 2.4128, + "theoretical_loss": 3.3501005579572567, + "tokens_seen": 2679177216 + }, + { + "epoch": 0.62, + "learning_rate": 3.7995506339271385e-05, + "loss": 2.4061, + "theoretical_loss": 3.3500876654549425, + "tokens_seen": 2679308288 + }, + { + "epoch": 0.62, + "learning_rate": 3.7987481945113145e-05, + "loss": 2.5278, + "theoretical_loss": 3.350074773759902, + "tokens_seen": 2679439360 + }, + { + "epoch": 0.62, + "learning_rate": 3.7979457550954905e-05, + "loss": 2.5432, + "theoretical_loss": 3.3500618828720454, + "tokens_seen": 2679570432 + }, + { + "epoch": 0.62, + "learning_rate": 3.7971433156796665e-05, + "loss": 2.5912, + "theoretical_loss": 3.3500489927912827, + "tokens_seen": 2679701504 + }, + { + "epoch": 0.62, + "learning_rate": 3.7963408762638425e-05, + "loss": 2.4684, + "theoretical_loss": 3.3500361035175232, + "tokens_seen": 2679832576 + }, + { + "epoch": 0.62, + "learning_rate": 3.795538436848018e-05, + "loss": 2.5635, + "theoretical_loss": 3.350023215050678, + "tokens_seen": 2679963648 + }, + { + "epoch": 0.62, + "learning_rate": 3.794735997432194e-05, + "loss": 2.6576, + "theoretical_loss": 3.350010327390656, + "tokens_seen": 2680094720 + }, + { + "epoch": 0.62, + "learning_rate": 3.79393355801637e-05, + "loss": 2.4394, + "theoretical_loss": 3.3499974405373676, + "tokens_seen": 2680225792 + }, + { + "epoch": 0.62, + "learning_rate": 3.793131118600546e-05, + "loss": 2.4439, + "theoretical_loss": 3.3499845544907236, + "tokens_seen": 2680356864 + }, + { + "epoch": 0.62, + "learning_rate": 3.792328679184722e-05, + "loss": 2.5219, + "theoretical_loss": 3.349971669250633, + "tokens_seen": 2680487936 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1469118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.400596857070923, + "objective/train/theoretical_loss": 3.3499652269330173, + "objective/train/tokens_used": 1051012576, + "theoretical_loss": 3.3499652269330173, + "tokens_seen": 2680553472 + }, + { + "epoch": 0.62, + "learning_rate": 3.791526239768898e-05, + "loss": 2.4775, + "theoretical_loss": 3.3499587848170065, + "tokens_seen": 2680619008 + }, + { + "epoch": 0.62, + "learning_rate": 3.790723800353074e-05, + "loss": 2.3737, + "theoretical_loss": 3.3499459011897543, + "tokens_seen": 2680750080 + }, + { + "epoch": 0.62, + "learning_rate": 3.789921360937249e-05, + "loss": 2.3904, + "theoretical_loss": 3.349933018368786, + "tokens_seen": 2680881152 + }, + { + "epoch": 0.62, + "learning_rate": 3.789118921521425e-05, + "loss": 2.3574, + "theoretical_loss": 3.349920136354012, + "tokens_seen": 2681012224 + }, + { + "epoch": 0.62, + "learning_rate": 3.788316482105601e-05, + "loss": 2.4671, + "theoretical_loss": 3.3499072551453426, + "tokens_seen": 2681143296 + }, + { + "epoch": 0.63, + "learning_rate": 3.787514042689777e-05, + "loss": 2.4756, + "theoretical_loss": 3.3498943747426875, + "tokens_seen": 2681274368 + }, + { + "epoch": 0.63, + "learning_rate": 3.786711603273953e-05, + "loss": 2.5548, + "theoretical_loss": 3.3498814951459575, + "tokens_seen": 2681405440 + }, + { + "epoch": 0.63, + "learning_rate": 3.785909163858129e-05, + "loss": 2.3916, + "theoretical_loss": 3.349868616355062, + "tokens_seen": 2681536512 + }, + { + "epoch": 0.63, + "learning_rate": 3.785106724442305e-05, + "loss": 2.4379, + "theoretical_loss": 3.349855738369912, + "tokens_seen": 2681667584 + }, + { + "epoch": 0.63, + "learning_rate": 3.784304285026481e-05, + "loss": 2.3015, + "theoretical_loss": 3.3498428611904174, + "tokens_seen": 2681798656 + }, + { + "epoch": 0.63, + "learning_rate": 3.783501845610656e-05, + "loss": 2.3653, + "theoretical_loss": 3.349829984816488, + "tokens_seen": 2681929728 + }, + { + "epoch": 0.63, + "learning_rate": 3.782699406194832e-05, + "loss": 2.4046, + "theoretical_loss": 3.3498171092480344, + "tokens_seen": 2682060800 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1470248, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6374380588531494, + "objective/train/theoretical_loss": 3.3498042344849672, + "objective/train/tokens_used": 1052650976, + "theoretical_loss": 3.3498042344849672, + "tokens_seen": 2682191872 + }, + { + "epoch": 0.63, + "learning_rate": 3.781896966779008e-05, + "loss": 2.538, + "theoretical_loss": 3.3498042344849672, + "tokens_seen": 2682191872 + }, + { + "epoch": 0.63, + "learning_rate": 3.781094527363184e-05, + "loss": 2.437, + "theoretical_loss": 3.3497913605271963, + "tokens_seen": 2682322944 + }, + { + "epoch": 0.63, + "learning_rate": 3.78029208794736e-05, + "loss": 2.4163, + "theoretical_loss": 3.349778487374632, + "tokens_seen": 2682454016 + }, + { + "epoch": 0.63, + "learning_rate": 3.779489648531536e-05, + "loss": 2.3257, + "theoretical_loss": 3.3497656150271844, + "tokens_seen": 2682585088 + }, + { + "epoch": 0.63, + "learning_rate": 3.778687209115712e-05, + "loss": 2.4177, + "theoretical_loss": 3.349752743484764, + "tokens_seen": 2682716160 + }, + { + "epoch": 0.63, + "learning_rate": 3.7778847696998876e-05, + "loss": 2.4259, + "theoretical_loss": 3.349739872747281, + "tokens_seen": 2682847232 + }, + { + "epoch": 0.63, + "learning_rate": 3.7770823302840635e-05, + "loss": 2.577, + "theoretical_loss": 3.3497270028146464, + "tokens_seen": 2682978304 + }, + { + "epoch": 0.63, + "learning_rate": 3.7762798908682395e-05, + "loss": 2.6435, + "theoretical_loss": 3.3497141336867697, + "tokens_seen": 2683109376 + }, + { + "epoch": 0.63, + "learning_rate": 3.7754774514524155e-05, + "loss": 2.3955, + "theoretical_loss": 3.349701265363562, + "tokens_seen": 2683240448 + }, + { + "epoch": 0.63, + "learning_rate": 3.7746750120365915e-05, + "loss": 2.3776, + "theoretical_loss": 3.3496883978449326, + "tokens_seen": 2683371520 + }, + { + "epoch": 0.63, + "learning_rate": 3.7738725726207675e-05, + "loss": 2.4147, + "theoretical_loss": 3.349675531130793, + "tokens_seen": 2683502592 + }, + { + "epoch": 0.63, + "learning_rate": 3.7730701332049435e-05, + "loss": 2.2998, + "theoretical_loss": 3.349662665221053, + "tokens_seen": 2683633664 + }, + { + "epoch": 0.63, + "learning_rate": 3.7722676937891195e-05, + "loss": 2.6147, + "theoretical_loss": 3.3496498001156234, + "tokens_seen": 2683764736 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1470921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1118478775024414, + "objective/train/theoretical_loss": 3.349643367864497, + "objective/train/tokens_used": 1054289376, + "theoretical_loss": 3.349643367864497, + "tokens_seen": 2683830272 + }, + { + "epoch": 0.63, + "learning_rate": 3.771465254373295e-05, + "loss": 2.4649, + "theoretical_loss": 3.3496369358144142, + "tokens_seen": 2683895808 + }, + { + "epoch": 0.63, + "learning_rate": 3.770662814957471e-05, + "loss": 2.4365, + "theoretical_loss": 3.3496240723173365, + "tokens_seen": 2684026880 + }, + { + "epoch": 0.63, + "learning_rate": 3.769860375541647e-05, + "loss": 2.4556, + "theoretical_loss": 3.3496112096243005, + "tokens_seen": 2684157952 + }, + { + "epoch": 0.63, + "learning_rate": 3.769057936125823e-05, + "loss": 2.6117, + "theoretical_loss": 3.349598347735216, + "tokens_seen": 2684289024 + }, + { + "epoch": 0.63, + "learning_rate": 3.768255496709999e-05, + "loss": 2.5112, + "theoretical_loss": 3.3495854866499943, + "tokens_seen": 2684420096 + }, + { + "epoch": 0.63, + "learning_rate": 3.767453057294175e-05, + "loss": 2.4603, + "theoretical_loss": 3.349572626368546, + "tokens_seen": 2684551168 + }, + { + "epoch": 0.63, + "learning_rate": 3.766650617878351e-05, + "loss": 2.5604, + "theoretical_loss": 3.349559766890781, + "tokens_seen": 2684682240 + }, + { + "epoch": 0.63, + "learning_rate": 3.765848178462526e-05, + "loss": 2.446, + "theoretical_loss": 3.3495469082166105, + "tokens_seen": 2684813312 + }, + { + "epoch": 0.63, + "learning_rate": 3.765045739046702e-05, + "loss": 2.342, + "theoretical_loss": 3.349534050345944, + "tokens_seen": 2684944384 + }, + { + "epoch": 0.63, + "learning_rate": 3.764243299630878e-05, + "loss": 2.4983, + "theoretical_loss": 3.3495211932786937, + "tokens_seen": 2685075456 + }, + { + "epoch": 0.63, + "learning_rate": 3.763440860215054e-05, + "loss": 2.4331, + "theoretical_loss": 3.3495083370147682, + "tokens_seen": 2685206528 + }, + { + "epoch": 0.63, + "learning_rate": 3.76263842079923e-05, + "loss": 2.3591, + "theoretical_loss": 3.3494954815540803, + "tokens_seen": 2685337600 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1472115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3866775035858154, + "objective/train/theoretical_loss": 3.3494826268965388, + "objective/train/tokens_used": 1055927776, + "theoretical_loss": 3.3494826268965388, + "tokens_seen": 2685468672 + }, + { + "epoch": 0.63, + "learning_rate": 3.761835981383406e-05, + "loss": 2.4225, + "theoretical_loss": 3.3494826268965388, + "tokens_seen": 2685468672 + }, + { + "epoch": 0.63, + "learning_rate": 3.761033541967582e-05, + "loss": 2.5182, + "theoretical_loss": 3.349469773042055, + "tokens_seen": 2685599744 + }, + { + "epoch": 0.63, + "learning_rate": 3.760231102551757e-05, + "loss": 2.4697, + "theoretical_loss": 3.3494569199905397, + "tokens_seen": 2685730816 + }, + { + "epoch": 0.63, + "learning_rate": 3.759428663135933e-05, + "loss": 2.4112, + "theoretical_loss": 3.3494440677419033, + "tokens_seen": 2685861888 + }, + { + "epoch": 0.63, + "learning_rate": 3.758626223720109e-05, + "loss": 2.2836, + "theoretical_loss": 3.3494312162960567, + "tokens_seen": 2685992960 + }, + { + "epoch": 0.63, + "learning_rate": 3.757823784304285e-05, + "loss": 2.4805, + "theoretical_loss": 3.34941836565291, + "tokens_seen": 2686124032 + }, + { + "epoch": 0.63, + "learning_rate": 3.757021344888461e-05, + "loss": 2.3856, + "theoretical_loss": 3.349405515812375, + "tokens_seen": 2686255104 + }, + { + "epoch": 0.63, + "learning_rate": 3.756218905472637e-05, + "loss": 2.4809, + "theoretical_loss": 3.3493926667743614, + "tokens_seen": 2686386176 + }, + { + "epoch": 0.63, + "learning_rate": 3.755416466056813e-05, + "loss": 2.3604, + "theoretical_loss": 3.3493798185387806, + "tokens_seen": 2686517248 + }, + { + "epoch": 0.63, + "learning_rate": 3.754614026640989e-05, + "loss": 2.4097, + "theoretical_loss": 3.3493669711055425, + "tokens_seen": 2686648320 + }, + { + "epoch": 0.63, + "learning_rate": 3.7538115872251646e-05, + "loss": 2.458, + "theoretical_loss": 3.349354124474559, + "tokens_seen": 2686779392 + }, + { + "epoch": 0.63, + "learning_rate": 3.7530091478093406e-05, + "loss": 2.4466, + "theoretical_loss": 3.34934127864574, + "tokens_seen": 2686910464 + }, + { + "epoch": 0.63, + "learning_rate": 3.7522067083935165e-05, + "loss": 2.6208, + "theoretical_loss": 3.3493284336189966, + "tokens_seen": 2687041536 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1473411, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.704242467880249, + "objective/train/theoretical_loss": 3.3493220114063753, + "objective/train/tokens_used": 1057566176, + "theoretical_loss": 3.3493220114063753, + "tokens_seen": 2687107072 + }, + { + "epoch": 0.63, + "learning_rate": 3.7514042689776925e-05, + "loss": 2.4411, + "theoretical_loss": 3.3493155893942395, + "tokens_seen": 2687172608 + }, + { + "epoch": 0.63, + "learning_rate": 3.7506018295618685e-05, + "loss": 2.2969, + "theoretical_loss": 3.349302745971379, + "tokens_seen": 2687303680 + }, + { + "epoch": 0.63, + "learning_rate": 3.7497993901460445e-05, + "loss": 2.4363, + "theoretical_loss": 3.349289903350327, + "tokens_seen": 2687434752 + }, + { + "epoch": 0.63, + "learning_rate": 3.7489969507302205e-05, + "loss": 2.6161, + "theoretical_loss": 3.349277061530994, + "tokens_seen": 2687565824 + }, + { + "epoch": 0.63, + "learning_rate": 3.748194511314396e-05, + "loss": 2.5701, + "theoretical_loss": 3.3492642205132905, + "tokens_seen": 2687696896 + }, + { + "epoch": 0.63, + "learning_rate": 3.747392071898572e-05, + "loss": 2.4847, + "theoretical_loss": 3.3492513802971278, + "tokens_seen": 2687827968 + }, + { + "epoch": 0.63, + "learning_rate": 3.746589632482748e-05, + "loss": 2.5666, + "theoretical_loss": 3.349238540882416, + "tokens_seen": 2687959040 + }, + { + "epoch": 0.63, + "learning_rate": 3.745787193066924e-05, + "loss": 2.6206, + "theoretical_loss": 3.3492257022690666, + "tokens_seen": 2688090112 + }, + { + "epoch": 0.63, + "learning_rate": 3.7449847536511e-05, + "loss": 2.4022, + "theoretical_loss": 3.349212864456991, + "tokens_seen": 2688221184 + }, + { + "epoch": 0.63, + "learning_rate": 3.744182314235276e-05, + "loss": 2.4668, + "theoretical_loss": 3.349200027446099, + "tokens_seen": 2688352256 + }, + { + "epoch": 0.63, + "learning_rate": 3.743379874819452e-05, + "loss": 2.6213, + "theoretical_loss": 3.349187191236302, + "tokens_seen": 2688483328 + }, + { + "epoch": 0.63, + "learning_rate": 3.742577435403627e-05, + "loss": 2.4976, + "theoretical_loss": 3.3491743558275116, + "tokens_seen": 2688614400 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1474073, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3861188888549805, + "objective/train/theoretical_loss": 3.349161521219638, + "objective/train/tokens_used": 1059204576, + "theoretical_loss": 3.349161521219638, + "tokens_seen": 2688745472 + }, + { + "epoch": 0.63, + "learning_rate": 3.741774995987803e-05, + "loss": 2.4648, + "theoretical_loss": 3.349161521219638, + "tokens_seen": 2688745472 + }, + { + "epoch": 0.63, + "learning_rate": 3.740972556571979e-05, + "loss": 2.5868, + "theoretical_loss": 3.349148687412592, + "tokens_seen": 2688876544 + }, + { + "epoch": 0.63, + "learning_rate": 3.740170117156155e-05, + "loss": 2.5468, + "theoretical_loss": 3.3491358544062857, + "tokens_seen": 2689007616 + }, + { + "epoch": 0.63, + "learning_rate": 3.739367677740331e-05, + "loss": 2.5238, + "theoretical_loss": 3.3491230222006294, + "tokens_seen": 2689138688 + }, + { + "epoch": 0.63, + "learning_rate": 3.738565238324507e-05, + "loss": 2.4706, + "theoretical_loss": 3.3491101907955336, + "tokens_seen": 2689269760 + }, + { + "epoch": 0.63, + "learning_rate": 3.737762798908683e-05, + "loss": 2.4537, + "theoretical_loss": 3.3490973601909104, + "tokens_seen": 2689400832 + }, + { + "epoch": 0.63, + "learning_rate": 3.736960359492859e-05, + "loss": 2.3524, + "theoretical_loss": 3.34908453038667, + "tokens_seen": 2689531904 + }, + { + "epoch": 0.63, + "learning_rate": 3.736157920077034e-05, + "loss": 2.2384, + "theoretical_loss": 3.3490717013827243, + "tokens_seen": 2689662976 + }, + { + "epoch": 0.63, + "learning_rate": 3.73535548066121e-05, + "loss": 2.2986, + "theoretical_loss": 3.3490588731789837, + "tokens_seen": 2689794048 + }, + { + "epoch": 0.63, + "learning_rate": 3.734553041245386e-05, + "loss": 2.3438, + "theoretical_loss": 3.349046045775359, + "tokens_seen": 2689925120 + }, + { + "epoch": 0.63, + "learning_rate": 3.733750601829562e-05, + "loss": 2.3496, + "theoretical_loss": 3.3490332191717624, + "tokens_seen": 2690056192 + }, + { + "epoch": 0.63, + "learning_rate": 3.732948162413738e-05, + "loss": 2.442, + "theoretical_loss": 3.349020393368104, + "tokens_seen": 2690187264 + }, + { + "epoch": 0.63, + "learning_rate": 3.732145722997914e-05, + "loss": 2.3775, + "theoretical_loss": 3.349007568364296, + "tokens_seen": 2690318336 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1475232, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1894705295562744, + "objective/train/theoretical_loss": 3.3490011561623074, + "objective/train/tokens_used": 1060842976, + "theoretical_loss": 3.3490011561623074, + "tokens_seen": 2690383872 + }, + { + "epoch": 0.63, + "learning_rate": 3.73134328358209e-05, + "loss": 2.4773, + "theoretical_loss": 3.3489947441602483, + "tokens_seen": 2690449408 + }, + { + "epoch": 0.63, + "learning_rate": 3.7305408441662656e-05, + "loss": 2.4679, + "theoretical_loss": 3.348981920755873, + "tokens_seen": 2690580480 + }, + { + "epoch": 0.63, + "learning_rate": 3.7297384047504416e-05, + "loss": 2.4106, + "theoretical_loss": 3.348969098151081, + "tokens_seen": 2690711552 + }, + { + "epoch": 0.63, + "learning_rate": 3.7289359653346176e-05, + "loss": 2.5059, + "theoretical_loss": 3.3489562763457834, + "tokens_seen": 2690842624 + }, + { + "epoch": 0.63, + "learning_rate": 3.7281335259187936e-05, + "loss": 2.3449, + "theoretical_loss": 3.3489434553398914, + "tokens_seen": 2690973696 + }, + { + "epoch": 0.63, + "learning_rate": 3.7273310865029695e-05, + "loss": 2.4938, + "theoretical_loss": 3.3489306351333163, + "tokens_seen": 2691104768 + }, + { + "epoch": 0.63, + "learning_rate": 3.7265286470871455e-05, + "loss": 2.4894, + "theoretical_loss": 3.3489178157259696, + "tokens_seen": 2691235840 + }, + { + "epoch": 0.63, + "learning_rate": 3.7257262076713215e-05, + "loss": 2.4974, + "theoretical_loss": 3.3489049971177622, + "tokens_seen": 2691366912 + }, + { + "epoch": 0.63, + "learning_rate": 3.724923768255497e-05, + "loss": 2.3135, + "theoretical_loss": 3.3488921793086055, + "tokens_seen": 2691497984 + }, + { + "epoch": 0.63, + "learning_rate": 3.724121328839673e-05, + "loss": 2.5897, + "theoretical_loss": 3.348879362298411, + "tokens_seen": 2691629056 + }, + { + "epoch": 0.63, + "learning_rate": 3.723318889423849e-05, + "loss": 2.5056, + "theoretical_loss": 3.3488665460870894, + "tokens_seen": 2691760128 + }, + { + "epoch": 0.63, + "learning_rate": 3.722516450008025e-05, + "loss": 2.4855, + "theoretical_loss": 3.348853730674552, + "tokens_seen": 2691891200 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1476402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4059083461761475, + "objective/train/theoretical_loss": 3.3488409160607113, + "objective/train/tokens_used": 1062481376, + "theoretical_loss": 3.3488409160607113, + "tokens_seen": 2692022272 + }, + { + "epoch": 0.63, + "learning_rate": 3.721714010592201e-05, + "loss": 2.5978, + "theoretical_loss": 3.3488409160607113, + "tokens_seen": 2692022272 + }, + { + "epoch": 0.63, + "learning_rate": 3.720911571176377e-05, + "loss": 2.3765, + "theoretical_loss": 3.3488281022454776, + "tokens_seen": 2692153344 + }, + { + "epoch": 0.63, + "learning_rate": 3.720109131760552e-05, + "loss": 2.4473, + "theoretical_loss": 3.3488152892287624, + "tokens_seen": 2692284416 + }, + { + "epoch": 0.63, + "learning_rate": 3.719306692344728e-05, + "loss": 2.4664, + "theoretical_loss": 3.348802477010477, + "tokens_seen": 2692415488 + }, + { + "epoch": 0.63, + "learning_rate": 3.718504252928904e-05, + "loss": 2.3889, + "theoretical_loss": 3.348789665590533, + "tokens_seen": 2692546560 + }, + { + "epoch": 0.63, + "learning_rate": 3.71770181351308e-05, + "loss": 2.4313, + "theoretical_loss": 3.3487768549688415, + "tokens_seen": 2692677632 + }, + { + "epoch": 0.63, + "learning_rate": 3.716899374097256e-05, + "loss": 2.3433, + "theoretical_loss": 3.3487640451453147, + "tokens_seen": 2692808704 + }, + { + "epoch": 0.63, + "learning_rate": 3.716096934681432e-05, + "loss": 2.3087, + "theoretical_loss": 3.3487512361198633, + "tokens_seen": 2692939776 + }, + { + "epoch": 0.63, + "learning_rate": 3.7152944952656074e-05, + "loss": 2.5218, + "theoretical_loss": 3.348738427892399, + "tokens_seen": 2693070848 + }, + { + "epoch": 0.63, + "learning_rate": 3.7144920558497834e-05, + "loss": 2.4962, + "theoretical_loss": 3.3487256204628326, + "tokens_seen": 2693201920 + }, + { + "epoch": 0.63, + "learning_rate": 3.7136896164339593e-05, + "loss": 2.4186, + "theoretical_loss": 3.348712813831076, + "tokens_seen": 2693332992 + }, + { + "epoch": 0.63, + "learning_rate": 3.7128871770181353e-05, + "loss": 2.442, + "theoretical_loss": 3.3487000079970417, + "tokens_seen": 2693464064 + }, + { + "epoch": 0.63, + "learning_rate": 3.712084737602311e-05, + "loss": 2.4236, + "theoretical_loss": 3.34868720296064, + "tokens_seen": 2693595136 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1476845, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.327301263809204, + "objective/train/theoretical_loss": 3.3486808007415236, + "objective/train/tokens_used": 1064119776, + "theoretical_loss": 3.3486808007415236, + "tokens_seen": 2693660672 + }, + { + "epoch": 0.63, + "learning_rate": 3.711282298186487e-05, + "loss": 2.4507, + "theoretical_loss": 3.348674398721782, + "tokens_seen": 2693726208 + }, + { + "epoch": 0.63, + "learning_rate": 3.7104798587706626e-05, + "loss": 2.6191, + "theoretical_loss": 3.348661595280381, + "tokens_seen": 2693857280 + }, + { + "epoch": 0.63, + "learning_rate": 3.7096774193548386e-05, + "loss": 2.3221, + "theoretical_loss": 3.3486487926363466, + "tokens_seen": 2693988352 + }, + { + "epoch": 0.63, + "learning_rate": 3.7088749799390146e-05, + "loss": 2.3309, + "theoretical_loss": 3.3486359907895915, + "tokens_seen": 2694119424 + }, + { + "epoch": 0.63, + "learning_rate": 3.7080725405231906e-05, + "loss": 2.2445, + "theoretical_loss": 3.348623189740027, + "tokens_seen": 2694250496 + }, + { + "epoch": 0.63, + "learning_rate": 3.7072701011073666e-05, + "loss": 2.5137, + "theoretical_loss": 3.3486103894875647, + "tokens_seen": 2694381568 + }, + { + "epoch": 0.63, + "learning_rate": 3.706467661691542e-05, + "loss": 2.535, + "theoretical_loss": 3.3485975900321163, + "tokens_seen": 2694512640 + }, + { + "epoch": 0.63, + "learning_rate": 3.705665222275718e-05, + "loss": 2.5019, + "theoretical_loss": 3.348584791373593, + "tokens_seen": 2694643712 + }, + { + "epoch": 0.63, + "learning_rate": 3.704862782859894e-05, + "loss": 2.482, + "theoretical_loss": 3.3485719935119067, + "tokens_seen": 2694774784 + }, + { + "epoch": 0.63, + "learning_rate": 3.70406034344407e-05, + "loss": 2.5847, + "theoretical_loss": 3.3485591964469688, + "tokens_seen": 2694905856 + }, + { + "epoch": 0.63, + "learning_rate": 3.703257904028246e-05, + "loss": 2.5493, + "theoretical_loss": 3.3485464001786918, + "tokens_seen": 2695036928 + }, + { + "epoch": 0.63, + "learning_rate": 3.702455464612422e-05, + "loss": 2.4057, + "theoretical_loss": 3.348533604706986, + "tokens_seen": 2695168000 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1477458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6382787227630615, + "objective/train/theoretical_loss": 3.348520810031764, + "objective/train/tokens_used": 1065758176, + "theoretical_loss": 3.348520810031764, + "tokens_seen": 2695299072 + }, + { + "epoch": 0.63, + "learning_rate": 3.701653025196598e-05, + "loss": 2.5525, + "theoretical_loss": 3.348520810031764, + "tokens_seen": 2695299072 + }, + { + "epoch": 0.63, + "learning_rate": 3.700850585780773e-05, + "loss": 2.4771, + "theoretical_loss": 3.3485080161529375, + "tokens_seen": 2695430144 + }, + { + "epoch": 0.63, + "learning_rate": 3.700048146364949e-05, + "loss": 2.5558, + "theoretical_loss": 3.348495223070418, + "tokens_seen": 2695561216 + }, + { + "epoch": 0.63, + "learning_rate": 3.699245706949125e-05, + "loss": 2.3694, + "theoretical_loss": 3.348482430784117, + "tokens_seen": 2695692288 + }, + { + "epoch": 0.63, + "learning_rate": 3.698443267533301e-05, + "loss": 2.4983, + "theoretical_loss": 3.3484696392939464, + "tokens_seen": 2695823360 + }, + { + "epoch": 0.63, + "learning_rate": 3.697640828117477e-05, + "loss": 2.5732, + "theoretical_loss": 3.3484568485998185, + "tokens_seen": 2695954432 + }, + { + "epoch": 0.63, + "learning_rate": 3.696838388701653e-05, + "loss": 2.457, + "theoretical_loss": 3.348444058701644, + "tokens_seen": 2696085504 + }, + { + "epoch": 0.63, + "learning_rate": 3.696035949285829e-05, + "loss": 2.5446, + "theoretical_loss": 3.3484312695993355, + "tokens_seen": 2696216576 + }, + { + "epoch": 0.63, + "learning_rate": 3.6952335098700044e-05, + "loss": 2.3598, + "theoretical_loss": 3.3484184812928044, + "tokens_seen": 2696347648 + }, + { + "epoch": 0.63, + "learning_rate": 3.6944310704541804e-05, + "loss": 2.4182, + "theoretical_loss": 3.348405693781963, + "tokens_seen": 2696478720 + }, + { + "epoch": 0.63, + "learning_rate": 3.6936286310383564e-05, + "loss": 2.5361, + "theoretical_loss": 3.348392907066722, + "tokens_seen": 2696609792 + }, + { + "epoch": 0.63, + "learning_rate": 3.6928261916225324e-05, + "loss": 2.4974, + "theoretical_loss": 3.348380121146995, + "tokens_seen": 2696740864 + }, + { + "epoch": 0.63, + "learning_rate": 3.6920237522067084e-05, + "loss": 2.5142, + "theoretical_loss": 3.348367336022692, + "tokens_seen": 2696871936 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1478411, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4757823944091797, + "objective/train/theoretical_loss": 3.3483609437587973, + "objective/train/tokens_used": 1067396576, + "theoretical_loss": 3.3483609437587973, + "tokens_seen": 2696937472 + }, + { + "epoch": 0.63, + "learning_rate": 3.6912213127908844e-05, + "loss": 2.4973, + "theoretical_loss": 3.348354551693726, + "tokens_seen": 2697003008 + }, + { + "epoch": 0.63, + "learning_rate": 3.6904188733750604e-05, + "loss": 2.4062, + "theoretical_loss": 3.3483417681600085, + "tokens_seen": 2697134080 + }, + { + "epoch": 0.63, + "learning_rate": 3.6896164339592364e-05, + "loss": 2.5997, + "theoretical_loss": 3.3483289854214515, + "tokens_seen": 2697265152 + }, + { + "epoch": 0.63, + "learning_rate": 3.688813994543412e-05, + "loss": 2.3938, + "theoretical_loss": 3.348316203477967, + "tokens_seen": 2697396224 + }, + { + "epoch": 0.63, + "learning_rate": 3.6880115551275877e-05, + "loss": 2.4181, + "theoretical_loss": 3.3483034223294665, + "tokens_seen": 2697527296 + }, + { + "epoch": 0.63, + "learning_rate": 3.6872091157117637e-05, + "loss": 2.5388, + "theoretical_loss": 3.3482906419758627, + "tokens_seen": 2697658368 + }, + { + "epoch": 0.64, + "learning_rate": 3.6864066762959396e-05, + "loss": 2.4772, + "theoretical_loss": 3.3482778624170666, + "tokens_seen": 2697789440 + }, + { + "epoch": 0.64, + "learning_rate": 3.6856042368801156e-05, + "loss": 2.3924, + "theoretical_loss": 3.348265083652991, + "tokens_seen": 2697920512 + }, + { + "epoch": 0.64, + "learning_rate": 3.6848017974642916e-05, + "loss": 2.4611, + "theoretical_loss": 3.348252305683547, + "tokens_seen": 2698051584 + }, + { + "epoch": 0.64, + "learning_rate": 3.6839993580484676e-05, + "loss": 2.3728, + "theoretical_loss": 3.348239528508647, + "tokens_seen": 2698182656 + }, + { + "epoch": 0.64, + "learning_rate": 3.683196918632643e-05, + "loss": 2.4528, + "theoretical_loss": 3.3482267521282036, + "tokens_seen": 2698313728 + }, + { + "epoch": 0.64, + "learning_rate": 3.682394479216819e-05, + "loss": 2.6252, + "theoretical_loss": 3.348213976542128, + "tokens_seen": 2698444800 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1479663, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.700819492340088, + "objective/train/theoretical_loss": 3.3482012017503324, + "objective/train/tokens_used": 1069034976, + "theoretical_loss": 3.3482012017503324, + "tokens_seen": 2698575872 + }, + { + "epoch": 0.64, + "learning_rate": 3.681592039800995e-05, + "loss": 2.5426, + "theoretical_loss": 3.3482012017503324, + "tokens_seen": 2698575872 + }, + { + "epoch": 0.64, + "learning_rate": 3.680789600385171e-05, + "loss": 2.3993, + "theoretical_loss": 3.348188427752729, + "tokens_seen": 2698706944 + }, + { + "epoch": 0.64, + "learning_rate": 3.679987160969347e-05, + "loss": 2.6285, + "theoretical_loss": 3.34817565454923, + "tokens_seen": 2698838016 + }, + { + "epoch": 0.64, + "learning_rate": 3.679184721553523e-05, + "loss": 2.4306, + "theoretical_loss": 3.3481628821397473, + "tokens_seen": 2698969088 + }, + { + "epoch": 0.64, + "learning_rate": 3.678382282137699e-05, + "loss": 2.3948, + "theoretical_loss": 3.348150110524193, + "tokens_seen": 2699100160 + }, + { + "epoch": 0.64, + "learning_rate": 3.677579842721875e-05, + "loss": 2.3846, + "theoretical_loss": 3.348137339702479, + "tokens_seen": 2699231232 + }, + { + "epoch": 0.64, + "learning_rate": 3.67677740330605e-05, + "loss": 2.324, + "theoretical_loss": 3.348124569674517, + "tokens_seen": 2699362304 + }, + { + "epoch": 0.64, + "learning_rate": 3.675974963890226e-05, + "loss": 2.4597, + "theoretical_loss": 3.3481118004402206, + "tokens_seen": 2699493376 + }, + { + "epoch": 0.64, + "learning_rate": 3.675172524474402e-05, + "loss": 2.3666, + "theoretical_loss": 3.3480990319995008, + "tokens_seen": 2699624448 + }, + { + "epoch": 0.64, + "learning_rate": 3.674370085058578e-05, + "loss": 2.3979, + "theoretical_loss": 3.34808626435227, + "tokens_seen": 2699755520 + }, + { + "epoch": 0.64, + "learning_rate": 3.673567645642754e-05, + "loss": 2.4638, + "theoretical_loss": 3.34807349749844, + "tokens_seen": 2699886592 + }, + { + "epoch": 0.64, + "learning_rate": 3.67276520622693e-05, + "loss": 2.4201, + "theoretical_loss": 3.348060731437924, + "tokens_seen": 2700017664 + }, + { + "epoch": 0.64, + "learning_rate": 3.671962766811106e-05, + "loss": 2.4681, + "theoretical_loss": 3.3480479661706335, + "tokens_seen": 2700148736 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1480304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.051276206970215, + "objective/train/theoretical_loss": 3.34804158383442, + "objective/train/tokens_used": 1070673376, + "theoretical_loss": 3.34804158383442, + "tokens_seen": 2700214272 + }, + { + "epoch": 0.64, + "learning_rate": 3.6711603273952814e-05, + "loss": 2.5699, + "theoretical_loss": 3.3480352016964803, + "tokens_seen": 2700279808 + }, + { + "epoch": 0.64, + "learning_rate": 3.6703578879794574e-05, + "loss": 2.4897, + "theoretical_loss": 3.348022438015377, + "tokens_seen": 2700410880 + }, + { + "epoch": 0.64, + "learning_rate": 3.6695554485636334e-05, + "loss": 2.2944, + "theoretical_loss": 3.3480096751272366, + "tokens_seen": 2700541952 + }, + { + "epoch": 0.64, + "learning_rate": 3.6687530091478094e-05, + "loss": 2.2985, + "theoretical_loss": 3.3479969130319702, + "tokens_seen": 2700673024 + }, + { + "epoch": 0.64, + "learning_rate": 3.6679505697319854e-05, + "loss": 2.4141, + "theoretical_loss": 3.3479841517294906, + "tokens_seen": 2700804096 + }, + { + "epoch": 0.64, + "learning_rate": 3.6671481303161614e-05, + "loss": 2.6012, + "theoretical_loss": 3.34797139121971, + "tokens_seen": 2700935168 + }, + { + "epoch": 0.64, + "learning_rate": 3.6663456909003374e-05, + "loss": 2.4951, + "theoretical_loss": 3.347958631502541, + "tokens_seen": 2701066240 + }, + { + "epoch": 0.64, + "learning_rate": 3.665543251484513e-05, + "loss": 2.527, + "theoretical_loss": 3.3479458725778954, + "tokens_seen": 2701197312 + }, + { + "epoch": 0.64, + "learning_rate": 3.664740812068689e-05, + "loss": 2.2992, + "theoretical_loss": 3.347933114445686, + "tokens_seen": 2701328384 + }, + { + "epoch": 0.64, + "learning_rate": 3.663938372652865e-05, + "loss": 2.2971, + "theoretical_loss": 3.3479203571058247, + "tokens_seen": 2701459456 + }, + { + "epoch": 0.64, + "learning_rate": 3.6631359332370407e-05, + "loss": 2.3731, + "theoretical_loss": 3.347907600558224, + "tokens_seen": 2701590528 + }, + { + "epoch": 0.64, + "learning_rate": 3.6623334938212166e-05, + "loss": 2.3502, + "theoretical_loss": 3.3478948448027963, + "tokens_seen": 2701721600 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1481453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6353137493133545, + "objective/train/theoretical_loss": 3.3478820898394543, + "objective/train/tokens_used": 1072311776, + "theoretical_loss": 3.3478820898394543, + "tokens_seen": 2701852672 + }, + { + "epoch": 0.64, + "learning_rate": 3.6615310544053926e-05, + "loss": 2.5426, + "theoretical_loss": 3.3478820898394543, + "tokens_seen": 2701852672 + }, + { + "epoch": 0.64, + "learning_rate": 3.6607286149895686e-05, + "loss": 2.5835, + "theoretical_loss": 3.3478693356681095, + "tokens_seen": 2701983744 + }, + { + "epoch": 0.64, + "learning_rate": 3.6599261755737446e-05, + "loss": 2.3755, + "theoretical_loss": 3.347856582288675, + "tokens_seen": 2702114816 + }, + { + "epoch": 0.64, + "learning_rate": 3.65912373615792e-05, + "loss": 2.3932, + "theoretical_loss": 3.3478438297010635, + "tokens_seen": 2702245888 + }, + { + "epoch": 0.64, + "learning_rate": 3.658321296742096e-05, + "loss": 2.4704, + "theoretical_loss": 3.347831077905187, + "tokens_seen": 2702376960 + }, + { + "epoch": 0.64, + "learning_rate": 3.657518857326272e-05, + "loss": 2.3924, + "theoretical_loss": 3.347818326900958, + "tokens_seen": 2702508032 + }, + { + "epoch": 0.64, + "learning_rate": 3.656716417910448e-05, + "loss": 2.4351, + "theoretical_loss": 3.3478055766882884, + "tokens_seen": 2702639104 + }, + { + "epoch": 0.64, + "learning_rate": 3.655913978494624e-05, + "loss": 2.5004, + "theoretical_loss": 3.347792827267092, + "tokens_seen": 2702770176 + }, + { + "epoch": 0.64, + "learning_rate": 3.6551115390788e-05, + "loss": 2.5438, + "theoretical_loss": 3.34778007863728, + "tokens_seen": 2702901248 + }, + { + "epoch": 0.64, + "learning_rate": 3.654309099662976e-05, + "loss": 2.3906, + "theoretical_loss": 3.3477673307987654, + "tokens_seen": 2703032320 + }, + { + "epoch": 0.64, + "learning_rate": 3.653506660247151e-05, + "loss": 2.3713, + "theoretical_loss": 3.347754583751461, + "tokens_seen": 2703163392 + }, + { + "epoch": 0.64, + "learning_rate": 3.652704220831327e-05, + "loss": 2.5048, + "theoretical_loss": 3.347741837495279, + "tokens_seen": 2703294464 + }, + { + "epoch": 0.64, + "learning_rate": 3.651901781415503e-05, + "loss": 2.5338, + "theoretical_loss": 3.347729092030132, + "tokens_seen": 2703425536 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1482015, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5750064849853516, + "objective/train/theoretical_loss": 3.3477227195941692, + "objective/train/tokens_used": 1073950176, + "theoretical_loss": 3.3477227195941692, + "tokens_seen": 2703491072 + }, + { + "epoch": 0.64, + "learning_rate": 3.651099341999679e-05, + "loss": 2.5279, + "theoretical_loss": 3.3477163473559326, + "tokens_seen": 2703556608 + }, + { + "epoch": 0.64, + "learning_rate": 3.650296902583855e-05, + "loss": 2.5356, + "theoretical_loss": 3.347703603472593, + "tokens_seen": 2703687680 + }, + { + "epoch": 0.64, + "learning_rate": 3.649494463168031e-05, + "loss": 2.4367, + "theoretical_loss": 3.3476908603800264, + "tokens_seen": 2703818752 + }, + { + "epoch": 0.64, + "learning_rate": 3.648692023752207e-05, + "loss": 2.5655, + "theoretical_loss": 3.347678118078145, + "tokens_seen": 2703949824 + }, + { + "epoch": 0.64, + "learning_rate": 3.6478895843363824e-05, + "loss": 2.5971, + "theoretical_loss": 3.347665376566862, + "tokens_seen": 2704080896 + }, + { + "epoch": 0.64, + "learning_rate": 3.6470871449205584e-05, + "loss": 2.495, + "theoretical_loss": 3.347652635846089, + "tokens_seen": 2704211968 + }, + { + "epoch": 0.64, + "learning_rate": 3.6462847055047344e-05, + "loss": 2.4159, + "theoretical_loss": 3.34763989591574, + "tokens_seen": 2704343040 + }, + { + "epoch": 0.64, + "learning_rate": 3.6454822660889104e-05, + "loss": 2.4748, + "theoretical_loss": 3.3476271567757263, + "tokens_seen": 2704474112 + }, + { + "epoch": 0.64, + "learning_rate": 3.6446798266730864e-05, + "loss": 2.3461, + "theoretical_loss": 3.347614418425961, + "tokens_seen": 2704605184 + }, + { + "epoch": 0.64, + "learning_rate": 3.6438773872572624e-05, + "loss": 2.4834, + "theoretical_loss": 3.347601680866357, + "tokens_seen": 2704736256 + }, + { + "epoch": 0.64, + "learning_rate": 3.6430749478414384e-05, + "loss": 2.5552, + "theoretical_loss": 3.3475889440968274, + "tokens_seen": 2704867328 + }, + { + "epoch": 0.64, + "learning_rate": 3.6422725084256144e-05, + "loss": 2.3128, + "theoretical_loss": 3.3475762081172844, + "tokens_seen": 2704998400 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1483223, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.347963571548462, + "objective/train/theoretical_loss": 3.3475634729276402, + "objective/train/tokens_used": 1075588576, + "theoretical_loss": 3.3475634729276402, + "tokens_seen": 2705129472 + }, + { + "epoch": 0.64, + "learning_rate": 3.64147006900979e-05, + "loss": 2.476, + "theoretical_loss": 3.3475634729276402, + "tokens_seen": 2705129472 + }, + { + "epoch": 0.64, + "learning_rate": 3.640667629593966e-05, + "loss": 2.4922, + "theoretical_loss": 3.3475507385278087, + "tokens_seen": 2705260544 + }, + { + "epoch": 0.64, + "learning_rate": 3.639865190178142e-05, + "loss": 2.4722, + "theoretical_loss": 3.347538004917702, + "tokens_seen": 2705391616 + }, + { + "epoch": 0.64, + "learning_rate": 3.639062750762318e-05, + "loss": 2.5027, + "theoretical_loss": 3.3475252720972324, + "tokens_seen": 2705522688 + }, + { + "epoch": 0.64, + "learning_rate": 3.6382603113464937e-05, + "loss": 2.3558, + "theoretical_loss": 3.3475125400663135, + "tokens_seen": 2705653760 + }, + { + "epoch": 0.64, + "learning_rate": 3.6374578719306696e-05, + "loss": 2.4779, + "theoretical_loss": 3.3474998088248578, + "tokens_seen": 2705784832 + }, + { + "epoch": 0.64, + "learning_rate": 3.6366554325148456e-05, + "loss": 2.4797, + "theoretical_loss": 3.347487078372778, + "tokens_seen": 2705915904 + }, + { + "epoch": 0.64, + "learning_rate": 3.635852993099021e-05, + "loss": 2.5574, + "theoretical_loss": 3.3474743487099876, + "tokens_seen": 2706046976 + }, + { + "epoch": 0.64, + "learning_rate": 3.635050553683197e-05, + "loss": 2.4294, + "theoretical_loss": 3.3474616198363982, + "tokens_seen": 2706178048 + }, + { + "epoch": 0.64, + "learning_rate": 3.634248114267373e-05, + "loss": 2.4985, + "theoretical_loss": 3.347448891751924, + "tokens_seen": 2706309120 + }, + { + "epoch": 0.64, + "learning_rate": 3.633445674851549e-05, + "loss": 2.4143, + "theoretical_loss": 3.3474361644564765, + "tokens_seen": 2706440192 + }, + { + "epoch": 0.64, + "learning_rate": 3.632643235435725e-05, + "loss": 2.5264, + "theoretical_loss": 3.3474234379499697, + "tokens_seen": 2706571264 + }, + { + "epoch": 0.64, + "learning_rate": 3.631840796019901e-05, + "loss": 2.5461, + "theoretical_loss": 3.3474107122323162, + "tokens_seen": 2706702336 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1483807, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7409844398498535, + "objective/train/theoretical_loss": 3.347404349669282, + "objective/train/tokens_used": 1077226976, + "theoretical_loss": 3.347404349669282, + "tokens_seen": 2706767872 + }, + { + "epoch": 0.64, + "learning_rate": 3.631038356604077e-05, + "loss": 2.5175, + "theoretical_loss": 3.3473979873034283, + "tokens_seen": 2706833408 + }, + { + "epoch": 0.64, + "learning_rate": 3.630235917188253e-05, + "loss": 2.4168, + "theoretical_loss": 3.3473852631632197, + "tokens_seen": 2706964480 + }, + { + "epoch": 0.64, + "learning_rate": 3.629433477772428e-05, + "loss": 2.3933, + "theoretical_loss": 3.347372539811603, + "tokens_seen": 2707095552 + }, + { + "epoch": 0.64, + "learning_rate": 3.628631038356604e-05, + "loss": 2.3679, + "theoretical_loss": 3.347359817248491, + "tokens_seen": 2707226624 + }, + { + "epoch": 0.64, + "learning_rate": 3.62782859894078e-05, + "loss": 2.4869, + "theoretical_loss": 3.347347095473797, + "tokens_seen": 2707357696 + }, + { + "epoch": 0.64, + "learning_rate": 3.627026159524956e-05, + "loss": 2.4295, + "theoretical_loss": 3.347334374487434, + "tokens_seen": 2707488768 + }, + { + "epoch": 0.64, + "learning_rate": 3.626223720109132e-05, + "loss": 2.5261, + "theoretical_loss": 3.3473216542893147, + "tokens_seen": 2707619840 + }, + { + "epoch": 0.64, + "learning_rate": 3.625421280693308e-05, + "loss": 2.4118, + "theoretical_loss": 3.347308934879352, + "tokens_seen": 2707750912 + }, + { + "epoch": 0.64, + "learning_rate": 3.624618841277484e-05, + "loss": 2.6683, + "theoretical_loss": 3.347296216257459, + "tokens_seen": 2707881984 + }, + { + "epoch": 0.64, + "learning_rate": 3.6238164018616595e-05, + "loss": 2.381, + "theoretical_loss": 3.347283498423549, + "tokens_seen": 2708013056 + }, + { + "epoch": 0.64, + "learning_rate": 3.6230139624458354e-05, + "loss": 2.4607, + "theoretical_loss": 3.347270781377535, + "tokens_seen": 2708144128 + }, + { + "epoch": 0.64, + "learning_rate": 3.6222115230300114e-05, + "loss": 2.6117, + "theoretical_loss": 3.34725806511933, + "tokens_seen": 2708275200 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1484425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9466347694396973, + "objective/train/theoretical_loss": 3.347245349648847, + "objective/train/tokens_used": 1078865376, + "theoretical_loss": 3.347245349648847, + "tokens_seen": 2708406272 + }, + { + "epoch": 0.64, + "learning_rate": 3.6214090836141874e-05, + "loss": 2.5172, + "theoretical_loss": 3.347245349648847, + "tokens_seen": 2708406272 + }, + { + "epoch": 0.64, + "learning_rate": 3.6206066441983634e-05, + "loss": 2.5483, + "theoretical_loss": 3.347232634965999, + "tokens_seen": 2708537344 + }, + { + "epoch": 0.64, + "learning_rate": 3.6198042047825394e-05, + "loss": 2.4855, + "theoretical_loss": 3.347219921070699, + "tokens_seen": 2708668416 + }, + { + "epoch": 0.64, + "learning_rate": 3.6190017653667154e-05, + "loss": 2.5187, + "theoretical_loss": 3.347207207962861, + "tokens_seen": 2708799488 + }, + { + "epoch": 0.64, + "learning_rate": 3.618199325950891e-05, + "loss": 2.4098, + "theoretical_loss": 3.3471944956423973, + "tokens_seen": 2708930560 + }, + { + "epoch": 0.64, + "learning_rate": 3.617396886535067e-05, + "loss": 2.3271, + "theoretical_loss": 3.3471817841092206, + "tokens_seen": 2709061632 + }, + { + "epoch": 0.64, + "learning_rate": 3.616594447119243e-05, + "loss": 2.4914, + "theoretical_loss": 3.3471690733632453, + "tokens_seen": 2709192704 + }, + { + "epoch": 0.64, + "learning_rate": 3.615792007703419e-05, + "loss": 2.3555, + "theoretical_loss": 3.3471563634043835, + "tokens_seen": 2709323776 + }, + { + "epoch": 0.64, + "learning_rate": 3.614989568287595e-05, + "loss": 2.456, + "theoretical_loss": 3.3471436542325486, + "tokens_seen": 2709454848 + }, + { + "epoch": 0.64, + "learning_rate": 3.614187128871771e-05, + "loss": 2.5037, + "theoretical_loss": 3.3471309458476544, + "tokens_seen": 2709585920 + }, + { + "epoch": 0.64, + "learning_rate": 3.6133846894559467e-05, + "loss": 2.4604, + "theoretical_loss": 3.347118238249614, + "tokens_seen": 2709716992 + }, + { + "epoch": 0.64, + "learning_rate": 3.6125822500401226e-05, + "loss": 2.5686, + "theoretical_loss": 3.3471055314383396, + "tokens_seen": 2709848064 + }, + { + "epoch": 0.64, + "learning_rate": 3.611779810624298e-05, + "loss": 2.37, + "theoretical_loss": 3.3470928254137453, + "tokens_seen": 2709979136 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1485722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7794363498687744, + "objective/train/theoretical_loss": 3.3470864726964265, + "objective/train/tokens_used": 1080503776, + "theoretical_loss": 3.3470864726964265, + "tokens_seen": 2710044672 + }, + { + "epoch": 0.64, + "learning_rate": 3.610977371208474e-05, + "loss": 2.5324, + "theoretical_loss": 3.3470801201757445, + "tokens_seen": 2710110208 + }, + { + "epoch": 0.64, + "learning_rate": 3.61017493179265e-05, + "loss": 2.6427, + "theoretical_loss": 3.34706741572425, + "tokens_seen": 2710241280 + }, + { + "epoch": 0.64, + "learning_rate": 3.609372492376826e-05, + "loss": 2.4225, + "theoretical_loss": 3.3470547120591756, + "tokens_seen": 2710372352 + }, + { + "epoch": 0.64, + "learning_rate": 3.608570052961002e-05, + "loss": 2.2942, + "theoretical_loss": 3.3470420091804343, + "tokens_seen": 2710503424 + }, + { + "epoch": 0.64, + "learning_rate": 3.607767613545178e-05, + "loss": 2.4839, + "theoretical_loss": 3.3470293070879387, + "tokens_seen": 2710634496 + }, + { + "epoch": 0.64, + "learning_rate": 3.606965174129354e-05, + "loss": 2.4786, + "theoretical_loss": 3.347016605781603, + "tokens_seen": 2710765568 + }, + { + "epoch": 0.64, + "learning_rate": 3.606162734713529e-05, + "loss": 2.3856, + "theoretical_loss": 3.3470039052613405, + "tokens_seen": 2710896640 + }, + { + "epoch": 0.64, + "learning_rate": 3.605360295297705e-05, + "loss": 2.5467, + "theoretical_loss": 3.3469912055270647, + "tokens_seen": 2711027712 + }, + { + "epoch": 0.64, + "learning_rate": 3.604557855881881e-05, + "loss": 2.5482, + "theoretical_loss": 3.3469785065786883, + "tokens_seen": 2711158784 + }, + { + "epoch": 0.64, + "learning_rate": 3.603755416466057e-05, + "loss": 2.3933, + "theoretical_loss": 3.3469658084161247, + "tokens_seen": 2711289856 + }, + { + "epoch": 0.64, + "learning_rate": 3.602952977050233e-05, + "loss": 2.5852, + "theoretical_loss": 3.346953111039288, + "tokens_seen": 2711420928 + }, + { + "epoch": 0.64, + "learning_rate": 3.602150537634409e-05, + "loss": 2.4401, + "theoretical_loss": 3.346940414448091, + "tokens_seen": 2711552000 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1486434, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0475034713745117, + "objective/train/theoretical_loss": 3.3469277186424473, + "objective/train/tokens_used": 1082142176, + "theoretical_loss": 3.3469277186424473, + "tokens_seen": 2711683072 + }, + { + "epoch": 0.64, + "learning_rate": 3.601348098218585e-05, + "loss": 2.374, + "theoretical_loss": 3.3469277186424473, + "tokens_seen": 2711683072 + }, + { + "epoch": 0.64, + "learning_rate": 3.6005456588027605e-05, + "loss": 2.5843, + "theoretical_loss": 3.3469150236222704, + "tokens_seen": 2711814144 + }, + { + "epoch": 0.64, + "learning_rate": 3.5997432193869365e-05, + "loss": 2.4526, + "theoretical_loss": 3.3469023293874733, + "tokens_seen": 2711945216 + }, + { + "epoch": 0.64, + "learning_rate": 3.5989407799711125e-05, + "loss": 2.4807, + "theoretical_loss": 3.3468896359379703, + "tokens_seen": 2712076288 + }, + { + "epoch": 0.64, + "learning_rate": 3.5981383405552884e-05, + "loss": 2.6072, + "theoretical_loss": 3.3468769432736742, + "tokens_seen": 2712207360 + }, + { + "epoch": 0.64, + "learning_rate": 3.5973359011394644e-05, + "loss": 2.5478, + "theoretical_loss": 3.346864251394499, + "tokens_seen": 2712338432 + }, + { + "epoch": 0.64, + "learning_rate": 3.5965334617236404e-05, + "loss": 2.5064, + "theoretical_loss": 3.3468515603003572, + "tokens_seen": 2712469504 + }, + { + "epoch": 0.64, + "learning_rate": 3.5957310223078164e-05, + "loss": 2.3661, + "theoretical_loss": 3.3468388699911635, + "tokens_seen": 2712600576 + }, + { + "epoch": 0.64, + "learning_rate": 3.5949285828919924e-05, + "loss": 2.4061, + "theoretical_loss": 3.3468261804668313, + "tokens_seen": 2712731648 + }, + { + "epoch": 0.64, + "learning_rate": 3.594126143476168e-05, + "loss": 2.4716, + "theoretical_loss": 3.346813491727273, + "tokens_seen": 2712862720 + }, + { + "epoch": 0.64, + "learning_rate": 3.593323704060344e-05, + "loss": 2.3752, + "theoretical_loss": 3.3468008037724033, + "tokens_seen": 2712993792 + }, + { + "epoch": 0.64, + "learning_rate": 3.59252126464452e-05, + "loss": 2.465, + "theoretical_loss": 3.3467881166021356, + "tokens_seen": 2713124864 + }, + { + "epoch": 0.64, + "learning_rate": 3.591718825228696e-05, + "loss": 2.3633, + "theoretical_loss": 3.346775430216383, + "tokens_seen": 2713255936 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1487851, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.7478755712509155, + "objective/train/theoretical_loss": 3.346769087317673, + "objective/train/tokens_used": 1083780576, + "theoretical_loss": 3.346769087317673, + "tokens_seen": 2713321472 + }, + { + "epoch": 0.64, + "learning_rate": 3.590916385812872e-05, + "loss": 2.3724, + "theoretical_loss": 3.3467627446150594, + "tokens_seen": 2713387008 + }, + { + "epoch": 0.64, + "learning_rate": 3.590113946397048e-05, + "loss": 2.2117, + "theoretical_loss": 3.3467500597980786, + "tokens_seen": 2713518080 + }, + { + "epoch": 0.64, + "learning_rate": 3.589311506981223e-05, + "loss": 2.2926, + "theoretical_loss": 3.3467373757653536, + "tokens_seen": 2713649152 + }, + { + "epoch": 0.64, + "learning_rate": 3.588509067565399e-05, + "loss": 2.4695, + "theoretical_loss": 3.3467246925167986, + "tokens_seen": 2713780224 + }, + { + "epoch": 0.64, + "learning_rate": 3.587706628149575e-05, + "loss": 2.6484, + "theoretical_loss": 3.346712010052327, + "tokens_seen": 2713911296 + }, + { + "epoch": 0.64, + "learning_rate": 3.586904188733751e-05, + "loss": 2.6275, + "theoretical_loss": 3.3466993283718525, + "tokens_seen": 2714042368 + }, + { + "epoch": 0.64, + "learning_rate": 3.586101749317927e-05, + "loss": 2.3635, + "theoretical_loss": 3.346686647475289, + "tokens_seen": 2714173440 + }, + { + "epoch": 0.65, + "learning_rate": 3.585299309902103e-05, + "loss": 2.4468, + "theoretical_loss": 3.34667396736255, + "tokens_seen": 2714304512 + }, + { + "epoch": 0.65, + "learning_rate": 3.584496870486278e-05, + "loss": 2.5112, + "theoretical_loss": 3.346661288033549, + "tokens_seen": 2714435584 + }, + { + "epoch": 0.65, + "learning_rate": 3.583694431070454e-05, + "loss": 2.5002, + "theoretical_loss": 3.3466486094882, + "tokens_seen": 2714566656 + }, + { + "epoch": 0.65, + "learning_rate": 3.58289199165463e-05, + "loss": 2.635, + "theoretical_loss": 3.346635931726417, + "tokens_seen": 2714697728 + }, + { + "epoch": 0.65, + "learning_rate": 3.582089552238806e-05, + "loss": 2.3643, + "theoretical_loss": 3.346623254748113, + "tokens_seen": 2714828800 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1488290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3086225986480713, + "objective/train/theoretical_loss": 3.3466105785532023, + "objective/train/tokens_used": 1085418976, + "theoretical_loss": 3.3466105785532023, + "tokens_seen": 2714959872 + }, + { + "epoch": 0.65, + "learning_rate": 3.581287112822982e-05, + "loss": 2.4433, + "theoretical_loss": 3.3466105785532023, + "tokens_seen": 2714959872 + }, + { + "epoch": 0.65, + "learning_rate": 3.580484673407158e-05, + "loss": 2.5208, + "theoretical_loss": 3.3465979031415984, + "tokens_seen": 2715090944 + }, + { + "epoch": 0.65, + "learning_rate": 3.5796822339913335e-05, + "loss": 2.4689, + "theoretical_loss": 3.346585228513215, + "tokens_seen": 2715222016 + }, + { + "epoch": 0.65, + "learning_rate": 3.5788797945755095e-05, + "loss": 2.5037, + "theoretical_loss": 3.3465725546679668, + "tokens_seen": 2715353088 + }, + { + "epoch": 0.65, + "learning_rate": 3.5780773551596855e-05, + "loss": 2.3635, + "theoretical_loss": 3.3465598816057667, + "tokens_seen": 2715484160 + }, + { + "epoch": 0.65, + "learning_rate": 3.5772749157438615e-05, + "loss": 2.5038, + "theoretical_loss": 3.3465472093265287, + "tokens_seen": 2715615232 + }, + { + "epoch": 0.65, + "learning_rate": 3.5764724763280375e-05, + "loss": 2.3655, + "theoretical_loss": 3.3465345378301663, + "tokens_seen": 2715746304 + }, + { + "epoch": 0.65, + "learning_rate": 3.5756700369122135e-05, + "loss": 2.3137, + "theoretical_loss": 3.346521867116594, + "tokens_seen": 2715877376 + }, + { + "epoch": 0.65, + "learning_rate": 3.574867597496389e-05, + "loss": 2.4967, + "theoretical_loss": 3.3465091971857257, + "tokens_seen": 2716008448 + }, + { + "epoch": 0.65, + "learning_rate": 3.574065158080565e-05, + "loss": 2.4123, + "theoretical_loss": 3.3464965280374748, + "tokens_seen": 2716139520 + }, + { + "epoch": 0.65, + "learning_rate": 3.573262718664741e-05, + "loss": 2.3733, + "theoretical_loss": 3.3464838596717557, + "tokens_seen": 2716270592 + }, + { + "epoch": 0.65, + "learning_rate": 3.572460279248917e-05, + "loss": 2.5354, + "theoretical_loss": 3.3464711920884813, + "tokens_seen": 2716401664 + }, + { + "epoch": 0.65, + "learning_rate": 3.571657839833093e-05, + "loss": 2.5163, + "theoretical_loss": 3.346458525287567, + "tokens_seen": 2716532736 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1489275, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.579836845397949, + "objective/train/theoretical_loss": 3.3464521921804673, + "objective/train/tokens_used": 1087057376, + "theoretical_loss": 3.3464521921804673, + "tokens_seen": 2716598272 + }, + { + "epoch": 0.65, + "learning_rate": 3.570855400417269e-05, + "loss": 2.455, + "theoretical_loss": 3.3464458592689255, + "tokens_seen": 2716663808 + }, + { + "epoch": 0.65, + "learning_rate": 3.570052961001444e-05, + "loss": 2.4979, + "theoretical_loss": 3.346433194032471, + "tokens_seen": 2716794880 + }, + { + "epoch": 0.65, + "learning_rate": 3.56925052158562e-05, + "loss": 2.4506, + "theoretical_loss": 3.3464205295781184, + "tokens_seen": 2716925952 + }, + { + "epoch": 0.65, + "learning_rate": 3.568448082169796e-05, + "loss": 2.3881, + "theoretical_loss": 3.3464078659057805, + "tokens_seen": 2717057024 + }, + { + "epoch": 0.65, + "learning_rate": 3.567645642753972e-05, + "loss": 2.268, + "theoretical_loss": 3.346395203015372, + "tokens_seen": 2717188096 + }, + { + "epoch": 0.65, + "learning_rate": 3.566843203338148e-05, + "loss": 2.5094, + "theoretical_loss": 3.346382540906806, + "tokens_seen": 2717319168 + }, + { + "epoch": 0.65, + "learning_rate": 3.566040763922324e-05, + "loss": 2.4769, + "theoretical_loss": 3.346369879579998, + "tokens_seen": 2717450240 + }, + { + "epoch": 0.65, + "learning_rate": 3.5652383245065e-05, + "loss": 2.4473, + "theoretical_loss": 3.3463572190348607, + "tokens_seen": 2717581312 + }, + { + "epoch": 0.65, + "learning_rate": 3.564435885090675e-05, + "loss": 2.546, + "theoretical_loss": 3.3463445592713086, + "tokens_seen": 2717712384 + }, + { + "epoch": 0.65, + "learning_rate": 3.563633445674851e-05, + "loss": 2.5489, + "theoretical_loss": 3.3463319002892558, + "tokens_seen": 2717843456 + }, + { + "epoch": 0.65, + "learning_rate": 3.562831006259027e-05, + "loss": 2.5903, + "theoretical_loss": 3.346319242088617, + "tokens_seen": 2717974528 + }, + { + "epoch": 0.65, + "learning_rate": 3.562028566843203e-05, + "loss": 2.4224, + "theoretical_loss": 3.3463065846693048, + "tokens_seen": 2718105600 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1489648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.355644941329956, + "objective/train/theoretical_loss": 3.3462939280312347, + "objective/train/tokens_used": 1088695776, + "theoretical_loss": 3.3462939280312347, + "tokens_seen": 2718236672 + }, + { + "epoch": 0.65, + "learning_rate": 3.561226127427379e-05, + "loss": 2.4112, + "theoretical_loss": 3.3462939280312347, + "tokens_seen": 2718236672 + }, + { + "epoch": 0.65, + "learning_rate": 3.560423688011555e-05, + "loss": 2.4575, + "theoretical_loss": 3.34628127217432, + "tokens_seen": 2718367744 + }, + { + "epoch": 0.65, + "learning_rate": 3.559621248595731e-05, + "loss": 2.4626, + "theoretical_loss": 3.3462686170984752, + "tokens_seen": 2718498816 + }, + { + "epoch": 0.65, + "learning_rate": 3.5588188091799066e-05, + "loss": 2.5048, + "theoretical_loss": 3.3462559628036144, + "tokens_seen": 2718629888 + }, + { + "epoch": 0.65, + "learning_rate": 3.5580163697640826e-05, + "loss": 2.445, + "theoretical_loss": 3.3462433092896515, + "tokens_seen": 2718760960 + }, + { + "epoch": 0.65, + "learning_rate": 3.5572139303482585e-05, + "loss": 2.4882, + "theoretical_loss": 3.346230656556501, + "tokens_seen": 2718892032 + }, + { + "epoch": 0.65, + "learning_rate": 3.5564114909324345e-05, + "loss": 2.5077, + "theoretical_loss": 3.3462180046040766, + "tokens_seen": 2719023104 + }, + { + "epoch": 0.65, + "learning_rate": 3.5556090515166105e-05, + "loss": 2.6728, + "theoretical_loss": 3.346205353432293, + "tokens_seen": 2719154176 + }, + { + "epoch": 0.65, + "learning_rate": 3.5548066121007865e-05, + "loss": 2.5287, + "theoretical_loss": 3.3461927030410648, + "tokens_seen": 2719285248 + }, + { + "epoch": 0.65, + "learning_rate": 3.5540041726849625e-05, + "loss": 2.4433, + "theoretical_loss": 3.346180053430305, + "tokens_seen": 2719416320 + }, + { + "epoch": 0.65, + "learning_rate": 3.553201733269138e-05, + "loss": 2.5832, + "theoretical_loss": 3.3461674045999286, + "tokens_seen": 2719547392 + }, + { + "epoch": 0.65, + "learning_rate": 3.552399293853314e-05, + "loss": 2.5982, + "theoretical_loss": 3.34615475654985, + "tokens_seen": 2719678464 + }, + { + "epoch": 0.65, + "learning_rate": 3.55159685443749e-05, + "loss": 2.5118, + "theoretical_loss": 3.346142109279983, + "tokens_seen": 2719809536 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1490990, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5563888549804688, + "objective/train/theoretical_loss": 3.3461357859376024, + "objective/train/tokens_used": 1090334176, + "theoretical_loss": 3.3461357859376024, + "tokens_seen": 2719875072 + }, + { + "epoch": 0.65, + "learning_rate": 3.550794415021666e-05, + "loss": 2.5369, + "theoretical_loss": 3.346129462790242, + "tokens_seen": 2719940608 + }, + { + "epoch": 0.65, + "learning_rate": 3.549991975605842e-05, + "loss": 2.7118, + "theoretical_loss": 3.3461168170805418, + "tokens_seen": 2720071680 + }, + { + "epoch": 0.65, + "learning_rate": 3.549189536190018e-05, + "loss": 2.3598, + "theoretical_loss": 3.346104172150796, + "tokens_seen": 2720202752 + }, + { + "epoch": 0.65, + "learning_rate": 3.548387096774194e-05, + "loss": 2.4812, + "theoretical_loss": 3.346091528000919, + "tokens_seen": 2720333824 + }, + { + "epoch": 0.65, + "learning_rate": 3.54758465735837e-05, + "loss": 2.4157, + "theoretical_loss": 3.3460788846308254, + "tokens_seen": 2720464896 + }, + { + "epoch": 0.65, + "learning_rate": 3.546782217942545e-05, + "loss": 2.4856, + "theoretical_loss": 3.3460662420404295, + "tokens_seen": 2720595968 + }, + { + "epoch": 0.65, + "learning_rate": 3.545979778526721e-05, + "loss": 2.4088, + "theoretical_loss": 3.3460536002296455, + "tokens_seen": 2720727040 + }, + { + "epoch": 0.65, + "learning_rate": 3.545177339110897e-05, + "loss": 2.5533, + "theoretical_loss": 3.346040959198388, + "tokens_seen": 2720858112 + }, + { + "epoch": 0.65, + "learning_rate": 3.544374899695073e-05, + "loss": 2.4315, + "theoretical_loss": 3.3460283189465714, + "tokens_seen": 2720989184 + }, + { + "epoch": 0.65, + "learning_rate": 3.543572460279249e-05, + "loss": 2.5689, + "theoretical_loss": 3.34601567947411, + "tokens_seen": 2721120256 + }, + { + "epoch": 0.65, + "learning_rate": 3.542770020863425e-05, + "loss": 2.4516, + "theoretical_loss": 3.346003040780918, + "tokens_seen": 2721251328 + }, + { + "epoch": 0.65, + "learning_rate": 3.541967581447601e-05, + "loss": 2.4676, + "theoretical_loss": 3.3459904028669096, + "tokens_seen": 2721382400 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1491566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.288203001022339, + "objective/train/theoretical_loss": 3.345977765732, + "objective/train/tokens_used": 1091972576, + "theoretical_loss": 3.345977765732, + "tokens_seen": 2721513472 + }, + { + "epoch": 0.65, + "learning_rate": 3.541165142031776e-05, + "loss": 2.4671, + "theoretical_loss": 3.345977765732, + "tokens_seen": 2721513472 + }, + { + "epoch": 0.65, + "learning_rate": 3.540362702615952e-05, + "loss": 2.524, + "theoretical_loss": 3.3459651293761032, + "tokens_seen": 2721644544 + }, + { + "epoch": 0.65, + "learning_rate": 3.539560263200128e-05, + "loss": 2.4028, + "theoretical_loss": 3.345952493799134, + "tokens_seen": 2721775616 + }, + { + "epoch": 0.65, + "learning_rate": 3.538757823784304e-05, + "loss": 2.4696, + "theoretical_loss": 3.3459398590010068, + "tokens_seen": 2721906688 + }, + { + "epoch": 0.65, + "learning_rate": 3.53795538436848e-05, + "loss": 2.4458, + "theoretical_loss": 3.3459272249816356, + "tokens_seen": 2722037760 + }, + { + "epoch": 0.65, + "learning_rate": 3.537152944952656e-05, + "loss": 2.4003, + "theoretical_loss": 3.345914591740935, + "tokens_seen": 2722168832 + }, + { + "epoch": 0.65, + "learning_rate": 3.536350505536832e-05, + "loss": 2.4835, + "theoretical_loss": 3.3459019592788204, + "tokens_seen": 2722299904 + }, + { + "epoch": 0.65, + "learning_rate": 3.535548066121008e-05, + "loss": 2.4951, + "theoretical_loss": 3.3458893275952053, + "tokens_seen": 2722430976 + }, + { + "epoch": 0.65, + "learning_rate": 3.5347456267051836e-05, + "loss": 2.5762, + "theoretical_loss": 3.3458766966900044, + "tokens_seen": 2722562048 + }, + { + "epoch": 0.65, + "learning_rate": 3.5339431872893596e-05, + "loss": 2.582, + "theoretical_loss": 3.3458640665631325, + "tokens_seen": 2722693120 + }, + { + "epoch": 0.65, + "learning_rate": 3.5331407478735356e-05, + "loss": 2.5187, + "theoretical_loss": 3.3458514372145047, + "tokens_seen": 2722824192 + }, + { + "epoch": 0.65, + "learning_rate": 3.5323383084577115e-05, + "loss": 2.4994, + "theoretical_loss": 3.3458388086440345, + "tokens_seen": 2722955264 + }, + { + "epoch": 0.65, + "learning_rate": 3.5315358690418875e-05, + "loss": 2.5744, + "theoretical_loss": 3.3458261808516374, + "tokens_seen": 2723086336 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1492723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.941817045211792, + "objective/train/theoretical_loss": 3.3458198672471893, + "objective/train/tokens_used": 1093610976, + "theoretical_loss": 3.3458198672471893, + "tokens_seen": 2723151872 + }, + { + "epoch": 0.65, + "learning_rate": 3.5307334296260635e-05, + "loss": 2.4774, + "theoretical_loss": 3.3458135538372273, + "tokens_seen": 2723217408 + }, + { + "epoch": 0.65, + "learning_rate": 3.5299309902102395e-05, + "loss": 2.5267, + "theoretical_loss": 3.3458009276007195, + "tokens_seen": 2723348480 + }, + { + "epoch": 0.65, + "learning_rate": 3.529128550794415e-05, + "loss": 2.4364, + "theoretical_loss": 3.3457883021420285, + "tokens_seen": 2723479552 + }, + { + "epoch": 0.65, + "learning_rate": 3.528326111378591e-05, + "loss": 2.5515, + "theoretical_loss": 3.3457756774610683, + "tokens_seen": 2723610624 + }, + { + "epoch": 0.65, + "learning_rate": 3.527523671962767e-05, + "loss": 2.3878, + "theoretical_loss": 3.3457630535577545, + "tokens_seen": 2723741696 + }, + { + "epoch": 0.65, + "learning_rate": 3.526721232546943e-05, + "loss": 2.4028, + "theoretical_loss": 3.345750430432001, + "tokens_seen": 2723872768 + }, + { + "epoch": 0.65, + "learning_rate": 3.525918793131119e-05, + "loss": 2.4922, + "theoretical_loss": 3.345737808083723, + "tokens_seen": 2724003840 + }, + { + "epoch": 0.65, + "learning_rate": 3.525116353715295e-05, + "loss": 2.363, + "theoretical_loss": 3.345725186512835, + "tokens_seen": 2724134912 + }, + { + "epoch": 0.65, + "learning_rate": 3.524313914299471e-05, + "loss": 2.5357, + "theoretical_loss": 3.3457125657192512, + "tokens_seen": 2724265984 + }, + { + "epoch": 0.65, + "learning_rate": 3.523511474883646e-05, + "loss": 2.5207, + "theoretical_loss": 3.3456999457028878, + "tokens_seen": 2724397056 + }, + { + "epoch": 0.65, + "learning_rate": 3.522709035467822e-05, + "loss": 2.5429, + "theoretical_loss": 3.345687326463658, + "tokens_seen": 2724528128 + }, + { + "epoch": 0.65, + "learning_rate": 3.521906596051998e-05, + "loss": 2.4954, + "theoretical_loss": 3.3456747080014773, + "tokens_seen": 2724659200 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1493382, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6638998985290527, + "objective/train/theoretical_loss": 3.3456620903162606, + "objective/train/tokens_used": 1095249376, + "theoretical_loss": 3.3456620903162606, + "tokens_seen": 2724790272 + }, + { + "epoch": 0.65, + "learning_rate": 3.521104156636174e-05, + "loss": 2.5952, + "theoretical_loss": 3.3456620903162606, + "tokens_seen": 2724790272 + }, + { + "epoch": 0.65, + "learning_rate": 3.52030171722035e-05, + "loss": 2.523, + "theoretical_loss": 3.3456494734079225, + "tokens_seen": 2724921344 + }, + { + "epoch": 0.65, + "learning_rate": 3.519499277804526e-05, + "loss": 2.4885, + "theoretical_loss": 3.3456368572763773, + "tokens_seen": 2725052416 + }, + { + "epoch": 0.65, + "learning_rate": 3.518696838388702e-05, + "loss": 2.4991, + "theoretical_loss": 3.3456242419215405, + "tokens_seen": 2725183488 + }, + { + "epoch": 0.65, + "learning_rate": 3.517894398972878e-05, + "loss": 2.392, + "theoretical_loss": 3.345611627343327, + "tokens_seen": 2725314560 + }, + { + "epoch": 0.65, + "learning_rate": 3.517091959557053e-05, + "loss": 2.365, + "theoretical_loss": 3.345599013541651, + "tokens_seen": 2725445632 + }, + { + "epoch": 0.65, + "learning_rate": 3.516289520141229e-05, + "loss": 2.4359, + "theoretical_loss": 3.345586400516428, + "tokens_seen": 2725576704 + }, + { + "epoch": 0.65, + "learning_rate": 3.515487080725405e-05, + "loss": 2.388, + "theoretical_loss": 3.345573788267572, + "tokens_seen": 2725707776 + }, + { + "epoch": 0.65, + "learning_rate": 3.514684641309581e-05, + "loss": 2.5227, + "theoretical_loss": 3.345561176794999, + "tokens_seen": 2725838848 + }, + { + "epoch": 0.65, + "learning_rate": 3.513882201893757e-05, + "loss": 2.6064, + "theoretical_loss": 3.345548566098623, + "tokens_seen": 2725969920 + }, + { + "epoch": 0.65, + "learning_rate": 3.513079762477933e-05, + "loss": 2.5136, + "theoretical_loss": 3.3455359561783595, + "tokens_seen": 2726100992 + }, + { + "epoch": 0.65, + "learning_rate": 3.512277323062109e-05, + "loss": 2.6152, + "theoretical_loss": 3.3455233470341232, + "tokens_seen": 2726232064 + }, + { + "epoch": 0.65, + "learning_rate": 3.5114748836462846e-05, + "loss": 2.4771, + "theoretical_loss": 3.345510738665829, + "tokens_seen": 2726363136 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1494501, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.746694564819336, + "objective/train/theoretical_loss": 3.3455044347726335, + "objective/train/tokens_used": 1096887776, + "theoretical_loss": 3.3455044347726335, + "tokens_seen": 2726428672 + }, + { + "epoch": 0.65, + "learning_rate": 3.5106724442304606e-05, + "loss": 2.5405, + "theoretical_loss": 3.345498131073392, + "tokens_seen": 2726494208 + }, + { + "epoch": 0.65, + "learning_rate": 3.5098700048146366e-05, + "loss": 2.6344, + "theoretical_loss": 3.3454855242567265, + "tokens_seen": 2726625280 + }, + { + "epoch": 0.65, + "learning_rate": 3.5090675653988126e-05, + "loss": 2.5141, + "theoretical_loss": 3.3454729182157483, + "tokens_seen": 2726756352 + }, + { + "epoch": 0.65, + "learning_rate": 3.5082651259829885e-05, + "loss": 2.4083, + "theoretical_loss": 3.345460312950372, + "tokens_seen": 2726887424 + }, + { + "epoch": 0.65, + "learning_rate": 3.5074626865671645e-05, + "loss": 2.3854, + "theoretical_loss": 3.345447708460513, + "tokens_seen": 2727018496 + }, + { + "epoch": 0.65, + "learning_rate": 3.5066602471513405e-05, + "loss": 2.5143, + "theoretical_loss": 3.3454351047460857, + "tokens_seen": 2727149568 + }, + { + "epoch": 0.65, + "learning_rate": 3.505857807735516e-05, + "loss": 2.3857, + "theoretical_loss": 3.3454225018070054, + "tokens_seen": 2727280640 + }, + { + "epoch": 0.65, + "learning_rate": 3.505055368319692e-05, + "loss": 2.5121, + "theoretical_loss": 3.3454098996431876, + "tokens_seen": 2727411712 + }, + { + "epoch": 0.65, + "learning_rate": 3.504252928903868e-05, + "loss": 2.4665, + "theoretical_loss": 3.3453972982545466, + "tokens_seen": 2727542784 + }, + { + "epoch": 0.65, + "learning_rate": 3.503450489488044e-05, + "loss": 2.5383, + "theoretical_loss": 3.345384697640998, + "tokens_seen": 2727673856 + }, + { + "epoch": 0.65, + "learning_rate": 3.50264805007222e-05, + "loss": 2.4387, + "theoretical_loss": 3.345372097802456, + "tokens_seen": 2727804928 + }, + { + "epoch": 0.65, + "learning_rate": 3.501845610656396e-05, + "loss": 2.4514, + "theoretical_loss": 3.3453594987388375, + "tokens_seen": 2727936000 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1495104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2885494232177734, + "objective/train/theoretical_loss": 3.345346900450056, + "objective/train/tokens_used": 1098526176, + "theoretical_loss": 3.345346900450056, + "tokens_seen": 2728067072 + }, + { + "epoch": 0.65, + "learning_rate": 3.501043171240572e-05, + "loss": 2.5054, + "theoretical_loss": 3.345346900450056, + "tokens_seen": 2728067072 + }, + { + "epoch": 0.65, + "learning_rate": 3.500240731824748e-05, + "loss": 2.4118, + "theoretical_loss": 3.3453343029360267, + "tokens_seen": 2728198144 + }, + { + "epoch": 0.65, + "learning_rate": 3.499438292408923e-05, + "loss": 2.51, + "theoretical_loss": 3.345321706196666, + "tokens_seen": 2728329216 + }, + { + "epoch": 0.65, + "learning_rate": 3.498635852993099e-05, + "loss": 2.6463, + "theoretical_loss": 3.345309110231888, + "tokens_seen": 2728460288 + }, + { + "epoch": 0.65, + "learning_rate": 3.497833413577275e-05, + "loss": 2.561, + "theoretical_loss": 3.345296515041608, + "tokens_seen": 2728591360 + }, + { + "epoch": 0.65, + "learning_rate": 3.497030974161451e-05, + "loss": 2.3536, + "theoretical_loss": 3.3452839206257408, + "tokens_seen": 2728722432 + }, + { + "epoch": 0.65, + "learning_rate": 3.496228534745627e-05, + "loss": 2.3834, + "theoretical_loss": 3.3452713269842023, + "tokens_seen": 2728853504 + }, + { + "epoch": 0.65, + "learning_rate": 3.495426095329803e-05, + "loss": 2.3928, + "theoretical_loss": 3.3452587341169076, + "tokens_seen": 2728984576 + }, + { + "epoch": 0.65, + "learning_rate": 3.494623655913979e-05, + "loss": 2.3082, + "theoretical_loss": 3.345246142023772, + "tokens_seen": 2729115648 + }, + { + "epoch": 0.65, + "learning_rate": 3.4938212164981543e-05, + "loss": 2.2811, + "theoretical_loss": 3.3452335507047097, + "tokens_seen": 2729246720 + }, + { + "epoch": 0.65, + "learning_rate": 3.49301877708233e-05, + "loss": 2.5952, + "theoretical_loss": 3.3452209601596374, + "tokens_seen": 2729377792 + }, + { + "epoch": 0.65, + "learning_rate": 3.492216337666506e-05, + "loss": 2.5046, + "theoretical_loss": 3.3452083703884696, + "tokens_seen": 2729508864 + }, + { + "epoch": 0.65, + "learning_rate": 3.491413898250682e-05, + "loss": 2.5722, + "theoretical_loss": 3.3451957813911215, + "tokens_seen": 2729639936 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1496319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5245237350463867, + "objective/train/theoretical_loss": 3.3451894871826036, + "objective/train/tokens_used": 1100164576, + "theoretical_loss": 3.3451894871826036, + "tokens_seen": 2729705472 + }, + { + "epoch": 0.65, + "learning_rate": 3.490611458834858e-05, + "loss": 2.5168, + "theoretical_loss": 3.3451831931675087, + "tokens_seen": 2729771008 + }, + { + "epoch": 0.65, + "learning_rate": 3.489809019419034e-05, + "loss": 2.4767, + "theoretical_loss": 3.3451706057175463, + "tokens_seen": 2729902080 + }, + { + "epoch": 0.65, + "learning_rate": 3.48900658000321e-05, + "loss": 2.5175, + "theoretical_loss": 3.3451580190411496, + "tokens_seen": 2730033152 + }, + { + "epoch": 0.65, + "learning_rate": 3.488204140587386e-05, + "loss": 2.5053, + "theoretical_loss": 3.345145433138234, + "tokens_seen": 2730164224 + }, + { + "epoch": 0.65, + "learning_rate": 3.4874017011715616e-05, + "loss": 2.3593, + "theoretical_loss": 3.345132848008715, + "tokens_seen": 2730295296 + }, + { + "epoch": 0.65, + "learning_rate": 3.4865992617557376e-05, + "loss": 2.5384, + "theoretical_loss": 3.3451202636525075, + "tokens_seen": 2730426368 + }, + { + "epoch": 0.65, + "learning_rate": 3.4857968223399136e-05, + "loss": 2.439, + "theoretical_loss": 3.3451076800695274, + "tokens_seen": 2730557440 + }, + { + "epoch": 0.65, + "learning_rate": 3.4849943829240896e-05, + "loss": 2.4717, + "theoretical_loss": 3.34509509725969, + "tokens_seen": 2730688512 + }, + { + "epoch": 0.66, + "learning_rate": 3.4841919435082656e-05, + "loss": 2.5443, + "theoretical_loss": 3.34508251522291, + "tokens_seen": 2730819584 + }, + { + "epoch": 0.66, + "learning_rate": 3.4833895040924415e-05, + "loss": 2.3643, + "theoretical_loss": 3.3450699339591035, + "tokens_seen": 2730950656 + }, + { + "epoch": 0.66, + "learning_rate": 3.4825870646766175e-05, + "loss": 2.3578, + "theoretical_loss": 3.3450573534681856, + "tokens_seen": 2731081728 + }, + { + "epoch": 0.66, + "learning_rate": 3.481784625260793e-05, + "loss": 2.4377, + "theoretical_loss": 3.3450447737500717, + "tokens_seen": 2731212800 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1497706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6082136631011963, + "objective/train/theoretical_loss": 3.345032194804678, + "objective/train/tokens_used": 1101802976, + "theoretical_loss": 3.345032194804678, + "tokens_seen": 2731343872 + }, + { + "epoch": 0.66, + "learning_rate": 3.480982185844969e-05, + "loss": 2.5727, + "theoretical_loss": 3.345032194804678, + "tokens_seen": 2731343872 + }, + { + "epoch": 0.66, + "learning_rate": 3.480179746429145e-05, + "loss": 2.5304, + "theoretical_loss": 3.345019616631919, + "tokens_seen": 2731474944 + }, + { + "epoch": 0.66, + "learning_rate": 3.479377307013321e-05, + "loss": 2.5214, + "theoretical_loss": 3.3450070392317106, + "tokens_seen": 2731606016 + }, + { + "epoch": 0.66, + "learning_rate": 3.478574867597497e-05, + "loss": 2.3767, + "theoretical_loss": 3.3449944626039683, + "tokens_seen": 2731737088 + }, + { + "epoch": 0.66, + "learning_rate": 3.477772428181673e-05, + "loss": 2.4831, + "theoretical_loss": 3.344981886748607, + "tokens_seen": 2731868160 + }, + { + "epoch": 0.66, + "learning_rate": 3.476969988765849e-05, + "loss": 2.5359, + "theoretical_loss": 3.344969311665543, + "tokens_seen": 2731999232 + }, + { + "epoch": 0.66, + "learning_rate": 3.476167549350024e-05, + "loss": 2.4655, + "theoretical_loss": 3.344956737354692, + "tokens_seen": 2732130304 + }, + { + "epoch": 0.66, + "learning_rate": 3.4753651099342e-05, + "loss": 2.5243, + "theoretical_loss": 3.3449441638159687, + "tokens_seen": 2732261376 + }, + { + "epoch": 0.66, + "learning_rate": 3.474562670518376e-05, + "loss": 2.3393, + "theoretical_loss": 3.344931591049289, + "tokens_seen": 2732392448 + }, + { + "epoch": 0.66, + "learning_rate": 3.473760231102552e-05, + "loss": 2.4912, + "theoretical_loss": 3.3449190190545686, + "tokens_seen": 2732523520 + }, + { + "epoch": 0.66, + "learning_rate": 3.472957791686728e-05, + "loss": 2.3429, + "theoretical_loss": 3.3449064478317228, + "tokens_seen": 2732654592 + }, + { + "epoch": 0.66, + "learning_rate": 3.472155352270904e-05, + "loss": 2.5234, + "theoretical_loss": 3.3448938773806676, + "tokens_seen": 2732785664 + }, + { + "epoch": 0.66, + "learning_rate": 3.47135291285508e-05, + "loss": 2.4067, + "theoretical_loss": 3.344881307701318, + "tokens_seen": 2732916736 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1498175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3263230323791504, + "objective/train/theoretical_loss": 3.344875023151007, + "objective/train/tokens_used": 1103441376, + "theoretical_loss": 3.344875023151007, + "tokens_seen": 2732982272 + }, + { + "epoch": 0.66, + "learning_rate": 3.470550473439256e-05, + "loss": 2.3915, + "theoretical_loss": 3.3448687387935903, + "tokens_seen": 2733047808 + }, + { + "epoch": 0.66, + "learning_rate": 3.4697480340234314e-05, + "loss": 2.6167, + "theoretical_loss": 3.3448561706574, + "tokens_seen": 2733178880 + }, + { + "epoch": 0.66, + "learning_rate": 3.4689455946076073e-05, + "loss": 2.6844, + "theoretical_loss": 3.344843603292662, + "tokens_seen": 2733309952 + }, + { + "epoch": 0.66, + "learning_rate": 3.468143155191783e-05, + "loss": 2.5562, + "theoretical_loss": 3.3448310366992926, + "tokens_seen": 2733441024 + }, + { + "epoch": 0.66, + "learning_rate": 3.467340715775959e-05, + "loss": 2.4987, + "theoretical_loss": 3.3448184708772075, + "tokens_seen": 2733572096 + }, + { + "epoch": 0.66, + "learning_rate": 3.466538276360135e-05, + "loss": 2.4844, + "theoretical_loss": 3.3448059058263224, + "tokens_seen": 2733703168 + }, + { + "epoch": 0.66, + "learning_rate": 3.465735836944311e-05, + "loss": 2.5608, + "theoretical_loss": 3.344793341546553, + "tokens_seen": 2733834240 + }, + { + "epoch": 0.66, + "learning_rate": 3.464933397528487e-05, + "loss": 2.3996, + "theoretical_loss": 3.3447807780378147, + "tokens_seen": 2733965312 + }, + { + "epoch": 0.66, + "learning_rate": 3.4641309581126626e-05, + "loss": 2.4886, + "theoretical_loss": 3.3447682153000233, + "tokens_seen": 2734096384 + }, + { + "epoch": 0.66, + "learning_rate": 3.4633285186968386e-05, + "loss": 2.5481, + "theoretical_loss": 3.3447556533330944, + "tokens_seen": 2734227456 + }, + { + "epoch": 0.66, + "learning_rate": 3.4625260792810146e-05, + "loss": 2.4264, + "theoretical_loss": 3.3447430921369445, + "tokens_seen": 2734358528 + }, + { + "epoch": 0.66, + "learning_rate": 3.4617236398651906e-05, + "loss": 2.447, + "theoretical_loss": 3.3447305317114884, + "tokens_seen": 2734489600 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1499307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0189573764801025, + "objective/train/theoretical_loss": 3.3447179720566425, + "objective/train/tokens_used": 1105079776, + "theoretical_loss": 3.3447179720566425, + "tokens_seen": 2734620672 + }, + { + "epoch": 0.66, + "learning_rate": 3.4609212004493666e-05, + "loss": 2.5755, + "theoretical_loss": 3.3447179720566425, + "tokens_seen": 2734620672 + }, + { + "epoch": 0.66, + "learning_rate": 3.4601187610335426e-05, + "loss": 2.5965, + "theoretical_loss": 3.3447054131723224, + "tokens_seen": 2734751744 + }, + { + "epoch": 0.66, + "learning_rate": 3.4593163216177186e-05, + "loss": 2.4773, + "theoretical_loss": 3.344692855058444, + "tokens_seen": 2734882816 + }, + { + "epoch": 0.66, + "learning_rate": 3.458513882201894e-05, + "loss": 2.5901, + "theoretical_loss": 3.344680297714923, + "tokens_seen": 2735013888 + }, + { + "epoch": 0.66, + "learning_rate": 3.45771144278607e-05, + "loss": 2.5768, + "theoretical_loss": 3.344667741141675, + "tokens_seen": 2735144960 + }, + { + "epoch": 0.66, + "learning_rate": 3.456909003370246e-05, + "loss": 2.4334, + "theoretical_loss": 3.3446551853386164, + "tokens_seen": 2735276032 + }, + { + "epoch": 0.66, + "learning_rate": 3.456106563954422e-05, + "loss": 2.4979, + "theoretical_loss": 3.3446426303056622, + "tokens_seen": 2735407104 + }, + { + "epoch": 0.66, + "learning_rate": 3.455304124538598e-05, + "loss": 2.3416, + "theoretical_loss": 3.3446300760427294, + "tokens_seen": 2735538176 + }, + { + "epoch": 0.66, + "learning_rate": 3.454501685122774e-05, + "loss": 2.3853, + "theoretical_loss": 3.344617522549733, + "tokens_seen": 2735669248 + }, + { + "epoch": 0.66, + "learning_rate": 3.453699245706949e-05, + "loss": 2.4853, + "theoretical_loss": 3.3446049698265896, + "tokens_seen": 2735800320 + }, + { + "epoch": 0.66, + "learning_rate": 3.452896806291125e-05, + "loss": 2.5385, + "theoretical_loss": 3.3445924178732143, + "tokens_seen": 2735931392 + }, + { + "epoch": 0.66, + "learning_rate": 3.452094366875301e-05, + "loss": 2.4644, + "theoretical_loss": 3.3445798666895237, + "tokens_seen": 2736062464 + }, + { + "epoch": 0.66, + "learning_rate": 3.451291927459477e-05, + "loss": 2.4318, + "theoretical_loss": 3.344567316275433, + "tokens_seen": 2736193536 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1499778, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5299994945526123, + "objective/train/theoretical_loss": 3.344561041356962, + "objective/train/tokens_used": 1106718176, + "theoretical_loss": 3.344561041356962, + "tokens_seen": 2736259072 + }, + { + "epoch": 0.66, + "learning_rate": 3.450489488043653e-05, + "loss": 2.397, + "theoretical_loss": 3.344554766630859, + "tokens_seen": 2736324608 + }, + { + "epoch": 0.66, + "learning_rate": 3.449687048627829e-05, + "loss": 2.4117, + "theoretical_loss": 3.344542217755717, + "tokens_seen": 2736455680 + }, + { + "epoch": 0.66, + "learning_rate": 3.4488846092120044e-05, + "loss": 2.4857, + "theoretical_loss": 3.3445296696499236, + "tokens_seen": 2736586752 + }, + { + "epoch": 0.66, + "learning_rate": 3.4480821697961804e-05, + "loss": 2.5434, + "theoretical_loss": 3.344517122313394, + "tokens_seen": 2736717824 + }, + { + "epoch": 0.66, + "learning_rate": 3.4472797303803564e-05, + "loss": 2.5334, + "theoretical_loss": 3.3445045757460448, + "tokens_seen": 2736848896 + }, + { + "epoch": 0.66, + "learning_rate": 3.4464772909645324e-05, + "loss": 2.448, + "theoretical_loss": 3.3444920299477916, + "tokens_seen": 2736979968 + }, + { + "epoch": 0.66, + "learning_rate": 3.4456748515487084e-05, + "loss": 2.4545, + "theoretical_loss": 3.3444794849185513, + "tokens_seen": 2737111040 + }, + { + "epoch": 0.66, + "learning_rate": 3.4448724121328844e-05, + "loss": 2.5572, + "theoretical_loss": 3.344466940658239, + "tokens_seen": 2737242112 + }, + { + "epoch": 0.66, + "learning_rate": 3.44406997271706e-05, + "loss": 2.5544, + "theoretical_loss": 3.344454397166771, + "tokens_seen": 2737373184 + }, + { + "epoch": 0.66, + "learning_rate": 3.4432675333012357e-05, + "loss": 2.5336, + "theoretical_loss": 3.3444418544440633, + "tokens_seen": 2737504256 + }, + { + "epoch": 0.66, + "learning_rate": 3.4424650938854116e-05, + "loss": 2.5966, + "theoretical_loss": 3.344429312490032, + "tokens_seen": 2737635328 + }, + { + "epoch": 0.66, + "learning_rate": 3.4416626544695876e-05, + "loss": 2.5815, + "theoretical_loss": 3.3444167713045934, + "tokens_seen": 2737766400 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1501422, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4461238384246826, + "objective/train/theoretical_loss": 3.3444042308876636, + "objective/train/tokens_used": 1108356576, + "theoretical_loss": 3.3444042308876636, + "tokens_seen": 2737897472 + }, + { + "epoch": 0.66, + "learning_rate": 3.4408602150537636e-05, + "loss": 2.453, + "theoretical_loss": 3.3444042308876636, + "tokens_seen": 2737897472 + }, + { + "epoch": 0.66, + "learning_rate": 3.4400577756379396e-05, + "loss": 2.5619, + "theoretical_loss": 3.3443916912391587, + "tokens_seen": 2738028544 + }, + { + "epoch": 0.66, + "learning_rate": 3.439255336222115e-05, + "loss": 2.6006, + "theoretical_loss": 3.3443791523589947, + "tokens_seen": 2738159616 + }, + { + "epoch": 0.66, + "learning_rate": 3.438452896806291e-05, + "loss": 2.4822, + "theoretical_loss": 3.344366614247088, + "tokens_seen": 2738290688 + }, + { + "epoch": 0.66, + "learning_rate": 3.437650457390467e-05, + "loss": 2.4062, + "theoretical_loss": 3.3443540769033544, + "tokens_seen": 2738421760 + }, + { + "epoch": 0.66, + "learning_rate": 3.436848017974643e-05, + "loss": 2.5756, + "theoretical_loss": 3.34434154032771, + "tokens_seen": 2738552832 + }, + { + "epoch": 0.66, + "learning_rate": 3.436045578558819e-05, + "loss": 2.5112, + "theoretical_loss": 3.3443290045200715, + "tokens_seen": 2738683904 + }, + { + "epoch": 0.66, + "learning_rate": 3.435243139142995e-05, + "loss": 2.5298, + "theoretical_loss": 3.3443164694803547, + "tokens_seen": 2738814976 + }, + { + "epoch": 0.66, + "learning_rate": 3.43444069972717e-05, + "loss": 2.5086, + "theoretical_loss": 3.344303935208476, + "tokens_seen": 2738946048 + }, + { + "epoch": 0.66, + "learning_rate": 3.433638260311346e-05, + "loss": 2.4469, + "theoretical_loss": 3.3442914017043512, + "tokens_seen": 2739077120 + }, + { + "epoch": 0.66, + "learning_rate": 3.432835820895522e-05, + "loss": 2.4485, + "theoretical_loss": 3.3442788689678973, + "tokens_seen": 2739208192 + }, + { + "epoch": 0.66, + "learning_rate": 3.432033381479698e-05, + "loss": 2.4571, + "theoretical_loss": 3.3442663369990298, + "tokens_seen": 2739339264 + }, + { + "epoch": 0.66, + "learning_rate": 3.431230942063874e-05, + "loss": 2.3059, + "theoretical_loss": 3.3442538057976656, + "tokens_seen": 2739470336 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1502092, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6665875911712646, + "objective/train/theoretical_loss": 3.344247540484771, + "objective/train/tokens_used": 1109994976, + "theoretical_loss": 3.344247540484771, + "tokens_seen": 2739535872 + }, + { + "epoch": 0.66, + "learning_rate": 3.43042850264805e-05, + "loss": 2.4032, + "theoretical_loss": 3.3442412753637205, + "tokens_seen": 2739601408 + }, + { + "epoch": 0.66, + "learning_rate": 3.429626063232226e-05, + "loss": 2.351, + "theoretical_loss": 3.3442287456971114, + "tokens_seen": 2739732480 + }, + { + "epoch": 0.66, + "learning_rate": 3.4288236238164015e-05, + "loss": 2.4643, + "theoretical_loss": 3.3442162167977534, + "tokens_seen": 2739863552 + }, + { + "epoch": 0.66, + "learning_rate": 3.4280211844005774e-05, + "loss": 2.4795, + "theoretical_loss": 3.344203688665564, + "tokens_seen": 2739994624 + }, + { + "epoch": 0.66, + "learning_rate": 3.4272187449847534e-05, + "loss": 2.4743, + "theoretical_loss": 3.344191161300459, + "tokens_seen": 2740125696 + }, + { + "epoch": 0.66, + "learning_rate": 3.4264163055689294e-05, + "loss": 2.3405, + "theoretical_loss": 3.344178634702355, + "tokens_seen": 2740256768 + }, + { + "epoch": 0.66, + "learning_rate": 3.4256138661531054e-05, + "loss": 2.5996, + "theoretical_loss": 3.3441661088711685, + "tokens_seen": 2740387840 + }, + { + "epoch": 0.66, + "learning_rate": 3.4248114267372814e-05, + "loss": 2.4444, + "theoretical_loss": 3.344153583806815, + "tokens_seen": 2740518912 + }, + { + "epoch": 0.66, + "learning_rate": 3.4240089873214574e-05, + "loss": 2.6266, + "theoretical_loss": 3.344141059509212, + "tokens_seen": 2740649984 + }, + { + "epoch": 0.66, + "learning_rate": 3.4232065479056334e-05, + "loss": 2.4787, + "theoretical_loss": 3.344128535978275, + "tokens_seen": 2740781056 + }, + { + "epoch": 0.66, + "learning_rate": 3.422404108489809e-05, + "loss": 2.4022, + "theoretical_loss": 3.3441160132139207, + "tokens_seen": 2740912128 + }, + { + "epoch": 0.66, + "learning_rate": 3.421601669073985e-05, + "loss": 2.5719, + "theoretical_loss": 3.3441034912160656, + "tokens_seen": 2741043200 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1503441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7105658054351807, + "objective/train/theoretical_loss": 3.3440909699846264, + "objective/train/tokens_used": 1111633376, + "theoretical_loss": 3.3440909699846264, + "tokens_seen": 2741174272 + }, + { + "epoch": 0.66, + "learning_rate": 3.420799229658161e-05, + "loss": 2.5341, + "theoretical_loss": 3.3440909699846264, + "tokens_seen": 2741174272 + }, + { + "epoch": 0.66, + "learning_rate": 3.419996790242337e-05, + "loss": 2.4649, + "theoretical_loss": 3.344078449519519, + "tokens_seen": 2741305344 + }, + { + "epoch": 0.66, + "learning_rate": 3.419194350826513e-05, + "loss": 2.5547, + "theoretical_loss": 3.3440659298206605, + "tokens_seen": 2741436416 + }, + { + "epoch": 0.66, + "learning_rate": 3.4183919114106887e-05, + "loss": 2.5456, + "theoretical_loss": 3.344053410887967, + "tokens_seen": 2741567488 + }, + { + "epoch": 0.66, + "learning_rate": 3.4175894719948646e-05, + "loss": 2.4532, + "theoretical_loss": 3.3440408927213547, + "tokens_seen": 2741698560 + }, + { + "epoch": 0.66, + "learning_rate": 3.41678703257904e-05, + "loss": 2.5769, + "theoretical_loss": 3.3440283753207405, + "tokens_seen": 2741829632 + }, + { + "epoch": 0.66, + "learning_rate": 3.415984593163216e-05, + "loss": 2.473, + "theoretical_loss": 3.3440158586860407, + "tokens_seen": 2741960704 + }, + { + "epoch": 0.66, + "learning_rate": 3.415182153747392e-05, + "loss": 2.5257, + "theoretical_loss": 3.344003342817172, + "tokens_seen": 2742091776 + }, + { + "epoch": 0.66, + "learning_rate": 3.414379714331568e-05, + "loss": 2.3966, + "theoretical_loss": 3.343990827714051, + "tokens_seen": 2742222848 + }, + { + "epoch": 0.66, + "learning_rate": 3.413577274915744e-05, + "loss": 2.4576, + "theoretical_loss": 3.343978313376594, + "tokens_seen": 2742353920 + }, + { + "epoch": 0.66, + "learning_rate": 3.41277483549992e-05, + "loss": 2.5117, + "theoretical_loss": 3.3439657998047174, + "tokens_seen": 2742484992 + }, + { + "epoch": 0.66, + "learning_rate": 3.411972396084096e-05, + "loss": 2.4824, + "theoretical_loss": 3.3439532869983384, + "tokens_seen": 2742616064 + }, + { + "epoch": 0.66, + "learning_rate": 3.411169956668271e-05, + "loss": 2.5898, + "theoretical_loss": 3.3439407749573733, + "tokens_seen": 2742747136 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1504071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.226562261581421, + "objective/train/theoretical_loss": 3.343934519223895, + "objective/train/tokens_used": 1113271776, + "theoretical_loss": 3.343934519223895, + "tokens_seen": 2742812672 + }, + { + "epoch": 0.66, + "learning_rate": 3.410367517252447e-05, + "loss": 2.3934, + "theoretical_loss": 3.3439282636817387, + "tokens_seen": 2742878208 + }, + { + "epoch": 0.66, + "learning_rate": 3.409565077836623e-05, + "loss": 2.4441, + "theoretical_loss": 3.343915753171351, + "tokens_seen": 2743009280 + }, + { + "epoch": 0.66, + "learning_rate": 3.408762638420799e-05, + "loss": 2.4898, + "theoretical_loss": 3.3439032434261273, + "tokens_seen": 2743140352 + }, + { + "epoch": 0.66, + "learning_rate": 3.407960199004975e-05, + "loss": 2.4536, + "theoretical_loss": 3.3438907344459836, + "tokens_seen": 2743271424 + }, + { + "epoch": 0.66, + "learning_rate": 3.407157759589151e-05, + "loss": 2.4249, + "theoretical_loss": 3.343878226230837, + "tokens_seen": 2743402496 + }, + { + "epoch": 0.66, + "learning_rate": 3.406355320173327e-05, + "loss": 2.4515, + "theoretical_loss": 3.343865718780604, + "tokens_seen": 2743533568 + }, + { + "epoch": 0.66, + "learning_rate": 3.405552880757503e-05, + "loss": 2.4402, + "theoretical_loss": 3.3438532120952016, + "tokens_seen": 2743664640 + }, + { + "epoch": 0.66, + "learning_rate": 3.4047504413416785e-05, + "loss": 2.4702, + "theoretical_loss": 3.343840706174546, + "tokens_seen": 2743795712 + }, + { + "epoch": 0.66, + "learning_rate": 3.4039480019258545e-05, + "loss": 2.5509, + "theoretical_loss": 3.3438282010185545, + "tokens_seen": 2743926784 + }, + { + "epoch": 0.66, + "learning_rate": 3.4031455625100304e-05, + "loss": 2.4796, + "theoretical_loss": 3.3438156966271433, + "tokens_seen": 2744057856 + }, + { + "epoch": 0.66, + "learning_rate": 3.4023431230942064e-05, + "loss": 2.4026, + "theoretical_loss": 3.3438031930002294, + "tokens_seen": 2744188928 + }, + { + "epoch": 0.66, + "learning_rate": 3.4015406836783824e-05, + "loss": 2.505, + "theoretical_loss": 3.3437906901377294, + "tokens_seen": 2744320000 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1505433, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.489396810531616, + "objective/train/theoretical_loss": 3.3437781880395603, + "objective/train/tokens_used": 1114910176, + "theoretical_loss": 3.3437781880395603, + "tokens_seen": 2744451072 + }, + { + "epoch": 0.66, + "learning_rate": 3.4007382442625584e-05, + "loss": 2.487, + "theoretical_loss": 3.3437781880395603, + "tokens_seen": 2744451072 + }, + { + "epoch": 0.66, + "learning_rate": 3.3999358048467344e-05, + "loss": 2.4969, + "theoretical_loss": 3.3437656867056385, + "tokens_seen": 2744582144 + }, + { + "epoch": 0.66, + "learning_rate": 3.39913336543091e-05, + "loss": 2.4676, + "theoretical_loss": 3.343753186135881, + "tokens_seen": 2744713216 + }, + { + "epoch": 0.66, + "learning_rate": 3.398330926015086e-05, + "loss": 2.471, + "theoretical_loss": 3.3437406863302046, + "tokens_seen": 2744844288 + }, + { + "epoch": 0.66, + "learning_rate": 3.397528486599262e-05, + "loss": 2.5135, + "theoretical_loss": 3.3437281872885265, + "tokens_seen": 2744975360 + }, + { + "epoch": 0.66, + "learning_rate": 3.396726047183438e-05, + "loss": 2.381, + "theoretical_loss": 3.3437156890107627, + "tokens_seen": 2745106432 + }, + { + "epoch": 0.66, + "learning_rate": 3.395923607767614e-05, + "loss": 2.3236, + "theoretical_loss": 3.3437031914968305, + "tokens_seen": 2745237504 + }, + { + "epoch": 0.66, + "learning_rate": 3.39512116835179e-05, + "loss": 2.5457, + "theoretical_loss": 3.343690694746647, + "tokens_seen": 2745368576 + }, + { + "epoch": 0.66, + "learning_rate": 3.394318728935966e-05, + "loss": 2.4858, + "theoretical_loss": 3.3436781987601285, + "tokens_seen": 2745499648 + }, + { + "epoch": 0.66, + "learning_rate": 3.3935162895201417e-05, + "loss": 2.4755, + "theoretical_loss": 3.3436657035371926, + "tokens_seen": 2745630720 + }, + { + "epoch": 0.66, + "learning_rate": 3.392713850104317e-05, + "loss": 2.4015, + "theoretical_loss": 3.3436532090777553, + "tokens_seen": 2745761792 + }, + { + "epoch": 0.66, + "learning_rate": 3.391911410688493e-05, + "loss": 2.5683, + "theoretical_loss": 3.343640715381734, + "tokens_seen": 2745892864 + }, + { + "epoch": 0.66, + "learning_rate": 3.391108971272669e-05, + "loss": 2.5567, + "theoretical_loss": 3.343628222449046, + "tokens_seen": 2746023936 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1506093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6272194385528564, + "objective/train/theoretical_loss": 3.343621976268926, + "objective/train/tokens_used": 1116548576, + "theoretical_loss": 3.343621976268926, + "tokens_seen": 2746089472 + }, + { + "epoch": 0.66, + "learning_rate": 3.390306531856845e-05, + "loss": 2.4929, + "theoretical_loss": 3.3436157302796072, + "tokens_seen": 2746155008 + }, + { + "epoch": 0.66, + "learning_rate": 3.389504092441021e-05, + "loss": 2.5207, + "theoretical_loss": 3.343603238873336, + "tokens_seen": 2746286080 + }, + { + "epoch": 0.66, + "learning_rate": 3.388701653025197e-05, + "loss": 2.3436, + "theoretical_loss": 3.343590748230148, + "tokens_seen": 2746417152 + }, + { + "epoch": 0.66, + "learning_rate": 3.387899213609373e-05, + "loss": 2.5794, + "theoretical_loss": 3.343578258349961, + "tokens_seen": 2746548224 + }, + { + "epoch": 0.66, + "learning_rate": 3.387096774193548e-05, + "loss": 2.3864, + "theoretical_loss": 3.343565769232691, + "tokens_seen": 2746679296 + }, + { + "epoch": 0.66, + "learning_rate": 3.386294334777724e-05, + "loss": 2.6166, + "theoretical_loss": 3.343553280878256, + "tokens_seen": 2746810368 + }, + { + "epoch": 0.66, + "learning_rate": 3.3854918953619e-05, + "loss": 2.3291, + "theoretical_loss": 3.3435407932865733, + "tokens_seen": 2746941440 + }, + { + "epoch": 0.66, + "learning_rate": 3.384689455946076e-05, + "loss": 2.6428, + "theoretical_loss": 3.3435283064575585, + "tokens_seen": 2747072512 + }, + { + "epoch": 0.66, + "learning_rate": 3.383887016530252e-05, + "loss": 2.4268, + "theoretical_loss": 3.3435158203911297, + "tokens_seen": 2747203584 + }, + { + "epoch": 0.67, + "learning_rate": 3.383084577114428e-05, + "loss": 2.4598, + "theoretical_loss": 3.3435033350872034, + "tokens_seen": 2747334656 + }, + { + "epoch": 0.67, + "learning_rate": 3.382282137698604e-05, + "loss": 2.462, + "theoretical_loss": 3.3434908505456975, + "tokens_seen": 2747465728 + }, + { + "epoch": 0.67, + "learning_rate": 3.3814796982827795e-05, + "loss": 2.509, + "theoretical_loss": 3.3434783667665284, + "tokens_seen": 2747596800 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1507199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.484710693359375, + "objective/train/theoretical_loss": 3.343465883749613, + "objective/train/tokens_used": 1118186976, + "theoretical_loss": 3.343465883749613, + "tokens_seen": 2747727872 + }, + { + "epoch": 0.67, + "learning_rate": 3.3806772588669555e-05, + "loss": 2.4121, + "theoretical_loss": 3.343465883749613, + "tokens_seen": 2747727872 + }, + { + "epoch": 0.67, + "learning_rate": 3.3798748194511315e-05, + "loss": 2.3471, + "theoretical_loss": 3.3434534014948687, + "tokens_seen": 2747858944 + }, + { + "epoch": 0.67, + "learning_rate": 3.3790723800353074e-05, + "loss": 2.5165, + "theoretical_loss": 3.343440920002213, + "tokens_seen": 2747990016 + }, + { + "epoch": 0.67, + "learning_rate": 3.3782699406194834e-05, + "loss": 2.4241, + "theoretical_loss": 3.3434284392715625, + "tokens_seen": 2748121088 + }, + { + "epoch": 0.67, + "learning_rate": 3.3774675012036594e-05, + "loss": 2.4719, + "theoretical_loss": 3.343415959302834, + "tokens_seen": 2748252160 + }, + { + "epoch": 0.67, + "learning_rate": 3.3766650617878354e-05, + "loss": 2.5406, + "theoretical_loss": 3.3434034800959456, + "tokens_seen": 2748383232 + }, + { + "epoch": 0.67, + "learning_rate": 3.3758626223720114e-05, + "loss": 2.4953, + "theoretical_loss": 3.343391001650814, + "tokens_seen": 2748514304 + }, + { + "epoch": 0.67, + "learning_rate": 3.375060182956187e-05, + "loss": 2.4465, + "theoretical_loss": 3.343378523967356, + "tokens_seen": 2748645376 + }, + { + "epoch": 0.67, + "learning_rate": 3.374257743540363e-05, + "loss": 2.5101, + "theoretical_loss": 3.34336604704549, + "tokens_seen": 2748776448 + }, + { + "epoch": 0.67, + "learning_rate": 3.373455304124539e-05, + "loss": 2.433, + "theoretical_loss": 3.3433535708851316, + "tokens_seen": 2748907520 + }, + { + "epoch": 0.67, + "learning_rate": 3.372652864708715e-05, + "loss": 2.5451, + "theoretical_loss": 3.3433410954861986, + "tokens_seen": 2749038592 + }, + { + "epoch": 0.67, + "learning_rate": 3.371850425292891e-05, + "loss": 2.7032, + "theoretical_loss": 3.343328620848609, + "tokens_seen": 2749169664 + }, + { + "epoch": 0.67, + "learning_rate": 3.371047985877067e-05, + "loss": 2.5373, + "theoretical_loss": 3.343316146972279, + "tokens_seen": 2749300736 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1508529, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.529550313949585, + "objective/train/theoretical_loss": 3.3433099103195607, + "objective/train/tokens_used": 1119825376, + "theoretical_loss": 3.3433099103195607, + "tokens_seen": 2749366272 + }, + { + "epoch": 0.67, + "learning_rate": 3.370245546461243e-05, + "loss": 2.4335, + "theoretical_loss": 3.3433036738571267, + "tokens_seen": 2749431808 + }, + { + "epoch": 0.67, + "learning_rate": 3.369443107045418e-05, + "loss": 2.2095, + "theoretical_loss": 3.3432912015030687, + "tokens_seen": 2749562880 + }, + { + "epoch": 0.67, + "learning_rate": 3.368640667629594e-05, + "loss": 2.5101, + "theoretical_loss": 3.343278729910023, + "tokens_seen": 2749693952 + }, + { + "epoch": 0.67, + "learning_rate": 3.36783822821377e-05, + "loss": 2.4012, + "theoretical_loss": 3.3432662590779056, + "tokens_seen": 2749825024 + }, + { + "epoch": 0.67, + "learning_rate": 3.367035788797946e-05, + "loss": 2.5454, + "theoretical_loss": 3.3432537890066354, + "tokens_seen": 2749956096 + }, + { + "epoch": 0.67, + "learning_rate": 3.366233349382122e-05, + "loss": 2.4135, + "theoretical_loss": 3.3432413196961286, + "tokens_seen": 2750087168 + }, + { + "epoch": 0.67, + "learning_rate": 3.365430909966298e-05, + "loss": 2.3647, + "theoretical_loss": 3.343228851146303, + "tokens_seen": 2750218240 + }, + { + "epoch": 0.67, + "learning_rate": 3.364628470550474e-05, + "loss": 2.567, + "theoretical_loss": 3.343216383357076, + "tokens_seen": 2750349312 + }, + { + "epoch": 0.67, + "learning_rate": 3.363826031134649e-05, + "loss": 2.4521, + "theoretical_loss": 3.3432039163283647, + "tokens_seen": 2750480384 + }, + { + "epoch": 0.67, + "learning_rate": 3.363023591718825e-05, + "loss": 2.4044, + "theoretical_loss": 3.3431914500600866, + "tokens_seen": 2750611456 + }, + { + "epoch": 0.67, + "learning_rate": 3.362221152303001e-05, + "loss": 2.5113, + "theoretical_loss": 3.343178984552159, + "tokens_seen": 2750742528 + }, + { + "epoch": 0.67, + "learning_rate": 3.361418712887177e-05, + "loss": 2.4718, + "theoretical_loss": 3.343166519804499, + "tokens_seen": 2750873600 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1509065, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.614013671875, + "objective/train/theoretical_loss": 3.343154055817025, + "objective/train/tokens_used": 1121463776, + "theoretical_loss": 3.343154055817025, + "tokens_seen": 2751004672 + }, + { + "epoch": 0.67, + "learning_rate": 3.360616273471353e-05, + "loss": 2.5216, + "theoretical_loss": 3.343154055817025, + "tokens_seen": 2751004672 + }, + { + "epoch": 0.67, + "learning_rate": 3.359813834055529e-05, + "loss": 2.4854, + "theoretical_loss": 3.343141592589654, + "tokens_seen": 2751135744 + }, + { + "epoch": 0.67, + "learning_rate": 3.359011394639705e-05, + "loss": 2.5276, + "theoretical_loss": 3.3431291301223025, + "tokens_seen": 2751266816 + }, + { + "epoch": 0.67, + "learning_rate": 3.358208955223881e-05, + "loss": 2.5203, + "theoretical_loss": 3.343116668414889, + "tokens_seen": 2751397888 + }, + { + "epoch": 0.67, + "learning_rate": 3.3574065158080565e-05, + "loss": 2.3878, + "theoretical_loss": 3.3431042074673307, + "tokens_seen": 2751528960 + }, + { + "epoch": 0.67, + "learning_rate": 3.3566040763922325e-05, + "loss": 2.5384, + "theoretical_loss": 3.3430917472795447, + "tokens_seen": 2751660032 + }, + { + "epoch": 0.67, + "learning_rate": 3.3558016369764085e-05, + "loss": 2.5459, + "theoretical_loss": 3.3430792878514493, + "tokens_seen": 2751791104 + }, + { + "epoch": 0.67, + "learning_rate": 3.3549991975605845e-05, + "loss": 2.4486, + "theoretical_loss": 3.343066829182961, + "tokens_seen": 2751922176 + }, + { + "epoch": 0.67, + "learning_rate": 3.3541967581447604e-05, + "loss": 2.4876, + "theoretical_loss": 3.343054371273998, + "tokens_seen": 2752053248 + }, + { + "epoch": 0.67, + "learning_rate": 3.3533943187289364e-05, + "loss": 2.5864, + "theoretical_loss": 3.343041914124478, + "tokens_seen": 2752184320 + }, + { + "epoch": 0.67, + "learning_rate": 3.3525918793131124e-05, + "loss": 2.4352, + "theoretical_loss": 3.3430294577343176, + "tokens_seen": 2752315392 + }, + { + "epoch": 0.67, + "learning_rate": 3.351789439897288e-05, + "loss": 2.4919, + "theoretical_loss": 3.3430170021034353, + "tokens_seen": 2752446464 + }, + { + "epoch": 0.67, + "learning_rate": 3.350987000481464e-05, + "loss": 2.4718, + "theoretical_loss": 3.3430045472317484, + "tokens_seen": 2752577536 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1510225, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4023191928863525, + "objective/train/theoretical_loss": 3.3429983200805773, + "objective/train/tokens_used": 1123102176, + "theoretical_loss": 3.3429983200805773, + "tokens_seen": 2752643072 + }, + { + "epoch": 0.67, + "learning_rate": 3.35018456106564e-05, + "loss": 2.4753, + "theoretical_loss": 3.342992093119174, + "tokens_seen": 2752708608 + }, + { + "epoch": 0.67, + "learning_rate": 3.349382121649816e-05, + "loss": 2.5409, + "theoretical_loss": 3.3429796397656304, + "tokens_seen": 2752839680 + }, + { + "epoch": 0.67, + "learning_rate": 3.348579682233992e-05, + "loss": 2.4779, + "theoretical_loss": 3.3429671871710345, + "tokens_seen": 2752970752 + }, + { + "epoch": 0.67, + "learning_rate": 3.347777242818168e-05, + "loss": 2.3324, + "theoretical_loss": 3.3429547353353044, + "tokens_seen": 2753101824 + }, + { + "epoch": 0.67, + "learning_rate": 3.346974803402344e-05, + "loss": 2.5782, + "theoretical_loss": 3.342942284258358, + "tokens_seen": 2753232896 + }, + { + "epoch": 0.67, + "learning_rate": 3.34617236398652e-05, + "loss": 2.5729, + "theoretical_loss": 3.3429298339401123, + "tokens_seen": 2753363968 + }, + { + "epoch": 0.67, + "learning_rate": 3.345369924570695e-05, + "loss": 2.5398, + "theoretical_loss": 3.342917384380485, + "tokens_seen": 2753495040 + }, + { + "epoch": 0.67, + "learning_rate": 3.344567485154871e-05, + "loss": 2.6094, + "theoretical_loss": 3.3429049355793943, + "tokens_seen": 2753626112 + }, + { + "epoch": 0.67, + "learning_rate": 3.343765045739047e-05, + "loss": 2.5743, + "theoretical_loss": 3.342892487536757, + "tokens_seen": 2753757184 + }, + { + "epoch": 0.67, + "learning_rate": 3.342962606323223e-05, + "loss": 2.4056, + "theoretical_loss": 3.342880040252492, + "tokens_seen": 2753888256 + }, + { + "epoch": 0.67, + "learning_rate": 3.342160166907399e-05, + "loss": 2.571, + "theoretical_loss": 3.342867593726516, + "tokens_seen": 2754019328 + }, + { + "epoch": 0.67, + "learning_rate": 3.341357727491575e-05, + "loss": 2.4542, + "theoretical_loss": 3.3428551479587476, + "tokens_seen": 2754150400 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1510639, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4005911350250244, + "objective/train/theoretical_loss": 3.3428427029491035, + "objective/train/tokens_used": 1124740576, + "theoretical_loss": 3.3428427029491035, + "tokens_seen": 2754281472 + }, + { + "epoch": 0.67, + "learning_rate": 3.340555288075751e-05, + "loss": 2.5115, + "theoretical_loss": 3.3428427029491035, + "tokens_seen": 2754281472 + }, + { + "epoch": 0.67, + "learning_rate": 3.339752848659926e-05, + "loss": 2.5324, + "theoretical_loss": 3.342830258697502, + "tokens_seen": 2754412544 + }, + { + "epoch": 0.67, + "learning_rate": 3.338950409244102e-05, + "loss": 2.4639, + "theoretical_loss": 3.342817815203861, + "tokens_seen": 2754543616 + }, + { + "epoch": 0.67, + "learning_rate": 3.338147969828278e-05, + "loss": 2.4533, + "theoretical_loss": 3.3428053724680975, + "tokens_seen": 2754674688 + }, + { + "epoch": 0.67, + "learning_rate": 3.337345530412454e-05, + "loss": 2.6524, + "theoretical_loss": 3.3427929304901305, + "tokens_seen": 2754805760 + }, + { + "epoch": 0.67, + "learning_rate": 3.33654309099663e-05, + "loss": 2.6015, + "theoretical_loss": 3.3427804892698774, + "tokens_seen": 2754936832 + }, + { + "epoch": 0.67, + "learning_rate": 3.335740651580806e-05, + "loss": 2.5677, + "theoretical_loss": 3.342768048807255, + "tokens_seen": 2755067904 + }, + { + "epoch": 0.67, + "learning_rate": 3.334938212164982e-05, + "loss": 2.604, + "theoretical_loss": 3.3427556091021824, + "tokens_seen": 2755198976 + }, + { + "epoch": 0.67, + "learning_rate": 3.3341357727491575e-05, + "loss": 2.6709, + "theoretical_loss": 3.3427431701545767, + "tokens_seen": 2755330048 + }, + { + "epoch": 0.67, + "learning_rate": 3.3333333333333335e-05, + "loss": 2.5263, + "theoretical_loss": 3.342730731964356, + "tokens_seen": 2755461120 + }, + { + "epoch": 0.67, + "learning_rate": 3.3325308939175095e-05, + "loss": 2.6037, + "theoretical_loss": 3.342718294531438, + "tokens_seen": 2755592192 + }, + { + "epoch": 0.67, + "learning_rate": 3.3317284545016855e-05, + "loss": 2.4935, + "theoretical_loss": 3.342705857855741, + "tokens_seen": 2755723264 + }, + { + "epoch": 0.67, + "learning_rate": 3.3309260150858615e-05, + "loss": 2.4643, + "theoretical_loss": 3.3426934219371827, + "tokens_seen": 2755854336 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1511832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5330140590667725, + "objective/train/theoretical_loss": 3.3426872042618045, + "objective/train/tokens_used": 1126378976, + "theoretical_loss": 3.3426872042618045, + "tokens_seen": 2755919872 + }, + { + "epoch": 0.67, + "learning_rate": 3.3301235756700375e-05, + "loss": 2.5681, + "theoretical_loss": 3.3426809867756804, + "tokens_seen": 2755985408 + }, + { + "epoch": 0.67, + "learning_rate": 3.3293211362542134e-05, + "loss": 2.4618, + "theoretical_loss": 3.342668552371153, + "tokens_seen": 2756116480 + }, + { + "epoch": 0.67, + "learning_rate": 3.3285186968383894e-05, + "loss": 2.5302, + "theoretical_loss": 3.3426561187235175, + "tokens_seen": 2756247552 + }, + { + "epoch": 0.67, + "learning_rate": 3.327716257422565e-05, + "loss": 2.4883, + "theoretical_loss": 3.3426436858326927, + "tokens_seen": 2756378624 + }, + { + "epoch": 0.67, + "learning_rate": 3.326913818006741e-05, + "loss": 2.4475, + "theoretical_loss": 3.3426312536985954, + "tokens_seen": 2756509696 + }, + { + "epoch": 0.67, + "learning_rate": 3.326111378590917e-05, + "loss": 2.5158, + "theoretical_loss": 3.3426188223211453, + "tokens_seen": 2756640768 + }, + { + "epoch": 0.67, + "learning_rate": 3.325308939175093e-05, + "loss": 2.5126, + "theoretical_loss": 3.3426063917002584, + "tokens_seen": 2756771840 + }, + { + "epoch": 0.67, + "learning_rate": 3.324506499759269e-05, + "loss": 2.5728, + "theoretical_loss": 3.342593961835854, + "tokens_seen": 2756902912 + }, + { + "epoch": 0.67, + "learning_rate": 3.323704060343445e-05, + "loss": 2.5345, + "theoretical_loss": 3.34258153272785, + "tokens_seen": 2757033984 + }, + { + "epoch": 0.67, + "learning_rate": 3.322901620927621e-05, + "loss": 2.6041, + "theoretical_loss": 3.342569104376164, + "tokens_seen": 2757165056 + }, + { + "epoch": 0.67, + "learning_rate": 3.322099181511796e-05, + "loss": 2.4431, + "theoretical_loss": 3.342556676780714, + "tokens_seen": 2757296128 + }, + { + "epoch": 0.67, + "learning_rate": 3.321296742095972e-05, + "loss": 2.4356, + "theoretical_loss": 3.3425442499414184, + "tokens_seen": 2757427200 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1512441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.350581645965576, + "objective/train/theoretical_loss": 3.342531823858195, + "objective/train/tokens_used": 1128017376, + "theoretical_loss": 3.342531823858195, + "tokens_seen": 2757558272 + }, + { + "epoch": 0.67, + "learning_rate": 3.320494302680148e-05, + "loss": 2.5186, + "theoretical_loss": 3.342531823858195, + "tokens_seen": 2757558272 + }, + { + "epoch": 0.67, + "learning_rate": 3.319691863264324e-05, + "loss": 2.5281, + "theoretical_loss": 3.342519398530962, + "tokens_seen": 2757689344 + }, + { + "epoch": 0.67, + "learning_rate": 3.3188894238485e-05, + "loss": 2.3466, + "theoretical_loss": 3.342506973959637, + "tokens_seen": 2757820416 + }, + { + "epoch": 0.67, + "learning_rate": 3.318086984432676e-05, + "loss": 2.2902, + "theoretical_loss": 3.342494550144139, + "tokens_seen": 2757951488 + }, + { + "epoch": 0.67, + "learning_rate": 3.317284545016851e-05, + "loss": 2.4138, + "theoretical_loss": 3.342482127084385, + "tokens_seen": 2758082560 + }, + { + "epoch": 0.67, + "learning_rate": 3.316482105601027e-05, + "loss": 2.418, + "theoretical_loss": 3.342469704780295, + "tokens_seen": 2758213632 + }, + { + "epoch": 0.67, + "learning_rate": 3.315679666185203e-05, + "loss": 2.5967, + "theoretical_loss": 3.3424572832317847, + "tokens_seen": 2758344704 + }, + { + "epoch": 0.67, + "learning_rate": 3.314877226769379e-05, + "loss": 2.5354, + "theoretical_loss": 3.3424448624387737, + "tokens_seen": 2758475776 + }, + { + "epoch": 0.67, + "learning_rate": 3.314074787353555e-05, + "loss": 2.7509, + "theoretical_loss": 3.34243244240118, + "tokens_seen": 2758606848 + }, + { + "epoch": 0.67, + "learning_rate": 3.313272347937731e-05, + "loss": 2.5234, + "theoretical_loss": 3.3424200231189216, + "tokens_seen": 2758737920 + }, + { + "epoch": 0.67, + "learning_rate": 3.3124699085219065e-05, + "loss": 2.5651, + "theoretical_loss": 3.3424076045919167, + "tokens_seen": 2758868992 + }, + { + "epoch": 0.67, + "learning_rate": 3.3116674691060825e-05, + "loss": 2.5245, + "theoretical_loss": 3.342395186820083, + "tokens_seen": 2759000064 + }, + { + "epoch": 0.67, + "learning_rate": 3.3108650296902585e-05, + "loss": 2.3784, + "theoretical_loss": 3.34238276980334, + "tokens_seen": 2759131136 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1513700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7759766578674316, + "objective/train/theoretical_loss": 3.342376561578101, + "objective/train/tokens_used": 1129655776, + "theoretical_loss": 3.342376561578101, + "tokens_seen": 2759196672 + }, + { + "epoch": 0.67, + "learning_rate": 3.3100625902744345e-05, + "loss": 2.5232, + "theoretical_loss": 3.3423703535416043, + "tokens_seen": 2759262208 + }, + { + "epoch": 0.67, + "learning_rate": 3.3092601508586105e-05, + "loss": 2.5183, + "theoretical_loss": 3.3423579380347954, + "tokens_seen": 2759393280 + }, + { + "epoch": 0.67, + "learning_rate": 3.3084577114427865e-05, + "loss": 2.5246, + "theoretical_loss": 3.342345523282831, + "tokens_seen": 2759524352 + }, + { + "epoch": 0.67, + "learning_rate": 3.307655272026962e-05, + "loss": 2.4896, + "theoretical_loss": 3.3423331092856294, + "tokens_seen": 2759655424 + }, + { + "epoch": 0.67, + "learning_rate": 3.306852832611138e-05, + "loss": 2.6971, + "theoretical_loss": 3.342320696043109, + "tokens_seen": 2759786496 + }, + { + "epoch": 0.67, + "learning_rate": 3.306050393195314e-05, + "loss": 2.4454, + "theoretical_loss": 3.342308283555188, + "tokens_seen": 2759917568 + }, + { + "epoch": 0.67, + "learning_rate": 3.30524795377949e-05, + "loss": 2.4226, + "theoretical_loss": 3.3422958718217846, + "tokens_seen": 2760048640 + }, + { + "epoch": 0.67, + "learning_rate": 3.304445514363666e-05, + "loss": 2.526, + "theoretical_loss": 3.3422834608428174, + "tokens_seen": 2760179712 + }, + { + "epoch": 0.67, + "learning_rate": 3.303643074947842e-05, + "loss": 2.5769, + "theoretical_loss": 3.342271050618204, + "tokens_seen": 2760310784 + }, + { + "epoch": 0.67, + "learning_rate": 3.302840635532017e-05, + "loss": 2.4153, + "theoretical_loss": 3.342258641147864, + "tokens_seen": 2760441856 + }, + { + "epoch": 0.67, + "learning_rate": 3.302038196116193e-05, + "loss": 2.436, + "theoretical_loss": 3.342246232431714, + "tokens_seen": 2760572928 + }, + { + "epoch": 0.67, + "learning_rate": 3.301235756700369e-05, + "loss": 2.4677, + "theoretical_loss": 3.342233824469674, + "tokens_seen": 2760704000 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1514208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.610041856765747, + "objective/train/theoretical_loss": 3.342221417261662, + "objective/train/tokens_used": 1131294176, + "theoretical_loss": 3.342221417261662, + "tokens_seen": 2760835072 + }, + { + "epoch": 0.67, + "learning_rate": 3.300433317284545e-05, + "loss": 2.4808, + "theoretical_loss": 3.342221417261662, + "tokens_seen": 2760835072 + }, + { + "epoch": 0.67, + "learning_rate": 3.299630877868721e-05, + "loss": 2.5169, + "theoretical_loss": 3.3422090108075957, + "tokens_seen": 2760966144 + }, + { + "epoch": 0.67, + "learning_rate": 3.298828438452897e-05, + "loss": 2.4137, + "theoretical_loss": 3.342196605107394, + "tokens_seen": 2761097216 + }, + { + "epoch": 0.67, + "learning_rate": 3.298025999037072e-05, + "loss": 2.6073, + "theoretical_loss": 3.342184200160975, + "tokens_seen": 2761228288 + }, + { + "epoch": 0.67, + "learning_rate": 3.297223559621248e-05, + "loss": 2.6416, + "theoretical_loss": 3.3421717959682575, + "tokens_seen": 2761359360 + }, + { + "epoch": 0.67, + "learning_rate": 3.296421120205424e-05, + "loss": 2.563, + "theoretical_loss": 3.34215939252916, + "tokens_seen": 2761490432 + }, + { + "epoch": 0.67, + "learning_rate": 3.2956186807896e-05, + "loss": 2.3484, + "theoretical_loss": 3.3421469898436005, + "tokens_seen": 2761621504 + }, + { + "epoch": 0.67, + "learning_rate": 3.294816241373776e-05, + "loss": 2.4974, + "theoretical_loss": 3.3421345879114974, + "tokens_seen": 2761752576 + }, + { + "epoch": 0.67, + "learning_rate": 3.294013801957952e-05, + "loss": 2.5213, + "theoretical_loss": 3.3421221867327695, + "tokens_seen": 2761883648 + }, + { + "epoch": 0.67, + "learning_rate": 3.293211362542128e-05, + "loss": 2.4602, + "theoretical_loss": 3.3421097863073355, + "tokens_seen": 2762014720 + }, + { + "epoch": 0.67, + "learning_rate": 3.2924089231263036e-05, + "loss": 2.5766, + "theoretical_loss": 3.3420973866351136, + "tokens_seen": 2762145792 + }, + { + "epoch": 0.67, + "learning_rate": 3.2916064837104796e-05, + "loss": 2.5199, + "theoretical_loss": 3.3420849877160226, + "tokens_seen": 2762276864 + }, + { + "epoch": 0.67, + "learning_rate": 3.2908040442946556e-05, + "loss": 2.5617, + "theoretical_loss": 3.3420725895499803, + "tokens_seen": 2762407936 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1514886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.244025230407715, + "objective/train/theoretical_loss": 3.342066390749327, + "objective/train/tokens_used": 1132932576, + "theoretical_loss": 3.342066390749327, + "tokens_seen": 2762473472 + }, + { + "epoch": 0.67, + "learning_rate": 3.2900016048788316e-05, + "loss": 2.4041, + "theoretical_loss": 3.3420601921369055, + "tokens_seen": 2762539008 + }, + { + "epoch": 0.67, + "learning_rate": 3.2891991654630076e-05, + "loss": 2.4773, + "theoretical_loss": 3.3420477954767174, + "tokens_seen": 2762670080 + }, + { + "epoch": 0.67, + "learning_rate": 3.2883967260471835e-05, + "loss": 2.5283, + "theoretical_loss": 3.342035399569334, + "tokens_seen": 2762801152 + }, + { + "epoch": 0.67, + "learning_rate": 3.2875942866313595e-05, + "loss": 2.5782, + "theoretical_loss": 3.3420230044146737, + "tokens_seen": 2762932224 + }, + { + "epoch": 0.67, + "learning_rate": 3.286791847215535e-05, + "loss": 2.6017, + "theoretical_loss": 3.3420106100126556, + "tokens_seen": 2763063296 + }, + { + "epoch": 0.67, + "learning_rate": 3.285989407799711e-05, + "loss": 2.5161, + "theoretical_loss": 3.3419982163631983, + "tokens_seen": 2763194368 + }, + { + "epoch": 0.67, + "learning_rate": 3.285186968383887e-05, + "loss": 2.512, + "theoretical_loss": 3.34198582346622, + "tokens_seen": 2763325440 + }, + { + "epoch": 0.67, + "learning_rate": 3.284384528968063e-05, + "loss": 2.5965, + "theoretical_loss": 3.341973431321639, + "tokens_seen": 2763456512 + }, + { + "epoch": 0.67, + "learning_rate": 3.283582089552239e-05, + "loss": 2.5986, + "theoretical_loss": 3.3419610399293753, + "tokens_seen": 2763587584 + }, + { + "epoch": 0.68, + "learning_rate": 3.282779650136415e-05, + "loss": 2.6114, + "theoretical_loss": 3.341948649289346, + "tokens_seen": 2763718656 + }, + { + "epoch": 0.68, + "learning_rate": 3.281977210720591e-05, + "loss": 2.5027, + "theoretical_loss": 3.3419362594014705, + "tokens_seen": 2763849728 + }, + { + "epoch": 0.68, + "learning_rate": 3.281174771304767e-05, + "loss": 2.6566, + "theoretical_loss": 3.341923870265668, + "tokens_seen": 2763980800 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1515515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5417497158050537, + "objective/train/theoretical_loss": 3.3419114818818563, + "objective/train/tokens_used": 1134570976, + "theoretical_loss": 3.3419114818818563, + "tokens_seen": 2764111872 + }, + { + "epoch": 0.68, + "learning_rate": 3.280372331888942e-05, + "loss": 2.6045, + "theoretical_loss": 3.3419114818818563, + "tokens_seen": 2764111872 + }, + { + "epoch": 0.68, + "learning_rate": 3.279569892473118e-05, + "loss": 2.6287, + "theoretical_loss": 3.3418990942499542, + "tokens_seen": 2764242944 + }, + { + "epoch": 0.68, + "learning_rate": 3.278767453057294e-05, + "loss": 2.5693, + "theoretical_loss": 3.3418867073698806, + "tokens_seen": 2764374016 + }, + { + "epoch": 0.68, + "learning_rate": 3.27796501364147e-05, + "loss": 2.6235, + "theoretical_loss": 3.341874321241555, + "tokens_seen": 2764505088 + }, + { + "epoch": 0.68, + "learning_rate": 3.277162574225646e-05, + "loss": 2.5625, + "theoretical_loss": 3.3418619358648947, + "tokens_seen": 2764636160 + }, + { + "epoch": 0.68, + "learning_rate": 3.276360134809822e-05, + "loss": 2.4813, + "theoretical_loss": 3.341849551239819, + "tokens_seen": 2764767232 + }, + { + "epoch": 0.68, + "learning_rate": 3.275557695393998e-05, + "loss": 2.6026, + "theoretical_loss": 3.341837167366247, + "tokens_seen": 2764898304 + }, + { + "epoch": 0.68, + "learning_rate": 3.2747552559781734e-05, + "loss": 2.5526, + "theoretical_loss": 3.3418247842440976, + "tokens_seen": 2765029376 + }, + { + "epoch": 0.68, + "learning_rate": 3.2739528165623493e-05, + "loss": 2.7275, + "theoretical_loss": 3.341812401873289, + "tokens_seen": 2765160448 + }, + { + "epoch": 0.68, + "learning_rate": 3.273150377146525e-05, + "loss": 2.6639, + "theoretical_loss": 3.34180002025374, + "tokens_seen": 2765291520 + }, + { + "epoch": 0.68, + "learning_rate": 3.272347937730701e-05, + "loss": 2.7329, + "theoretical_loss": 3.34178763938537, + "tokens_seen": 2765422592 + }, + { + "epoch": 0.68, + "learning_rate": 3.271545498314877e-05, + "loss": 2.6393, + "theoretical_loss": 3.341775259268098, + "tokens_seen": 2765553664 + }, + { + "epoch": 0.68, + "learning_rate": 3.270743058899053e-05, + "loss": 2.5101, + "theoretical_loss": 3.3417628799018413, + "tokens_seen": 2765684736 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1516638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1252388954162598, + "objective/train/theoretical_loss": 3.3417566905003193, + "objective/train/tokens_used": 1136209376, + "theoretical_loss": 3.3417566905003193, + "tokens_seen": 2765750272 + }, + { + "epoch": 0.68, + "learning_rate": 3.269940619483229e-05, + "loss": 2.4778, + "theoretical_loss": 3.3417505012865205, + "tokens_seen": 2765815808 + }, + { + "epoch": 0.68, + "learning_rate": 3.2691381800674046e-05, + "loss": 2.4849, + "theoretical_loss": 3.3417381234220533, + "tokens_seen": 2765946880 + }, + { + "epoch": 0.68, + "learning_rate": 3.2683357406515806e-05, + "loss": 2.6644, + "theoretical_loss": 3.3417257463083594, + "tokens_seen": 2766077952 + }, + { + "epoch": 0.68, + "learning_rate": 3.2675333012357566e-05, + "loss": 2.5996, + "theoretical_loss": 3.3417133699453574, + "tokens_seen": 2766209024 + }, + { + "epoch": 0.68, + "learning_rate": 3.2667308618199326e-05, + "loss": 2.5837, + "theoretical_loss": 3.341700994332966, + "tokens_seen": 2766340096 + }, + { + "epoch": 0.68, + "learning_rate": 3.2659284224041086e-05, + "loss": 2.4346, + "theoretical_loss": 3.341688619471104, + "tokens_seen": 2766471168 + }, + { + "epoch": 0.68, + "learning_rate": 3.2651259829882846e-05, + "loss": 2.6304, + "theoretical_loss": 3.3416762453596913, + "tokens_seen": 2766602240 + }, + { + "epoch": 0.68, + "learning_rate": 3.2643235435724606e-05, + "loss": 2.5374, + "theoretical_loss": 3.3416638719986453, + "tokens_seen": 2766733312 + }, + { + "epoch": 0.68, + "learning_rate": 3.2635211041566365e-05, + "loss": 2.6512, + "theoretical_loss": 3.3416514993878863, + "tokens_seen": 2766864384 + }, + { + "epoch": 0.68, + "learning_rate": 3.262718664740812e-05, + "loss": 2.6426, + "theoretical_loss": 3.3416391275273325, + "tokens_seen": 2766995456 + }, + { + "epoch": 0.68, + "learning_rate": 3.261916225324988e-05, + "loss": 2.4941, + "theoretical_loss": 3.341626756416903, + "tokens_seen": 2767126528 + }, + { + "epoch": 0.68, + "learning_rate": 3.261113785909164e-05, + "loss": 2.4554, + "theoretical_loss": 3.3416143860565173, + "tokens_seen": 2767257600 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1517741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.414573907852173, + "objective/train/theoretical_loss": 3.341602016446094, + "objective/train/tokens_used": 1137847776, + "theoretical_loss": 3.341602016446094, + "tokens_seen": 2767388672 + }, + { + "epoch": 0.68, + "learning_rate": 3.26031134649334e-05, + "loss": 2.3742, + "theoretical_loss": 3.341602016446094, + "tokens_seen": 2767388672 + }, + { + "epoch": 0.68, + "learning_rate": 3.259508907077516e-05, + "loss": 2.4867, + "theoretical_loss": 3.3415896475855513, + "tokens_seen": 2767519744 + }, + { + "epoch": 0.68, + "learning_rate": 3.258706467661692e-05, + "loss": 2.5839, + "theoretical_loss": 3.34157727947481, + "tokens_seen": 2767650816 + }, + { + "epoch": 0.68, + "learning_rate": 3.257904028245868e-05, + "loss": 2.5469, + "theoretical_loss": 3.341564912113788, + "tokens_seen": 2767781888 + }, + { + "epoch": 0.68, + "learning_rate": 3.257101588830043e-05, + "loss": 2.4627, + "theoretical_loss": 3.341552545502404, + "tokens_seen": 2767912960 + }, + { + "epoch": 0.68, + "learning_rate": 3.256299149414219e-05, + "loss": 2.4721, + "theoretical_loss": 3.341540179640578, + "tokens_seen": 2768044032 + }, + { + "epoch": 0.68, + "learning_rate": 3.255496709998395e-05, + "loss": 2.5981, + "theoretical_loss": 3.341527814528229, + "tokens_seen": 2768175104 + }, + { + "epoch": 0.68, + "learning_rate": 3.254694270582571e-05, + "loss": 2.6825, + "theoretical_loss": 3.341515450165275, + "tokens_seen": 2768306176 + }, + { + "epoch": 0.68, + "learning_rate": 3.253891831166747e-05, + "loss": 2.6159, + "theoretical_loss": 3.3415030865516364, + "tokens_seen": 2768437248 + }, + { + "epoch": 0.68, + "learning_rate": 3.253089391750923e-05, + "loss": 2.5181, + "theoretical_loss": 3.341490723687232, + "tokens_seen": 2768568320 + }, + { + "epoch": 0.68, + "learning_rate": 3.252286952335099e-05, + "loss": 2.32, + "theoretical_loss": 3.3414783615719807, + "tokens_seen": 2768699392 + }, + { + "epoch": 0.68, + "learning_rate": 3.251484512919275e-05, + "loss": 2.5059, + "theoretical_loss": 3.341466000205801, + "tokens_seen": 2768830464 + }, + { + "epoch": 0.68, + "learning_rate": 3.2506820735034504e-05, + "loss": 2.5147, + "theoretical_loss": 3.3414536395886136, + "tokens_seen": 2768961536 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1518343, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4138665199279785, + "objective/train/theoretical_loss": 3.3414474595608663, + "objective/train/tokens_used": 1139486176, + "theoretical_loss": 3.3414474595608663, + "tokens_seen": 2769027072 + }, + { + "epoch": 0.68, + "learning_rate": 3.2498796340876264e-05, + "loss": 2.6642, + "theoretical_loss": 3.3414412797203363, + "tokens_seen": 2769092608 + }, + { + "epoch": 0.68, + "learning_rate": 3.2490771946718023e-05, + "loss": 2.5414, + "theoretical_loss": 3.3414289206008885, + "tokens_seen": 2769223680 + }, + { + "epoch": 0.68, + "learning_rate": 3.248274755255978e-05, + "loss": 2.6604, + "theoretical_loss": 3.34141656223019, + "tokens_seen": 2769354752 + }, + { + "epoch": 0.68, + "learning_rate": 3.247472315840154e-05, + "loss": 2.6459, + "theoretical_loss": 3.34140420460816, + "tokens_seen": 2769485824 + }, + { + "epoch": 0.68, + "learning_rate": 3.24666987642433e-05, + "loss": 2.5674, + "theoretical_loss": 3.341391847734717, + "tokens_seen": 2769616896 + }, + { + "epoch": 0.68, + "learning_rate": 3.245867437008506e-05, + "loss": 2.6249, + "theoretical_loss": 3.3413794916097808, + "tokens_seen": 2769747968 + }, + { + "epoch": 0.68, + "learning_rate": 3.2450649975926816e-05, + "loss": 2.5689, + "theoretical_loss": 3.34136713623327, + "tokens_seen": 2769879040 + }, + { + "epoch": 0.68, + "learning_rate": 3.2442625581768576e-05, + "loss": 2.5172, + "theoretical_loss": 3.341354781605105, + "tokens_seen": 2770010112 + }, + { + "epoch": 0.68, + "learning_rate": 3.2434601187610336e-05, + "loss": 2.443, + "theoretical_loss": 3.3413424277252037, + "tokens_seen": 2770141184 + }, + { + "epoch": 0.68, + "learning_rate": 3.2426576793452096e-05, + "loss": 2.4569, + "theoretical_loss": 3.341330074593486, + "tokens_seen": 2770272256 + }, + { + "epoch": 0.68, + "learning_rate": 3.2418552399293856e-05, + "loss": 2.4347, + "theoretical_loss": 3.341317722209872, + "tokens_seen": 2770403328 + }, + { + "epoch": 0.68, + "learning_rate": 3.2410528005135616e-05, + "loss": 2.4839, + "theoretical_loss": 3.34130537057428, + "tokens_seen": 2770534400 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1519063, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5506296157836914, + "objective/train/theoretical_loss": 3.3412930196866295, + "objective/train/tokens_used": 1141124576, + "theoretical_loss": 3.3412930196866295, + "tokens_seen": 2770665472 + }, + { + "epoch": 0.68, + "learning_rate": 3.2402503610977376e-05, + "loss": 2.5008, + "theoretical_loss": 3.3412930196866295, + "tokens_seen": 2770665472 + }, + { + "epoch": 0.68, + "learning_rate": 3.239447921681913e-05, + "loss": 2.4328, + "theoretical_loss": 3.3412806695468396, + "tokens_seen": 2770796544 + }, + { + "epoch": 0.68, + "learning_rate": 3.238645482266089e-05, + "loss": 2.5201, + "theoretical_loss": 3.34126832015483, + "tokens_seen": 2770927616 + }, + { + "epoch": 0.68, + "learning_rate": 3.237843042850265e-05, + "loss": 2.5948, + "theoretical_loss": 3.3412559715105203, + "tokens_seen": 2771058688 + }, + { + "epoch": 0.68, + "learning_rate": 3.237040603434441e-05, + "loss": 2.6841, + "theoretical_loss": 3.3412436236138294, + "tokens_seen": 2771189760 + }, + { + "epoch": 0.68, + "learning_rate": 3.236238164018617e-05, + "loss": 2.4125, + "theoretical_loss": 3.3412312764646765, + "tokens_seen": 2771320832 + }, + { + "epoch": 0.68, + "learning_rate": 3.235435724602793e-05, + "loss": 2.4078, + "theoretical_loss": 3.3412189300629818, + "tokens_seen": 2771451904 + }, + { + "epoch": 0.68, + "learning_rate": 3.234633285186969e-05, + "loss": 2.6153, + "theoretical_loss": 3.3412065844086642, + "tokens_seen": 2771582976 + }, + { + "epoch": 0.68, + "learning_rate": 3.233830845771145e-05, + "loss": 2.4019, + "theoretical_loss": 3.341194239501643, + "tokens_seen": 2771714048 + }, + { + "epoch": 0.68, + "learning_rate": 3.23302840635532e-05, + "loss": 2.5149, + "theoretical_loss": 3.341181895341838, + "tokens_seen": 2771845120 + }, + { + "epoch": 0.68, + "learning_rate": 3.232225966939496e-05, + "loss": 2.4509, + "theoretical_loss": 3.3411695519291684, + "tokens_seen": 2771976192 + }, + { + "epoch": 0.68, + "learning_rate": 3.231423527523672e-05, + "loss": 2.4817, + "theoretical_loss": 3.3411572092635535, + "tokens_seen": 2772107264 + }, + { + "epoch": 0.68, + "learning_rate": 3.230621088107848e-05, + "loss": 2.5289, + "theoretical_loss": 3.341144867344913, + "tokens_seen": 2772238336 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1519662, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.863593816757202, + "objective/train/theoretical_loss": 3.3411386966656833, + "objective/train/tokens_used": 1142762976, + "theoretical_loss": 3.3411386966656833, + "tokens_seen": 2772303872 + }, + { + "epoch": 0.68, + "learning_rate": 3.229818648692024e-05, + "loss": 2.5754, + "theoretical_loss": 3.3411325261731664, + "tokens_seen": 2772369408 + }, + { + "epoch": 0.68, + "learning_rate": 3.2290162092762e-05, + "loss": 2.4861, + "theoretical_loss": 3.341120185748233, + "tokens_seen": 2772500480 + }, + { + "epoch": 0.68, + "learning_rate": 3.228213769860376e-05, + "loss": 2.5871, + "theoretical_loss": 3.3411078460700327, + "tokens_seen": 2772631552 + }, + { + "epoch": 0.68, + "learning_rate": 3.2274113304445514e-05, + "loss": 2.7053, + "theoretical_loss": 3.3410955071384842, + "tokens_seen": 2772762624 + }, + { + "epoch": 0.68, + "learning_rate": 3.2266088910287274e-05, + "loss": 2.5365, + "theoretical_loss": 3.341083168953508, + "tokens_seen": 2772893696 + }, + { + "epoch": 0.68, + "learning_rate": 3.2258064516129034e-05, + "loss": 2.4959, + "theoretical_loss": 3.341070831515023, + "tokens_seen": 2773024768 + }, + { + "epoch": 0.68, + "learning_rate": 3.2250040121970793e-05, + "loss": 2.4418, + "theoretical_loss": 3.341058494822949, + "tokens_seen": 2773155840 + }, + { + "epoch": 0.68, + "learning_rate": 3.2242015727812553e-05, + "loss": 2.4594, + "theoretical_loss": 3.3410461588772056, + "tokens_seen": 2773286912 + }, + { + "epoch": 0.68, + "learning_rate": 3.223399133365431e-05, + "loss": 2.4591, + "theoretical_loss": 3.341033823677712, + "tokens_seen": 2773417984 + }, + { + "epoch": 0.68, + "learning_rate": 3.222596693949607e-05, + "loss": 2.4975, + "theoretical_loss": 3.3410214892243886, + "tokens_seen": 2773549056 + }, + { + "epoch": 0.68, + "learning_rate": 3.2217942545337826e-05, + "loss": 2.6172, + "theoretical_loss": 3.3410091555171544, + "tokens_seen": 2773680128 + }, + { + "epoch": 0.68, + "learning_rate": 3.2209918151179586e-05, + "loss": 2.5399, + "theoretical_loss": 3.3409968225559292, + "tokens_seen": 2773811200 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1520187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7829596996307373, + "objective/train/theoretical_loss": 3.3409844903406323, + "objective/train/tokens_used": 1144401376, + "theoretical_loss": 3.3409844903406323, + "tokens_seen": 2773942272 + }, + { + "epoch": 0.68, + "learning_rate": 3.2201893757021346e-05, + "loss": 2.5858, + "theoretical_loss": 3.3409844903406323, + "tokens_seen": 2773942272 + }, + { + "epoch": 0.68, + "learning_rate": 3.2193869362863106e-05, + "loss": 2.6247, + "theoretical_loss": 3.3409721588711836, + "tokens_seen": 2774073344 + }, + { + "epoch": 0.68, + "learning_rate": 3.2185844968704866e-05, + "loss": 2.5113, + "theoretical_loss": 3.340959828147503, + "tokens_seen": 2774204416 + }, + { + "epoch": 0.68, + "learning_rate": 3.2177820574546626e-05, + "loss": 2.5454, + "theoretical_loss": 3.3409474981695095, + "tokens_seen": 2774335488 + }, + { + "epoch": 0.68, + "learning_rate": 3.2169796180388386e-05, + "loss": 2.7144, + "theoretical_loss": 3.3409351689371234, + "tokens_seen": 2774466560 + }, + { + "epoch": 0.68, + "learning_rate": 3.2161771786230146e-05, + "loss": 2.4494, + "theoretical_loss": 3.340922840450264, + "tokens_seen": 2774597632 + }, + { + "epoch": 0.68, + "learning_rate": 3.21537473920719e-05, + "loss": 2.4793, + "theoretical_loss": 3.3409105127088514, + "tokens_seen": 2774728704 + }, + { + "epoch": 0.68, + "learning_rate": 3.214572299791366e-05, + "loss": 2.5809, + "theoretical_loss": 3.3408981857128053, + "tokens_seen": 2774859776 + }, + { + "epoch": 0.68, + "learning_rate": 3.213769860375542e-05, + "loss": 2.5672, + "theoretical_loss": 3.3408858594620447, + "tokens_seen": 2774990848 + }, + { + "epoch": 0.68, + "learning_rate": 3.212967420959718e-05, + "loss": 2.5098, + "theoretical_loss": 3.3408735339564903, + "tokens_seen": 2775121920 + }, + { + "epoch": 0.68, + "learning_rate": 3.212164981543894e-05, + "loss": 2.4314, + "theoretical_loss": 3.3408612091960617, + "tokens_seen": 2775252992 + }, + { + "epoch": 0.68, + "learning_rate": 3.21136254212807e-05, + "loss": 2.5536, + "theoretical_loss": 3.340848885180678, + "tokens_seen": 2775384064 + }, + { + "epoch": 0.68, + "learning_rate": 3.210560102712246e-05, + "loss": 2.5052, + "theoretical_loss": 3.340836561910259, + "tokens_seen": 2775515136 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1521387, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.528700351715088, + "objective/train/theoretical_loss": 3.340830400554387, + "objective/train/tokens_used": 1146039776, + "theoretical_loss": 3.340830400554387, + "tokens_seen": 2775580672 + }, + { + "epoch": 0.68, + "learning_rate": 3.209757663296421e-05, + "loss": 2.3747, + "theoretical_loss": 3.340824239384726, + "tokens_seen": 2775646208 + }, + { + "epoch": 0.68, + "learning_rate": 3.208955223880597e-05, + "loss": 2.7068, + "theoretical_loss": 3.3408119176039963, + "tokens_seen": 2775777280 + }, + { + "epoch": 0.68, + "learning_rate": 3.208152784464773e-05, + "loss": 2.6546, + "theoretical_loss": 3.340799596567992, + "tokens_seen": 2775908352 + }, + { + "epoch": 0.68, + "learning_rate": 3.207350345048949e-05, + "loss": 2.5614, + "theoretical_loss": 3.3407872762766315, + "tokens_seen": 2776039424 + }, + { + "epoch": 0.68, + "learning_rate": 3.206547905633125e-05, + "loss": 2.5554, + "theoretical_loss": 3.3407749567298355, + "tokens_seen": 2776170496 + }, + { + "epoch": 0.68, + "learning_rate": 3.205745466217301e-05, + "loss": 2.6451, + "theoretical_loss": 3.3407626379275235, + "tokens_seen": 2776301568 + }, + { + "epoch": 0.68, + "learning_rate": 3.204943026801477e-05, + "loss": 2.471, + "theoretical_loss": 3.340750319869615, + "tokens_seen": 2776432640 + }, + { + "epoch": 0.68, + "learning_rate": 3.2041405873856524e-05, + "loss": 2.4504, + "theoretical_loss": 3.3407380025560305, + "tokens_seen": 2776563712 + }, + { + "epoch": 0.68, + "learning_rate": 3.2033381479698284e-05, + "loss": 2.6381, + "theoretical_loss": 3.3407256859866896, + "tokens_seen": 2776694784 + }, + { + "epoch": 0.68, + "learning_rate": 3.2025357085540044e-05, + "loss": 2.6021, + "theoretical_loss": 3.340713370161512, + "tokens_seen": 2776825856 + }, + { + "epoch": 0.68, + "learning_rate": 3.2017332691381804e-05, + "loss": 2.6289, + "theoretical_loss": 3.3407010550804186, + "tokens_seen": 2776956928 + }, + { + "epoch": 0.68, + "learning_rate": 3.2009308297223564e-05, + "loss": 2.5138, + "theoretical_loss": 3.340688740743328, + "tokens_seen": 2777088000 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1522596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.167062759399414, + "objective/train/theoretical_loss": 3.3406764271501608, + "objective/train/tokens_used": 1147678176, + "theoretical_loss": 3.3406764271501608, + "tokens_seen": 2777219072 + }, + { + "epoch": 0.68, + "learning_rate": 3.2001283903065323e-05, + "loss": 2.4633, + "theoretical_loss": 3.3406764271501608, + "tokens_seen": 2777219072 + }, + { + "epoch": 0.68, + "learning_rate": 3.199325950890708e-05, + "loss": 2.4218, + "theoretical_loss": 3.3406641143008367, + "tokens_seen": 2777350144 + }, + { + "epoch": 0.68, + "learning_rate": 3.198523511474884e-05, + "loss": 2.73, + "theoretical_loss": 3.3406518021952762, + "tokens_seen": 2777481216 + }, + { + "epoch": 0.68, + "learning_rate": 3.1977210720590596e-05, + "loss": 2.5067, + "theoretical_loss": 3.3406394908333987, + "tokens_seen": 2777612288 + }, + { + "epoch": 0.68, + "learning_rate": 3.1969186326432356e-05, + "loss": 2.7427, + "theoretical_loss": 3.340627180215124, + "tokens_seen": 2777743360 + }, + { + "epoch": 0.68, + "learning_rate": 3.1961161932274116e-05, + "loss": 2.5786, + "theoretical_loss": 3.3406148703403735, + "tokens_seen": 2777874432 + }, + { + "epoch": 0.68, + "learning_rate": 3.1953137538115876e-05, + "loss": 2.5493, + "theoretical_loss": 3.3406025612090655, + "tokens_seen": 2778005504 + }, + { + "epoch": 0.68, + "learning_rate": 3.1945113143957636e-05, + "loss": 2.4784, + "theoretical_loss": 3.340590252821121, + "tokens_seen": 2778136576 + }, + { + "epoch": 0.68, + "learning_rate": 3.1937088749799396e-05, + "loss": 2.6933, + "theoretical_loss": 3.3405779451764595, + "tokens_seen": 2778267648 + }, + { + "epoch": 0.68, + "learning_rate": 3.1929064355641156e-05, + "loss": 2.5031, + "theoretical_loss": 3.3405656382750015, + "tokens_seen": 2778398720 + }, + { + "epoch": 0.68, + "learning_rate": 3.192103996148291e-05, + "loss": 2.6326, + "theoretical_loss": 3.3405533321166674, + "tokens_seen": 2778529792 + }, + { + "epoch": 0.68, + "learning_rate": 3.191301556732467e-05, + "loss": 2.4469, + "theoretical_loss": 3.340541026701376, + "tokens_seen": 2778660864 + }, + { + "epoch": 0.68, + "learning_rate": 3.190499117316643e-05, + "loss": 2.6159, + "theoretical_loss": 3.3405287220290485, + "tokens_seen": 2778791936 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1523177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.411984443664551, + "objective/train/theoretical_loss": 3.3405225699714713, + "objective/train/tokens_used": 1149316576, + "theoretical_loss": 3.3405225699714713, + "tokens_seen": 2778857472 + }, + { + "epoch": 0.68, + "learning_rate": 3.189696677900819e-05, + "loss": 2.5797, + "theoretical_loss": 3.3405164180996048, + "tokens_seen": 2778923008 + }, + { + "epoch": 0.68, + "learning_rate": 3.188894238484995e-05, + "loss": 2.548, + "theoretical_loss": 3.3405041149129646, + "tokens_seen": 2779054080 + }, + { + "epoch": 0.68, + "learning_rate": 3.188091799069171e-05, + "loss": 2.5632, + "theoretical_loss": 3.3404918124690486, + "tokens_seen": 2779185152 + }, + { + "epoch": 0.68, + "learning_rate": 3.187289359653347e-05, + "loss": 2.5135, + "theoretical_loss": 3.3404795107677763, + "tokens_seen": 2779316224 + }, + { + "epoch": 0.68, + "learning_rate": 3.186486920237522e-05, + "loss": 2.4697, + "theoretical_loss": 3.340467209809068, + "tokens_seen": 2779447296 + }, + { + "epoch": 0.68, + "learning_rate": 3.185684480821698e-05, + "loss": 2.6371, + "theoretical_loss": 3.340454909592845, + "tokens_seen": 2779578368 + }, + { + "epoch": 0.68, + "learning_rate": 3.184882041405874e-05, + "loss": 2.6328, + "theoretical_loss": 3.340442610119026, + "tokens_seen": 2779709440 + }, + { + "epoch": 0.68, + "learning_rate": 3.18407960199005e-05, + "loss": 2.5303, + "theoretical_loss": 3.3404303113875313, + "tokens_seen": 2779840512 + }, + { + "epoch": 0.68, + "learning_rate": 3.183277162574226e-05, + "loss": 2.6089, + "theoretical_loss": 3.340418013398282, + "tokens_seen": 2779971584 + }, + { + "epoch": 0.68, + "learning_rate": 3.182474723158402e-05, + "loss": 2.5975, + "theoretical_loss": 3.3404057161511975, + "tokens_seen": 2780102656 + }, + { + "epoch": 0.69, + "learning_rate": 3.1816722837425774e-05, + "loss": 2.6039, + "theoretical_loss": 3.340393419646199, + "tokens_seen": 2780233728 + }, + { + "epoch": 0.69, + "learning_rate": 3.1808698443267534e-05, + "loss": 2.5849, + "theoretical_loss": 3.3403811238832053, + "tokens_seen": 2780364800 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1524216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.94921875, + "objective/train/theoretical_loss": 3.340368828862138, + "objective/train/tokens_used": 1150954976, + "theoretical_loss": 3.340368828862138, + "tokens_seen": 2780495872 + }, + { + "epoch": 0.69, + "learning_rate": 3.1800674049109294e-05, + "loss": 2.7077, + "theoretical_loss": 3.340368828862138, + "tokens_seen": 2780495872 + }, + { + "epoch": 0.69, + "learning_rate": 3.1792649654951054e-05, + "loss": 2.495, + "theoretical_loss": 3.3403565345829165, + "tokens_seen": 2780626944 + }, + { + "epoch": 0.69, + "learning_rate": 3.1784625260792814e-05, + "loss": 2.6486, + "theoretical_loss": 3.3403442410454613, + "tokens_seen": 2780758016 + }, + { + "epoch": 0.69, + "learning_rate": 3.1776600866634574e-05, + "loss": 2.5794, + "theoretical_loss": 3.340331948249693, + "tokens_seen": 2780889088 + }, + { + "epoch": 0.69, + "learning_rate": 3.176857647247633e-05, + "loss": 2.5194, + "theoretical_loss": 3.3403196561955317, + "tokens_seen": 2781020160 + }, + { + "epoch": 0.69, + "learning_rate": 3.176055207831809e-05, + "loss": 2.6748, + "theoretical_loss": 3.3403073648828974, + "tokens_seen": 2781151232 + }, + { + "epoch": 0.69, + "learning_rate": 3.175252768415985e-05, + "loss": 2.601, + "theoretical_loss": 3.340295074311711, + "tokens_seen": 2781282304 + }, + { + "epoch": 0.69, + "learning_rate": 3.1744503290001607e-05, + "loss": 2.5017, + "theoretical_loss": 3.340282784481892, + "tokens_seen": 2781413376 + }, + { + "epoch": 0.69, + "learning_rate": 3.1736478895843366e-05, + "loss": 2.5009, + "theoretical_loss": 3.3402704953933617, + "tokens_seen": 2781544448 + }, + { + "epoch": 0.69, + "learning_rate": 3.1728454501685126e-05, + "loss": 2.5676, + "theoretical_loss": 3.34025820704604, + "tokens_seen": 2781675520 + }, + { + "epoch": 0.69, + "learning_rate": 3.172043010752688e-05, + "loss": 2.4712, + "theoretical_loss": 3.3402459194398473, + "tokens_seen": 2781806592 + }, + { + "epoch": 0.69, + "learning_rate": 3.171240571336864e-05, + "loss": 2.523, + "theoretical_loss": 3.3402336325747037, + "tokens_seen": 2781937664 + }, + { + "epoch": 0.69, + "learning_rate": 3.17043813192104e-05, + "loss": 2.6011, + "theoretical_loss": 3.34022134645053, + "tokens_seen": 2782068736 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1524928, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2948381900787354, + "objective/train/theoretical_loss": 3.340215203666282, + "objective/train/tokens_used": 1152593376, + "theoretical_loss": 3.340215203666282, + "tokens_seen": 2782134272 + }, + { + "epoch": 0.69, + "learning_rate": 3.169635692505216e-05, + "loss": 2.6525, + "theoretical_loss": 3.340209061067247, + "tokens_seen": 2782199808 + }, + { + "epoch": 0.69, + "learning_rate": 3.168833253089392e-05, + "loss": 2.569, + "theoretical_loss": 3.340196776424774, + "tokens_seen": 2782330880 + }, + { + "epoch": 0.69, + "learning_rate": 3.168030813673568e-05, + "loss": 2.5758, + "theoretical_loss": 3.3401844925230324, + "tokens_seen": 2782461952 + }, + { + "epoch": 0.69, + "learning_rate": 3.167228374257743e-05, + "loss": 2.5313, + "theoretical_loss": 3.340172209361942, + "tokens_seen": 2782593024 + }, + { + "epoch": 0.69, + "learning_rate": 3.166425934841919e-05, + "loss": 2.5996, + "theoretical_loss": 3.3401599269414235, + "tokens_seen": 2782724096 + }, + { + "epoch": 0.69, + "learning_rate": 3.165623495426095e-05, + "loss": 2.5491, + "theoretical_loss": 3.340147645261398, + "tokens_seen": 2782855168 + }, + { + "epoch": 0.69, + "learning_rate": 3.164821056010271e-05, + "loss": 2.423, + "theoretical_loss": 3.340135364321785, + "tokens_seen": 2782986240 + }, + { + "epoch": 0.69, + "learning_rate": 3.164018616594447e-05, + "loss": 2.4026, + "theoretical_loss": 3.340123084122505, + "tokens_seen": 2783117312 + }, + { + "epoch": 0.69, + "learning_rate": 3.163216177178623e-05, + "loss": 2.5138, + "theoretical_loss": 3.3401108046634795, + "tokens_seen": 2783248384 + }, + { + "epoch": 0.69, + "learning_rate": 3.1624137377627985e-05, + "loss": 2.5064, + "theoretical_loss": 3.3400985259446285, + "tokens_seen": 2783379456 + }, + { + "epoch": 0.69, + "learning_rate": 3.1616112983469745e-05, + "loss": 2.6569, + "theoretical_loss": 3.340086247965872, + "tokens_seen": 2783510528 + }, + { + "epoch": 0.69, + "learning_rate": 3.1608088589311505e-05, + "loss": 2.4419, + "theoretical_loss": 3.3400739707271314, + "tokens_seen": 2783641600 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1526321, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.55534291267395, + "objective/train/theoretical_loss": 3.3400616942283268, + "objective/train/tokens_used": 1154231776, + "theoretical_loss": 3.3400616942283268, + "tokens_seen": 2783772672 + }, + { + "epoch": 0.69, + "learning_rate": 3.1600064195153265e-05, + "loss": 2.4114, + "theoretical_loss": 3.3400616942283268, + "tokens_seen": 2783772672 + }, + { + "epoch": 0.69, + "learning_rate": 3.1592039800995024e-05, + "loss": 2.5541, + "theoretical_loss": 3.3400494184693788, + "tokens_seen": 2783903744 + }, + { + "epoch": 0.69, + "learning_rate": 3.1584015406836784e-05, + "loss": 2.5932, + "theoretical_loss": 3.340037143450208, + "tokens_seen": 2784034816 + }, + { + "epoch": 0.69, + "learning_rate": 3.1575991012678544e-05, + "loss": 2.6955, + "theoretical_loss": 3.3400248691707346, + "tokens_seen": 2784165888 + }, + { + "epoch": 0.69, + "learning_rate": 3.15679666185203e-05, + "loss": 2.6207, + "theoretical_loss": 3.34001259563088, + "tokens_seen": 2784296960 + }, + { + "epoch": 0.69, + "learning_rate": 3.155994222436206e-05, + "loss": 2.5871, + "theoretical_loss": 3.340000322830564, + "tokens_seen": 2784428032 + }, + { + "epoch": 0.69, + "learning_rate": 3.155191783020382e-05, + "loss": 2.4799, + "theoretical_loss": 3.3399880507697084, + "tokens_seen": 2784559104 + }, + { + "epoch": 0.69, + "learning_rate": 3.154389343604558e-05, + "loss": 2.4971, + "theoretical_loss": 3.3399757794482325, + "tokens_seen": 2784690176 + }, + { + "epoch": 0.69, + "learning_rate": 3.153586904188734e-05, + "loss": 2.5771, + "theoretical_loss": 3.3399635088660578, + "tokens_seen": 2784821248 + }, + { + "epoch": 0.69, + "learning_rate": 3.15278446477291e-05, + "loss": 2.6527, + "theoretical_loss": 3.3399512390231045, + "tokens_seen": 2784952320 + }, + { + "epoch": 0.69, + "learning_rate": 3.151982025357086e-05, + "loss": 2.7049, + "theoretical_loss": 3.339938969919294, + "tokens_seen": 2785083392 + }, + { + "epoch": 0.69, + "learning_rate": 3.151179585941262e-05, + "loss": 2.4972, + "theoretical_loss": 3.3399267015545457, + "tokens_seen": 2785214464 + }, + { + "epoch": 0.69, + "learning_rate": 3.150377146525437e-05, + "loss": 2.5467, + "theoretical_loss": 3.3399144339287816, + "tokens_seen": 2785345536 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1526979, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.399793863296509, + "objective/train/theoretical_loss": 3.339908300392994, + "objective/train/tokens_used": 1155870176, + "theoretical_loss": 3.339908300392994, + "tokens_seen": 2785411072 + }, + { + "epoch": 0.69, + "learning_rate": 3.149574707109613e-05, + "loss": 2.3941, + "theoretical_loss": 3.339902167041922, + "tokens_seen": 2785476608 + }, + { + "epoch": 0.69, + "learning_rate": 3.148772267693789e-05, + "loss": 2.5574, + "theoretical_loss": 3.3398899008938874, + "tokens_seen": 2785607680 + }, + { + "epoch": 0.69, + "learning_rate": 3.147969828277965e-05, + "loss": 2.5154, + "theoretical_loss": 3.3398776354845987, + "tokens_seen": 2785738752 + }, + { + "epoch": 0.69, + "learning_rate": 3.147167388862141e-05, + "loss": 2.5914, + "theoretical_loss": 3.3398653708139765, + "tokens_seen": 2785869824 + }, + { + "epoch": 0.69, + "learning_rate": 3.146364949446317e-05, + "loss": 2.5472, + "theoretical_loss": 3.3398531068819417, + "tokens_seen": 2786000896 + }, + { + "epoch": 0.69, + "learning_rate": 3.145562510030493e-05, + "loss": 2.6109, + "theoretical_loss": 3.3398408436884153, + "tokens_seen": 2786131968 + }, + { + "epoch": 0.69, + "learning_rate": 3.144760070614668e-05, + "loss": 2.5574, + "theoretical_loss": 3.3398285812333173, + "tokens_seen": 2786263040 + }, + { + "epoch": 0.69, + "learning_rate": 3.143957631198844e-05, + "loss": 2.4479, + "theoretical_loss": 3.3398163195165695, + "tokens_seen": 2786394112 + }, + { + "epoch": 0.69, + "learning_rate": 3.14315519178302e-05, + "loss": 2.4651, + "theoretical_loss": 3.3398040585380926, + "tokens_seen": 2786525184 + }, + { + "epoch": 0.69, + "learning_rate": 3.142352752367196e-05, + "loss": 2.484, + "theoretical_loss": 3.3397917982978065, + "tokens_seen": 2786656256 + }, + { + "epoch": 0.69, + "learning_rate": 3.141550312951372e-05, + "loss": 2.5685, + "theoretical_loss": 3.339779538795633, + "tokens_seen": 2786787328 + }, + { + "epoch": 0.69, + "learning_rate": 3.140747873535548e-05, + "loss": 2.603, + "theoretical_loss": 3.3397672800314924, + "tokens_seen": 2786918400 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1528257, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.879302501678467, + "objective/train/theoretical_loss": 3.3397550220053054, + "objective/train/tokens_used": 1157508576, + "theoretical_loss": 3.3397550220053054, + "tokens_seen": 2787049472 + }, + { + "epoch": 0.69, + "learning_rate": 3.139945434119724e-05, + "loss": 2.6802, + "theoretical_loss": 3.3397550220053054, + "tokens_seen": 2787049472 + }, + { + "epoch": 0.69, + "learning_rate": 3.1391429947039e-05, + "loss": 2.5828, + "theoretical_loss": 3.339742764716994, + "tokens_seen": 2787180544 + }, + { + "epoch": 0.69, + "learning_rate": 3.1383405552880755e-05, + "loss": 2.5858, + "theoretical_loss": 3.3397305081664777, + "tokens_seen": 2787311616 + }, + { + "epoch": 0.69, + "learning_rate": 3.1375381158722515e-05, + "loss": 2.625, + "theoretical_loss": 3.3397182523536784, + "tokens_seen": 2787442688 + }, + { + "epoch": 0.69, + "learning_rate": 3.1367356764564275e-05, + "loss": 2.6322, + "theoretical_loss": 3.339705997278516, + "tokens_seen": 2787573760 + }, + { + "epoch": 0.69, + "learning_rate": 3.1359332370406035e-05, + "loss": 2.5538, + "theoretical_loss": 3.339693742940913, + "tokens_seen": 2787704832 + }, + { + "epoch": 0.69, + "learning_rate": 3.1351307976247795e-05, + "loss": 2.5159, + "theoretical_loss": 3.339681489340789, + "tokens_seen": 2787835904 + }, + { + "epoch": 0.69, + "learning_rate": 3.1343283582089554e-05, + "loss": 2.5059, + "theoretical_loss": 3.339669236478065, + "tokens_seen": 2787966976 + }, + { + "epoch": 0.69, + "learning_rate": 3.1335259187931314e-05, + "loss": 2.6706, + "theoretical_loss": 3.3396569843526622, + "tokens_seen": 2788098048 + }, + { + "epoch": 0.69, + "learning_rate": 3.132723479377307e-05, + "loss": 2.4964, + "theoretical_loss": 3.3396447329645023, + "tokens_seen": 2788229120 + }, + { + "epoch": 0.69, + "learning_rate": 3.131921039961483e-05, + "loss": 2.5161, + "theoretical_loss": 3.339632482313505, + "tokens_seen": 2788360192 + }, + { + "epoch": 0.69, + "learning_rate": 3.131118600545659e-05, + "loss": 2.4169, + "theoretical_loss": 3.339620232399592, + "tokens_seen": 2788491264 + }, + { + "epoch": 0.69, + "learning_rate": 3.130316161129835e-05, + "loss": 2.5836, + "theoretical_loss": 3.3396079832226846, + "tokens_seen": 2788622336 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1528798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.782625436782837, + "objective/train/theoretical_loss": 3.339601858910583, + "objective/train/tokens_used": 1159146976, + "theoretical_loss": 3.339601858910583, + "tokens_seen": 2788687872 + }, + { + "epoch": 0.69, + "learning_rate": 3.129513721714011e-05, + "loss": 2.5638, + "theoretical_loss": 3.339595734782703, + "tokens_seen": 2788753408 + }, + { + "epoch": 0.69, + "learning_rate": 3.128711282298187e-05, + "loss": 2.5055, + "theoretical_loss": 3.3395834870795698, + "tokens_seen": 2788884480 + }, + { + "epoch": 0.69, + "learning_rate": 3.127908842882363e-05, + "loss": 2.471, + "theoretical_loss": 3.339571240113204, + "tokens_seen": 2789015552 + }, + { + "epoch": 0.69, + "learning_rate": 3.127106403466538e-05, + "loss": 2.5459, + "theoretical_loss": 3.3395589938835277, + "tokens_seen": 2789146624 + }, + { + "epoch": 0.69, + "learning_rate": 3.126303964050714e-05, + "loss": 2.6837, + "theoretical_loss": 3.339546748390462, + "tokens_seen": 2789277696 + }, + { + "epoch": 0.69, + "learning_rate": 3.12550152463489e-05, + "loss": 2.4113, + "theoretical_loss": 3.339534503633928, + "tokens_seen": 2789408768 + }, + { + "epoch": 0.69, + "learning_rate": 3.124699085219066e-05, + "loss": 2.6231, + "theoretical_loss": 3.339522259613846, + "tokens_seen": 2789539840 + }, + { + "epoch": 0.69, + "learning_rate": 3.123896645803242e-05, + "loss": 2.3677, + "theoretical_loss": 3.3395100163301383, + "tokens_seen": 2789670912 + }, + { + "epoch": 0.69, + "learning_rate": 3.123094206387418e-05, + "loss": 2.5877, + "theoretical_loss": 3.3394977737827256, + "tokens_seen": 2789801984 + }, + { + "epoch": 0.69, + "learning_rate": 3.122291766971594e-05, + "loss": 2.4771, + "theoretical_loss": 3.3394855319715284, + "tokens_seen": 2789933056 + }, + { + "epoch": 0.69, + "learning_rate": 3.12148932755577e-05, + "loss": 2.4385, + "theoretical_loss": 3.3394732908964686, + "tokens_seen": 2790064128 + }, + { + "epoch": 0.69, + "learning_rate": 3.120686888139945e-05, + "loss": 2.6108, + "theoretical_loss": 3.3394610505574676, + "tokens_seen": 2790195200 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1530080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.245954751968384, + "objective/train/theoretical_loss": 3.3394488109544453, + "objective/train/tokens_used": 1160785376, + "theoretical_loss": 3.3394488109544453, + "tokens_seen": 2790326272 + }, + { + "epoch": 0.69, + "learning_rate": 3.119884448724121e-05, + "loss": 2.5196, + "theoretical_loss": 3.3394488109544453, + "tokens_seen": 2790326272 + }, + { + "epoch": 0.69, + "learning_rate": 3.119082009308297e-05, + "loss": 2.5565, + "theoretical_loss": 3.3394365720873243, + "tokens_seen": 2790457344 + }, + { + "epoch": 0.69, + "learning_rate": 3.118279569892473e-05, + "loss": 2.5119, + "theoretical_loss": 3.3394243339560243, + "tokens_seen": 2790588416 + }, + { + "epoch": 0.69, + "learning_rate": 3.117477130476649e-05, + "loss": 2.5716, + "theoretical_loss": 3.339412096560468, + "tokens_seen": 2790719488 + }, + { + "epoch": 0.69, + "learning_rate": 3.116674691060825e-05, + "loss": 2.4334, + "theoretical_loss": 3.339399859900576, + "tokens_seen": 2790850560 + }, + { + "epoch": 0.69, + "learning_rate": 3.115872251645001e-05, + "loss": 2.6583, + "theoretical_loss": 3.339387623976269, + "tokens_seen": 2790981632 + }, + { + "epoch": 0.69, + "learning_rate": 3.1150698122291765e-05, + "loss": 2.507, + "theoretical_loss": 3.3393753887874693, + "tokens_seen": 2791112704 + }, + { + "epoch": 0.69, + "learning_rate": 3.1142673728133525e-05, + "loss": 2.6068, + "theoretical_loss": 3.339363154334097, + "tokens_seen": 2791243776 + }, + { + "epoch": 0.69, + "learning_rate": 3.1134649333975285e-05, + "loss": 2.5607, + "theoretical_loss": 3.339350920616074, + "tokens_seen": 2791374848 + }, + { + "epoch": 0.69, + "learning_rate": 3.1126624939817045e-05, + "loss": 2.5854, + "theoretical_loss": 3.339338687633322, + "tokens_seen": 2791505920 + }, + { + "epoch": 0.69, + "learning_rate": 3.1118600545658805e-05, + "loss": 2.4616, + "theoretical_loss": 3.3393264553857613, + "tokens_seen": 2791636992 + }, + { + "epoch": 0.69, + "learning_rate": 3.1110576151500565e-05, + "loss": 2.6009, + "theoretical_loss": 3.339314223873314, + "tokens_seen": 2791768064 + }, + { + "epoch": 0.69, + "learning_rate": 3.1102551757342325e-05, + "loss": 2.6568, + "theoretical_loss": 3.339301993095901, + "tokens_seen": 2791899136 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1530718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.394282341003418, + "objective/train/theoretical_loss": 3.3392958779828077, + "objective/train/tokens_used": 1162423776, + "theoretical_loss": 3.3392958779828077, + "tokens_seen": 2791964672 + }, + { + "epoch": 0.69, + "learning_rate": 3.109452736318408e-05, + "loss": 2.3955, + "theoretical_loss": 3.339289763053444, + "tokens_seen": 2792030208 + }, + { + "epoch": 0.69, + "learning_rate": 3.108650296902584e-05, + "loss": 2.4857, + "theoretical_loss": 3.3392775337458636, + "tokens_seen": 2792161280 + }, + { + "epoch": 0.69, + "learning_rate": 3.10784785748676e-05, + "loss": 2.5342, + "theoretical_loss": 3.3392653051730816, + "tokens_seen": 2792292352 + }, + { + "epoch": 0.69, + "learning_rate": 3.107045418070936e-05, + "loss": 2.6158, + "theoretical_loss": 3.33925307733502, + "tokens_seen": 2792423424 + }, + { + "epoch": 0.69, + "learning_rate": 3.106242978655112e-05, + "loss": 2.4196, + "theoretical_loss": 3.339240850231599, + "tokens_seen": 2792554496 + }, + { + "epoch": 0.69, + "learning_rate": 3.105440539239288e-05, + "loss": 2.4217, + "theoretical_loss": 3.339228623862741, + "tokens_seen": 2792685568 + }, + { + "epoch": 0.69, + "learning_rate": 3.104638099823464e-05, + "loss": 2.4502, + "theoretical_loss": 3.3392163982283667, + "tokens_seen": 2792816640 + }, + { + "epoch": 0.69, + "learning_rate": 3.10383566040764e-05, + "loss": 2.7538, + "theoretical_loss": 3.3392041733283975, + "tokens_seen": 2792947712 + }, + { + "epoch": 0.69, + "learning_rate": 3.103033220991815e-05, + "loss": 2.4247, + "theoretical_loss": 3.3391919491627555, + "tokens_seen": 2793078784 + }, + { + "epoch": 0.69, + "learning_rate": 3.102230781575991e-05, + "loss": 2.6041, + "theoretical_loss": 3.3391797257313613, + "tokens_seen": 2793209856 + }, + { + "epoch": 0.69, + "learning_rate": 3.101428342160167e-05, + "loss": 2.3878, + "theoretical_loss": 3.3391675030341372, + "tokens_seen": 2793340928 + }, + { + "epoch": 0.69, + "learning_rate": 3.100625902744343e-05, + "loss": 2.4185, + "theoretical_loss": 3.3391552810710037, + "tokens_seen": 2793472000 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1531819, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5151286125183105, + "objective/train/theoretical_loss": 3.3391430598418834, + "objective/train/tokens_used": 1164062176, + "theoretical_loss": 3.3391430598418834, + "tokens_seen": 2793603072 + }, + { + "epoch": 0.69, + "learning_rate": 3.099823463328519e-05, + "loss": 2.5516, + "theoretical_loss": 3.3391430598418834, + "tokens_seen": 2793603072 + }, + { + "epoch": 0.69, + "learning_rate": 3.099021023912695e-05, + "loss": 2.4662, + "theoretical_loss": 3.3391308393466965, + "tokens_seen": 2793734144 + }, + { + "epoch": 0.69, + "learning_rate": 3.098218584496871e-05, + "loss": 2.4532, + "theoretical_loss": 3.3391186195853657, + "tokens_seen": 2793865216 + }, + { + "epoch": 0.69, + "learning_rate": 3.097416145081046e-05, + "loss": 2.2864, + "theoretical_loss": 3.3391064005578115, + "tokens_seen": 2793996288 + }, + { + "epoch": 0.69, + "learning_rate": 3.096613705665222e-05, + "loss": 2.5493, + "theoretical_loss": 3.339094182263956, + "tokens_seen": 2794127360 + }, + { + "epoch": 0.69, + "learning_rate": 3.095811266249398e-05, + "loss": 2.5273, + "theoretical_loss": 3.339081964703721, + "tokens_seen": 2794258432 + }, + { + "epoch": 0.69, + "learning_rate": 3.095008826833574e-05, + "loss": 2.5587, + "theoretical_loss": 3.3390697478770273, + "tokens_seen": 2794389504 + }, + { + "epoch": 0.69, + "learning_rate": 3.09420638741775e-05, + "loss": 2.4987, + "theoretical_loss": 3.339057531783797, + "tokens_seen": 2794520576 + }, + { + "epoch": 0.69, + "learning_rate": 3.093403948001926e-05, + "loss": 2.746, + "theoretical_loss": 3.339045316423951, + "tokens_seen": 2794651648 + }, + { + "epoch": 0.69, + "learning_rate": 3.092601508586102e-05, + "loss": 2.4318, + "theoretical_loss": 3.339033101797412, + "tokens_seen": 2794782720 + }, + { + "epoch": 0.69, + "learning_rate": 3.091799069170278e-05, + "loss": 2.4048, + "theoretical_loss": 3.3390208879041006, + "tokens_seen": 2794913792 + }, + { + "epoch": 0.69, + "learning_rate": 3.0909966297544535e-05, + "loss": 2.5786, + "theoretical_loss": 3.3390086747439387, + "tokens_seen": 2795044864 + }, + { + "epoch": 0.69, + "learning_rate": 3.0901941903386295e-05, + "loss": 2.506, + "theoretical_loss": 3.3389964623168478, + "tokens_seen": 2795175936 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1532927, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7780046463012695, + "objective/train/theoretical_loss": 3.3389903563781798, + "objective/train/tokens_used": 1165700576, + "theoretical_loss": 3.3389903563781798, + "tokens_seen": 2795241472 + }, + { + "epoch": 0.69, + "learning_rate": 3.0893917509228055e-05, + "loss": 2.5247, + "theoretical_loss": 3.33898425062275, + "tokens_seen": 2795307008 + }, + { + "epoch": 0.69, + "learning_rate": 3.0885893115069815e-05, + "loss": 2.7444, + "theoretical_loss": 3.3389720396615665, + "tokens_seen": 2795438080 + }, + { + "epoch": 0.69, + "learning_rate": 3.0877868720911575e-05, + "loss": 2.4664, + "theoretical_loss": 3.338959829433219, + "tokens_seen": 2795569152 + }, + { + "epoch": 0.69, + "learning_rate": 3.0869844326753335e-05, + "loss": 2.4605, + "theoretical_loss": 3.3389476199376293, + "tokens_seen": 2795700224 + }, + { + "epoch": 0.69, + "learning_rate": 3.0861819932595095e-05, + "loss": 2.4172, + "theoretical_loss": 3.3389354111747194, + "tokens_seen": 2795831296 + }, + { + "epoch": 0.69, + "learning_rate": 3.085379553843685e-05, + "loss": 2.5662, + "theoretical_loss": 3.33892320314441, + "tokens_seen": 2795962368 + }, + { + "epoch": 0.69, + "learning_rate": 3.084577114427861e-05, + "loss": 2.5004, + "theoretical_loss": 3.3389109958466237, + "tokens_seen": 2796093440 + }, + { + "epoch": 0.69, + "learning_rate": 3.083774675012037e-05, + "loss": 2.5391, + "theoretical_loss": 3.3388987892812816, + "tokens_seen": 2796224512 + }, + { + "epoch": 0.69, + "learning_rate": 3.082972235596213e-05, + "loss": 2.3743, + "theoretical_loss": 3.338886583448306, + "tokens_seen": 2796355584 + }, + { + "epoch": 0.69, + "learning_rate": 3.082169796180389e-05, + "loss": 2.5168, + "theoretical_loss": 3.3388743783476182, + "tokens_seen": 2796486656 + }, + { + "epoch": 0.69, + "learning_rate": 3.081367356764565e-05, + "loss": 2.3847, + "theoretical_loss": 3.33886217397914, + "tokens_seen": 2796617728 + }, + { + "epoch": 0.7, + "learning_rate": 3.080564917348741e-05, + "loss": 2.3921, + "theoretical_loss": 3.338849970342794, + "tokens_seen": 2796748800 + }, + { + "debugging/Self-BLEU-5": 0.4352260950139857, + "debugging/distinct-1-grams": 0.795556155839227, + "debugging/distinct-2-grams": 0.9594928292100138, + "debugging/entropy-1-grams": 5.546456218535589, + "debugging/entropy-2-grams": 6.296094978862367, + "debugging/length": 466.2, + "debugging/num_segments": 10, + "debugging/score": 0.0046013380158723746, + "debugging/score_std": 0.005243219871494746, + "epoch": 0.7, + "objective/train/docs_used": 1533490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.615194082260132, + "objective/train/theoretical_loss": 3.3388377674385006, + "objective/train/tokens_used": 1167338976, + "theoretical_loss": 3.3388377674385006, + "tokens_seen": 2796879872 + }, + { + "epoch": 0.7, + "learning_rate": 3.079762477932916e-05, + "loss": 2.6006, + "theoretical_loss": 3.3388377674385006, + "tokens_seen": 2796879872 + }, + { + "epoch": 0.7, + "learning_rate": 3.078960038517092e-05, + "loss": 2.5173, + "theoretical_loss": 3.3388255652661822, + "tokens_seen": 2797010944 + }, + { + "epoch": 0.7, + "learning_rate": 3.078157599101268e-05, + "loss": 2.5252, + "theoretical_loss": 3.3388133638257607, + "tokens_seen": 2797142016 + }, + { + "epoch": 0.7, + "learning_rate": 3.077355159685444e-05, + "loss": 2.5496, + "theoretical_loss": 3.338801163117158, + "tokens_seen": 2797273088 + }, + { + "epoch": 0.7, + "learning_rate": 3.07655272026962e-05, + "loss": 2.6301, + "theoretical_loss": 3.3387889631402956, + "tokens_seen": 2797404160 + }, + { + "epoch": 0.7, + "learning_rate": 3.075750280853796e-05, + "loss": 2.548, + "theoretical_loss": 3.338776763895096, + "tokens_seen": 2797535232 + }, + { + "epoch": 0.7, + "learning_rate": 3.074947841437972e-05, + "loss": 2.6746, + "theoretical_loss": 3.3387645653814797, + "tokens_seen": 2797666304 + }, + { + "epoch": 0.7, + "learning_rate": 3.074145402022148e-05, + "loss": 2.5866, + "theoretical_loss": 3.3387523675993696, + "tokens_seen": 2797797376 + }, + { + "epoch": 0.7, + "learning_rate": 3.073342962606323e-05, + "loss": 2.6281, + "theoretical_loss": 3.3387401705486877, + "tokens_seen": 2797928448 + }, + { + "epoch": 0.7, + "learning_rate": 3.072540523190499e-05, + "loss": 2.5841, + "theoretical_loss": 3.338727974229355, + "tokens_seen": 2798059520 + }, + { + "epoch": 0.7, + "learning_rate": 3.071738083774675e-05, + "loss": 2.4539, + "theoretical_loss": 3.338715778641294, + "tokens_seen": 2798190592 + }, + { + "epoch": 0.7, + "learning_rate": 3.070935644358851e-05, + "loss": 2.4044, + "theoretical_loss": 3.3387035837844268, + "tokens_seen": 2798321664 + }, + { + "epoch": 0.7, + "learning_rate": 3.070133204943027e-05, + "loss": 2.4808, + "theoretical_loss": 3.338691389658675, + "tokens_seen": 2798452736 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1534684, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4507222175598145, + "objective/train/theoretical_loss": 3.3386852928699433, + "objective/train/tokens_used": 1168977376, + "theoretical_loss": 3.3386852928699433, + "tokens_seen": 2798518272 + }, + { + "epoch": 0.7, + "learning_rate": 3.069330765527203e-05, + "loss": 2.5493, + "theoretical_loss": 3.3386791962639606, + "tokens_seen": 2798583808 + }, + { + "epoch": 0.7, + "learning_rate": 3.068528326111379e-05, + "loss": 2.6466, + "theoretical_loss": 3.3386670036002055, + "tokens_seen": 2798714880 + }, + { + "epoch": 0.7, + "learning_rate": 3.0677258866955545e-05, + "loss": 2.5329, + "theoretical_loss": 3.3386548116673316, + "tokens_seen": 2798845952 + }, + { + "epoch": 0.7, + "learning_rate": 3.0669234472797305e-05, + "loss": 2.4976, + "theoretical_loss": 3.3386426204652606, + "tokens_seen": 2798977024 + }, + { + "epoch": 0.7, + "learning_rate": 3.0661210078639065e-05, + "loss": 2.5084, + "theoretical_loss": 3.338630429993915, + "tokens_seen": 2799108096 + }, + { + "epoch": 0.7, + "learning_rate": 3.0653185684480825e-05, + "loss": 2.5853, + "theoretical_loss": 3.3386182402532167, + "tokens_seen": 2799239168 + }, + { + "epoch": 0.7, + "learning_rate": 3.0645161290322585e-05, + "loss": 2.5732, + "theoretical_loss": 3.3386060512430875, + "tokens_seen": 2799370240 + }, + { + "epoch": 0.7, + "learning_rate": 3.0637136896164345e-05, + "loss": 2.6384, + "theoretical_loss": 3.3385938629634495, + "tokens_seen": 2799501312 + }, + { + "epoch": 0.7, + "learning_rate": 3.0629112502006105e-05, + "loss": 2.5419, + "theoretical_loss": 3.3385816754142246, + "tokens_seen": 2799632384 + }, + { + "epoch": 0.7, + "learning_rate": 3.062108810784786e-05, + "loss": 2.3459, + "theoretical_loss": 3.338569488595335, + "tokens_seen": 2799763456 + }, + { + "epoch": 0.7, + "learning_rate": 3.061306371368962e-05, + "loss": 2.4346, + "theoretical_loss": 3.3385573025067026, + "tokens_seen": 2799894528 + }, + { + "epoch": 0.7, + "learning_rate": 3.060503931953138e-05, + "loss": 2.4697, + "theoretical_loss": 3.3385451171482496, + "tokens_seen": 2800025600 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1535069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.967625379562378, + "objective/train/theoretical_loss": 3.338532932519898, + "objective/train/tokens_used": 1170615776, + "theoretical_loss": 3.338532932519898, + "tokens_seen": 2800156672 + }, + { + "epoch": 0.7, + "learning_rate": 3.059701492537314e-05, + "loss": 2.4706, + "theoretical_loss": 3.338532932519898, + "tokens_seen": 2800156672 + }, + { + "epoch": 0.7, + "learning_rate": 3.05889905312149e-05, + "loss": 2.6365, + "theoretical_loss": 3.33852074862157, + "tokens_seen": 2800287744 + }, + { + "epoch": 0.7, + "learning_rate": 3.058096613705666e-05, + "loss": 2.5331, + "theoretical_loss": 3.338508565453188, + "tokens_seen": 2800418816 + }, + { + "epoch": 0.7, + "learning_rate": 3.057294174289842e-05, + "loss": 2.4858, + "theoretical_loss": 3.338496383014673, + "tokens_seen": 2800549888 + }, + { + "epoch": 0.7, + "learning_rate": 3.056491734874018e-05, + "loss": 2.5371, + "theoretical_loss": 3.338484201305948, + "tokens_seen": 2800680960 + }, + { + "epoch": 0.7, + "learning_rate": 3.055689295458193e-05, + "loss": 2.5871, + "theoretical_loss": 3.338472020326935, + "tokens_seen": 2800812032 + }, + { + "epoch": 0.7, + "learning_rate": 3.054886856042369e-05, + "loss": 2.5273, + "theoretical_loss": 3.3384598400775563, + "tokens_seen": 2800943104 + }, + { + "epoch": 0.7, + "learning_rate": 3.054084416626545e-05, + "loss": 2.5335, + "theoretical_loss": 3.3384476605577333, + "tokens_seen": 2801074176 + }, + { + "epoch": 0.7, + "learning_rate": 3.053281977210721e-05, + "loss": 2.6222, + "theoretical_loss": 3.338435481767389, + "tokens_seen": 2801205248 + }, + { + "epoch": 0.7, + "learning_rate": 3.052479537794897e-05, + "loss": 2.4284, + "theoretical_loss": 3.3384233037064455, + "tokens_seen": 2801336320 + }, + { + "epoch": 0.7, + "learning_rate": 3.0516770983790727e-05, + "loss": 2.5942, + "theoretical_loss": 3.338411126374825, + "tokens_seen": 2801467392 + }, + { + "epoch": 0.7, + "learning_rate": 3.0508746589632486e-05, + "loss": 2.5676, + "theoretical_loss": 3.3383989497724493, + "tokens_seen": 2801598464 + }, + { + "epoch": 0.7, + "learning_rate": 3.050072219547424e-05, + "loss": 2.4867, + "theoretical_loss": 3.3383867738992405, + "tokens_seen": 2801729536 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1536202, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6087231636047363, + "objective/train/theoretical_loss": 3.3383806862360497, + "objective/train/tokens_used": 1172254176, + "theoretical_loss": 3.3383806862360497, + "tokens_seen": 2801795072 + }, + { + "epoch": 0.7, + "learning_rate": 3.0492697801316e-05, + "loss": 2.5052, + "theoretical_loss": 3.3383745987551214, + "tokens_seen": 2801860608 + }, + { + "epoch": 0.7, + "learning_rate": 3.048467340715776e-05, + "loss": 2.5109, + "theoretical_loss": 3.338362424340014, + "tokens_seen": 2801991680 + }, + { + "epoch": 0.7, + "learning_rate": 3.047664901299952e-05, + "loss": 2.5483, + "theoretical_loss": 3.33835025065384, + "tokens_seen": 2802122752 + }, + { + "epoch": 0.7, + "learning_rate": 3.046862461884128e-05, + "loss": 2.5764, + "theoretical_loss": 3.338338077696523, + "tokens_seen": 2802253824 + }, + { + "epoch": 0.7, + "learning_rate": 3.046060022468304e-05, + "loss": 2.5046, + "theoretical_loss": 3.338325905467984, + "tokens_seen": 2802384896 + }, + { + "epoch": 0.7, + "learning_rate": 3.04525758305248e-05, + "loss": 2.6157, + "theoretical_loss": 3.3383137339681457, + "tokens_seen": 2802515968 + }, + { + "epoch": 0.7, + "learning_rate": 3.044455143636656e-05, + "loss": 2.4584, + "theoretical_loss": 3.3383015631969304, + "tokens_seen": 2802647040 + }, + { + "epoch": 0.7, + "learning_rate": 3.0436527042208312e-05, + "loss": 2.5256, + "theoretical_loss": 3.3382893931542603, + "tokens_seen": 2802778112 + }, + { + "epoch": 0.7, + "learning_rate": 3.0428502648050072e-05, + "loss": 2.4961, + "theoretical_loss": 3.338277223840058, + "tokens_seen": 2802909184 + }, + { + "epoch": 0.7, + "learning_rate": 3.0420478253891832e-05, + "loss": 2.4958, + "theoretical_loss": 3.3382650552542463, + "tokens_seen": 2803040256 + }, + { + "epoch": 0.7, + "learning_rate": 3.0412453859733592e-05, + "loss": 2.6271, + "theoretical_loss": 3.338252887396746, + "tokens_seen": 2803171328 + }, + { + "epoch": 0.7, + "learning_rate": 3.040442946557535e-05, + "loss": 2.5715, + "theoretical_loss": 3.338240720267481, + "tokens_seen": 2803302400 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1537011, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.325948476791382, + "objective/train/theoretical_loss": 3.338228553866373, + "objective/train/tokens_used": 1173892576, + "theoretical_loss": 3.338228553866373, + "tokens_seen": 2803433472 + }, + { + "epoch": 0.7, + "learning_rate": 3.039640507141711e-05, + "loss": 2.5399, + "theoretical_loss": 3.338228553866373, + "tokens_seen": 2803433472 + }, + { + "epoch": 0.7, + "learning_rate": 3.038838067725887e-05, + "loss": 2.3867, + "theoretical_loss": 3.3382163881933447, + "tokens_seen": 2803564544 + }, + { + "epoch": 0.7, + "learning_rate": 3.0380356283100625e-05, + "loss": 2.5344, + "theoretical_loss": 3.338204223248318, + "tokens_seen": 2803695616 + }, + { + "epoch": 0.7, + "learning_rate": 3.0372331888942384e-05, + "loss": 2.5792, + "theoretical_loss": 3.3381920590312153, + "tokens_seen": 2803826688 + }, + { + "epoch": 0.7, + "learning_rate": 3.0364307494784144e-05, + "loss": 2.6544, + "theoretical_loss": 3.3381798955419595, + "tokens_seen": 2803957760 + }, + { + "epoch": 0.7, + "learning_rate": 3.0356283100625904e-05, + "loss": 2.5496, + "theoretical_loss": 3.3381677327804726, + "tokens_seen": 2804088832 + }, + { + "epoch": 0.7, + "learning_rate": 3.0348258706467664e-05, + "loss": 2.5003, + "theoretical_loss": 3.3381555707466775, + "tokens_seen": 2804219904 + }, + { + "epoch": 0.7, + "learning_rate": 3.0340234312309424e-05, + "loss": 2.4555, + "theoretical_loss": 3.3381434094404963, + "tokens_seen": 2804350976 + }, + { + "epoch": 0.7, + "learning_rate": 3.0332209918151184e-05, + "loss": 2.6091, + "theoretical_loss": 3.3381312488618518, + "tokens_seen": 2804482048 + }, + { + "epoch": 0.7, + "learning_rate": 3.0324185523992937e-05, + "loss": 2.5544, + "theoretical_loss": 3.338119089010666, + "tokens_seen": 2804613120 + }, + { + "epoch": 0.7, + "learning_rate": 3.0316161129834697e-05, + "loss": 2.6375, + "theoretical_loss": 3.3381069298868615, + "tokens_seen": 2804744192 + }, + { + "epoch": 0.7, + "learning_rate": 3.0308136735676457e-05, + "loss": 2.5532, + "theoretical_loss": 3.3380947714903613, + "tokens_seen": 2804875264 + }, + { + "epoch": 0.7, + "learning_rate": 3.0300112341518217e-05, + "loss": 2.5765, + "theoretical_loss": 3.3380826138210873, + "tokens_seen": 2805006336 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1538239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.309110403060913, + "objective/train/theoretical_loss": 3.3380765352591357, + "objective/train/tokens_used": 1175530976, + "theoretical_loss": 3.3380765352591357, + "tokens_seen": 2805071872 + }, + { + "epoch": 0.7, + "learning_rate": 3.0292087947359977e-05, + "loss": 2.5106, + "theoretical_loss": 3.3380704568789623, + "tokens_seen": 2805137408 + }, + { + "epoch": 0.7, + "learning_rate": 3.0284063553201737e-05, + "loss": 2.6774, + "theoretical_loss": 3.3380583006639086, + "tokens_seen": 2805268480 + }, + { + "epoch": 0.7, + "learning_rate": 3.0276039159043497e-05, + "loss": 2.5012, + "theoretical_loss": 3.338046145175849, + "tokens_seen": 2805399552 + }, + { + "epoch": 0.7, + "learning_rate": 3.0268014764885256e-05, + "loss": 2.4883, + "theoretical_loss": 3.338033990414706, + "tokens_seen": 2805530624 + }, + { + "epoch": 0.7, + "learning_rate": 3.025999037072701e-05, + "loss": 2.4929, + "theoretical_loss": 3.338021836380402, + "tokens_seen": 2805661696 + }, + { + "epoch": 0.7, + "learning_rate": 3.025196597656877e-05, + "loss": 2.5377, + "theoretical_loss": 3.33800968307286, + "tokens_seen": 2805792768 + }, + { + "epoch": 0.7, + "learning_rate": 3.024394158241053e-05, + "loss": 2.4495, + "theoretical_loss": 3.337997530492003, + "tokens_seen": 2805923840 + }, + { + "epoch": 0.7, + "learning_rate": 3.023591718825229e-05, + "loss": 2.4912, + "theoretical_loss": 3.337985378637752, + "tokens_seen": 2806054912 + }, + { + "epoch": 0.7, + "learning_rate": 3.022789279409405e-05, + "loss": 2.5285, + "theoretical_loss": 3.337973227510031, + "tokens_seen": 2806185984 + }, + { + "epoch": 0.7, + "learning_rate": 3.021986839993581e-05, + "loss": 2.4353, + "theoretical_loss": 3.337961077108762, + "tokens_seen": 2806317056 + }, + { + "epoch": 0.7, + "learning_rate": 3.021184400577757e-05, + "loss": 2.4247, + "theoretical_loss": 3.337948927433868, + "tokens_seen": 2806448128 + }, + { + "epoch": 0.7, + "learning_rate": 3.0203819611619322e-05, + "loss": 2.4742, + "theoretical_loss": 3.3379367784852714, + "tokens_seen": 2806579200 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1538833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5450992584228516, + "objective/train/theoretical_loss": 3.3379246302628953, + "objective/train/tokens_used": 1177169376, + "theoretical_loss": 3.3379246302628953, + "tokens_seen": 2806710272 + }, + { + "epoch": 0.7, + "learning_rate": 3.0195795217461082e-05, + "loss": 2.5642, + "theoretical_loss": 3.3379246302628953, + "tokens_seen": 2806710272 + }, + { + "epoch": 0.7, + "learning_rate": 3.0187770823302842e-05, + "loss": 2.4604, + "theoretical_loss": 3.3379124827666615, + "tokens_seen": 2806841344 + }, + { + "epoch": 0.7, + "learning_rate": 3.0179746429144602e-05, + "loss": 2.5778, + "theoretical_loss": 3.3379003359964936, + "tokens_seen": 2806972416 + }, + { + "epoch": 0.7, + "learning_rate": 3.0171722034986362e-05, + "loss": 2.6396, + "theoretical_loss": 3.337888189952314, + "tokens_seen": 2807103488 + }, + { + "epoch": 0.7, + "learning_rate": 3.0163697640828122e-05, + "loss": 2.6705, + "theoretical_loss": 3.337876044634045, + "tokens_seen": 2807234560 + }, + { + "epoch": 0.7, + "learning_rate": 3.0155673246669878e-05, + "loss": 2.62, + "theoretical_loss": 3.33786390004161, + "tokens_seen": 2807365632 + }, + { + "epoch": 0.7, + "learning_rate": 3.0147648852511635e-05, + "loss": 2.5888, + "theoretical_loss": 3.3378517561749312, + "tokens_seen": 2807496704 + }, + { + "epoch": 0.7, + "learning_rate": 3.0139624458353395e-05, + "loss": 2.5649, + "theoretical_loss": 3.337839613033932, + "tokens_seen": 2807627776 + }, + { + "epoch": 0.7, + "learning_rate": 3.0131600064195155e-05, + "loss": 2.4295, + "theoretical_loss": 3.3378274706185342, + "tokens_seen": 2807758848 + }, + { + "epoch": 0.7, + "learning_rate": 3.0123575670036914e-05, + "loss": 2.6212, + "theoretical_loss": 3.3378153289286616, + "tokens_seen": 2807889920 + }, + { + "epoch": 0.7, + "learning_rate": 3.0115551275878674e-05, + "loss": 2.4942, + "theoretical_loss": 3.3378031879642363, + "tokens_seen": 2808020992 + }, + { + "epoch": 0.7, + "learning_rate": 3.010752688172043e-05, + "loss": 2.3465, + "theoretical_loss": 3.337791047725181, + "tokens_seen": 2808152064 + }, + { + "epoch": 0.7, + "learning_rate": 3.009950248756219e-05, + "loss": 2.3874, + "theoretical_loss": 3.337778908211419, + "tokens_seen": 2808283136 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1540092, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3799989223480225, + "objective/train/theoretical_loss": 3.3377728387264987, + "objective/train/tokens_used": 1178807776, + "theoretical_loss": 3.3377728387264987, + "tokens_seen": 2808348672 + }, + { + "epoch": 0.7, + "learning_rate": 3.009147809340395e-05, + "loss": 2.4335, + "theoretical_loss": 3.3377667694228728, + "tokens_seen": 2808414208 + }, + { + "epoch": 0.7, + "learning_rate": 3.0083453699245707e-05, + "loss": 2.5369, + "theoretical_loss": 3.3377546313594655, + "tokens_seen": 2808545280 + }, + { + "epoch": 0.7, + "learning_rate": 3.0075429305087467e-05, + "loss": 2.557, + "theoretical_loss": 3.3377424940211196, + "tokens_seen": 2808676352 + }, + { + "epoch": 0.7, + "learning_rate": 3.0067404910929227e-05, + "loss": 2.5052, + "theoretical_loss": 3.337730357407758, + "tokens_seen": 2808807424 + }, + { + "epoch": 0.7, + "learning_rate": 3.0059380516770984e-05, + "loss": 2.5175, + "theoretical_loss": 3.3377182215193044, + "tokens_seen": 2808938496 + }, + { + "epoch": 0.7, + "learning_rate": 3.0051356122612743e-05, + "loss": 2.6032, + "theoretical_loss": 3.3377060863556807, + "tokens_seen": 2809069568 + }, + { + "epoch": 0.7, + "learning_rate": 3.0043331728454503e-05, + "loss": 2.5068, + "theoretical_loss": 3.3376939519168096, + "tokens_seen": 2809200640 + }, + { + "epoch": 0.7, + "learning_rate": 3.0035307334296263e-05, + "loss": 2.4617, + "theoretical_loss": 3.337681818202615, + "tokens_seen": 2809331712 + }, + { + "epoch": 0.7, + "learning_rate": 3.002728294013802e-05, + "loss": 2.5549, + "theoretical_loss": 3.337669685213019, + "tokens_seen": 2809462784 + }, + { + "epoch": 0.7, + "learning_rate": 3.001925854597978e-05, + "loss": 2.6509, + "theoretical_loss": 3.3376575529479453, + "tokens_seen": 2809593856 + }, + { + "epoch": 0.7, + "learning_rate": 3.0011234151821536e-05, + "loss": 2.5383, + "theoretical_loss": 3.3376454214073155, + "tokens_seen": 2809724928 + }, + { + "epoch": 0.7, + "learning_rate": 3.0003209757663296e-05, + "loss": 2.5261, + "theoretical_loss": 3.337633290591054, + "tokens_seen": 2809856000 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1540729, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.77140212059021, + "objective/train/theoretical_loss": 3.3376211604990833, + "objective/train/tokens_used": 1180446176, + "theoretical_loss": 3.3376211604990833, + "tokens_seen": 2809987072 + }, + { + "epoch": 0.7, + "learning_rate": 2.9995185363505056e-05, + "loss": 2.6624, + "theoretical_loss": 3.3376211604990833, + "tokens_seen": 2809987072 + }, + { + "epoch": 0.7, + "learning_rate": 2.9987160969346816e-05, + "loss": 2.386, + "theoretical_loss": 3.337609031131326, + "tokens_seen": 2810118144 + }, + { + "epoch": 0.7, + "learning_rate": 2.9979136575188576e-05, + "loss": 2.6401, + "theoretical_loss": 3.3375969024877055, + "tokens_seen": 2810249216 + }, + { + "epoch": 0.7, + "learning_rate": 2.9971112181030336e-05, + "loss": 2.4412, + "theoretical_loss": 3.3375847745681444, + "tokens_seen": 2810380288 + }, + { + "epoch": 0.7, + "learning_rate": 2.996308778687209e-05, + "loss": 2.5676, + "theoretical_loss": 3.337572647372566, + "tokens_seen": 2810511360 + }, + { + "epoch": 0.7, + "learning_rate": 2.995506339271385e-05, + "loss": 2.4744, + "theoretical_loss": 3.337560520900893, + "tokens_seen": 2810642432 + }, + { + "epoch": 0.7, + "learning_rate": 2.994703899855561e-05, + "loss": 2.586, + "theoretical_loss": 3.337548395153049, + "tokens_seen": 2810773504 + }, + { + "epoch": 0.7, + "learning_rate": 2.993901460439737e-05, + "loss": 2.5455, + "theoretical_loss": 3.3375362701289566, + "tokens_seen": 2810904576 + }, + { + "epoch": 0.7, + "learning_rate": 2.993099021023913e-05, + "loss": 2.4537, + "theoretical_loss": 3.337524145828539, + "tokens_seen": 2811035648 + }, + { + "epoch": 0.7, + "learning_rate": 2.992296581608089e-05, + "loss": 2.5086, + "theoretical_loss": 3.33751202225172, + "tokens_seen": 2811166720 + }, + { + "epoch": 0.7, + "learning_rate": 2.991494142192265e-05, + "loss": 2.4171, + "theoretical_loss": 3.337499899398421, + "tokens_seen": 2811297792 + }, + { + "epoch": 0.7, + "learning_rate": 2.99069170277644e-05, + "loss": 2.5328, + "theoretical_loss": 3.3374877772685663, + "tokens_seen": 2811428864 + }, + { + "epoch": 0.7, + "learning_rate": 2.989889263360616e-05, + "loss": 2.5414, + "theoretical_loss": 3.337475655862079, + "tokens_seen": 2811559936 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1541868, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4373865127563477, + "objective/train/theoretical_loss": 3.3374695954300737, + "objective/train/tokens_used": 1182084576, + "theoretical_loss": 3.3374695954300737, + "tokens_seen": 2811625472 + }, + { + "epoch": 0.7, + "learning_rate": 2.989086823944792e-05, + "loss": 2.331, + "theoretical_loss": 3.3374635351788813, + "tokens_seen": 2811691008 + }, + { + "epoch": 0.7, + "learning_rate": 2.988284384528968e-05, + "loss": 2.5662, + "theoretical_loss": 3.337451415218897, + "tokens_seen": 2811822080 + }, + { + "epoch": 0.7, + "learning_rate": 2.987481945113144e-05, + "loss": 2.5865, + "theoretical_loss": 3.33743929598205, + "tokens_seen": 2811953152 + }, + { + "epoch": 0.7, + "learning_rate": 2.98667950569732e-05, + "loss": 2.2868, + "theoretical_loss": 3.3374271774682622, + "tokens_seen": 2812084224 + }, + { + "epoch": 0.7, + "learning_rate": 2.985877066281496e-05, + "loss": 2.63, + "theoretical_loss": 3.337415059677457, + "tokens_seen": 2812215296 + }, + { + "epoch": 0.7, + "learning_rate": 2.9850746268656714e-05, + "loss": 2.5081, + "theoretical_loss": 3.3374029426095584, + "tokens_seen": 2812346368 + }, + { + "epoch": 0.7, + "learning_rate": 2.9842721874498474e-05, + "loss": 2.3956, + "theoretical_loss": 3.3373908262644885, + "tokens_seen": 2812477440 + }, + { + "epoch": 0.7, + "learning_rate": 2.9834697480340234e-05, + "loss": 2.4097, + "theoretical_loss": 3.337378710642171, + "tokens_seen": 2812608512 + }, + { + "epoch": 0.7, + "learning_rate": 2.9826673086181994e-05, + "loss": 2.5387, + "theoretical_loss": 3.3373665957425294, + "tokens_seen": 2812739584 + }, + { + "epoch": 0.7, + "learning_rate": 2.9818648692023754e-05, + "loss": 2.6679, + "theoretical_loss": 3.337354481565486, + "tokens_seen": 2812870656 + }, + { + "epoch": 0.7, + "learning_rate": 2.9810624297865514e-05, + "loss": 2.6541, + "theoretical_loss": 3.3373423681109653, + "tokens_seen": 2813001728 + }, + { + "epoch": 0.7, + "learning_rate": 2.9802599903707273e-05, + "loss": 2.4281, + "theoretical_loss": 3.3373302553788897, + "tokens_seen": 2813132800 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1543196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6425278186798096, + "objective/train/theoretical_loss": 3.3373181433691825, + "objective/train/tokens_used": 1183722976, + "theoretical_loss": 3.3373181433691825, + "tokens_seen": 2813263872 + }, + { + "epoch": 0.71, + "learning_rate": 2.9794575509549033e-05, + "loss": 2.411, + "theoretical_loss": 3.3373181433691825, + "tokens_seen": 2813263872 + }, + { + "epoch": 0.71, + "learning_rate": 2.9786551115390786e-05, + "loss": 2.5334, + "theoretical_loss": 3.337306032081767, + "tokens_seen": 2813394944 + }, + { + "epoch": 0.71, + "learning_rate": 2.9778526721232546e-05, + "loss": 2.489, + "theoretical_loss": 3.337293921516567, + "tokens_seen": 2813526016 + }, + { + "epoch": 0.71, + "learning_rate": 2.9770502327074306e-05, + "loss": 2.5091, + "theoretical_loss": 3.337281811673505, + "tokens_seen": 2813657088 + }, + { + "epoch": 0.71, + "learning_rate": 2.9762477932916066e-05, + "loss": 2.4243, + "theoretical_loss": 3.3372697025525047, + "tokens_seen": 2813788160 + }, + { + "epoch": 0.71, + "learning_rate": 2.9754453538757826e-05, + "loss": 2.4699, + "theoretical_loss": 3.337257594153489, + "tokens_seen": 2813919232 + }, + { + "epoch": 0.71, + "learning_rate": 2.9746429144599586e-05, + "loss": 2.4699, + "theoretical_loss": 3.3372454864763825, + "tokens_seen": 2814050304 + }, + { + "epoch": 0.71, + "learning_rate": 2.9738404750441346e-05, + "loss": 2.5367, + "theoretical_loss": 3.3372333795211073, + "tokens_seen": 2814181376 + }, + { + "epoch": 0.71, + "learning_rate": 2.97303803562831e-05, + "loss": 2.3836, + "theoretical_loss": 3.337221273287587, + "tokens_seen": 2814312448 + }, + { + "epoch": 0.71, + "learning_rate": 2.972235596212486e-05, + "loss": 2.5073, + "theoretical_loss": 3.337209167775745, + "tokens_seen": 2814443520 + }, + { + "epoch": 0.71, + "learning_rate": 2.971433156796662e-05, + "loss": 2.6079, + "theoretical_loss": 3.337197062985505, + "tokens_seen": 2814574592 + }, + { + "epoch": 0.71, + "learning_rate": 2.970630717380838e-05, + "loss": 2.5136, + "theoretical_loss": 3.33718495891679, + "tokens_seen": 2814705664 + }, + { + "epoch": 0.71, + "learning_rate": 2.969828277965014e-05, + "loss": 2.4699, + "theoretical_loss": 3.3371728555695235, + "tokens_seen": 2814836736 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1543701, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.649340867996216, + "objective/train/theoretical_loss": 3.3371668041664098, + "objective/train/tokens_used": 1185361376, + "theoretical_loss": 3.3371668041664098, + "tokens_seen": 2814902272 + }, + { + "epoch": 0.71, + "learning_rate": 2.96902583854919e-05, + "loss": 2.3623, + "theoretical_loss": 3.337160752943629, + "tokens_seen": 2814967808 + }, + { + "epoch": 0.71, + "learning_rate": 2.968223399133366e-05, + "loss": 2.4494, + "theoretical_loss": 3.33714865103903, + "tokens_seen": 2815098880 + }, + { + "epoch": 0.71, + "learning_rate": 2.967420959717541e-05, + "loss": 2.6194, + "theoretical_loss": 3.337136549855649, + "tokens_seen": 2815229952 + }, + { + "epoch": 0.71, + "learning_rate": 2.966618520301717e-05, + "loss": 2.5349, + "theoretical_loss": 3.337124449393411, + "tokens_seen": 2815361024 + }, + { + "epoch": 0.71, + "learning_rate": 2.965816080885893e-05, + "loss": 2.3989, + "theoretical_loss": 3.3371123496522386, + "tokens_seen": 2815492096 + }, + { + "epoch": 0.71, + "learning_rate": 2.965013641470069e-05, + "loss": 2.5976, + "theoretical_loss": 3.337100250632055, + "tokens_seen": 2815623168 + }, + { + "epoch": 0.71, + "learning_rate": 2.964211202054245e-05, + "loss": 2.5118, + "theoretical_loss": 3.337088152332784, + "tokens_seen": 2815754240 + }, + { + "epoch": 0.71, + "learning_rate": 2.963408762638421e-05, + "loss": 2.5726, + "theoretical_loss": 3.3370760547543497, + "tokens_seen": 2815885312 + }, + { + "epoch": 0.71, + "learning_rate": 2.962606323222597e-05, + "loss": 2.5291, + "theoretical_loss": 3.3370639578966745, + "tokens_seen": 2816016384 + }, + { + "epoch": 0.71, + "learning_rate": 2.961803883806773e-05, + "loss": 2.3767, + "theoretical_loss": 3.337051861759683, + "tokens_seen": 2816147456 + }, + { + "epoch": 0.71, + "learning_rate": 2.9610014443909484e-05, + "loss": 2.4788, + "theoretical_loss": 3.3370397663432976, + "tokens_seen": 2816278528 + }, + { + "epoch": 0.71, + "learning_rate": 2.9601990049751244e-05, + "loss": 2.6541, + "theoretical_loss": 3.3370276716474425, + "tokens_seen": 2816409600 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1544731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5291428565979004, + "objective/train/theoretical_loss": 3.337015577672041, + "objective/train/tokens_used": 1186999776, + "theoretical_loss": 3.337015577672041, + "tokens_seen": 2816540672 + }, + { + "epoch": 0.71, + "learning_rate": 2.9593965655593004e-05, + "loss": 2.483, + "theoretical_loss": 3.337015577672041, + "tokens_seen": 2816540672 + }, + { + "epoch": 0.71, + "learning_rate": 2.9585941261434764e-05, + "loss": 2.5772, + "theoretical_loss": 3.337003484417017, + "tokens_seen": 2816671744 + }, + { + "epoch": 0.71, + "learning_rate": 2.9577916867276524e-05, + "loss": 2.5148, + "theoretical_loss": 3.336991391882294, + "tokens_seen": 2816802816 + }, + { + "epoch": 0.71, + "learning_rate": 2.9569892473118284e-05, + "loss": 2.5, + "theoretical_loss": 3.336979300067795, + "tokens_seen": 2816933888 + }, + { + "epoch": 0.71, + "learning_rate": 2.9561868078960044e-05, + "loss": 2.6741, + "theoretical_loss": 3.3369672089734443, + "tokens_seen": 2817064960 + }, + { + "epoch": 0.71, + "learning_rate": 2.9553843684801797e-05, + "loss": 2.4193, + "theoretical_loss": 3.3369551185991653, + "tokens_seen": 2817196032 + }, + { + "epoch": 0.71, + "learning_rate": 2.9545819290643557e-05, + "loss": 2.6252, + "theoretical_loss": 3.3369430289448814, + "tokens_seen": 2817327104 + }, + { + "epoch": 0.71, + "learning_rate": 2.9537794896485316e-05, + "loss": 2.3723, + "theoretical_loss": 3.3369309400105163, + "tokens_seen": 2817458176 + }, + { + "epoch": 0.71, + "learning_rate": 2.9529770502327076e-05, + "loss": 2.4889, + "theoretical_loss": 3.3369188517959945, + "tokens_seen": 2817589248 + }, + { + "epoch": 0.71, + "learning_rate": 2.9521746108168836e-05, + "loss": 2.4626, + "theoretical_loss": 3.3369067643012382, + "tokens_seen": 2817720320 + }, + { + "epoch": 0.71, + "learning_rate": 2.9513721714010596e-05, + "loss": 2.4912, + "theoretical_loss": 3.3368946775261716, + "tokens_seen": 2817851392 + }, + { + "epoch": 0.71, + "learning_rate": 2.9505697319852356e-05, + "loss": 2.407, + "theoretical_loss": 3.3368825914707188, + "tokens_seen": 2817982464 + }, + { + "epoch": 0.71, + "learning_rate": 2.9497672925694113e-05, + "loss": 2.5216, + "theoretical_loss": 3.3368705061348036, + "tokens_seen": 2818113536 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1545316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3830935955047607, + "objective/train/theoretical_loss": 3.336864463736648, + "objective/train/tokens_used": 1188638176, + "theoretical_loss": 3.336864463736648, + "tokens_seen": 2818179072 + }, + { + "epoch": 0.71, + "learning_rate": 2.948964853153587e-05, + "loss": 2.46, + "theoretical_loss": 3.3368584215183485, + "tokens_seen": 2818244608 + }, + { + "epoch": 0.71, + "learning_rate": 2.948162413737763e-05, + "loss": 2.5155, + "theoretical_loss": 3.336846337621279, + "tokens_seen": 2818375680 + }, + { + "epoch": 0.71, + "learning_rate": 2.947359974321939e-05, + "loss": 2.4112, + "theoretical_loss": 3.336834254443517, + "tokens_seen": 2818506752 + }, + { + "epoch": 0.71, + "learning_rate": 2.946557534906115e-05, + "loss": 2.48, + "theoretical_loss": 3.3368221719849873, + "tokens_seen": 2818637824 + }, + { + "epoch": 0.71, + "learning_rate": 2.945755095490291e-05, + "loss": 2.6649, + "theoretical_loss": 3.3368100902456135, + "tokens_seen": 2818768896 + }, + { + "epoch": 0.71, + "learning_rate": 2.9449526560744665e-05, + "loss": 2.5015, + "theoretical_loss": 3.3367980092253196, + "tokens_seen": 2818899968 + }, + { + "epoch": 0.71, + "learning_rate": 2.9441502166586425e-05, + "loss": 2.6761, + "theoretical_loss": 3.3367859289240287, + "tokens_seen": 2819031040 + }, + { + "epoch": 0.71, + "learning_rate": 2.943347777242818e-05, + "loss": 2.5658, + "theoretical_loss": 3.3367738493416654, + "tokens_seen": 2819162112 + }, + { + "epoch": 0.71, + "learning_rate": 2.942545337826994e-05, + "loss": 2.5448, + "theoretical_loss": 3.3367617704781525, + "tokens_seen": 2819293184 + }, + { + "epoch": 0.71, + "learning_rate": 2.94174289841117e-05, + "loss": 2.5258, + "theoretical_loss": 3.336749692333415, + "tokens_seen": 2819424256 + }, + { + "epoch": 0.71, + "learning_rate": 2.940940458995346e-05, + "loss": 2.6859, + "theoretical_loss": 3.336737614907375, + "tokens_seen": 2819555328 + }, + { + "epoch": 0.71, + "learning_rate": 2.9401380195795218e-05, + "loss": 2.5097, + "theoretical_loss": 3.336725538199958, + "tokens_seen": 2819686400 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1546442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0588791370391846, + "objective/train/theoretical_loss": 3.3367134622110877, + "objective/train/tokens_used": 1190276576, + "theoretical_loss": 3.3367134622110877, + "tokens_seen": 2819817472 + }, + { + "epoch": 0.71, + "learning_rate": 2.9393355801636978e-05, + "loss": 2.6196, + "theoretical_loss": 3.3367134622110877, + "tokens_seen": 2819817472 + }, + { + "epoch": 0.71, + "learning_rate": 2.9385331407478738e-05, + "loss": 2.6077, + "theoretical_loss": 3.3367013869406867, + "tokens_seen": 2819948544 + }, + { + "epoch": 0.71, + "learning_rate": 2.9377307013320494e-05, + "loss": 2.5406, + "theoretical_loss": 3.33668931238868, + "tokens_seen": 2820079616 + }, + { + "epoch": 0.71, + "learning_rate": 2.9369282619162254e-05, + "loss": 2.4328, + "theoretical_loss": 3.3366772385549908, + "tokens_seen": 2820210688 + }, + { + "epoch": 0.71, + "learning_rate": 2.9361258225004014e-05, + "loss": 2.5112, + "theoretical_loss": 3.336665165439544, + "tokens_seen": 2820341760 + }, + { + "epoch": 0.71, + "learning_rate": 2.935323383084577e-05, + "loss": 2.5181, + "theoretical_loss": 3.336653093042262, + "tokens_seen": 2820472832 + }, + { + "epoch": 0.71, + "learning_rate": 2.934520943668753e-05, + "loss": 2.5061, + "theoretical_loss": 3.3366410213630697, + "tokens_seen": 2820603904 + }, + { + "epoch": 0.71, + "learning_rate": 2.933718504252929e-05, + "loss": 2.5894, + "theoretical_loss": 3.336628950401891, + "tokens_seen": 2820734976 + }, + { + "epoch": 0.71, + "learning_rate": 2.932916064837105e-05, + "loss": 2.5016, + "theoretical_loss": 3.3366168801586498, + "tokens_seen": 2820866048 + }, + { + "epoch": 0.71, + "learning_rate": 2.932113625421281e-05, + "loss": 2.6328, + "theoretical_loss": 3.3366048106332697, + "tokens_seen": 2820997120 + }, + { + "epoch": 0.71, + "learning_rate": 2.9313111860054567e-05, + "loss": 2.5409, + "theoretical_loss": 3.3365927418256747, + "tokens_seen": 2821128192 + }, + { + "epoch": 0.71, + "learning_rate": 2.9305087465896323e-05, + "loss": 2.5167, + "theoretical_loss": 3.336580673735789, + "tokens_seen": 2821259264 + }, + { + "epoch": 0.71, + "learning_rate": 2.9297063071738083e-05, + "loss": 2.5224, + "theoretical_loss": 3.336568606363537, + "tokens_seen": 2821390336 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1547104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4776675701141357, + "objective/train/theoretical_loss": 3.3365625729464994, + "objective/train/tokens_used": 1191914976, + "theoretical_loss": 3.3365625729464994, + "tokens_seen": 2821455872 + }, + { + "epoch": 0.71, + "learning_rate": 2.9289038677579843e-05, + "loss": 2.3769, + "theoretical_loss": 3.3365565397088415, + "tokens_seen": 2821521408 + }, + { + "epoch": 0.71, + "learning_rate": 2.9281014283421603e-05, + "loss": 2.5575, + "theoretical_loss": 3.336544473771627, + "tokens_seen": 2821652480 + }, + { + "epoch": 0.71, + "learning_rate": 2.9272989889263363e-05, + "loss": 2.4788, + "theoretical_loss": 3.3365324085518187, + "tokens_seen": 2821783552 + }, + { + "epoch": 0.71, + "learning_rate": 2.9264965495105123e-05, + "loss": 2.6253, + "theoretical_loss": 3.336520344049339, + "tokens_seen": 2821914624 + }, + { + "epoch": 0.71, + "learning_rate": 2.9256941100946876e-05, + "loss": 2.5184, + "theoretical_loss": 3.3365082802641126, + "tokens_seen": 2822045696 + }, + { + "epoch": 0.71, + "learning_rate": 2.9248916706788636e-05, + "loss": 2.4687, + "theoretical_loss": 3.3364962171960633, + "tokens_seen": 2822176768 + }, + { + "epoch": 0.71, + "learning_rate": 2.9240892312630396e-05, + "loss": 2.5589, + "theoretical_loss": 3.336484154845116, + "tokens_seen": 2822307840 + }, + { + "epoch": 0.71, + "learning_rate": 2.9232867918472156e-05, + "loss": 2.5615, + "theoretical_loss": 3.3364720932111935, + "tokens_seen": 2822438912 + }, + { + "epoch": 0.71, + "learning_rate": 2.9224843524313916e-05, + "loss": 2.5391, + "theoretical_loss": 3.336460032294221, + "tokens_seen": 2822569984 + }, + { + "epoch": 0.71, + "learning_rate": 2.9216819130155675e-05, + "loss": 2.5234, + "theoretical_loss": 3.336447972094122, + "tokens_seen": 2822701056 + }, + { + "epoch": 0.71, + "learning_rate": 2.9208794735997435e-05, + "loss": 2.5914, + "theoretical_loss": 3.3364359126108205, + "tokens_seen": 2822832128 + }, + { + "epoch": 0.71, + "learning_rate": 2.920077034183919e-05, + "loss": 2.5605, + "theoretical_loss": 3.336423853844241, + "tokens_seen": 2822963200 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1548160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.736697196960449, + "objective/train/theoretical_loss": 3.3364117957943074, + "objective/train/tokens_used": 1193553376, + "theoretical_loss": 3.3364117957943074, + "tokens_seen": 2823094272 + }, + { + "epoch": 0.71, + "learning_rate": 2.919274594768095e-05, + "loss": 2.5636, + "theoretical_loss": 3.3364117957943074, + "tokens_seen": 2823094272 + }, + { + "epoch": 0.71, + "learning_rate": 2.9184721553522708e-05, + "loss": 2.5388, + "theoretical_loss": 3.336399738460944, + "tokens_seen": 2823225344 + }, + { + "epoch": 0.71, + "learning_rate": 2.9176697159364468e-05, + "loss": 2.581, + "theoretical_loss": 3.3363876818440747, + "tokens_seen": 2823356416 + }, + { + "epoch": 0.71, + "learning_rate": 2.9168672765206228e-05, + "loss": 2.4907, + "theoretical_loss": 3.336375625943624, + "tokens_seen": 2823487488 + }, + { + "epoch": 0.71, + "learning_rate": 2.9160648371047988e-05, + "loss": 2.4659, + "theoretical_loss": 3.3363635707595156, + "tokens_seen": 2823618560 + }, + { + "epoch": 0.71, + "learning_rate": 2.9152623976889748e-05, + "loss": 2.4414, + "theoretical_loss": 3.3363515162916744, + "tokens_seen": 2823749632 + }, + { + "epoch": 0.71, + "learning_rate": 2.9144599582731508e-05, + "loss": 2.4496, + "theoretical_loss": 3.3363394625400242, + "tokens_seen": 2823880704 + }, + { + "epoch": 0.71, + "learning_rate": 2.913657518857326e-05, + "loss": 2.5732, + "theoretical_loss": 3.336327409504489, + "tokens_seen": 2824011776 + }, + { + "epoch": 0.71, + "learning_rate": 2.912855079441502e-05, + "loss": 2.5549, + "theoretical_loss": 3.336315357184993, + "tokens_seen": 2824142848 + }, + { + "epoch": 0.71, + "learning_rate": 2.912052640025678e-05, + "loss": 2.5019, + "theoretical_loss": 3.336303305581461, + "tokens_seen": 2824273920 + }, + { + "epoch": 0.71, + "learning_rate": 2.911250200609854e-05, + "loss": 2.5645, + "theoretical_loss": 3.336291254693817, + "tokens_seen": 2824404992 + }, + { + "epoch": 0.71, + "learning_rate": 2.91044776119403e-05, + "loss": 2.6796, + "theoretical_loss": 3.3362792045219845, + "tokens_seen": 2824536064 + }, + { + "epoch": 0.71, + "learning_rate": 2.909645321778206e-05, + "loss": 2.5172, + "theoretical_loss": 3.336267155065889, + "tokens_seen": 2824667136 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1548671, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.283342123031616, + "objective/train/theoretical_loss": 3.3362611306062187, + "objective/train/tokens_used": 1195191776, + "theoretical_loss": 3.3362611306062187, + "tokens_seen": 2824732672 + }, + { + "epoch": 0.71, + "learning_rate": 2.908842882362382e-05, + "loss": 2.4625, + "theoretical_loss": 3.3362551063254537, + "tokens_seen": 2824798208 + }, + { + "epoch": 0.71, + "learning_rate": 2.9080404429465574e-05, + "loss": 2.3523, + "theoretical_loss": 3.336243058300604, + "tokens_seen": 2824929280 + }, + { + "epoch": 0.71, + "learning_rate": 2.9072380035307333e-05, + "loss": 2.3789, + "theoretical_loss": 3.336231010991263, + "tokens_seen": 2825060352 + }, + { + "epoch": 0.71, + "learning_rate": 2.9064355641149093e-05, + "loss": 2.5625, + "theoretical_loss": 3.3362189643973563, + "tokens_seen": 2825191424 + }, + { + "epoch": 0.71, + "learning_rate": 2.9056331246990853e-05, + "loss": 2.5052, + "theoretical_loss": 3.336206918518807, + "tokens_seen": 2825322496 + }, + { + "epoch": 0.71, + "learning_rate": 2.9048306852832613e-05, + "loss": 2.6922, + "theoretical_loss": 3.3361948733555398, + "tokens_seen": 2825453568 + }, + { + "epoch": 0.71, + "learning_rate": 2.9040282458674373e-05, + "loss": 2.5149, + "theoretical_loss": 3.3361828289074795, + "tokens_seen": 2825584640 + }, + { + "epoch": 0.71, + "learning_rate": 2.9032258064516133e-05, + "loss": 2.5262, + "theoretical_loss": 3.3361707851745503, + "tokens_seen": 2825715712 + }, + { + "epoch": 0.71, + "learning_rate": 2.9024233670357893e-05, + "loss": 2.4821, + "theoretical_loss": 3.3361587421566763, + "tokens_seen": 2825846784 + }, + { + "epoch": 0.71, + "learning_rate": 2.9016209276199646e-05, + "loss": 2.5052, + "theoretical_loss": 3.3361466998537823, + "tokens_seen": 2825977856 + }, + { + "epoch": 0.71, + "learning_rate": 2.9008184882041406e-05, + "loss": 2.5508, + "theoretical_loss": 3.336134658265792, + "tokens_seen": 2826108928 + }, + { + "epoch": 0.71, + "learning_rate": 2.9000160487883166e-05, + "loss": 2.5166, + "theoretical_loss": 3.33612261739263, + "tokens_seen": 2826240000 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1549878, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6296541690826416, + "objective/train/theoretical_loss": 3.3361105772342214, + "objective/train/tokens_used": 1196830176, + "theoretical_loss": 3.3361105772342214, + "tokens_seen": 2826371072 + }, + { + "epoch": 0.71, + "learning_rate": 2.8992136093724926e-05, + "loss": 2.5283, + "theoretical_loss": 3.3361105772342214, + "tokens_seen": 2826371072 + }, + { + "epoch": 0.71, + "learning_rate": 2.8984111699566686e-05, + "loss": 2.6464, + "theoretical_loss": 3.33609853779049, + "tokens_seen": 2826502144 + }, + { + "epoch": 0.71, + "learning_rate": 2.8976087305408446e-05, + "loss": 2.6093, + "theoretical_loss": 3.3360864990613606, + "tokens_seen": 2826633216 + }, + { + "epoch": 0.71, + "learning_rate": 2.8968062911250205e-05, + "loss": 2.6076, + "theoretical_loss": 3.336074461046757, + "tokens_seen": 2826764288 + }, + { + "epoch": 0.71, + "learning_rate": 2.896003851709196e-05, + "loss": 2.5031, + "theoretical_loss": 3.336062423746605, + "tokens_seen": 2826895360 + }, + { + "epoch": 0.71, + "learning_rate": 2.895201412293372e-05, + "loss": 2.4735, + "theoretical_loss": 3.336050387160827, + "tokens_seen": 2827026432 + }, + { + "epoch": 0.71, + "learning_rate": 2.894398972877548e-05, + "loss": 2.618, + "theoretical_loss": 3.33603835128935, + "tokens_seen": 2827157504 + }, + { + "epoch": 0.71, + "learning_rate": 2.8935965334617238e-05, + "loss": 2.708, + "theoretical_loss": 3.336026316132096, + "tokens_seen": 2827288576 + }, + { + "epoch": 0.71, + "learning_rate": 2.8927940940458998e-05, + "loss": 2.4509, + "theoretical_loss": 3.3360142816889913, + "tokens_seen": 2827419648 + }, + { + "epoch": 0.71, + "learning_rate": 2.8919916546300758e-05, + "loss": 2.2834, + "theoretical_loss": 3.33600224795996, + "tokens_seen": 2827550720 + }, + { + "epoch": 0.71, + "learning_rate": 2.8911892152142518e-05, + "loss": 2.5137, + "theoretical_loss": 3.3359902149449256, + "tokens_seen": 2827681792 + }, + { + "epoch": 0.71, + "learning_rate": 2.890386775798427e-05, + "loss": 2.5556, + "theoretical_loss": 3.335978182643814, + "tokens_seen": 2827812864 + }, + { + "epoch": 0.71, + "learning_rate": 2.889584336382603e-05, + "loss": 2.5618, + "theoretical_loss": 3.3359661510565495, + "tokens_seen": 2827943936 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1550467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9742480516433716, + "objective/train/theoretical_loss": 3.335960135530586, + "objective/train/tokens_used": 1198468576, + "theoretical_loss": 3.335960135530586, + "tokens_seen": 2828009472 + }, + { + "epoch": 0.71, + "learning_rate": 2.888781896966779e-05, + "loss": 2.5818, + "theoretical_loss": 3.335954120183056, + "tokens_seen": 2828075008 + }, + { + "epoch": 0.71, + "learning_rate": 2.887979457550955e-05, + "loss": 2.5264, + "theoretical_loss": 3.3359420900232584, + "tokens_seen": 2828206080 + }, + { + "epoch": 0.71, + "learning_rate": 2.887177018135131e-05, + "loss": 2.5877, + "theoretical_loss": 3.335930060577082, + "tokens_seen": 2828337152 + }, + { + "epoch": 0.71, + "learning_rate": 2.886374578719307e-05, + "loss": 2.6022, + "theoretical_loss": 3.33591803184445, + "tokens_seen": 2828468224 + }, + { + "epoch": 0.71, + "learning_rate": 2.885572139303483e-05, + "loss": 2.6192, + "theoretical_loss": 3.335906003825288, + "tokens_seen": 2828599296 + }, + { + "epoch": 0.71, + "learning_rate": 2.884769699887659e-05, + "loss": 2.4822, + "theoretical_loss": 3.335893976519521, + "tokens_seen": 2828730368 + }, + { + "epoch": 0.71, + "learning_rate": 2.8839672604718344e-05, + "loss": 2.4179, + "theoretical_loss": 3.3358819499270727, + "tokens_seen": 2828861440 + }, + { + "epoch": 0.71, + "learning_rate": 2.8831648210560103e-05, + "loss": 2.4044, + "theoretical_loss": 3.3358699240478678, + "tokens_seen": 2828992512 + }, + { + "epoch": 0.71, + "learning_rate": 2.8823623816401863e-05, + "loss": 2.5494, + "theoretical_loss": 3.335857898881831, + "tokens_seen": 2829123584 + }, + { + "epoch": 0.71, + "learning_rate": 2.8815599422243623e-05, + "loss": 2.3656, + "theoretical_loss": 3.335845874428888, + "tokens_seen": 2829254656 + }, + { + "epoch": 0.71, + "learning_rate": 2.8807575028085383e-05, + "loss": 2.4916, + "theoretical_loss": 3.335833850688962, + "tokens_seen": 2829385728 + }, + { + "epoch": 0.71, + "learning_rate": 2.8799550633927143e-05, + "loss": 2.6347, + "theoretical_loss": 3.335821827661979, + "tokens_seen": 2829516800 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1551304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3311545848846436, + "objective/train/theoretical_loss": 3.335809805347863, + "objective/train/tokens_used": 1200106976, + "theoretical_loss": 3.335809805347863, + "tokens_seen": 2829647872 + }, + { + "epoch": 0.71, + "learning_rate": 2.87915262397689e-05, + "loss": 2.4843, + "theoretical_loss": 3.335809805347863, + "tokens_seen": 2829647872 + }, + { + "epoch": 0.72, + "learning_rate": 2.8783501845610656e-05, + "loss": 2.6467, + "theoretical_loss": 3.3357977837465382, + "tokens_seen": 2829778944 + }, + { + "epoch": 0.72, + "learning_rate": 2.8775477451452416e-05, + "loss": 2.5553, + "theoretical_loss": 3.3357857628579306, + "tokens_seen": 2829910016 + }, + { + "epoch": 0.72, + "learning_rate": 2.8767453057294176e-05, + "loss": 2.4768, + "theoretical_loss": 3.335773742681964, + "tokens_seen": 2830041088 + }, + { + "epoch": 0.72, + "learning_rate": 2.8759428663135936e-05, + "loss": 2.5271, + "theoretical_loss": 3.335761723218563, + "tokens_seen": 2830172160 + }, + { + "epoch": 0.72, + "learning_rate": 2.8751404268977696e-05, + "loss": 2.4898, + "theoretical_loss": 3.3357497044676534, + "tokens_seen": 2830303232 + }, + { + "epoch": 0.72, + "learning_rate": 2.8743379874819452e-05, + "loss": 2.5668, + "theoretical_loss": 3.3357376864291592, + "tokens_seen": 2830434304 + }, + { + "epoch": 0.72, + "learning_rate": 2.8735355480661212e-05, + "loss": 2.5593, + "theoretical_loss": 3.3357256691030055, + "tokens_seen": 2830565376 + }, + { + "epoch": 0.72, + "learning_rate": 2.872733108650297e-05, + "loss": 2.5946, + "theoretical_loss": 3.3357136524891167, + "tokens_seen": 2830696448 + }, + { + "epoch": 0.72, + "learning_rate": 2.871930669234473e-05, + "loss": 2.678, + "theoretical_loss": 3.3357016365874177, + "tokens_seen": 2830827520 + }, + { + "epoch": 0.72, + "learning_rate": 2.871128229818649e-05, + "loss": 2.2981, + "theoretical_loss": 3.3356896213978335, + "tokens_seen": 2830958592 + }, + { + "epoch": 0.72, + "learning_rate": 2.870325790402825e-05, + "loss": 2.4455, + "theoretical_loss": 3.335677606920289, + "tokens_seen": 2831089664 + }, + { + "epoch": 0.72, + "learning_rate": 2.8695233509870005e-05, + "loss": 2.5223, + "theoretical_loss": 3.335665593154709, + "tokens_seen": 2831220736 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1551942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2895348072052, + "objective/train/theoretical_loss": 3.335659586538882, + "objective/train/tokens_used": 1201745376, + "theoretical_loss": 3.335659586538882, + "tokens_seen": 2831286272 + }, + { + "epoch": 0.72, + "learning_rate": 2.8687209115711765e-05, + "loss": 2.5018, + "theoretical_loss": 3.335653580101018, + "tokens_seen": 2831351808 + }, + { + "epoch": 0.72, + "learning_rate": 2.8679184721553525e-05, + "loss": 2.5282, + "theoretical_loss": 3.3356415677591413, + "tokens_seen": 2831482880 + }, + { + "epoch": 0.72, + "learning_rate": 2.8671160327395285e-05, + "loss": 2.4455, + "theoretical_loss": 3.3356295561290037, + "tokens_seen": 2831613952 + }, + { + "epoch": 0.72, + "learning_rate": 2.866313593323704e-05, + "loss": 2.4923, + "theoretical_loss": 3.33561754521053, + "tokens_seen": 2831745024 + }, + { + "epoch": 0.72, + "learning_rate": 2.86551115390788e-05, + "loss": 2.6073, + "theoretical_loss": 3.335605535003645, + "tokens_seen": 2831876096 + }, + { + "epoch": 0.72, + "learning_rate": 2.8647087144920558e-05, + "loss": 2.5256, + "theoretical_loss": 3.335593525508274, + "tokens_seen": 2832007168 + }, + { + "epoch": 0.72, + "learning_rate": 2.8639062750762318e-05, + "loss": 2.7181, + "theoretical_loss": 3.3355815167243414, + "tokens_seen": 2832138240 + }, + { + "epoch": 0.72, + "learning_rate": 2.8631038356604077e-05, + "loss": 2.5226, + "theoretical_loss": 3.3355695086517723, + "tokens_seen": 2832269312 + }, + { + "epoch": 0.72, + "learning_rate": 2.8623013962445837e-05, + "loss": 2.5795, + "theoretical_loss": 3.335557501290492, + "tokens_seen": 2832400384 + }, + { + "epoch": 0.72, + "learning_rate": 2.8614989568287597e-05, + "loss": 2.473, + "theoretical_loss": 3.335545494640425, + "tokens_seen": 2832531456 + }, + { + "epoch": 0.72, + "learning_rate": 2.8606965174129354e-05, + "loss": 2.6299, + "theoretical_loss": 3.335533488701496, + "tokens_seen": 2832662528 + }, + { + "epoch": 0.72, + "learning_rate": 2.859894077997111e-05, + "loss": 2.7204, + "theoretical_loss": 3.335521483473631, + "tokens_seen": 2832793600 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1553090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.629032611846924, + "objective/train/theoretical_loss": 3.335509478956754, + "objective/train/tokens_used": 1203383776, + "theoretical_loss": 3.335509478956754, + "tokens_seen": 2832924672 + }, + { + "epoch": 0.72, + "learning_rate": 2.859091638581287e-05, + "loss": 2.5743, + "theoretical_loss": 3.335509478956754, + "tokens_seen": 2832924672 + }, + { + "epoch": 0.72, + "learning_rate": 2.858289199165463e-05, + "loss": 2.337, + "theoretical_loss": 3.3354974751507904, + "tokens_seen": 2833055744 + }, + { + "epoch": 0.72, + "learning_rate": 2.857486759749639e-05, + "loss": 2.4407, + "theoretical_loss": 3.3354854720556655, + "tokens_seen": 2833186816 + }, + { + "epoch": 0.72, + "learning_rate": 2.856684320333815e-05, + "loss": 2.7173, + "theoretical_loss": 3.335473469671304, + "tokens_seen": 2833317888 + }, + { + "epoch": 0.72, + "learning_rate": 2.855881880917991e-05, + "loss": 2.5421, + "theoretical_loss": 3.3354614679976304, + "tokens_seen": 2833448960 + }, + { + "epoch": 0.72, + "learning_rate": 2.855079441502167e-05, + "loss": 2.4906, + "theoretical_loss": 3.3354494670345707, + "tokens_seen": 2833580032 + }, + { + "epoch": 0.72, + "learning_rate": 2.8542770020863423e-05, + "loss": 2.3566, + "theoretical_loss": 3.33543746678205, + "tokens_seen": 2833711104 + }, + { + "epoch": 0.72, + "learning_rate": 2.8534745626705183e-05, + "loss": 2.4478, + "theoretical_loss": 3.3354254672399923, + "tokens_seen": 2833842176 + }, + { + "epoch": 0.72, + "learning_rate": 2.8526721232546943e-05, + "loss": 2.5143, + "theoretical_loss": 3.3354134684083236, + "tokens_seen": 2833973248 + }, + { + "epoch": 0.72, + "learning_rate": 2.8518696838388703e-05, + "loss": 2.6133, + "theoretical_loss": 3.3354014702869685, + "tokens_seen": 2834104320 + }, + { + "epoch": 0.72, + "learning_rate": 2.8510672444230462e-05, + "loss": 2.5619, + "theoretical_loss": 3.335389472875853, + "tokens_seen": 2834235392 + }, + { + "epoch": 0.72, + "learning_rate": 2.8502648050072222e-05, + "loss": 2.5558, + "theoretical_loss": 3.3353774761749007, + "tokens_seen": 2834366464 + }, + { + "epoch": 0.72, + "learning_rate": 2.8494623655913982e-05, + "loss": 2.5911, + "theoretical_loss": 3.335365480184038, + "tokens_seen": 2834497536 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1554323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.297001838684082, + "objective/train/theoretical_loss": 3.3353594824548662, + "objective/train/tokens_used": 1205022176, + "theoretical_loss": 3.3353594824548662, + "tokens_seen": 2834563072 + }, + { + "epoch": 0.72, + "learning_rate": 2.8486599261755735e-05, + "loss": 2.3246, + "theoretical_loss": 3.335353484903189, + "tokens_seen": 2834628608 + }, + { + "epoch": 0.72, + "learning_rate": 2.8478574867597495e-05, + "loss": 2.4736, + "theoretical_loss": 3.3353414903322802, + "tokens_seen": 2834759680 + }, + { + "epoch": 0.72, + "learning_rate": 2.8470550473439255e-05, + "loss": 2.6413, + "theoretical_loss": 3.3353294964712354, + "tokens_seen": 2834890752 + }, + { + "epoch": 0.72, + "learning_rate": 2.8462526079281015e-05, + "loss": 2.6, + "theoretical_loss": 3.3353175033199807, + "tokens_seen": 2835021824 + }, + { + "epoch": 0.72, + "learning_rate": 2.8454501685122775e-05, + "loss": 2.6082, + "theoretical_loss": 3.335305510878441, + "tokens_seen": 2835152896 + }, + { + "epoch": 0.72, + "learning_rate": 2.8446477290964535e-05, + "loss": 2.5055, + "theoretical_loss": 3.335293519146541, + "tokens_seen": 2835283968 + }, + { + "epoch": 0.72, + "learning_rate": 2.8438452896806295e-05, + "loss": 2.4753, + "theoretical_loss": 3.3352815281242068, + "tokens_seen": 2835415040 + }, + { + "epoch": 0.72, + "learning_rate": 2.8430428502648048e-05, + "loss": 2.4762, + "theoretical_loss": 3.335269537811363, + "tokens_seen": 2835546112 + }, + { + "epoch": 0.72, + "learning_rate": 2.8422404108489808e-05, + "loss": 2.5491, + "theoretical_loss": 3.335257548207935, + "tokens_seen": 2835677184 + }, + { + "epoch": 0.72, + "learning_rate": 2.8414379714331568e-05, + "loss": 2.5353, + "theoretical_loss": 3.335245559313848, + "tokens_seen": 2835808256 + }, + { + "epoch": 0.72, + "learning_rate": 2.8406355320173328e-05, + "loss": 2.4891, + "theoretical_loss": 3.3352335711290273, + "tokens_seen": 2835939328 + }, + { + "epoch": 0.72, + "learning_rate": 2.8398330926015088e-05, + "loss": 2.5988, + "theoretical_loss": 3.3352215836533983, + "tokens_seen": 2836070400 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1554852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6612417697906494, + "objective/train/theoretical_loss": 3.3352095968868856, + "objective/train/tokens_used": 1206660576, + "theoretical_loss": 3.3352095968868856, + "tokens_seen": 2836201472 + }, + { + "epoch": 0.72, + "learning_rate": 2.8390306531856847e-05, + "loss": 2.5803, + "theoretical_loss": 3.3352095968868856, + "tokens_seen": 2836201472 + }, + { + "epoch": 0.72, + "learning_rate": 2.8382282137698607e-05, + "loss": 2.7953, + "theoretical_loss": 3.3351976108294155, + "tokens_seen": 2836332544 + }, + { + "epoch": 0.72, + "learning_rate": 2.8374257743540367e-05, + "loss": 2.7288, + "theoretical_loss": 3.3351856254809125, + "tokens_seen": 2836463616 + }, + { + "epoch": 0.72, + "learning_rate": 2.836623334938212e-05, + "loss": 2.546, + "theoretical_loss": 3.3351736408413024, + "tokens_seen": 2836594688 + }, + { + "epoch": 0.72, + "learning_rate": 2.835820895522388e-05, + "loss": 2.5879, + "theoretical_loss": 3.3351616569105103, + "tokens_seen": 2836725760 + }, + { + "epoch": 0.72, + "learning_rate": 2.835018456106564e-05, + "loss": 2.4247, + "theoretical_loss": 3.3351496736884614, + "tokens_seen": 2836856832 + }, + { + "epoch": 0.72, + "learning_rate": 2.83421601669074e-05, + "loss": 2.4702, + "theoretical_loss": 3.335137691175081, + "tokens_seen": 2836987904 + }, + { + "epoch": 0.72, + "learning_rate": 2.833413577274916e-05, + "loss": 2.598, + "theoretical_loss": 3.335125709370295, + "tokens_seen": 2837118976 + }, + { + "epoch": 0.72, + "learning_rate": 2.832611137859092e-05, + "loss": 2.3395, + "theoretical_loss": 3.3351137282740284, + "tokens_seen": 2837250048 + }, + { + "epoch": 0.72, + "learning_rate": 2.831808698443268e-05, + "loss": 2.4792, + "theoretical_loss": 3.335101747886206, + "tokens_seen": 2837381120 + }, + { + "epoch": 0.72, + "learning_rate": 2.8310062590274433e-05, + "loss": 2.5302, + "theoretical_loss": 3.3350897682067546, + "tokens_seen": 2837512192 + }, + { + "epoch": 0.72, + "learning_rate": 2.8302038196116193e-05, + "loss": 2.6181, + "theoretical_loss": 3.3350777892355983, + "tokens_seen": 2837643264 + }, + { + "epoch": 0.72, + "learning_rate": 2.8294013801957953e-05, + "loss": 2.4896, + "theoretical_loss": 3.3350658109726634, + "tokens_seen": 2837774336 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1555900, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8633480072021484, + "objective/train/theoretical_loss": 3.335059822106755, + "objective/train/tokens_used": 1208298976, + "theoretical_loss": 3.335059822106755, + "tokens_seen": 2837839872 + }, + { + "epoch": 0.72, + "learning_rate": 2.8285989407799713e-05, + "loss": 2.5653, + "theoretical_loss": 3.3350538334178745, + "tokens_seen": 2837905408 + }, + { + "epoch": 0.72, + "learning_rate": 2.8277965013641473e-05, + "loss": 2.5218, + "theoretical_loss": 3.3350418565711575, + "tokens_seen": 2838036480 + }, + { + "epoch": 0.72, + "learning_rate": 2.8269940619483233e-05, + "loss": 2.3429, + "theoretical_loss": 3.3350298804324376, + "tokens_seen": 2838167552 + }, + { + "epoch": 0.72, + "learning_rate": 2.8261916225324992e-05, + "loss": 2.6138, + "theoretical_loss": 3.3350179050016404, + "tokens_seen": 2838298624 + }, + { + "epoch": 0.72, + "learning_rate": 2.8253891831166746e-05, + "loss": 2.4787, + "theoretical_loss": 3.3350059302786916, + "tokens_seen": 2838429696 + }, + { + "epoch": 0.72, + "learning_rate": 2.8245867437008505e-05, + "loss": 2.5838, + "theoretical_loss": 3.3349939562635162, + "tokens_seen": 2838560768 + }, + { + "epoch": 0.72, + "learning_rate": 2.8237843042850265e-05, + "loss": 2.5966, + "theoretical_loss": 3.33498198295604, + "tokens_seen": 2838691840 + }, + { + "epoch": 0.72, + "learning_rate": 2.8229818648692025e-05, + "loss": 2.5996, + "theoretical_loss": 3.3349700103561886, + "tokens_seen": 2838822912 + }, + { + "epoch": 0.72, + "learning_rate": 2.8221794254533785e-05, + "loss": 2.5997, + "theoretical_loss": 3.3349580384638875, + "tokens_seen": 2838953984 + }, + { + "epoch": 0.72, + "learning_rate": 2.8213769860375545e-05, + "loss": 2.5872, + "theoretical_loss": 3.334946067279062, + "tokens_seen": 2839085056 + }, + { + "epoch": 0.72, + "learning_rate": 2.8205745466217305e-05, + "loss": 2.6567, + "theoretical_loss": 3.3349340968016374, + "tokens_seen": 2839216128 + }, + { + "epoch": 0.72, + "learning_rate": 2.8197721072059065e-05, + "loss": 2.5247, + "theoretical_loss": 3.3349221270315397, + "tokens_seen": 2839347200 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1556560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2388436794281006, + "objective/train/theoretical_loss": 3.3349101579686944, + "objective/train/tokens_used": 1209937376, + "theoretical_loss": 3.3349101579686944, + "tokens_seen": 2839478272 + }, + { + "epoch": 0.72, + "learning_rate": 2.8189696677900818e-05, + "loss": 2.5183, + "theoretical_loss": 3.3349101579686944, + "tokens_seen": 2839478272 + }, + { + "epoch": 0.72, + "learning_rate": 2.8181672283742578e-05, + "loss": 2.5197, + "theoretical_loss": 3.334898189613027, + "tokens_seen": 2839609344 + }, + { + "epoch": 0.72, + "learning_rate": 2.8173647889584338e-05, + "loss": 2.4118, + "theoretical_loss": 3.334886221964463, + "tokens_seen": 2839740416 + }, + { + "epoch": 0.72, + "learning_rate": 2.8165623495426098e-05, + "loss": 2.4823, + "theoretical_loss": 3.334874255022928, + "tokens_seen": 2839871488 + }, + { + "epoch": 0.72, + "learning_rate": 2.8157599101267858e-05, + "loss": 2.4639, + "theoretical_loss": 3.3348622887883472, + "tokens_seen": 2840002560 + }, + { + "epoch": 0.72, + "learning_rate": 2.8149574707109618e-05, + "loss": 2.5711, + "theoretical_loss": 3.3348503232606475, + "tokens_seen": 2840133632 + }, + { + "epoch": 0.72, + "learning_rate": 2.8141550312951377e-05, + "loss": 2.4458, + "theoretical_loss": 3.334838358439753, + "tokens_seen": 2840264704 + }, + { + "epoch": 0.72, + "learning_rate": 2.813352591879313e-05, + "loss": 2.5102, + "theoretical_loss": 3.3348263943255905, + "tokens_seen": 2840395776 + }, + { + "epoch": 0.72, + "learning_rate": 2.812550152463489e-05, + "loss": 2.4841, + "theoretical_loss": 3.334814430918085, + "tokens_seen": 2840526848 + }, + { + "epoch": 0.72, + "learning_rate": 2.811747713047665e-05, + "loss": 2.5125, + "theoretical_loss": 3.334802468217162, + "tokens_seen": 2840657920 + }, + { + "epoch": 0.72, + "learning_rate": 2.810945273631841e-05, + "loss": 2.7811, + "theoretical_loss": 3.334790506222748, + "tokens_seen": 2840788992 + }, + { + "epoch": 0.72, + "learning_rate": 2.810142834216017e-05, + "loss": 2.5244, + "theoretical_loss": 3.3347785449347676, + "tokens_seen": 2840920064 + }, + { + "epoch": 0.72, + "learning_rate": 2.8093403948001927e-05, + "loss": 2.5685, + "theoretical_loss": 3.3347665843531473, + "tokens_seen": 2841051136 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1557653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2519896030426025, + "objective/train/theoretical_loss": 3.334760604327199, + "objective/train/tokens_used": 1211575776, + "theoretical_loss": 3.334760604327199, + "tokens_seen": 2841116672 + }, + { + "epoch": 0.72, + "learning_rate": 2.8085379553843687e-05, + "loss": 2.4631, + "theoretical_loss": 3.334754624477813, + "tokens_seen": 2841182208 + }, + { + "epoch": 0.72, + "learning_rate": 2.8077355159685447e-05, + "loss": 2.5316, + "theoretical_loss": 3.3347426653086893, + "tokens_seen": 2841313280 + }, + { + "epoch": 0.72, + "learning_rate": 2.8069330765527203e-05, + "loss": 2.4728, + "theoretical_loss": 3.334730706845703, + "tokens_seen": 2841444352 + }, + { + "epoch": 0.72, + "learning_rate": 2.8061306371368963e-05, + "loss": 2.5714, + "theoretical_loss": 3.334718749088779, + "tokens_seen": 2841575424 + }, + { + "epoch": 0.72, + "learning_rate": 2.8053281977210723e-05, + "loss": 2.5049, + "theoretical_loss": 3.334706792037844, + "tokens_seen": 2841706496 + }, + { + "epoch": 0.72, + "learning_rate": 2.804525758305248e-05, + "loss": 2.5302, + "theoretical_loss": 3.3346948356928228, + "tokens_seen": 2841837568 + }, + { + "epoch": 0.72, + "learning_rate": 2.803723318889424e-05, + "loss": 2.616, + "theoretical_loss": 3.334682880053642, + "tokens_seen": 2841968640 + }, + { + "epoch": 0.72, + "learning_rate": 2.8029208794736e-05, + "loss": 2.6402, + "theoretical_loss": 3.3346709251202267, + "tokens_seen": 2842099712 + }, + { + "epoch": 0.72, + "learning_rate": 2.802118440057776e-05, + "loss": 2.4817, + "theoretical_loss": 3.334658970892503, + "tokens_seen": 2842230784 + }, + { + "epoch": 0.72, + "learning_rate": 2.8013160006419516e-05, + "loss": 2.5685, + "theoretical_loss": 3.3346470173703966, + "tokens_seen": 2842361856 + }, + { + "epoch": 0.72, + "learning_rate": 2.8005135612261276e-05, + "loss": 2.5174, + "theoretical_loss": 3.3346350645538334, + "tokens_seen": 2842492928 + }, + { + "epoch": 0.72, + "learning_rate": 2.7997111218103032e-05, + "loss": 2.5316, + "theoretical_loss": 3.3346231124427392, + "tokens_seen": 2842624000 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1558297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5437302589416504, + "objective/train/theoretical_loss": 3.33461116103704, + "objective/train/tokens_used": 1213214176, + "theoretical_loss": 3.33461116103704, + "tokens_seen": 2842755072 + }, + { + "epoch": 0.72, + "learning_rate": 2.7989086823944792e-05, + "loss": 2.5199, + "theoretical_loss": 3.33461116103704, + "tokens_seen": 2842755072 + }, + { + "epoch": 0.72, + "learning_rate": 2.7981062429786552e-05, + "loss": 2.5567, + "theoretical_loss": 3.3345992103366613, + "tokens_seen": 2842886144 + }, + { + "epoch": 0.72, + "learning_rate": 2.7973038035628312e-05, + "loss": 2.5956, + "theoretical_loss": 3.334587260341529, + "tokens_seen": 2843017216 + }, + { + "epoch": 0.72, + "learning_rate": 2.796501364147007e-05, + "loss": 2.5821, + "theoretical_loss": 3.3345753110515695, + "tokens_seen": 2843148288 + }, + { + "epoch": 0.72, + "learning_rate": 2.7956989247311828e-05, + "loss": 2.3708, + "theoretical_loss": 3.3345633624667084, + "tokens_seen": 2843279360 + }, + { + "epoch": 0.72, + "learning_rate": 2.7948964853153585e-05, + "loss": 2.4114, + "theoretical_loss": 3.3345514145868713, + "tokens_seen": 2843410432 + }, + { + "epoch": 0.72, + "learning_rate": 2.7940940458995345e-05, + "loss": 2.3745, + "theoretical_loss": 3.334539467411984, + "tokens_seen": 2843541504 + }, + { + "epoch": 0.72, + "learning_rate": 2.7932916064837105e-05, + "loss": 2.5646, + "theoretical_loss": 3.3345275209419736, + "tokens_seen": 2843672576 + }, + { + "epoch": 0.72, + "learning_rate": 2.7924891670678864e-05, + "loss": 2.625, + "theoretical_loss": 3.334515575176764, + "tokens_seen": 2843803648 + }, + { + "epoch": 0.72, + "learning_rate": 2.7916867276520624e-05, + "loss": 2.4451, + "theoretical_loss": 3.334503630116283, + "tokens_seen": 2843934720 + }, + { + "epoch": 0.72, + "learning_rate": 2.7908842882362384e-05, + "loss": 2.3877, + "theoretical_loss": 3.334491685760456, + "tokens_seen": 2844065792 + }, + { + "epoch": 0.72, + "learning_rate": 2.7900818488204144e-05, + "loss": 2.4848, + "theoretical_loss": 3.3344797421092083, + "tokens_seen": 2844196864 + }, + { + "epoch": 0.72, + "learning_rate": 2.7892794094045897e-05, + "loss": 2.4033, + "theoretical_loss": 3.3344677991624665, + "tokens_seen": 2844327936 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1559506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6845366954803467, + "objective/train/theoretical_loss": 3.334461827953262, + "objective/train/tokens_used": 1214852576, + "theoretical_loss": 3.334461827953262, + "tokens_seen": 2844393472 + }, + { + "epoch": 0.72, + "learning_rate": 2.7884769699887657e-05, + "loss": 2.6387, + "theoretical_loss": 3.3344558569201563, + "tokens_seen": 2844459008 + }, + { + "epoch": 0.72, + "learning_rate": 2.7876745305729417e-05, + "loss": 2.5429, + "theoretical_loss": 3.334443915382204, + "tokens_seen": 2844590080 + }, + { + "epoch": 0.72, + "learning_rate": 2.7868720911571177e-05, + "loss": 2.5695, + "theoretical_loss": 3.3344319745485356, + "tokens_seen": 2844721152 + }, + { + "epoch": 0.72, + "learning_rate": 2.7860696517412937e-05, + "loss": 2.5009, + "theoretical_loss": 3.3344200344190766, + "tokens_seen": 2844852224 + }, + { + "epoch": 0.72, + "learning_rate": 2.7852672123254697e-05, + "loss": 2.4684, + "theoretical_loss": 3.3344080949937536, + "tokens_seen": 2844983296 + }, + { + "epoch": 0.72, + "learning_rate": 2.7844647729096457e-05, + "loss": 2.5834, + "theoretical_loss": 3.3343961562724926, + "tokens_seen": 2845114368 + }, + { + "epoch": 0.72, + "learning_rate": 2.783662333493821e-05, + "loss": 2.3993, + "theoretical_loss": 3.3343842182552192, + "tokens_seen": 2845245440 + }, + { + "epoch": 0.72, + "learning_rate": 2.782859894077997e-05, + "loss": 2.4374, + "theoretical_loss": 3.3343722809418597, + "tokens_seen": 2845376512 + }, + { + "epoch": 0.72, + "learning_rate": 2.782057454662173e-05, + "loss": 2.5798, + "theoretical_loss": 3.33436034433234, + "tokens_seen": 2845507584 + }, + { + "epoch": 0.72, + "learning_rate": 2.781255015246349e-05, + "loss": 2.6322, + "theoretical_loss": 3.3343484084265866, + "tokens_seen": 2845638656 + }, + { + "epoch": 0.72, + "learning_rate": 2.780452575830525e-05, + "loss": 2.7562, + "theoretical_loss": 3.3343364732245258, + "tokens_seen": 2845769728 + }, + { + "epoch": 0.72, + "learning_rate": 2.779650136414701e-05, + "loss": 2.5461, + "theoretical_loss": 3.334324538726083, + "tokens_seen": 2845900800 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1560188, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6349122524261475, + "objective/train/theoretical_loss": 3.3343126049311844, + "objective/train/tokens_used": 1216490976, + "theoretical_loss": 3.3343126049311844, + "tokens_seen": 2846031872 + }, + { + "epoch": 0.72, + "learning_rate": 2.778847696998877e-05, + "loss": 2.6005, + "theoretical_loss": 3.3343126049311844, + "tokens_seen": 2846031872 + }, + { + "epoch": 0.72, + "learning_rate": 2.7780452575830522e-05, + "loss": 2.5172, + "theoretical_loss": 3.334300671839756, + "tokens_seen": 2846162944 + }, + { + "epoch": 0.73, + "learning_rate": 2.7772428181672282e-05, + "loss": 2.4359, + "theoretical_loss": 3.334288739451725, + "tokens_seen": 2846294016 + }, + { + "epoch": 0.73, + "learning_rate": 2.7764403787514042e-05, + "loss": 2.6681, + "theoretical_loss": 3.334276807767017, + "tokens_seen": 2846425088 + }, + { + "epoch": 0.73, + "learning_rate": 2.7756379393355802e-05, + "loss": 2.4633, + "theoretical_loss": 3.334264876785557, + "tokens_seen": 2846556160 + }, + { + "epoch": 0.73, + "learning_rate": 2.7748354999197562e-05, + "loss": 2.4452, + "theoretical_loss": 3.3342529465072728, + "tokens_seen": 2846687232 + }, + { + "epoch": 0.73, + "learning_rate": 2.7740330605039322e-05, + "loss": 2.364, + "theoretical_loss": 3.3342410169320895, + "tokens_seen": 2846818304 + }, + { + "epoch": 0.73, + "learning_rate": 2.7732306210881082e-05, + "loss": 2.4245, + "theoretical_loss": 3.334229088059934, + "tokens_seen": 2846949376 + }, + { + "epoch": 0.73, + "learning_rate": 2.7724281816722842e-05, + "loss": 2.4924, + "theoretical_loss": 3.3342171598907324, + "tokens_seen": 2847080448 + }, + { + "epoch": 0.73, + "learning_rate": 2.7716257422564595e-05, + "loss": 2.616, + "theoretical_loss": 3.3342052324244107, + "tokens_seen": 2847211520 + }, + { + "epoch": 0.73, + "learning_rate": 2.7708233028406355e-05, + "loss": 2.5437, + "theoretical_loss": 3.334193305660895, + "tokens_seen": 2847342592 + }, + { + "epoch": 0.73, + "learning_rate": 2.7700208634248115e-05, + "loss": 2.3779, + "theoretical_loss": 3.3341813796001114, + "tokens_seen": 2847473664 + }, + { + "epoch": 0.73, + "learning_rate": 2.7692184240089875e-05, + "loss": 2.5432, + "theoretical_loss": 3.3341694542419873, + "tokens_seen": 2847604736 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1561382, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.027073860168457, + "objective/train/theoretical_loss": 3.3341634918263985, + "objective/train/tokens_used": 1218129376, + "theoretical_loss": 3.3341634918263985, + "tokens_seen": 2847670272 + }, + { + "epoch": 0.73, + "learning_rate": 2.7684159845931635e-05, + "loss": 2.4138, + "theoretical_loss": 3.3341575295864474, + "tokens_seen": 2847735808 + }, + { + "epoch": 0.73, + "learning_rate": 2.7676135451773394e-05, + "loss": 2.6646, + "theoretical_loss": 3.334145605633419, + "tokens_seen": 2847866880 + }, + { + "epoch": 0.73, + "learning_rate": 2.7668111057615154e-05, + "loss": 2.6212, + "theoretical_loss": 3.334133682382828, + "tokens_seen": 2847997952 + }, + { + "epoch": 0.73, + "learning_rate": 2.7660086663456907e-05, + "loss": 2.5608, + "theoretical_loss": 3.3341217598346007, + "tokens_seen": 2848129024 + }, + { + "epoch": 0.73, + "learning_rate": 2.7652062269298667e-05, + "loss": 2.6368, + "theoretical_loss": 3.3341098379886636, + "tokens_seen": 2848260096 + }, + { + "epoch": 0.73, + "learning_rate": 2.7644037875140427e-05, + "loss": 2.5323, + "theoretical_loss": 3.334097916844943, + "tokens_seen": 2848391168 + }, + { + "epoch": 0.73, + "learning_rate": 2.7636013480982187e-05, + "loss": 2.6112, + "theoretical_loss": 3.3340859964033647, + "tokens_seen": 2848522240 + }, + { + "epoch": 0.73, + "learning_rate": 2.7627989086823947e-05, + "loss": 2.4983, + "theoretical_loss": 3.334074076663856, + "tokens_seen": 2848653312 + }, + { + "epoch": 0.73, + "learning_rate": 2.7619964692665707e-05, + "loss": 2.5251, + "theoretical_loss": 3.3340621576263425, + "tokens_seen": 2848784384 + }, + { + "epoch": 0.73, + "learning_rate": 2.7611940298507467e-05, + "loss": 2.3739, + "theoretical_loss": 3.3340502392907503, + "tokens_seen": 2848915456 + }, + { + "epoch": 0.73, + "learning_rate": 2.7603915904349227e-05, + "loss": 2.6097, + "theoretical_loss": 3.334038321657007, + "tokens_seen": 2849046528 + }, + { + "epoch": 0.73, + "learning_rate": 2.759589151019098e-05, + "loss": 2.4595, + "theoretical_loss": 3.334026404725038, + "tokens_seen": 2849177600 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1562134, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6647844314575195, + "objective/train/theoretical_loss": 3.33401448849477, + "objective/train/tokens_used": 1219767776, + "theoretical_loss": 3.33401448849477, + "tokens_seen": 2849308672 + }, + { + "epoch": 0.73, + "learning_rate": 2.758786711603274e-05, + "loss": 2.5983, + "theoretical_loss": 3.33401448849477, + "tokens_seen": 2849308672 + }, + { + "epoch": 0.73, + "learning_rate": 2.75798427218745e-05, + "loss": 2.4583, + "theoretical_loss": 3.3340025729661287, + "tokens_seen": 2849439744 + }, + { + "epoch": 0.73, + "learning_rate": 2.757181832771626e-05, + "loss": 2.5359, + "theoretical_loss": 3.333990658139042, + "tokens_seen": 2849570816 + }, + { + "epoch": 0.73, + "learning_rate": 2.756379393355802e-05, + "loss": 2.3428, + "theoretical_loss": 3.333978744013435, + "tokens_seen": 2849701888 + }, + { + "epoch": 0.73, + "learning_rate": 2.755576953939978e-05, + "loss": 2.5183, + "theoretical_loss": 3.3339668305892345, + "tokens_seen": 2849832960 + }, + { + "epoch": 0.73, + "learning_rate": 2.754774514524154e-05, + "loss": 2.5226, + "theoretical_loss": 3.333954917866367, + "tokens_seen": 2849964032 + }, + { + "epoch": 0.73, + "learning_rate": 2.7539720751083292e-05, + "loss": 2.4743, + "theoretical_loss": 3.3339430058447594, + "tokens_seen": 2850095104 + }, + { + "epoch": 0.73, + "learning_rate": 2.7531696356925052e-05, + "loss": 2.4899, + "theoretical_loss": 3.333931094524338, + "tokens_seen": 2850226176 + }, + { + "epoch": 0.73, + "learning_rate": 2.7523671962766812e-05, + "loss": 2.4979, + "theoretical_loss": 3.333919183905029, + "tokens_seen": 2850357248 + }, + { + "epoch": 0.73, + "learning_rate": 2.7515647568608572e-05, + "loss": 2.6242, + "theoretical_loss": 3.3339072739867586, + "tokens_seen": 2850488320 + }, + { + "epoch": 0.73, + "learning_rate": 2.7507623174450332e-05, + "loss": 2.5262, + "theoretical_loss": 3.333895364769454, + "tokens_seen": 2850619392 + }, + { + "epoch": 0.73, + "learning_rate": 2.7499598780292092e-05, + "loss": 2.5585, + "theoretical_loss": 3.333883456253041, + "tokens_seen": 2850750464 + }, + { + "epoch": 0.73, + "learning_rate": 2.7491574386133852e-05, + "loss": 2.4632, + "theoretical_loss": 3.333871548437447, + "tokens_seen": 2850881536 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1563534, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.207017183303833, + "objective/train/theoretical_loss": 3.3338655947924334, + "objective/train/tokens_used": 1221406176, + "theoretical_loss": 3.3338655947924334, + "tokens_seen": 2850947072 + }, + { + "epoch": 0.73, + "learning_rate": 2.7483549991975605e-05, + "loss": 2.4203, + "theoretical_loss": 3.333859641322597, + "tokens_seen": 2851012608 + }, + { + "epoch": 0.73, + "learning_rate": 2.7475525597817365e-05, + "loss": 2.4526, + "theoretical_loss": 3.33384773490842, + "tokens_seen": 2851143680 + }, + { + "epoch": 0.73, + "learning_rate": 2.7467501203659125e-05, + "loss": 2.6056, + "theoretical_loss": 3.3338358291948404, + "tokens_seen": 2851274752 + }, + { + "epoch": 0.73, + "learning_rate": 2.7459476809500885e-05, + "loss": 2.5653, + "theoretical_loss": 3.3338239241817855, + "tokens_seen": 2851405824 + }, + { + "epoch": 0.73, + "learning_rate": 2.7451452415342645e-05, + "loss": 2.5201, + "theoretical_loss": 3.3338120198691823, + "tokens_seen": 2851536896 + }, + { + "epoch": 0.73, + "learning_rate": 2.7443428021184405e-05, + "loss": 2.4608, + "theoretical_loss": 3.333800116256957, + "tokens_seen": 2851667968 + }, + { + "epoch": 0.73, + "learning_rate": 2.743540362702616e-05, + "loss": 2.6562, + "theoretical_loss": 3.333788213345036, + "tokens_seen": 2851799040 + }, + { + "epoch": 0.73, + "learning_rate": 2.742737923286792e-05, + "loss": 2.3807, + "theoretical_loss": 3.333776311133346, + "tokens_seen": 2851930112 + }, + { + "epoch": 0.73, + "learning_rate": 2.7419354838709678e-05, + "loss": 2.2986, + "theoretical_loss": 3.333764409621814, + "tokens_seen": 2852061184 + }, + { + "epoch": 0.73, + "learning_rate": 2.7411330444551437e-05, + "loss": 2.5259, + "theoretical_loss": 3.3337525088103668, + "tokens_seen": 2852192256 + }, + { + "epoch": 0.73, + "learning_rate": 2.7403306050393197e-05, + "loss": 2.5677, + "theoretical_loss": 3.33374060869893, + "tokens_seen": 2852323328 + }, + { + "epoch": 0.73, + "learning_rate": 2.7395281656234957e-05, + "loss": 2.5207, + "theoretical_loss": 3.3337287092874313, + "tokens_seen": 2852454400 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1564244, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.576831340789795, + "objective/train/theoretical_loss": 3.333716810575797, + "objective/train/tokens_used": 1223044576, + "theoretical_loss": 3.333716810575797, + "tokens_seen": 2852585472 + }, + { + "epoch": 0.73, + "learning_rate": 2.7387257262076714e-05, + "loss": 2.4139, + "theoretical_loss": 3.333716810575797, + "tokens_seen": 2852585472 + }, + { + "epoch": 0.73, + "learning_rate": 2.7379232867918474e-05, + "loss": 2.5057, + "theoretical_loss": 3.3337049125639537, + "tokens_seen": 2852716544 + }, + { + "epoch": 0.73, + "learning_rate": 2.7371208473760234e-05, + "loss": 2.5661, + "theoretical_loss": 3.333693015251828, + "tokens_seen": 2852847616 + }, + { + "epoch": 0.73, + "learning_rate": 2.736318407960199e-05, + "loss": 2.6085, + "theoretical_loss": 3.333681118639347, + "tokens_seen": 2852978688 + }, + { + "epoch": 0.73, + "learning_rate": 2.735515968544375e-05, + "loss": 2.4476, + "theoretical_loss": 3.3336692227264373, + "tokens_seen": 2853109760 + }, + { + "epoch": 0.73, + "learning_rate": 2.734713529128551e-05, + "loss": 2.3648, + "theoretical_loss": 3.3336573275130252, + "tokens_seen": 2853240832 + }, + { + "epoch": 0.73, + "learning_rate": 2.7339110897127266e-05, + "loss": 2.5255, + "theoretical_loss": 3.3336454329990377, + "tokens_seen": 2853371904 + }, + { + "epoch": 0.73, + "learning_rate": 2.7331086502969026e-05, + "loss": 2.5227, + "theoretical_loss": 3.333633539184402, + "tokens_seen": 2853502976 + }, + { + "epoch": 0.73, + "learning_rate": 2.7323062108810786e-05, + "loss": 2.4498, + "theoretical_loss": 3.333621646069044, + "tokens_seen": 2853634048 + }, + { + "epoch": 0.73, + "learning_rate": 2.7315037714652546e-05, + "loss": 2.42, + "theoretical_loss": 3.3336097536528912, + "tokens_seen": 2853765120 + }, + { + "epoch": 0.73, + "learning_rate": 2.7307013320494303e-05, + "loss": 2.4954, + "theoretical_loss": 3.3335978619358704, + "tokens_seen": 2853896192 + }, + { + "epoch": 0.73, + "learning_rate": 2.7298988926336063e-05, + "loss": 2.2811, + "theoretical_loss": 3.3335859709179076, + "tokens_seen": 2854027264 + }, + { + "epoch": 0.73, + "learning_rate": 2.729096453217782e-05, + "loss": 2.5083, + "theoretical_loss": 3.3335740805989302, + "tokens_seen": 2854158336 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1565503, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.558499336242676, + "objective/train/theoretical_loss": 3.333568135701538, + "objective/train/tokens_used": 1224682976, + "theoretical_loss": 3.333568135701538, + "tokens_seen": 2854223872 + }, + { + "epoch": 0.73, + "learning_rate": 2.728294013801958e-05, + "loss": 2.5039, + "theoretical_loss": 3.3335621909788653, + "tokens_seen": 2854289408 + }, + { + "epoch": 0.73, + "learning_rate": 2.727491574386134e-05, + "loss": 2.6232, + "theoretical_loss": 3.3335503020576387, + "tokens_seen": 2854420480 + }, + { + "epoch": 0.73, + "learning_rate": 2.72668913497031e-05, + "loss": 2.3615, + "theoretical_loss": 3.3335384138351785, + "tokens_seen": 2854551552 + }, + { + "epoch": 0.73, + "learning_rate": 2.725886695554486e-05, + "loss": 2.5467, + "theoretical_loss": 3.3335265263114104, + "tokens_seen": 2854682624 + }, + { + "epoch": 0.73, + "learning_rate": 2.725084256138662e-05, + "loss": 2.5787, + "theoretical_loss": 3.3335146394862623, + "tokens_seen": 2854813696 + }, + { + "epoch": 0.73, + "learning_rate": 2.7242818167228372e-05, + "loss": 2.5972, + "theoretical_loss": 3.33350275335966, + "tokens_seen": 2854944768 + }, + { + "epoch": 0.73, + "learning_rate": 2.723479377307013e-05, + "loss": 2.5292, + "theoretical_loss": 3.333490867931532, + "tokens_seen": 2855075840 + }, + { + "epoch": 0.73, + "learning_rate": 2.722676937891189e-05, + "loss": 2.2502, + "theoretical_loss": 3.333478983201803, + "tokens_seen": 2855206912 + }, + { + "epoch": 0.73, + "learning_rate": 2.721874498475365e-05, + "loss": 2.5668, + "theoretical_loss": 3.3334670991704014, + "tokens_seen": 2855337984 + }, + { + "epoch": 0.73, + "learning_rate": 2.721072059059541e-05, + "loss": 2.4293, + "theoretical_loss": 3.333455215837254, + "tokens_seen": 2855469056 + }, + { + "epoch": 0.73, + "learning_rate": 2.720269619643717e-05, + "loss": 2.5271, + "theoretical_loss": 3.333443333202287, + "tokens_seen": 2855600128 + }, + { + "epoch": 0.73, + "learning_rate": 2.719467180227893e-05, + "loss": 2.4124, + "theoretical_loss": 3.3334314512654286, + "tokens_seen": 2855731200 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1566517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3041694164276123, + "objective/train/theoretical_loss": 3.3334195700266043, + "objective/train/tokens_used": 1226321376, + "theoretical_loss": 3.3334195700266043, + "tokens_seen": 2855862272 + }, + { + "epoch": 0.73, + "learning_rate": 2.7186647408120684e-05, + "loss": 2.5483, + "theoretical_loss": 3.3334195700266043, + "tokens_seen": 2855862272 + }, + { + "epoch": 0.73, + "learning_rate": 2.7178623013962444e-05, + "loss": 2.5432, + "theoretical_loss": 3.333407689485742, + "tokens_seen": 2855993344 + }, + { + "epoch": 0.73, + "learning_rate": 2.7170598619804204e-05, + "loss": 2.5794, + "theoretical_loss": 3.3333958096427683, + "tokens_seen": 2856124416 + }, + { + "epoch": 0.73, + "learning_rate": 2.7162574225645964e-05, + "loss": 2.3993, + "theoretical_loss": 3.33338393049761, + "tokens_seen": 2856255488 + }, + { + "epoch": 0.73, + "learning_rate": 2.7154549831487724e-05, + "loss": 2.4666, + "theoretical_loss": 3.333372052050195, + "tokens_seen": 2856386560 + }, + { + "epoch": 0.73, + "learning_rate": 2.7146525437329484e-05, + "loss": 2.4215, + "theoretical_loss": 3.3333601743004495, + "tokens_seen": 2856517632 + }, + { + "epoch": 0.73, + "learning_rate": 2.7138501043171244e-05, + "loss": 2.5435, + "theoretical_loss": 3.3333482972483, + "tokens_seen": 2856648704 + }, + { + "epoch": 0.73, + "learning_rate": 2.7130476649013004e-05, + "loss": 2.4497, + "theoretical_loss": 3.333336420893675, + "tokens_seen": 2856779776 + }, + { + "epoch": 0.73, + "learning_rate": 2.7122452254854757e-05, + "loss": 2.4225, + "theoretical_loss": 3.3333245452365006, + "tokens_seen": 2856910848 + }, + { + "epoch": 0.73, + "learning_rate": 2.7114427860696517e-05, + "loss": 2.4123, + "theoretical_loss": 3.3333126702767037, + "tokens_seen": 2857041920 + }, + { + "epoch": 0.73, + "learning_rate": 2.7106403466538277e-05, + "loss": 2.5359, + "theoretical_loss": 3.333300796014212, + "tokens_seen": 2857172992 + }, + { + "epoch": 0.73, + "learning_rate": 2.7098379072380037e-05, + "loss": 2.4594, + "theoretical_loss": 3.3332889224489515, + "tokens_seen": 2857304064 + }, + { + "epoch": 0.73, + "learning_rate": 2.7090354678221796e-05, + "loss": 2.5669, + "theoretical_loss": 3.333277049580851, + "tokens_seen": 2857435136 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1567257, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.916217565536499, + "objective/train/theoretical_loss": 3.333271113408212, + "objective/train/tokens_used": 1227959776, + "theoretical_loss": 3.333271113408212, + "tokens_seen": 2857500672 + }, + { + "epoch": 0.73, + "learning_rate": 2.7082330284063556e-05, + "loss": 2.434, + "theoretical_loss": 3.3332651774098356, + "tokens_seen": 2857566208 + }, + { + "epoch": 0.73, + "learning_rate": 2.7074305889905316e-05, + "loss": 2.6498, + "theoretical_loss": 3.333253305935834, + "tokens_seen": 2857697280 + }, + { + "epoch": 0.73, + "learning_rate": 2.706628149574707e-05, + "loss": 2.5622, + "theoretical_loss": 3.3332414351587727, + "tokens_seen": 2857828352 + }, + { + "epoch": 0.73, + "learning_rate": 2.705825710158883e-05, + "loss": 2.5632, + "theoretical_loss": 3.3332295650785784, + "tokens_seen": 2857959424 + }, + { + "epoch": 0.73, + "learning_rate": 2.705023270743059e-05, + "loss": 2.5541, + "theoretical_loss": 3.333217695695179, + "tokens_seen": 2858090496 + }, + { + "epoch": 0.73, + "learning_rate": 2.704220831327235e-05, + "loss": 2.5119, + "theoretical_loss": 3.333205827008501, + "tokens_seen": 2858221568 + }, + { + "epoch": 0.73, + "learning_rate": 2.703418391911411e-05, + "loss": 2.6879, + "theoretical_loss": 3.3331939590184723, + "tokens_seen": 2858352640 + }, + { + "epoch": 0.73, + "learning_rate": 2.702615952495587e-05, + "loss": 2.3777, + "theoretical_loss": 3.333182091725019, + "tokens_seen": 2858483712 + }, + { + "epoch": 0.73, + "learning_rate": 2.701813513079763e-05, + "loss": 2.4396, + "theoretical_loss": 3.333170225128069, + "tokens_seen": 2858614784 + }, + { + "epoch": 0.73, + "learning_rate": 2.7010110736639382e-05, + "loss": 2.477, + "theoretical_loss": 3.33315835922755, + "tokens_seen": 2858745856 + }, + { + "epoch": 0.73, + "learning_rate": 2.7002086342481142e-05, + "loss": 2.3739, + "theoretical_loss": 3.333146494023388, + "tokens_seen": 2858876928 + }, + { + "epoch": 0.73, + "learning_rate": 2.6994061948322902e-05, + "loss": 2.597, + "theoretical_loss": 3.333134629515511, + "tokens_seen": 2859008000 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1568436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2539758682250977, + "objective/train/theoretical_loss": 3.333122765703846, + "objective/train/tokens_used": 1229598176, + "theoretical_loss": 3.333122765703846, + "tokens_seen": 2859139072 + }, + { + "epoch": 0.73, + "learning_rate": 2.698603755416466e-05, + "loss": 2.464, + "theoretical_loss": 3.333122765703846, + "tokens_seen": 2859139072 + }, + { + "epoch": 0.73, + "learning_rate": 2.697801316000642e-05, + "loss": 2.7074, + "theoretical_loss": 3.33311090258832, + "tokens_seen": 2859270144 + }, + { + "epoch": 0.73, + "learning_rate": 2.696998876584818e-05, + "loss": 2.6594, + "theoretical_loss": 3.333099040168861, + "tokens_seen": 2859401216 + }, + { + "epoch": 0.73, + "learning_rate": 2.696196437168994e-05, + "loss": 2.3883, + "theoretical_loss": 3.3330871784453953, + "tokens_seen": 2859532288 + }, + { + "epoch": 0.73, + "learning_rate": 2.69539399775317e-05, + "loss": 2.4671, + "theoretical_loss": 3.333075317417851, + "tokens_seen": 2859663360 + }, + { + "epoch": 0.73, + "learning_rate": 2.6945915583373454e-05, + "loss": 2.5927, + "theoretical_loss": 3.333063457086155, + "tokens_seen": 2859794432 + }, + { + "epoch": 0.73, + "learning_rate": 2.6937891189215214e-05, + "loss": 2.5472, + "theoretical_loss": 3.333051597450234, + "tokens_seen": 2859925504 + }, + { + "epoch": 0.73, + "learning_rate": 2.6929866795056974e-05, + "loss": 2.4657, + "theoretical_loss": 3.333039738510016, + "tokens_seen": 2860056576 + }, + { + "epoch": 0.73, + "learning_rate": 2.6921842400898734e-05, + "loss": 2.448, + "theoretical_loss": 3.3330278802654285, + "tokens_seen": 2860187648 + }, + { + "epoch": 0.73, + "learning_rate": 2.6913818006740494e-05, + "loss": 2.5272, + "theoretical_loss": 3.3330160227163983, + "tokens_seen": 2860318720 + }, + { + "epoch": 0.73, + "learning_rate": 2.6905793612582254e-05, + "loss": 2.601, + "theoretical_loss": 3.333004165862853, + "tokens_seen": 2860449792 + }, + { + "epoch": 0.73, + "learning_rate": 2.6897769218424014e-05, + "loss": 2.5212, + "theoretical_loss": 3.3329923097047196, + "tokens_seen": 2860580864 + }, + { + "epoch": 0.73, + "learning_rate": 2.6889744824265767e-05, + "loss": 2.4972, + "theoretical_loss": 3.3329804542419263, + "tokens_seen": 2860711936 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1568975, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8592374324798584, + "objective/train/theoretical_loss": 3.332974526771259, + "objective/train/tokens_used": 1231236576, + "theoretical_loss": 3.332974526771259, + "tokens_seen": 2860777472 + }, + { + "epoch": 0.73, + "learning_rate": 2.6881720430107527e-05, + "loss": 2.6253, + "theoretical_loss": 3.3329685994743996, + "tokens_seen": 2860843008 + }, + { + "epoch": 0.73, + "learning_rate": 2.6873696035949287e-05, + "loss": 2.5361, + "theoretical_loss": 3.332956745402067, + "tokens_seen": 2860974080 + }, + { + "epoch": 0.73, + "learning_rate": 2.6865671641791047e-05, + "loss": 2.6042, + "theoretical_loss": 3.332944892024856, + "tokens_seen": 2861105152 + }, + { + "epoch": 0.73, + "learning_rate": 2.6857647247632807e-05, + "loss": 2.4368, + "theoretical_loss": 3.3329330393426946, + "tokens_seen": 2861236224 + }, + { + "epoch": 0.73, + "learning_rate": 2.6849622853474566e-05, + "loss": 2.5555, + "theoretical_loss": 3.332921187355509, + "tokens_seen": 2861367296 + }, + { + "epoch": 0.73, + "learning_rate": 2.6841598459316326e-05, + "loss": 2.5294, + "theoretical_loss": 3.332909336063228, + "tokens_seen": 2861498368 + }, + { + "epoch": 0.73, + "learning_rate": 2.683357406515808e-05, + "loss": 2.5727, + "theoretical_loss": 3.3328974854657774, + "tokens_seen": 2861629440 + }, + { + "epoch": 0.73, + "learning_rate": 2.682554967099984e-05, + "loss": 2.4163, + "theoretical_loss": 3.332885635563086, + "tokens_seen": 2861760512 + }, + { + "epoch": 0.73, + "learning_rate": 2.68175252768416e-05, + "loss": 2.5375, + "theoretical_loss": 3.332873786355081, + "tokens_seen": 2861891584 + }, + { + "epoch": 0.73, + "learning_rate": 2.680950088268336e-05, + "loss": 2.5582, + "theoretical_loss": 3.332861937841689, + "tokens_seen": 2862022656 + }, + { + "epoch": 0.73, + "learning_rate": 2.680147648852512e-05, + "loss": 2.6093, + "theoretical_loss": 3.3328500900228386, + "tokens_seen": 2862153728 + }, + { + "epoch": 0.73, + "learning_rate": 2.679345209436688e-05, + "loss": 2.6518, + "theoretical_loss": 3.3328382428984566, + "tokens_seen": 2862284800 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1570126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.868828296661377, + "objective/train/theoretical_loss": 3.3328263964684712, + "objective/train/tokens_used": 1232874976, + "theoretical_loss": 3.3328263964684712, + "tokens_seen": 2862415872 + }, + { + "epoch": 0.73, + "learning_rate": 2.678542770020864e-05, + "loss": 2.6085, + "theoretical_loss": 3.3328263964684712, + "tokens_seen": 2862415872 + }, + { + "epoch": 0.73, + "learning_rate": 2.6777403306050395e-05, + "loss": 2.5142, + "theoretical_loss": 3.3328145507328086, + "tokens_seen": 2862546944 + }, + { + "epoch": 0.73, + "learning_rate": 2.6769378911892152e-05, + "loss": 2.487, + "theoretical_loss": 3.332802705691398, + "tokens_seen": 2862678016 + }, + { + "epoch": 0.74, + "learning_rate": 2.6761354517733912e-05, + "loss": 2.5374, + "theoretical_loss": 3.3327908613441655, + "tokens_seen": 2862809088 + }, + { + "epoch": 0.74, + "learning_rate": 2.6753330123575672e-05, + "loss": 2.3176, + "theoretical_loss": 3.3327790176910392, + "tokens_seen": 2862940160 + }, + { + "epoch": 0.74, + "learning_rate": 2.6745305729417432e-05, + "loss": 2.4174, + "theoretical_loss": 3.3327671747319467, + "tokens_seen": 2863071232 + }, + { + "epoch": 0.74, + "learning_rate": 2.673728133525919e-05, + "loss": 2.546, + "theoretical_loss": 3.3327553324668155, + "tokens_seen": 2863202304 + }, + { + "epoch": 0.74, + "learning_rate": 2.6729256941100948e-05, + "loss": 2.4666, + "theoretical_loss": 3.332743490895573, + "tokens_seen": 2863333376 + }, + { + "epoch": 0.74, + "learning_rate": 2.6721232546942708e-05, + "loss": 2.393, + "theoretical_loss": 3.332731650018147, + "tokens_seen": 2863464448 + }, + { + "epoch": 0.74, + "learning_rate": 2.6713208152784465e-05, + "loss": 2.4734, + "theoretical_loss": 3.3327198098344653, + "tokens_seen": 2863595520 + }, + { + "epoch": 0.74, + "learning_rate": 2.6705183758626224e-05, + "loss": 2.5006, + "theoretical_loss": 3.332707970344455, + "tokens_seen": 2863726592 + }, + { + "epoch": 0.74, + "learning_rate": 2.6697159364467984e-05, + "loss": 2.5084, + "theoretical_loss": 3.3326961315480443, + "tokens_seen": 2863857664 + }, + { + "epoch": 0.74, + "learning_rate": 2.6689134970309744e-05, + "loss": 2.6834, + "theoretical_loss": 3.33268429344516, + "tokens_seen": 2863988736 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1570761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8785791397094727, + "objective/train/theoretical_loss": 3.332678374653768, + "objective/train/tokens_used": 1234513376, + "theoretical_loss": 3.332678374653768, + "tokens_seen": 2864054272 + }, + { + "epoch": 0.74, + "learning_rate": 2.66811105761515e-05, + "loss": 2.4869, + "theoretical_loss": 3.3326724560357306, + "tokens_seen": 2864119808 + }, + { + "epoch": 0.74, + "learning_rate": 2.667308618199326e-05, + "loss": 2.497, + "theoretical_loss": 3.3326606193196833, + "tokens_seen": 2864250880 + }, + { + "epoch": 0.74, + "learning_rate": 2.666506178783502e-05, + "loss": 2.5159, + "theoretical_loss": 3.3326487832969454, + "tokens_seen": 2864381952 + }, + { + "epoch": 0.74, + "learning_rate": 2.665703739367678e-05, + "loss": 2.3481, + "theoretical_loss": 3.3326369479674454, + "tokens_seen": 2864513024 + }, + { + "epoch": 0.74, + "learning_rate": 2.6649012999518537e-05, + "loss": 2.4118, + "theoretical_loss": 3.3326251133311104, + "tokens_seen": 2864644096 + }, + { + "epoch": 0.74, + "learning_rate": 2.6640988605360297e-05, + "loss": 2.4629, + "theoretical_loss": 3.332613279387868, + "tokens_seen": 2864775168 + }, + { + "epoch": 0.74, + "learning_rate": 2.6632964211202053e-05, + "loss": 2.4658, + "theoretical_loss": 3.3326014461376468, + "tokens_seen": 2864906240 + }, + { + "epoch": 0.74, + "learning_rate": 2.6624939817043813e-05, + "loss": 2.4515, + "theoretical_loss": 3.3325896135803736, + "tokens_seen": 2865037312 + }, + { + "epoch": 0.74, + "learning_rate": 2.6616915422885573e-05, + "loss": 2.4715, + "theoretical_loss": 3.332577781715976, + "tokens_seen": 2865168384 + }, + { + "epoch": 0.74, + "learning_rate": 2.6608891028727333e-05, + "loss": 2.3861, + "theoretical_loss": 3.3325659505443825, + "tokens_seen": 2865299456 + }, + { + "epoch": 0.74, + "learning_rate": 2.6600866634569093e-05, + "loss": 2.5326, + "theoretical_loss": 3.3325541200655207, + "tokens_seen": 2865430528 + }, + { + "epoch": 0.74, + "learning_rate": 2.659284224041085e-05, + "loss": 2.5623, + "theoretical_loss": 3.332542290279318, + "tokens_seen": 2865561600 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1571930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5435543060302734, + "objective/train/theoretical_loss": 3.3325304611857023, + "objective/train/tokens_used": 1236151776, + "theoretical_loss": 3.3325304611857023, + "tokens_seen": 2865692672 + }, + { + "epoch": 0.74, + "learning_rate": 2.6584817846252606e-05, + "loss": 2.4993, + "theoretical_loss": 3.3325304611857023, + "tokens_seen": 2865692672 + }, + { + "epoch": 0.74, + "learning_rate": 2.6576793452094366e-05, + "loss": 2.5309, + "theoretical_loss": 3.332518632784601, + "tokens_seen": 2865823744 + }, + { + "epoch": 0.74, + "learning_rate": 2.6568769057936126e-05, + "loss": 2.5003, + "theoretical_loss": 3.3325068050759423, + "tokens_seen": 2865954816 + }, + { + "epoch": 0.74, + "learning_rate": 2.6560744663777886e-05, + "loss": 2.5074, + "theoretical_loss": 3.3324949780596542, + "tokens_seen": 2866085888 + }, + { + "epoch": 0.74, + "learning_rate": 2.6552720269619646e-05, + "loss": 2.4174, + "theoretical_loss": 3.3324831517356643, + "tokens_seen": 2866216960 + }, + { + "epoch": 0.74, + "learning_rate": 2.6544695875461406e-05, + "loss": 2.4669, + "theoretical_loss": 3.3324713261039003, + "tokens_seen": 2866348032 + }, + { + "epoch": 0.74, + "learning_rate": 2.653667148130316e-05, + "loss": 2.5586, + "theoretical_loss": 3.33245950116429, + "tokens_seen": 2866479104 + }, + { + "epoch": 0.74, + "learning_rate": 2.652864708714492e-05, + "loss": 2.4374, + "theoretical_loss": 3.3324476769167615, + "tokens_seen": 2866610176 + }, + { + "epoch": 0.74, + "learning_rate": 2.652062269298668e-05, + "loss": 2.4853, + "theoretical_loss": 3.3324358533612424, + "tokens_seen": 2866741248 + }, + { + "epoch": 0.74, + "learning_rate": 2.651259829882844e-05, + "loss": 2.4316, + "theoretical_loss": 3.332424030497661, + "tokens_seen": 2866872320 + }, + { + "epoch": 0.74, + "learning_rate": 2.65045739046702e-05, + "loss": 2.5868, + "theoretical_loss": 3.332412208325944, + "tokens_seen": 2867003392 + }, + { + "epoch": 0.74, + "learning_rate": 2.6496549510511958e-05, + "loss": 2.5822, + "theoretical_loss": 3.3324003868460212, + "tokens_seen": 2867134464 + }, + { + "epoch": 0.74, + "learning_rate": 2.6488525116353718e-05, + "loss": 2.5035, + "theoretical_loss": 3.3323885660578187, + "tokens_seen": 2867265536 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1573131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3161399364471436, + "objective/train/theoretical_loss": 3.3323826559230905, + "objective/train/tokens_used": 1237790176, + "theoretical_loss": 3.3323826559230905, + "tokens_seen": 2867331072 + }, + { + "epoch": 0.74, + "learning_rate": 2.6480500722195478e-05, + "loss": 2.3721, + "theoretical_loss": 3.332376745961265, + "tokens_seen": 2867396608 + }, + { + "epoch": 0.74, + "learning_rate": 2.647247632803723e-05, + "loss": 2.5244, + "theoretical_loss": 3.332364926556289, + "tokens_seen": 2867527680 + }, + { + "epoch": 0.74, + "learning_rate": 2.646445193387899e-05, + "loss": 2.4998, + "theoretical_loss": 3.3323531078428172, + "tokens_seen": 2867658752 + }, + { + "epoch": 0.74, + "learning_rate": 2.645642753972075e-05, + "loss": 2.4182, + "theoretical_loss": 3.3323412898207785, + "tokens_seen": 2867789824 + }, + { + "epoch": 0.74, + "learning_rate": 2.644840314556251e-05, + "loss": 2.5279, + "theoretical_loss": 3.3323294724901, + "tokens_seen": 2867920896 + }, + { + "epoch": 0.74, + "learning_rate": 2.644037875140427e-05, + "loss": 2.4983, + "theoretical_loss": 3.3323176558507104, + "tokens_seen": 2868051968 + }, + { + "epoch": 0.74, + "learning_rate": 2.643235435724603e-05, + "loss": 2.5557, + "theoretical_loss": 3.332305839902537, + "tokens_seen": 2868183040 + }, + { + "epoch": 0.74, + "learning_rate": 2.642432996308779e-05, + "loss": 2.3907, + "theoretical_loss": 3.332294024645509, + "tokens_seen": 2868314112 + }, + { + "epoch": 0.74, + "learning_rate": 2.6416305568929544e-05, + "loss": 2.5527, + "theoretical_loss": 3.3322822100795526, + "tokens_seen": 2868445184 + }, + { + "epoch": 0.74, + "learning_rate": 2.6408281174771304e-05, + "loss": 2.5576, + "theoretical_loss": 3.3322703962045974, + "tokens_seen": 2868576256 + }, + { + "epoch": 0.74, + "learning_rate": 2.6400256780613064e-05, + "loss": 2.5593, + "theoretical_loss": 3.332258583020571, + "tokens_seen": 2868707328 + }, + { + "epoch": 0.74, + "learning_rate": 2.6392232386454824e-05, + "loss": 2.3494, + "theoretical_loss": 3.3322467705274006, + "tokens_seen": 2868838400 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1573762, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7825424671173096, + "objective/train/theoretical_loss": 3.332234958725015, + "objective/train/tokens_used": 1239428576, + "theoretical_loss": 3.332234958725015, + "tokens_seen": 2868969472 + }, + { + "epoch": 0.74, + "learning_rate": 2.6384207992296583e-05, + "loss": 2.5043, + "theoretical_loss": 3.332234958725015, + "tokens_seen": 2868969472 + }, + { + "epoch": 0.74, + "learning_rate": 2.6376183598138343e-05, + "loss": 2.328, + "theoretical_loss": 3.3322231476133424, + "tokens_seen": 2869100544 + }, + { + "epoch": 0.74, + "learning_rate": 2.6368159203980103e-05, + "loss": 2.4885, + "theoretical_loss": 3.3322113371923106, + "tokens_seen": 2869231616 + }, + { + "epoch": 0.74, + "learning_rate": 2.6360134809821856e-05, + "loss": 2.4281, + "theoretical_loss": 3.332199527461847, + "tokens_seen": 2869362688 + }, + { + "epoch": 0.74, + "learning_rate": 2.6352110415663616e-05, + "loss": 2.6309, + "theoretical_loss": 3.332187718421881, + "tokens_seen": 2869493760 + }, + { + "epoch": 0.74, + "learning_rate": 2.6344086021505376e-05, + "loss": 2.5793, + "theoretical_loss": 3.3321759100723396, + "tokens_seen": 2869624832 + }, + { + "epoch": 0.74, + "learning_rate": 2.6336061627347136e-05, + "loss": 2.5014, + "theoretical_loss": 3.3321641024131514, + "tokens_seen": 2869755904 + }, + { + "epoch": 0.74, + "learning_rate": 2.6328037233188896e-05, + "loss": 2.3978, + "theoretical_loss": 3.3321522954442444, + "tokens_seen": 2869886976 + }, + { + "epoch": 0.74, + "learning_rate": 2.6320012839030656e-05, + "loss": 2.4782, + "theoretical_loss": 3.3321404891655466, + "tokens_seen": 2870018048 + }, + { + "epoch": 0.74, + "learning_rate": 2.6311988444872416e-05, + "loss": 2.4287, + "theoretical_loss": 3.3321286835769865, + "tokens_seen": 2870149120 + }, + { + "epoch": 0.74, + "learning_rate": 2.6303964050714176e-05, + "loss": 2.411, + "theoretical_loss": 3.332116878678492, + "tokens_seen": 2870280192 + }, + { + "epoch": 0.74, + "learning_rate": 2.629593965655593e-05, + "loss": 2.4743, + "theoretical_loss": 3.3321050744699914, + "tokens_seen": 2870411264 + }, + { + "epoch": 0.74, + "learning_rate": 2.628791526239769e-05, + "loss": 2.5238, + "theoretical_loss": 3.3320932709514124, + "tokens_seen": 2870542336 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1574954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.376718044281006, + "objective/train/theoretical_loss": 3.3320873694508215, + "objective/train/tokens_used": 1241066976, + "theoretical_loss": 3.3320873694508215, + "tokens_seen": 2870607872 + }, + { + "epoch": 0.74, + "learning_rate": 2.627989086823945e-05, + "loss": 2.4198, + "theoretical_loss": 3.332081468122684, + "tokens_seen": 2870673408 + }, + { + "epoch": 0.74, + "learning_rate": 2.627186647408121e-05, + "loss": 2.4103, + "theoretical_loss": 3.332069665983733, + "tokens_seen": 2870804480 + }, + { + "epoch": 0.74, + "learning_rate": 2.626384207992297e-05, + "loss": 2.5129, + "theoretical_loss": 3.332057864534489, + "tokens_seen": 2870935552 + }, + { + "epoch": 0.74, + "learning_rate": 2.625581768576473e-05, + "loss": 2.5286, + "theoretical_loss": 3.33204606377488, + "tokens_seen": 2871066624 + }, + { + "epoch": 0.74, + "learning_rate": 2.6247793291606488e-05, + "loss": 2.4894, + "theoretical_loss": 3.3320342637048337, + "tokens_seen": 2871197696 + }, + { + "epoch": 0.74, + "learning_rate": 2.623976889744824e-05, + "loss": 2.4167, + "theoretical_loss": 3.3320224643242784, + "tokens_seen": 2871328768 + }, + { + "epoch": 0.74, + "learning_rate": 2.623174450329e-05, + "loss": 2.562, + "theoretical_loss": 3.3320106656331427, + "tokens_seen": 2871459840 + }, + { + "epoch": 0.74, + "learning_rate": 2.622372010913176e-05, + "loss": 2.3728, + "theoretical_loss": 3.3319988676313543, + "tokens_seen": 2871590912 + }, + { + "epoch": 0.74, + "learning_rate": 2.621569571497352e-05, + "loss": 2.5538, + "theoretical_loss": 3.331987070318842, + "tokens_seen": 2871721984 + }, + { + "epoch": 0.74, + "learning_rate": 2.620767132081528e-05, + "loss": 2.325, + "theoretical_loss": 3.331975273695534, + "tokens_seen": 2871853056 + }, + { + "epoch": 0.74, + "learning_rate": 2.619964692665704e-05, + "loss": 2.5688, + "theoretical_loss": 3.3319634777613585, + "tokens_seen": 2871984128 + }, + { + "epoch": 0.74, + "learning_rate": 2.61916225324988e-05, + "loss": 2.5696, + "theoretical_loss": 3.3319516825162436, + "tokens_seen": 2872115200 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1575619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.732257604598999, + "objective/train/theoretical_loss": 3.331939887960118, + "objective/train/tokens_used": 1242705376, + "theoretical_loss": 3.331939887960118, + "tokens_seen": 2872246272 + }, + { + "epoch": 0.74, + "learning_rate": 2.618359813834056e-05, + "loss": 2.5838, + "theoretical_loss": 3.331939887960118, + "tokens_seen": 2872246272 + }, + { + "epoch": 0.74, + "learning_rate": 2.6175573744182314e-05, + "loss": 2.5334, + "theoretical_loss": 3.3319280940929095, + "tokens_seen": 2872377344 + }, + { + "epoch": 0.74, + "learning_rate": 2.6167549350024074e-05, + "loss": 2.6445, + "theoretical_loss": 3.3319163009145467, + "tokens_seen": 2872508416 + }, + { + "epoch": 0.74, + "learning_rate": 2.6159524955865834e-05, + "loss": 2.5977, + "theoretical_loss": 3.331904508424958, + "tokens_seen": 2872639488 + }, + { + "epoch": 0.74, + "learning_rate": 2.6151500561707594e-05, + "loss": 2.5116, + "theoretical_loss": 3.331892716624072, + "tokens_seen": 2872770560 + }, + { + "epoch": 0.74, + "learning_rate": 2.6143476167549354e-05, + "loss": 2.3174, + "theoretical_loss": 3.3318809255118165, + "tokens_seen": 2872901632 + }, + { + "epoch": 0.74, + "learning_rate": 2.6135451773391113e-05, + "loss": 2.492, + "theoretical_loss": 3.33186913508812, + "tokens_seen": 2873032704 + }, + { + "epoch": 0.74, + "learning_rate": 2.6127427379232873e-05, + "loss": 2.5228, + "theoretical_loss": 3.331857345352911, + "tokens_seen": 2873163776 + }, + { + "epoch": 0.74, + "learning_rate": 2.6119402985074626e-05, + "loss": 2.67, + "theoretical_loss": 3.331845556306118, + "tokens_seen": 2873294848 + }, + { + "epoch": 0.74, + "learning_rate": 2.6111378590916386e-05, + "loss": 2.518, + "theoretical_loss": 3.331833767947669, + "tokens_seen": 2873425920 + }, + { + "epoch": 0.74, + "learning_rate": 2.6103354196758146e-05, + "loss": 2.5106, + "theoretical_loss": 3.331821980277493, + "tokens_seen": 2873556992 + }, + { + "epoch": 0.74, + "learning_rate": 2.6095329802599906e-05, + "loss": 2.3514, + "theoretical_loss": 3.331810193295518, + "tokens_seen": 2873688064 + }, + { + "epoch": 0.74, + "learning_rate": 2.6087305408441666e-05, + "loss": 2.4918, + "theoretical_loss": 3.331798407001673, + "tokens_seen": 2873819136 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1576835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.46124005317688, + "objective/train/theoretical_loss": 3.3317925141127764, + "objective/train/tokens_used": 1244343776, + "theoretical_loss": 3.3317925141127764, + "tokens_seen": 2873884672 + }, + { + "epoch": 0.74, + "learning_rate": 2.6079281014283426e-05, + "loss": 2.3727, + "theoretical_loss": 3.3317866213958856, + "tokens_seen": 2873950208 + }, + { + "epoch": 0.74, + "learning_rate": 2.6071256620125182e-05, + "loss": 2.3579, + "theoretical_loss": 3.3317748364780844, + "tokens_seen": 2874081280 + }, + { + "epoch": 0.74, + "learning_rate": 2.606323222596694e-05, + "loss": 2.4175, + "theoretical_loss": 3.3317630522481987, + "tokens_seen": 2874212352 + }, + { + "epoch": 0.74, + "learning_rate": 2.60552078318087e-05, + "loss": 2.5451, + "theoretical_loss": 3.331751268706156, + "tokens_seen": 2874343424 + }, + { + "epoch": 0.74, + "learning_rate": 2.604718343765046e-05, + "loss": 2.5829, + "theoretical_loss": 3.3317394858518856, + "tokens_seen": 2874474496 + }, + { + "epoch": 0.74, + "learning_rate": 2.603915904349222e-05, + "loss": 2.5416, + "theoretical_loss": 3.3317277036853152, + "tokens_seen": 2874605568 + }, + { + "epoch": 0.74, + "learning_rate": 2.6031134649333975e-05, + "loss": 2.6133, + "theoretical_loss": 3.3317159222063735, + "tokens_seen": 2874736640 + }, + { + "epoch": 0.74, + "learning_rate": 2.6023110255175735e-05, + "loss": 2.5903, + "theoretical_loss": 3.3317041414149897, + "tokens_seen": 2874867712 + }, + { + "epoch": 0.74, + "learning_rate": 2.6015085861017495e-05, + "loss": 2.5154, + "theoretical_loss": 3.3316923613110916, + "tokens_seen": 2874998784 + }, + { + "epoch": 0.74, + "learning_rate": 2.6007061466859255e-05, + "loss": 2.4855, + "theoretical_loss": 3.331680581894608, + "tokens_seen": 2875129856 + }, + { + "epoch": 0.74, + "learning_rate": 2.599903707270101e-05, + "loss": 2.4813, + "theoretical_loss": 3.3316688031654675, + "tokens_seen": 2875260928 + }, + { + "epoch": 0.74, + "learning_rate": 2.599101267854277e-05, + "loss": 2.4103, + "theoretical_loss": 3.3316570251235986, + "tokens_seen": 2875392000 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1577362, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7285592555999756, + "objective/train/theoretical_loss": 3.33164524776893, + "objective/train/tokens_used": 1245982176, + "theoretical_loss": 3.33164524776893, + "tokens_seen": 2875523072 + }, + { + "epoch": 0.74, + "learning_rate": 2.5982988284384528e-05, + "loss": 2.6044, + "theoretical_loss": 3.33164524776893, + "tokens_seen": 2875523072 + }, + { + "epoch": 0.74, + "learning_rate": 2.5974963890226288e-05, + "loss": 2.4341, + "theoretical_loss": 3.33163347110139, + "tokens_seen": 2875654144 + }, + { + "epoch": 0.74, + "learning_rate": 2.5966939496068048e-05, + "loss": 2.4686, + "theoretical_loss": 3.3316216951209072, + "tokens_seen": 2875785216 + }, + { + "epoch": 0.74, + "learning_rate": 2.5958915101909808e-05, + "loss": 2.4627, + "theoretical_loss": 3.3316099198274105, + "tokens_seen": 2875916288 + }, + { + "epoch": 0.74, + "learning_rate": 2.5950890707751568e-05, + "loss": 2.3936, + "theoretical_loss": 3.3315981452208288, + "tokens_seen": 2876047360 + }, + { + "epoch": 0.74, + "learning_rate": 2.5942866313593324e-05, + "loss": 2.4873, + "theoretical_loss": 3.3315863713010896, + "tokens_seen": 2876178432 + }, + { + "epoch": 0.74, + "learning_rate": 2.593484191943508e-05, + "loss": 2.52, + "theoretical_loss": 3.331574598068123, + "tokens_seen": 2876309504 + }, + { + "epoch": 0.74, + "learning_rate": 2.592681752527684e-05, + "loss": 2.3663, + "theoretical_loss": 3.331562825521856, + "tokens_seen": 2876440576 + }, + { + "epoch": 0.74, + "learning_rate": 2.59187931311186e-05, + "loss": 2.4195, + "theoretical_loss": 3.331551053662219, + "tokens_seen": 2876571648 + }, + { + "epoch": 0.74, + "learning_rate": 2.591076873696036e-05, + "loss": 2.4857, + "theoretical_loss": 3.331539282489139, + "tokens_seen": 2876702720 + }, + { + "epoch": 0.74, + "learning_rate": 2.590274434280212e-05, + "loss": 2.3449, + "theoretical_loss": 3.3315275120025456, + "tokens_seen": 2876833792 + }, + { + "epoch": 0.74, + "learning_rate": 2.589471994864388e-05, + "loss": 2.438, + "theoretical_loss": 3.3315157422023676, + "tokens_seen": 2876964864 + }, + { + "epoch": 0.74, + "learning_rate": 2.5886695554485633e-05, + "loss": 2.4269, + "theoretical_loss": 3.331503973088534, + "tokens_seen": 2877095936 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1578509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.293755531311035, + "objective/train/theoretical_loss": 3.3314980887889734, + "objective/train/tokens_used": 1247620576, + "theoretical_loss": 3.3314980887889734, + "tokens_seen": 2877161472 + }, + { + "epoch": 0.74, + "learning_rate": 2.5878671160327393e-05, + "loss": 2.4526, + "theoretical_loss": 3.331492204660972, + "tokens_seen": 2877227008 + }, + { + "epoch": 0.74, + "learning_rate": 2.5870646766169153e-05, + "loss": 2.3764, + "theoretical_loss": 3.331480436919612, + "tokens_seen": 2877358080 + }, + { + "epoch": 0.74, + "learning_rate": 2.5862622372010913e-05, + "loss": 2.5021, + "theoretical_loss": 3.331468669864382, + "tokens_seen": 2877489152 + }, + { + "epoch": 0.74, + "learning_rate": 2.5854597977852673e-05, + "loss": 2.469, + "theoretical_loss": 3.3314569034952104, + "tokens_seen": 2877620224 + }, + { + "epoch": 0.74, + "learning_rate": 2.5846573583694433e-05, + "loss": 2.4949, + "theoretical_loss": 3.3314451378120267, + "tokens_seen": 2877751296 + }, + { + "epoch": 0.74, + "learning_rate": 2.5838549189536193e-05, + "loss": 2.6846, + "theoretical_loss": 3.331433372814759, + "tokens_seen": 2877882368 + }, + { + "epoch": 0.74, + "learning_rate": 2.5830524795377953e-05, + "loss": 2.494, + "theoretical_loss": 3.331421608503337, + "tokens_seen": 2878013440 + }, + { + "epoch": 0.74, + "learning_rate": 2.5822500401219706e-05, + "loss": 2.494, + "theoretical_loss": 3.331409844877688, + "tokens_seen": 2878144512 + }, + { + "epoch": 0.74, + "learning_rate": 2.5814476007061466e-05, + "loss": 2.5297, + "theoretical_loss": 3.3313980819377425, + "tokens_seen": 2878275584 + }, + { + "epoch": 0.74, + "learning_rate": 2.5806451612903226e-05, + "loss": 2.3394, + "theoretical_loss": 3.331386319683428, + "tokens_seen": 2878406656 + }, + { + "epoch": 0.74, + "learning_rate": 2.5798427218744985e-05, + "loss": 2.4572, + "theoretical_loss": 3.331374558114674, + "tokens_seen": 2878537728 + }, + { + "epoch": 0.74, + "learning_rate": 2.5790402824586745e-05, + "loss": 2.4709, + "theoretical_loss": 3.331362797231409, + "tokens_seen": 2878668800 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1579013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.363574981689453, + "objective/train/theoretical_loss": 3.3313510370335617, + "objective/train/tokens_used": 1249258976, + "theoretical_loss": 3.3313510370335617, + "tokens_seen": 2878799872 + }, + { + "epoch": 0.74, + "learning_rate": 2.5782378430428505e-05, + "loss": 2.4975, + "theoretical_loss": 3.3313510370335617, + "tokens_seen": 2878799872 + }, + { + "epoch": 0.74, + "learning_rate": 2.5774354036270265e-05, + "loss": 2.6713, + "theoretical_loss": 3.3313392775210615, + "tokens_seen": 2878930944 + }, + { + "epoch": 0.74, + "learning_rate": 2.5766329642112018e-05, + "loss": 2.3749, + "theoretical_loss": 3.3313275186938367, + "tokens_seen": 2879062016 + }, + { + "epoch": 0.74, + "learning_rate": 2.5758305247953778e-05, + "loss": 2.4006, + "theoretical_loss": 3.3313157605518167, + "tokens_seen": 2879193088 + }, + { + "epoch": 0.75, + "learning_rate": 2.5750280853795538e-05, + "loss": 2.5832, + "theoretical_loss": 3.3313040030949304, + "tokens_seen": 2879324160 + }, + { + "epoch": 0.75, + "learning_rate": 2.5742256459637298e-05, + "loss": 2.4333, + "theoretical_loss": 3.331292246323106, + "tokens_seen": 2879455232 + }, + { + "epoch": 0.75, + "learning_rate": 2.5734232065479058e-05, + "loss": 2.4775, + "theoretical_loss": 3.331280490236273, + "tokens_seen": 2879586304 + }, + { + "epoch": 0.75, + "learning_rate": 2.5726207671320818e-05, + "loss": 2.5637, + "theoretical_loss": 3.33126873483436, + "tokens_seen": 2879717376 + }, + { + "epoch": 0.75, + "learning_rate": 2.5718183277162578e-05, + "loss": 2.4927, + "theoretical_loss": 3.331256980117296, + "tokens_seen": 2879848448 + }, + { + "epoch": 0.75, + "learning_rate": 2.5710158883004338e-05, + "loss": 2.5344, + "theoretical_loss": 3.3312452260850103, + "tokens_seen": 2879979520 + }, + { + "epoch": 0.75, + "learning_rate": 2.570213448884609e-05, + "loss": 2.437, + "theoretical_loss": 3.331233472737431, + "tokens_seen": 2880110592 + }, + { + "epoch": 0.75, + "learning_rate": 2.569411009468785e-05, + "loss": 2.5171, + "theoretical_loss": 3.3312217200744882, + "tokens_seen": 2880241664 + }, + { + "epoch": 0.75, + "learning_rate": 2.568608570052961e-05, + "loss": 2.4984, + "theoretical_loss": 3.33120996809611, + "tokens_seen": 2880372736 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1579013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3296008110046387, + "objective/train/theoretical_loss": 3.3312040923636106, + "objective/train/tokens_used": 1250897376, + "theoretical_loss": 3.3312040923636106, + "tokens_seen": 2880438272 + }, + { + "epoch": 0.75, + "learning_rate": 2.567806130637137e-05, + "loss": 2.4728, + "theoretical_loss": 3.3311982168022256, + "tokens_seen": 2880503808 + }, + { + "epoch": 0.75, + "learning_rate": 2.567003691221313e-05, + "loss": 2.5106, + "theoretical_loss": 3.3311864661927637, + "tokens_seen": 2880634880 + }, + { + "epoch": 0.75, + "learning_rate": 2.566201251805489e-05, + "loss": 2.6279, + "theoretical_loss": 3.331174716267654, + "tokens_seen": 2880765952 + }, + { + "epoch": 0.75, + "learning_rate": 2.565398812389665e-05, + "loss": 2.6496, + "theoretical_loss": 3.3311629670268252, + "tokens_seen": 2880897024 + }, + { + "epoch": 0.75, + "learning_rate": 2.5645963729738403e-05, + "loss": 2.5266, + "theoretical_loss": 3.331151218470206, + "tokens_seen": 2881028096 + }, + { + "epoch": 0.75, + "learning_rate": 2.5637939335580163e-05, + "loss": 2.7341, + "theoretical_loss": 3.331139470597726, + "tokens_seen": 2881159168 + }, + { + "epoch": 0.75, + "learning_rate": 2.5629914941421923e-05, + "loss": 2.6319, + "theoretical_loss": 3.3311277234093137, + "tokens_seen": 2881290240 + }, + { + "epoch": 0.75, + "learning_rate": 2.5621890547263683e-05, + "loss": 2.51, + "theoretical_loss": 3.3311159769048984, + "tokens_seen": 2881421312 + }, + { + "epoch": 0.75, + "learning_rate": 2.5613866153105443e-05, + "loss": 2.5693, + "theoretical_loss": 3.3311042310844092, + "tokens_seen": 2881552384 + }, + { + "epoch": 0.75, + "learning_rate": 2.5605841758947203e-05, + "loss": 2.683, + "theoretical_loss": 3.331092485947775, + "tokens_seen": 2881683456 + }, + { + "epoch": 0.75, + "learning_rate": 2.5597817364788963e-05, + "loss": 2.4699, + "theoretical_loss": 3.331080741494925, + "tokens_seen": 2881814528 + }, + { + "epoch": 0.75, + "learning_rate": 2.5589792970630716e-05, + "loss": 2.6009, + "theoretical_loss": 3.3310689977257883, + "tokens_seen": 2881945600 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1579536, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.476306438446045, + "objective/train/theoretical_loss": 3.331057254640294, + "objective/train/tokens_used": 1252535776, + "theoretical_loss": 3.331057254640294, + "tokens_seen": 2882076672 + }, + { + "epoch": 0.75, + "learning_rate": 2.5581768576472476e-05, + "loss": 2.6539, + "theoretical_loss": 3.331057254640294, + "tokens_seen": 2882076672 + }, + { + "epoch": 0.75, + "learning_rate": 2.5573744182314236e-05, + "loss": 2.6187, + "theoretical_loss": 3.3310455122383713, + "tokens_seen": 2882207744 + }, + { + "epoch": 0.75, + "learning_rate": 2.5565719788155996e-05, + "loss": 2.5934, + "theoretical_loss": 3.331033770519949, + "tokens_seen": 2882338816 + }, + { + "epoch": 0.75, + "learning_rate": 2.5557695393997755e-05, + "loss": 2.6068, + "theoretical_loss": 3.3310220294849566, + "tokens_seen": 2882469888 + }, + { + "epoch": 0.75, + "learning_rate": 2.5549670999839515e-05, + "loss": 2.5722, + "theoretical_loss": 3.331010289133323, + "tokens_seen": 2882600960 + }, + { + "epoch": 0.75, + "learning_rate": 2.5541646605681275e-05, + "loss": 2.5261, + "theoretical_loss": 3.3309985494649776, + "tokens_seen": 2882732032 + }, + { + "epoch": 0.75, + "learning_rate": 2.5533622211523035e-05, + "loss": 2.6064, + "theoretical_loss": 3.3309868104798492, + "tokens_seen": 2882863104 + }, + { + "epoch": 0.75, + "learning_rate": 2.552559781736479e-05, + "loss": 2.627, + "theoretical_loss": 3.3309750721778677, + "tokens_seen": 2882994176 + }, + { + "epoch": 0.75, + "learning_rate": 2.5517573423206548e-05, + "loss": 2.6114, + "theoretical_loss": 3.3309633345589615, + "tokens_seen": 2883125248 + }, + { + "epoch": 0.75, + "learning_rate": 2.5509549029048308e-05, + "loss": 2.6035, + "theoretical_loss": 3.33095159762306, + "tokens_seen": 2883256320 + }, + { + "epoch": 0.75, + "learning_rate": 2.5501524634890068e-05, + "loss": 2.5675, + "theoretical_loss": 3.3309398613700925, + "tokens_seen": 2883387392 + }, + { + "epoch": 0.75, + "learning_rate": 2.5493500240731828e-05, + "loss": 2.5169, + "theoretical_loss": 3.3309281257999883, + "tokens_seen": 2883518464 + }, + { + "epoch": 0.75, + "learning_rate": 2.5485475846573588e-05, + "loss": 2.5947, + "theoretical_loss": 3.3309163909126767, + "tokens_seen": 2883649536 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1580744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0638628005981445, + "objective/train/theoretical_loss": 3.330910523725046, + "objective/train/tokens_used": 1254174176, + "theoretical_loss": 3.330910523725046, + "tokens_seen": 2883715072 + }, + { + "epoch": 0.75, + "learning_rate": 2.5477451452415348e-05, + "loss": 2.6187, + "theoretical_loss": 3.330904656708087, + "tokens_seen": 2883780608 + }, + { + "epoch": 0.75, + "learning_rate": 2.54694270582571e-05, + "loss": 2.4664, + "theoretical_loss": 3.3308929231861475, + "tokens_seen": 2883911680 + }, + { + "epoch": 0.75, + "learning_rate": 2.546140266409886e-05, + "loss": 2.6551, + "theoretical_loss": 3.3308811903467888, + "tokens_seen": 2884042752 + }, + { + "epoch": 0.75, + "learning_rate": 2.545337826994062e-05, + "loss": 2.5027, + "theoretical_loss": 3.3308694581899396, + "tokens_seen": 2884173824 + }, + { + "epoch": 0.75, + "learning_rate": 2.544535387578238e-05, + "loss": 2.4939, + "theoretical_loss": 3.330857726715529, + "tokens_seen": 2884304896 + }, + { + "epoch": 0.75, + "learning_rate": 2.543732948162414e-05, + "loss": 2.5993, + "theoretical_loss": 3.3308459959234864, + "tokens_seen": 2884435968 + }, + { + "epoch": 0.75, + "learning_rate": 2.54293050874659e-05, + "loss": 2.4914, + "theoretical_loss": 3.330834265813741, + "tokens_seen": 2884567040 + }, + { + "epoch": 0.75, + "learning_rate": 2.5421280693307657e-05, + "loss": 2.6339, + "theoretical_loss": 3.330822536386223, + "tokens_seen": 2884698112 + }, + { + "epoch": 0.75, + "learning_rate": 2.5413256299149413e-05, + "loss": 2.6268, + "theoretical_loss": 3.3308108076408605, + "tokens_seen": 2884829184 + }, + { + "epoch": 0.75, + "learning_rate": 2.5405231904991173e-05, + "loss": 2.5674, + "theoretical_loss": 3.3307990795775835, + "tokens_seen": 2884960256 + }, + { + "epoch": 0.75, + "learning_rate": 2.5397207510832933e-05, + "loss": 2.5833, + "theoretical_loss": 3.330787352196321, + "tokens_seen": 2885091328 + }, + { + "epoch": 0.75, + "learning_rate": 2.5389183116674693e-05, + "loss": 2.6078, + "theoretical_loss": 3.3307756254970027, + "tokens_seen": 2885222400 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1581203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7565793991088867, + "objective/train/theoretical_loss": 3.330763899479558, + "objective/train/tokens_used": 1255812576, + "theoretical_loss": 3.330763899479558, + "tokens_seen": 2885353472 + }, + { + "epoch": 0.75, + "learning_rate": 2.5381158722516453e-05, + "loss": 2.8068, + "theoretical_loss": 3.330763899479558, + "tokens_seen": 2885353472 + }, + { + "epoch": 0.75, + "learning_rate": 2.537313432835821e-05, + "loss": 2.5891, + "theoretical_loss": 3.3307521741439157, + "tokens_seen": 2885484544 + }, + { + "epoch": 0.75, + "learning_rate": 2.536510993419997e-05, + "loss": 2.4497, + "theoretical_loss": 3.330740449490006, + "tokens_seen": 2885615616 + }, + { + "epoch": 0.75, + "learning_rate": 2.535708554004173e-05, + "loss": 2.4295, + "theoretical_loss": 3.3307287255177576, + "tokens_seen": 2885746688 + }, + { + "epoch": 0.75, + "learning_rate": 2.5349061145883486e-05, + "loss": 2.4693, + "theoretical_loss": 3.3307170022271, + "tokens_seen": 2885877760 + }, + { + "epoch": 0.75, + "learning_rate": 2.5341036751725246e-05, + "loss": 2.4541, + "theoretical_loss": 3.3307052796179635, + "tokens_seen": 2886008832 + }, + { + "epoch": 0.75, + "learning_rate": 2.5333012357567006e-05, + "loss": 2.6991, + "theoretical_loss": 3.3306935576902763, + "tokens_seen": 2886139904 + }, + { + "epoch": 0.75, + "learning_rate": 2.5324987963408762e-05, + "loss": 2.5633, + "theoretical_loss": 3.3306818364439685, + "tokens_seen": 2886270976 + }, + { + "epoch": 0.75, + "learning_rate": 2.5316963569250522e-05, + "loss": 2.4777, + "theoretical_loss": 3.33067011587897, + "tokens_seen": 2886402048 + }, + { + "epoch": 0.75, + "learning_rate": 2.5308939175092282e-05, + "loss": 2.5478, + "theoretical_loss": 3.330658395995209, + "tokens_seen": 2886533120 + }, + { + "epoch": 0.75, + "learning_rate": 2.5300914780934042e-05, + "loss": 2.6698, + "theoretical_loss": 3.330646676792616, + "tokens_seen": 2886664192 + }, + { + "epoch": 0.75, + "learning_rate": 2.52928903867758e-05, + "loss": 2.5255, + "theoretical_loss": 3.33063495827112, + "tokens_seen": 2886795264 + }, + { + "epoch": 0.75, + "learning_rate": 2.528486599261756e-05, + "loss": 2.6515, + "theoretical_loss": 3.3306232404306506, + "tokens_seen": 2886926336 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1582413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.529496908187866, + "objective/train/theoretical_loss": 3.330617381765779, + "objective/train/tokens_used": 1257450976, + "theoretical_loss": 3.330617381765779, + "tokens_seen": 2886991872 + }, + { + "epoch": 0.75, + "learning_rate": 2.5276841598459315e-05, + "loss": 2.5228, + "theoretical_loss": 3.3306115232711377, + "tokens_seen": 2887057408 + }, + { + "epoch": 0.75, + "learning_rate": 2.5268817204301075e-05, + "loss": 2.4839, + "theoretical_loss": 3.33059980679251, + "tokens_seen": 2887188480 + }, + { + "epoch": 0.75, + "learning_rate": 2.5260792810142835e-05, + "loss": 2.5443, + "theoretical_loss": 3.3305880909946977, + "tokens_seen": 2887319552 + }, + { + "epoch": 0.75, + "learning_rate": 2.5252768415984595e-05, + "loss": 2.3887, + "theoretical_loss": 3.3305763758776306, + "tokens_seen": 2887450624 + }, + { + "epoch": 0.75, + "learning_rate": 2.5244744021826355e-05, + "loss": 2.5118, + "theoretical_loss": 3.330564661441237, + "tokens_seen": 2887581696 + }, + { + "epoch": 0.75, + "learning_rate": 2.5236719627668114e-05, + "loss": 2.5059, + "theoretical_loss": 3.330552947685448, + "tokens_seen": 2887712768 + }, + { + "epoch": 0.75, + "learning_rate": 2.5228695233509868e-05, + "loss": 2.4984, + "theoretical_loss": 3.330541234610192, + "tokens_seen": 2887843840 + }, + { + "epoch": 0.75, + "learning_rate": 2.5220670839351628e-05, + "loss": 2.5882, + "theoretical_loss": 3.330529522215399, + "tokens_seen": 2887974912 + }, + { + "epoch": 0.75, + "learning_rate": 2.5212646445193387e-05, + "loss": 2.7447, + "theoretical_loss": 3.3305178105009987, + "tokens_seen": 2888105984 + }, + { + "epoch": 0.75, + "learning_rate": 2.5204622051035147e-05, + "loss": 2.5045, + "theoretical_loss": 3.3305060994669207, + "tokens_seen": 2888237056 + }, + { + "epoch": 0.75, + "learning_rate": 2.5196597656876907e-05, + "loss": 2.6241, + "theoretical_loss": 3.330494389113094, + "tokens_seen": 2888368128 + }, + { + "epoch": 0.75, + "learning_rate": 2.5188573262718667e-05, + "loss": 2.7022, + "theoretical_loss": 3.330482679439449, + "tokens_seen": 2888499200 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1583588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.430612802505493, + "objective/train/theoretical_loss": 3.3304709704459152, + "objective/train/tokens_used": 1259089376, + "theoretical_loss": 3.3304709704459152, + "tokens_seen": 2888630272 + }, + { + "epoch": 0.75, + "learning_rate": 2.5180548868560427e-05, + "loss": 2.7269, + "theoretical_loss": 3.3304709704459152, + "tokens_seen": 2888630272 + }, + { + "epoch": 0.75, + "learning_rate": 2.517252447440218e-05, + "loss": 2.5736, + "theoretical_loss": 3.330459262132422, + "tokens_seen": 2888761344 + }, + { + "epoch": 0.75, + "learning_rate": 2.516450008024394e-05, + "loss": 2.4107, + "theoretical_loss": 3.3304475544988987, + "tokens_seen": 2888892416 + }, + { + "epoch": 0.75, + "learning_rate": 2.51564756860857e-05, + "loss": 2.6884, + "theoretical_loss": 3.330435847545276, + "tokens_seen": 2889023488 + }, + { + "epoch": 0.75, + "learning_rate": 2.514845129192746e-05, + "loss": 2.658, + "theoretical_loss": 3.330424141271483, + "tokens_seen": 2889154560 + }, + { + "epoch": 0.75, + "learning_rate": 2.514042689776922e-05, + "loss": 2.5616, + "theoretical_loss": 3.3304124356774487, + "tokens_seen": 2889285632 + }, + { + "epoch": 0.75, + "learning_rate": 2.513240250361098e-05, + "loss": 2.5694, + "theoretical_loss": 3.330400730763104, + "tokens_seen": 2889416704 + }, + { + "epoch": 0.75, + "learning_rate": 2.512437810945274e-05, + "loss": 2.6229, + "theoretical_loss": 3.3303890265283775, + "tokens_seen": 2889547776 + }, + { + "epoch": 0.75, + "learning_rate": 2.5116353715294493e-05, + "loss": 2.5287, + "theoretical_loss": 3.3303773229731997, + "tokens_seen": 2889678848 + }, + { + "epoch": 0.75, + "learning_rate": 2.5108329321136253e-05, + "loss": 2.6392, + "theoretical_loss": 3.3303656200975005, + "tokens_seen": 2889809920 + }, + { + "epoch": 0.75, + "learning_rate": 2.5100304926978013e-05, + "loss": 2.5251, + "theoretical_loss": 3.330353917901209, + "tokens_seen": 2889940992 + }, + { + "epoch": 0.75, + "learning_rate": 2.5092280532819772e-05, + "loss": 2.6107, + "theoretical_loss": 3.330342216384255, + "tokens_seen": 2890072064 + }, + { + "epoch": 0.75, + "learning_rate": 2.5084256138661532e-05, + "loss": 2.6284, + "theoretical_loss": 3.3303305155465686, + "tokens_seen": 2890203136 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1583937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.51579213142395, + "objective/train/theoretical_loss": 3.3303246653824283, + "objective/train/tokens_used": 1260727776, + "theoretical_loss": 3.3303246653824283, + "tokens_seen": 2890268672 + }, + { + "epoch": 0.75, + "learning_rate": 2.5076231744503292e-05, + "loss": 2.6178, + "theoretical_loss": 3.330318815388079, + "tokens_seen": 2890334208 + }, + { + "epoch": 0.75, + "learning_rate": 2.5068207350345052e-05, + "loss": 2.5528, + "theoretical_loss": 3.3303071159087168, + "tokens_seen": 2890465280 + }, + { + "epoch": 0.75, + "learning_rate": 2.5060182956186812e-05, + "loss": 2.5801, + "theoretical_loss": 3.3302954171084114, + "tokens_seen": 2890596352 + }, + { + "epoch": 0.75, + "learning_rate": 2.5052158562028565e-05, + "loss": 2.5291, + "theoretical_loss": 3.330283718987092, + "tokens_seen": 2890727424 + }, + { + "epoch": 0.75, + "learning_rate": 2.5044134167870325e-05, + "loss": 2.5727, + "theoretical_loss": 3.3302720215446895, + "tokens_seen": 2890858496 + }, + { + "epoch": 0.75, + "learning_rate": 2.5036109773712085e-05, + "loss": 2.4702, + "theoretical_loss": 3.3302603247811327, + "tokens_seen": 2890989568 + }, + { + "epoch": 0.75, + "learning_rate": 2.5028085379553845e-05, + "loss": 2.6296, + "theoretical_loss": 3.3302486286963524, + "tokens_seen": 2891120640 + }, + { + "epoch": 0.75, + "learning_rate": 2.5020060985395605e-05, + "loss": 2.6613, + "theoretical_loss": 3.3302369332902777, + "tokens_seen": 2891251712 + }, + { + "epoch": 0.75, + "learning_rate": 2.5012036591237365e-05, + "loss": 2.5165, + "theoretical_loss": 3.3302252385628384, + "tokens_seen": 2891382784 + }, + { + "epoch": 0.75, + "learning_rate": 2.5004012197079125e-05, + "loss": 2.3854, + "theoretical_loss": 3.330213544513965, + "tokens_seen": 2891513856 + }, + { + "epoch": 0.75, + "learning_rate": 2.499598780292088e-05, + "loss": 2.4419, + "theoretical_loss": 3.3302018511435874, + "tokens_seen": 2891644928 + }, + { + "epoch": 0.75, + "learning_rate": 2.498796340876264e-05, + "loss": 2.6644, + "theoretical_loss": 3.3301901584516345, + "tokens_seen": 2891776000 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1585254, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.459376096725464, + "objective/train/theoretical_loss": 3.3301784664380367, + "objective/train/tokens_used": 1262366176, + "theoretical_loss": 3.3301784664380367, + "tokens_seen": 2891907072 + }, + { + "epoch": 0.75, + "learning_rate": 2.4979939014604398e-05, + "loss": 2.6464, + "theoretical_loss": 3.3301784664380367, + "tokens_seen": 2891907072 + }, + { + "epoch": 0.75, + "learning_rate": 2.4971914620446157e-05, + "loss": 2.6311, + "theoretical_loss": 3.3301667751027244, + "tokens_seen": 2892038144 + }, + { + "epoch": 0.75, + "learning_rate": 2.4963890226287917e-05, + "loss": 2.5207, + "theoretical_loss": 3.330155084445627, + "tokens_seen": 2892169216 + }, + { + "epoch": 0.75, + "learning_rate": 2.4955865832129677e-05, + "loss": 2.6175, + "theoretical_loss": 3.3301433944666745, + "tokens_seen": 2892300288 + }, + { + "epoch": 0.75, + "learning_rate": 2.4947841437971434e-05, + "loss": 2.5369, + "theoretical_loss": 3.330131705165797, + "tokens_seen": 2892431360 + }, + { + "epoch": 0.75, + "learning_rate": 2.4939817043813194e-05, + "loss": 2.7184, + "theoretical_loss": 3.3301200165429243, + "tokens_seen": 2892562432 + }, + { + "epoch": 0.75, + "learning_rate": 2.4931792649654954e-05, + "loss": 2.6193, + "theoretical_loss": 3.3301083285979862, + "tokens_seen": 2892693504 + }, + { + "epoch": 0.75, + "learning_rate": 2.492376825549671e-05, + "loss": 2.6195, + "theoretical_loss": 3.330096641330913, + "tokens_seen": 2892824576 + }, + { + "epoch": 0.75, + "learning_rate": 2.491574386133847e-05, + "loss": 2.6155, + "theoretical_loss": 3.3300849547416345, + "tokens_seen": 2892955648 + }, + { + "epoch": 0.75, + "learning_rate": 2.490771946718023e-05, + "loss": 2.5419, + "theoretical_loss": 3.3300732688300805, + "tokens_seen": 2893086720 + }, + { + "epoch": 0.75, + "learning_rate": 2.489969507302199e-05, + "loss": 2.7091, + "theoretical_loss": 3.3300615835961813, + "tokens_seen": 2893217792 + }, + { + "epoch": 0.75, + "learning_rate": 2.4891670678863746e-05, + "loss": 2.4441, + "theoretical_loss": 3.330049899039867, + "tokens_seen": 2893348864 + }, + { + "epoch": 0.75, + "learning_rate": 2.4883646284705506e-05, + "loss": 2.5439, + "theoretical_loss": 3.3300382151610672, + "tokens_seen": 2893479936 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1585876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.157543182373047, + "objective/train/theoretical_loss": 3.330032373475713, + "objective/train/tokens_used": 1264004576, + "theoretical_loss": 3.330032373475713, + "tokens_seen": 2893545472 + }, + { + "epoch": 0.75, + "learning_rate": 2.4875621890547266e-05, + "loss": 2.4847, + "theoretical_loss": 3.330026531959712, + "tokens_seen": 2893611008 + }, + { + "epoch": 0.75, + "learning_rate": 2.4867597496389026e-05, + "loss": 2.6497, + "theoretical_loss": 3.330014849435732, + "tokens_seen": 2893742080 + }, + { + "epoch": 0.75, + "learning_rate": 2.4859573102230783e-05, + "loss": 2.5221, + "theoretical_loss": 3.3300031675890565, + "tokens_seen": 2893873152 + }, + { + "epoch": 0.75, + "learning_rate": 2.4851548708072543e-05, + "loss": 2.5889, + "theoretical_loss": 3.329991486419616, + "tokens_seen": 2894004224 + }, + { + "epoch": 0.75, + "learning_rate": 2.4843524313914302e-05, + "loss": 2.4886, + "theoretical_loss": 3.32997980592734, + "tokens_seen": 2894135296 + }, + { + "epoch": 0.75, + "learning_rate": 2.483549991975606e-05, + "loss": 2.4688, + "theoretical_loss": 3.3299681261121594, + "tokens_seen": 2894266368 + }, + { + "epoch": 0.75, + "learning_rate": 2.482747552559782e-05, + "loss": 2.681, + "theoretical_loss": 3.329956446974004, + "tokens_seen": 2894397440 + }, + { + "epoch": 0.75, + "learning_rate": 2.481945113143958e-05, + "loss": 2.6061, + "theoretical_loss": 3.329944768512804, + "tokens_seen": 2894528512 + }, + { + "epoch": 0.75, + "learning_rate": 2.481142673728134e-05, + "loss": 2.5827, + "theoretical_loss": 3.3299330907284888, + "tokens_seen": 2894659584 + }, + { + "epoch": 0.75, + "learning_rate": 2.4803402343123095e-05, + "loss": 2.5738, + "theoretical_loss": 3.3299214136209896, + "tokens_seen": 2894790656 + }, + { + "epoch": 0.75, + "learning_rate": 2.4795377948964855e-05, + "loss": 2.643, + "theoretical_loss": 3.3299097371902358, + "tokens_seen": 2894921728 + }, + { + "epoch": 0.75, + "learning_rate": 2.4787353554806615e-05, + "loss": 2.6112, + "theoretical_loss": 3.3298980614361575, + "tokens_seen": 2895052800 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1586924, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7644100189208984, + "objective/train/theoretical_loss": 3.3298863863586847, + "objective/train/tokens_used": 1265642976, + "theoretical_loss": 3.3298863863586847, + "tokens_seen": 2895183872 + }, + { + "epoch": 0.75, + "learning_rate": 2.4779329160648375e-05, + "loss": 2.6174, + "theoretical_loss": 3.3298863863586847, + "tokens_seen": 2895183872 + }, + { + "epoch": 0.75, + "learning_rate": 2.477130476649013e-05, + "loss": 2.5583, + "theoretical_loss": 3.329874711957749, + "tokens_seen": 2895314944 + }, + { + "epoch": 0.75, + "learning_rate": 2.476328037233189e-05, + "loss": 2.4138, + "theoretical_loss": 3.329863038233279, + "tokens_seen": 2895446016 + }, + { + "epoch": 0.75, + "learning_rate": 2.475525597817365e-05, + "loss": 2.4434, + "theoretical_loss": 3.329851365185205, + "tokens_seen": 2895577088 + }, + { + "epoch": 0.76, + "learning_rate": 2.4747231584015408e-05, + "loss": 2.576, + "theoretical_loss": 3.329839692813458, + "tokens_seen": 2895708160 + }, + { + "epoch": 0.76, + "learning_rate": 2.4739207189857168e-05, + "loss": 2.6187, + "theoretical_loss": 3.3298280211179674, + "tokens_seen": 2895839232 + }, + { + "epoch": 0.76, + "learning_rate": 2.4731182795698928e-05, + "loss": 2.5579, + "theoretical_loss": 3.329816350098664, + "tokens_seen": 2895970304 + }, + { + "epoch": 0.76, + "learning_rate": 2.4723158401540687e-05, + "loss": 2.4086, + "theoretical_loss": 3.329804679755478, + "tokens_seen": 2896101376 + }, + { + "epoch": 0.76, + "learning_rate": 2.4715134007382444e-05, + "loss": 2.6609, + "theoretical_loss": 3.3297930100883395, + "tokens_seen": 2896232448 + }, + { + "epoch": 0.76, + "learning_rate": 2.4707109613224204e-05, + "loss": 2.5488, + "theoretical_loss": 3.3297813410971786, + "tokens_seen": 2896363520 + }, + { + "epoch": 0.76, + "learning_rate": 2.4699085219065964e-05, + "loss": 2.6263, + "theoretical_loss": 3.3297696727819255, + "tokens_seen": 2896494592 + }, + { + "epoch": 0.76, + "learning_rate": 2.469106082490772e-05, + "loss": 2.5277, + "theoretical_loss": 3.329758005142511, + "tokens_seen": 2896625664 + }, + { + "epoch": 0.76, + "learning_rate": 2.468303643074948e-05, + "loss": 2.6617, + "theoretical_loss": 3.3297463381788646, + "tokens_seen": 2896756736 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1587920, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.492361068725586, + "objective/train/theoretical_loss": 3.3297405049504327, + "objective/train/tokens_used": 1267281376, + "theoretical_loss": 3.3297405049504327, + "tokens_seen": 2896822272 + }, + { + "epoch": 0.76, + "learning_rate": 2.467501203659124e-05, + "loss": 2.4768, + "theoretical_loss": 3.3297346718909173, + "tokens_seen": 2896887808 + }, + { + "epoch": 0.76, + "learning_rate": 2.4666987642432997e-05, + "loss": 2.4666, + "theoretical_loss": 3.3297230062785985, + "tokens_seen": 2897018880 + }, + { + "epoch": 0.76, + "learning_rate": 2.4658963248274757e-05, + "loss": 2.522, + "theoretical_loss": 3.3297113413418398, + "tokens_seen": 2897149952 + }, + { + "epoch": 0.76, + "learning_rate": 2.4650938854116516e-05, + "loss": 2.5601, + "theoretical_loss": 3.3296996770805705, + "tokens_seen": 2897281024 + }, + { + "epoch": 0.76, + "learning_rate": 2.4642914459958273e-05, + "loss": 2.5957, + "theoretical_loss": 3.3296880134947213, + "tokens_seen": 2897412096 + }, + { + "epoch": 0.76, + "learning_rate": 2.4634890065800033e-05, + "loss": 2.4985, + "theoretical_loss": 3.329676350584222, + "tokens_seen": 2897543168 + }, + { + "epoch": 0.76, + "learning_rate": 2.4626865671641793e-05, + "loss": 2.5963, + "theoretical_loss": 3.329664688349004, + "tokens_seen": 2897674240 + }, + { + "epoch": 0.76, + "learning_rate": 2.461884127748355e-05, + "loss": 2.3936, + "theoretical_loss": 3.329653026788997, + "tokens_seen": 2897805312 + }, + { + "epoch": 0.76, + "learning_rate": 2.461081688332531e-05, + "loss": 2.5022, + "theoretical_loss": 3.3296413659041315, + "tokens_seen": 2897936384 + }, + { + "epoch": 0.76, + "learning_rate": 2.460279248916707e-05, + "loss": 2.6187, + "theoretical_loss": 3.329629705694338, + "tokens_seen": 2898067456 + }, + { + "epoch": 0.76, + "learning_rate": 2.4594768095008826e-05, + "loss": 2.6595, + "theoretical_loss": 3.329618046159546, + "tokens_seen": 2898198528 + }, + { + "epoch": 0.76, + "learning_rate": 2.4586743700850586e-05, + "loss": 2.6449, + "theoretical_loss": 3.329606387299687, + "tokens_seen": 2898329600 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1588625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8180313110351562, + "objective/train/theoretical_loss": 3.3295947291146915, + "objective/train/tokens_used": 1268919776, + "theoretical_loss": 3.3295947291146915, + "tokens_seen": 2898460672 + }, + { + "epoch": 0.76, + "learning_rate": 2.4578719306692345e-05, + "loss": 2.5801, + "theoretical_loss": 3.3295947291146915, + "tokens_seen": 2898460672 + }, + { + "epoch": 0.76, + "learning_rate": 2.4570694912534102e-05, + "loss": 2.604, + "theoretical_loss": 3.329583071604489, + "tokens_seen": 2898591744 + }, + { + "epoch": 0.76, + "learning_rate": 2.4562670518375862e-05, + "loss": 2.5077, + "theoretical_loss": 3.32957141476901, + "tokens_seen": 2898722816 + }, + { + "epoch": 0.76, + "learning_rate": 2.4554646124217622e-05, + "loss": 2.5093, + "theoretical_loss": 3.3295597586081858, + "tokens_seen": 2898853888 + }, + { + "epoch": 0.76, + "learning_rate": 2.454662173005938e-05, + "loss": 2.6125, + "theoretical_loss": 3.3295481031219465, + "tokens_seen": 2898984960 + }, + { + "epoch": 0.76, + "learning_rate": 2.4538597335901138e-05, + "loss": 2.5741, + "theoretical_loss": 3.329536448310222, + "tokens_seen": 2899116032 + }, + { + "epoch": 0.76, + "learning_rate": 2.4530572941742898e-05, + "loss": 2.5092, + "theoretical_loss": 3.3295247941729436, + "tokens_seen": 2899247104 + }, + { + "epoch": 0.76, + "learning_rate": 2.4522548547584658e-05, + "loss": 2.5146, + "theoretical_loss": 3.3295131407100413, + "tokens_seen": 2899378176 + }, + { + "epoch": 0.76, + "learning_rate": 2.4514524153426418e-05, + "loss": 2.5698, + "theoretical_loss": 3.3295014879214455, + "tokens_seen": 2899509248 + }, + { + "epoch": 0.76, + "learning_rate": 2.4506499759268174e-05, + "loss": 2.7011, + "theoretical_loss": 3.329489835807087, + "tokens_seen": 2899640320 + }, + { + "epoch": 0.76, + "learning_rate": 2.4498475365109934e-05, + "loss": 2.602, + "theoretical_loss": 3.329478184366896, + "tokens_seen": 2899771392 + }, + { + "epoch": 0.76, + "learning_rate": 2.4490450970951694e-05, + "loss": 2.5897, + "theoretical_loss": 3.3294665336008036, + "tokens_seen": 2899902464 + }, + { + "epoch": 0.76, + "learning_rate": 2.4482426576793454e-05, + "loss": 2.3861, + "theoretical_loss": 3.3294548835087396, + "tokens_seen": 2900033536 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1589812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.532057046890259, + "objective/train/theoretical_loss": 3.329449058715447, + "objective/train/tokens_used": 1270558176, + "theoretical_loss": 3.329449058715447, + "tokens_seen": 2900099072 + }, + { + "epoch": 0.76, + "learning_rate": 2.447440218263521e-05, + "loss": 2.5234, + "theoretical_loss": 3.3294432340906353, + "tokens_seen": 2900164608 + }, + { + "epoch": 0.76, + "learning_rate": 2.446637778847697e-05, + "loss": 2.5843, + "theoretical_loss": 3.3294315853464207, + "tokens_seen": 2900295680 + }, + { + "epoch": 0.76, + "learning_rate": 2.445835339431873e-05, + "loss": 2.4283, + "theoretical_loss": 3.3294199372760263, + "tokens_seen": 2900426752 + }, + { + "epoch": 0.76, + "learning_rate": 2.4450329000160487e-05, + "loss": 2.5161, + "theoretical_loss": 3.329408289879383, + "tokens_seen": 2900557824 + }, + { + "epoch": 0.76, + "learning_rate": 2.4442304606002247e-05, + "loss": 2.4876, + "theoretical_loss": 3.3293966431564215, + "tokens_seen": 2900688896 + }, + { + "epoch": 0.76, + "learning_rate": 2.4434280211844007e-05, + "loss": 2.5458, + "theoretical_loss": 3.3293849971070717, + "tokens_seen": 2900819968 + }, + { + "epoch": 0.76, + "learning_rate": 2.4426255817685767e-05, + "loss": 2.6196, + "theoretical_loss": 3.329373351731265, + "tokens_seen": 2900951040 + }, + { + "epoch": 0.76, + "learning_rate": 2.4418231423527523e-05, + "loss": 2.6463, + "theoretical_loss": 3.329361707028932, + "tokens_seen": 2901082112 + }, + { + "epoch": 0.76, + "learning_rate": 2.4410207029369283e-05, + "loss": 2.5262, + "theoretical_loss": 3.3293500630000024, + "tokens_seen": 2901213184 + }, + { + "epoch": 0.76, + "learning_rate": 2.4402182635211043e-05, + "loss": 2.6352, + "theoretical_loss": 3.329338419644408, + "tokens_seen": 2901344256 + }, + { + "epoch": 0.76, + "learning_rate": 2.4394158241052803e-05, + "loss": 2.5687, + "theoretical_loss": 3.3293267769620787, + "tokens_seen": 2901475328 + }, + { + "epoch": 0.76, + "learning_rate": 2.438613384689456e-05, + "loss": 2.6855, + "theoretical_loss": 3.329315134952945, + "tokens_seen": 2901606400 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1590498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.820100784301758, + "objective/train/theoretical_loss": 3.3293034936169383, + "objective/train/tokens_used": 1272196576, + "theoretical_loss": 3.3293034936169383, + "tokens_seen": 2901737472 + }, + { + "epoch": 0.76, + "learning_rate": 2.437810945273632e-05, + "loss": 2.5648, + "theoretical_loss": 3.3293034936169383, + "tokens_seen": 2901737472 + }, + { + "epoch": 0.76, + "learning_rate": 2.437008505857808e-05, + "loss": 2.5316, + "theoretical_loss": 3.3292918529539888, + "tokens_seen": 2901868544 + }, + { + "epoch": 0.76, + "learning_rate": 2.4362060664419836e-05, + "loss": 2.6159, + "theoretical_loss": 3.3292802129640275, + "tokens_seen": 2901999616 + }, + { + "epoch": 0.76, + "learning_rate": 2.4354036270261596e-05, + "loss": 2.3878, + "theoretical_loss": 3.329268573646985, + "tokens_seen": 2902130688 + }, + { + "epoch": 0.76, + "learning_rate": 2.4346011876103356e-05, + "loss": 2.6399, + "theoretical_loss": 3.3292569350027916, + "tokens_seen": 2902261760 + }, + { + "epoch": 0.76, + "learning_rate": 2.4337987481945116e-05, + "loss": 2.5045, + "theoretical_loss": 3.3292452970313784, + "tokens_seen": 2902392832 + }, + { + "epoch": 0.76, + "learning_rate": 2.4329963087786872e-05, + "loss": 2.5863, + "theoretical_loss": 3.329233659732676, + "tokens_seen": 2902523904 + }, + { + "epoch": 0.76, + "learning_rate": 2.4321938693628632e-05, + "loss": 2.672, + "theoretical_loss": 3.3292220231066154, + "tokens_seen": 2902654976 + }, + { + "epoch": 0.76, + "learning_rate": 2.4313914299470392e-05, + "loss": 2.7881, + "theoretical_loss": 3.329210387153127, + "tokens_seen": 2902786048 + }, + { + "epoch": 0.76, + "learning_rate": 2.4305889905312152e-05, + "loss": 2.4103, + "theoretical_loss": 3.3291987518721418, + "tokens_seen": 2902917120 + }, + { + "epoch": 0.76, + "learning_rate": 2.4297865511153908e-05, + "loss": 2.4887, + "theoretical_loss": 3.3291871172635905, + "tokens_seen": 2903048192 + }, + { + "epoch": 0.76, + "learning_rate": 2.4289841116995668e-05, + "loss": 2.5342, + "theoretical_loss": 3.3291754833274037, + "tokens_seen": 2903179264 + }, + { + "epoch": 0.76, + "learning_rate": 2.4281816722837428e-05, + "loss": 2.433, + "theoretical_loss": 3.3291638500635123, + "tokens_seen": 2903310336 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1591523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1737945079803467, + "objective/train/theoretical_loss": 3.329158033683656, + "objective/train/tokens_used": 1273834976, + "theoretical_loss": 3.329158033683656, + "tokens_seen": 2903375872 + }, + { + "epoch": 0.76, + "learning_rate": 2.4273792328679185e-05, + "loss": 2.5738, + "theoretical_loss": 3.3291522174718473, + "tokens_seen": 2903441408 + }, + { + "epoch": 0.76, + "learning_rate": 2.4265767934520945e-05, + "loss": 2.5013, + "theoretical_loss": 3.3291405855523393, + "tokens_seen": 2903572480 + }, + { + "epoch": 0.76, + "learning_rate": 2.4257743540362704e-05, + "loss": 2.566, + "theoretical_loss": 3.3291289543049194, + "tokens_seen": 2903703552 + }, + { + "epoch": 0.76, + "learning_rate": 2.4249719146204464e-05, + "loss": 2.6115, + "theoretical_loss": 3.329117323729518, + "tokens_seen": 2903834624 + }, + { + "epoch": 0.76, + "learning_rate": 2.424169475204622e-05, + "loss": 2.5392, + "theoretical_loss": 3.329105693826066, + "tokens_seen": 2903965696 + }, + { + "epoch": 0.76, + "learning_rate": 2.423367035788798e-05, + "loss": 2.6932, + "theoretical_loss": 3.3290940645944946, + "tokens_seen": 2904096768 + }, + { + "epoch": 0.76, + "learning_rate": 2.422564596372974e-05, + "loss": 2.4283, + "theoretical_loss": 3.3290824360347346, + "tokens_seen": 2904227840 + }, + { + "epoch": 0.76, + "learning_rate": 2.42176215695715e-05, + "loss": 2.4441, + "theoretical_loss": 3.329070808146717, + "tokens_seen": 2904358912 + }, + { + "epoch": 0.76, + "learning_rate": 2.4209597175413257e-05, + "loss": 2.5265, + "theoretical_loss": 3.329059180930372, + "tokens_seen": 2904489984 + }, + { + "epoch": 0.76, + "learning_rate": 2.4201572781255017e-05, + "loss": 2.5377, + "theoretical_loss": 3.329047554385631, + "tokens_seen": 2904621056 + }, + { + "epoch": 0.76, + "learning_rate": 2.4193548387096777e-05, + "loss": 2.4765, + "theoretical_loss": 3.3290359285124245, + "tokens_seen": 2904752128 + }, + { + "epoch": 0.76, + "learning_rate": 2.4185523992938533e-05, + "loss": 2.6014, + "theoretical_loss": 3.329024303310684, + "tokens_seen": 2904883200 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1591966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6324024200439453, + "objective/train/theoretical_loss": 3.32901267878034, + "objective/train/tokens_used": 1275473376, + "theoretical_loss": 3.32901267878034, + "tokens_seen": 2905014272 + }, + { + "epoch": 0.76, + "learning_rate": 2.4177499598780293e-05, + "loss": 2.6151, + "theoretical_loss": 3.32901267878034, + "tokens_seen": 2905014272 + }, + { + "epoch": 0.76, + "learning_rate": 2.4169475204622053e-05, + "loss": 2.5197, + "theoretical_loss": 3.3290010549213243, + "tokens_seen": 2905145344 + }, + { + "epoch": 0.76, + "learning_rate": 2.4161450810463813e-05, + "loss": 2.5164, + "theoretical_loss": 3.3289894317335667, + "tokens_seen": 2905276416 + }, + { + "epoch": 0.76, + "learning_rate": 2.415342641630557e-05, + "loss": 2.5791, + "theoretical_loss": 3.3289778092169984, + "tokens_seen": 2905407488 + }, + { + "epoch": 0.76, + "learning_rate": 2.414540202214733e-05, + "loss": 2.5896, + "theoretical_loss": 3.3289661873715506, + "tokens_seen": 2905538560 + }, + { + "epoch": 0.76, + "learning_rate": 2.413737762798909e-05, + "loss": 2.5454, + "theoretical_loss": 3.3289545661971545, + "tokens_seen": 2905669632 + }, + { + "epoch": 0.76, + "learning_rate": 2.412935323383085e-05, + "loss": 2.6444, + "theoretical_loss": 3.3289429456937403, + "tokens_seen": 2905800704 + }, + { + "epoch": 0.76, + "learning_rate": 2.4121328839672606e-05, + "loss": 2.464, + "theoretical_loss": 3.32893132586124, + "tokens_seen": 2905931776 + }, + { + "epoch": 0.76, + "learning_rate": 2.4113304445514366e-05, + "loss": 2.6204, + "theoretical_loss": 3.328919706699584, + "tokens_seen": 2906062848 + }, + { + "epoch": 0.76, + "learning_rate": 2.4105280051356126e-05, + "loss": 2.5604, + "theoretical_loss": 3.3289080882087037, + "tokens_seen": 2906193920 + }, + { + "epoch": 0.76, + "learning_rate": 2.4097255657197882e-05, + "loss": 2.5527, + "theoretical_loss": 3.3288964703885293, + "tokens_seen": 2906324992 + }, + { + "epoch": 0.76, + "learning_rate": 2.4089231263039642e-05, + "loss": 2.6141, + "theoretical_loss": 3.3288848532389927, + "tokens_seen": 2906456064 + }, + { + "epoch": 0.76, + "learning_rate": 2.4081206868881402e-05, + "loss": 2.5182, + "theoretical_loss": 3.3288732367600247, + "tokens_seen": 2906587136 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1593138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.510889768600464, + "objective/train/theoretical_loss": 3.328867428771982, + "objective/train/tokens_used": 1277111776, + "theoretical_loss": 3.328867428771982, + "tokens_seen": 2906652672 + }, + { + "epoch": 0.76, + "learning_rate": 2.4073182474723162e-05, + "loss": 2.5843, + "theoretical_loss": 3.3288616209515562, + "tokens_seen": 2906718208 + }, + { + "epoch": 0.76, + "learning_rate": 2.406515808056492e-05, + "loss": 2.4194, + "theoretical_loss": 3.3288500058135186, + "tokens_seen": 2906849280 + }, + { + "epoch": 0.76, + "learning_rate": 2.405713368640668e-05, + "loss": 2.7761, + "theoretical_loss": 3.3288383913458426, + "tokens_seen": 2906980352 + }, + { + "epoch": 0.76, + "learning_rate": 2.4049109292248438e-05, + "loss": 2.5779, + "theoretical_loss": 3.3288267775484592, + "tokens_seen": 2907111424 + }, + { + "epoch": 0.76, + "learning_rate": 2.4041084898090198e-05, + "loss": 2.6168, + "theoretical_loss": 3.3288151644213, + "tokens_seen": 2907242496 + }, + { + "epoch": 0.76, + "learning_rate": 2.4033060503931955e-05, + "loss": 2.744, + "theoretical_loss": 3.328803551964296, + "tokens_seen": 2907373568 + }, + { + "epoch": 0.76, + "learning_rate": 2.4025036109773715e-05, + "loss": 2.6414, + "theoretical_loss": 3.3287919401773776, + "tokens_seen": 2907504640 + }, + { + "epoch": 0.76, + "learning_rate": 2.4017011715615474e-05, + "loss": 2.5549, + "theoretical_loss": 3.3287803290604767, + "tokens_seen": 2907635712 + }, + { + "epoch": 0.76, + "learning_rate": 2.400898732145723e-05, + "loss": 2.5827, + "theoretical_loss": 3.3287687186135244, + "tokens_seen": 2907766784 + }, + { + "epoch": 0.76, + "learning_rate": 2.400096292729899e-05, + "loss": 2.4176, + "theoretical_loss": 3.328757108836452, + "tokens_seen": 2907897856 + }, + { + "epoch": 0.76, + "learning_rate": 2.399293853314075e-05, + "loss": 2.5718, + "theoretical_loss": 3.3287454997291896, + "tokens_seen": 2908028928 + }, + { + "epoch": 0.76, + "learning_rate": 2.3984914138982507e-05, + "loss": 2.5868, + "theoretical_loss": 3.3287338912916695, + "tokens_seen": 2908160000 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1594326, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5529305934906006, + "objective/train/theoretical_loss": 3.3287222835238226, + "objective/train/tokens_used": 1278750176, + "theoretical_loss": 3.3287222835238226, + "tokens_seen": 2908291072 + }, + { + "epoch": 0.76, + "learning_rate": 2.3976889744824267e-05, + "loss": 2.6648, + "theoretical_loss": 3.3287222835238226, + "tokens_seen": 2908291072 + }, + { + "epoch": 0.76, + "learning_rate": 2.3968865350666027e-05, + "loss": 2.5351, + "theoretical_loss": 3.3287106764255796, + "tokens_seen": 2908422144 + }, + { + "epoch": 0.76, + "learning_rate": 2.3960840956507784e-05, + "loss": 2.5594, + "theoretical_loss": 3.3286990699968726, + "tokens_seen": 2908553216 + }, + { + "epoch": 0.76, + "learning_rate": 2.3952816562349544e-05, + "loss": 2.6094, + "theoretical_loss": 3.328687464237632, + "tokens_seen": 2908684288 + }, + { + "epoch": 0.76, + "learning_rate": 2.39447921681913e-05, + "loss": 2.5469, + "theoretical_loss": 3.328675859147789, + "tokens_seen": 2908815360 + }, + { + "epoch": 0.76, + "learning_rate": 2.393676777403306e-05, + "loss": 2.5171, + "theoretical_loss": 3.328664254727275, + "tokens_seen": 2908946432 + }, + { + "epoch": 0.76, + "learning_rate": 2.392874337987482e-05, + "loss": 2.5231, + "theoretical_loss": 3.328652650976022, + "tokens_seen": 2909077504 + }, + { + "epoch": 0.76, + "learning_rate": 2.392071898571658e-05, + "loss": 2.7872, + "theoretical_loss": 3.32864104789396, + "tokens_seen": 2909208576 + }, + { + "epoch": 0.76, + "learning_rate": 2.3912694591558336e-05, + "loss": 2.7629, + "theoretical_loss": 3.3286294454810217, + "tokens_seen": 2909339648 + }, + { + "epoch": 0.76, + "learning_rate": 2.3904670197400096e-05, + "loss": 2.5999, + "theoretical_loss": 3.328617843737137, + "tokens_seen": 2909470720 + }, + { + "epoch": 0.76, + "learning_rate": 2.3896645803241856e-05, + "loss": 2.4985, + "theoretical_loss": 3.3286062426622376, + "tokens_seen": 2909601792 + }, + { + "epoch": 0.76, + "learning_rate": 2.3888621409083613e-05, + "loss": 2.6355, + "theoretical_loss": 3.328594642256255, + "tokens_seen": 2909732864 + }, + { + "epoch": 0.76, + "learning_rate": 2.3880597014925373e-05, + "loss": 2.5797, + "theoretical_loss": 3.3285830425191207, + "tokens_seen": 2909863936 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1594956, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6602909564971924, + "objective/train/theoretical_loss": 3.32857724290135, + "objective/train/tokens_used": 1280388576, + "theoretical_loss": 3.32857724290135, + "tokens_seen": 2909929472 + }, + { + "epoch": 0.76, + "learning_rate": 2.3872572620767132e-05, + "loss": 2.6056, + "theoretical_loss": 3.3285714434507656, + "tokens_seen": 2909995008 + }, + { + "epoch": 0.76, + "learning_rate": 2.3864548226608892e-05, + "loss": 2.5946, + "theoretical_loss": 3.328559845051121, + "tokens_seen": 2910126080 + }, + { + "epoch": 0.76, + "learning_rate": 2.385652383245065e-05, + "loss": 2.4256, + "theoretical_loss": 3.3285482473201187, + "tokens_seen": 2910257152 + }, + { + "epoch": 0.76, + "learning_rate": 2.384849943829241e-05, + "loss": 2.6181, + "theoretical_loss": 3.328536650257689, + "tokens_seen": 2910388224 + }, + { + "epoch": 0.76, + "learning_rate": 2.384047504413417e-05, + "loss": 2.6696, + "theoretical_loss": 3.328525053863765, + "tokens_seen": 2910519296 + }, + { + "epoch": 0.76, + "learning_rate": 2.383245064997593e-05, + "loss": 2.7521, + "theoretical_loss": 3.3285134581382763, + "tokens_seen": 2910650368 + }, + { + "epoch": 0.76, + "learning_rate": 2.3824426255817685e-05, + "loss": 2.598, + "theoretical_loss": 3.328501863081155, + "tokens_seen": 2910781440 + }, + { + "epoch": 0.76, + "learning_rate": 2.3816401861659445e-05, + "loss": 2.5603, + "theoretical_loss": 3.328490268692333, + "tokens_seen": 2910912512 + }, + { + "epoch": 0.76, + "learning_rate": 2.3808377467501205e-05, + "loss": 2.6252, + "theoretical_loss": 3.328478674971741, + "tokens_seen": 2911043584 + }, + { + "epoch": 0.76, + "learning_rate": 2.380035307334296e-05, + "loss": 2.5996, + "theoretical_loss": 3.3284670819193103, + "tokens_seen": 2911174656 + }, + { + "epoch": 0.76, + "learning_rate": 2.379232867918472e-05, + "loss": 2.5814, + "theoretical_loss": 3.328455489534973, + "tokens_seen": 2911305728 + }, + { + "epoch": 0.76, + "learning_rate": 2.378430428502648e-05, + "loss": 2.4858, + "theoretical_loss": 3.32844389781866, + "tokens_seen": 2911436800 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1595936, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.731386423110962, + "objective/train/theoretical_loss": 3.3284323067703028, + "objective/train/tokens_used": 1282026976, + "theoretical_loss": 3.3284323067703028, + "tokens_seen": 2911567872 + }, + { + "epoch": 0.76, + "learning_rate": 2.377627989086824e-05, + "loss": 2.639, + "theoretical_loss": 3.3284323067703028, + "tokens_seen": 2911567872 + }, + { + "epoch": 0.76, + "learning_rate": 2.3768255496709998e-05, + "loss": 2.4485, + "theoretical_loss": 3.3284207163898327, + "tokens_seen": 2911698944 + }, + { + "epoch": 0.76, + "learning_rate": 2.3760231102551758e-05, + "loss": 2.5002, + "theoretical_loss": 3.328409126677182, + "tokens_seen": 2911830016 + }, + { + "epoch": 0.76, + "learning_rate": 2.3752206708393518e-05, + "loss": 2.7108, + "theoretical_loss": 3.3283975376322807, + "tokens_seen": 2911961088 + }, + { + "epoch": 0.76, + "learning_rate": 2.3744182314235277e-05, + "loss": 2.5345, + "theoretical_loss": 3.328385949255061, + "tokens_seen": 2912092160 + }, + { + "epoch": 0.77, + "learning_rate": 2.3736157920077034e-05, + "loss": 2.5079, + "theoretical_loss": 3.328374361545455, + "tokens_seen": 2912223232 + }, + { + "epoch": 0.77, + "learning_rate": 2.3728133525918794e-05, + "loss": 2.525, + "theoretical_loss": 3.3283627745033937, + "tokens_seen": 2912354304 + }, + { + "epoch": 0.77, + "learning_rate": 2.3720109131760554e-05, + "loss": 2.3641, + "theoretical_loss": 3.3283511881288086, + "tokens_seen": 2912485376 + }, + { + "epoch": 0.77, + "learning_rate": 2.371208473760231e-05, + "loss": 2.5768, + "theoretical_loss": 3.328339602421631, + "tokens_seen": 2912616448 + }, + { + "epoch": 0.77, + "learning_rate": 2.370406034344407e-05, + "loss": 2.657, + "theoretical_loss": 3.3283280173817924, + "tokens_seen": 2912747520 + }, + { + "epoch": 0.77, + "learning_rate": 2.369603594928583e-05, + "loss": 2.5195, + "theoretical_loss": 3.328316433009225, + "tokens_seen": 2912878592 + }, + { + "epoch": 0.77, + "learning_rate": 2.368801155512759e-05, + "loss": 2.5212, + "theoretical_loss": 3.3283048493038594, + "tokens_seen": 2913009664 + }, + { + "epoch": 0.77, + "learning_rate": 2.3679987160969346e-05, + "loss": 2.5772, + "theoretical_loss": 3.3282932662656277, + "tokens_seen": 2913140736 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1596519, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.513598680496216, + "objective/train/theoretical_loss": 3.328287474996666, + "objective/train/tokens_used": 1283665376, + "theoretical_loss": 3.328287474996666, + "tokens_seen": 2913206272 + }, + { + "epoch": 0.77, + "learning_rate": 2.3671962766811106e-05, + "loss": 2.5505, + "theoretical_loss": 3.3282816838944616, + "tokens_seen": 2913271808 + }, + { + "epoch": 0.77, + "learning_rate": 2.3663938372652866e-05, + "loss": 2.5471, + "theoretical_loss": 3.3282701021902925, + "tokens_seen": 2913402880 + }, + { + "epoch": 0.77, + "learning_rate": 2.3655913978494626e-05, + "loss": 2.5365, + "theoretical_loss": 3.3282585211530518, + "tokens_seen": 2913533952 + }, + { + "epoch": 0.77, + "learning_rate": 2.3647889584336383e-05, + "loss": 2.6141, + "theoretical_loss": 3.3282469407826714, + "tokens_seen": 2913665024 + }, + { + "epoch": 0.77, + "learning_rate": 2.3639865190178143e-05, + "loss": 2.5661, + "theoretical_loss": 3.3282353610790825, + "tokens_seen": 2913796096 + }, + { + "epoch": 0.77, + "learning_rate": 2.3631840796019903e-05, + "loss": 2.644, + "theoretical_loss": 3.328223782042217, + "tokens_seen": 2913927168 + }, + { + "epoch": 0.77, + "learning_rate": 2.362381640186166e-05, + "loss": 2.6101, + "theoretical_loss": 3.328212203672007, + "tokens_seen": 2914058240 + }, + { + "epoch": 0.77, + "learning_rate": 2.361579200770342e-05, + "loss": 2.5837, + "theoretical_loss": 3.328200625968383, + "tokens_seen": 2914189312 + }, + { + "epoch": 0.77, + "learning_rate": 2.360776761354518e-05, + "loss": 2.6299, + "theoretical_loss": 3.3281890489312778, + "tokens_seen": 2914320384 + }, + { + "epoch": 0.77, + "learning_rate": 2.359974321938694e-05, + "loss": 2.5812, + "theoretical_loss": 3.3281774725606224, + "tokens_seen": 2914451456 + }, + { + "epoch": 0.77, + "learning_rate": 2.3591718825228695e-05, + "loss": 2.4741, + "theoretical_loss": 3.3281658968563486, + "tokens_seen": 2914582528 + }, + { + "epoch": 0.77, + "learning_rate": 2.3583694431070455e-05, + "loss": 2.6254, + "theoretical_loss": 3.3281543218183876, + "tokens_seen": 2914713600 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1597740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8263800144195557, + "objective/train/theoretical_loss": 3.3281427474466723, + "objective/train/tokens_used": 1285303776, + "theoretical_loss": 3.3281427474466723, + "tokens_seen": 2914844672 + }, + { + "epoch": 0.77, + "learning_rate": 2.3575670036912215e-05, + "loss": 2.6411, + "theoretical_loss": 3.3281427474466723, + "tokens_seen": 2914844672 + }, + { + "epoch": 0.77, + "learning_rate": 2.3567645642753975e-05, + "loss": 2.5572, + "theoretical_loss": 3.328131173741133, + "tokens_seen": 2914975744 + }, + { + "epoch": 0.77, + "learning_rate": 2.355962124859573e-05, + "loss": 2.4096, + "theoretical_loss": 3.328119600701702, + "tokens_seen": 2915106816 + }, + { + "epoch": 0.77, + "learning_rate": 2.355159685443749e-05, + "loss": 2.4784, + "theoretical_loss": 3.3281080283283115, + "tokens_seen": 2915237888 + }, + { + "epoch": 0.77, + "learning_rate": 2.354357246027925e-05, + "loss": 2.5974, + "theoretical_loss": 3.3280964566208926, + "tokens_seen": 2915368960 + }, + { + "epoch": 0.77, + "learning_rate": 2.353554806612101e-05, + "loss": 2.421, + "theoretical_loss": 3.3280848855793774, + "tokens_seen": 2915500032 + }, + { + "epoch": 0.77, + "learning_rate": 2.3527523671962768e-05, + "loss": 2.6717, + "theoretical_loss": 3.3280733152036976, + "tokens_seen": 2915631104 + }, + { + "epoch": 0.77, + "learning_rate": 2.3519499277804528e-05, + "loss": 2.6119, + "theoretical_loss": 3.328061745493785, + "tokens_seen": 2915762176 + }, + { + "epoch": 0.77, + "learning_rate": 2.3511474883646288e-05, + "loss": 2.6485, + "theoretical_loss": 3.3280501764495707, + "tokens_seen": 2915893248 + }, + { + "epoch": 0.77, + "learning_rate": 2.3503450489488044e-05, + "loss": 2.5186, + "theoretical_loss": 3.3280386080709867, + "tokens_seen": 2916024320 + }, + { + "epoch": 0.77, + "learning_rate": 2.3495426095329804e-05, + "loss": 2.6661, + "theoretical_loss": 3.3280270403579655, + "tokens_seen": 2916155392 + }, + { + "epoch": 0.77, + "learning_rate": 2.3487401701171564e-05, + "loss": 2.4079, + "theoretical_loss": 3.3280154733104386, + "tokens_seen": 2916286464 + }, + { + "epoch": 0.77, + "learning_rate": 2.3479377307013324e-05, + "loss": 2.636, + "theoretical_loss": 3.3280039069283376, + "tokens_seen": 2916417536 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1598807, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7069852352142334, + "objective/train/theoretical_loss": 3.3279981239868004, + "objective/train/tokens_used": 1286942176, + "theoretical_loss": 3.3279981239868004, + "tokens_seen": 2916483072 + }, + { + "epoch": 0.77, + "learning_rate": 2.347135291285508e-05, + "loss": 2.5134, + "theoretical_loss": 3.327992341211594, + "tokens_seen": 2916548608 + }, + { + "epoch": 0.77, + "learning_rate": 2.346332851869684e-05, + "loss": 2.603, + "theoretical_loss": 3.3279807761601403, + "tokens_seen": 2916679680 + }, + { + "epoch": 0.77, + "learning_rate": 2.34553041245386e-05, + "loss": 2.4866, + "theoretical_loss": 3.327969211773908, + "tokens_seen": 2916810752 + }, + { + "epoch": 0.77, + "learning_rate": 2.344727973038036e-05, + "loss": 2.6634, + "theoretical_loss": 3.327957648052829, + "tokens_seen": 2916941824 + }, + { + "epoch": 0.77, + "learning_rate": 2.3439255336222117e-05, + "loss": 2.6067, + "theoretical_loss": 3.327946084996835, + "tokens_seen": 2917072896 + }, + { + "epoch": 0.77, + "learning_rate": 2.3431230942063876e-05, + "loss": 2.4639, + "theoretical_loss": 3.327934522605858, + "tokens_seen": 2917203968 + }, + { + "epoch": 0.77, + "learning_rate": 2.3423206547905636e-05, + "loss": 2.6279, + "theoretical_loss": 3.3279229608798304, + "tokens_seen": 2917335040 + }, + { + "epoch": 0.77, + "learning_rate": 2.3415182153747393e-05, + "loss": 2.4701, + "theoretical_loss": 3.3279113998186833, + "tokens_seen": 2917466112 + }, + { + "epoch": 0.77, + "learning_rate": 2.3407157759589153e-05, + "loss": 2.597, + "theoretical_loss": 3.3278998394223485, + "tokens_seen": 2917597184 + }, + { + "epoch": 0.77, + "learning_rate": 2.3399133365430913e-05, + "loss": 2.6115, + "theoretical_loss": 3.3278882796907587, + "tokens_seen": 2917728256 + }, + { + "epoch": 0.77, + "learning_rate": 2.3391108971272673e-05, + "loss": 2.58, + "theoretical_loss": 3.3278767206238453, + "tokens_seen": 2917859328 + }, + { + "epoch": 0.77, + "learning_rate": 2.338308457711443e-05, + "loss": 2.5901, + "theoretical_loss": 3.3278651622215403, + "tokens_seen": 2917990400 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1599566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.475285053253174, + "objective/train/theoretical_loss": 3.327853604483775, + "objective/train/tokens_used": 1288580576, + "theoretical_loss": 3.327853604483775, + "tokens_seen": 2918121472 + }, + { + "epoch": 0.77, + "learning_rate": 2.337506018295619e-05, + "loss": 2.4965, + "theoretical_loss": 3.327853604483775, + "tokens_seen": 2918121472 + }, + { + "epoch": 0.77, + "learning_rate": 2.336703578879795e-05, + "loss": 2.5344, + "theoretical_loss": 3.327842047410483, + "tokens_seen": 2918252544 + }, + { + "epoch": 0.77, + "learning_rate": 2.3359011394639705e-05, + "loss": 2.5949, + "theoretical_loss": 3.327830491001595, + "tokens_seen": 2918383616 + }, + { + "epoch": 0.77, + "learning_rate": 2.3350987000481465e-05, + "loss": 2.6555, + "theoretical_loss": 3.327818935257043, + "tokens_seen": 2918514688 + }, + { + "epoch": 0.77, + "learning_rate": 2.3342962606323225e-05, + "loss": 2.5427, + "theoretical_loss": 3.3278073801767594, + "tokens_seen": 2918645760 + }, + { + "epoch": 0.77, + "learning_rate": 2.3334938212164982e-05, + "loss": 2.6132, + "theoretical_loss": 3.327795825760676, + "tokens_seen": 2918776832 + }, + { + "epoch": 0.77, + "learning_rate": 2.3326913818006742e-05, + "loss": 2.3645, + "theoretical_loss": 3.3277842720087243, + "tokens_seen": 2918907904 + }, + { + "epoch": 0.77, + "learning_rate": 2.33188894238485e-05, + "loss": 2.535, + "theoretical_loss": 3.3277727189208375, + "tokens_seen": 2919038976 + }, + { + "epoch": 0.77, + "learning_rate": 2.3310865029690258e-05, + "loss": 2.5458, + "theoretical_loss": 3.3277611664969466, + "tokens_seen": 2919170048 + }, + { + "epoch": 0.77, + "learning_rate": 2.3302840635532018e-05, + "loss": 2.3878, + "theoretical_loss": 3.327749614736984, + "tokens_seen": 2919301120 + }, + { + "epoch": 0.77, + "learning_rate": 2.3294816241373778e-05, + "loss": 2.6375, + "theoretical_loss": 3.327738063640881, + "tokens_seen": 2919432192 + }, + { + "epoch": 0.77, + "learning_rate": 2.3286791847215534e-05, + "loss": 2.5007, + "theoretical_loss": 3.3277265132085714, + "tokens_seen": 2919563264 + }, + { + "epoch": 0.77, + "learning_rate": 2.3278767453057294e-05, + "loss": 2.4987, + "theoretical_loss": 3.3277149634399854, + "tokens_seen": 2919694336 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1600503, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6972103118896484, + "objective/train/theoretical_loss": 3.327709188804568, + "objective/train/tokens_used": 1290218976, + "theoretical_loss": 3.327709188804568, + "tokens_seen": 2919759872 + }, + { + "epoch": 0.77, + "learning_rate": 2.3270743058899054e-05, + "loss": 2.6001, + "theoretical_loss": 3.327703414335056, + "tokens_seen": 2919825408 + }, + { + "epoch": 0.77, + "learning_rate": 2.326271866474081e-05, + "loss": 2.5022, + "theoretical_loss": 3.3276918658937156, + "tokens_seen": 2919956480 + }, + { + "epoch": 0.77, + "learning_rate": 2.325469427058257e-05, + "loss": 2.5821, + "theoretical_loss": 3.3276803181158954, + "tokens_seen": 2920087552 + }, + { + "epoch": 0.77, + "learning_rate": 2.324666987642433e-05, + "loss": 2.3761, + "theoretical_loss": 3.3276687710015276, + "tokens_seen": 2920218624 + }, + { + "epoch": 0.77, + "learning_rate": 2.3238645482266087e-05, + "loss": 2.5287, + "theoretical_loss": 3.327657224550545, + "tokens_seen": 2920349696 + }, + { + "epoch": 0.77, + "learning_rate": 2.3230621088107847e-05, + "loss": 2.4937, + "theoretical_loss": 3.327645678762879, + "tokens_seen": 2920480768 + }, + { + "epoch": 0.77, + "learning_rate": 2.3222596693949607e-05, + "loss": 2.6609, + "theoretical_loss": 3.3276341336384627, + "tokens_seen": 2920611840 + }, + { + "epoch": 0.77, + "learning_rate": 2.3214572299791367e-05, + "loss": 2.4948, + "theoretical_loss": 3.3276225891772273, + "tokens_seen": 2920742912 + }, + { + "epoch": 0.77, + "learning_rate": 2.3206547905633123e-05, + "loss": 2.5396, + "theoretical_loss": 3.327611045379105, + "tokens_seen": 2920873984 + }, + { + "epoch": 0.77, + "learning_rate": 2.3198523511474883e-05, + "loss": 2.3764, + "theoretical_loss": 3.3275995022440283, + "tokens_seen": 2921005056 + }, + { + "epoch": 0.77, + "learning_rate": 2.3190499117316643e-05, + "loss": 2.4582, + "theoretical_loss": 3.3275879597719293, + "tokens_seen": 2921136128 + }, + { + "epoch": 0.77, + "learning_rate": 2.3182474723158403e-05, + "loss": 2.5642, + "theoretical_loss": 3.32757641796274, + "tokens_seen": 2921267200 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1601768, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7379355430603027, + "objective/train/theoretical_loss": 3.327564876816393, + "objective/train/tokens_used": 1291857376, + "theoretical_loss": 3.327564876816393, + "tokens_seen": 2921398272 + }, + { + "epoch": 0.77, + "learning_rate": 2.317445032900016e-05, + "loss": 2.5506, + "theoretical_loss": 3.327564876816393, + "tokens_seen": 2921398272 + }, + { + "epoch": 0.77, + "learning_rate": 2.316642593484192e-05, + "loss": 2.3653, + "theoretical_loss": 3.3275533363328202, + "tokens_seen": 2921529344 + }, + { + "epoch": 0.77, + "learning_rate": 2.315840154068368e-05, + "loss": 2.4172, + "theoretical_loss": 3.3275417965119534, + "tokens_seen": 2921660416 + }, + { + "epoch": 0.77, + "learning_rate": 2.3150377146525436e-05, + "loss": 2.4557, + "theoretical_loss": 3.3275302573537258, + "tokens_seen": 2921791488 + }, + { + "epoch": 0.77, + "learning_rate": 2.3142352752367196e-05, + "loss": 2.5714, + "theoretical_loss": 3.3275187188580686, + "tokens_seen": 2921922560 + }, + { + "epoch": 0.77, + "learning_rate": 2.3134328358208956e-05, + "loss": 2.516, + "theoretical_loss": 3.327507181024915, + "tokens_seen": 2922053632 + }, + { + "epoch": 0.77, + "learning_rate": 2.3126303964050716e-05, + "loss": 2.5968, + "theoretical_loss": 3.3274956438541965, + "tokens_seen": 2922184704 + }, + { + "epoch": 0.77, + "learning_rate": 2.3118279569892472e-05, + "loss": 2.4174, + "theoretical_loss": 3.327484107345845, + "tokens_seen": 2922315776 + }, + { + "epoch": 0.77, + "learning_rate": 2.3110255175734232e-05, + "loss": 2.3954, + "theoretical_loss": 3.3274725714997944, + "tokens_seen": 2922446848 + }, + { + "epoch": 0.77, + "learning_rate": 2.3102230781575992e-05, + "loss": 2.6324, + "theoretical_loss": 3.3274610363159756, + "tokens_seen": 2922577920 + }, + { + "epoch": 0.77, + "learning_rate": 2.3094206387417752e-05, + "loss": 2.5569, + "theoretical_loss": 3.327449501794321, + "tokens_seen": 2922708992 + }, + { + "epoch": 0.77, + "learning_rate": 2.308618199325951e-05, + "loss": 2.3876, + "theoretical_loss": 3.327437967934763, + "tokens_seen": 2922840064 + }, + { + "epoch": 0.77, + "learning_rate": 2.3078157599101268e-05, + "loss": 2.583, + "theoretical_loss": 3.3274264347372347, + "tokens_seen": 2922971136 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1602436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.276355266571045, + "objective/train/theoretical_loss": 3.3274206683867096, + "objective/train/tokens_used": 1293495776, + "theoretical_loss": 3.3274206683867096, + "tokens_seen": 2923036672 + }, + { + "epoch": 0.77, + "learning_rate": 2.3070133204943028e-05, + "loss": 2.532, + "theoretical_loss": 3.3274149022016672, + "tokens_seen": 2923102208 + }, + { + "epoch": 0.77, + "learning_rate": 2.3062108810784785e-05, + "loss": 2.4712, + "theoretical_loss": 3.3274033703279935, + "tokens_seen": 2923233280 + }, + { + "epoch": 0.77, + "learning_rate": 2.3054084416626545e-05, + "loss": 2.4571, + "theoretical_loss": 3.327391839116146, + "tokens_seen": 2923364352 + }, + { + "epoch": 0.77, + "learning_rate": 2.3046060022468305e-05, + "loss": 2.5444, + "theoretical_loss": 3.3273803085660565, + "tokens_seen": 2923495424 + }, + { + "epoch": 0.77, + "learning_rate": 2.3038035628310064e-05, + "loss": 2.5438, + "theoretical_loss": 3.327368778677658, + "tokens_seen": 2923626496 + }, + { + "epoch": 0.77, + "learning_rate": 2.303001123415182e-05, + "loss": 2.5622, + "theoretical_loss": 3.3273572494508823, + "tokens_seen": 2923757568 + }, + { + "epoch": 0.77, + "learning_rate": 2.302198683999358e-05, + "loss": 2.5591, + "theoretical_loss": 3.3273457208856625, + "tokens_seen": 2923888640 + }, + { + "epoch": 0.77, + "learning_rate": 2.301396244583534e-05, + "loss": 2.5, + "theoretical_loss": 3.32733419298193, + "tokens_seen": 2924019712 + }, + { + "epoch": 0.77, + "learning_rate": 2.30059380516771e-05, + "loss": 2.623, + "theoretical_loss": 3.327322665739618, + "tokens_seen": 2924150784 + }, + { + "epoch": 0.77, + "learning_rate": 2.2997913657518857e-05, + "loss": 2.5349, + "theoretical_loss": 3.3273111391586587, + "tokens_seen": 2924281856 + }, + { + "epoch": 0.77, + "learning_rate": 2.2989889263360617e-05, + "loss": 2.5961, + "theoretical_loss": 3.3272996132389845, + "tokens_seen": 2924412928 + }, + { + "epoch": 0.77, + "learning_rate": 2.2981864869202377e-05, + "loss": 2.5555, + "theoretical_loss": 3.327288087980528, + "tokens_seen": 2924544000 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1603558, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5634658336639404, + "objective/train/theoretical_loss": 3.327276563383221, + "objective/train/tokens_used": 1295134176, + "theoretical_loss": 3.327276563383221, + "tokens_seen": 2924675072 + }, + { + "epoch": 0.77, + "learning_rate": 2.2973840475044137e-05, + "loss": 2.5436, + "theoretical_loss": 3.327276563383221, + "tokens_seen": 2924675072 + }, + { + "epoch": 0.77, + "learning_rate": 2.2965816080885893e-05, + "loss": 2.7136, + "theoretical_loss": 3.3272650394469965, + "tokens_seen": 2924806144 + }, + { + "epoch": 0.77, + "learning_rate": 2.2957791686727653e-05, + "loss": 2.4659, + "theoretical_loss": 3.327253516171787, + "tokens_seen": 2924937216 + }, + { + "epoch": 0.77, + "learning_rate": 2.2949767292569413e-05, + "loss": 2.5202, + "theoretical_loss": 3.3272419935575246, + "tokens_seen": 2925068288 + }, + { + "epoch": 0.77, + "learning_rate": 2.294174289841117e-05, + "loss": 2.4398, + "theoretical_loss": 3.327230471604142, + "tokens_seen": 2925199360 + }, + { + "epoch": 0.77, + "learning_rate": 2.293371850425293e-05, + "loss": 2.4818, + "theoretical_loss": 3.327218950311572, + "tokens_seen": 2925330432 + }, + { + "epoch": 0.77, + "learning_rate": 2.292569411009469e-05, + "loss": 2.5487, + "theoretical_loss": 3.327207429679746, + "tokens_seen": 2925461504 + }, + { + "epoch": 0.77, + "learning_rate": 2.291766971593645e-05, + "loss": 2.438, + "theoretical_loss": 3.327195909708598, + "tokens_seen": 2925592576 + }, + { + "epoch": 0.77, + "learning_rate": 2.2909645321778206e-05, + "loss": 2.4553, + "theoretical_loss": 3.327184390398059, + "tokens_seen": 2925723648 + }, + { + "epoch": 0.77, + "learning_rate": 2.2901620927619966e-05, + "loss": 2.4753, + "theoretical_loss": 3.327172871748063, + "tokens_seen": 2925854720 + }, + { + "epoch": 0.77, + "learning_rate": 2.2893596533461726e-05, + "loss": 2.676, + "theoretical_loss": 3.3271613537585414, + "tokens_seen": 2925985792 + }, + { + "epoch": 0.77, + "learning_rate": 2.2885572139303486e-05, + "loss": 2.5227, + "theoretical_loss": 3.3271498364294274, + "tokens_seen": 2926116864 + }, + { + "epoch": 0.77, + "learning_rate": 2.2877547745145242e-05, + "loss": 2.561, + "theoretical_loss": 3.327138319760653, + "tokens_seen": 2926247936 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1604805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5546083450317383, + "objective/train/theoretical_loss": 3.3271325616738725, + "objective/train/tokens_used": 1296772576, + "theoretical_loss": 3.3271325616738725, + "tokens_seen": 2926313472 + }, + { + "epoch": 0.77, + "learning_rate": 2.2869523350987002e-05, + "loss": 2.5934, + "theoretical_loss": 3.3271268037521518, + "tokens_seen": 2926379008 + }, + { + "epoch": 0.77, + "learning_rate": 2.2861498956828762e-05, + "loss": 2.6673, + "theoretical_loss": 3.327115288403855, + "tokens_seen": 2926510080 + }, + { + "epoch": 0.77, + "learning_rate": 2.285347456267052e-05, + "loss": 2.4269, + "theoretical_loss": 3.327103773715696, + "tokens_seen": 2926641152 + }, + { + "epoch": 0.77, + "learning_rate": 2.284545016851228e-05, + "loss": 2.3486, + "theoretical_loss": 3.3270922596876074, + "tokens_seen": 2926772224 + }, + { + "epoch": 0.77, + "learning_rate": 2.283742577435404e-05, + "loss": 2.5719, + "theoretical_loss": 3.3270807463195213, + "tokens_seen": 2926903296 + }, + { + "epoch": 0.77, + "learning_rate": 2.2829401380195798e-05, + "loss": 2.5748, + "theoretical_loss": 3.327069233611371, + "tokens_seen": 2927034368 + }, + { + "epoch": 0.77, + "learning_rate": 2.2821376986037555e-05, + "loss": 2.6935, + "theoretical_loss": 3.3270577215630888, + "tokens_seen": 2927165440 + }, + { + "epoch": 0.77, + "learning_rate": 2.2813352591879315e-05, + "loss": 2.5626, + "theoretical_loss": 3.327046210174607, + "tokens_seen": 2927296512 + }, + { + "epoch": 0.77, + "learning_rate": 2.2805328197721075e-05, + "loss": 2.3529, + "theoretical_loss": 3.327034699445859, + "tokens_seen": 2927427584 + }, + { + "epoch": 0.77, + "learning_rate": 2.2797303803562835e-05, + "loss": 2.6046, + "theoretical_loss": 3.327023189376777, + "tokens_seen": 2927558656 + }, + { + "epoch": 0.77, + "learning_rate": 2.278927940940459e-05, + "loss": 2.5889, + "theoretical_loss": 3.3270116799672933, + "tokens_seen": 2927689728 + }, + { + "epoch": 0.77, + "learning_rate": 2.278125501524635e-05, + "loss": 2.4459, + "theoretical_loss": 3.327000171217341, + "tokens_seen": 2927820800 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1605428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.591338634490967, + "objective/train/theoretical_loss": 3.326988663126853, + "objective/train/tokens_used": 1298410976, + "theoretical_loss": 3.326988663126853, + "tokens_seen": 2927951872 + }, + { + "epoch": 0.77, + "learning_rate": 2.277323062108811e-05, + "loss": 2.6359, + "theoretical_loss": 3.326988663126853, + "tokens_seen": 2927951872 + }, + { + "epoch": 0.77, + "learning_rate": 2.2765206226929867e-05, + "loss": 2.5749, + "theoretical_loss": 3.3269771556957615, + "tokens_seen": 2928082944 + }, + { + "epoch": 0.77, + "learning_rate": 2.2757181832771627e-05, + "loss": 2.6781, + "theoretical_loss": 3.3269656489239994, + "tokens_seen": 2928214016 + }, + { + "epoch": 0.77, + "learning_rate": 2.2749157438613387e-05, + "loss": 2.4255, + "theoretical_loss": 3.3269541428114997, + "tokens_seen": 2928345088 + }, + { + "epoch": 0.77, + "learning_rate": 2.2741133044455147e-05, + "loss": 2.6718, + "theoretical_loss": 3.3269426373581945, + "tokens_seen": 2928476160 + }, + { + "epoch": 0.77, + "learning_rate": 2.2733108650296904e-05, + "loss": 2.4245, + "theoretical_loss": 3.326931132564017, + "tokens_seen": 2928607232 + }, + { + "epoch": 0.78, + "learning_rate": 2.2725084256138663e-05, + "loss": 2.6238, + "theoretical_loss": 3.3269196284289, + "tokens_seen": 2928738304 + }, + { + "epoch": 0.78, + "learning_rate": 2.2717059861980423e-05, + "loss": 2.5922, + "theoretical_loss": 3.3269081249527765, + "tokens_seen": 2928869376 + }, + { + "epoch": 0.78, + "learning_rate": 2.2709035467822183e-05, + "loss": 2.549, + "theoretical_loss": 3.3268966221355782, + "tokens_seen": 2929000448 + }, + { + "epoch": 0.78, + "learning_rate": 2.270101107366394e-05, + "loss": 2.575, + "theoretical_loss": 3.3268851199772387, + "tokens_seen": 2929131520 + }, + { + "epoch": 0.78, + "learning_rate": 2.26929866795057e-05, + "loss": 2.6852, + "theoretical_loss": 3.326873618477691, + "tokens_seen": 2929262592 + }, + { + "epoch": 0.78, + "learning_rate": 2.268496228534746e-05, + "loss": 2.5874, + "theoretical_loss": 3.3268621176368667, + "tokens_seen": 2929393664 + }, + { + "epoch": 0.78, + "learning_rate": 2.2676937891189216e-05, + "loss": 2.6313, + "theoretical_loss": 3.3268506174547, + "tokens_seen": 2929524736 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1606103, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.476057291030884, + "objective/train/theoretical_loss": 3.326844867610592, + "objective/train/tokens_used": 1300049376, + "theoretical_loss": 3.326844867610592, + "tokens_seen": 2929590272 + }, + { + "epoch": 0.78, + "learning_rate": 2.2668913497030976e-05, + "loss": 2.514, + "theoretical_loss": 3.326839117931123, + "tokens_seen": 2929655808 + }, + { + "epoch": 0.78, + "learning_rate": 2.2660889102872736e-05, + "loss": 2.5924, + "theoretical_loss": 3.3268276190660684, + "tokens_seen": 2929786880 + }, + { + "epoch": 0.78, + "learning_rate": 2.2652864708714492e-05, + "loss": 2.5995, + "theoretical_loss": 3.32681612085947, + "tokens_seen": 2929917952 + }, + { + "epoch": 0.78, + "learning_rate": 2.2644840314556252e-05, + "loss": 2.3415, + "theoretical_loss": 3.326804623311259, + "tokens_seen": 2930049024 + }, + { + "epoch": 0.78, + "learning_rate": 2.2636815920398012e-05, + "loss": 2.6126, + "theoretical_loss": 3.3267931264213697, + "tokens_seen": 2930180096 + }, + { + "epoch": 0.78, + "learning_rate": 2.262879152623977e-05, + "loss": 2.5638, + "theoretical_loss": 3.326781630189734, + "tokens_seen": 2930311168 + }, + { + "epoch": 0.78, + "learning_rate": 2.262076713208153e-05, + "loss": 2.3671, + "theoretical_loss": 3.326770134616286, + "tokens_seen": 2930442240 + }, + { + "epoch": 0.78, + "learning_rate": 2.261274273792329e-05, + "loss": 2.5749, + "theoretical_loss": 3.326758639700957, + "tokens_seen": 2930573312 + }, + { + "epoch": 0.78, + "learning_rate": 2.2604718343765045e-05, + "loss": 2.647, + "theoretical_loss": 3.3267471454436808, + "tokens_seen": 2930704384 + }, + { + "epoch": 0.78, + "learning_rate": 2.2596693949606805e-05, + "loss": 2.5466, + "theoretical_loss": 3.3267356518443902, + "tokens_seen": 2930835456 + }, + { + "epoch": 0.78, + "learning_rate": 2.2588669555448565e-05, + "loss": 2.6272, + "theoretical_loss": 3.3267241589030183, + "tokens_seen": 2930966528 + }, + { + "epoch": 0.78, + "learning_rate": 2.258064516129032e-05, + "loss": 2.5944, + "theoretical_loss": 3.3267126666194975, + "tokens_seen": 2931097600 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1606733, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.799635887145996, + "objective/train/theoretical_loss": 3.326701174993761, + "objective/train/tokens_used": 1301687776, + "theoretical_loss": 3.326701174993761, + "tokens_seen": 2931228672 + }, + { + "epoch": 0.78, + "learning_rate": 2.257262076713208e-05, + "loss": 2.5523, + "theoretical_loss": 3.326701174993761, + "tokens_seen": 2931228672 + }, + { + "epoch": 0.78, + "learning_rate": 2.256459637297384e-05, + "loss": 2.4369, + "theoretical_loss": 3.3266896840257423, + "tokens_seen": 2931359744 + }, + { + "epoch": 0.78, + "learning_rate": 2.2556571978815598e-05, + "loss": 2.5218, + "theoretical_loss": 3.3266781937153733, + "tokens_seen": 2931490816 + }, + { + "epoch": 0.78, + "learning_rate": 2.2548547584657358e-05, + "loss": 2.6261, + "theoretical_loss": 3.3266667040625877, + "tokens_seen": 2931621888 + }, + { + "epoch": 0.78, + "learning_rate": 2.2540523190499118e-05, + "loss": 2.5724, + "theoretical_loss": 3.3266552150673183, + "tokens_seen": 2931752960 + }, + { + "epoch": 0.78, + "learning_rate": 2.2532498796340878e-05, + "loss": 2.683, + "theoretical_loss": 3.3266437267294977, + "tokens_seen": 2931884032 + }, + { + "epoch": 0.78, + "learning_rate": 2.2524474402182634e-05, + "loss": 2.6451, + "theoretical_loss": 3.3266322390490592, + "tokens_seen": 2932015104 + }, + { + "epoch": 0.78, + "learning_rate": 2.2516450008024394e-05, + "loss": 2.5258, + "theoretical_loss": 3.326620752025936, + "tokens_seen": 2932146176 + }, + { + "epoch": 0.78, + "learning_rate": 2.2508425613866154e-05, + "loss": 2.5195, + "theoretical_loss": 3.326609265660061, + "tokens_seen": 2932277248 + }, + { + "epoch": 0.78, + "learning_rate": 2.2500401219707914e-05, + "loss": 2.6457, + "theoretical_loss": 3.326597779951367, + "tokens_seen": 2932408320 + }, + { + "epoch": 0.78, + "learning_rate": 2.249237682554967e-05, + "loss": 2.5557, + "theoretical_loss": 3.326586294899787, + "tokens_seen": 2932539392 + }, + { + "epoch": 0.78, + "learning_rate": 2.248435243139143e-05, + "loss": 2.6602, + "theoretical_loss": 3.3265748105052544, + "tokens_seen": 2932670464 + }, + { + "epoch": 0.78, + "learning_rate": 2.247632803723319e-05, + "loss": 2.5251, + "theoretical_loss": 3.326563326767702, + "tokens_seen": 2932801536 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1608143, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.717588186264038, + "objective/train/theoretical_loss": 3.326557585145273, + "objective/train/tokens_used": 1303326176, + "theoretical_loss": 3.326557585145273, + "tokens_seen": 2932867072 + }, + { + "epoch": 0.78, + "learning_rate": 2.2468303643074947e-05, + "loss": 2.7513, + "theoretical_loss": 3.326551843687063, + "tokens_seen": 2932932608 + }, + { + "epoch": 0.78, + "learning_rate": 2.2460279248916707e-05, + "loss": 2.4716, + "theoretical_loss": 3.3265403612632705, + "tokens_seen": 2933063680 + }, + { + "epoch": 0.78, + "learning_rate": 2.2452254854758466e-05, + "loss": 2.7236, + "theoretical_loss": 3.3265288794962573, + "tokens_seen": 2933194752 + }, + { + "epoch": 0.78, + "learning_rate": 2.2444230460600226e-05, + "loss": 2.4855, + "theoretical_loss": 3.3265173983859566, + "tokens_seen": 2933325824 + }, + { + "epoch": 0.78, + "learning_rate": 2.2436206066441983e-05, + "loss": 2.6096, + "theoretical_loss": 3.3265059179323013, + "tokens_seen": 2933456896 + }, + { + "epoch": 0.78, + "learning_rate": 2.2428181672283743e-05, + "loss": 2.4952, + "theoretical_loss": 3.3264944381352253, + "tokens_seen": 2933587968 + }, + { + "epoch": 0.78, + "learning_rate": 2.2420157278125503e-05, + "loss": 2.589, + "theoretical_loss": 3.3264829589946605, + "tokens_seen": 2933719040 + }, + { + "epoch": 0.78, + "learning_rate": 2.2412132883967263e-05, + "loss": 2.5315, + "theoretical_loss": 3.3264714805105413, + "tokens_seen": 2933850112 + }, + { + "epoch": 0.78, + "learning_rate": 2.240410848980902e-05, + "loss": 2.5208, + "theoretical_loss": 3.3264600026827997, + "tokens_seen": 2933981184 + }, + { + "epoch": 0.78, + "learning_rate": 2.239608409565078e-05, + "loss": 2.574, + "theoretical_loss": 3.3264485255113696, + "tokens_seen": 2934112256 + }, + { + "epoch": 0.78, + "learning_rate": 2.238805970149254e-05, + "loss": 2.5782, + "theoretical_loss": 3.326437048996184, + "tokens_seen": 2934243328 + }, + { + "epoch": 0.78, + "learning_rate": 2.2380035307334295e-05, + "loss": 2.5453, + "theoretical_loss": 3.326425573137176, + "tokens_seen": 2934374400 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1608844, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.229003667831421, + "objective/train/theoretical_loss": 3.3264140979342787, + "objective/train/tokens_used": 1304964576, + "theoretical_loss": 3.3264140979342787, + "tokens_seen": 2934505472 + }, + { + "epoch": 0.78, + "learning_rate": 2.2372010913176055e-05, + "loss": 2.584, + "theoretical_loss": 3.3264140979342787, + "tokens_seen": 2934505472 + }, + { + "epoch": 0.78, + "learning_rate": 2.2363986519017815e-05, + "loss": 2.7305, + "theoretical_loss": 3.326402623387425, + "tokens_seen": 2934636544 + }, + { + "epoch": 0.78, + "learning_rate": 2.2355962124859575e-05, + "loss": 2.5251, + "theoretical_loss": 3.3263911494965486, + "tokens_seen": 2934767616 + }, + { + "epoch": 0.78, + "learning_rate": 2.234793773070133e-05, + "loss": 2.6973, + "theoretical_loss": 3.326379676261583, + "tokens_seen": 2934898688 + }, + { + "epoch": 0.78, + "learning_rate": 2.233991333654309e-05, + "loss": 2.4717, + "theoretical_loss": 3.3263682036824607, + "tokens_seen": 2935029760 + }, + { + "epoch": 0.78, + "learning_rate": 2.233188894238485e-05, + "loss": 2.4199, + "theoretical_loss": 3.326356731759115, + "tokens_seen": 2935160832 + }, + { + "epoch": 0.78, + "learning_rate": 2.232386454822661e-05, + "loss": 2.6605, + "theoretical_loss": 3.3263452604914794, + "tokens_seen": 2935291904 + }, + { + "epoch": 0.78, + "learning_rate": 2.2315840154068368e-05, + "loss": 2.5986, + "theoretical_loss": 3.326333789879487, + "tokens_seen": 2935422976 + }, + { + "epoch": 0.78, + "learning_rate": 2.2307815759910128e-05, + "loss": 2.6759, + "theoretical_loss": 3.3263223199230714, + "tokens_seen": 2935554048 + }, + { + "epoch": 0.78, + "learning_rate": 2.2299791365751888e-05, + "loss": 2.717, + "theoretical_loss": 3.326310850622165, + "tokens_seen": 2935685120 + }, + { + "epoch": 0.78, + "learning_rate": 2.2291766971593644e-05, + "loss": 2.6044, + "theoretical_loss": 3.3262993819767024, + "tokens_seen": 2935816192 + }, + { + "epoch": 0.78, + "learning_rate": 2.2283742577435404e-05, + "loss": 2.5585, + "theoretical_loss": 3.3262879139866155, + "tokens_seen": 2935947264 + }, + { + "epoch": 0.78, + "learning_rate": 2.2275718183277164e-05, + "loss": 2.6063, + "theoretical_loss": 3.3262764466518386, + "tokens_seen": 2936078336 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1609868, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9224324226379395, + "objective/train/theoretical_loss": 3.32627071323017, + "objective/train/tokens_used": 1306602976, + "theoretical_loss": 3.32627071323017, + "tokens_seen": 2936143872 + }, + { + "epoch": 0.78, + "learning_rate": 2.2267693789118924e-05, + "loss": 2.691, + "theoretical_loss": 3.3262649799723043, + "tokens_seen": 2936209408 + }, + { + "epoch": 0.78, + "learning_rate": 2.225966939496068e-05, + "loss": 2.5873, + "theoretical_loss": 3.3262535139479468, + "tokens_seen": 2936340480 + }, + { + "epoch": 0.78, + "learning_rate": 2.225164500080244e-05, + "loss": 2.6164, + "theoretical_loss": 3.326242048578698, + "tokens_seen": 2936471552 + }, + { + "epoch": 0.78, + "learning_rate": 2.22436206066442e-05, + "loss": 2.5749, + "theoretical_loss": 3.326230583864493, + "tokens_seen": 2936602624 + }, + { + "epoch": 0.78, + "learning_rate": 2.223559621248596e-05, + "loss": 2.6236, + "theoretical_loss": 3.3262191198052635, + "tokens_seen": 2936733696 + }, + { + "epoch": 0.78, + "learning_rate": 2.2227571818327717e-05, + "loss": 2.5877, + "theoretical_loss": 3.3262076564009444, + "tokens_seen": 2936864768 + }, + { + "epoch": 0.78, + "learning_rate": 2.2219547424169477e-05, + "loss": 2.4783, + "theoretical_loss": 3.3261961936514677, + "tokens_seen": 2936995840 + }, + { + "epoch": 0.78, + "learning_rate": 2.2211523030011236e-05, + "loss": 2.5755, + "theoretical_loss": 3.326184731556767, + "tokens_seen": 2937126912 + }, + { + "epoch": 0.78, + "learning_rate": 2.2203498635852993e-05, + "loss": 2.4573, + "theoretical_loss": 3.3261732701167768, + "tokens_seen": 2937257984 + }, + { + "epoch": 0.78, + "learning_rate": 2.2195474241694753e-05, + "loss": 2.5559, + "theoretical_loss": 3.326161809331429, + "tokens_seen": 2937389056 + }, + { + "epoch": 0.78, + "learning_rate": 2.2187449847536513e-05, + "loss": 2.5465, + "theoretical_loss": 3.326150349200658, + "tokens_seen": 2937520128 + }, + { + "epoch": 0.78, + "learning_rate": 2.2179425453378273e-05, + "loss": 2.5197, + "theoretical_loss": 3.3261388897243966, + "tokens_seen": 2937651200 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1610441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5121865272521973, + "objective/train/theoretical_loss": 3.326127430902579, + "objective/train/tokens_used": 1308241376, + "theoretical_loss": 3.326127430902579, + "tokens_seen": 2937782272 + }, + { + "epoch": 0.78, + "learning_rate": 2.217140105922003e-05, + "loss": 2.4828, + "theoretical_loss": 3.326127430902579, + "tokens_seen": 2937782272 + }, + { + "epoch": 0.78, + "learning_rate": 2.216337666506179e-05, + "loss": 2.5691, + "theoretical_loss": 3.3261159727351375, + "tokens_seen": 2937913344 + }, + { + "epoch": 0.78, + "learning_rate": 2.215535227090355e-05, + "loss": 2.4652, + "theoretical_loss": 3.326104515222007, + "tokens_seen": 2938044416 + }, + { + "epoch": 0.78, + "learning_rate": 2.214732787674531e-05, + "loss": 2.7075, + "theoretical_loss": 3.32609305836312, + "tokens_seen": 2938175488 + }, + { + "epoch": 0.78, + "learning_rate": 2.2139303482587065e-05, + "loss": 2.6151, + "theoretical_loss": 3.3260816021584096, + "tokens_seen": 2938306560 + }, + { + "epoch": 0.78, + "learning_rate": 2.2131279088428825e-05, + "loss": 2.5251, + "theoretical_loss": 3.3260701466078095, + "tokens_seen": 2938437632 + }, + { + "epoch": 0.78, + "learning_rate": 2.2123254694270585e-05, + "loss": 2.5961, + "theoretical_loss": 3.3260586917112542, + "tokens_seen": 2938568704 + }, + { + "epoch": 0.78, + "learning_rate": 2.2115230300112342e-05, + "loss": 2.6311, + "theoretical_loss": 3.326047237468676, + "tokens_seen": 2938699776 + }, + { + "epoch": 0.78, + "learning_rate": 2.2107205905954102e-05, + "loss": 2.7539, + "theoretical_loss": 3.326035783880009, + "tokens_seen": 2938830848 + }, + { + "epoch": 0.78, + "learning_rate": 2.209918151179586e-05, + "loss": 2.6187, + "theoretical_loss": 3.3260243309451862, + "tokens_seen": 2938961920 + }, + { + "epoch": 0.78, + "learning_rate": 2.209115711763762e-05, + "loss": 2.701, + "theoretical_loss": 3.326012878664142, + "tokens_seen": 2939092992 + }, + { + "epoch": 0.78, + "learning_rate": 2.2083132723479378e-05, + "loss": 2.4374, + "theoretical_loss": 3.326001427036809, + "tokens_seen": 2939224064 + }, + { + "epoch": 0.78, + "learning_rate": 2.2075108329321138e-05, + "loss": 2.5354, + "theoretical_loss": 3.325989976063121, + "tokens_seen": 2939355136 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1611444, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.553744316101074, + "objective/train/theoretical_loss": 3.3259842508213735, + "objective/train/tokens_used": 1309879776, + "theoretical_loss": 3.3259842508213735, + "tokens_seen": 2939420672 + }, + { + "epoch": 0.78, + "learning_rate": 2.2067083935162898e-05, + "loss": 2.5655, + "theoretical_loss": 3.325978525743012, + "tokens_seen": 2939486208 + }, + { + "epoch": 0.78, + "learning_rate": 2.2059059541004658e-05, + "loss": 2.7218, + "theoretical_loss": 3.325967076076415, + "tokens_seen": 2939617280 + }, + { + "epoch": 0.78, + "learning_rate": 2.2051035146846414e-05, + "loss": 2.607, + "theoretical_loss": 3.3259556270632635, + "tokens_seen": 2939748352 + }, + { + "epoch": 0.78, + "learning_rate": 2.2043010752688174e-05, + "loss": 2.7019, + "theoretical_loss": 3.325944178703492, + "tokens_seen": 2939879424 + }, + { + "epoch": 0.78, + "learning_rate": 2.2034986358529934e-05, + "loss": 2.618, + "theoretical_loss": 3.3259327309970326, + "tokens_seen": 2940010496 + }, + { + "epoch": 0.78, + "learning_rate": 2.2026961964371694e-05, + "loss": 2.673, + "theoretical_loss": 3.3259212839438206, + "tokens_seen": 2940141568 + }, + { + "epoch": 0.78, + "learning_rate": 2.201893757021345e-05, + "loss": 2.5706, + "theoretical_loss": 3.3259098375437883, + "tokens_seen": 2940272640 + }, + { + "epoch": 0.78, + "learning_rate": 2.201091317605521e-05, + "loss": 2.6264, + "theoretical_loss": 3.3258983917968696, + "tokens_seen": 2940403712 + }, + { + "epoch": 0.78, + "learning_rate": 2.200288878189697e-05, + "loss": 2.4964, + "theoretical_loss": 3.3258869467029983, + "tokens_seen": 2940534784 + }, + { + "epoch": 0.78, + "learning_rate": 2.1994864387738727e-05, + "loss": 2.6815, + "theoretical_loss": 3.3258755022621083, + "tokens_seen": 2940665856 + }, + { + "epoch": 0.78, + "learning_rate": 2.1986839993580487e-05, + "loss": 2.5363, + "theoretical_loss": 3.325864058474133, + "tokens_seen": 2940796928 + }, + { + "epoch": 0.78, + "learning_rate": 2.1978815599422247e-05, + "loss": 2.5316, + "theoretical_loss": 3.325852615339006, + "tokens_seen": 2940928000 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1612588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.314429759979248, + "objective/train/theoretical_loss": 3.32584117285666, + "objective/train/tokens_used": 1311518176, + "theoretical_loss": 3.32584117285666, + "tokens_seen": 2941059072 + }, + { + "epoch": 0.78, + "learning_rate": 2.1970791205264003e-05, + "loss": 2.4991, + "theoretical_loss": 3.32584117285666, + "tokens_seen": 2941059072 + }, + { + "epoch": 0.78, + "learning_rate": 2.1962766811105763e-05, + "loss": 2.6013, + "theoretical_loss": 3.3258297310270306, + "tokens_seen": 2941190144 + }, + { + "epoch": 0.78, + "learning_rate": 2.1954742416947523e-05, + "loss": 2.4898, + "theoretical_loss": 3.3258182898500506, + "tokens_seen": 2941321216 + }, + { + "epoch": 0.78, + "learning_rate": 2.194671802278928e-05, + "loss": 2.5939, + "theoretical_loss": 3.325806849325653, + "tokens_seen": 2941452288 + }, + { + "epoch": 0.78, + "learning_rate": 2.193869362863104e-05, + "loss": 2.6567, + "theoretical_loss": 3.3257954094537725, + "tokens_seen": 2941583360 + }, + { + "epoch": 0.78, + "learning_rate": 2.19306692344728e-05, + "loss": 2.6704, + "theoretical_loss": 3.325783970234342, + "tokens_seen": 2941714432 + }, + { + "epoch": 0.78, + "learning_rate": 2.1922644840314556e-05, + "loss": 2.5994, + "theoretical_loss": 3.325772531667296, + "tokens_seen": 2941845504 + }, + { + "epoch": 0.78, + "learning_rate": 2.1914620446156316e-05, + "loss": 2.7089, + "theoretical_loss": 3.325761093752568, + "tokens_seen": 2941976576 + }, + { + "epoch": 0.78, + "learning_rate": 2.1906596051998076e-05, + "loss": 2.7531, + "theoretical_loss": 3.3257496564900912, + "tokens_seen": 2942107648 + }, + { + "epoch": 0.78, + "learning_rate": 2.1898571657839832e-05, + "loss": 2.6421, + "theoretical_loss": 3.3257382198798, + "tokens_seen": 2942238720 + }, + { + "epoch": 0.78, + "learning_rate": 2.1890547263681592e-05, + "loss": 2.4586, + "theoretical_loss": 3.325726783921628, + "tokens_seen": 2942369792 + }, + { + "epoch": 0.78, + "learning_rate": 2.1882522869523352e-05, + "loss": 2.55, + "theoretical_loss": 3.3257153486155087, + "tokens_seen": 2942500864 + }, + { + "epoch": 0.78, + "learning_rate": 2.187449847536511e-05, + "loss": 2.743, + "theoretical_loss": 3.325703913961376, + "tokens_seen": 2942631936 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1613186, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.919858694076538, + "objective/train/theoretical_loss": 3.325698196878784, + "objective/train/tokens_used": 1313156576, + "theoretical_loss": 3.325698196878784, + "tokens_seen": 2942697472 + }, + { + "epoch": 0.78, + "learning_rate": 2.186647408120687e-05, + "loss": 2.6785, + "theoretical_loss": 3.325692479959164, + "tokens_seen": 2942763008 + }, + { + "epoch": 0.78, + "learning_rate": 2.185844968704863e-05, + "loss": 2.4959, + "theoretical_loss": 3.325681046608806, + "tokens_seen": 2942894080 + }, + { + "epoch": 0.78, + "learning_rate": 2.1850425292890388e-05, + "loss": 2.8182, + "theoretical_loss": 3.3256696139102364, + "tokens_seen": 2943025152 + }, + { + "epoch": 0.78, + "learning_rate": 2.1842400898732145e-05, + "loss": 2.5602, + "theoretical_loss": 3.3256581818633886, + "tokens_seen": 2943156224 + }, + { + "epoch": 0.78, + "learning_rate": 2.1834376504573905e-05, + "loss": 2.4934, + "theoretical_loss": 3.325646750468196, + "tokens_seen": 2943287296 + }, + { + "epoch": 0.78, + "learning_rate": 2.1826352110415665e-05, + "loss": 2.5294, + "theoretical_loss": 3.3256353197245936, + "tokens_seen": 2943418368 + }, + { + "epoch": 0.78, + "learning_rate": 2.181832771625742e-05, + "loss": 2.5524, + "theoretical_loss": 3.325623889632514, + "tokens_seen": 2943549440 + }, + { + "epoch": 0.78, + "learning_rate": 2.181030332209918e-05, + "loss": 2.7133, + "theoretical_loss": 3.325612460191892, + "tokens_seen": 2943680512 + }, + { + "epoch": 0.78, + "learning_rate": 2.180227892794094e-05, + "loss": 2.6147, + "theoretical_loss": 3.325601031402661, + "tokens_seen": 2943811584 + }, + { + "epoch": 0.78, + "learning_rate": 2.17942545337827e-05, + "loss": 2.633, + "theoretical_loss": 3.325589603264755, + "tokens_seen": 2943942656 + }, + { + "epoch": 0.78, + "learning_rate": 2.1786230139624457e-05, + "loss": 2.6837, + "theoretical_loss": 3.3255781757781078, + "tokens_seen": 2944073728 + }, + { + "epoch": 0.78, + "learning_rate": 2.1778205745466217e-05, + "loss": 2.6237, + "theoretical_loss": 3.3255667489426535, + "tokens_seen": 2944204800 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1614093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7392966747283936, + "objective/train/theoretical_loss": 3.3255553227583254, + "objective/train/tokens_used": 1314794976, + "theoretical_loss": 3.3255553227583254, + "tokens_seen": 2944335872 + }, + { + "epoch": 0.78, + "learning_rate": 2.1770181351307977e-05, + "loss": 2.6611, + "theoretical_loss": 3.3255553227583254, + "tokens_seen": 2944335872 + }, + { + "epoch": 0.78, + "learning_rate": 2.1762156957149737e-05, + "loss": 2.5186, + "theoretical_loss": 3.3255438972250584, + "tokens_seen": 2944466944 + }, + { + "epoch": 0.78, + "learning_rate": 2.1754132562991494e-05, + "loss": 2.5476, + "theoretical_loss": 3.325532472342786, + "tokens_seen": 2944598016 + }, + { + "epoch": 0.78, + "learning_rate": 2.1746108168833253e-05, + "loss": 2.4779, + "theoretical_loss": 3.3255210481114417, + "tokens_seen": 2944729088 + }, + { + "epoch": 0.78, + "learning_rate": 2.1738083774675013e-05, + "loss": 2.5679, + "theoretical_loss": 3.3255096245309597, + "tokens_seen": 2944860160 + }, + { + "epoch": 0.78, + "learning_rate": 2.173005938051677e-05, + "loss": 2.5374, + "theoretical_loss": 3.3254982016012744, + "tokens_seen": 2944991232 + }, + { + "epoch": 0.78, + "learning_rate": 2.172203498635853e-05, + "loss": 2.5329, + "theoretical_loss": 3.3254867793223193, + "tokens_seen": 2945122304 + }, + { + "epoch": 0.79, + "learning_rate": 2.171401059220029e-05, + "loss": 2.6374, + "theoretical_loss": 3.325475357694028, + "tokens_seen": 2945253376 + }, + { + "epoch": 0.79, + "learning_rate": 2.170598619804205e-05, + "loss": 2.5712, + "theoretical_loss": 3.3254639367163357, + "tokens_seen": 2945384448 + }, + { + "epoch": 0.79, + "learning_rate": 2.1697961803883806e-05, + "loss": 2.722, + "theoretical_loss": 3.325452516389175, + "tokens_seen": 2945515520 + }, + { + "epoch": 0.79, + "learning_rate": 2.1689937409725566e-05, + "loss": 2.5377, + "theoretical_loss": 3.325441096712481, + "tokens_seen": 2945646592 + }, + { + "epoch": 0.79, + "learning_rate": 2.1681913015567326e-05, + "loss": 2.6625, + "theoretical_loss": 3.325429677686187, + "tokens_seen": 2945777664 + }, + { + "epoch": 0.79, + "learning_rate": 2.1673888621409086e-05, + "loss": 2.6017, + "theoretical_loss": 3.325418259310227, + "tokens_seen": 2945908736 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1615476, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.480139970779419, + "objective/train/theoretical_loss": 3.3254125503661016, + "objective/train/tokens_used": 1316433376, + "theoretical_loss": 3.3254125503661016, + "tokens_seen": 2945974272 + }, + { + "epoch": 0.79, + "learning_rate": 2.1665864227250842e-05, + "loss": 2.4988, + "theoretical_loss": 3.3254068415845355, + "tokens_seen": 2946039808 + }, + { + "epoch": 0.79, + "learning_rate": 2.1657839833092602e-05, + "loss": 2.3924, + "theoretical_loss": 3.3253954245090465, + "tokens_seen": 2946170880 + }, + { + "epoch": 0.79, + "learning_rate": 2.1649815438934362e-05, + "loss": 2.6267, + "theoretical_loss": 3.325384008083694, + "tokens_seen": 2946301952 + }, + { + "epoch": 0.79, + "learning_rate": 2.164179104477612e-05, + "loss": 2.7339, + "theoretical_loss": 3.3253725923084114, + "tokens_seen": 2946433024 + }, + { + "epoch": 0.79, + "learning_rate": 2.163376665061788e-05, + "loss": 2.6113, + "theoretical_loss": 3.3253611771831335, + "tokens_seen": 2946564096 + }, + { + "epoch": 0.79, + "learning_rate": 2.162574225645964e-05, + "loss": 2.6112, + "theoretical_loss": 3.3253497627077944, + "tokens_seen": 2946695168 + }, + { + "epoch": 0.79, + "learning_rate": 2.16177178623014e-05, + "loss": 2.5369, + "theoretical_loss": 3.325338348882328, + "tokens_seen": 2946826240 + }, + { + "epoch": 0.79, + "learning_rate": 2.1609693468143155e-05, + "loss": 2.6685, + "theoretical_loss": 3.325326935706668, + "tokens_seen": 2946957312 + }, + { + "epoch": 0.79, + "learning_rate": 2.1601669073984915e-05, + "loss": 2.5944, + "theoretical_loss": 3.325315523180749, + "tokens_seen": 2947088384 + }, + { + "epoch": 0.79, + "learning_rate": 2.1593644679826675e-05, + "loss": 2.5084, + "theoretical_loss": 3.325304111304505, + "tokens_seen": 2947219456 + }, + { + "epoch": 0.79, + "learning_rate": 2.1585620285668435e-05, + "loss": 2.592, + "theoretical_loss": 3.3252927000778705, + "tokens_seen": 2947350528 + }, + { + "epoch": 0.79, + "learning_rate": 2.157759589151019e-05, + "loss": 2.572, + "theoretical_loss": 3.3252812895007793, + "tokens_seen": 2947481600 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1616167, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.593876838684082, + "objective/train/theoretical_loss": 3.325269879573165, + "objective/train/tokens_used": 1318071776, + "theoretical_loss": 3.325269879573165, + "tokens_seen": 2947612672 + }, + { + "epoch": 0.79, + "learning_rate": 2.156957149735195e-05, + "loss": 2.7078, + "theoretical_loss": 3.325269879573165, + "tokens_seen": 2947612672 + }, + { + "epoch": 0.79, + "learning_rate": 2.156154710319371e-05, + "loss": 2.5807, + "theoretical_loss": 3.325258470294963, + "tokens_seen": 2947743744 + }, + { + "epoch": 0.79, + "learning_rate": 2.155352270903547e-05, + "loss": 2.5902, + "theoretical_loss": 3.3252470616661056, + "tokens_seen": 2947874816 + }, + { + "epoch": 0.79, + "learning_rate": 2.1545498314877227e-05, + "loss": 2.6355, + "theoretical_loss": 3.3252356536865286, + "tokens_seen": 2948005888 + }, + { + "epoch": 0.79, + "learning_rate": 2.1537473920718987e-05, + "loss": 2.6438, + "theoretical_loss": 3.325224246356166, + "tokens_seen": 2948136960 + }, + { + "epoch": 0.79, + "learning_rate": 2.1529449526560747e-05, + "loss": 2.6121, + "theoretical_loss": 3.3252128396749514, + "tokens_seen": 2948268032 + }, + { + "epoch": 0.79, + "learning_rate": 2.1521425132402504e-05, + "loss": 2.7788, + "theoretical_loss": 3.3252014336428193, + "tokens_seen": 2948399104 + }, + { + "epoch": 0.79, + "learning_rate": 2.1513400738244264e-05, + "loss": 2.6676, + "theoretical_loss": 3.325190028259704, + "tokens_seen": 2948530176 + }, + { + "epoch": 0.79, + "learning_rate": 2.1505376344086024e-05, + "loss": 2.5768, + "theoretical_loss": 3.325178623525539, + "tokens_seen": 2948661248 + }, + { + "epoch": 0.79, + "learning_rate": 2.1497351949927783e-05, + "loss": 2.4448, + "theoretical_loss": 3.3251672194402597, + "tokens_seen": 2948792320 + }, + { + "epoch": 0.79, + "learning_rate": 2.148932755576954e-05, + "loss": 2.5104, + "theoretical_loss": 3.3251558160037997, + "tokens_seen": 2948923392 + }, + { + "epoch": 0.79, + "learning_rate": 2.14813031616113e-05, + "loss": 2.5448, + "theoretical_loss": 3.325144413216093, + "tokens_seen": 2949054464 + }, + { + "epoch": 0.79, + "learning_rate": 2.147327876745306e-05, + "loss": 2.5319, + "theoretical_loss": 3.325133011077075, + "tokens_seen": 2949185536 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1616875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7418766021728516, + "objective/train/theoretical_loss": 3.325127310250803, + "objective/train/tokens_used": 1319710176, + "theoretical_loss": 3.325127310250803, + "tokens_seen": 2949251072 + }, + { + "epoch": 0.79, + "learning_rate": 2.146525437329482e-05, + "loss": 2.6188, + "theoretical_loss": 3.3251216095866782, + "tokens_seen": 2949316608 + }, + { + "epoch": 0.79, + "learning_rate": 2.1457229979136576e-05, + "loss": 2.6206, + "theoretical_loss": 3.3251102087448388, + "tokens_seen": 2949447680 + }, + { + "epoch": 0.79, + "learning_rate": 2.1449205584978336e-05, + "loss": 2.7846, + "theoretical_loss": 3.3250988085514894, + "tokens_seen": 2949578752 + }, + { + "epoch": 0.79, + "learning_rate": 2.1441181190820096e-05, + "loss": 2.5935, + "theoretical_loss": 3.3250874090065654, + "tokens_seen": 2949709824 + }, + { + "epoch": 0.79, + "learning_rate": 2.1433156796661853e-05, + "loss": 2.5883, + "theoretical_loss": 3.3250760101100005, + "tokens_seen": 2949840896 + }, + { + "epoch": 0.79, + "learning_rate": 2.1425132402503612e-05, + "loss": 2.6725, + "theoretical_loss": 3.325064611861729, + "tokens_seen": 2949971968 + }, + { + "epoch": 0.79, + "learning_rate": 2.1417108008345372e-05, + "loss": 2.568, + "theoretical_loss": 3.325053214261686, + "tokens_seen": 2950103040 + }, + { + "epoch": 0.79, + "learning_rate": 2.1409083614187132e-05, + "loss": 2.5653, + "theoretical_loss": 3.3250418173098053, + "tokens_seen": 2950234112 + }, + { + "epoch": 0.79, + "learning_rate": 2.140105922002889e-05, + "loss": 2.697, + "theoretical_loss": 3.325030421006021, + "tokens_seen": 2950365184 + }, + { + "epoch": 0.79, + "learning_rate": 2.139303482587065e-05, + "loss": 2.6576, + "theoretical_loss": 3.325019025350268, + "tokens_seen": 2950496256 + }, + { + "epoch": 0.79, + "learning_rate": 2.138501043171241e-05, + "loss": 2.6494, + "theoretical_loss": 3.32500763034248, + "tokens_seen": 2950627328 + }, + { + "epoch": 0.79, + "learning_rate": 2.137698603755417e-05, + "loss": 2.7028, + "theoretical_loss": 3.3249962359825918, + "tokens_seen": 2950758400 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1617523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.730886936187744, + "objective/train/theoretical_loss": 3.3249848422705375, + "objective/train/tokens_used": 1321348576, + "theoretical_loss": 3.3249848422705375, + "tokens_seen": 2950889472 + }, + { + "epoch": 0.79, + "learning_rate": 2.1368961643395925e-05, + "loss": 2.7468, + "theoretical_loss": 3.3249848422705375, + "tokens_seen": 2950889472 + }, + { + "epoch": 0.79, + "learning_rate": 2.1360937249237685e-05, + "loss": 2.5487, + "theoretical_loss": 3.324973449206252, + "tokens_seen": 2951020544 + }, + { + "epoch": 0.79, + "learning_rate": 2.1352912855079445e-05, + "loss": 2.6857, + "theoretical_loss": 3.3249620567896696, + "tokens_seen": 2951151616 + }, + { + "epoch": 0.79, + "learning_rate": 2.13448884609212e-05, + "loss": 2.6324, + "theoretical_loss": 3.3249506650207246, + "tokens_seen": 2951282688 + }, + { + "epoch": 0.79, + "learning_rate": 2.133686406676296e-05, + "loss": 2.7054, + "theoretical_loss": 3.3249392738993513, + "tokens_seen": 2951413760 + }, + { + "epoch": 0.79, + "learning_rate": 2.132883967260472e-05, + "loss": 2.6233, + "theoretical_loss": 3.324927883425484, + "tokens_seen": 2951544832 + }, + { + "epoch": 0.79, + "learning_rate": 2.1320815278446478e-05, + "loss": 2.4883, + "theoretical_loss": 3.3249164935990576, + "tokens_seen": 2951675904 + }, + { + "epoch": 0.79, + "learning_rate": 2.1312790884288238e-05, + "loss": 2.558, + "theoretical_loss": 3.324905104420006, + "tokens_seen": 2951806976 + }, + { + "epoch": 0.79, + "learning_rate": 2.1304766490129997e-05, + "loss": 2.6255, + "theoretical_loss": 3.3248937158882645, + "tokens_seen": 2951938048 + }, + { + "epoch": 0.79, + "learning_rate": 2.1296742095971754e-05, + "loss": 2.5062, + "theoretical_loss": 3.3248823280037665, + "tokens_seen": 2952069120 + }, + { + "epoch": 0.79, + "learning_rate": 2.1288717701813514e-05, + "loss": 2.4717, + "theoretical_loss": 3.324870940766447, + "tokens_seen": 2952200192 + }, + { + "epoch": 0.79, + "learning_rate": 2.1280693307655274e-05, + "loss": 2.6335, + "theoretical_loss": 3.324859554176241, + "tokens_seen": 2952331264 + }, + { + "epoch": 0.79, + "learning_rate": 2.127266891349703e-05, + "loss": 2.4299, + "theoretical_loss": 3.324848168233082, + "tokens_seen": 2952462336 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1618766, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.576566457748413, + "objective/train/theoretical_loss": 3.324842475504125, + "objective/train/tokens_used": 1322986976, + "theoretical_loss": 3.324842475504125, + "tokens_seen": 2952527872 + }, + { + "epoch": 0.79, + "learning_rate": 2.126464451933879e-05, + "loss": 2.5455, + "theoretical_loss": 3.324836782936905, + "tokens_seen": 2952593408 + }, + { + "epoch": 0.79, + "learning_rate": 2.125662012518055e-05, + "loss": 2.5397, + "theoretical_loss": 3.324825398287645, + "tokens_seen": 2952724480 + }, + { + "epoch": 0.79, + "learning_rate": 2.1248595731022307e-05, + "loss": 2.5749, + "theoretical_loss": 3.3248140142852356, + "tokens_seen": 2952855552 + }, + { + "epoch": 0.79, + "learning_rate": 2.1240571336864067e-05, + "loss": 2.4549, + "theoretical_loss": 3.3248026309296117, + "tokens_seen": 2952986624 + }, + { + "epoch": 0.79, + "learning_rate": 2.1232546942705826e-05, + "loss": 2.656, + "theoretical_loss": 3.3247912482207083, + "tokens_seen": 2953117696 + }, + { + "epoch": 0.79, + "learning_rate": 2.1224522548547583e-05, + "loss": 2.6772, + "theoretical_loss": 3.3247798661584596, + "tokens_seen": 2953248768 + }, + { + "epoch": 0.79, + "learning_rate": 2.1216498154389343e-05, + "loss": 2.4604, + "theoretical_loss": 3.3247684847428, + "tokens_seen": 2953379840 + }, + { + "epoch": 0.79, + "learning_rate": 2.1208473760231103e-05, + "loss": 2.5826, + "theoretical_loss": 3.3247571039736643, + "tokens_seen": 2953510912 + }, + { + "epoch": 0.79, + "learning_rate": 2.1200449366072863e-05, + "loss": 2.5554, + "theoretical_loss": 3.3247457238509868, + "tokens_seen": 2953641984 + }, + { + "epoch": 0.79, + "learning_rate": 2.119242497191462e-05, + "loss": 2.4845, + "theoretical_loss": 3.3247343443747024, + "tokens_seen": 2953773056 + }, + { + "epoch": 0.79, + "learning_rate": 2.118440057775638e-05, + "loss": 2.5982, + "theoretical_loss": 3.324722965544746, + "tokens_seen": 2953904128 + }, + { + "epoch": 0.79, + "learning_rate": 2.117637618359814e-05, + "loss": 2.6455, + "theoretical_loss": 3.3247115873610515, + "tokens_seen": 2954035200 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1619446, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6044845581054688, + "objective/train/theoretical_loss": 3.324700209823554, + "objective/train/tokens_used": 1324625376, + "theoretical_loss": 3.324700209823554, + "tokens_seen": 2954166272 + }, + { + "epoch": 0.79, + "learning_rate": 2.1168351789439896e-05, + "loss": 2.5965, + "theoretical_loss": 3.324700209823554, + "tokens_seen": 2954166272 + }, + { + "epoch": 0.79, + "learning_rate": 2.1160327395281655e-05, + "loss": 2.5551, + "theoretical_loss": 3.3246888329321878, + "tokens_seen": 2954297344 + }, + { + "epoch": 0.79, + "learning_rate": 2.1152303001123415e-05, + "loss": 2.3478, + "theoretical_loss": 3.3246774566868877, + "tokens_seen": 2954428416 + }, + { + "epoch": 0.79, + "learning_rate": 2.1144278606965175e-05, + "loss": 2.529, + "theoretical_loss": 3.3246660810875883, + "tokens_seen": 2954559488 + }, + { + "epoch": 0.79, + "learning_rate": 2.1136254212806932e-05, + "loss": 2.4704, + "theoretical_loss": 3.3246547061342246, + "tokens_seen": 2954690560 + }, + { + "epoch": 0.79, + "learning_rate": 2.112822981864869e-05, + "loss": 2.5546, + "theoretical_loss": 3.324643331826731, + "tokens_seen": 2954821632 + }, + { + "epoch": 0.79, + "learning_rate": 2.112020542449045e-05, + "loss": 2.522, + "theoretical_loss": 3.324631958165042, + "tokens_seen": 2954952704 + }, + { + "epoch": 0.79, + "learning_rate": 2.111218103033221e-05, + "loss": 2.5326, + "theoretical_loss": 3.3246205851490926, + "tokens_seen": 2955083776 + }, + { + "epoch": 0.79, + "learning_rate": 2.1104156636173968e-05, + "loss": 2.4738, + "theoretical_loss": 3.3246092127788174, + "tokens_seen": 2955214848 + }, + { + "epoch": 0.79, + "learning_rate": 2.1096132242015728e-05, + "loss": 2.4675, + "theoretical_loss": 3.324597841054151, + "tokens_seen": 2955345920 + }, + { + "epoch": 0.79, + "learning_rate": 2.1088107847857488e-05, + "loss": 2.5449, + "theoretical_loss": 3.3245864699750287, + "tokens_seen": 2955476992 + }, + { + "epoch": 0.79, + "learning_rate": 2.1080083453699248e-05, + "loss": 2.47, + "theoretical_loss": 3.3245750995413843, + "tokens_seen": 2955608064 + }, + { + "epoch": 0.79, + "learning_rate": 2.1072059059541004e-05, + "loss": 2.5513, + "theoretical_loss": 3.3245637297531525, + "tokens_seen": 2955739136 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1620788, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.282139301300049, + "objective/train/theoretical_loss": 3.3245580451010466, + "objective/train/tokens_used": 1326263776, + "theoretical_loss": 3.3245580451010466, + "tokens_seen": 2955804672 + }, + { + "epoch": 0.79, + "learning_rate": 2.1064034665382764e-05, + "loss": 2.5094, + "theoretical_loss": 3.3245523606102694, + "tokens_seen": 2955870208 + }, + { + "epoch": 0.79, + "learning_rate": 2.1056010271224524e-05, + "loss": 2.549, + "theoretical_loss": 3.3245409921126683, + "tokens_seen": 2956001280 + }, + { + "epoch": 0.79, + "learning_rate": 2.104798587706628e-05, + "loss": 2.444, + "theoretical_loss": 3.3245296242602844, + "tokens_seen": 2956132352 + }, + { + "epoch": 0.79, + "learning_rate": 2.103996148290804e-05, + "loss": 2.5692, + "theoretical_loss": 3.324518257053053, + "tokens_seen": 2956263424 + }, + { + "epoch": 0.79, + "learning_rate": 2.10319370887498e-05, + "loss": 2.6679, + "theoretical_loss": 3.3245068904909085, + "tokens_seen": 2956394496 + }, + { + "epoch": 0.79, + "learning_rate": 2.102391269459156e-05, + "loss": 2.4208, + "theoretical_loss": 3.324495524573786, + "tokens_seen": 2956525568 + }, + { + "epoch": 0.79, + "learning_rate": 2.1015888300433317e-05, + "loss": 2.4886, + "theoretical_loss": 3.324484159301619, + "tokens_seen": 2956656640 + }, + { + "epoch": 0.79, + "learning_rate": 2.1007863906275077e-05, + "loss": 2.4968, + "theoretical_loss": 3.324472794674344, + "tokens_seen": 2956787712 + }, + { + "epoch": 0.79, + "learning_rate": 2.0999839512116837e-05, + "loss": 2.4537, + "theoretical_loss": 3.324461430691895, + "tokens_seen": 2956918784 + }, + { + "epoch": 0.79, + "learning_rate": 2.0991815117958597e-05, + "loss": 2.3955, + "theoretical_loss": 3.3244500673542072, + "tokens_seen": 2957049856 + }, + { + "epoch": 0.79, + "learning_rate": 2.0983790723800353e-05, + "loss": 2.3785, + "theoretical_loss": 3.324438704661215, + "tokens_seen": 2957180928 + }, + { + "epoch": 0.79, + "learning_rate": 2.0975766329642113e-05, + "loss": 2.5959, + "theoretical_loss": 3.324427342612853, + "tokens_seen": 2957312000 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1621459, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.433659791946411, + "objective/train/theoretical_loss": 3.3244159812090572, + "objective/train/tokens_used": 1327902176, + "theoretical_loss": 3.3244159812090572, + "tokens_seen": 2957443072 + }, + { + "epoch": 0.79, + "learning_rate": 2.0967741935483873e-05, + "loss": 2.68, + "theoretical_loss": 3.3244159812090572, + "tokens_seen": 2957443072 + }, + { + "epoch": 0.79, + "learning_rate": 2.095971754132563e-05, + "loss": 2.3856, + "theoretical_loss": 3.3244046204497617, + "tokens_seen": 2957574144 + }, + { + "epoch": 0.79, + "learning_rate": 2.095169314716739e-05, + "loss": 2.6877, + "theoretical_loss": 3.324393260334901, + "tokens_seen": 2957705216 + }, + { + "epoch": 0.79, + "learning_rate": 2.094366875300915e-05, + "loss": 2.5539, + "theoretical_loss": 3.324381900864411, + "tokens_seen": 2957836288 + }, + { + "epoch": 0.79, + "learning_rate": 2.093564435885091e-05, + "loss": 2.66, + "theoretical_loss": 3.324370542038226, + "tokens_seen": 2957967360 + }, + { + "epoch": 0.79, + "learning_rate": 2.0927619964692666e-05, + "loss": 2.5402, + "theoretical_loss": 3.3243591838562807, + "tokens_seen": 2958098432 + }, + { + "epoch": 0.79, + "learning_rate": 2.0919595570534426e-05, + "loss": 2.6158, + "theoretical_loss": 3.32434782631851, + "tokens_seen": 2958229504 + }, + { + "epoch": 0.79, + "learning_rate": 2.0911571176376185e-05, + "loss": 2.4832, + "theoretical_loss": 3.3243364694248494, + "tokens_seen": 2958360576 + }, + { + "epoch": 0.79, + "learning_rate": 2.0903546782217945e-05, + "loss": 2.4227, + "theoretical_loss": 3.3243251131752336, + "tokens_seen": 2958491648 + }, + { + "epoch": 0.79, + "learning_rate": 2.0895522388059702e-05, + "loss": 2.5677, + "theoretical_loss": 3.3243137575695973, + "tokens_seen": 2958622720 + }, + { + "epoch": 0.79, + "learning_rate": 2.0887497993901462e-05, + "loss": 2.4359, + "theoretical_loss": 3.3243024026078762, + "tokens_seen": 2958753792 + }, + { + "epoch": 0.79, + "learning_rate": 2.087947359974322e-05, + "loss": 2.3556, + "theoretical_loss": 3.324291048290004, + "tokens_seen": 2958884864 + }, + { + "epoch": 0.79, + "learning_rate": 2.0871449205584978e-05, + "loss": 2.5178, + "theoretical_loss": 3.3242796946159165, + "tokens_seen": 2959015936 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1622436, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2249538898468018, + "objective/train/theoretical_loss": 3.324274018020272, + "objective/train/tokens_used": 1329540576, + "theoretical_loss": 3.324274018020272, + "tokens_seen": 2959081472 + }, + { + "epoch": 0.79, + "learning_rate": 2.0863424811426738e-05, + "loss": 2.5765, + "theoretical_loss": 3.3242683415855487, + "tokens_seen": 2959147008 + }, + { + "epoch": 0.79, + "learning_rate": 2.0855400417268498e-05, + "loss": 2.276, + "theoretical_loss": 3.324256989198836, + "tokens_seen": 2959278080 + }, + { + "epoch": 0.79, + "learning_rate": 2.0847376023110258e-05, + "loss": 2.4127, + "theoretical_loss": 3.324245637455712, + "tokens_seen": 2959409152 + }, + { + "epoch": 0.79, + "learning_rate": 2.0839351628952014e-05, + "loss": 2.445, + "theoretical_loss": 3.3242342863561127, + "tokens_seen": 2959540224 + }, + { + "epoch": 0.79, + "learning_rate": 2.0831327234793774e-05, + "loss": 2.7036, + "theoretical_loss": 3.3242229358999733, + "tokens_seen": 2959671296 + }, + { + "epoch": 0.79, + "learning_rate": 2.0823302840635534e-05, + "loss": 2.4791, + "theoretical_loss": 3.3242115860872286, + "tokens_seen": 2959802368 + }, + { + "epoch": 0.79, + "learning_rate": 2.0815278446477294e-05, + "loss": 2.4712, + "theoretical_loss": 3.324200236917813, + "tokens_seen": 2959933440 + }, + { + "epoch": 0.79, + "learning_rate": 2.080725405231905e-05, + "loss": 2.6667, + "theoretical_loss": 3.3241888883916624, + "tokens_seen": 2960064512 + }, + { + "epoch": 0.79, + "learning_rate": 2.079922965816081e-05, + "loss": 2.5988, + "theoretical_loss": 3.3241775405087117, + "tokens_seen": 2960195584 + }, + { + "epoch": 0.79, + "learning_rate": 2.079120526400257e-05, + "loss": 2.4597, + "theoretical_loss": 3.324166193268896, + "tokens_seen": 2960326656 + }, + { + "epoch": 0.79, + "learning_rate": 2.0783180869844327e-05, + "loss": 2.5717, + "theoretical_loss": 3.3241548466721498, + "tokens_seen": 2960457728 + }, + { + "epoch": 0.79, + "learning_rate": 2.0775156475686087e-05, + "loss": 2.5125, + "theoretical_loss": 3.3241435007184084, + "tokens_seen": 2960588800 + }, + { + "debugging/Self-BLEU-5": 0.35205097333386615, + "debugging/distinct-1-grams": 0.7499603426983117, + "debugging/distinct-2-grams": 0.9617863867698168, + "debugging/entropy-1-grams": 5.186053991514431, + "debugging/entropy-2-grams": 5.731636343902597, + "debugging/length": 480.2, + "debugging/num_segments": 5, + "debugging/score": 0.008241509832664893, + "debugging/score_std": 0.003930832595700179, + "epoch": 0.79, + "objective/train/docs_used": 1623305, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2661898136138916, + "objective/train/theoretical_loss": 3.3241321554076078, + "objective/train/tokens_used": 1331178976, + "theoretical_loss": 3.3241321554076078, + "tokens_seen": 2960719872 + }, + { + "epoch": 0.79, + "learning_rate": 2.0767132081527847e-05, + "loss": 2.4978, + "theoretical_loss": 3.3241321554076078, + "tokens_seen": 2960719872 + }, + { + "epoch": 0.79, + "learning_rate": 2.0759107687369607e-05, + "loss": 2.496, + "theoretical_loss": 3.324120810739682, + "tokens_seen": 2960850944 + }, + { + "epoch": 0.79, + "learning_rate": 2.0751083293211363e-05, + "loss": 2.5284, + "theoretical_loss": 3.3241094667145665, + "tokens_seen": 2960982016 + }, + { + "epoch": 0.79, + "learning_rate": 2.0743058899053123e-05, + "loss": 2.5115, + "theoretical_loss": 3.3240981233321962, + "tokens_seen": 2961113088 + }, + { + "epoch": 0.79, + "learning_rate": 2.0735034504894883e-05, + "loss": 2.4531, + "theoretical_loss": 3.324086780592507, + "tokens_seen": 2961244160 + }, + { + "epoch": 0.79, + "learning_rate": 2.0727010110736643e-05, + "loss": 2.4015, + "theoretical_loss": 3.324075438495433, + "tokens_seen": 2961375232 + }, + { + "epoch": 0.79, + "learning_rate": 2.07189857165784e-05, + "loss": 2.5002, + "theoretical_loss": 3.3240640970409103, + "tokens_seen": 2961506304 + }, + { + "epoch": 0.79, + "learning_rate": 2.071096132242016e-05, + "loss": 2.4718, + "theoretical_loss": 3.3240527562288733, + "tokens_seen": 2961637376 + }, + { + "epoch": 0.8, + "learning_rate": 2.070293692826192e-05, + "loss": 2.4963, + "theoretical_loss": 3.3240414160592575, + "tokens_seen": 2961768448 + }, + { + "epoch": 0.8, + "learning_rate": 2.0694912534103676e-05, + "loss": 2.6047, + "theoretical_loss": 3.324030076531998, + "tokens_seen": 2961899520 + }, + { + "epoch": 0.8, + "learning_rate": 2.0686888139945436e-05, + "loss": 2.3912, + "theoretical_loss": 3.3240187376470307, + "tokens_seen": 2962030592 + }, + { + "epoch": 0.8, + "learning_rate": 2.0678863745787196e-05, + "loss": 2.6866, + "theoretical_loss": 3.3240073994042896, + "tokens_seen": 2962161664 + }, + { + "epoch": 0.8, + "learning_rate": 2.0670839351628955e-05, + "loss": 2.4675, + "theoretical_loss": 3.3239960618037108, + "tokens_seen": 2962292736 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1624009, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.642270565032959, + "objective/train/theoretical_loss": 3.3239903932442116, + "objective/train/tokens_used": 1332817376, + "theoretical_loss": 3.3239903932442116, + "tokens_seen": 2962358272 + }, + { + "epoch": 0.8, + "learning_rate": 2.0662814957470712e-05, + "loss": 2.7445, + "theoretical_loss": 3.323984724845229, + "tokens_seen": 2962423808 + }, + { + "epoch": 0.8, + "learning_rate": 2.0654790563312472e-05, + "loss": 2.5756, + "theoretical_loss": 3.3239733885287794, + "tokens_seen": 2962554880 + }, + { + "epoch": 0.8, + "learning_rate": 2.0646766169154232e-05, + "loss": 2.5787, + "theoretical_loss": 3.323962052854298, + "tokens_seen": 2962685952 + }, + { + "epoch": 0.8, + "learning_rate": 2.063874177499599e-05, + "loss": 2.5351, + "theoretical_loss": 3.3239507178217194, + "tokens_seen": 2962817024 + }, + { + "epoch": 0.8, + "learning_rate": 2.0630717380837748e-05, + "loss": 2.5195, + "theoretical_loss": 3.3239393834309787, + "tokens_seen": 2962948096 + }, + { + "epoch": 0.8, + "learning_rate": 2.0622692986679508e-05, + "loss": 2.5652, + "theoretical_loss": 3.323928049682012, + "tokens_seen": 2963079168 + }, + { + "epoch": 0.8, + "learning_rate": 2.0614668592521265e-05, + "loss": 2.5436, + "theoretical_loss": 3.323916716574753, + "tokens_seen": 2963210240 + }, + { + "epoch": 0.8, + "learning_rate": 2.0606644198363025e-05, + "loss": 2.4063, + "theoretical_loss": 3.323905384109139, + "tokens_seen": 2963341312 + }, + { + "epoch": 0.8, + "learning_rate": 2.0598619804204784e-05, + "loss": 2.5569, + "theoretical_loss": 3.3238940522851035, + "tokens_seen": 2963472384 + }, + { + "epoch": 0.8, + "learning_rate": 2.059059541004654e-05, + "loss": 2.5281, + "theoretical_loss": 3.323882721102583, + "tokens_seen": 2963603456 + }, + { + "epoch": 0.8, + "learning_rate": 2.05825710158883e-05, + "loss": 2.6272, + "theoretical_loss": 3.3238713905615125, + "tokens_seen": 2963734528 + }, + { + "epoch": 0.8, + "learning_rate": 2.057454662173006e-05, + "loss": 2.428, + "theoretical_loss": 3.323860060661827, + "tokens_seen": 2963865600 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1624849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4922640323638916, + "objective/train/theoretical_loss": 3.323848731403462, + "objective/train/tokens_used": 1334455776, + "theoretical_loss": 3.323848731403462, + "tokens_seen": 2963996672 + }, + { + "epoch": 0.8, + "learning_rate": 2.0566522227571817e-05, + "loss": 2.4807, + "theoretical_loss": 3.323848731403462, + "tokens_seen": 2963996672 + }, + { + "epoch": 0.8, + "learning_rate": 2.0558497833413577e-05, + "loss": 2.4042, + "theoretical_loss": 3.3238374027863533, + "tokens_seen": 2964127744 + }, + { + "epoch": 0.8, + "learning_rate": 2.0550473439255337e-05, + "loss": 2.5383, + "theoretical_loss": 3.3238260748104356, + "tokens_seen": 2964258816 + }, + { + "epoch": 0.8, + "learning_rate": 2.0542449045097094e-05, + "loss": 2.6045, + "theoretical_loss": 3.3238147474756445, + "tokens_seen": 2964389888 + }, + { + "epoch": 0.8, + "learning_rate": 2.0534424650938854e-05, + "loss": 2.6891, + "theoretical_loss": 3.3238034207819154, + "tokens_seen": 2964520960 + }, + { + "epoch": 0.8, + "learning_rate": 2.0526400256780613e-05, + "loss": 2.558, + "theoretical_loss": 3.3237920947291837, + "tokens_seen": 2964652032 + }, + { + "epoch": 0.8, + "learning_rate": 2.0518375862622373e-05, + "loss": 2.7355, + "theoretical_loss": 3.3237807693173846, + "tokens_seen": 2964783104 + }, + { + "epoch": 0.8, + "learning_rate": 2.051035146846413e-05, + "loss": 2.6267, + "theoretical_loss": 3.323769444546454, + "tokens_seen": 2964914176 + }, + { + "epoch": 0.8, + "learning_rate": 2.050232707430589e-05, + "loss": 2.4482, + "theoretical_loss": 3.3237581204163265, + "tokens_seen": 2965045248 + }, + { + "epoch": 0.8, + "learning_rate": 2.049430268014765e-05, + "loss": 2.6104, + "theoretical_loss": 3.323746796926938, + "tokens_seen": 2965176320 + }, + { + "epoch": 0.8, + "learning_rate": 2.0486278285989406e-05, + "loss": 2.4691, + "theoretical_loss": 3.3237354740782243, + "tokens_seen": 2965307392 + }, + { + "epoch": 0.8, + "learning_rate": 2.0478253891831166e-05, + "loss": 2.3647, + "theoretical_loss": 3.32372415187012, + "tokens_seen": 2965438464 + }, + { + "epoch": 0.8, + "learning_rate": 2.0470229497672926e-05, + "loss": 2.4767, + "theoretical_loss": 3.3237128303025614, + "tokens_seen": 2965569536 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1625444, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.670814037322998, + "objective/train/theoretical_loss": 3.323707169758966, + "objective/train/tokens_used": 1336094176, + "theoretical_loss": 3.323707169758966, + "tokens_seen": 2965635072 + }, + { + "epoch": 0.8, + "learning_rate": 2.0462205103514686e-05, + "loss": 2.5349, + "theoretical_loss": 3.323701509375483, + "tokens_seen": 2965700608 + }, + { + "epoch": 0.8, + "learning_rate": 2.0454180709356442e-05, + "loss": 2.5464, + "theoretical_loss": 3.323690189088821, + "tokens_seen": 2965831680 + }, + { + "epoch": 0.8, + "learning_rate": 2.0446156315198202e-05, + "loss": 2.5129, + "theoretical_loss": 3.323678869442511, + "tokens_seen": 2965962752 + }, + { + "epoch": 0.8, + "learning_rate": 2.0438131921039962e-05, + "loss": 2.6334, + "theoretical_loss": 3.323667550436488, + "tokens_seen": 2966093824 + }, + { + "epoch": 0.8, + "learning_rate": 2.0430107526881722e-05, + "loss": 2.5554, + "theoretical_loss": 3.3236562320706873, + "tokens_seen": 2966224896 + }, + { + "epoch": 0.8, + "learning_rate": 2.042208313272348e-05, + "loss": 2.4347, + "theoretical_loss": 3.323644914345045, + "tokens_seen": 2966355968 + }, + { + "epoch": 0.8, + "learning_rate": 2.041405873856524e-05, + "loss": 2.6176, + "theoretical_loss": 3.323633597259496, + "tokens_seen": 2966487040 + }, + { + "epoch": 0.8, + "learning_rate": 2.0406034344407e-05, + "loss": 2.5767, + "theoretical_loss": 3.3236222808139764, + "tokens_seen": 2966618112 + }, + { + "epoch": 0.8, + "learning_rate": 2.0398009950248755e-05, + "loss": 2.5855, + "theoretical_loss": 3.3236109650084216, + "tokens_seen": 2966749184 + }, + { + "epoch": 0.8, + "learning_rate": 2.0389985556090515e-05, + "loss": 2.6168, + "theoretical_loss": 3.323599649842767, + "tokens_seen": 2966880256 + }, + { + "epoch": 0.8, + "learning_rate": 2.0381961161932275e-05, + "loss": 2.6103, + "theoretical_loss": 3.323588335316948, + "tokens_seen": 2967011328 + }, + { + "epoch": 0.8, + "learning_rate": 2.0373936767774035e-05, + "loss": 2.6075, + "theoretical_loss": 3.3235770214309, + "tokens_seen": 2967142400 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1626732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5070440769195557, + "objective/train/theoretical_loss": 3.3235657081845593, + "objective/train/tokens_used": 1337732576, + "theoretical_loss": 3.3235657081845593, + "tokens_seen": 2967273472 + }, + { + "epoch": 0.8, + "learning_rate": 2.036591237361579e-05, + "loss": 2.5088, + "theoretical_loss": 3.3235657081845593, + "tokens_seen": 2967273472 + }, + { + "epoch": 0.8, + "learning_rate": 2.035788797945755e-05, + "loss": 2.5748, + "theoretical_loss": 3.3235543955778613, + "tokens_seen": 2967404544 + }, + { + "epoch": 0.8, + "learning_rate": 2.034986358529931e-05, + "loss": 2.5809, + "theoretical_loss": 3.323543083610741, + "tokens_seen": 2967535616 + }, + { + "epoch": 0.8, + "learning_rate": 2.034183919114107e-05, + "loss": 2.4675, + "theoretical_loss": 3.3235317722831343, + "tokens_seen": 2967666688 + }, + { + "epoch": 0.8, + "learning_rate": 2.0333814796982827e-05, + "loss": 2.5752, + "theoretical_loss": 3.323520461594977, + "tokens_seen": 2967797760 + }, + { + "epoch": 0.8, + "learning_rate": 2.0325790402824587e-05, + "loss": 2.7191, + "theoretical_loss": 3.323509151546205, + "tokens_seen": 2967928832 + }, + { + "epoch": 0.8, + "learning_rate": 2.0317766008666347e-05, + "loss": 2.4836, + "theoretical_loss": 3.323497842136753, + "tokens_seen": 2968059904 + }, + { + "epoch": 0.8, + "learning_rate": 2.0309741614508104e-05, + "loss": 2.5233, + "theoretical_loss": 3.323486533366557, + "tokens_seen": 2968190976 + }, + { + "epoch": 0.8, + "learning_rate": 2.0301717220349864e-05, + "loss": 2.55, + "theoretical_loss": 3.3234752252355526, + "tokens_seen": 2968322048 + }, + { + "epoch": 0.8, + "learning_rate": 2.0293692826191624e-05, + "loss": 2.5582, + "theoretical_loss": 3.323463917743676, + "tokens_seen": 2968453120 + }, + { + "epoch": 0.8, + "learning_rate": 2.0285668432033384e-05, + "loss": 2.3836, + "theoretical_loss": 3.3234526108908624, + "tokens_seen": 2968584192 + }, + { + "epoch": 0.8, + "learning_rate": 2.027764403787514e-05, + "loss": 2.5392, + "theoretical_loss": 3.3234413046770475, + "tokens_seen": 2968715264 + }, + { + "epoch": 0.8, + "learning_rate": 2.02696196437169e-05, + "loss": 2.4786, + "theoretical_loss": 3.323429999102167, + "tokens_seen": 2968846336 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1627892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9074361324310303, + "objective/train/theoretical_loss": 3.323424346554307, + "objective/train/tokens_used": 1339370976, + "theoretical_loss": 3.323424346554307, + "tokens_seen": 2968911872 + }, + { + "epoch": 0.8, + "learning_rate": 2.026159524955866e-05, + "loss": 2.5777, + "theoretical_loss": 3.3234186941661568, + "tokens_seen": 2968977408 + }, + { + "epoch": 0.8, + "learning_rate": 2.025357085540042e-05, + "loss": 2.4939, + "theoretical_loss": 3.323407389868952, + "tokens_seen": 2969108480 + }, + { + "epoch": 0.8, + "learning_rate": 2.0245546461242176e-05, + "loss": 2.5926, + "theoretical_loss": 3.3233960862104888, + "tokens_seen": 2969239552 + }, + { + "epoch": 0.8, + "learning_rate": 2.0237522067083936e-05, + "loss": 2.5601, + "theoretical_loss": 3.323384783190703, + "tokens_seen": 2969370624 + }, + { + "epoch": 0.8, + "learning_rate": 2.0229497672925696e-05, + "loss": 2.4749, + "theoretical_loss": 3.3233734808095297, + "tokens_seen": 2969501696 + }, + { + "epoch": 0.8, + "learning_rate": 2.0221473278767453e-05, + "loss": 2.5282, + "theoretical_loss": 3.3233621790669052, + "tokens_seen": 2969632768 + }, + { + "epoch": 0.8, + "learning_rate": 2.0213448884609213e-05, + "loss": 2.4961, + "theoretical_loss": 3.3233508779627656, + "tokens_seen": 2969763840 + }, + { + "epoch": 0.8, + "learning_rate": 2.0205424490450972e-05, + "loss": 2.667, + "theoretical_loss": 3.3233395774970456, + "tokens_seen": 2969894912 + }, + { + "epoch": 0.8, + "learning_rate": 2.0197400096292732e-05, + "loss": 2.5399, + "theoretical_loss": 3.323328277669682, + "tokens_seen": 2970025984 + }, + { + "epoch": 0.8, + "learning_rate": 2.018937570213449e-05, + "loss": 2.5013, + "theoretical_loss": 3.3233169784806096, + "tokens_seen": 2970157056 + }, + { + "epoch": 0.8, + "learning_rate": 2.018135130797625e-05, + "loss": 2.5534, + "theoretical_loss": 3.323305679929765, + "tokens_seen": 2970288128 + }, + { + "epoch": 0.8, + "learning_rate": 2.017332691381801e-05, + "loss": 2.6064, + "theoretical_loss": 3.3232943820170835, + "tokens_seen": 2970419200 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1628471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4990170001983643, + "objective/train/theoretical_loss": 3.323283084742501, + "objective/train/tokens_used": 1341009376, + "theoretical_loss": 3.323283084742501, + "tokens_seen": 2970550272 + }, + { + "epoch": 0.8, + "learning_rate": 2.016530251965977e-05, + "loss": 2.672, + "theoretical_loss": 3.323283084742501, + "tokens_seen": 2970550272 + }, + { + "epoch": 0.8, + "learning_rate": 2.0157278125501525e-05, + "loss": 2.5752, + "theoretical_loss": 3.3232717881059535, + "tokens_seen": 2970681344 + }, + { + "epoch": 0.8, + "learning_rate": 2.0149253731343285e-05, + "loss": 2.6169, + "theoretical_loss": 3.3232604921073765, + "tokens_seen": 2970812416 + }, + { + "epoch": 0.8, + "learning_rate": 2.0141229337185045e-05, + "loss": 2.6416, + "theoretical_loss": 3.323249196746706, + "tokens_seen": 2970943488 + }, + { + "epoch": 0.8, + "learning_rate": 2.0133204943026805e-05, + "loss": 2.5641, + "theoretical_loss": 3.323237902023878, + "tokens_seen": 2971074560 + }, + { + "epoch": 0.8, + "learning_rate": 2.012518054886856e-05, + "loss": 2.5236, + "theoretical_loss": 3.323226607938828, + "tokens_seen": 2971205632 + }, + { + "epoch": 0.8, + "learning_rate": 2.011715615471032e-05, + "loss": 2.5128, + "theoretical_loss": 3.3232153144914927, + "tokens_seen": 2971336704 + }, + { + "epoch": 0.8, + "learning_rate": 2.010913176055208e-05, + "loss": 2.541, + "theoretical_loss": 3.3232040216818066, + "tokens_seen": 2971467776 + }, + { + "epoch": 0.8, + "learning_rate": 2.0101107366393838e-05, + "loss": 2.5905, + "theoretical_loss": 3.3231927295097066, + "tokens_seen": 2971598848 + }, + { + "epoch": 0.8, + "learning_rate": 2.0093082972235598e-05, + "loss": 2.4747, + "theoretical_loss": 3.323181437975128, + "tokens_seen": 2971729920 + }, + { + "epoch": 0.8, + "learning_rate": 2.0085058578077357e-05, + "loss": 2.4937, + "theoretical_loss": 3.3231701470780073, + "tokens_seen": 2971860992 + }, + { + "epoch": 0.8, + "learning_rate": 2.0077034183919117e-05, + "loss": 2.5655, + "theoretical_loss": 3.3231588568182797, + "tokens_seen": 2971992064 + }, + { + "epoch": 0.8, + "learning_rate": 2.0069009789760874e-05, + "loss": 2.5899, + "theoretical_loss": 3.3231475671958814, + "tokens_seen": 2972123136 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1629627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.844027519226074, + "objective/train/theoretical_loss": 3.323141922623661, + "objective/train/tokens_used": 1342647776, + "theoretical_loss": 3.323141922623661, + "tokens_seen": 2972188672 + }, + { + "epoch": 0.8, + "learning_rate": 2.0060985395602634e-05, + "loss": 2.4155, + "theoretical_loss": 3.323136278210749, + "tokens_seen": 2972254208 + }, + { + "epoch": 0.8, + "learning_rate": 2.0052961001444394e-05, + "loss": 2.5375, + "theoretical_loss": 3.3231249898628175, + "tokens_seen": 2972385280 + }, + { + "epoch": 0.8, + "learning_rate": 2.0044936607286154e-05, + "loss": 2.5414, + "theoretical_loss": 3.323113702152023, + "tokens_seen": 2972516352 + }, + { + "epoch": 0.8, + "learning_rate": 2.003691221312791e-05, + "loss": 2.5304, + "theoretical_loss": 3.323102415078301, + "tokens_seen": 2972647424 + }, + { + "epoch": 0.8, + "learning_rate": 2.002888781896967e-05, + "loss": 2.6251, + "theoretical_loss": 3.323091128641589, + "tokens_seen": 2972778496 + }, + { + "epoch": 0.8, + "learning_rate": 2.002086342481143e-05, + "loss": 2.4168, + "theoretical_loss": 3.3230798428418216, + "tokens_seen": 2972909568 + }, + { + "epoch": 0.8, + "learning_rate": 2.0012839030653186e-05, + "loss": 2.5065, + "theoretical_loss": 3.323068557678935, + "tokens_seen": 2973040640 + }, + { + "epoch": 0.8, + "learning_rate": 2.0004814636494946e-05, + "loss": 2.4552, + "theoretical_loss": 3.323057273152866, + "tokens_seen": 2973171712 + }, + { + "epoch": 0.8, + "learning_rate": 1.9996790242336706e-05, + "loss": 2.5342, + "theoretical_loss": 3.3230459892635498, + "tokens_seen": 2973302784 + }, + { + "epoch": 0.8, + "learning_rate": 1.9988765848178466e-05, + "loss": 2.5816, + "theoretical_loss": 3.3230347060109224, + "tokens_seen": 2973433856 + }, + { + "epoch": 0.8, + "learning_rate": 1.9980741454020223e-05, + "loss": 2.4117, + "theoretical_loss": 3.3230234233949196, + "tokens_seen": 2973564928 + }, + { + "epoch": 0.8, + "learning_rate": 1.9972717059861983e-05, + "loss": 2.5884, + "theoretical_loss": 3.3230121414154783, + "tokens_seen": 2973696000 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1630214, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.507581949234009, + "objective/train/theoretical_loss": 3.323000860072534, + "objective/train/tokens_used": 1344286176, + "theoretical_loss": 3.323000860072534, + "tokens_seen": 2973827072 + }, + { + "epoch": 0.8, + "learning_rate": 1.9964692665703743e-05, + "loss": 2.5451, + "theoretical_loss": 3.323000860072534, + "tokens_seen": 2973827072 + }, + { + "epoch": 0.8, + "learning_rate": 1.99566682715455e-05, + "loss": 2.6348, + "theoretical_loss": 3.3229895793660225, + "tokens_seen": 2973958144 + }, + { + "epoch": 0.8, + "learning_rate": 1.994864387738726e-05, + "loss": 2.7013, + "theoretical_loss": 3.3229782992958805, + "tokens_seen": 2974089216 + }, + { + "epoch": 0.8, + "learning_rate": 1.994061948322902e-05, + "loss": 2.6042, + "theoretical_loss": 3.3229670198620433, + "tokens_seen": 2974220288 + }, + { + "epoch": 0.8, + "learning_rate": 1.9932595089070775e-05, + "loss": 2.4578, + "theoretical_loss": 3.3229557410644475, + "tokens_seen": 2974351360 + }, + { + "epoch": 0.8, + "learning_rate": 1.9924570694912535e-05, + "loss": 2.5525, + "theoretical_loss": 3.322944462903029, + "tokens_seen": 2974482432 + }, + { + "epoch": 0.8, + "learning_rate": 1.9916546300754295e-05, + "loss": 2.4676, + "theoretical_loss": 3.3229331853777238, + "tokens_seen": 2974613504 + }, + { + "epoch": 0.8, + "learning_rate": 1.990852190659605e-05, + "loss": 2.682, + "theoretical_loss": 3.3229219084884685, + "tokens_seen": 2974744576 + }, + { + "epoch": 0.8, + "learning_rate": 1.990049751243781e-05, + "loss": 2.5241, + "theoretical_loss": 3.3229106322351982, + "tokens_seen": 2974875648 + }, + { + "epoch": 0.8, + "learning_rate": 1.989247311827957e-05, + "loss": 2.5275, + "theoretical_loss": 3.32289935661785, + "tokens_seen": 2975006720 + }, + { + "epoch": 0.8, + "learning_rate": 1.9884448724121328e-05, + "loss": 2.6142, + "theoretical_loss": 3.3228880816363593, + "tokens_seen": 2975137792 + }, + { + "epoch": 0.8, + "learning_rate": 1.9876424329963088e-05, + "loss": 2.5595, + "theoretical_loss": 3.3228768072906627, + "tokens_seen": 2975268864 + }, + { + "epoch": 0.8, + "learning_rate": 1.9868399935804848e-05, + "loss": 2.5049, + "theoretical_loss": 3.3228655335806967, + "tokens_seen": 2975399936 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1631487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.623626947402954, + "objective/train/theoretical_loss": 3.322859896964092, + "objective/train/tokens_used": 1345924576, + "theoretical_loss": 3.322859896964092, + "tokens_seen": 2975465472 + }, + { + "epoch": 0.8, + "learning_rate": 1.9860375541646604e-05, + "loss": 2.5238, + "theoretical_loss": 3.3228542605063964, + "tokens_seen": 2975531008 + }, + { + "epoch": 0.8, + "learning_rate": 1.9852351147488364e-05, + "loss": 2.5424, + "theoretical_loss": 3.322842988067699, + "tokens_seen": 2975662080 + }, + { + "epoch": 0.8, + "learning_rate": 1.9844326753330124e-05, + "loss": 2.5265, + "theoretical_loss": 3.3228317162645395, + "tokens_seen": 2975793152 + }, + { + "epoch": 0.8, + "learning_rate": 1.983630235917188e-05, + "loss": 2.6107, + "theoretical_loss": 3.3228204450968555, + "tokens_seen": 2975924224 + }, + { + "epoch": 0.8, + "learning_rate": 1.982827796501364e-05, + "loss": 2.4192, + "theoretical_loss": 3.322809174564582, + "tokens_seen": 2976055296 + }, + { + "epoch": 0.8, + "learning_rate": 1.98202535708554e-05, + "loss": 2.5773, + "theoretical_loss": 3.322797904667656, + "tokens_seen": 2976186368 + }, + { + "epoch": 0.8, + "learning_rate": 1.981222917669716e-05, + "loss": 2.4716, + "theoretical_loss": 3.322786635406013, + "tokens_seen": 2976317440 + }, + { + "epoch": 0.8, + "learning_rate": 1.9804204782538917e-05, + "loss": 2.4703, + "theoretical_loss": 3.3227753667795894, + "tokens_seen": 2976448512 + }, + { + "epoch": 0.8, + "learning_rate": 1.9796180388380677e-05, + "loss": 2.5786, + "theoretical_loss": 3.322764098788322, + "tokens_seen": 2976579584 + }, + { + "epoch": 0.8, + "learning_rate": 1.9788155994222437e-05, + "loss": 2.5134, + "theoretical_loss": 3.3227528314321466, + "tokens_seen": 2976710656 + }, + { + "epoch": 0.8, + "learning_rate": 1.9780131600064197e-05, + "loss": 2.4181, + "theoretical_loss": 3.3227415647109995, + "tokens_seen": 2976841728 + }, + { + "epoch": 0.8, + "learning_rate": 1.9772107205905953e-05, + "loss": 2.5052, + "theoretical_loss": 3.3227302986248164, + "tokens_seen": 2976972800 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1632196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4567806720733643, + "objective/train/theoretical_loss": 3.3227190331735343, + "objective/train/tokens_used": 1347562976, + "theoretical_loss": 3.3227190331735343, + "tokens_seen": 2977103872 + }, + { + "epoch": 0.8, + "learning_rate": 1.9764082811747713e-05, + "loss": 2.5594, + "theoretical_loss": 3.3227190331735343, + "tokens_seen": 2977103872 + }, + { + "epoch": 0.8, + "learning_rate": 1.9756058417589473e-05, + "loss": 2.5378, + "theoretical_loss": 3.32270776835709, + "tokens_seen": 2977234944 + }, + { + "epoch": 0.8, + "learning_rate": 1.974803402343123e-05, + "loss": 2.4604, + "theoretical_loss": 3.3226965041754184, + "tokens_seen": 2977366016 + }, + { + "epoch": 0.8, + "learning_rate": 1.974000962927299e-05, + "loss": 2.4363, + "theoretical_loss": 3.322685240628456, + "tokens_seen": 2977497088 + }, + { + "epoch": 0.8, + "learning_rate": 1.973198523511475e-05, + "loss": 2.6001, + "theoretical_loss": 3.32267397771614, + "tokens_seen": 2977628160 + }, + { + "epoch": 0.8, + "learning_rate": 1.972396084095651e-05, + "loss": 2.4608, + "theoretical_loss": 3.3226627154384065, + "tokens_seen": 2977759232 + }, + { + "epoch": 0.8, + "learning_rate": 1.9715936446798266e-05, + "loss": 2.4968, + "theoretical_loss": 3.3226514537951912, + "tokens_seen": 2977890304 + }, + { + "epoch": 0.8, + "learning_rate": 1.9707912052640026e-05, + "loss": 2.5335, + "theoretical_loss": 3.3226401927864306, + "tokens_seen": 2978021376 + }, + { + "epoch": 0.8, + "learning_rate": 1.9699887658481786e-05, + "loss": 2.5739, + "theoretical_loss": 3.3226289324120617, + "tokens_seen": 2978152448 + }, + { + "epoch": 0.81, + "learning_rate": 1.9691863264323545e-05, + "loss": 2.5026, + "theoretical_loss": 3.32261767267202, + "tokens_seen": 2978283520 + }, + { + "epoch": 0.81, + "learning_rate": 1.9683838870165302e-05, + "loss": 2.4909, + "theoretical_loss": 3.322606413566242, + "tokens_seen": 2978414592 + }, + { + "epoch": 0.81, + "learning_rate": 1.9675814476007062e-05, + "loss": 2.5611, + "theoretical_loss": 3.322595155094665, + "tokens_seen": 2978545664 + }, + { + "epoch": 0.81, + "learning_rate": 1.9667790081848822e-05, + "loss": 2.4954, + "theoretical_loss": 3.322583897257224, + "tokens_seen": 2978676736 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1633119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.672278881072998, + "objective/train/theoretical_loss": 3.322578268576285, + "objective/train/tokens_used": 1349201376, + "theoretical_loss": 3.322578268576285, + "tokens_seen": 2978742272 + }, + { + "epoch": 0.81, + "learning_rate": 1.965976568769058e-05, + "loss": 2.5836, + "theoretical_loss": 3.322572640053856, + "tokens_seen": 2978807808 + }, + { + "epoch": 0.81, + "learning_rate": 1.9651741293532338e-05, + "loss": 2.5816, + "theoretical_loss": 3.3225613834844974, + "tokens_seen": 2978938880 + }, + { + "epoch": 0.81, + "learning_rate": 1.9643716899374098e-05, + "loss": 2.4913, + "theoretical_loss": 3.3225501275490847, + "tokens_seen": 2979069952 + }, + { + "epoch": 0.81, + "learning_rate": 1.9635692505215858e-05, + "loss": 2.529, + "theoretical_loss": 3.3225388722475544, + "tokens_seen": 2979201024 + }, + { + "epoch": 0.81, + "learning_rate": 1.9627668111057615e-05, + "loss": 2.7282, + "theoretical_loss": 3.3225276175798424, + "tokens_seen": 2979332096 + }, + { + "epoch": 0.81, + "learning_rate": 1.9619643716899374e-05, + "loss": 2.6273, + "theoretical_loss": 3.3225163635458856, + "tokens_seen": 2979463168 + }, + { + "epoch": 0.81, + "learning_rate": 1.9611619322741134e-05, + "loss": 2.4989, + "theoretical_loss": 3.32250511014562, + "tokens_seen": 2979594240 + }, + { + "epoch": 0.81, + "learning_rate": 1.9603594928582894e-05, + "loss": 2.3487, + "theoretical_loss": 3.322493857378983, + "tokens_seen": 2979725312 + }, + { + "epoch": 0.81, + "learning_rate": 1.959557053442465e-05, + "loss": 2.5914, + "theoretical_loss": 3.32248260524591, + "tokens_seen": 2979856384 + }, + { + "epoch": 0.81, + "learning_rate": 1.958754614026641e-05, + "loss": 2.5876, + "theoretical_loss": 3.3224713537463377, + "tokens_seen": 2979987456 + }, + { + "epoch": 0.81, + "learning_rate": 1.957952174610817e-05, + "loss": 2.5894, + "theoretical_loss": 3.322460102880203, + "tokens_seen": 2980118528 + }, + { + "epoch": 0.81, + "learning_rate": 1.957149735194993e-05, + "loss": 2.5152, + "theoretical_loss": 3.322448852647442, + "tokens_seen": 2980249600 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1634363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.772721767425537, + "objective/train/theoretical_loss": 3.322437603047991, + "objective/train/tokens_used": 1350839776, + "theoretical_loss": 3.322437603047991, + "tokens_seen": 2980380672 + }, + { + "epoch": 0.81, + "learning_rate": 1.9563472957791687e-05, + "loss": 2.7406, + "theoretical_loss": 3.322437603047991, + "tokens_seen": 2980380672 + }, + { + "epoch": 0.81, + "learning_rate": 1.9555448563633447e-05, + "loss": 2.5031, + "theoretical_loss": 3.322426354081787, + "tokens_seen": 2980511744 + }, + { + "epoch": 0.81, + "learning_rate": 1.9547424169475207e-05, + "loss": 2.5133, + "theoretical_loss": 3.3224151057487665, + "tokens_seen": 2980642816 + }, + { + "epoch": 0.81, + "learning_rate": 1.9539399775316963e-05, + "loss": 2.5667, + "theoretical_loss": 3.3224038580488653, + "tokens_seen": 2980773888 + }, + { + "epoch": 0.81, + "learning_rate": 1.9531375381158723e-05, + "loss": 2.5665, + "theoretical_loss": 3.3223926109820208, + "tokens_seen": 2980904960 + }, + { + "epoch": 0.81, + "learning_rate": 1.9523350987000483e-05, + "loss": 2.4836, + "theoretical_loss": 3.3223813645481695, + "tokens_seen": 2981036032 + }, + { + "epoch": 0.81, + "learning_rate": 1.9515326592842243e-05, + "loss": 2.6129, + "theoretical_loss": 3.322370118747247, + "tokens_seen": 2981167104 + }, + { + "epoch": 0.81, + "learning_rate": 1.9507302198684e-05, + "loss": 2.6632, + "theoretical_loss": 3.3223588735791907, + "tokens_seen": 2981298176 + }, + { + "epoch": 0.81, + "learning_rate": 1.949927780452576e-05, + "loss": 2.4869, + "theoretical_loss": 3.3223476290439367, + "tokens_seen": 2981429248 + }, + { + "epoch": 0.81, + "learning_rate": 1.949125341036752e-05, + "loss": 2.6638, + "theoretical_loss": 3.3223363851414223, + "tokens_seen": 2981560320 + }, + { + "epoch": 0.81, + "learning_rate": 1.948322901620928e-05, + "loss": 2.5038, + "theoretical_loss": 3.322325141871583, + "tokens_seen": 2981691392 + }, + { + "epoch": 0.81, + "learning_rate": 1.9475204622051036e-05, + "loss": 2.6047, + "theoretical_loss": 3.3223138992343566, + "tokens_seen": 2981822464 + }, + { + "epoch": 0.81, + "learning_rate": 1.9467180227892796e-05, + "loss": 2.5197, + "theoretical_loss": 3.322302657229679, + "tokens_seen": 2981953536 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1634841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.645475387573242, + "objective/train/theoretical_loss": 3.322297036464526, + "objective/train/tokens_used": 1352478176, + "theoretical_loss": 3.322297036464526, + "tokens_seen": 2982019072 + }, + { + "epoch": 0.81, + "learning_rate": 1.9459155833734556e-05, + "loss": 2.5284, + "theoretical_loss": 3.3222914158574866, + "tokens_seen": 2982084608 + }, + { + "epoch": 0.81, + "learning_rate": 1.9451131439576312e-05, + "loss": 2.5282, + "theoretical_loss": 3.3222801751177164, + "tokens_seen": 2982215680 + }, + { + "epoch": 0.81, + "learning_rate": 1.9443107045418072e-05, + "loss": 2.5006, + "theoretical_loss": 3.322268935010305, + "tokens_seen": 2982346752 + }, + { + "epoch": 0.81, + "learning_rate": 1.9435082651259832e-05, + "loss": 2.729, + "theoretical_loss": 3.322257695535189, + "tokens_seen": 2982477824 + }, + { + "epoch": 0.81, + "learning_rate": 1.9427058257101592e-05, + "loss": 2.4444, + "theoretical_loss": 3.322246456692305, + "tokens_seen": 2982608896 + }, + { + "epoch": 0.81, + "learning_rate": 1.941903386294335e-05, + "loss": 2.5602, + "theoretical_loss": 3.3222352184815893, + "tokens_seen": 2982739968 + }, + { + "epoch": 0.81, + "learning_rate": 1.9411009468785108e-05, + "loss": 2.5188, + "theoretical_loss": 3.3222239809029794, + "tokens_seen": 2982871040 + }, + { + "epoch": 0.81, + "learning_rate": 1.9402985074626868e-05, + "loss": 2.6108, + "theoretical_loss": 3.3222127439564115, + "tokens_seen": 2983002112 + }, + { + "epoch": 0.81, + "learning_rate": 1.9394960680468628e-05, + "loss": 2.5719, + "theoretical_loss": 3.322201507641822, + "tokens_seen": 2983133184 + }, + { + "epoch": 0.81, + "learning_rate": 1.9386936286310385e-05, + "loss": 2.6332, + "theoretical_loss": 3.3221902719591485, + "tokens_seen": 2983264256 + }, + { + "epoch": 0.81, + "learning_rate": 1.9378911892152144e-05, + "loss": 2.5889, + "theoretical_loss": 3.3221790369083264, + "tokens_seen": 2983395328 + }, + { + "epoch": 0.81, + "learning_rate": 1.9370887497993904e-05, + "loss": 2.6551, + "theoretical_loss": 3.322167802489293, + "tokens_seen": 2983526400 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1636104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5192723274230957, + "objective/train/theoretical_loss": 3.3221565687019856, + "objective/train/tokens_used": 1354116576, + "theoretical_loss": 3.3221565687019856, + "tokens_seen": 2983657472 + }, + { + "epoch": 0.81, + "learning_rate": 1.936286310383566e-05, + "loss": 2.569, + "theoretical_loss": 3.3221565687019856, + "tokens_seen": 2983657472 + }, + { + "epoch": 0.81, + "learning_rate": 1.935483870967742e-05, + "loss": 2.4781, + "theoretical_loss": 3.322145335546341, + "tokens_seen": 2983788544 + }, + { + "epoch": 0.81, + "learning_rate": 1.934681431551918e-05, + "loss": 2.7399, + "theoretical_loss": 3.3221341030222944, + "tokens_seen": 2983919616 + }, + { + "epoch": 0.81, + "learning_rate": 1.933878992136094e-05, + "loss": 2.5772, + "theoretical_loss": 3.3221228711297837, + "tokens_seen": 2984050688 + }, + { + "epoch": 0.81, + "learning_rate": 1.9330765527202697e-05, + "loss": 2.5531, + "theoretical_loss": 3.3221116398687456, + "tokens_seen": 2984181760 + }, + { + "epoch": 0.81, + "learning_rate": 1.9322741133044457e-05, + "loss": 2.5363, + "theoretical_loss": 3.3221004092391166, + "tokens_seen": 2984312832 + }, + { + "epoch": 0.81, + "learning_rate": 1.9314716738886217e-05, + "loss": 2.5448, + "theoretical_loss": 3.322089179240834, + "tokens_seen": 2984443904 + }, + { + "epoch": 0.81, + "learning_rate": 1.9306692344727973e-05, + "loss": 2.7139, + "theoretical_loss": 3.322077949873834, + "tokens_seen": 2984574976 + }, + { + "epoch": 0.81, + "learning_rate": 1.9298667950569733e-05, + "loss": 2.5138, + "theoretical_loss": 3.3220667211380537, + "tokens_seen": 2984706048 + }, + { + "epoch": 0.81, + "learning_rate": 1.9290643556411493e-05, + "loss": 2.6452, + "theoretical_loss": 3.3220554930334294, + "tokens_seen": 2984837120 + }, + { + "epoch": 0.81, + "learning_rate": 1.928261916225325e-05, + "loss": 2.6549, + "theoretical_loss": 3.3220442655598985, + "tokens_seen": 2984968192 + }, + { + "epoch": 0.81, + "learning_rate": 1.927459476809501e-05, + "loss": 2.6688, + "theoretical_loss": 3.322033038717398, + "tokens_seen": 2985099264 + }, + { + "epoch": 0.81, + "learning_rate": 1.926657037393677e-05, + "loss": 2.5945, + "theoretical_loss": 3.322021812505864, + "tokens_seen": 2985230336 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1636535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.291861057281494, + "objective/train/theoretical_loss": 3.32201619963669, + "objective/train/tokens_used": 1355754976, + "theoretical_loss": 3.32201619963669, + "tokens_seen": 2985295872 + }, + { + "epoch": 0.81, + "learning_rate": 1.9258545979778526e-05, + "loss": 2.3832, + "theoretical_loss": 3.3220105869252334, + "tokens_seen": 2985361408 + }, + { + "epoch": 0.81, + "learning_rate": 1.9250521585620286e-05, + "loss": 2.5043, + "theoretical_loss": 3.3219993619754438, + "tokens_seen": 2985492480 + }, + { + "epoch": 0.81, + "learning_rate": 1.9242497191462046e-05, + "loss": 2.5863, + "theoretical_loss": 3.3219881376564317, + "tokens_seen": 2985623552 + }, + { + "epoch": 0.81, + "learning_rate": 1.9234472797303802e-05, + "loss": 2.3261, + "theoretical_loss": 3.3219769139681334, + "tokens_seen": 2985754624 + }, + { + "epoch": 0.81, + "learning_rate": 1.9226448403145562e-05, + "loss": 2.5225, + "theoretical_loss": 3.3219656909104867, + "tokens_seen": 2985885696 + }, + { + "epoch": 0.81, + "learning_rate": 1.9218424008987322e-05, + "loss": 2.5418, + "theoretical_loss": 3.3219544684834275, + "tokens_seen": 2986016768 + }, + { + "epoch": 0.81, + "learning_rate": 1.921039961482908e-05, + "loss": 2.6172, + "theoretical_loss": 3.3219432466868937, + "tokens_seen": 2986147840 + }, + { + "epoch": 0.81, + "learning_rate": 1.920237522067084e-05, + "loss": 2.5011, + "theoretical_loss": 3.3219320255208213, + "tokens_seen": 2986278912 + }, + { + "epoch": 0.81, + "learning_rate": 1.91943508265126e-05, + "loss": 2.482, + "theoretical_loss": 3.321920804985148, + "tokens_seen": 2986409984 + }, + { + "epoch": 0.81, + "learning_rate": 1.9186326432354355e-05, + "loss": 2.6611, + "theoretical_loss": 3.32190958507981, + "tokens_seen": 2986541056 + }, + { + "epoch": 0.81, + "learning_rate": 1.9178302038196115e-05, + "loss": 2.4212, + "theoretical_loss": 3.3218983658047447, + "tokens_seen": 2986672128 + }, + { + "epoch": 0.81, + "learning_rate": 1.9170277644037875e-05, + "loss": 2.6422, + "theoretical_loss": 3.321887147159889, + "tokens_seen": 2986803200 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1637653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5154991149902344, + "objective/train/theoretical_loss": 3.3218759291451794, + "objective/train/tokens_used": 1357393376, + "theoretical_loss": 3.3218759291451794, + "tokens_seen": 2986934272 + }, + { + "epoch": 0.81, + "learning_rate": 1.9162253249879635e-05, + "loss": 2.5331, + "theoretical_loss": 3.3218759291451794, + "tokens_seen": 2986934272 + }, + { + "epoch": 0.81, + "learning_rate": 1.915422885572139e-05, + "loss": 2.491, + "theoretical_loss": 3.3218647117605533, + "tokens_seen": 2987065344 + }, + { + "epoch": 0.81, + "learning_rate": 1.914620446156315e-05, + "loss": 2.6404, + "theoretical_loss": 3.321853495005948, + "tokens_seen": 2987196416 + }, + { + "epoch": 0.81, + "learning_rate": 1.913818006740491e-05, + "loss": 2.3663, + "theoretical_loss": 3.3218422788812996, + "tokens_seen": 2987327488 + }, + { + "epoch": 0.81, + "learning_rate": 1.913015567324667e-05, + "loss": 2.4659, + "theoretical_loss": 3.3218310633865458, + "tokens_seen": 2987458560 + }, + { + "epoch": 0.81, + "learning_rate": 1.9122131279088428e-05, + "loss": 2.4597, + "theoretical_loss": 3.3218198485216233, + "tokens_seen": 2987589632 + }, + { + "epoch": 0.81, + "learning_rate": 1.9114106884930188e-05, + "loss": 2.5271, + "theoretical_loss": 3.3218086342864686, + "tokens_seen": 2987720704 + }, + { + "epoch": 0.81, + "learning_rate": 1.9106082490771947e-05, + "loss": 2.4177, + "theoretical_loss": 3.32179742068102, + "tokens_seen": 2987851776 + }, + { + "epoch": 0.81, + "learning_rate": 1.9098058096613707e-05, + "loss": 2.6712, + "theoretical_loss": 3.3217862077052134, + "tokens_seen": 2987982848 + }, + { + "epoch": 0.81, + "learning_rate": 1.9090033702455464e-05, + "loss": 2.4935, + "theoretical_loss": 3.321774995358986, + "tokens_seen": 2988113920 + }, + { + "epoch": 0.81, + "learning_rate": 1.9082009308297224e-05, + "loss": 2.4095, + "theoretical_loss": 3.321763783642275, + "tokens_seen": 2988244992 + }, + { + "epoch": 0.81, + "learning_rate": 1.9073984914138984e-05, + "loss": 2.4904, + "theoretical_loss": 3.3217525725550177, + "tokens_seen": 2988376064 + }, + { + "epoch": 0.81, + "learning_rate": 1.906596051998074e-05, + "loss": 2.4734, + "theoretical_loss": 3.3217413620971508, + "tokens_seen": 2988507136 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1638307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6184804439544678, + "objective/train/theoretical_loss": 3.3217357571042188, + "objective/train/tokens_used": 1359031776, + "theoretical_loss": 3.3217357571042188, + "tokens_seen": 2988572672 + }, + { + "epoch": 0.81, + "learning_rate": 1.90579361258225e-05, + "loss": 2.5475, + "theoretical_loss": 3.321730152268611, + "tokens_seen": 2988638208 + }, + { + "epoch": 0.81, + "learning_rate": 1.904991173166426e-05, + "loss": 2.6722, + "theoretical_loss": 3.3217189430693366, + "tokens_seen": 2988769280 + }, + { + "epoch": 0.81, + "learning_rate": 1.904188733750602e-05, + "loss": 2.5831, + "theoretical_loss": 3.3217077344992636, + "tokens_seen": 2988900352 + }, + { + "epoch": 0.81, + "learning_rate": 1.9033862943347776e-05, + "loss": 2.6092, + "theoretical_loss": 3.3216965265583296, + "tokens_seen": 2989031424 + }, + { + "epoch": 0.81, + "learning_rate": 1.9025838549189536e-05, + "loss": 2.5338, + "theoretical_loss": 3.3216853192464715, + "tokens_seen": 2989162496 + }, + { + "epoch": 0.81, + "learning_rate": 1.9017814155031296e-05, + "loss": 2.5576, + "theoretical_loss": 3.321674112563626, + "tokens_seen": 2989293568 + }, + { + "epoch": 0.81, + "learning_rate": 1.9009789760873056e-05, + "loss": 2.451, + "theoretical_loss": 3.321662906509731, + "tokens_seen": 2989424640 + }, + { + "epoch": 0.81, + "learning_rate": 1.9001765366714813e-05, + "loss": 2.6769, + "theoretical_loss": 3.3216517010847233, + "tokens_seen": 2989555712 + }, + { + "epoch": 0.81, + "learning_rate": 1.8993740972556573e-05, + "loss": 2.5636, + "theoretical_loss": 3.3216404962885395, + "tokens_seen": 2989686784 + }, + { + "epoch": 0.81, + "learning_rate": 1.8985716578398332e-05, + "loss": 2.4565, + "theoretical_loss": 3.321629292121118, + "tokens_seen": 2989817856 + }, + { + "epoch": 0.81, + "learning_rate": 1.897769218424009e-05, + "loss": 2.641, + "theoretical_loss": 3.3216180885823947, + "tokens_seen": 2989948928 + }, + { + "epoch": 0.81, + "learning_rate": 1.896966779008185e-05, + "loss": 2.4403, + "theoretical_loss": 3.3216068856723076, + "tokens_seen": 2990080000 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1639722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.678413152694702, + "objective/train/theoretical_loss": 3.3215956833907936, + "objective/train/tokens_used": 1360670176, + "theoretical_loss": 3.3215956833907936, + "tokens_seen": 2990211072 + }, + { + "epoch": 0.81, + "learning_rate": 1.896164339592361e-05, + "loss": 2.6318, + "theoretical_loss": 3.3215956833907936, + "tokens_seen": 2990211072 + }, + { + "epoch": 0.81, + "learning_rate": 1.895361900176537e-05, + "loss": 2.5696, + "theoretical_loss": 3.3215844817377898, + "tokens_seen": 2990342144 + }, + { + "epoch": 0.81, + "learning_rate": 1.8945594607607125e-05, + "loss": 2.5756, + "theoretical_loss": 3.321573280713233, + "tokens_seen": 2990473216 + }, + { + "epoch": 0.81, + "learning_rate": 1.8937570213448885e-05, + "loss": 2.5371, + "theoretical_loss": 3.321562080317061, + "tokens_seen": 2990604288 + }, + { + "epoch": 0.81, + "learning_rate": 1.8929545819290645e-05, + "loss": 2.5994, + "theoretical_loss": 3.321550880549211, + "tokens_seen": 2990735360 + }, + { + "epoch": 0.81, + "learning_rate": 1.8921521425132405e-05, + "loss": 2.5435, + "theoretical_loss": 3.32153968140962, + "tokens_seen": 2990866432 + }, + { + "epoch": 0.81, + "learning_rate": 1.891349703097416e-05, + "loss": 2.446, + "theoretical_loss": 3.321528482898225, + "tokens_seen": 2990997504 + }, + { + "epoch": 0.81, + "learning_rate": 1.890547263681592e-05, + "loss": 2.6237, + "theoretical_loss": 3.3215172850149637, + "tokens_seen": 2991128576 + }, + { + "epoch": 0.81, + "learning_rate": 1.889744824265768e-05, + "loss": 2.6811, + "theoretical_loss": 3.3215060877597735, + "tokens_seen": 2991259648 + }, + { + "epoch": 0.81, + "learning_rate": 1.8889423848499438e-05, + "loss": 2.6433, + "theoretical_loss": 3.3214948911325908, + "tokens_seen": 2991390720 + }, + { + "epoch": 0.81, + "learning_rate": 1.8881399454341198e-05, + "loss": 2.4884, + "theoretical_loss": 3.3214836951333537, + "tokens_seen": 2991521792 + }, + { + "epoch": 0.81, + "learning_rate": 1.8873375060182958e-05, + "loss": 2.4188, + "theoretical_loss": 3.321472499761999, + "tokens_seen": 2991652864 + }, + { + "epoch": 0.81, + "learning_rate": 1.8865350666024717e-05, + "loss": 2.4454, + "theoretical_loss": 3.321461305018464, + "tokens_seen": 2991783936 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1640856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8433732986450195, + "objective/train/theoretical_loss": 3.3214557078821096, + "objective/train/tokens_used": 1362308576, + "theoretical_loss": 3.3214557078821096, + "tokens_seen": 2991849472 + }, + { + "epoch": 0.81, + "learning_rate": 1.8857326271866474e-05, + "loss": 2.6711, + "theoretical_loss": 3.3214501109026866, + "tokens_seen": 2991915008 + }, + { + "epoch": 0.81, + "learning_rate": 1.8849301877708234e-05, + "loss": 2.3747, + "theoretical_loss": 3.321438917414603, + "tokens_seen": 2992046080 + }, + { + "epoch": 0.81, + "learning_rate": 1.8841277483549994e-05, + "loss": 2.4829, + "theoretical_loss": 3.3214277245541513, + "tokens_seen": 2992177152 + }, + { + "epoch": 0.81, + "learning_rate": 1.8833253089391754e-05, + "loss": 2.531, + "theoretical_loss": 3.321416532321269, + "tokens_seen": 2992308224 + }, + { + "epoch": 0.81, + "learning_rate": 1.882522869523351e-05, + "loss": 2.4599, + "theoretical_loss": 3.3214053407158923, + "tokens_seen": 2992439296 + }, + { + "epoch": 0.81, + "learning_rate": 1.881720430107527e-05, + "loss": 2.4871, + "theoretical_loss": 3.32139414973796, + "tokens_seen": 2992570368 + }, + { + "epoch": 0.81, + "learning_rate": 1.880917990691703e-05, + "loss": 2.4085, + "theoretical_loss": 3.3213829593874085, + "tokens_seen": 2992701440 + }, + { + "epoch": 0.81, + "learning_rate": 1.8801155512758787e-05, + "loss": 2.577, + "theoretical_loss": 3.321371769664175, + "tokens_seen": 2992832512 + }, + { + "epoch": 0.81, + "learning_rate": 1.8793131118600546e-05, + "loss": 2.6843, + "theoretical_loss": 3.321360580568198, + "tokens_seen": 2992963584 + }, + { + "epoch": 0.81, + "learning_rate": 1.8785106724442306e-05, + "loss": 2.6063, + "theoretical_loss": 3.321349392099414, + "tokens_seen": 2993094656 + }, + { + "epoch": 0.81, + "learning_rate": 1.8777082330284066e-05, + "loss": 2.5131, + "theoretical_loss": 3.32133820425776, + "tokens_seen": 2993225728 + }, + { + "epoch": 0.81, + "learning_rate": 1.8769057936125823e-05, + "loss": 2.3793, + "theoretical_loss": 3.321327017043174, + "tokens_seen": 2993356800 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1641461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4305381774902344, + "objective/train/theoretical_loss": 3.321315830455594, + "objective/train/tokens_used": 1363946976, + "theoretical_loss": 3.321315830455594, + "tokens_seen": 2993487872 + }, + { + "epoch": 0.81, + "learning_rate": 1.8761033541967583e-05, + "loss": 2.5237, + "theoretical_loss": 3.321315830455594, + "tokens_seen": 2993487872 + }, + { + "epoch": 0.81, + "learning_rate": 1.8753009147809343e-05, + "loss": 2.5376, + "theoretical_loss": 3.321304644494956, + "tokens_seen": 2993618944 + }, + { + "epoch": 0.81, + "learning_rate": 1.8744984753651103e-05, + "loss": 2.4436, + "theoretical_loss": 3.3212934591611982, + "tokens_seen": 2993750016 + }, + { + "epoch": 0.81, + "learning_rate": 1.873696035949286e-05, + "loss": 2.4605, + "theoretical_loss": 3.321282274454258, + "tokens_seen": 2993881088 + }, + { + "epoch": 0.81, + "learning_rate": 1.872893596533462e-05, + "loss": 2.5246, + "theoretical_loss": 3.321271090374073, + "tokens_seen": 2994012160 + }, + { + "epoch": 0.81, + "learning_rate": 1.872091157117638e-05, + "loss": 2.5197, + "theoretical_loss": 3.32125990692058, + "tokens_seen": 2994143232 + }, + { + "epoch": 0.81, + "learning_rate": 1.8712887177018135e-05, + "loss": 2.6335, + "theoretical_loss": 3.321248724093717, + "tokens_seen": 2994274304 + }, + { + "epoch": 0.81, + "learning_rate": 1.8704862782859895e-05, + "loss": 2.5897, + "theoretical_loss": 3.3212375418934217, + "tokens_seen": 2994405376 + }, + { + "epoch": 0.81, + "learning_rate": 1.8696838388701655e-05, + "loss": 2.401, + "theoretical_loss": 3.321226360319631, + "tokens_seen": 2994536448 + }, + { + "epoch": 0.81, + "learning_rate": 1.8688813994543415e-05, + "loss": 2.5462, + "theoretical_loss": 3.3212151793722824, + "tokens_seen": 2994667520 + }, + { + "epoch": 0.82, + "learning_rate": 1.868078960038517e-05, + "loss": 2.5192, + "theoretical_loss": 3.321203999051314, + "tokens_seen": 2994798592 + }, + { + "epoch": 0.82, + "learning_rate": 1.867276520622693e-05, + "loss": 2.5558, + "theoretical_loss": 3.321192819356662, + "tokens_seen": 2994929664 + }, + { + "epoch": 0.82, + "learning_rate": 1.866474081206869e-05, + "loss": 2.6745, + "theoretical_loss": 3.3211816402882652, + "tokens_seen": 2995060736 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1642666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0282912254333496, + "objective/train/theoretical_loss": 3.321176050988893, + "objective/train/tokens_used": 1365585376, + "theoretical_loss": 3.321176050988893, + "tokens_seen": 2995126272 + }, + { + "epoch": 0.82, + "learning_rate": 1.865671641791045e-05, + "loss": 2.4081, + "theoretical_loss": 3.3211704618460614, + "tokens_seen": 2995191808 + }, + { + "epoch": 0.82, + "learning_rate": 1.8648692023752208e-05, + "loss": 2.6275, + "theoretical_loss": 3.3211592840299864, + "tokens_seen": 2995322880 + }, + { + "epoch": 0.82, + "learning_rate": 1.8640667629593968e-05, + "loss": 2.5476, + "theoretical_loss": 3.321148106839979, + "tokens_seen": 2995453952 + }, + { + "epoch": 0.82, + "learning_rate": 1.8632643235435728e-05, + "loss": 2.5656, + "theoretical_loss": 3.321136930275977, + "tokens_seen": 2995585024 + }, + { + "epoch": 0.82, + "learning_rate": 1.8624618841277484e-05, + "loss": 2.5605, + "theoretical_loss": 3.321125754337917, + "tokens_seen": 2995716096 + }, + { + "epoch": 0.82, + "learning_rate": 1.8616594447119244e-05, + "loss": 2.4721, + "theoretical_loss": 3.3211145790257373, + "tokens_seen": 2995847168 + }, + { + "epoch": 0.82, + "learning_rate": 1.8608570052961004e-05, + "loss": 2.4505, + "theoretical_loss": 3.3211034043393743, + "tokens_seen": 2995978240 + }, + { + "epoch": 0.82, + "learning_rate": 1.860054565880276e-05, + "loss": 2.609, + "theoretical_loss": 3.3210922302787673, + "tokens_seen": 2996109312 + }, + { + "epoch": 0.82, + "learning_rate": 1.859252126464452e-05, + "loss": 2.4113, + "theoretical_loss": 3.3210810568438527, + "tokens_seen": 2996240384 + }, + { + "epoch": 0.82, + "learning_rate": 1.858449687048628e-05, + "loss": 2.6803, + "theoretical_loss": 3.3210698840345687, + "tokens_seen": 2996371456 + }, + { + "epoch": 0.82, + "learning_rate": 1.8576472476328037e-05, + "loss": 2.6266, + "theoretical_loss": 3.3210587118508523, + "tokens_seen": 2996502528 + }, + { + "epoch": 0.82, + "learning_rate": 1.8568448082169797e-05, + "loss": 2.5338, + "theoretical_loss": 3.3210475402926414, + "tokens_seen": 2996633600 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1643300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.658358097076416, + "objective/train/theoretical_loss": 3.3210363693598737, + "objective/train/tokens_used": 1367223776, + "theoretical_loss": 3.3210363693598737, + "tokens_seen": 2996764672 + }, + { + "epoch": 0.82, + "learning_rate": 1.8560423688011557e-05, + "loss": 2.4897, + "theoretical_loss": 3.3210363693598737, + "tokens_seen": 2996764672 + }, + { + "epoch": 0.82, + "learning_rate": 1.8552399293853313e-05, + "loss": 2.4023, + "theoretical_loss": 3.321025199052487, + "tokens_seen": 2996895744 + }, + { + "epoch": 0.82, + "learning_rate": 1.8544374899695073e-05, + "loss": 2.4781, + "theoretical_loss": 3.321014029370419, + "tokens_seen": 2997026816 + }, + { + "epoch": 0.82, + "learning_rate": 1.8536350505536833e-05, + "loss": 2.3616, + "theoretical_loss": 3.3210028603136066, + "tokens_seen": 2997157888 + }, + { + "epoch": 0.82, + "learning_rate": 1.852832611137859e-05, + "loss": 2.3119, + "theoretical_loss": 3.320991691881988, + "tokens_seen": 2997288960 + }, + { + "epoch": 0.82, + "learning_rate": 1.852030171722035e-05, + "loss": 2.3814, + "theoretical_loss": 3.320980524075501, + "tokens_seen": 2997420032 + }, + { + "epoch": 0.82, + "learning_rate": 1.851227732306211e-05, + "loss": 2.5025, + "theoretical_loss": 3.3209693568940826, + "tokens_seen": 2997551104 + }, + { + "epoch": 0.82, + "learning_rate": 1.8504252928903866e-05, + "loss": 2.6205, + "theoretical_loss": 3.3209581903376715, + "tokens_seen": 2997682176 + }, + { + "epoch": 0.82, + "learning_rate": 1.8496228534745626e-05, + "loss": 2.5281, + "theoretical_loss": 3.3209470244062045, + "tokens_seen": 2997813248 + }, + { + "epoch": 0.82, + "learning_rate": 1.8488204140587386e-05, + "loss": 2.4274, + "theoretical_loss": 3.32093585909962, + "tokens_seen": 2997944320 + }, + { + "epoch": 0.82, + "learning_rate": 1.8480179746429146e-05, + "loss": 2.4285, + "theoretical_loss": 3.320924694417855, + "tokens_seen": 2998075392 + }, + { + "epoch": 0.82, + "learning_rate": 1.8472155352270902e-05, + "loss": 2.6746, + "theoretical_loss": 3.3209135303608477, + "tokens_seen": 2998206464 + }, + { + "epoch": 0.82, + "learning_rate": 1.8464130958112662e-05, + "loss": 2.6209, + "theoretical_loss": 3.320902366928536, + "tokens_seen": 2998337536 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1644380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.322796583175659, + "objective/train/theoretical_loss": 3.3208967854466214, + "objective/train/tokens_used": 1368862176, + "theoretical_loss": 3.3208967854466214, + "tokens_seen": 2998403072 + }, + { + "epoch": 0.82, + "learning_rate": 1.8456106563954422e-05, + "loss": 2.5389, + "theoretical_loss": 3.320891204120857, + "tokens_seen": 2998468608 + }, + { + "epoch": 0.82, + "learning_rate": 1.8448082169796182e-05, + "loss": 2.6199, + "theoretical_loss": 3.320880041937749, + "tokens_seen": 2998599680 + }, + { + "epoch": 0.82, + "learning_rate": 1.8440057775637938e-05, + "loss": 2.4422, + "theoretical_loss": 3.3208688803791495, + "tokens_seen": 2998730752 + }, + { + "epoch": 0.82, + "learning_rate": 1.8432033381479698e-05, + "loss": 2.4386, + "theoretical_loss": 3.3208577194449966, + "tokens_seen": 2998861824 + }, + { + "epoch": 0.82, + "learning_rate": 1.8424008987321458e-05, + "loss": 2.4746, + "theoretical_loss": 3.3208465591352274, + "tokens_seen": 2998992896 + }, + { + "epoch": 0.82, + "learning_rate": 1.8415984593163215e-05, + "loss": 2.5119, + "theoretical_loss": 3.3208353994497806, + "tokens_seen": 2999123968 + }, + { + "epoch": 0.82, + "learning_rate": 1.8407960199004975e-05, + "loss": 2.4155, + "theoretical_loss": 3.320824240388593, + "tokens_seen": 2999255040 + }, + { + "epoch": 0.82, + "learning_rate": 1.8399935804846734e-05, + "loss": 2.4545, + "theoretical_loss": 3.320813081951603, + "tokens_seen": 2999386112 + }, + { + "epoch": 0.82, + "learning_rate": 1.8391911410688494e-05, + "loss": 2.6917, + "theoretical_loss": 3.320801924138748, + "tokens_seen": 2999517184 + }, + { + "epoch": 0.82, + "learning_rate": 1.838388701653025e-05, + "loss": 2.6181, + "theoretical_loss": 3.3207907669499663, + "tokens_seen": 2999648256 + }, + { + "epoch": 0.82, + "learning_rate": 1.837586262237201e-05, + "loss": 2.4262, + "theoretical_loss": 3.320779610385196, + "tokens_seen": 2999779328 + }, + { + "epoch": 0.82, + "learning_rate": 1.836783822821377e-05, + "loss": 2.4375, + "theoretical_loss": 3.320768454444374, + "tokens_seen": 2999910400 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1645056, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5875041484832764, + "objective/train/theoretical_loss": 3.3207572991274388, + "objective/train/tokens_used": 1370500576, + "theoretical_loss": 3.3207572991274388, + "tokens_seen": 3000041472 + }, + { + "epoch": 0.82, + "learning_rate": 1.835981383405553e-05, + "loss": 2.5155, + "theoretical_loss": 3.3207572991274388, + "tokens_seen": 3000041472 + }, + { + "epoch": 0.82, + "learning_rate": 1.8351789439897287e-05, + "loss": 2.5403, + "theoretical_loss": 3.3207461444343283, + "tokens_seen": 3000172544 + }, + { + "epoch": 0.82, + "learning_rate": 1.8343765045739047e-05, + "loss": 2.5076, + "theoretical_loss": 3.3207349903649797, + "tokens_seen": 3000303616 + }, + { + "epoch": 0.82, + "learning_rate": 1.8335740651580807e-05, + "loss": 2.4339, + "theoretical_loss": 3.3207238369193313, + "tokens_seen": 3000434688 + }, + { + "epoch": 0.82, + "learning_rate": 1.8327716257422563e-05, + "loss": 2.586, + "theoretical_loss": 3.3207126840973213, + "tokens_seen": 3000565760 + }, + { + "epoch": 0.82, + "learning_rate": 1.8319691863264323e-05, + "loss": 2.533, + "theoretical_loss": 3.3207015318988873, + "tokens_seen": 3000696832 + }, + { + "epoch": 0.82, + "learning_rate": 1.8311667469106083e-05, + "loss": 2.584, + "theoretical_loss": 3.320690380323967, + "tokens_seen": 3000827904 + }, + { + "epoch": 0.82, + "learning_rate": 1.8303643074947843e-05, + "loss": 2.5551, + "theoretical_loss": 3.3206792293724985, + "tokens_seen": 3000958976 + }, + { + "epoch": 0.82, + "learning_rate": 1.82956186807896e-05, + "loss": 2.5314, + "theoretical_loss": 3.3206680790444194, + "tokens_seen": 3001090048 + }, + { + "epoch": 0.82, + "learning_rate": 1.828759428663136e-05, + "loss": 2.569, + "theoretical_loss": 3.3206569293396684, + "tokens_seen": 3001221120 + }, + { + "epoch": 0.82, + "learning_rate": 1.827956989247312e-05, + "loss": 2.4471, + "theoretical_loss": 3.320645780258183, + "tokens_seen": 3001352192 + }, + { + "epoch": 0.82, + "learning_rate": 1.827154549831488e-05, + "loss": 2.4888, + "theoretical_loss": 3.3206346317999005, + "tokens_seen": 3001483264 + }, + { + "epoch": 0.82, + "learning_rate": 1.8263521104156636e-05, + "loss": 2.5283, + "theoretical_loss": 3.32062348396476, + "tokens_seen": 3001614336 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1646327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5438573360443115, + "objective/train/theoretical_loss": 3.3206179102808484, + "objective/train/tokens_used": 1372138976, + "theoretical_loss": 3.3206179102808484, + "tokens_seen": 3001679872 + }, + { + "epoch": 0.82, + "learning_rate": 1.8255496709998396e-05, + "loss": 2.4421, + "theoretical_loss": 3.3206123367526987, + "tokens_seen": 3001745408 + }, + { + "epoch": 0.82, + "learning_rate": 1.8247472315840156e-05, + "loss": 2.5707, + "theoretical_loss": 3.320601190163655, + "tokens_seen": 3001876480 + }, + { + "epoch": 0.82, + "learning_rate": 1.8239447921681912e-05, + "loss": 2.4456, + "theoretical_loss": 3.320590044197566, + "tokens_seen": 3002007552 + }, + { + "epoch": 0.82, + "learning_rate": 1.8231423527523672e-05, + "loss": 2.3618, + "theoretical_loss": 3.320578898854371, + "tokens_seen": 3002138624 + }, + { + "epoch": 0.82, + "learning_rate": 1.8223399133365432e-05, + "loss": 2.6431, + "theoretical_loss": 3.3205677541340073, + "tokens_seen": 3002269696 + }, + { + "epoch": 0.82, + "learning_rate": 1.8215374739207192e-05, + "loss": 2.3917, + "theoretical_loss": 3.3205566100364132, + "tokens_seen": 3002400768 + }, + { + "epoch": 0.82, + "learning_rate": 1.820735034504895e-05, + "loss": 2.4533, + "theoretical_loss": 3.320545466561526, + "tokens_seen": 3002531840 + }, + { + "epoch": 0.82, + "learning_rate": 1.819932595089071e-05, + "loss": 2.6189, + "theoretical_loss": 3.3205343237092846, + "tokens_seen": 3002662912 + }, + { + "epoch": 0.82, + "learning_rate": 1.8191301556732468e-05, + "loss": 2.346, + "theoretical_loss": 3.320523181479626, + "tokens_seen": 3002793984 + }, + { + "epoch": 0.82, + "learning_rate": 1.8183277162574228e-05, + "loss": 2.4582, + "theoretical_loss": 3.320512039872489, + "tokens_seen": 3002925056 + }, + { + "epoch": 0.82, + "learning_rate": 1.8175252768415985e-05, + "loss": 2.5696, + "theoretical_loss": 3.320500898887812, + "tokens_seen": 3003056128 + }, + { + "epoch": 0.82, + "learning_rate": 1.8167228374257745e-05, + "loss": 2.6059, + "theoretical_loss": 3.3204897585255324, + "tokens_seen": 3003187200 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1646835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.389035701751709, + "objective/train/theoretical_loss": 3.3204786187855886, + "objective/train/tokens_used": 1373777376, + "theoretical_loss": 3.3204786187855886, + "tokens_seen": 3003318272 + }, + { + "epoch": 0.82, + "learning_rate": 1.8159203980099505e-05, + "loss": 2.6538, + "theoretical_loss": 3.3204786187855886, + "tokens_seen": 3003318272 + }, + { + "epoch": 0.82, + "learning_rate": 1.8151179585941264e-05, + "loss": 2.4947, + "theoretical_loss": 3.320467479667918, + "tokens_seen": 3003449344 + }, + { + "epoch": 0.82, + "learning_rate": 1.814315519178302e-05, + "loss": 2.3192, + "theoretical_loss": 3.3204563411724597, + "tokens_seen": 3003580416 + }, + { + "epoch": 0.82, + "learning_rate": 1.813513079762478e-05, + "loss": 2.3717, + "theoretical_loss": 3.320445203299151, + "tokens_seen": 3003711488 + }, + { + "epoch": 0.82, + "learning_rate": 1.812710640346654e-05, + "loss": 2.6063, + "theoretical_loss": 3.3204340660479303, + "tokens_seen": 3003842560 + }, + { + "epoch": 0.82, + "learning_rate": 1.8119082009308297e-05, + "loss": 2.5886, + "theoretical_loss": 3.320422929418736, + "tokens_seen": 3003973632 + }, + { + "epoch": 0.82, + "learning_rate": 1.8111057615150057e-05, + "loss": 2.4073, + "theoretical_loss": 3.320411793411506, + "tokens_seen": 3004104704 + }, + { + "epoch": 0.82, + "learning_rate": 1.8103033220991817e-05, + "loss": 2.3654, + "theoretical_loss": 3.320400658026178, + "tokens_seen": 3004235776 + }, + { + "epoch": 0.82, + "learning_rate": 1.8095008826833577e-05, + "loss": 2.5072, + "theoretical_loss": 3.3203895232626905, + "tokens_seen": 3004366848 + }, + { + "epoch": 0.82, + "learning_rate": 1.8086984432675334e-05, + "loss": 2.6141, + "theoretical_loss": 3.3203783891209815, + "tokens_seen": 3004497920 + }, + { + "epoch": 0.82, + "learning_rate": 1.8078960038517093e-05, + "loss": 2.5234, + "theoretical_loss": 3.3203672556009898, + "tokens_seen": 3004628992 + }, + { + "epoch": 0.82, + "learning_rate": 1.8070935644358853e-05, + "loss": 2.4729, + "theoretical_loss": 3.320356122702653, + "tokens_seen": 3004760064 + }, + { + "epoch": 0.82, + "learning_rate": 1.8062911250200613e-05, + "loss": 2.4337, + "theoretical_loss": 3.320344990425909, + "tokens_seen": 3004891136 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1648109, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4846203327178955, + "objective/train/theoretical_loss": 3.3203394245206153, + "objective/train/tokens_used": 1375415776, + "theoretical_loss": 3.3203394245206153, + "tokens_seen": 3004956672 + }, + { + "epoch": 0.82, + "learning_rate": 1.805488685604237e-05, + "loss": 2.3613, + "theoretical_loss": 3.3203338587706965, + "tokens_seen": 3005022208 + }, + { + "epoch": 0.82, + "learning_rate": 1.804686246188413e-05, + "loss": 2.5654, + "theoretical_loss": 3.3203227277369534, + "tokens_seen": 3005153280 + }, + { + "epoch": 0.82, + "learning_rate": 1.803883806772589e-05, + "loss": 2.4953, + "theoretical_loss": 3.3203115973246184, + "tokens_seen": 3005284352 + }, + { + "epoch": 0.82, + "learning_rate": 1.8030813673567646e-05, + "loss": 2.4103, + "theoretical_loss": 3.320300467533629, + "tokens_seen": 3005415424 + }, + { + "epoch": 0.82, + "learning_rate": 1.8022789279409406e-05, + "loss": 2.4644, + "theoretical_loss": 3.320289338363924, + "tokens_seen": 3005546496 + }, + { + "epoch": 0.82, + "learning_rate": 1.8014764885251166e-05, + "loss": 2.5077, + "theoretical_loss": 3.320278209815441, + "tokens_seen": 3005677568 + }, + { + "epoch": 0.82, + "learning_rate": 1.8006740491092926e-05, + "loss": 2.7203, + "theoretical_loss": 3.320267081888119, + "tokens_seen": 3005808640 + }, + { + "epoch": 0.82, + "learning_rate": 1.7998716096934682e-05, + "loss": 2.461, + "theoretical_loss": 3.3202559545818957, + "tokens_seen": 3005939712 + }, + { + "epoch": 0.82, + "learning_rate": 1.7990691702776442e-05, + "loss": 2.5659, + "theoretical_loss": 3.320244827896709, + "tokens_seen": 3006070784 + }, + { + "epoch": 0.82, + "learning_rate": 1.7982667308618202e-05, + "loss": 2.3618, + "theoretical_loss": 3.3202337018324983, + "tokens_seen": 3006201856 + }, + { + "epoch": 0.82, + "learning_rate": 1.7974642914459962e-05, + "loss": 2.3227, + "theoretical_loss": 3.3202225763892015, + "tokens_seen": 3006332928 + }, + { + "epoch": 0.82, + "learning_rate": 1.796661852030172e-05, + "loss": 2.4426, + "theoretical_loss": 3.320211451566756, + "tokens_seen": 3006464000 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1648729, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6376848220825195, + "objective/train/theoretical_loss": 3.320200327365101, + "objective/train/tokens_used": 1377054176, + "theoretical_loss": 3.320200327365101, + "tokens_seen": 3006595072 + }, + { + "epoch": 0.82, + "learning_rate": 1.795859412614348e-05, + "loss": 2.5347, + "theoretical_loss": 3.320200327365101, + "tokens_seen": 3006595072 + }, + { + "epoch": 0.82, + "learning_rate": 1.795056973198524e-05, + "loss": 2.5554, + "theoretical_loss": 3.3201892037841745, + "tokens_seen": 3006726144 + }, + { + "epoch": 0.82, + "learning_rate": 1.7942545337826995e-05, + "loss": 2.6373, + "theoretical_loss": 3.3201780808239145, + "tokens_seen": 3006857216 + }, + { + "epoch": 0.82, + "learning_rate": 1.7934520943668755e-05, + "loss": 2.5802, + "theoretical_loss": 3.32016695848426, + "tokens_seen": 3006988288 + }, + { + "epoch": 0.82, + "learning_rate": 1.7926496549510515e-05, + "loss": 2.3462, + "theoretical_loss": 3.3201558367651485, + "tokens_seen": 3007119360 + }, + { + "epoch": 0.82, + "learning_rate": 1.791847215535227e-05, + "loss": 2.4609, + "theoretical_loss": 3.3201447156665194, + "tokens_seen": 3007250432 + }, + { + "epoch": 0.82, + "learning_rate": 1.791044776119403e-05, + "loss": 2.5349, + "theoretical_loss": 3.32013359518831, + "tokens_seen": 3007381504 + }, + { + "epoch": 0.82, + "learning_rate": 1.790242336703579e-05, + "loss": 2.4267, + "theoretical_loss": 3.320122475330459, + "tokens_seen": 3007512576 + }, + { + "epoch": 0.82, + "learning_rate": 1.7894398972877548e-05, + "loss": 2.5003, + "theoretical_loss": 3.320111356092905, + "tokens_seen": 3007643648 + }, + { + "epoch": 0.82, + "learning_rate": 1.7886374578719307e-05, + "loss": 2.4669, + "theoretical_loss": 3.320100237475587, + "tokens_seen": 3007774720 + }, + { + "epoch": 0.82, + "learning_rate": 1.7878350184561067e-05, + "loss": 2.4279, + "theoretical_loss": 3.320089119478441, + "tokens_seen": 3007905792 + }, + { + "epoch": 0.82, + "learning_rate": 1.7870325790402824e-05, + "loss": 2.5875, + "theoretical_loss": 3.3200780021014085, + "tokens_seen": 3008036864 + }, + { + "epoch": 0.82, + "learning_rate": 1.7862301396244584e-05, + "loss": 2.4346, + "theoretical_loss": 3.320066885344425, + "tokens_seen": 3008167936 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1649940, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1491355895996094, + "objective/train/theoretical_loss": 3.3200613271984336, + "objective/train/tokens_used": 1378692576, + "theoretical_loss": 3.3200613271984336, + "tokens_seen": 3008233472 + }, + { + "epoch": 0.82, + "learning_rate": 1.7854277002086344e-05, + "loss": 2.5467, + "theoretical_loss": 3.320055769207431, + "tokens_seen": 3008299008 + }, + { + "epoch": 0.82, + "learning_rate": 1.78462526079281e-05, + "loss": 2.4229, + "theoretical_loss": 3.3200446536903643, + "tokens_seen": 3008430080 + }, + { + "epoch": 0.82, + "learning_rate": 1.783822821376986e-05, + "loss": 2.4845, + "theoretical_loss": 3.320033538793163, + "tokens_seen": 3008561152 + }, + { + "epoch": 0.82, + "learning_rate": 1.783020381961162e-05, + "loss": 2.5522, + "theoretical_loss": 3.320022424515766, + "tokens_seen": 3008692224 + }, + { + "epoch": 0.82, + "learning_rate": 1.7822179425453377e-05, + "loss": 2.5308, + "theoretical_loss": 3.320011310858111, + "tokens_seen": 3008823296 + }, + { + "epoch": 0.82, + "learning_rate": 1.7814155031295136e-05, + "loss": 2.5673, + "theoretical_loss": 3.320000197820137, + "tokens_seen": 3008954368 + }, + { + "epoch": 0.82, + "learning_rate": 1.7806130637136896e-05, + "loss": 2.4331, + "theoretical_loss": 3.3199890854017826, + "tokens_seen": 3009085440 + }, + { + "epoch": 0.82, + "learning_rate": 1.7798106242978656e-05, + "loss": 2.4339, + "theoretical_loss": 3.319977973602986, + "tokens_seen": 3009216512 + }, + { + "epoch": 0.82, + "learning_rate": 1.7790081848820413e-05, + "loss": 2.5498, + "theoretical_loss": 3.3199668624236853, + "tokens_seen": 3009347584 + }, + { + "epoch": 0.82, + "learning_rate": 1.7782057454662173e-05, + "loss": 2.4248, + "theoretical_loss": 3.31995575186382, + "tokens_seen": 3009478656 + }, + { + "epoch": 0.82, + "learning_rate": 1.7774033060503933e-05, + "loss": 2.483, + "theoretical_loss": 3.3199446419233274, + "tokens_seen": 3009609728 + }, + { + "epoch": 0.82, + "learning_rate": 1.776600866634569e-05, + "loss": 2.4577, + "theoretical_loss": 3.319933532602147, + "tokens_seen": 3009740800 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1650677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.524388551712036, + "objective/train/theoretical_loss": 3.3199224239002167, + "objective/train/tokens_used": 1380330976, + "theoretical_loss": 3.3199224239002167, + "tokens_seen": 3009871872 + }, + { + "epoch": 0.82, + "learning_rate": 1.775798427218745e-05, + "loss": 2.5308, + "theoretical_loss": 3.3199224239002167, + "tokens_seen": 3009871872 + }, + { + "epoch": 0.82, + "learning_rate": 1.774995987802921e-05, + "loss": 2.4998, + "theoretical_loss": 3.3199113158174756, + "tokens_seen": 3010002944 + }, + { + "epoch": 0.82, + "learning_rate": 1.774193548387097e-05, + "loss": 2.5707, + "theoretical_loss": 3.3199002083538613, + "tokens_seen": 3010134016 + }, + { + "epoch": 0.82, + "learning_rate": 1.7733911089712725e-05, + "loss": 2.4654, + "theoretical_loss": 3.319889101509313, + "tokens_seen": 3010265088 + }, + { + "epoch": 0.82, + "learning_rate": 1.7725886695554485e-05, + "loss": 2.4882, + "theoretical_loss": 3.319877995283769, + "tokens_seen": 3010396160 + }, + { + "epoch": 0.82, + "learning_rate": 1.7717862301396245e-05, + "loss": 2.3004, + "theoretical_loss": 3.3198668896771686, + "tokens_seen": 3010527232 + }, + { + "epoch": 0.82, + "learning_rate": 1.7709837907238005e-05, + "loss": 2.4743, + "theoretical_loss": 3.319855784689449, + "tokens_seen": 3010658304 + }, + { + "epoch": 0.82, + "learning_rate": 1.770181351307976e-05, + "loss": 2.4036, + "theoretical_loss": 3.31984468032055, + "tokens_seen": 3010789376 + }, + { + "epoch": 0.82, + "learning_rate": 1.769378911892152e-05, + "loss": 2.3568, + "theoretical_loss": 3.3198335765704092, + "tokens_seen": 3010920448 + }, + { + "epoch": 0.82, + "learning_rate": 1.768576472476328e-05, + "loss": 2.481, + "theoretical_loss": 3.319822473438966, + "tokens_seen": 3011051520 + }, + { + "epoch": 0.82, + "learning_rate": 1.767774033060504e-05, + "loss": 2.6658, + "theoretical_loss": 3.3198113709261587, + "tokens_seen": 3011182592 + }, + { + "epoch": 0.83, + "learning_rate": 1.7669715936446798e-05, + "loss": 2.4001, + "theoretical_loss": 3.3198002690319255, + "tokens_seen": 3011313664 + }, + { + "epoch": 0.83, + "learning_rate": 1.7661691542288558e-05, + "loss": 2.3963, + "theoretical_loss": 3.319789167756206, + "tokens_seen": 3011444736 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1651905, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4369382858276367, + "objective/train/theoretical_loss": 3.319783617350269, + "objective/train/tokens_used": 1381969376, + "theoretical_loss": 3.319783617350269, + "tokens_seen": 3011510272 + }, + { + "epoch": 0.83, + "learning_rate": 1.7653667148130318e-05, + "loss": 2.6097, + "theoretical_loss": 3.3197780670989374, + "tokens_seen": 3011575808 + }, + { + "epoch": 0.83, + "learning_rate": 1.7645642753972074e-05, + "loss": 2.393, + "theoretical_loss": 3.31976696706006, + "tokens_seen": 3011706880 + }, + { + "epoch": 0.83, + "learning_rate": 1.7637618359813834e-05, + "loss": 2.306, + "theoretical_loss": 3.319755867639511, + "tokens_seen": 3011837952 + }, + { + "epoch": 0.83, + "learning_rate": 1.7629593965655594e-05, + "loss": 2.4724, + "theoretical_loss": 3.3197447688372295, + "tokens_seen": 3011969024 + }, + { + "epoch": 0.83, + "learning_rate": 1.7621569571497354e-05, + "loss": 2.4638, + "theoretical_loss": 3.3197336706531546, + "tokens_seen": 3012100096 + }, + { + "epoch": 0.83, + "learning_rate": 1.761354517733911e-05, + "loss": 2.667, + "theoretical_loss": 3.319722573087225, + "tokens_seen": 3012231168 + }, + { + "epoch": 0.83, + "learning_rate": 1.760552078318087e-05, + "loss": 2.4856, + "theoretical_loss": 3.3197114761393784, + "tokens_seen": 3012362240 + }, + { + "epoch": 0.83, + "learning_rate": 1.759749638902263e-05, + "loss": 2.494, + "theoretical_loss": 3.3197003798095546, + "tokens_seen": 3012493312 + }, + { + "epoch": 0.83, + "learning_rate": 1.758947199486439e-05, + "loss": 2.4326, + "theoretical_loss": 3.3196892840976915, + "tokens_seen": 3012624384 + }, + { + "epoch": 0.83, + "learning_rate": 1.7581447600706147e-05, + "loss": 2.6103, + "theoretical_loss": 3.319678189003728, + "tokens_seen": 3012755456 + }, + { + "epoch": 0.83, + "learning_rate": 1.7573423206547907e-05, + "loss": 2.5432, + "theoretical_loss": 3.319667094527603, + "tokens_seen": 3012886528 + }, + { + "epoch": 0.83, + "learning_rate": 1.7565398812389666e-05, + "loss": 2.614, + "theoretical_loss": 3.319656000669255, + "tokens_seen": 3013017600 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1652271, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.725615978240967, + "objective/train/theoretical_loss": 3.319644907428623, + "objective/train/tokens_used": 1383607776, + "theoretical_loss": 3.319644907428623, + "tokens_seen": 3013148672 + }, + { + "epoch": 0.83, + "learning_rate": 1.7557374418231423e-05, + "loss": 2.4089, + "theoretical_loss": 3.319644907428623, + "tokens_seen": 3013148672 + }, + { + "epoch": 0.83, + "learning_rate": 1.7549350024073183e-05, + "loss": 2.4162, + "theoretical_loss": 3.3196338148056457, + "tokens_seen": 3013279744 + }, + { + "epoch": 0.83, + "learning_rate": 1.7541325629914943e-05, + "loss": 2.4537, + "theoretical_loss": 3.3196227228002617, + "tokens_seen": 3013410816 + }, + { + "epoch": 0.83, + "learning_rate": 1.7533301235756703e-05, + "loss": 2.4924, + "theoretical_loss": 3.31961163141241, + "tokens_seen": 3013541888 + }, + { + "epoch": 0.83, + "learning_rate": 1.752527684159846e-05, + "loss": 2.469, + "theoretical_loss": 3.319600540642029, + "tokens_seen": 3013672960 + }, + { + "epoch": 0.83, + "learning_rate": 1.751725244744022e-05, + "loss": 2.5548, + "theoretical_loss": 3.3195894504890573, + "tokens_seen": 3013804032 + }, + { + "epoch": 0.83, + "learning_rate": 1.750922805328198e-05, + "loss": 2.6446, + "theoretical_loss": 3.3195783609534346, + "tokens_seen": 3013935104 + }, + { + "epoch": 0.83, + "learning_rate": 1.750120365912374e-05, + "loss": 2.6265, + "theoretical_loss": 3.3195672720350986, + "tokens_seen": 3014066176 + }, + { + "epoch": 0.83, + "learning_rate": 1.7493179264965495e-05, + "loss": 2.4619, + "theoretical_loss": 3.319556183733989, + "tokens_seen": 3014197248 + }, + { + "epoch": 0.83, + "learning_rate": 1.7485154870807255e-05, + "loss": 2.4752, + "theoretical_loss": 3.3195450960500437, + "tokens_seen": 3014328320 + }, + { + "epoch": 0.83, + "learning_rate": 1.7477130476649015e-05, + "loss": 2.4024, + "theoretical_loss": 3.3195340089832026, + "tokens_seen": 3014459392 + }, + { + "epoch": 0.83, + "learning_rate": 1.7469106082490772e-05, + "loss": 2.6356, + "theoretical_loss": 3.3195229225334035, + "tokens_seen": 3014590464 + }, + { + "epoch": 0.83, + "learning_rate": 1.746108168833253e-05, + "loss": 2.5769, + "theoretical_loss": 3.3195118367005856, + "tokens_seen": 3014721536 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1653310, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4980628490448, + "objective/train/theoretical_loss": 3.3195062940155258, + "objective/train/tokens_used": 1385246176, + "theoretical_loss": 3.3195062940155258, + "tokens_seen": 3014787072 + }, + { + "epoch": 0.83, + "learning_rate": 1.745305729417429e-05, + "loss": 2.4191, + "theoretical_loss": 3.3195007514846884, + "tokens_seen": 3014852608 + }, + { + "epoch": 0.83, + "learning_rate": 1.744503290001605e-05, + "loss": 2.5988, + "theoretical_loss": 3.3194896668856497, + "tokens_seen": 3014983680 + }, + { + "epoch": 0.83, + "learning_rate": 1.7437008505857808e-05, + "loss": 2.5424, + "theoretical_loss": 3.319478582903409, + "tokens_seen": 3015114752 + }, + { + "epoch": 0.83, + "learning_rate": 1.7428984111699568e-05, + "loss": 2.4683, + "theoretical_loss": 3.3194674995379048, + "tokens_seen": 3015245824 + }, + { + "epoch": 0.83, + "learning_rate": 1.7420959717541328e-05, + "loss": 2.5158, + "theoretical_loss": 3.3194564167890763, + "tokens_seen": 3015376896 + }, + { + "epoch": 0.83, + "learning_rate": 1.7412935323383088e-05, + "loss": 2.6395, + "theoretical_loss": 3.319445334656862, + "tokens_seen": 3015507968 + }, + { + "epoch": 0.83, + "learning_rate": 1.7404910929224844e-05, + "loss": 2.5144, + "theoretical_loss": 3.3194342531412016, + "tokens_seen": 3015639040 + }, + { + "epoch": 0.83, + "learning_rate": 1.7396886535066604e-05, + "loss": 2.4719, + "theoretical_loss": 3.319423172242033, + "tokens_seen": 3015770112 + }, + { + "epoch": 0.83, + "learning_rate": 1.7388862140908364e-05, + "loss": 2.5879, + "theoretical_loss": 3.319412091959296, + "tokens_seen": 3015901184 + }, + { + "epoch": 0.83, + "learning_rate": 1.738083774675012e-05, + "loss": 2.2957, + "theoretical_loss": 3.319401012292929, + "tokens_seen": 3016032256 + }, + { + "epoch": 0.83, + "learning_rate": 1.737281335259188e-05, + "loss": 2.5007, + "theoretical_loss": 3.3193899332428707, + "tokens_seen": 3016163328 + }, + { + "epoch": 0.83, + "learning_rate": 1.736478895843364e-05, + "loss": 2.4641, + "theoretical_loss": 3.3193788548090604, + "tokens_seen": 3016294400 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1654644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.614122152328491, + "objective/train/theoretical_loss": 3.319367776991437, + "objective/train/tokens_used": 1386884576, + "theoretical_loss": 3.319367776991437, + "tokens_seen": 3016425472 + }, + { + "epoch": 0.83, + "learning_rate": 1.73567645642754e-05, + "loss": 2.5554, + "theoretical_loss": 3.319367776991437, + "tokens_seen": 3016425472 + }, + { + "epoch": 0.83, + "learning_rate": 1.7348740170117157e-05, + "loss": 2.4762, + "theoretical_loss": 3.3193566997899397, + "tokens_seen": 3016556544 + }, + { + "epoch": 0.83, + "learning_rate": 1.7340715775958917e-05, + "loss": 2.4625, + "theoretical_loss": 3.319345623204507, + "tokens_seen": 3016687616 + }, + { + "epoch": 0.83, + "learning_rate": 1.7332691381800677e-05, + "loss": 2.5713, + "theoretical_loss": 3.319334547235078, + "tokens_seen": 3016818688 + }, + { + "epoch": 0.83, + "learning_rate": 1.7324666987642436e-05, + "loss": 2.5408, + "theoretical_loss": 3.319323471881592, + "tokens_seen": 3016949760 + }, + { + "epoch": 0.83, + "learning_rate": 1.7316642593484193e-05, + "loss": 2.4578, + "theoretical_loss": 3.3193123971439875, + "tokens_seen": 3017080832 + }, + { + "epoch": 0.83, + "learning_rate": 1.7308618199325953e-05, + "loss": 2.533, + "theoretical_loss": 3.319301323022204, + "tokens_seen": 3017211904 + }, + { + "epoch": 0.83, + "learning_rate": 1.7300593805167713e-05, + "loss": 2.5082, + "theoretical_loss": 3.3192902495161802, + "tokens_seen": 3017342976 + }, + { + "epoch": 0.83, + "learning_rate": 1.729256941100947e-05, + "loss": 2.4146, + "theoretical_loss": 3.319279176625855, + "tokens_seen": 3017474048 + }, + { + "epoch": 0.83, + "learning_rate": 1.728454501685123e-05, + "loss": 2.4658, + "theoretical_loss": 3.319268104351168, + "tokens_seen": 3017605120 + }, + { + "epoch": 0.83, + "learning_rate": 1.727652062269299e-05, + "loss": 2.6769, + "theoretical_loss": 3.3192570326920574, + "tokens_seen": 3017736192 + }, + { + "epoch": 0.83, + "learning_rate": 1.7268496228534746e-05, + "loss": 2.4696, + "theoretical_loss": 3.3192459616484626, + "tokens_seen": 3017867264 + }, + { + "epoch": 0.83, + "learning_rate": 1.7260471834376506e-05, + "loss": 2.5972, + "theoretical_loss": 3.319234891220323, + "tokens_seen": 3017998336 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1655335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3214454650878906, + "objective/train/theoretical_loss": 3.31922935623703, + "objective/train/tokens_used": 1388522976, + "theoretical_loss": 3.31922935623703, + "tokens_seen": 3018063872 + }, + { + "epoch": 0.83, + "learning_rate": 1.7252447440218265e-05, + "loss": 2.5928, + "theoretical_loss": 3.3192238214075775, + "tokens_seen": 3018129408 + }, + { + "epoch": 0.83, + "learning_rate": 1.7244423046060022e-05, + "loss": 2.6309, + "theoretical_loss": 3.319212752210165, + "tokens_seen": 3018260480 + }, + { + "epoch": 0.83, + "learning_rate": 1.7236398651901782e-05, + "loss": 2.3643, + "theoretical_loss": 3.3192016836280245, + "tokens_seen": 3018391552 + }, + { + "epoch": 0.83, + "learning_rate": 1.7228374257743542e-05, + "loss": 2.5669, + "theoretical_loss": 3.3191906156610953, + "tokens_seen": 3018522624 + }, + { + "epoch": 0.83, + "learning_rate": 1.72203498635853e-05, + "loss": 2.4952, + "theoretical_loss": 3.3191795483093163, + "tokens_seen": 3018653696 + }, + { + "epoch": 0.83, + "learning_rate": 1.7212325469427058e-05, + "loss": 2.4728, + "theoretical_loss": 3.3191684815726266, + "tokens_seen": 3018784768 + }, + { + "epoch": 0.83, + "learning_rate": 1.7204301075268818e-05, + "loss": 2.7568, + "theoretical_loss": 3.3191574154509658, + "tokens_seen": 3018915840 + }, + { + "epoch": 0.83, + "learning_rate": 1.7196276681110575e-05, + "loss": 2.4808, + "theoretical_loss": 3.319146349944272, + "tokens_seen": 3019046912 + }, + { + "epoch": 0.83, + "learning_rate": 1.7188252286952335e-05, + "loss": 2.4977, + "theoretical_loss": 3.319135285052486, + "tokens_seen": 3019177984 + }, + { + "epoch": 0.83, + "learning_rate": 1.7180227892794094e-05, + "loss": 2.4989, + "theoretical_loss": 3.319124220775545, + "tokens_seen": 3019309056 + }, + { + "epoch": 0.83, + "learning_rate": 1.717220349863585e-05, + "loss": 2.5469, + "theoretical_loss": 3.319113157113389, + "tokens_seen": 3019440128 + }, + { + "epoch": 0.83, + "learning_rate": 1.716417910447761e-05, + "loss": 2.5285, + "theoretical_loss": 3.3191020940659577, + "tokens_seen": 3019571200 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1656670, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3002569675445557, + "objective/train/theoretical_loss": 3.3190910316331896, + "objective/train/tokens_used": 1390161376, + "theoretical_loss": 3.3190910316331896, + "tokens_seen": 3019702272 + }, + { + "epoch": 0.83, + "learning_rate": 1.715615471031937e-05, + "loss": 2.3861, + "theoretical_loss": 3.3190910316331896, + "tokens_seen": 3019702272 + }, + { + "epoch": 0.83, + "learning_rate": 1.714813031616113e-05, + "loss": 2.6544, + "theoretical_loss": 3.319079969815024, + "tokens_seen": 3019833344 + }, + { + "epoch": 0.83, + "learning_rate": 1.7140105922002887e-05, + "loss": 2.3871, + "theoretical_loss": 3.3190689086114005, + "tokens_seen": 3019964416 + }, + { + "epoch": 0.83, + "learning_rate": 1.7132081527844647e-05, + "loss": 2.3084, + "theoretical_loss": 3.3190578480222577, + "tokens_seen": 3020095488 + }, + { + "epoch": 0.83, + "learning_rate": 1.7124057133686407e-05, + "loss": 2.4189, + "theoretical_loss": 3.3190467880475345, + "tokens_seen": 3020226560 + }, + { + "epoch": 0.83, + "learning_rate": 1.7116032739528167e-05, + "loss": 2.4361, + "theoretical_loss": 3.319035728687171, + "tokens_seen": 3020357632 + }, + { + "epoch": 0.83, + "learning_rate": 1.7108008345369923e-05, + "loss": 2.4857, + "theoretical_loss": 3.319024669941106, + "tokens_seen": 3020488704 + }, + { + "epoch": 0.83, + "learning_rate": 1.7099983951211683e-05, + "loss": 2.3905, + "theoretical_loss": 3.319013611809279, + "tokens_seen": 3020619776 + }, + { + "epoch": 0.83, + "learning_rate": 1.7091959557053443e-05, + "loss": 2.561, + "theoretical_loss": 3.319002554291629, + "tokens_seen": 3020750848 + }, + { + "epoch": 0.83, + "learning_rate": 1.70839351628952e-05, + "loss": 2.3982, + "theoretical_loss": 3.3189914973880947, + "tokens_seen": 3020881920 + }, + { + "epoch": 0.83, + "learning_rate": 1.707591076873696e-05, + "loss": 2.524, + "theoretical_loss": 3.318980441098616, + "tokens_seen": 3021012992 + }, + { + "epoch": 0.83, + "learning_rate": 1.706788637457872e-05, + "loss": 2.467, + "theoretical_loss": 3.3189693854231326, + "tokens_seen": 3021144064 + }, + { + "epoch": 0.83, + "learning_rate": 1.705986198042048e-05, + "loss": 2.5579, + "theoretical_loss": 3.3189583303615824, + "tokens_seen": 3021275136 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1657192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6645877361297607, + "objective/train/theoretical_loss": 3.3189528030610136, + "objective/train/tokens_used": 1391799776, + "theoretical_loss": 3.3189528030610136, + "tokens_seen": 3021340672 + }, + { + "epoch": 0.83, + "learning_rate": 1.7051837586262236e-05, + "loss": 2.5696, + "theoretical_loss": 3.318947275913906, + "tokens_seen": 3021406208 + }, + { + "epoch": 0.83, + "learning_rate": 1.7043813192103996e-05, + "loss": 2.5559, + "theoretical_loss": 3.318936222080042, + "tokens_seen": 3021537280 + }, + { + "epoch": 0.83, + "learning_rate": 1.7035788797945756e-05, + "loss": 2.499, + "theoretical_loss": 3.3189251688599297, + "tokens_seen": 3021668352 + }, + { + "epoch": 0.83, + "learning_rate": 1.7027764403787516e-05, + "loss": 2.5007, + "theoretical_loss": 3.318914116253509, + "tokens_seen": 3021799424 + }, + { + "epoch": 0.83, + "learning_rate": 1.7019740009629272e-05, + "loss": 2.4537, + "theoretical_loss": 3.3189030642607182, + "tokens_seen": 3021930496 + }, + { + "epoch": 0.83, + "learning_rate": 1.7011715615471032e-05, + "loss": 2.4056, + "theoretical_loss": 3.3188920128814976, + "tokens_seen": 3022061568 + }, + { + "epoch": 0.83, + "learning_rate": 1.7003691221312792e-05, + "loss": 2.6592, + "theoretical_loss": 3.3188809621157858, + "tokens_seen": 3022192640 + }, + { + "epoch": 0.83, + "learning_rate": 1.699566682715455e-05, + "loss": 2.526, + "theoretical_loss": 3.3188699119635228, + "tokens_seen": 3022323712 + }, + { + "epoch": 0.83, + "learning_rate": 1.698764243299631e-05, + "loss": 2.4016, + "theoretical_loss": 3.3188588624246473, + "tokens_seen": 3022454784 + }, + { + "epoch": 0.83, + "learning_rate": 1.697961803883807e-05, + "loss": 2.5709, + "theoretical_loss": 3.318847813499099, + "tokens_seen": 3022585856 + }, + { + "epoch": 0.83, + "learning_rate": 1.697159364467983e-05, + "loss": 2.4417, + "theoretical_loss": 3.318836765186817, + "tokens_seen": 3022716928 + }, + { + "epoch": 0.83, + "learning_rate": 1.6963569250521585e-05, + "loss": 2.4194, + "theoretical_loss": 3.3188257174877416, + "tokens_seen": 3022848000 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1658380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.086480140686035, + "objective/train/theoretical_loss": 3.3188146704018107, + "objective/train/tokens_used": 1393438176, + "theoretical_loss": 3.3188146704018107, + "tokens_seen": 3022979072 + }, + { + "epoch": 0.83, + "learning_rate": 1.6955544856363345e-05, + "loss": 2.3074, + "theoretical_loss": 3.3188146704018107, + "tokens_seen": 3022979072 + }, + { + "epoch": 0.83, + "learning_rate": 1.6947520462205105e-05, + "loss": 2.4012, + "theoretical_loss": 3.318803623928965, + "tokens_seen": 3023110144 + }, + { + "epoch": 0.83, + "learning_rate": 1.6939496068046865e-05, + "loss": 2.35, + "theoretical_loss": 3.318792578069143, + "tokens_seen": 3023241216 + }, + { + "epoch": 0.83, + "learning_rate": 1.693147167388862e-05, + "loss": 2.4078, + "theoretical_loss": 3.3187815328222845, + "tokens_seen": 3023372288 + }, + { + "epoch": 0.83, + "learning_rate": 1.692344727973038e-05, + "loss": 2.5395, + "theoretical_loss": 3.3187704881883286, + "tokens_seen": 3023503360 + }, + { + "epoch": 0.83, + "learning_rate": 1.691542288557214e-05, + "loss": 2.4872, + "theoretical_loss": 3.3187594441672155, + "tokens_seen": 3023634432 + }, + { + "epoch": 0.83, + "learning_rate": 1.6907398491413897e-05, + "loss": 2.4905, + "theoretical_loss": 3.3187484007588837, + "tokens_seen": 3023765504 + }, + { + "epoch": 0.83, + "learning_rate": 1.6899374097255657e-05, + "loss": 2.4156, + "theoretical_loss": 3.3187373579632733, + "tokens_seen": 3023896576 + }, + { + "epoch": 0.83, + "learning_rate": 1.6891349703097417e-05, + "loss": 2.536, + "theoretical_loss": 3.3187263157803235, + "tokens_seen": 3024027648 + }, + { + "epoch": 0.83, + "learning_rate": 1.6883325308939177e-05, + "loss": 2.4519, + "theoretical_loss": 3.3187152742099735, + "tokens_seen": 3024158720 + }, + { + "epoch": 0.83, + "learning_rate": 1.6875300914780934e-05, + "loss": 2.4461, + "theoretical_loss": 3.3187042332521632, + "tokens_seen": 3024289792 + }, + { + "epoch": 0.83, + "learning_rate": 1.6867276520622694e-05, + "loss": 2.3906, + "theoretical_loss": 3.318693192906832, + "tokens_seen": 3024420864 + }, + { + "epoch": 0.83, + "learning_rate": 1.6859252126464453e-05, + "loss": 2.6269, + "theoretical_loss": 3.318682153173919, + "tokens_seen": 3024551936 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1658975, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6726958751678467, + "objective/train/theoretical_loss": 3.3186766335371005, + "objective/train/tokens_used": 1395076576, + "theoretical_loss": 3.3186766335371005, + "tokens_seen": 3024617472 + }, + { + "epoch": 0.83, + "learning_rate": 1.6851227732306213e-05, + "loss": 2.493, + "theoretical_loss": 3.3186711140533642, + "tokens_seen": 3024683008 + }, + { + "epoch": 0.83, + "learning_rate": 1.684320333814797e-05, + "loss": 2.4229, + "theoretical_loss": 3.3186600755451066, + "tokens_seen": 3024814080 + }, + { + "epoch": 0.83, + "learning_rate": 1.683517894398973e-05, + "loss": 2.408, + "theoretical_loss": 3.3186490376490863, + "tokens_seen": 3024945152 + }, + { + "epoch": 0.83, + "learning_rate": 1.682715454983149e-05, + "loss": 2.437, + "theoretical_loss": 3.318638000365242, + "tokens_seen": 3025076224 + }, + { + "epoch": 0.83, + "learning_rate": 1.6819130155673246e-05, + "loss": 2.4496, + "theoretical_loss": 3.318626963693514, + "tokens_seen": 3025207296 + }, + { + "epoch": 0.83, + "learning_rate": 1.6811105761515006e-05, + "loss": 2.3594, + "theoretical_loss": 3.3186159276338416, + "tokens_seen": 3025338368 + }, + { + "epoch": 0.83, + "learning_rate": 1.6803081367356766e-05, + "loss": 2.5156, + "theoretical_loss": 3.3186048921861646, + "tokens_seen": 3025469440 + }, + { + "epoch": 0.83, + "learning_rate": 1.6795056973198526e-05, + "loss": 2.6181, + "theoretical_loss": 3.3185938573504217, + "tokens_seen": 3025600512 + }, + { + "epoch": 0.83, + "learning_rate": 1.6787032579040282e-05, + "loss": 2.2243, + "theoretical_loss": 3.3185828231265533, + "tokens_seen": 3025731584 + }, + { + "epoch": 0.83, + "learning_rate": 1.6779008184882042e-05, + "loss": 2.4931, + "theoretical_loss": 3.3185717895144986, + "tokens_seen": 3025862656 + }, + { + "epoch": 0.83, + "learning_rate": 1.6770983790723802e-05, + "loss": 2.376, + "theoretical_loss": 3.3185607565141972, + "tokens_seen": 3025993728 + }, + { + "epoch": 0.83, + "learning_rate": 1.6762959396565562e-05, + "loss": 2.5968, + "theoretical_loss": 3.318549724125589, + "tokens_seen": 3026124800 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1660130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.368717670440674, + "objective/train/theoretical_loss": 3.318538692348613, + "objective/train/tokens_used": 1396714976, + "theoretical_loss": 3.318538692348613, + "tokens_seen": 3026255872 + }, + { + "epoch": 0.83, + "learning_rate": 1.675493500240732e-05, + "loss": 2.4947, + "theoretical_loss": 3.318538692348613, + "tokens_seen": 3026255872 + }, + { + "epoch": 0.83, + "learning_rate": 1.674691060824908e-05, + "loss": 2.458, + "theoretical_loss": 3.318527661183209, + "tokens_seen": 3026386944 + }, + { + "epoch": 0.83, + "learning_rate": 1.673888621409084e-05, + "loss": 2.4933, + "theoretical_loss": 3.318516630629317, + "tokens_seen": 3026518016 + }, + { + "epoch": 0.83, + "learning_rate": 1.67308618199326e-05, + "loss": 2.466, + "theoretical_loss": 3.3185056006868763, + "tokens_seen": 3026649088 + }, + { + "epoch": 0.83, + "learning_rate": 1.6722837425774355e-05, + "loss": 2.5016, + "theoretical_loss": 3.318494571355827, + "tokens_seen": 3026780160 + }, + { + "epoch": 0.83, + "learning_rate": 1.6714813031616115e-05, + "loss": 2.465, + "theoretical_loss": 3.3184835426361077, + "tokens_seen": 3026911232 + }, + { + "epoch": 0.83, + "learning_rate": 1.6706788637457875e-05, + "loss": 2.4581, + "theoretical_loss": 3.318472514527659, + "tokens_seen": 3027042304 + }, + { + "epoch": 0.83, + "learning_rate": 1.669876424329963e-05, + "loss": 2.4784, + "theoretical_loss": 3.31846148703042, + "tokens_seen": 3027173376 + }, + { + "epoch": 0.83, + "learning_rate": 1.669073984914139e-05, + "loss": 2.254, + "theoretical_loss": 3.3184504601443305, + "tokens_seen": 3027304448 + }, + { + "epoch": 0.83, + "learning_rate": 1.668271545498315e-05, + "loss": 2.4988, + "theoretical_loss": 3.318439433869331, + "tokens_seen": 3027435520 + }, + { + "epoch": 0.83, + "learning_rate": 1.667469106082491e-05, + "loss": 2.3611, + "theoretical_loss": 3.3184284082053597, + "tokens_seen": 3027566592 + }, + { + "epoch": 0.84, + "learning_rate": 1.6666666666666667e-05, + "loss": 2.6435, + "theoretical_loss": 3.318417383152357, + "tokens_seen": 3027697664 + }, + { + "epoch": 0.84, + "learning_rate": 1.6658642272508427e-05, + "loss": 2.5639, + "theoretical_loss": 3.3184063587102632, + "tokens_seen": 3027828736 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1660772, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.246469020843506, + "objective/train/theoretical_loss": 3.318400846718288, + "objective/train/tokens_used": 1398353376, + "theoretical_loss": 3.318400846718288, + "tokens_seen": 3027894272 + }, + { + "epoch": 0.84, + "learning_rate": 1.6650617878350187e-05, + "loss": 2.3443, + "theoretical_loss": 3.318395334879017, + "tokens_seen": 3027959808 + }, + { + "epoch": 0.84, + "learning_rate": 1.6642593484191947e-05, + "loss": 2.4742, + "theoretical_loss": 3.3183843116585585, + "tokens_seen": 3028090880 + }, + { + "epoch": 0.84, + "learning_rate": 1.6634569090033704e-05, + "loss": 2.5776, + "theoretical_loss": 3.318373289048828, + "tokens_seen": 3028221952 + }, + { + "epoch": 0.84, + "learning_rate": 1.6626544695875464e-05, + "loss": 2.4474, + "theoretical_loss": 3.318362267049764, + "tokens_seen": 3028353024 + }, + { + "epoch": 0.84, + "learning_rate": 1.6618520301717224e-05, + "loss": 2.3702, + "theoretical_loss": 3.318351245661307, + "tokens_seen": 3028484096 + }, + { + "epoch": 0.84, + "learning_rate": 1.661049590755898e-05, + "loss": 2.3199, + "theoretical_loss": 3.3183402248833973, + "tokens_seen": 3028615168 + }, + { + "epoch": 0.84, + "learning_rate": 1.660247151340074e-05, + "loss": 2.5319, + "theoretical_loss": 3.318329204715974, + "tokens_seen": 3028746240 + }, + { + "epoch": 0.84, + "learning_rate": 1.65944471192425e-05, + "loss": 2.5275, + "theoretical_loss": 3.3183181851589763, + "tokens_seen": 3028877312 + }, + { + "epoch": 0.84, + "learning_rate": 1.6586422725084256e-05, + "loss": 2.4632, + "theoretical_loss": 3.318307166212345, + "tokens_seen": 3029008384 + }, + { + "epoch": 0.84, + "learning_rate": 1.6578398330926016e-05, + "loss": 2.567, + "theoretical_loss": 3.318296147876019, + "tokens_seen": 3029139456 + }, + { + "epoch": 0.84, + "learning_rate": 1.6570373936767776e-05, + "loss": 2.5145, + "theoretical_loss": 3.318285130149939, + "tokens_seen": 3029270528 + }, + { + "epoch": 0.84, + "learning_rate": 1.6562349542609533e-05, + "loss": 2.4646, + "theoretical_loss": 3.3182741130340445, + "tokens_seen": 3029401600 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1662079, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.748887538909912, + "objective/train/theoretical_loss": 3.318263096528275, + "objective/train/tokens_used": 1399991776, + "theoretical_loss": 3.318263096528275, + "tokens_seen": 3029532672 + }, + { + "epoch": 0.84, + "learning_rate": 1.6554325148451293e-05, + "loss": 2.4014, + "theoretical_loss": 3.318263096528275, + "tokens_seen": 3029532672 + }, + { + "epoch": 0.84, + "learning_rate": 1.6546300754293053e-05, + "loss": 2.5034, + "theoretical_loss": 3.31825208063257, + "tokens_seen": 3029663744 + }, + { + "epoch": 0.84, + "learning_rate": 1.653827636013481e-05, + "loss": 2.25, + "theoretical_loss": 3.3182410653468706, + "tokens_seen": 3029794816 + }, + { + "epoch": 0.84, + "learning_rate": 1.653025196597657e-05, + "loss": 2.4943, + "theoretical_loss": 3.3182300506711155, + "tokens_seen": 3029925888 + }, + { + "epoch": 0.84, + "learning_rate": 1.652222757181833e-05, + "loss": 2.5707, + "theoretical_loss": 3.318219036605245, + "tokens_seen": 3030056960 + }, + { + "epoch": 0.84, + "learning_rate": 1.6514203177660085e-05, + "loss": 2.5159, + "theoretical_loss": 3.3182080231491984, + "tokens_seen": 3030188032 + }, + { + "epoch": 0.84, + "learning_rate": 1.6506178783501845e-05, + "loss": 2.3304, + "theoretical_loss": 3.3181970103029164, + "tokens_seen": 3030319104 + }, + { + "epoch": 0.84, + "learning_rate": 1.6498154389343605e-05, + "loss": 2.4665, + "theoretical_loss": 3.3181859980663386, + "tokens_seen": 3030450176 + }, + { + "epoch": 0.84, + "learning_rate": 1.649012999518536e-05, + "loss": 2.3757, + "theoretical_loss": 3.3181749864394043, + "tokens_seen": 3030581248 + }, + { + "epoch": 0.84, + "learning_rate": 1.648210560102712e-05, + "loss": 2.3745, + "theoretical_loss": 3.318163975422054, + "tokens_seen": 3030712320 + }, + { + "epoch": 0.84, + "learning_rate": 1.647408120686888e-05, + "loss": 2.4452, + "theoretical_loss": 3.318152965014227, + "tokens_seen": 3030843392 + }, + { + "epoch": 0.84, + "learning_rate": 1.646605681271064e-05, + "loss": 2.4241, + "theoretical_loss": 3.3181419552158644, + "tokens_seen": 3030974464 + }, + { + "epoch": 0.84, + "learning_rate": 1.6458032418552398e-05, + "loss": 2.3636, + "theoretical_loss": 3.318130946026905, + "tokens_seen": 3031105536 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1662642, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4066145420074463, + "objective/train/theoretical_loss": 3.318125441660933, + "objective/train/tokens_used": 1401630176, + "theoretical_loss": 3.318125441660933, + "tokens_seen": 3031171072 + }, + { + "epoch": 0.84, + "learning_rate": 1.6450008024394158e-05, + "loss": 2.4663, + "theoretical_loss": 3.318119937447289, + "tokens_seen": 3031236608 + }, + { + "epoch": 0.84, + "learning_rate": 1.6441983630235918e-05, + "loss": 2.5431, + "theoretical_loss": 3.3181089294769563, + "tokens_seen": 3031367680 + }, + { + "epoch": 0.84, + "learning_rate": 1.6433959236077674e-05, + "loss": 2.4624, + "theoretical_loss": 3.318097922115847, + "tokens_seen": 3031498752 + }, + { + "epoch": 0.84, + "learning_rate": 1.6425934841919434e-05, + "loss": 2.5936, + "theoretical_loss": 3.318086915363901, + "tokens_seen": 3031629824 + }, + { + "epoch": 0.84, + "learning_rate": 1.6417910447761194e-05, + "loss": 2.3127, + "theoretical_loss": 3.318075909221058, + "tokens_seen": 3031760896 + }, + { + "epoch": 0.84, + "learning_rate": 1.6409886053602954e-05, + "loss": 2.4347, + "theoretical_loss": 3.3180649036872585, + "tokens_seen": 3031891968 + }, + { + "epoch": 0.84, + "learning_rate": 1.640186165944471e-05, + "loss": 2.4715, + "theoretical_loss": 3.3180538987624417, + "tokens_seen": 3032023040 + }, + { + "epoch": 0.84, + "learning_rate": 1.639383726528647e-05, + "loss": 2.6148, + "theoretical_loss": 3.3180428944465485, + "tokens_seen": 3032154112 + }, + { + "epoch": 0.84, + "learning_rate": 1.638581287112823e-05, + "loss": 2.5188, + "theoretical_loss": 3.318031890739518, + "tokens_seen": 3032285184 + }, + { + "epoch": 0.84, + "learning_rate": 1.637778847696999e-05, + "loss": 2.3767, + "theoretical_loss": 3.318020887641291, + "tokens_seen": 3032416256 + }, + { + "epoch": 0.84, + "learning_rate": 1.6369764082811747e-05, + "loss": 2.5238, + "theoretical_loss": 3.318009885151807, + "tokens_seen": 3032547328 + }, + { + "epoch": 0.84, + "learning_rate": 1.6361739688653507e-05, + "loss": 2.4487, + "theoretical_loss": 3.317998883271006, + "tokens_seen": 3032678400 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1663221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6590020656585693, + "objective/train/theoretical_loss": 3.317987881998828, + "objective/train/tokens_used": 1403268576, + "theoretical_loss": 3.317987881998828, + "tokens_seen": 3032809472 + }, + { + "epoch": 0.84, + "learning_rate": 1.6353715294495267e-05, + "loss": 2.4898, + "theoretical_loss": 3.317987881998828, + "tokens_seen": 3032809472 + }, + { + "epoch": 0.84, + "learning_rate": 1.6345690900337023e-05, + "loss": 2.58, + "theoretical_loss": 3.3179768813352135, + "tokens_seen": 3032940544 + }, + { + "epoch": 0.84, + "learning_rate": 1.6337666506178783e-05, + "loss": 2.4908, + "theoretical_loss": 3.317965881280102, + "tokens_seen": 3033071616 + }, + { + "epoch": 0.84, + "learning_rate": 1.6329642112020543e-05, + "loss": 2.61, + "theoretical_loss": 3.3179548818334337, + "tokens_seen": 3033202688 + }, + { + "epoch": 0.84, + "learning_rate": 1.6321617717862303e-05, + "loss": 2.4658, + "theoretical_loss": 3.3179438829951486, + "tokens_seen": 3033333760 + }, + { + "epoch": 0.84, + "learning_rate": 1.631359332370406e-05, + "loss": 2.399, + "theoretical_loss": 3.317932884765187, + "tokens_seen": 3033464832 + }, + { + "epoch": 0.84, + "learning_rate": 1.630556892954582e-05, + "loss": 2.5633, + "theoretical_loss": 3.317921887143489, + "tokens_seen": 3033595904 + }, + { + "epoch": 0.84, + "learning_rate": 1.629754453538758e-05, + "loss": 2.5227, + "theoretical_loss": 3.3179108901299945, + "tokens_seen": 3033726976 + }, + { + "epoch": 0.84, + "learning_rate": 1.628952014122934e-05, + "loss": 2.4982, + "theoretical_loss": 3.317899893724644, + "tokens_seen": 3033858048 + }, + { + "epoch": 0.84, + "learning_rate": 1.6281495747071096e-05, + "loss": 2.4184, + "theoretical_loss": 3.3178888979273764, + "tokens_seen": 3033989120 + }, + { + "epoch": 0.84, + "learning_rate": 1.6273471352912855e-05, + "loss": 2.4728, + "theoretical_loss": 3.3178779027381333, + "tokens_seen": 3034120192 + }, + { + "epoch": 0.84, + "learning_rate": 1.6265446958754615e-05, + "loss": 2.4488, + "theoretical_loss": 3.3178669081568537, + "tokens_seen": 3034251264 + }, + { + "epoch": 0.84, + "learning_rate": 1.6257422564596375e-05, + "loss": 2.5035, + "theoretical_loss": 3.317855914183478, + "tokens_seen": 3034382336 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1664363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4554569721221924, + "objective/train/theoretical_loss": 3.3178504174247356, + "objective/train/tokens_used": 1404906976, + "theoretical_loss": 3.3178504174247356, + "tokens_seen": 3034447872 + }, + { + "epoch": 0.84, + "learning_rate": 1.6249398170438132e-05, + "loss": 2.545, + "theoretical_loss": 3.317844920817947, + "tokens_seen": 3034513408 + }, + { + "epoch": 0.84, + "learning_rate": 1.624137377627989e-05, + "loss": 2.4892, + "theoretical_loss": 3.3178339280602, + "tokens_seen": 3034644480 + }, + { + "epoch": 0.84, + "learning_rate": 1.623334938212165e-05, + "loss": 2.3942, + "theoretical_loss": 3.3178229359101774, + "tokens_seen": 3034775552 + }, + { + "epoch": 0.84, + "learning_rate": 1.6225324987963408e-05, + "loss": 2.3399, + "theoretical_loss": 3.31781194436782, + "tokens_seen": 3034906624 + }, + { + "epoch": 0.84, + "learning_rate": 1.6217300593805168e-05, + "loss": 2.4212, + "theoretical_loss": 3.317800953433067, + "tokens_seen": 3035037696 + }, + { + "epoch": 0.84, + "learning_rate": 1.6209276199646928e-05, + "loss": 2.5644, + "theoretical_loss": 3.317789963105859, + "tokens_seen": 3035168768 + }, + { + "epoch": 0.84, + "learning_rate": 1.6201251805488688e-05, + "loss": 2.565, + "theoretical_loss": 3.3177789733861363, + "tokens_seen": 3035299840 + }, + { + "epoch": 0.84, + "learning_rate": 1.6193227411330444e-05, + "loss": 2.5336, + "theoretical_loss": 3.3177679842738383, + "tokens_seen": 3035430912 + }, + { + "epoch": 0.84, + "learning_rate": 1.6185203017172204e-05, + "loss": 2.526, + "theoretical_loss": 3.3177569957689066, + "tokens_seen": 3035561984 + }, + { + "epoch": 0.84, + "learning_rate": 1.6177178623013964e-05, + "loss": 2.5815, + "theoretical_loss": 3.3177460078712806, + "tokens_seen": 3035693056 + }, + { + "epoch": 0.84, + "learning_rate": 1.6169154228855724e-05, + "loss": 2.4693, + "theoretical_loss": 3.3177350205809004, + "tokens_seen": 3035824128 + }, + { + "epoch": 0.84, + "learning_rate": 1.616112983469748e-05, + "loss": 2.5013, + "theoretical_loss": 3.317724033897706, + "tokens_seen": 3035955200 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1664891, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8782825469970703, + "objective/train/theoretical_loss": 3.3177130478216386, + "objective/train/tokens_used": 1406545376, + "theoretical_loss": 3.3177130478216386, + "tokens_seen": 3036086272 + }, + { + "epoch": 0.84, + "learning_rate": 1.615310544053924e-05, + "loss": 2.6436, + "theoretical_loss": 3.3177130478216386, + "tokens_seen": 3036086272 + }, + { + "epoch": 0.84, + "learning_rate": 1.6145081046381e-05, + "loss": 2.3684, + "theoretical_loss": 3.3177020623526374, + "tokens_seen": 3036217344 + }, + { + "epoch": 0.84, + "learning_rate": 1.6137056652222757e-05, + "loss": 2.3742, + "theoretical_loss": 3.3176910774906436, + "tokens_seen": 3036348416 + }, + { + "epoch": 0.84, + "learning_rate": 1.6129032258064517e-05, + "loss": 2.5547, + "theoretical_loss": 3.3176800932355968, + "tokens_seen": 3036479488 + }, + { + "epoch": 0.84, + "learning_rate": 1.6121007863906277e-05, + "loss": 2.6758, + "theoretical_loss": 3.3176691095874373, + "tokens_seen": 3036610560 + }, + { + "epoch": 0.84, + "learning_rate": 1.6112983469748037e-05, + "loss": 2.4446, + "theoretical_loss": 3.317658126546106, + "tokens_seen": 3036741632 + }, + { + "epoch": 0.84, + "learning_rate": 1.6104959075589793e-05, + "loss": 2.5401, + "theoretical_loss": 3.3176471441115423, + "tokens_seen": 3036872704 + }, + { + "epoch": 0.84, + "learning_rate": 1.6096934681431553e-05, + "loss": 2.5432, + "theoretical_loss": 3.317636162283687, + "tokens_seen": 3037003776 + }, + { + "epoch": 0.84, + "learning_rate": 1.6088910287273313e-05, + "loss": 2.5467, + "theoretical_loss": 3.31762518106248, + "tokens_seen": 3037134848 + }, + { + "epoch": 0.84, + "learning_rate": 1.6080885893115073e-05, + "loss": 2.4923, + "theoretical_loss": 3.317614200447862, + "tokens_seen": 3037265920 + }, + { + "epoch": 0.84, + "learning_rate": 1.607286149895683e-05, + "loss": 2.3481, + "theoretical_loss": 3.3176032204397736, + "tokens_seen": 3037396992 + }, + { + "epoch": 0.84, + "learning_rate": 1.606483710479859e-05, + "loss": 2.4739, + "theoretical_loss": 3.3175922410381546, + "tokens_seen": 3037528064 + }, + { + "epoch": 0.84, + "learning_rate": 1.605681271064035e-05, + "loss": 2.6104, + "theoretical_loss": 3.3175812622429453, + "tokens_seen": 3037659136 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1665791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2066845893859863, + "objective/train/theoretical_loss": 3.317575773072726, + "objective/train/tokens_used": 1408183776, + "theoretical_loss": 3.317575773072726, + "tokens_seen": 3037724672 + }, + { + "epoch": 0.84, + "learning_rate": 1.6048788316482106e-05, + "loss": 2.2528, + "theoretical_loss": 3.317570284054087, + "tokens_seen": 3037790208 + }, + { + "epoch": 0.84, + "learning_rate": 1.6040763922323866e-05, + "loss": 2.4986, + "theoretical_loss": 3.317559306471518, + "tokens_seen": 3037921280 + }, + { + "epoch": 0.84, + "learning_rate": 1.6032739528165626e-05, + "loss": 2.5177, + "theoretical_loss": 3.317548329495181, + "tokens_seen": 3038052352 + }, + { + "epoch": 0.84, + "learning_rate": 1.6024715134007385e-05, + "loss": 2.6081, + "theoretical_loss": 3.3175373531250147, + "tokens_seen": 3038183424 + }, + { + "epoch": 0.84, + "learning_rate": 1.6016690739849142e-05, + "loss": 2.6728, + "theoretical_loss": 3.3175263773609602, + "tokens_seen": 3038314496 + }, + { + "epoch": 0.84, + "learning_rate": 1.6008666345690902e-05, + "loss": 2.6007, + "theoretical_loss": 3.3175154022029583, + "tokens_seen": 3038445568 + }, + { + "epoch": 0.84, + "learning_rate": 1.6000641951532662e-05, + "loss": 2.4508, + "theoretical_loss": 3.317504427650948, + "tokens_seen": 3038576640 + }, + { + "epoch": 0.84, + "learning_rate": 1.599261755737442e-05, + "loss": 2.537, + "theoretical_loss": 3.3174934537048713, + "tokens_seen": 3038707712 + }, + { + "epoch": 0.84, + "learning_rate": 1.5984593163216178e-05, + "loss": 2.5027, + "theoretical_loss": 3.317482480364667, + "tokens_seen": 3038838784 + }, + { + "epoch": 0.84, + "learning_rate": 1.5976568769057938e-05, + "loss": 2.6133, + "theoretical_loss": 3.3174715076302776, + "tokens_seen": 3038969856 + }, + { + "epoch": 0.84, + "learning_rate": 1.5968544374899698e-05, + "loss": 2.4922, + "theoretical_loss": 3.3174605355016418, + "tokens_seen": 3039100928 + }, + { + "epoch": 0.84, + "learning_rate": 1.5960519980741454e-05, + "loss": 2.4346, + "theoretical_loss": 3.3174495639787005, + "tokens_seen": 3039232000 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1666368, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5738983154296875, + "objective/train/theoretical_loss": 3.317438593061394, + "objective/train/tokens_used": 1409822176, + "theoretical_loss": 3.317438593061394, + "tokens_seen": 3039363072 + }, + { + "epoch": 0.84, + "learning_rate": 1.5952495586583214e-05, + "loss": 2.4412, + "theoretical_loss": 3.317438593061394, + "tokens_seen": 3039363072 + }, + { + "epoch": 0.84, + "learning_rate": 1.5944471192424974e-05, + "loss": 2.6364, + "theoretical_loss": 3.3174276227496633, + "tokens_seen": 3039494144 + }, + { + "epoch": 0.84, + "learning_rate": 1.5936446798266734e-05, + "loss": 2.4192, + "theoretical_loss": 3.3174166530434483, + "tokens_seen": 3039625216 + }, + { + "epoch": 0.84, + "learning_rate": 1.592842240410849e-05, + "loss": 2.5044, + "theoretical_loss": 3.3174056839426895, + "tokens_seen": 3039756288 + }, + { + "epoch": 0.84, + "learning_rate": 1.592039800995025e-05, + "loss": 2.5201, + "theoretical_loss": 3.3173947154473282, + "tokens_seen": 3039887360 + }, + { + "epoch": 0.84, + "learning_rate": 1.591237361579201e-05, + "loss": 2.664, + "theoretical_loss": 3.3173837475573036, + "tokens_seen": 3040018432 + }, + { + "epoch": 0.84, + "learning_rate": 1.5904349221633767e-05, + "loss": 2.5798, + "theoretical_loss": 3.3173727802725574, + "tokens_seen": 3040149504 + }, + { + "epoch": 0.84, + "learning_rate": 1.5896324827475527e-05, + "loss": 2.4587, + "theoretical_loss": 3.3173618135930294, + "tokens_seen": 3040280576 + }, + { + "epoch": 0.84, + "learning_rate": 1.5888300433317287e-05, + "loss": 2.6078, + "theoretical_loss": 3.31735084751866, + "tokens_seen": 3040411648 + }, + { + "epoch": 0.84, + "learning_rate": 1.5880276039159043e-05, + "loss": 2.537, + "theoretical_loss": 3.31733988204939, + "tokens_seen": 3040542720 + }, + { + "epoch": 0.84, + "learning_rate": 1.5872251645000803e-05, + "loss": 2.5461, + "theoretical_loss": 3.31732891718516, + "tokens_seen": 3040673792 + }, + { + "epoch": 0.84, + "learning_rate": 1.5864227250842563e-05, + "loss": 2.4755, + "theoretical_loss": 3.3173179529259107, + "tokens_seen": 3040804864 + }, + { + "epoch": 0.84, + "learning_rate": 1.585620285668432e-05, + "loss": 2.5608, + "theoretical_loss": 3.3173069892715823, + "tokens_seen": 3040935936 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1667402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6003267765045166, + "objective/train/theoretical_loss": 3.3173015076712447, + "objective/train/tokens_used": 1411460576, + "theoretical_loss": 3.3173015076712447, + "tokens_seen": 3041001472 + }, + { + "epoch": 0.84, + "learning_rate": 1.584817846252608e-05, + "loss": 2.4349, + "theoretical_loss": 3.3172960262221154, + "tokens_seen": 3041067008 + }, + { + "epoch": 0.84, + "learning_rate": 1.584015406836784e-05, + "loss": 2.4019, + "theoretical_loss": 3.317285063777451, + "tokens_seen": 3041198080 + }, + { + "epoch": 0.84, + "learning_rate": 1.5832129674209596e-05, + "loss": 2.5457, + "theoretical_loss": 3.3172741019375285, + "tokens_seen": 3041329152 + }, + { + "epoch": 0.84, + "learning_rate": 1.5824105280051356e-05, + "loss": 2.4927, + "theoretical_loss": 3.3172631407022894, + "tokens_seen": 3041460224 + }, + { + "epoch": 0.84, + "learning_rate": 1.5816080885893116e-05, + "loss": 2.5248, + "theoretical_loss": 3.3172521800716748, + "tokens_seen": 3041591296 + }, + { + "epoch": 0.84, + "learning_rate": 1.5808056491734872e-05, + "loss": 2.5397, + "theoretical_loss": 3.317241220045624, + "tokens_seen": 3041722368 + }, + { + "epoch": 0.84, + "learning_rate": 1.5800032097576632e-05, + "loss": 2.4543, + "theoretical_loss": 3.3172302606240787, + "tokens_seen": 3041853440 + }, + { + "epoch": 0.84, + "learning_rate": 1.5792007703418392e-05, + "loss": 2.5292, + "theoretical_loss": 3.3172193018069787, + "tokens_seen": 3041984512 + }, + { + "epoch": 0.84, + "learning_rate": 1.578398330926015e-05, + "loss": 2.5867, + "theoretical_loss": 3.317208343594265, + "tokens_seen": 3042115584 + }, + { + "epoch": 0.84, + "learning_rate": 1.577595891510191e-05, + "loss": 2.6072, + "theoretical_loss": 3.3171973859858785, + "tokens_seen": 3042246656 + }, + { + "epoch": 0.84, + "learning_rate": 1.576793452094367e-05, + "loss": 2.6158, + "theoretical_loss": 3.3171864289817594, + "tokens_seen": 3042377728 + }, + { + "epoch": 0.84, + "learning_rate": 1.575991012678543e-05, + "loss": 2.5423, + "theoretical_loss": 3.3171754725818485, + "tokens_seen": 3042508800 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1667965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5699822902679443, + "objective/train/theoretical_loss": 3.3171645167860864, + "objective/train/tokens_used": 1413098976, + "theoretical_loss": 3.3171645167860864, + "tokens_seen": 3042639872 + }, + { + "epoch": 0.84, + "learning_rate": 1.5751885732627185e-05, + "loss": 2.5357, + "theoretical_loss": 3.3171645167860864, + "tokens_seen": 3042639872 + }, + { + "epoch": 0.84, + "learning_rate": 1.5743861338468945e-05, + "loss": 2.6788, + "theoretical_loss": 3.317153561594414, + "tokens_seen": 3042770944 + }, + { + "epoch": 0.84, + "learning_rate": 1.5735836944310705e-05, + "loss": 2.4735, + "theoretical_loss": 3.3171426070067715, + "tokens_seen": 3042902016 + }, + { + "epoch": 0.84, + "learning_rate": 1.5727812550152465e-05, + "loss": 2.5951, + "theoretical_loss": 3.3171316530231003, + "tokens_seen": 3043033088 + }, + { + "epoch": 0.84, + "learning_rate": 1.571978815599422e-05, + "loss": 2.4411, + "theoretical_loss": 3.31712069964334, + "tokens_seen": 3043164160 + }, + { + "epoch": 0.84, + "learning_rate": 1.571176376183598e-05, + "loss": 2.45, + "theoretical_loss": 3.317109746867432, + "tokens_seen": 3043295232 + }, + { + "epoch": 0.84, + "learning_rate": 1.570373936767774e-05, + "loss": 2.4725, + "theoretical_loss": 3.3170987946953177, + "tokens_seen": 3043426304 + }, + { + "epoch": 0.84, + "learning_rate": 1.56957149735195e-05, + "loss": 2.634, + "theoretical_loss": 3.3170878431269366, + "tokens_seen": 3043557376 + }, + { + "epoch": 0.84, + "learning_rate": 1.5687690579361257e-05, + "loss": 2.4361, + "theoretical_loss": 3.31707689216223, + "tokens_seen": 3043688448 + }, + { + "epoch": 0.84, + "learning_rate": 1.5679666185203017e-05, + "loss": 2.4865, + "theoretical_loss": 3.317065941801139, + "tokens_seen": 3043819520 + }, + { + "epoch": 0.84, + "learning_rate": 1.5671641791044777e-05, + "loss": 2.6059, + "theoretical_loss": 3.317054992043603, + "tokens_seen": 3043950592 + }, + { + "epoch": 0.84, + "learning_rate": 1.5663617396886534e-05, + "loss": 2.4695, + "theoretical_loss": 3.317044042889564, + "tokens_seen": 3044081664 + }, + { + "epoch": 0.85, + "learning_rate": 1.5655593002728294e-05, + "loss": 2.4768, + "theoretical_loss": 3.317033094338962, + "tokens_seen": 3044212736 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1668980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5335159301757812, + "objective/train/theoretical_loss": 3.317027620289932, + "objective/train/tokens_used": 1414737376, + "theoretical_loss": 3.317027620289932, + "tokens_seen": 3044278272 + }, + { + "epoch": 0.85, + "learning_rate": 1.5647568608570054e-05, + "loss": 2.5277, + "theoretical_loss": 3.3170221463917384, + "tokens_seen": 3044343808 + }, + { + "epoch": 0.85, + "learning_rate": 1.5639544214411813e-05, + "loss": 2.4658, + "theoretical_loss": 3.3170111990478337, + "tokens_seen": 3044474880 + }, + { + "epoch": 0.85, + "learning_rate": 1.563151982025357e-05, + "loss": 2.6097, + "theoretical_loss": 3.3170002523071886, + "tokens_seen": 3044605952 + }, + { + "epoch": 0.85, + "learning_rate": 1.562349542609533e-05, + "loss": 2.501, + "theoretical_loss": 3.316989306169744, + "tokens_seen": 3044737024 + }, + { + "epoch": 0.85, + "learning_rate": 1.561547103193709e-05, + "loss": 2.3754, + "theoretical_loss": 3.3169783606354413, + "tokens_seen": 3044868096 + }, + { + "epoch": 0.85, + "learning_rate": 1.560744663777885e-05, + "loss": 2.5464, + "theoretical_loss": 3.3169674157042195, + "tokens_seen": 3044999168 + }, + { + "epoch": 0.85, + "learning_rate": 1.5599422243620606e-05, + "loss": 2.5343, + "theoretical_loss": 3.3169564713760216, + "tokens_seen": 3045130240 + }, + { + "epoch": 0.85, + "learning_rate": 1.5591397849462366e-05, + "loss": 2.4912, + "theoretical_loss": 3.3169455276507867, + "tokens_seen": 3045261312 + }, + { + "epoch": 0.85, + "learning_rate": 1.5583373455304126e-05, + "loss": 2.6209, + "theoretical_loss": 3.3169345845284566, + "tokens_seen": 3045392384 + }, + { + "epoch": 0.85, + "learning_rate": 1.5575349061145883e-05, + "loss": 2.4422, + "theoretical_loss": 3.316923642008972, + "tokens_seen": 3045523456 + }, + { + "epoch": 0.85, + "learning_rate": 1.5567324666987642e-05, + "loss": 2.6853, + "theoretical_loss": 3.3169127000922733, + "tokens_seen": 3045654528 + }, + { + "epoch": 0.85, + "learning_rate": 1.5559300272829402e-05, + "loss": 2.5424, + "theoretical_loss": 3.316901758778302, + "tokens_seen": 3045785600 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1669400, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.407111167907715, + "objective/train/theoretical_loss": 3.3168908180669985, + "objective/train/tokens_used": 1416375776, + "theoretical_loss": 3.3168908180669985, + "tokens_seen": 3045916672 + }, + { + "epoch": 0.85, + "learning_rate": 1.5551275878671162e-05, + "loss": 2.5602, + "theoretical_loss": 3.3168908180669985, + "tokens_seen": 3045916672 + }, + { + "epoch": 0.85, + "learning_rate": 1.554325148451292e-05, + "loss": 2.5502, + "theoretical_loss": 3.3168798779583035, + "tokens_seen": 3046047744 + }, + { + "epoch": 0.85, + "learning_rate": 1.553522709035468e-05, + "loss": 2.6937, + "theoretical_loss": 3.3168689384521586, + "tokens_seen": 3046178816 + }, + { + "epoch": 0.85, + "learning_rate": 1.552720269619644e-05, + "loss": 2.5193, + "theoretical_loss": 3.316857999548504, + "tokens_seen": 3046309888 + }, + { + "epoch": 0.85, + "learning_rate": 1.55191783020382e-05, + "loss": 2.4718, + "theoretical_loss": 3.316847061247281, + "tokens_seen": 3046440960 + }, + { + "epoch": 0.85, + "learning_rate": 1.5511153907879955e-05, + "loss": 2.4381, + "theoretical_loss": 3.3168361235484305, + "tokens_seen": 3046572032 + }, + { + "epoch": 0.85, + "learning_rate": 1.5503129513721715e-05, + "loss": 2.5585, + "theoretical_loss": 3.3168251864518936, + "tokens_seen": 3046703104 + }, + { + "epoch": 0.85, + "learning_rate": 1.5495105119563475e-05, + "loss": 2.5928, + "theoretical_loss": 3.31681424995761, + "tokens_seen": 3046834176 + }, + { + "epoch": 0.85, + "learning_rate": 1.548708072540523e-05, + "loss": 2.4617, + "theoretical_loss": 3.316803314065522, + "tokens_seen": 3046965248 + }, + { + "epoch": 0.85, + "learning_rate": 1.547905633124699e-05, + "loss": 2.7113, + "theoretical_loss": 3.3167923787755704, + "tokens_seen": 3047096320 + }, + { + "epoch": 0.85, + "learning_rate": 1.547103193708875e-05, + "loss": 2.5256, + "theoretical_loss": 3.3167814440876957, + "tokens_seen": 3047227392 + }, + { + "epoch": 0.85, + "learning_rate": 1.546300754293051e-05, + "loss": 2.5155, + "theoretical_loss": 3.316770510001839, + "tokens_seen": 3047358464 + }, + { + "epoch": 0.85, + "learning_rate": 1.5454983148772268e-05, + "loss": 2.3391, + "theoretical_loss": 3.316759576517941, + "tokens_seen": 3047489536 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1670628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.565459966659546, + "objective/train/theoretical_loss": 3.316754110001708, + "objective/train/tokens_used": 1418014176, + "theoretical_loss": 3.316754110001708, + "tokens_seen": 3047555072 + }, + { + "epoch": 0.85, + "learning_rate": 1.5446958754614027e-05, + "loss": 2.4615, + "theoretical_loss": 3.3167486436359432, + "tokens_seen": 3047620608 + }, + { + "epoch": 0.85, + "learning_rate": 1.5438934360455787e-05, + "loss": 2.7149, + "theoretical_loss": 3.316737711355786, + "tokens_seen": 3047751680 + }, + { + "epoch": 0.85, + "learning_rate": 1.5430909966297547e-05, + "loss": 2.4278, + "theoretical_loss": 3.3167267796774107, + "tokens_seen": 3047882752 + }, + { + "epoch": 0.85, + "learning_rate": 1.5422885572139304e-05, + "loss": 2.5207, + "theoretical_loss": 3.316715848600759, + "tokens_seen": 3048013824 + }, + { + "epoch": 0.85, + "learning_rate": 1.5414861177981064e-05, + "loss": 2.5722, + "theoretical_loss": 3.3167049181257706, + "tokens_seen": 3048144896 + }, + { + "epoch": 0.85, + "learning_rate": 1.5406836783822824e-05, + "loss": 2.6322, + "theoretical_loss": 3.316693988252387, + "tokens_seen": 3048275968 + }, + { + "epoch": 0.85, + "learning_rate": 1.539881238966458e-05, + "loss": 2.6071, + "theoretical_loss": 3.3166830589805496, + "tokens_seen": 3048407040 + }, + { + "epoch": 0.85, + "learning_rate": 1.539078799550634e-05, + "loss": 2.6415, + "theoretical_loss": 3.3166721303101987, + "tokens_seen": 3048538112 + }, + { + "epoch": 0.85, + "learning_rate": 1.53827636013481e-05, + "loss": 2.6135, + "theoretical_loss": 3.316661202241276, + "tokens_seen": 3048669184 + }, + { + "epoch": 0.85, + "learning_rate": 1.537473920718986e-05, + "loss": 2.5045, + "theoretical_loss": 3.316650274773723, + "tokens_seen": 3048800256 + }, + { + "epoch": 0.85, + "learning_rate": 1.5366714813031616e-05, + "loss": 2.5821, + "theoretical_loss": 3.316639347907479, + "tokens_seen": 3048931328 + }, + { + "epoch": 0.85, + "learning_rate": 1.5358690418873376e-05, + "loss": 2.5712, + "theoretical_loss": 3.316628421642487, + "tokens_seen": 3049062400 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1671752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6042397022247314, + "objective/train/theoretical_loss": 3.316617495978687, + "objective/train/tokens_used": 1419652576, + "theoretical_loss": 3.316617495978687, + "tokens_seen": 3049193472 + }, + { + "epoch": 0.85, + "learning_rate": 1.5350666024715136e-05, + "loss": 2.5875, + "theoretical_loss": 3.316617495978687, + "tokens_seen": 3049193472 + }, + { + "epoch": 0.85, + "learning_rate": 1.5342641630556896e-05, + "loss": 2.65, + "theoretical_loss": 3.3166065709160204, + "tokens_seen": 3049324544 + }, + { + "epoch": 0.85, + "learning_rate": 1.5334617236398653e-05, + "loss": 2.5318, + "theoretical_loss": 3.316595646454428, + "tokens_seen": 3049455616 + }, + { + "epoch": 0.85, + "learning_rate": 1.5326592842240413e-05, + "loss": 2.6331, + "theoretical_loss": 3.316584722593851, + "tokens_seen": 3049586688 + }, + { + "epoch": 0.85, + "learning_rate": 1.5318568448082172e-05, + "loss": 2.5451, + "theoretical_loss": 3.3165737993342312, + "tokens_seen": 3049717760 + }, + { + "epoch": 0.85, + "learning_rate": 1.531054405392393e-05, + "loss": 2.5625, + "theoretical_loss": 3.3165628766755084, + "tokens_seen": 3049848832 + }, + { + "epoch": 0.85, + "learning_rate": 1.530251965976569e-05, + "loss": 2.5471, + "theoretical_loss": 3.316551954617625, + "tokens_seen": 3049979904 + }, + { + "epoch": 0.85, + "learning_rate": 1.529449526560745e-05, + "loss": 2.6976, + "theoretical_loss": 3.3165410331605214, + "tokens_seen": 3050110976 + }, + { + "epoch": 0.85, + "learning_rate": 1.528647087144921e-05, + "loss": 2.6216, + "theoretical_loss": 3.316530112304139, + "tokens_seen": 3050242048 + }, + { + "epoch": 0.85, + "learning_rate": 1.5278446477290965e-05, + "loss": 2.6002, + "theoretical_loss": 3.3165191920484185, + "tokens_seen": 3050373120 + }, + { + "epoch": 0.85, + "learning_rate": 1.5270422083132725e-05, + "loss": 2.5463, + "theoretical_loss": 3.316508272393302, + "tokens_seen": 3050504192 + }, + { + "epoch": 0.85, + "learning_rate": 1.5262397688974485e-05, + "loss": 2.6211, + "theoretical_loss": 3.3164973533387294, + "tokens_seen": 3050635264 + }, + { + "epoch": 0.85, + "learning_rate": 1.5254373294816243e-05, + "loss": 2.566, + "theoretical_loss": 3.3164864348846432, + "tokens_seen": 3050766336 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1672176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.466606616973877, + "objective/train/theoretical_loss": 3.3164809758827634, + "objective/train/tokens_used": 1421290976, + "theoretical_loss": 3.3164809758827634, + "tokens_seen": 3050831872 + }, + { + "epoch": 0.85, + "learning_rate": 1.5246348900658e-05, + "loss": 2.5025, + "theoretical_loss": 3.3164755170309834, + "tokens_seen": 3050897408 + }, + { + "epoch": 0.85, + "learning_rate": 1.523832450649976e-05, + "loss": 2.4682, + "theoretical_loss": 3.316464599777692, + "tokens_seen": 3051028480 + }, + { + "epoch": 0.85, + "learning_rate": 1.523030011234152e-05, + "loss": 2.5364, + "theoretical_loss": 3.31645368312471, + "tokens_seen": 3051159552 + }, + { + "epoch": 0.85, + "learning_rate": 1.522227571818328e-05, + "loss": 2.5779, + "theoretical_loss": 3.3164427670719783, + "tokens_seen": 3051290624 + }, + { + "epoch": 0.85, + "learning_rate": 1.5214251324025036e-05, + "loss": 2.3921, + "theoretical_loss": 3.316431851619438, + "tokens_seen": 3051421696 + }, + { + "epoch": 0.85, + "learning_rate": 1.5206226929866796e-05, + "loss": 2.5992, + "theoretical_loss": 3.316420936767031, + "tokens_seen": 3051552768 + }, + { + "epoch": 0.85, + "learning_rate": 1.5198202535708556e-05, + "loss": 2.6406, + "theoretical_loss": 3.3164100225146984, + "tokens_seen": 3051683840 + }, + { + "epoch": 0.85, + "learning_rate": 1.5190178141550312e-05, + "loss": 2.5572, + "theoretical_loss": 3.316399108862381, + "tokens_seen": 3051814912 + }, + { + "epoch": 0.85, + "learning_rate": 1.5182153747392072e-05, + "loss": 2.6631, + "theoretical_loss": 3.3163881958100205, + "tokens_seen": 3051945984 + }, + { + "epoch": 0.85, + "learning_rate": 1.5174129353233832e-05, + "loss": 2.6822, + "theoretical_loss": 3.3163772833575575, + "tokens_seen": 3052077056 + }, + { + "epoch": 0.85, + "learning_rate": 1.5166104959075592e-05, + "loss": 2.6556, + "theoretical_loss": 3.316366371504934, + "tokens_seen": 3052208128 + }, + { + "epoch": 0.85, + "learning_rate": 1.5158080564917349e-05, + "loss": 2.5627, + "theoretical_loss": 3.316355460252091, + "tokens_seen": 3052339200 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1673387, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.769145965576172, + "objective/train/theoretical_loss": 3.3163445495989694, + "objective/train/tokens_used": 1422929376, + "theoretical_loss": 3.3163445495989694, + "tokens_seen": 3052470272 + }, + { + "epoch": 0.85, + "learning_rate": 1.5150056170759108e-05, + "loss": 2.6315, + "theoretical_loss": 3.3163445495989694, + "tokens_seen": 3052470272 + }, + { + "epoch": 0.85, + "learning_rate": 1.5142031776600868e-05, + "loss": 2.4792, + "theoretical_loss": 3.316333639545511, + "tokens_seen": 3052601344 + }, + { + "epoch": 0.85, + "learning_rate": 1.5134007382442628e-05, + "loss": 2.5526, + "theoretical_loss": 3.316322730091657, + "tokens_seen": 3052732416 + }, + { + "epoch": 0.85, + "learning_rate": 1.5125982988284385e-05, + "loss": 2.6901, + "theoretical_loss": 3.3163118212373486, + "tokens_seen": 3052863488 + }, + { + "epoch": 0.85, + "learning_rate": 1.5117958594126145e-05, + "loss": 2.5541, + "theoretical_loss": 3.316300912982527, + "tokens_seen": 3052994560 + }, + { + "epoch": 0.85, + "learning_rate": 1.5109934199967905e-05, + "loss": 2.5325, + "theoretical_loss": 3.316290005327134, + "tokens_seen": 3053125632 + }, + { + "epoch": 0.85, + "learning_rate": 1.5101909805809661e-05, + "loss": 2.4847, + "theoretical_loss": 3.31627909827111, + "tokens_seen": 3053256704 + }, + { + "epoch": 0.85, + "learning_rate": 1.5093885411651421e-05, + "loss": 2.4819, + "theoretical_loss": 3.3162681918143972, + "tokens_seen": 3053387776 + }, + { + "epoch": 0.85, + "learning_rate": 1.5085861017493181e-05, + "loss": 2.557, + "theoretical_loss": 3.3162572859569366, + "tokens_seen": 3053518848 + }, + { + "epoch": 0.85, + "learning_rate": 1.5077836623334939e-05, + "loss": 2.5447, + "theoretical_loss": 3.3162463806986695, + "tokens_seen": 3053649920 + }, + { + "epoch": 0.85, + "learning_rate": 1.5069812229176697e-05, + "loss": 2.4972, + "theoretical_loss": 3.316235476039538, + "tokens_seen": 3053780992 + }, + { + "epoch": 0.85, + "learning_rate": 1.5061787835018457e-05, + "loss": 2.3541, + "theoretical_loss": 3.316224571979482, + "tokens_seen": 3053912064 + }, + { + "epoch": 0.85, + "learning_rate": 1.5053763440860215e-05, + "loss": 2.5233, + "theoretical_loss": 3.3162136685184445, + "tokens_seen": 3054043136 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1673816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.776703357696533, + "objective/train/theoretical_loss": 3.3162082170125387, + "objective/train/tokens_used": 1424567776, + "theoretical_loss": 3.3162082170125387, + "tokens_seen": 3054108672 + }, + { + "epoch": 0.85, + "learning_rate": 1.5045739046701975e-05, + "loss": 2.4799, + "theoretical_loss": 3.3162027656563655, + "tokens_seen": 3054174208 + }, + { + "epoch": 0.85, + "learning_rate": 1.5037714652543734e-05, + "loss": 2.5283, + "theoretical_loss": 3.316191863393187, + "tokens_seen": 3054305280 + }, + { + "epoch": 0.85, + "learning_rate": 1.5029690258385492e-05, + "loss": 2.5764, + "theoretical_loss": 3.316180961728851, + "tokens_seen": 3054436352 + }, + { + "epoch": 0.85, + "learning_rate": 1.5021665864227252e-05, + "loss": 2.6314, + "theoretical_loss": 3.316170060663298, + "tokens_seen": 3054567424 + }, + { + "epoch": 0.85, + "learning_rate": 1.501364147006901e-05, + "loss": 2.4294, + "theoretical_loss": 3.3161591601964697, + "tokens_seen": 3054698496 + }, + { + "epoch": 0.85, + "learning_rate": 1.5005617075910768e-05, + "loss": 2.5307, + "theoretical_loss": 3.3161482603283075, + "tokens_seen": 3054829568 + }, + { + "epoch": 0.85, + "learning_rate": 1.4997592681752528e-05, + "loss": 2.5783, + "theoretical_loss": 3.316137361058753, + "tokens_seen": 3054960640 + }, + { + "epoch": 0.85, + "learning_rate": 1.4989568287594288e-05, + "loss": 2.5835, + "theoretical_loss": 3.3161264623877473, + "tokens_seen": 3055091712 + }, + { + "epoch": 0.85, + "learning_rate": 1.4981543893436044e-05, + "loss": 2.5967, + "theoretical_loss": 3.3161155643152327, + "tokens_seen": 3055222784 + }, + { + "epoch": 0.85, + "learning_rate": 1.4973519499277804e-05, + "loss": 2.5167, + "theoretical_loss": 3.3161046668411496, + "tokens_seen": 3055353856 + }, + { + "epoch": 0.85, + "learning_rate": 1.4965495105119564e-05, + "loss": 2.4681, + "theoretical_loss": 3.3160937699654403, + "tokens_seen": 3055484928 + }, + { + "epoch": 0.85, + "learning_rate": 1.4957470710961324e-05, + "loss": 2.4444, + "theoretical_loss": 3.316082873688045, + "tokens_seen": 3055616000 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1674780, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.520395040512085, + "objective/train/theoretical_loss": 3.316071978008907, + "objective/train/tokens_used": 1426206176, + "theoretical_loss": 3.316071978008907, + "tokens_seen": 3055747072 + }, + { + "epoch": 0.85, + "learning_rate": 1.494944631680308e-05, + "loss": 2.5961, + "theoretical_loss": 3.316071978008907, + "tokens_seen": 3055747072 + }, + { + "epoch": 0.85, + "learning_rate": 1.494142192264484e-05, + "loss": 2.5469, + "theoretical_loss": 3.316061082927967, + "tokens_seen": 3055878144 + }, + { + "epoch": 0.85, + "learning_rate": 1.49333975284866e-05, + "loss": 2.497, + "theoretical_loss": 3.316050188445166, + "tokens_seen": 3056009216 + }, + { + "epoch": 0.85, + "learning_rate": 1.4925373134328357e-05, + "loss": 2.6637, + "theoretical_loss": 3.3160392945604458, + "tokens_seen": 3056140288 + }, + { + "epoch": 0.85, + "learning_rate": 1.4917348740170117e-05, + "loss": 2.4947, + "theoretical_loss": 3.316028401273748, + "tokens_seen": 3056271360 + }, + { + "epoch": 0.85, + "learning_rate": 1.4909324346011877e-05, + "loss": 2.5498, + "theoretical_loss": 3.3160175085850145, + "tokens_seen": 3056402432 + }, + { + "epoch": 0.85, + "learning_rate": 1.4901299951853637e-05, + "loss": 2.5845, + "theoretical_loss": 3.316006616494186, + "tokens_seen": 3056533504 + }, + { + "epoch": 0.85, + "learning_rate": 1.4893275557695393e-05, + "loss": 2.5745, + "theoretical_loss": 3.315995725001205, + "tokens_seen": 3056664576 + }, + { + "epoch": 0.85, + "learning_rate": 1.4885251163537153e-05, + "loss": 2.6664, + "theoretical_loss": 3.3159848341060125, + "tokens_seen": 3056795648 + }, + { + "epoch": 0.85, + "learning_rate": 1.4877226769378913e-05, + "loss": 2.5379, + "theoretical_loss": 3.31597394380855, + "tokens_seen": 3056926720 + }, + { + "epoch": 0.85, + "learning_rate": 1.4869202375220673e-05, + "loss": 2.4157, + "theoretical_loss": 3.3159630541087592, + "tokens_seen": 3057057792 + }, + { + "epoch": 0.85, + "learning_rate": 1.486117798106243e-05, + "loss": 2.5781, + "theoretical_loss": 3.3159521650065815, + "tokens_seen": 3057188864 + }, + { + "epoch": 0.85, + "learning_rate": 1.485315358690419e-05, + "loss": 2.485, + "theoretical_loss": 3.3159412765019587, + "tokens_seen": 3057319936 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1675313, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.691657781600952, + "objective/train/theoretical_loss": 3.3159358324737123, + "objective/train/tokens_used": 1427844576, + "theoretical_loss": 3.3159358324737123, + "tokens_seen": 3057385472 + }, + { + "epoch": 0.85, + "learning_rate": 1.484512919274595e-05, + "loss": 2.5423, + "theoretical_loss": 3.3159303885948326, + "tokens_seen": 3057451008 + }, + { + "epoch": 0.85, + "learning_rate": 1.4837104798587706e-05, + "loss": 2.5224, + "theoretical_loss": 3.3159195012851446, + "tokens_seen": 3057582080 + }, + { + "epoch": 0.85, + "learning_rate": 1.4829080404429466e-05, + "loss": 2.4144, + "theoretical_loss": 3.315908614572836, + "tokens_seen": 3057713152 + }, + { + "epoch": 0.85, + "learning_rate": 1.4821056010271226e-05, + "loss": 2.6158, + "theoretical_loss": 3.315897728457849, + "tokens_seen": 3057844224 + }, + { + "epoch": 0.85, + "learning_rate": 1.4813031616112986e-05, + "loss": 2.4262, + "theoretical_loss": 3.3158868429401247, + "tokens_seen": 3057975296 + }, + { + "epoch": 0.85, + "learning_rate": 1.4805007221954742e-05, + "loss": 2.5168, + "theoretical_loss": 3.3158759580196047, + "tokens_seen": 3058106368 + }, + { + "epoch": 0.85, + "learning_rate": 1.4796982827796502e-05, + "loss": 2.5399, + "theoretical_loss": 3.315865073696231, + "tokens_seen": 3058237440 + }, + { + "epoch": 0.85, + "learning_rate": 1.4788958433638262e-05, + "loss": 2.5254, + "theoretical_loss": 3.3158541899699454, + "tokens_seen": 3058368512 + }, + { + "epoch": 0.85, + "learning_rate": 1.4780934039480022e-05, + "loss": 2.5443, + "theoretical_loss": 3.315843306840689, + "tokens_seen": 3058499584 + }, + { + "epoch": 0.85, + "learning_rate": 1.4772909645321778e-05, + "loss": 2.6038, + "theoretical_loss": 3.315832424308404, + "tokens_seen": 3058630656 + }, + { + "epoch": 0.85, + "learning_rate": 1.4764885251163538e-05, + "loss": 2.6214, + "theoretical_loss": 3.3158215423730315, + "tokens_seen": 3058761728 + }, + { + "epoch": 0.85, + "learning_rate": 1.4756860857005298e-05, + "loss": 2.4357, + "theoretical_loss": 3.315810661034514, + "tokens_seen": 3058892800 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1676633, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7599308490753174, + "objective/train/theoretical_loss": 3.3157997802927923, + "objective/train/tokens_used": 1429482976, + "theoretical_loss": 3.3157997802927923, + "tokens_seen": 3059023872 + }, + { + "epoch": 0.85, + "learning_rate": 1.4748836462847056e-05, + "loss": 2.5609, + "theoretical_loss": 3.3157997802927923, + "tokens_seen": 3059023872 + }, + { + "epoch": 0.85, + "learning_rate": 1.4740812068688815e-05, + "loss": 2.4203, + "theoretical_loss": 3.3157889001478087, + "tokens_seen": 3059154944 + }, + { + "epoch": 0.85, + "learning_rate": 1.4732787674530574e-05, + "loss": 2.6502, + "theoretical_loss": 3.3157780205995047, + "tokens_seen": 3059286016 + }, + { + "epoch": 0.85, + "learning_rate": 1.4724763280372333e-05, + "loss": 2.5964, + "theoretical_loss": 3.3157671416478216, + "tokens_seen": 3059417088 + }, + { + "epoch": 0.85, + "learning_rate": 1.471673888621409e-05, + "loss": 2.5115, + "theoretical_loss": 3.3157562632927022, + "tokens_seen": 3059548160 + }, + { + "epoch": 0.85, + "learning_rate": 1.470871449205585e-05, + "loss": 2.5497, + "theoretical_loss": 3.315745385534087, + "tokens_seen": 3059679232 + }, + { + "epoch": 0.85, + "learning_rate": 1.4700690097897609e-05, + "loss": 2.6019, + "theoretical_loss": 3.3157345083719187, + "tokens_seen": 3059810304 + }, + { + "epoch": 0.85, + "learning_rate": 1.4692665703739369e-05, + "loss": 2.6461, + "theoretical_loss": 3.3157236318061387, + "tokens_seen": 3059941376 + }, + { + "epoch": 0.85, + "learning_rate": 1.4684641309581127e-05, + "loss": 2.4945, + "theoretical_loss": 3.315712755836689, + "tokens_seen": 3060072448 + }, + { + "epoch": 0.85, + "learning_rate": 1.4676616915422885e-05, + "loss": 2.6303, + "theoretical_loss": 3.3157018804635103, + "tokens_seen": 3060203520 + }, + { + "epoch": 0.85, + "learning_rate": 1.4668592521264645e-05, + "loss": 2.4958, + "theoretical_loss": 3.3156910056865456, + "tokens_seen": 3060334592 + }, + { + "epoch": 0.85, + "learning_rate": 1.4660568127106405e-05, + "loss": 2.3388, + "theoretical_loss": 3.315680131505736, + "tokens_seen": 3060465664 + }, + { + "epoch": 0.85, + "learning_rate": 1.4652543732948162e-05, + "loss": 2.504, + "theoretical_loss": 3.315669257921024, + "tokens_seen": 3060596736 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1677094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.020873785018921, + "objective/train/theoretical_loss": 3.315663821352186, + "objective/train/tokens_used": 1431121376, + "theoretical_loss": 3.315663821352186, + "tokens_seen": 3060662272 + }, + { + "epoch": 0.86, + "learning_rate": 1.4644519338789922e-05, + "loss": 2.5688, + "theoretical_loss": 3.3156583849323504, + "tokens_seen": 3060727808 + }, + { + "epoch": 0.86, + "learning_rate": 1.4636494944631681e-05, + "loss": 2.5035, + "theoretical_loss": 3.315647512539658, + "tokens_seen": 3060858880 + }, + { + "epoch": 0.86, + "learning_rate": 1.4628470550473438e-05, + "loss": 2.5132, + "theoretical_loss": 3.3156366407428877, + "tokens_seen": 3060989952 + }, + { + "epoch": 0.86, + "learning_rate": 1.4620446156315198e-05, + "loss": 2.6009, + "theoretical_loss": 3.315625769541982, + "tokens_seen": 3061121024 + }, + { + "epoch": 0.86, + "learning_rate": 1.4612421762156958e-05, + "loss": 2.5412, + "theoretical_loss": 3.3156148989368823, + "tokens_seen": 3061252096 + }, + { + "epoch": 0.86, + "learning_rate": 1.4604397367998718e-05, + "loss": 2.5552, + "theoretical_loss": 3.315604028927531, + "tokens_seen": 3061383168 + }, + { + "epoch": 0.86, + "learning_rate": 1.4596372973840474e-05, + "loss": 2.4837, + "theoretical_loss": 3.3155931595138695, + "tokens_seen": 3061514240 + }, + { + "epoch": 0.86, + "learning_rate": 1.4588348579682234e-05, + "loss": 2.5179, + "theoretical_loss": 3.3155822906958394, + "tokens_seen": 3061645312 + }, + { + "epoch": 0.86, + "learning_rate": 1.4580324185523994e-05, + "loss": 2.4506, + "theoretical_loss": 3.315571422473383, + "tokens_seen": 3061776384 + }, + { + "epoch": 0.86, + "learning_rate": 1.4572299791365754e-05, + "loss": 2.5928, + "theoretical_loss": 3.315560554846442, + "tokens_seen": 3061907456 + }, + { + "epoch": 0.86, + "learning_rate": 1.456427539720751e-05, + "loss": 2.575, + "theoretical_loss": 3.315549687814959, + "tokens_seen": 3062038528 + }, + { + "epoch": 0.86, + "learning_rate": 1.455625100304927e-05, + "loss": 2.6156, + "theoretical_loss": 3.315538821378875, + "tokens_seen": 3062169600 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1678209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6883559226989746, + "objective/train/theoretical_loss": 3.3155279555381316, + "objective/train/tokens_used": 1432759776, + "theoretical_loss": 3.3155279555381316, + "tokens_seen": 3062300672 + }, + { + "epoch": 0.86, + "learning_rate": 1.454822660889103e-05, + "loss": 2.4816, + "theoretical_loss": 3.3155279555381316, + "tokens_seen": 3062300672 + }, + { + "epoch": 0.86, + "learning_rate": 1.4540202214732787e-05, + "loss": 2.7469, + "theoretical_loss": 3.315517090292672, + "tokens_seen": 3062431744 + }, + { + "epoch": 0.86, + "learning_rate": 1.4532177820574547e-05, + "loss": 2.4916, + "theoretical_loss": 3.3155062256424364, + "tokens_seen": 3062562816 + }, + { + "epoch": 0.86, + "learning_rate": 1.4524153426416307e-05, + "loss": 2.5447, + "theoretical_loss": 3.3154953615873684, + "tokens_seen": 3062693888 + }, + { + "epoch": 0.86, + "learning_rate": 1.4516129032258066e-05, + "loss": 2.4958, + "theoretical_loss": 3.3154844981274088, + "tokens_seen": 3062824960 + }, + { + "epoch": 0.86, + "learning_rate": 1.4508104638099823e-05, + "loss": 2.5403, + "theoretical_loss": 3.3154736352625003, + "tokens_seen": 3062956032 + }, + { + "epoch": 0.86, + "learning_rate": 1.4500080243941583e-05, + "loss": 2.6656, + "theoretical_loss": 3.315462772992584, + "tokens_seen": 3063087104 + }, + { + "epoch": 0.86, + "learning_rate": 1.4492055849783343e-05, + "loss": 2.6808, + "theoretical_loss": 3.3154519113176026, + "tokens_seen": 3063218176 + }, + { + "epoch": 0.86, + "learning_rate": 1.4484031455625103e-05, + "loss": 2.5495, + "theoretical_loss": 3.3154410502374976, + "tokens_seen": 3063349248 + }, + { + "epoch": 0.86, + "learning_rate": 1.447600706146686e-05, + "loss": 2.4814, + "theoretical_loss": 3.3154301897522114, + "tokens_seen": 3063480320 + }, + { + "epoch": 0.86, + "learning_rate": 1.4467982667308619e-05, + "loss": 2.5933, + "theoretical_loss": 3.315419329861686, + "tokens_seen": 3063611392 + }, + { + "epoch": 0.86, + "learning_rate": 1.4459958273150379e-05, + "loss": 2.5648, + "theoretical_loss": 3.3154084705658624, + "tokens_seen": 3063742464 + }, + { + "epoch": 0.86, + "learning_rate": 1.4451933878992136e-05, + "loss": 2.4628, + "theoretical_loss": 3.3153976118646837, + "tokens_seen": 3063873536 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1678783, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.77242374420166, + "objective/train/theoretical_loss": 3.3153921827370683, + "objective/train/tokens_used": 1434398176, + "theoretical_loss": 3.3153921827370683, + "tokens_seen": 3063939072 + }, + { + "epoch": 0.86, + "learning_rate": 1.4443909484833895e-05, + "loss": 2.3325, + "theoretical_loss": 3.3153867537580917, + "tokens_seen": 3064004608 + }, + { + "epoch": 0.86, + "learning_rate": 1.4435885090675655e-05, + "loss": 2.6655, + "theoretical_loss": 3.315375896246028, + "tokens_seen": 3064135680 + }, + { + "epoch": 0.86, + "learning_rate": 1.4427860696517415e-05, + "loss": 2.6586, + "theoretical_loss": 3.3153650393284346, + "tokens_seen": 3064266752 + }, + { + "epoch": 0.86, + "learning_rate": 1.4419836302359172e-05, + "loss": 2.5659, + "theoretical_loss": 3.315354183005254, + "tokens_seen": 3064397824 + }, + { + "epoch": 0.86, + "learning_rate": 1.4411811908200932e-05, + "loss": 2.4379, + "theoretical_loss": 3.315343327276428, + "tokens_seen": 3064528896 + }, + { + "epoch": 0.86, + "learning_rate": 1.4403787514042692e-05, + "loss": 2.4609, + "theoretical_loss": 3.315332472141899, + "tokens_seen": 3064659968 + }, + { + "epoch": 0.86, + "learning_rate": 1.439576311988445e-05, + "loss": 2.5342, + "theoretical_loss": 3.315321617601608, + "tokens_seen": 3064791040 + }, + { + "epoch": 0.86, + "learning_rate": 1.4387738725726208e-05, + "loss": 2.595, + "theoretical_loss": 3.315310763655498, + "tokens_seen": 3064922112 + }, + { + "epoch": 0.86, + "learning_rate": 1.4379714331567968e-05, + "loss": 2.6508, + "theoretical_loss": 3.3152999103035112, + "tokens_seen": 3065053184 + }, + { + "epoch": 0.86, + "learning_rate": 1.4371689937409726e-05, + "loss": 2.3388, + "theoretical_loss": 3.315289057545589, + "tokens_seen": 3065184256 + }, + { + "epoch": 0.86, + "learning_rate": 1.4363665543251484e-05, + "loss": 2.5525, + "theoretical_loss": 3.3152782053816736, + "tokens_seen": 3065315328 + }, + { + "epoch": 0.86, + "learning_rate": 1.4355641149093244e-05, + "loss": 2.6307, + "theoretical_loss": 3.3152673538117075, + "tokens_seen": 3065446400 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1679875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8023416996002197, + "objective/train/theoretical_loss": 3.3152565028356324, + "objective/train/tokens_used": 1436036576, + "theoretical_loss": 3.3152565028356324, + "tokens_seen": 3065577472 + }, + { + "epoch": 0.86, + "learning_rate": 1.4347616754935002e-05, + "loss": 2.5914, + "theoretical_loss": 3.3152565028356324, + "tokens_seen": 3065577472 + }, + { + "epoch": 0.86, + "learning_rate": 1.4339592360776762e-05, + "loss": 2.6115, + "theoretical_loss": 3.315245652453391, + "tokens_seen": 3065708544 + }, + { + "epoch": 0.86, + "learning_rate": 1.433156796661852e-05, + "loss": 2.5214, + "theoretical_loss": 3.3152348026649245, + "tokens_seen": 3065839616 + }, + { + "epoch": 0.86, + "learning_rate": 1.4323543572460279e-05, + "loss": 2.4591, + "theoretical_loss": 3.3152239534701753, + "tokens_seen": 3065970688 + }, + { + "epoch": 0.86, + "learning_rate": 1.4315519178302039e-05, + "loss": 2.5935, + "theoretical_loss": 3.3152131048690863, + "tokens_seen": 3066101760 + }, + { + "epoch": 0.86, + "learning_rate": 1.4307494784143799e-05, + "loss": 2.5806, + "theoretical_loss": 3.315202256861599, + "tokens_seen": 3066232832 + }, + { + "epoch": 0.86, + "learning_rate": 1.4299470389985555e-05, + "loss": 2.4617, + "theoretical_loss": 3.3151914094476553, + "tokens_seen": 3066363904 + }, + { + "epoch": 0.86, + "learning_rate": 1.4291445995827315e-05, + "loss": 2.4192, + "theoretical_loss": 3.315180562627198, + "tokens_seen": 3066494976 + }, + { + "epoch": 0.86, + "learning_rate": 1.4283421601669075e-05, + "loss": 2.6678, + "theoretical_loss": 3.3151697164001686, + "tokens_seen": 3066626048 + }, + { + "epoch": 0.86, + "learning_rate": 1.4275397207510835e-05, + "loss": 2.6263, + "theoretical_loss": 3.31515887076651, + "tokens_seen": 3066757120 + }, + { + "epoch": 0.86, + "learning_rate": 1.4267372813352591e-05, + "loss": 2.3305, + "theoretical_loss": 3.3151480257261636, + "tokens_seen": 3066888192 + }, + { + "epoch": 0.86, + "learning_rate": 1.4259348419194351e-05, + "loss": 2.5424, + "theoretical_loss": 3.315137181279072, + "tokens_seen": 3067019264 + }, + { + "epoch": 0.86, + "learning_rate": 1.4251324025036111e-05, + "loss": 2.5744, + "theoretical_loss": 3.3151263374251774, + "tokens_seen": 3067150336 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1680552, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.414252996444702, + "objective/train/theoretical_loss": 3.315120915720661, + "objective/train/tokens_used": 1437674976, + "theoretical_loss": 3.315120915720661, + "tokens_seen": 3067215872 + }, + { + "epoch": 0.86, + "learning_rate": 1.4243299630877868e-05, + "loss": 2.6072, + "theoretical_loss": 3.3151154941644223, + "tokens_seen": 3067281408 + }, + { + "epoch": 0.86, + "learning_rate": 1.4235275236719628e-05, + "loss": 2.5786, + "theoretical_loss": 3.3151046514967484, + "tokens_seen": 3067412480 + }, + { + "epoch": 0.86, + "learning_rate": 1.4227250842561388e-05, + "loss": 2.408, + "theoretical_loss": 3.3150938094220983, + "tokens_seen": 3067543552 + }, + { + "epoch": 0.86, + "learning_rate": 1.4219226448403147e-05, + "loss": 2.5059, + "theoretical_loss": 3.3150829679404135, + "tokens_seen": 3067674624 + }, + { + "epoch": 0.86, + "learning_rate": 1.4211202054244904e-05, + "loss": 2.5236, + "theoretical_loss": 3.3150721270516375, + "tokens_seen": 3067805696 + }, + { + "epoch": 0.86, + "learning_rate": 1.4203177660086664e-05, + "loss": 2.5923, + "theoretical_loss": 3.315061286755711, + "tokens_seen": 3067936768 + }, + { + "epoch": 0.86, + "learning_rate": 1.4195153265928424e-05, + "loss": 2.5668, + "theoretical_loss": 3.3150504470525775, + "tokens_seen": 3068067840 + }, + { + "epoch": 0.86, + "learning_rate": 1.4187128871770184e-05, + "loss": 2.5885, + "theoretical_loss": 3.315039607942179, + "tokens_seen": 3068198912 + }, + { + "epoch": 0.86, + "learning_rate": 1.417910447761194e-05, + "loss": 2.4659, + "theoretical_loss": 3.3150287694244573, + "tokens_seen": 3068329984 + }, + { + "epoch": 0.86, + "learning_rate": 1.41710800834537e-05, + "loss": 2.452, + "theoretical_loss": 3.3150179314993555, + "tokens_seen": 3068461056 + }, + { + "epoch": 0.86, + "learning_rate": 1.416305568929546e-05, + "loss": 2.4895, + "theoretical_loss": 3.315007094166815, + "tokens_seen": 3068592128 + }, + { + "epoch": 0.86, + "learning_rate": 1.4155031295137216e-05, + "loss": 2.4029, + "theoretical_loss": 3.3149962574267784, + "tokens_seen": 3068723200 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1681818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.268106460571289, + "objective/train/theoretical_loss": 3.314985421279188, + "objective/train/tokens_used": 1439313376, + "theoretical_loss": 3.314985421279188, + "tokens_seen": 3068854272 + }, + { + "epoch": 0.86, + "learning_rate": 1.4147006900978976e-05, + "loss": 2.4844, + "theoretical_loss": 3.314985421279188, + "tokens_seen": 3068854272 + }, + { + "epoch": 0.86, + "learning_rate": 1.4138982506820736e-05, + "loss": 2.492, + "theoretical_loss": 3.3149745857239865, + "tokens_seen": 3068985344 + }, + { + "epoch": 0.86, + "learning_rate": 1.4130958112662496e-05, + "loss": 2.3199, + "theoretical_loss": 3.3149637507611156, + "tokens_seen": 3069116416 + }, + { + "epoch": 0.86, + "learning_rate": 1.4122933718504253e-05, + "loss": 2.4783, + "theoretical_loss": 3.314952916390518, + "tokens_seen": 3069247488 + }, + { + "epoch": 0.86, + "learning_rate": 1.4114909324346013e-05, + "loss": 2.571, + "theoretical_loss": 3.314942082612136, + "tokens_seen": 3069378560 + }, + { + "epoch": 0.86, + "learning_rate": 1.4106884930187773e-05, + "loss": 2.5443, + "theoretical_loss": 3.3149312494259116, + "tokens_seen": 3069509632 + }, + { + "epoch": 0.86, + "learning_rate": 1.4098860536029532e-05, + "loss": 2.5285, + "theoretical_loss": 3.3149204168317876, + "tokens_seen": 3069640704 + }, + { + "epoch": 0.86, + "learning_rate": 1.4090836141871289e-05, + "loss": 2.576, + "theoretical_loss": 3.3149095848297065, + "tokens_seen": 3069771776 + }, + { + "epoch": 0.86, + "learning_rate": 1.4082811747713049e-05, + "loss": 2.4494, + "theoretical_loss": 3.31489875341961, + "tokens_seen": 3069902848 + }, + { + "epoch": 0.86, + "learning_rate": 1.4074787353554809e-05, + "loss": 2.5832, + "theoretical_loss": 3.314887922601441, + "tokens_seen": 3070033920 + }, + { + "epoch": 0.86, + "learning_rate": 1.4066762959396565e-05, + "loss": 2.3942, + "theoretical_loss": 3.314877092375142, + "tokens_seen": 3070164992 + }, + { + "epoch": 0.86, + "learning_rate": 1.4058738565238325e-05, + "loss": 2.5213, + "theoretical_loss": 3.3148662627406544, + "tokens_seen": 3070296064 + }, + { + "epoch": 0.86, + "learning_rate": 1.4050714171080085e-05, + "loss": 2.4577, + "theoretical_loss": 3.3148554336979217, + "tokens_seen": 3070427136 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1682472, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5107250213623047, + "objective/train/theoretical_loss": 3.3148500193984454, + "objective/train/tokens_used": 1440951776, + "theoretical_loss": 3.3148500193984454, + "tokens_seen": 3070492672 + }, + { + "epoch": 0.86, + "learning_rate": 1.4042689776921843e-05, + "loss": 2.4559, + "theoretical_loss": 3.3148446052468863, + "tokens_seen": 3070558208 + }, + { + "epoch": 0.86, + "learning_rate": 1.4034665382763602e-05, + "loss": 2.3871, + "theoretical_loss": 3.3148337773874896, + "tokens_seen": 3070689280 + }, + { + "epoch": 0.86, + "learning_rate": 1.4026640988605361e-05, + "loss": 2.5899, + "theoretical_loss": 3.314822950119675, + "tokens_seen": 3070820352 + }, + { + "epoch": 0.86, + "learning_rate": 1.401861659444712e-05, + "loss": 2.5563, + "theoretical_loss": 3.3148121234433843, + "tokens_seen": 3070951424 + }, + { + "epoch": 0.86, + "learning_rate": 1.401059220028888e-05, + "loss": 2.4696, + "theoretical_loss": 3.3148012973585606, + "tokens_seen": 3071082496 + }, + { + "epoch": 0.86, + "learning_rate": 1.4002567806130638e-05, + "loss": 2.3517, + "theoretical_loss": 3.3147904718651455, + "tokens_seen": 3071213568 + }, + { + "epoch": 0.86, + "learning_rate": 1.3994543411972396e-05, + "loss": 2.5484, + "theoretical_loss": 3.3147796469630824, + "tokens_seen": 3071344640 + }, + { + "epoch": 0.86, + "learning_rate": 1.3986519017814156e-05, + "loss": 2.3598, + "theoretical_loss": 3.314768822652313, + "tokens_seen": 3071475712 + }, + { + "epoch": 0.86, + "learning_rate": 1.3978494623655914e-05, + "loss": 2.4922, + "theoretical_loss": 3.3147579989327802, + "tokens_seen": 3071606784 + }, + { + "epoch": 0.86, + "learning_rate": 1.3970470229497672e-05, + "loss": 2.5397, + "theoretical_loss": 3.3147471758044262, + "tokens_seen": 3071737856 + }, + { + "epoch": 0.86, + "learning_rate": 1.3962445835339432e-05, + "loss": 2.5214, + "theoretical_loss": 3.3147363532671936, + "tokens_seen": 3071868928 + }, + { + "epoch": 0.86, + "learning_rate": 1.3954421441181192e-05, + "loss": 2.4816, + "theoretical_loss": 3.3147255313210247, + "tokens_seen": 3072000000 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1683394, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.594682216644287, + "objective/train/theoretical_loss": 3.3147147099658625, + "objective/train/tokens_used": 1442590176, + "theoretical_loss": 3.3147147099658625, + "tokens_seen": 3072131072 + }, + { + "epoch": 0.86, + "learning_rate": 1.3946397047022949e-05, + "loss": 2.5986, + "theoretical_loss": 3.3147147099658625, + "tokens_seen": 3072131072 + }, + { + "epoch": 0.86, + "learning_rate": 1.3938372652864709e-05, + "loss": 2.5196, + "theoretical_loss": 3.3147038892016494, + "tokens_seen": 3072262144 + }, + { + "epoch": 0.86, + "learning_rate": 1.3930348258706468e-05, + "loss": 2.5555, + "theoretical_loss": 3.3146930690283276, + "tokens_seen": 3072393216 + }, + { + "epoch": 0.86, + "learning_rate": 1.3922323864548228e-05, + "loss": 2.5361, + "theoretical_loss": 3.3146822494458394, + "tokens_seen": 3072524288 + }, + { + "epoch": 0.86, + "learning_rate": 1.3914299470389985e-05, + "loss": 2.5483, + "theoretical_loss": 3.3146714304541285, + "tokens_seen": 3072655360 + }, + { + "epoch": 0.86, + "learning_rate": 1.3906275076231745e-05, + "loss": 2.5655, + "theoretical_loss": 3.314660612053136, + "tokens_seen": 3072786432 + }, + { + "epoch": 0.86, + "learning_rate": 1.3898250682073505e-05, + "loss": 2.5168, + "theoretical_loss": 3.314649794242805, + "tokens_seen": 3072917504 + }, + { + "epoch": 0.86, + "learning_rate": 1.3890226287915261e-05, + "loss": 2.6042, + "theoretical_loss": 3.314638977023079, + "tokens_seen": 3073048576 + }, + { + "epoch": 0.86, + "learning_rate": 1.3882201893757021e-05, + "loss": 2.5633, + "theoretical_loss": 3.314628160393899, + "tokens_seen": 3073179648 + }, + { + "epoch": 0.86, + "learning_rate": 1.3874177499598781e-05, + "loss": 2.642, + "theoretical_loss": 3.3146173443552085, + "tokens_seen": 3073310720 + }, + { + "epoch": 0.86, + "learning_rate": 1.3866153105440541e-05, + "loss": 2.4514, + "theoretical_loss": 3.3146065289069497, + "tokens_seen": 3073441792 + }, + { + "epoch": 0.86, + "learning_rate": 1.3858128711282297e-05, + "loss": 2.4957, + "theoretical_loss": 3.3145957140490654, + "tokens_seen": 3073572864 + }, + { + "epoch": 0.86, + "learning_rate": 1.3850104317124057e-05, + "loss": 2.6561, + "theoretical_loss": 3.3145848997814986, + "tokens_seen": 3073703936 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1684169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4573862552642822, + "objective/train/theoretical_loss": 3.3145794928690657, + "objective/train/tokens_used": 1444228576, + "theoretical_loss": 3.3145794928690657, + "tokens_seen": 3073769472 + }, + { + "epoch": 0.86, + "learning_rate": 1.3842079922965817e-05, + "loss": 2.4924, + "theoretical_loss": 3.314574086104191, + "tokens_seen": 3073835008 + }, + { + "epoch": 0.86, + "learning_rate": 1.3834055528807577e-05, + "loss": 2.6227, + "theoretical_loss": 3.314563273017086, + "tokens_seen": 3073966080 + }, + { + "epoch": 0.86, + "learning_rate": 1.3826031134649334e-05, + "loss": 2.5439, + "theoretical_loss": 3.314552460520126, + "tokens_seen": 3074097152 + }, + { + "epoch": 0.86, + "learning_rate": 1.3818006740491094e-05, + "loss": 2.5019, + "theoretical_loss": 3.314541648613253, + "tokens_seen": 3074228224 + }, + { + "epoch": 0.86, + "learning_rate": 1.3809982346332853e-05, + "loss": 2.436, + "theoretical_loss": 3.314530837296411, + "tokens_seen": 3074359296 + }, + { + "epoch": 0.86, + "learning_rate": 1.3801957952174613e-05, + "loss": 2.5657, + "theoretical_loss": 3.314520026569541, + "tokens_seen": 3074490368 + }, + { + "epoch": 0.86, + "learning_rate": 1.379393355801637e-05, + "loss": 2.6493, + "theoretical_loss": 3.3145092164325867, + "tokens_seen": 3074621440 + }, + { + "epoch": 0.86, + "learning_rate": 1.378590916385813e-05, + "loss": 2.6587, + "theoretical_loss": 3.314498406885491, + "tokens_seen": 3074752512 + }, + { + "epoch": 0.86, + "learning_rate": 1.377788476969989e-05, + "loss": 2.6773, + "theoretical_loss": 3.314487597928196, + "tokens_seen": 3074883584 + }, + { + "epoch": 0.86, + "learning_rate": 1.3769860375541646e-05, + "loss": 2.7063, + "theoretical_loss": 3.3144767895606444, + "tokens_seen": 3075014656 + }, + { + "epoch": 0.86, + "learning_rate": 1.3761835981383406e-05, + "loss": 2.6697, + "theoretical_loss": 3.314465981782779, + "tokens_seen": 3075145728 + }, + { + "epoch": 0.86, + "learning_rate": 1.3753811587225166e-05, + "loss": 2.6753, + "theoretical_loss": 3.3144551745945425, + "tokens_seen": 3075276800 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1684867, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.679647922515869, + "objective/train/theoretical_loss": 3.3144443679958773, + "objective/train/tokens_used": 1445866976, + "theoretical_loss": 3.3144443679958773, + "tokens_seen": 3075407872 + }, + { + "epoch": 0.86, + "learning_rate": 1.3745787193066926e-05, + "loss": 2.4399, + "theoretical_loss": 3.3144443679958773, + "tokens_seen": 3075407872 + }, + { + "epoch": 0.86, + "learning_rate": 1.3737762798908682e-05, + "loss": 2.5885, + "theoretical_loss": 3.3144335619867267, + "tokens_seen": 3075538944 + }, + { + "epoch": 0.86, + "learning_rate": 1.3729738404750442e-05, + "loss": 2.4694, + "theoretical_loss": 3.3144227565670334, + "tokens_seen": 3075670016 + }, + { + "epoch": 0.86, + "learning_rate": 1.3721714010592202e-05, + "loss": 2.6594, + "theoretical_loss": 3.3144119517367394, + "tokens_seen": 3075801088 + }, + { + "epoch": 0.86, + "learning_rate": 1.371368961643396e-05, + "loss": 2.5288, + "theoretical_loss": 3.314401147495788, + "tokens_seen": 3075932160 + }, + { + "epoch": 0.86, + "learning_rate": 1.3705665222275719e-05, + "loss": 2.591, + "theoretical_loss": 3.3143903438441225, + "tokens_seen": 3076063232 + }, + { + "epoch": 0.86, + "learning_rate": 1.3697640828117479e-05, + "loss": 2.5402, + "theoretical_loss": 3.314379540781684, + "tokens_seen": 3076194304 + }, + { + "epoch": 0.86, + "learning_rate": 1.3689616433959237e-05, + "loss": 2.5118, + "theoretical_loss": 3.314368738308417, + "tokens_seen": 3076325376 + }, + { + "epoch": 0.86, + "learning_rate": 1.3681592039800995e-05, + "loss": 2.7106, + "theoretical_loss": 3.314357936424263, + "tokens_seen": 3076456448 + }, + { + "epoch": 0.86, + "learning_rate": 1.3673567645642755e-05, + "loss": 2.5054, + "theoretical_loss": 3.3143471351291653, + "tokens_seen": 3076587520 + }, + { + "epoch": 0.86, + "learning_rate": 1.3665543251484513e-05, + "loss": 2.4516, + "theoretical_loss": 3.314336334423067, + "tokens_seen": 3076718592 + }, + { + "epoch": 0.86, + "learning_rate": 1.3657518857326273e-05, + "loss": 2.6456, + "theoretical_loss": 3.3143255343059104, + "tokens_seen": 3076849664 + }, + { + "epoch": 0.86, + "learning_rate": 1.3649494463168031e-05, + "loss": 2.3957, + "theoretical_loss": 3.3143147347776383, + "tokens_seen": 3076980736 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1686119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.411834478378296, + "objective/train/theoretical_loss": 3.3143093352343165, + "objective/train/tokens_used": 1447505376, + "theoretical_loss": 3.3143093352343165, + "tokens_seen": 3077046272 + }, + { + "epoch": 0.86, + "learning_rate": 1.364147006900979e-05, + "loss": 2.4902, + "theoretical_loss": 3.314303935838194, + "tokens_seen": 3077111808 + }, + { + "epoch": 0.87, + "learning_rate": 1.363344567485155e-05, + "loss": 2.6203, + "theoretical_loss": 3.3142931374875197, + "tokens_seen": 3077242880 + }, + { + "epoch": 0.87, + "learning_rate": 1.362542128069331e-05, + "loss": 2.5606, + "theoretical_loss": 3.3142823397255587, + "tokens_seen": 3077373952 + }, + { + "epoch": 0.87, + "learning_rate": 1.3617396886535066e-05, + "loss": 2.5854, + "theoretical_loss": 3.3142715425522535, + "tokens_seen": 3077505024 + }, + { + "epoch": 0.87, + "learning_rate": 1.3609372492376826e-05, + "loss": 2.5253, + "theoretical_loss": 3.3142607459675473, + "tokens_seen": 3077636096 + }, + { + "epoch": 0.87, + "learning_rate": 1.3601348098218586e-05, + "loss": 2.444, + "theoretical_loss": 3.3142499499713827, + "tokens_seen": 3077767168 + }, + { + "epoch": 0.87, + "learning_rate": 1.3593323704060342e-05, + "loss": 2.5814, + "theoretical_loss": 3.3142391545637024, + "tokens_seen": 3077898240 + }, + { + "epoch": 0.87, + "learning_rate": 1.3585299309902102e-05, + "loss": 2.5994, + "theoretical_loss": 3.3142283597444493, + "tokens_seen": 3078029312 + }, + { + "epoch": 0.87, + "learning_rate": 1.3577274915743862e-05, + "loss": 2.6163, + "theoretical_loss": 3.314217565513567, + "tokens_seen": 3078160384 + }, + { + "epoch": 0.87, + "learning_rate": 1.3569250521585622e-05, + "loss": 2.7563, + "theoretical_loss": 3.3142067718709973, + "tokens_seen": 3078291456 + }, + { + "epoch": 0.87, + "learning_rate": 1.3561226127427378e-05, + "loss": 2.5761, + "theoretical_loss": 3.3141959788166835, + "tokens_seen": 3078422528 + }, + { + "epoch": 0.87, + "learning_rate": 1.3553201733269138e-05, + "loss": 2.5992, + "theoretical_loss": 3.314185186350569, + "tokens_seen": 3078553600 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1686853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.854732036590576, + "objective/train/theoretical_loss": 3.3141743944725963, + "objective/train/tokens_used": 1449143776, + "theoretical_loss": 3.3141743944725963, + "tokens_seen": 3078684672 + }, + { + "epoch": 0.87, + "learning_rate": 1.3545177339110898e-05, + "loss": 2.7221, + "theoretical_loss": 3.3141743944725963, + "tokens_seen": 3078684672 + }, + { + "epoch": 0.87, + "learning_rate": 1.3537152944952658e-05, + "loss": 2.7556, + "theoretical_loss": 3.314163603182708, + "tokens_seen": 3078815744 + }, + { + "epoch": 0.87, + "learning_rate": 1.3529128550794415e-05, + "loss": 2.3897, + "theoretical_loss": 3.3141528124808475, + "tokens_seen": 3078946816 + }, + { + "epoch": 0.87, + "learning_rate": 1.3521104156636175e-05, + "loss": 2.6311, + "theoretical_loss": 3.3141420223669575, + "tokens_seen": 3079077888 + }, + { + "epoch": 0.87, + "learning_rate": 1.3513079762477934e-05, + "loss": 2.5274, + "theoretical_loss": 3.314131232840981, + "tokens_seen": 3079208960 + }, + { + "epoch": 0.87, + "learning_rate": 1.3505055368319691e-05, + "loss": 2.5689, + "theoretical_loss": 3.3141204439028606, + "tokens_seen": 3079340032 + }, + { + "epoch": 0.87, + "learning_rate": 1.3497030974161451e-05, + "loss": 2.605, + "theoretical_loss": 3.31410965555254, + "tokens_seen": 3079471104 + }, + { + "epoch": 0.87, + "learning_rate": 1.348900658000321e-05, + "loss": 2.5474, + "theoretical_loss": 3.3140988677899617, + "tokens_seen": 3079602176 + }, + { + "epoch": 0.87, + "learning_rate": 1.348098218584497e-05, + "loss": 2.691, + "theoretical_loss": 3.3140880806150683, + "tokens_seen": 3079733248 + }, + { + "epoch": 0.87, + "learning_rate": 1.3472957791686727e-05, + "loss": 2.5329, + "theoretical_loss": 3.3140772940278036, + "tokens_seen": 3079864320 + }, + { + "epoch": 0.87, + "learning_rate": 1.3464933397528487e-05, + "loss": 2.5694, + "theoretical_loss": 3.31406650802811, + "tokens_seen": 3079995392 + }, + { + "epoch": 0.87, + "learning_rate": 1.3456909003370247e-05, + "loss": 2.6529, + "theoretical_loss": 3.314055722615931, + "tokens_seen": 3080126464 + }, + { + "epoch": 0.87, + "learning_rate": 1.3448884609212007e-05, + "loss": 2.6881, + "theoretical_loss": 3.3140449377912087, + "tokens_seen": 3080257536 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1688263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.429612636566162, + "objective/train/theoretical_loss": 3.314039545599126, + "objective/train/tokens_used": 1450782176, + "theoretical_loss": 3.314039545599126, + "tokens_seen": 3080323072 + }, + { + "epoch": 0.87, + "learning_rate": 1.3440860215053763e-05, + "loss": 2.5227, + "theoretical_loss": 3.314034153553887, + "tokens_seen": 3080388608 + }, + { + "epoch": 0.87, + "learning_rate": 1.3432835820895523e-05, + "loss": 2.5157, + "theoretical_loss": 3.314023369903908, + "tokens_seen": 3080519680 + }, + { + "epoch": 0.87, + "learning_rate": 1.3424811426737283e-05, + "loss": 2.6338, + "theoretical_loss": 3.314012586841216, + "tokens_seen": 3080650752 + }, + { + "epoch": 0.87, + "learning_rate": 1.341678703257904e-05, + "loss": 2.5241, + "theoretical_loss": 3.3140018043657533, + "tokens_seen": 3080781824 + }, + { + "epoch": 0.87, + "learning_rate": 1.34087626384208e-05, + "loss": 2.6226, + "theoretical_loss": 3.3139910224774627, + "tokens_seen": 3080912896 + }, + { + "epoch": 0.87, + "learning_rate": 1.340073824426256e-05, + "loss": 2.5899, + "theoretical_loss": 3.3139802411762873, + "tokens_seen": 3081043968 + }, + { + "epoch": 0.87, + "learning_rate": 1.339271385010432e-05, + "loss": 2.6454, + "theoretical_loss": 3.313969460462171, + "tokens_seen": 3081175040 + }, + { + "epoch": 0.87, + "learning_rate": 1.3384689455946076e-05, + "loss": 2.4173, + "theoretical_loss": 3.3139586803350554, + "tokens_seen": 3081306112 + }, + { + "epoch": 0.87, + "learning_rate": 1.3376665061787836e-05, + "loss": 2.5636, + "theoretical_loss": 3.3139479007948847, + "tokens_seen": 3081437184 + }, + { + "epoch": 0.87, + "learning_rate": 1.3368640667629596e-05, + "loss": 2.4222, + "theoretical_loss": 3.313937121841602, + "tokens_seen": 3081568256 + }, + { + "epoch": 0.87, + "learning_rate": 1.3360616273471354e-05, + "loss": 2.4218, + "theoretical_loss": 3.31392634347515, + "tokens_seen": 3081699328 + }, + { + "epoch": 0.87, + "learning_rate": 1.3352591879313112e-05, + "loss": 2.5662, + "theoretical_loss": 3.3139155656954715, + "tokens_seen": 3081830400 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1688735, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3998351097106934, + "objective/train/theoretical_loss": 3.3139047885025104, + "objective/train/tokens_used": 1452420576, + "theoretical_loss": 3.3139047885025104, + "tokens_seen": 3081961472 + }, + { + "epoch": 0.87, + "learning_rate": 1.3344567485154872e-05, + "loss": 2.4915, + "theoretical_loss": 3.3139047885025104, + "tokens_seen": 3081961472 + }, + { + "epoch": 0.87, + "learning_rate": 1.333654309099663e-05, + "loss": 2.5263, + "theoretical_loss": 3.313894011896209, + "tokens_seen": 3082092544 + }, + { + "epoch": 0.87, + "learning_rate": 1.332851869683839e-05, + "loss": 2.4879, + "theoretical_loss": 3.313883235876511, + "tokens_seen": 3082223616 + }, + { + "epoch": 0.87, + "learning_rate": 1.3320494302680148e-05, + "loss": 2.6281, + "theoretical_loss": 3.3138724604433594, + "tokens_seen": 3082354688 + }, + { + "epoch": 0.87, + "learning_rate": 1.3312469908521907e-05, + "loss": 2.3639, + "theoretical_loss": 3.313861685596697, + "tokens_seen": 3082485760 + }, + { + "epoch": 0.87, + "learning_rate": 1.3304445514363667e-05, + "loss": 2.5163, + "theoretical_loss": 3.313850911336467, + "tokens_seen": 3082616832 + }, + { + "epoch": 0.87, + "learning_rate": 1.3296421120205425e-05, + "loss": 2.4787, + "theoretical_loss": 3.313840137662613, + "tokens_seen": 3082747904 + }, + { + "epoch": 0.87, + "learning_rate": 1.3288396726047183e-05, + "loss": 2.5259, + "theoretical_loss": 3.3138293645750783, + "tokens_seen": 3082878976 + }, + { + "epoch": 0.87, + "learning_rate": 1.3280372331888943e-05, + "loss": 2.694, + "theoretical_loss": 3.3138185920738055, + "tokens_seen": 3083010048 + }, + { + "epoch": 0.87, + "learning_rate": 1.3272347937730703e-05, + "loss": 2.6132, + "theoretical_loss": 3.3138078201587375, + "tokens_seen": 3083141120 + }, + { + "epoch": 0.87, + "learning_rate": 1.326432354357246e-05, + "loss": 2.6371, + "theoretical_loss": 3.313797048829818, + "tokens_seen": 3083272192 + }, + { + "epoch": 0.87, + "learning_rate": 1.325629914941422e-05, + "loss": 2.6071, + "theoretical_loss": 3.3137862780869902, + "tokens_seen": 3083403264 + }, + { + "epoch": 0.87, + "learning_rate": 1.3248274755255979e-05, + "loss": 2.6976, + "theoretical_loss": 3.313775507930197, + "tokens_seen": 3083534336 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1689720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0052433013916016, + "objective/train/theoretical_loss": 3.313770123071546, + "objective/train/tokens_used": 1454058976, + "theoretical_loss": 3.313770123071546, + "tokens_seen": 3083599872 + }, + { + "epoch": 0.87, + "learning_rate": 1.3240250361097739e-05, + "loss": 2.7115, + "theoretical_loss": 3.3137647383593825, + "tokens_seen": 3083665408 + }, + { + "epoch": 0.87, + "learning_rate": 1.3232225966939496e-05, + "loss": 2.6481, + "theoretical_loss": 3.313753969374489, + "tokens_seen": 3083796480 + }, + { + "epoch": 0.87, + "learning_rate": 1.3224201572781255e-05, + "loss": 2.5421, + "theoretical_loss": 3.3137432009754595, + "tokens_seen": 3083927552 + }, + { + "epoch": 0.87, + "learning_rate": 1.3216177178623015e-05, + "loss": 2.6339, + "theoretical_loss": 3.313732433162238, + "tokens_seen": 3084058624 + }, + { + "epoch": 0.87, + "learning_rate": 1.3208152784464772e-05, + "loss": 2.5555, + "theoretical_loss": 3.313721665934767, + "tokens_seen": 3084189696 + }, + { + "epoch": 0.87, + "learning_rate": 1.3200128390306532e-05, + "loss": 2.5437, + "theoretical_loss": 3.3137108992929907, + "tokens_seen": 3084320768 + }, + { + "epoch": 0.87, + "learning_rate": 1.3192103996148292e-05, + "loss": 2.553, + "theoretical_loss": 3.3137001332368516, + "tokens_seen": 3084451840 + }, + { + "epoch": 0.87, + "learning_rate": 1.3184079601990052e-05, + "loss": 2.5865, + "theoretical_loss": 3.313689367766293, + "tokens_seen": 3084582912 + }, + { + "epoch": 0.87, + "learning_rate": 1.3176055207831808e-05, + "loss": 2.4876, + "theoretical_loss": 3.3136786028812586, + "tokens_seen": 3084713984 + }, + { + "epoch": 0.87, + "learning_rate": 1.3168030813673568e-05, + "loss": 2.4894, + "theoretical_loss": 3.3136678385816913, + "tokens_seen": 3084845056 + }, + { + "epoch": 0.87, + "learning_rate": 1.3160006419515328e-05, + "loss": 2.553, + "theoretical_loss": 3.3136570748675345, + "tokens_seen": 3084976128 + }, + { + "epoch": 0.87, + "learning_rate": 1.3151982025357088e-05, + "loss": 2.5838, + "theoretical_loss": 3.313646311738731, + "tokens_seen": 3085107200 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1690993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7996625900268555, + "objective/train/theoretical_loss": 3.3136355491952254, + "objective/train/tokens_used": 1455697376, + "theoretical_loss": 3.3136355491952254, + "tokens_seen": 3085238272 + }, + { + "epoch": 0.87, + "learning_rate": 1.3143957631198844e-05, + "loss": 2.6144, + "theoretical_loss": 3.3136355491952254, + "tokens_seen": 3085238272 + }, + { + "epoch": 0.87, + "learning_rate": 1.3135933237040604e-05, + "loss": 2.5368, + "theoretical_loss": 3.31362478723696, + "tokens_seen": 3085369344 + }, + { + "epoch": 0.87, + "learning_rate": 1.3127908842882364e-05, + "loss": 2.4758, + "theoretical_loss": 3.313614025863878, + "tokens_seen": 3085500416 + }, + { + "epoch": 0.87, + "learning_rate": 1.311988444872412e-05, + "loss": 2.6187, + "theoretical_loss": 3.3136032650759235, + "tokens_seen": 3085631488 + }, + { + "epoch": 0.87, + "learning_rate": 1.311186005456588e-05, + "loss": 2.5921, + "theoretical_loss": 3.313592504873039, + "tokens_seen": 3085762560 + }, + { + "epoch": 0.87, + "learning_rate": 1.310383566040764e-05, + "loss": 2.5815, + "theoretical_loss": 3.3135817452551684, + "tokens_seen": 3085893632 + }, + { + "epoch": 0.87, + "learning_rate": 1.30958112662494e-05, + "loss": 2.5357, + "theoretical_loss": 3.3135709862222544, + "tokens_seen": 3086024704 + }, + { + "epoch": 0.87, + "learning_rate": 1.3087786872091157e-05, + "loss": 2.4781, + "theoretical_loss": 3.3135602277742415, + "tokens_seen": 3086155776 + }, + { + "epoch": 0.87, + "learning_rate": 1.3079762477932917e-05, + "loss": 2.5106, + "theoretical_loss": 3.313549469911072, + "tokens_seen": 3086286848 + }, + { + "epoch": 0.87, + "learning_rate": 1.3071738083774677e-05, + "loss": 2.3946, + "theoretical_loss": 3.3135387126326896, + "tokens_seen": 3086417920 + }, + { + "epoch": 0.87, + "learning_rate": 1.3063713689616437e-05, + "loss": 2.6383, + "theoretical_loss": 3.313527955939038, + "tokens_seen": 3086548992 + }, + { + "epoch": 0.87, + "learning_rate": 1.3055689295458193e-05, + "loss": 2.5887, + "theoretical_loss": 3.31351719983006, + "tokens_seen": 3086680064 + }, + { + "epoch": 0.87, + "learning_rate": 1.3047664901299953e-05, + "loss": 2.533, + "theoretical_loss": 3.3135064443056996, + "tokens_seen": 3086811136 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1691628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4921112060546875, + "objective/train/theoretical_loss": 3.313501066762733, + "objective/train/tokens_used": 1457335776, + "theoretical_loss": 3.313501066762733, + "tokens_seen": 3086876672 + }, + { + "epoch": 0.87, + "learning_rate": 1.3039640507141713e-05, + "loss": 2.6214, + "theoretical_loss": 3.3134956893658996, + "tokens_seen": 3086942208 + }, + { + "epoch": 0.87, + "learning_rate": 1.303161611298347e-05, + "loss": 2.6238, + "theoretical_loss": 3.313484935010604, + "tokens_seen": 3087073280 + }, + { + "epoch": 0.87, + "learning_rate": 1.302359171882523e-05, + "loss": 2.4525, + "theoretical_loss": 3.313474181239756, + "tokens_seen": 3087204352 + }, + { + "epoch": 0.87, + "learning_rate": 1.3015567324666988e-05, + "loss": 2.555, + "theoretical_loss": 3.3134634280532986, + "tokens_seen": 3087335424 + }, + { + "epoch": 0.87, + "learning_rate": 1.3007542930508748e-05, + "loss": 2.6606, + "theoretical_loss": 3.3134526754511757, + "tokens_seen": 3087466496 + }, + { + "epoch": 0.87, + "learning_rate": 1.2999518536350506e-05, + "loss": 2.6111, + "theoretical_loss": 3.3134419234333308, + "tokens_seen": 3087597568 + }, + { + "epoch": 0.87, + "learning_rate": 1.2991494142192264e-05, + "loss": 2.7454, + "theoretical_loss": 3.3134311719997074, + "tokens_seen": 3087728640 + }, + { + "epoch": 0.87, + "learning_rate": 1.2983469748034024e-05, + "loss": 2.5254, + "theoretical_loss": 3.3134204211502487, + "tokens_seen": 3087859712 + }, + { + "epoch": 0.87, + "learning_rate": 1.2975445353875784e-05, + "loss": 2.4625, + "theoretical_loss": 3.313409670884898, + "tokens_seen": 3087990784 + }, + { + "epoch": 0.87, + "learning_rate": 1.296742095971754e-05, + "loss": 2.622, + "theoretical_loss": 3.313398921203599, + "tokens_seen": 3088121856 + }, + { + "epoch": 0.87, + "learning_rate": 1.29593965655593e-05, + "loss": 2.6915, + "theoretical_loss": 3.313388172106295, + "tokens_seen": 3088252928 + }, + { + "epoch": 0.87, + "learning_rate": 1.295137217140106e-05, + "loss": 2.602, + "theoretical_loss": 3.3133774235929296, + "tokens_seen": 3088384000 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1692791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.525712490081787, + "objective/train/theoretical_loss": 3.313366675663447, + "objective/train/tokens_used": 1458974176, + "theoretical_loss": 3.313366675663447, + "tokens_seen": 3088515072 + }, + { + "epoch": 0.87, + "learning_rate": 1.2943347777242817e-05, + "loss": 2.5719, + "theoretical_loss": 3.313366675663447, + "tokens_seen": 3088515072 + }, + { + "epoch": 0.87, + "learning_rate": 1.2935323383084577e-05, + "loss": 2.6227, + "theoretical_loss": 3.3133559283177894, + "tokens_seen": 3088646144 + }, + { + "epoch": 0.87, + "learning_rate": 1.2927298988926336e-05, + "loss": 2.4152, + "theoretical_loss": 3.3133451815559014, + "tokens_seen": 3088777216 + }, + { + "epoch": 0.87, + "learning_rate": 1.2919274594768096e-05, + "loss": 2.5023, + "theoretical_loss": 3.313334435377726, + "tokens_seen": 3088908288 + }, + { + "epoch": 0.87, + "learning_rate": 1.2911250200609853e-05, + "loss": 2.6108, + "theoretical_loss": 3.3133236897832066, + "tokens_seen": 3089039360 + }, + { + "epoch": 0.87, + "learning_rate": 1.2903225806451613e-05, + "loss": 2.6461, + "theoretical_loss": 3.3133129447722873, + "tokens_seen": 3089170432 + }, + { + "epoch": 0.87, + "learning_rate": 1.2895201412293373e-05, + "loss": 2.6468, + "theoretical_loss": 3.313302200344911, + "tokens_seen": 3089301504 + }, + { + "epoch": 0.87, + "learning_rate": 1.2887177018135133e-05, + "loss": 2.6447, + "theoretical_loss": 3.313291456501022, + "tokens_seen": 3089432576 + }, + { + "epoch": 0.87, + "learning_rate": 1.2879152623976889e-05, + "loss": 2.4829, + "theoretical_loss": 3.3132807132405633, + "tokens_seen": 3089563648 + }, + { + "epoch": 0.87, + "learning_rate": 1.2871128229818649e-05, + "loss": 2.6226, + "theoretical_loss": 3.3132699705634785, + "tokens_seen": 3089694720 + }, + { + "epoch": 0.87, + "learning_rate": 1.2863103835660409e-05, + "loss": 2.6299, + "theoretical_loss": 3.313259228469711, + "tokens_seen": 3089825792 + }, + { + "epoch": 0.87, + "learning_rate": 1.2855079441502169e-05, + "loss": 2.4978, + "theoretical_loss": 3.3132484869592047, + "tokens_seen": 3089956864 + }, + { + "epoch": 0.87, + "learning_rate": 1.2847055047343925e-05, + "loss": 2.5865, + "theoretical_loss": 3.3132377460319034, + "tokens_seen": 3090087936 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1693139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8856472969055176, + "objective/train/theoretical_loss": 3.313232375786937, + "objective/train/tokens_used": 1460612576, + "theoretical_loss": 3.313232375786937, + "tokens_seen": 3090153472 + }, + { + "epoch": 0.87, + "learning_rate": 1.2839030653185685e-05, + "loss": 2.715, + "theoretical_loss": 3.31322700568775, + "tokens_seen": 3090219008 + }, + { + "epoch": 0.87, + "learning_rate": 1.2831006259027445e-05, + "loss": 2.5266, + "theoretical_loss": 3.3132162659266893, + "tokens_seen": 3090350080 + }, + { + "epoch": 0.87, + "learning_rate": 1.2822981864869202e-05, + "loss": 2.6794, + "theoretical_loss": 3.3132055267486638, + "tokens_seen": 3090481152 + }, + { + "epoch": 0.87, + "learning_rate": 1.2814957470710962e-05, + "loss": 2.6612, + "theoretical_loss": 3.3131947881536172, + "tokens_seen": 3090612224 + }, + { + "epoch": 0.87, + "learning_rate": 1.2806933076552721e-05, + "loss": 2.5723, + "theoretical_loss": 3.3131840501414938, + "tokens_seen": 3090743296 + }, + { + "epoch": 0.87, + "learning_rate": 1.2798908682394481e-05, + "loss": 2.7091, + "theoretical_loss": 3.313173312712237, + "tokens_seen": 3090874368 + }, + { + "epoch": 0.87, + "learning_rate": 1.2790884288236238e-05, + "loss": 2.7139, + "theoretical_loss": 3.31316257586579, + "tokens_seen": 3091005440 + }, + { + "epoch": 0.87, + "learning_rate": 1.2782859894077998e-05, + "loss": 2.6649, + "theoretical_loss": 3.3131518396020967, + "tokens_seen": 3091136512 + }, + { + "epoch": 0.87, + "learning_rate": 1.2774835499919758e-05, + "loss": 2.5484, + "theoretical_loss": 3.313141103921101, + "tokens_seen": 3091267584 + }, + { + "epoch": 0.87, + "learning_rate": 1.2766811105761518e-05, + "loss": 2.5596, + "theoretical_loss": 3.3131303688227463, + "tokens_seen": 3091398656 + }, + { + "epoch": 0.87, + "learning_rate": 1.2758786711603274e-05, + "loss": 2.6279, + "theoretical_loss": 3.313119634306976, + "tokens_seen": 3091529728 + }, + { + "epoch": 0.87, + "learning_rate": 1.2750762317445034e-05, + "loss": 2.6194, + "theoretical_loss": 3.313108900373735, + "tokens_seen": 3091660800 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1694249, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.546267509460449, + "objective/train/theoretical_loss": 3.3130981670229653, + "objective/train/tokens_used": 1462250976, + "theoretical_loss": 3.3130981670229653, + "tokens_seen": 3091791872 + }, + { + "epoch": 0.87, + "learning_rate": 1.2742737923286794e-05, + "loss": 2.5968, + "theoretical_loss": 3.3130981670229653, + "tokens_seen": 3091791872 + }, + { + "epoch": 0.87, + "learning_rate": 1.273471352912855e-05, + "loss": 2.5337, + "theoretical_loss": 3.3130874342546117, + "tokens_seen": 3091922944 + }, + { + "epoch": 0.87, + "learning_rate": 1.272668913497031e-05, + "loss": 2.5643, + "theoretical_loss": 3.313076702068618, + "tokens_seen": 3092054016 + }, + { + "epoch": 0.87, + "learning_rate": 1.271866474081207e-05, + "loss": 2.5365, + "theoretical_loss": 3.313065970464927, + "tokens_seen": 3092185088 + }, + { + "epoch": 0.87, + "learning_rate": 1.2710640346653828e-05, + "loss": 2.4118, + "theoretical_loss": 3.3130552394434836, + "tokens_seen": 3092316160 + }, + { + "epoch": 0.87, + "learning_rate": 1.2702615952495587e-05, + "loss": 2.7855, + "theoretical_loss": 3.313044509004231, + "tokens_seen": 3092447232 + }, + { + "epoch": 0.87, + "learning_rate": 1.2694591558337347e-05, + "loss": 2.6017, + "theoretical_loss": 3.313033779147112, + "tokens_seen": 3092578304 + }, + { + "epoch": 0.87, + "learning_rate": 1.2686567164179105e-05, + "loss": 2.7318, + "theoretical_loss": 3.313023049872072, + "tokens_seen": 3092709376 + }, + { + "epoch": 0.87, + "learning_rate": 1.2678542770020865e-05, + "loss": 2.5262, + "theoretical_loss": 3.3130123211790536, + "tokens_seen": 3092840448 + }, + { + "epoch": 0.87, + "learning_rate": 1.2670518375862623e-05, + "loss": 2.555, + "theoretical_loss": 3.3130015930680012, + "tokens_seen": 3092971520 + }, + { + "epoch": 0.87, + "learning_rate": 1.2662493981704381e-05, + "loss": 2.641, + "theoretical_loss": 3.3129908655388585, + "tokens_seen": 3093102592 + }, + { + "epoch": 0.87, + "learning_rate": 1.2654469587546141e-05, + "loss": 2.6281, + "theoretical_loss": 3.3129801385915685, + "tokens_seen": 3093233664 + }, + { + "epoch": 0.87, + "learning_rate": 1.26464451933879e-05, + "loss": 2.6426, + "theoretical_loss": 3.3129694122260758, + "tokens_seen": 3093364736 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1695060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.517820358276367, + "objective/train/theoretical_loss": 3.312964049261486, + "objective/train/tokens_used": 1463889376, + "theoretical_loss": 3.312964049261486, + "tokens_seen": 3093430272 + }, + { + "epoch": 0.87, + "learning_rate": 1.2638420799229657e-05, + "loss": 2.6531, + "theoretical_loss": 3.3129586864423244, + "tokens_seen": 3093495808 + }, + { + "epoch": 0.87, + "learning_rate": 1.2630396405071417e-05, + "loss": 2.5879, + "theoretical_loss": 3.312947961240257, + "tokens_seen": 3093626880 + }, + { + "epoch": 0.88, + "learning_rate": 1.2622372010913177e-05, + "loss": 2.6323, + "theoretical_loss": 3.3129372366198186, + "tokens_seen": 3093757952 + }, + { + "epoch": 0.88, + "learning_rate": 1.2614347616754934e-05, + "loss": 2.5436, + "theoretical_loss": 3.3129265125809524, + "tokens_seen": 3093889024 + }, + { + "epoch": 0.88, + "learning_rate": 1.2606323222596694e-05, + "loss": 2.7345, + "theoretical_loss": 3.3129157891236023, + "tokens_seen": 3094020096 + }, + { + "epoch": 0.88, + "learning_rate": 1.2598298828438454e-05, + "loss": 2.6091, + "theoretical_loss": 3.312905066247712, + "tokens_seen": 3094151168 + }, + { + "epoch": 0.88, + "learning_rate": 1.2590274434280214e-05, + "loss": 2.6091, + "theoretical_loss": 3.312894343953226, + "tokens_seen": 3094282240 + }, + { + "epoch": 0.88, + "learning_rate": 1.258225004012197e-05, + "loss": 2.6067, + "theoretical_loss": 3.312883622240087, + "tokens_seen": 3094413312 + }, + { + "epoch": 0.88, + "learning_rate": 1.257422564596373e-05, + "loss": 2.6258, + "theoretical_loss": 3.3128729011082396, + "tokens_seen": 3094544384 + }, + { + "epoch": 0.88, + "learning_rate": 1.256620125180549e-05, + "loss": 2.6245, + "theoretical_loss": 3.312862180557628, + "tokens_seen": 3094675456 + }, + { + "epoch": 0.88, + "learning_rate": 1.2558176857647246e-05, + "loss": 2.4965, + "theoretical_loss": 3.3128514605881954, + "tokens_seen": 3094806528 + }, + { + "epoch": 0.88, + "learning_rate": 1.2550152463489006e-05, + "loss": 2.7031, + "theoretical_loss": 3.312840741199886, + "tokens_seen": 3094937600 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1695507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.743687629699707, + "objective/train/theoretical_loss": 3.3128300223926437, + "objective/train/tokens_used": 1465527776, + "theoretical_loss": 3.3128300223926437, + "tokens_seen": 3095068672 + }, + { + "epoch": 0.88, + "learning_rate": 1.2542128069330766e-05, + "loss": 2.4924, + "theoretical_loss": 3.3128300223926437, + "tokens_seen": 3095068672 + }, + { + "epoch": 0.88, + "learning_rate": 1.2534103675172526e-05, + "loss": 2.6238, + "theoretical_loss": 3.3128193041664122, + "tokens_seen": 3095199744 + }, + { + "epoch": 0.88, + "learning_rate": 1.2526079281014283e-05, + "loss": 2.4503, + "theoretical_loss": 3.312808586521135, + "tokens_seen": 3095330816 + }, + { + "epoch": 0.88, + "learning_rate": 1.2518054886856043e-05, + "loss": 2.5579, + "theoretical_loss": 3.3127978694567575, + "tokens_seen": 3095461888 + }, + { + "epoch": 0.88, + "learning_rate": 1.2510030492697802e-05, + "loss": 2.6154, + "theoretical_loss": 3.3127871529732222, + "tokens_seen": 3095592960 + }, + { + "epoch": 0.88, + "learning_rate": 1.2502006098539562e-05, + "loss": 2.6283, + "theoretical_loss": 3.3127764370704735, + "tokens_seen": 3095724032 + }, + { + "epoch": 0.88, + "learning_rate": 1.249398170438132e-05, + "loss": 2.6665, + "theoretical_loss": 3.3127657217484554, + "tokens_seen": 3095855104 + }, + { + "epoch": 0.88, + "learning_rate": 1.2485957310223079e-05, + "loss": 2.5384, + "theoretical_loss": 3.312755007007112, + "tokens_seen": 3095986176 + }, + { + "epoch": 0.88, + "learning_rate": 1.2477932916064839e-05, + "loss": 2.6241, + "theoretical_loss": 3.3127442928463866, + "tokens_seen": 3096117248 + }, + { + "epoch": 0.88, + "learning_rate": 1.2469908521906597e-05, + "loss": 2.6001, + "theoretical_loss": 3.312733579266224, + "tokens_seen": 3096248320 + }, + { + "epoch": 0.88, + "learning_rate": 1.2461884127748355e-05, + "loss": 2.568, + "theoretical_loss": 3.3127228662665673, + "tokens_seen": 3096379392 + }, + { + "epoch": 0.88, + "learning_rate": 1.2453859733590115e-05, + "loss": 2.7273, + "theoretical_loss": 3.312712153847361, + "tokens_seen": 3096510464 + }, + { + "epoch": 0.88, + "learning_rate": 1.2445835339431873e-05, + "loss": 2.5691, + "theoretical_loss": 3.3127014420085494, + "tokens_seen": 3096641536 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1696585, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.869546890258789, + "objective/train/theoretical_loss": 3.3126960863067736, + "objective/train/tokens_used": 1467166176, + "theoretical_loss": 3.3126960863067736, + "tokens_seen": 3096707072 + }, + { + "epoch": 0.88, + "learning_rate": 1.2437810945273633e-05, + "loss": 2.6561, + "theoretical_loss": 3.3126907307500755, + "tokens_seen": 3096772608 + }, + { + "epoch": 0.88, + "learning_rate": 1.2429786551115391e-05, + "loss": 2.6248, + "theoretical_loss": 3.3126800200718844, + "tokens_seen": 3096903680 + }, + { + "epoch": 0.88, + "learning_rate": 1.2421762156957151e-05, + "loss": 2.7087, + "theoretical_loss": 3.312669309973919, + "tokens_seen": 3097034752 + }, + { + "epoch": 0.88, + "learning_rate": 1.241373776279891e-05, + "loss": 2.7328, + "theoretical_loss": 3.3126586004561247, + "tokens_seen": 3097165824 + }, + { + "epoch": 0.88, + "learning_rate": 1.240571336864067e-05, + "loss": 2.7871, + "theoretical_loss": 3.312647891518444, + "tokens_seen": 3097296896 + }, + { + "epoch": 0.88, + "learning_rate": 1.2397688974482428e-05, + "loss": 2.6973, + "theoretical_loss": 3.312637183160822, + "tokens_seen": 3097427968 + }, + { + "epoch": 0.88, + "learning_rate": 1.2389664580324187e-05, + "loss": 2.5717, + "theoretical_loss": 3.3126264753832024, + "tokens_seen": 3097559040 + }, + { + "epoch": 0.88, + "learning_rate": 1.2381640186165946e-05, + "loss": 2.4592, + "theoretical_loss": 3.312615768185529, + "tokens_seen": 3097690112 + }, + { + "epoch": 0.88, + "learning_rate": 1.2373615792007704e-05, + "loss": 2.5213, + "theoretical_loss": 3.3126050615677465, + "tokens_seen": 3097821184 + }, + { + "epoch": 0.88, + "learning_rate": 1.2365591397849464e-05, + "loss": 2.5601, + "theoretical_loss": 3.3125943555297983, + "tokens_seen": 3097952256 + }, + { + "epoch": 0.88, + "learning_rate": 1.2357567003691222e-05, + "loss": 2.5821, + "theoretical_loss": 3.3125836500716286, + "tokens_seen": 3098083328 + }, + { + "epoch": 0.88, + "learning_rate": 1.2349542609532982e-05, + "loss": 2.5393, + "theoretical_loss": 3.312572945193182, + "tokens_seen": 3098214400 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1697196, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9544224739074707, + "objective/train/theoretical_loss": 3.312562240894402, + "objective/train/tokens_used": 1468804576, + "theoretical_loss": 3.312562240894402, + "tokens_seen": 3098345472 + }, + { + "epoch": 0.88, + "learning_rate": 1.234151821537474e-05, + "loss": 2.6779, + "theoretical_loss": 3.312562240894402, + "tokens_seen": 3098345472 + }, + { + "epoch": 0.88, + "learning_rate": 1.2333493821216498e-05, + "loss": 2.6656, + "theoretical_loss": 3.3125515371752323, + "tokens_seen": 3098476544 + }, + { + "epoch": 0.88, + "learning_rate": 1.2325469427058258e-05, + "loss": 2.5373, + "theoretical_loss": 3.3125408340356186, + "tokens_seen": 3098607616 + }, + { + "epoch": 0.88, + "learning_rate": 1.2317445032900016e-05, + "loss": 2.7178, + "theoretical_loss": 3.312530131475503, + "tokens_seen": 3098738688 + }, + { + "epoch": 0.88, + "learning_rate": 1.2309420638741775e-05, + "loss": 2.5468, + "theoretical_loss": 3.3125194294948312, + "tokens_seen": 3098869760 + }, + { + "epoch": 0.88, + "learning_rate": 1.2301396244583535e-05, + "loss": 2.6067, + "theoretical_loss": 3.3125087280935466, + "tokens_seen": 3099000832 + }, + { + "epoch": 0.88, + "learning_rate": 1.2293371850425293e-05, + "loss": 2.5229, + "theoretical_loss": 3.3124980272715936, + "tokens_seen": 3099131904 + }, + { + "epoch": 0.88, + "learning_rate": 1.2285347456267051e-05, + "loss": 2.6516, + "theoretical_loss": 3.3124873270289155, + "tokens_seen": 3099262976 + }, + { + "epoch": 0.88, + "learning_rate": 1.2277323062108811e-05, + "loss": 2.4737, + "theoretical_loss": 3.3124766273654576, + "tokens_seen": 3099394048 + }, + { + "epoch": 0.88, + "learning_rate": 1.2269298667950569e-05, + "loss": 2.6402, + "theoretical_loss": 3.3124659282811635, + "tokens_seen": 3099525120 + }, + { + "epoch": 0.88, + "learning_rate": 1.2261274273792329e-05, + "loss": 2.5667, + "theoretical_loss": 3.3124552297759777, + "tokens_seen": 3099656192 + }, + { + "epoch": 0.88, + "learning_rate": 1.2253249879634087e-05, + "loss": 2.6581, + "theoretical_loss": 3.312444531849844, + "tokens_seen": 3099787264 + }, + { + "epoch": 0.88, + "learning_rate": 1.2245225485475847e-05, + "loss": 2.4983, + "theoretical_loss": 3.3124338345027065, + "tokens_seen": 3099918336 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1698465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.516152858734131, + "objective/train/theoretical_loss": 3.312428486046244, + "objective/train/tokens_used": 1470442976, + "theoretical_loss": 3.312428486046244, + "tokens_seen": 3099983872 + }, + { + "epoch": 0.88, + "learning_rate": 1.2237201091317605e-05, + "loss": 2.4846, + "theoretical_loss": 3.3124231377345095, + "tokens_seen": 3100049408 + }, + { + "epoch": 0.88, + "learning_rate": 1.2229176697159365e-05, + "loss": 2.6195, + "theoretical_loss": 3.3124124415451974, + "tokens_seen": 3100180480 + }, + { + "epoch": 0.88, + "learning_rate": 1.2221152303001123e-05, + "loss": 2.455, + "theoretical_loss": 3.3124017459347144, + "tokens_seen": 3100311552 + }, + { + "epoch": 0.88, + "learning_rate": 1.2213127908842883e-05, + "loss": 2.5777, + "theoretical_loss": 3.3123910509030043, + "tokens_seen": 3100442624 + }, + { + "epoch": 0.88, + "learning_rate": 1.2205103514684642e-05, + "loss": 2.6749, + "theoretical_loss": 3.3123803564500114, + "tokens_seen": 3100573696 + }, + { + "epoch": 0.88, + "learning_rate": 1.2197079120526401e-05, + "loss": 2.667, + "theoretical_loss": 3.3123696625756804, + "tokens_seen": 3100704768 + }, + { + "epoch": 0.88, + "learning_rate": 1.218905472636816e-05, + "loss": 2.6271, + "theoretical_loss": 3.312358969279955, + "tokens_seen": 3100835840 + }, + { + "epoch": 0.88, + "learning_rate": 1.2181030332209918e-05, + "loss": 2.4664, + "theoretical_loss": 3.31234827656278, + "tokens_seen": 3100966912 + }, + { + "epoch": 0.88, + "learning_rate": 1.2173005938051678e-05, + "loss": 2.5187, + "theoretical_loss": 3.312337584424099, + "tokens_seen": 3101097984 + }, + { + "epoch": 0.88, + "learning_rate": 1.2164981543893436e-05, + "loss": 2.5555, + "theoretical_loss": 3.312326892863857, + "tokens_seen": 3101229056 + }, + { + "epoch": 0.88, + "learning_rate": 1.2156957149735196e-05, + "loss": 2.6103, + "theoretical_loss": 3.3123162018819974, + "tokens_seen": 3101360128 + }, + { + "epoch": 0.88, + "learning_rate": 1.2148932755576954e-05, + "loss": 2.4043, + "theoretical_loss": 3.312305511478465, + "tokens_seen": 3101491200 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1699106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.67728590965271, + "objective/train/theoretical_loss": 3.312294821653204, + "objective/train/tokens_used": 1472081376, + "theoretical_loss": 3.312294821653204, + "tokens_seen": 3101622272 + }, + { + "epoch": 0.88, + "learning_rate": 1.2140908361418714e-05, + "loss": 2.6168, + "theoretical_loss": 3.312294821653204, + "tokens_seen": 3101622272 + }, + { + "epoch": 0.88, + "learning_rate": 1.2132883967260472e-05, + "loss": 2.5399, + "theoretical_loss": 3.312284132406159, + "tokens_seen": 3101753344 + }, + { + "epoch": 0.88, + "learning_rate": 1.2124859573102232e-05, + "loss": 2.492, + "theoretical_loss": 3.312273443737274, + "tokens_seen": 3101884416 + }, + { + "epoch": 0.88, + "learning_rate": 1.211683517894399e-05, + "loss": 2.6553, + "theoretical_loss": 3.3122627556464925, + "tokens_seen": 3102015488 + }, + { + "epoch": 0.88, + "learning_rate": 1.210881078478575e-05, + "loss": 2.7154, + "theoretical_loss": 3.31225206813376, + "tokens_seen": 3102146560 + }, + { + "epoch": 0.88, + "learning_rate": 1.2100786390627508e-05, + "loss": 2.6884, + "theoretical_loss": 3.3122413811990206, + "tokens_seen": 3102277632 + }, + { + "epoch": 0.88, + "learning_rate": 1.2092761996469267e-05, + "loss": 2.508, + "theoretical_loss": 3.312230694842218, + "tokens_seen": 3102408704 + }, + { + "epoch": 0.88, + "learning_rate": 1.2084737602311027e-05, + "loss": 2.5564, + "theoretical_loss": 3.312220009063297, + "tokens_seen": 3102539776 + }, + { + "epoch": 0.88, + "learning_rate": 1.2076713208152785e-05, + "loss": 2.587, + "theoretical_loss": 3.312209323862202, + "tokens_seen": 3102670848 + }, + { + "epoch": 0.88, + "learning_rate": 1.2068688813994545e-05, + "loss": 2.5942, + "theoretical_loss": 3.3121986392388774, + "tokens_seen": 3102801920 + }, + { + "epoch": 0.88, + "learning_rate": 1.2060664419836303e-05, + "loss": 2.505, + "theoretical_loss": 3.312187955193267, + "tokens_seen": 3102932992 + }, + { + "epoch": 0.88, + "learning_rate": 1.2052640025678063e-05, + "loss": 2.4273, + "theoretical_loss": 3.3121772717253157, + "tokens_seen": 3103064064 + }, + { + "epoch": 0.88, + "learning_rate": 1.2044615631519821e-05, + "loss": 2.6151, + "theoretical_loss": 3.312166588834968, + "tokens_seen": 3103195136 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1699977, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.289055109024048, + "objective/train/theoretical_loss": 3.3121612476063778, + "objective/train/tokens_used": 1473719776, + "theoretical_loss": 3.3121612476063778, + "tokens_seen": 3103260672 + }, + { + "epoch": 0.88, + "learning_rate": 1.2036591237361581e-05, + "loss": 2.5031, + "theoretical_loss": 3.3121559065221673, + "tokens_seen": 3103326208 + }, + { + "epoch": 0.88, + "learning_rate": 1.202856684320334e-05, + "loss": 2.6014, + "theoretical_loss": 3.312145224786859, + "tokens_seen": 3103457280 + }, + { + "epoch": 0.88, + "learning_rate": 1.2020542449045099e-05, + "loss": 2.6565, + "theoretical_loss": 3.312134543628987, + "tokens_seen": 3103588352 + }, + { + "epoch": 0.88, + "learning_rate": 1.2012518054886857e-05, + "loss": 2.5438, + "theoretical_loss": 3.3121238630484964, + "tokens_seen": 3103719424 + }, + { + "epoch": 0.88, + "learning_rate": 1.2004493660728616e-05, + "loss": 2.5147, + "theoretical_loss": 3.3121131830453305, + "tokens_seen": 3103850496 + }, + { + "epoch": 0.88, + "learning_rate": 1.1996469266570375e-05, + "loss": 2.526, + "theoretical_loss": 3.3121025036194345, + "tokens_seen": 3103981568 + }, + { + "epoch": 0.88, + "learning_rate": 1.1988444872412134e-05, + "loss": 2.6292, + "theoretical_loss": 3.3120918247707527, + "tokens_seen": 3104112640 + }, + { + "epoch": 0.88, + "learning_rate": 1.1980420478253892e-05, + "loss": 2.6156, + "theoretical_loss": 3.3120811464992292, + "tokens_seen": 3104243712 + }, + { + "epoch": 0.88, + "learning_rate": 1.197239608409565e-05, + "loss": 2.7622, + "theoretical_loss": 3.3120704688048086, + "tokens_seen": 3104374784 + }, + { + "epoch": 0.88, + "learning_rate": 1.196437168993741e-05, + "loss": 2.6547, + "theoretical_loss": 3.3120597916874357, + "tokens_seen": 3104505856 + }, + { + "epoch": 0.88, + "learning_rate": 1.1956347295779168e-05, + "loss": 2.5428, + "theoretical_loss": 3.3120491151470546, + "tokens_seen": 3104636928 + }, + { + "epoch": 0.88, + "learning_rate": 1.1948322901620928e-05, + "loss": 2.5223, + "theoretical_loss": 3.3120384391836097, + "tokens_seen": 3104768000 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1701288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.463628053665161, + "objective/train/theoretical_loss": 3.3120277637970457, + "objective/train/tokens_used": 1475358176, + "theoretical_loss": 3.3120277637970457, + "tokens_seen": 3104899072 + }, + { + "epoch": 0.88, + "learning_rate": 1.1940298507462686e-05, + "loss": 2.6639, + "theoretical_loss": 3.3120277637970457, + "tokens_seen": 3104899072 + }, + { + "epoch": 0.88, + "learning_rate": 1.1932274113304446e-05, + "loss": 2.5559, + "theoretical_loss": 3.312017088987307, + "tokens_seen": 3105030144 + }, + { + "epoch": 0.88, + "learning_rate": 1.1924249719146204e-05, + "loss": 2.7103, + "theoretical_loss": 3.3120064147543378, + "tokens_seen": 3105161216 + }, + { + "epoch": 0.88, + "learning_rate": 1.1916225324987964e-05, + "loss": 2.652, + "theoretical_loss": 3.311995741098083, + "tokens_seen": 3105292288 + }, + { + "epoch": 0.88, + "learning_rate": 1.1908200930829723e-05, + "loss": 2.4478, + "theoretical_loss": 3.3119850680184872, + "tokens_seen": 3105423360 + }, + { + "epoch": 0.88, + "learning_rate": 1.190017653667148e-05, + "loss": 2.4615, + "theoretical_loss": 3.3119743955154943, + "tokens_seen": 3105554432 + }, + { + "epoch": 0.88, + "learning_rate": 1.189215214251324e-05, + "loss": 2.6877, + "theoretical_loss": 3.3119637235890496, + "tokens_seen": 3105685504 + }, + { + "epoch": 0.88, + "learning_rate": 1.1884127748354999e-05, + "loss": 2.6198, + "theoretical_loss": 3.311953052239097, + "tokens_seen": 3105816576 + }, + { + "epoch": 0.88, + "learning_rate": 1.1876103354196759e-05, + "loss": 2.602, + "theoretical_loss": 3.311942381465581, + "tokens_seen": 3105947648 + }, + { + "epoch": 0.88, + "learning_rate": 1.1868078960038517e-05, + "loss": 2.5661, + "theoretical_loss": 3.3119317112684468, + "tokens_seen": 3106078720 + }, + { + "epoch": 0.88, + "learning_rate": 1.1860054565880277e-05, + "loss": 2.686, + "theoretical_loss": 3.311921041647638, + "tokens_seen": 3106209792 + }, + { + "epoch": 0.88, + "learning_rate": 1.1852030171722035e-05, + "loss": 2.557, + "theoretical_loss": 3.3119103726031005, + "tokens_seen": 3106340864 + }, + { + "epoch": 0.88, + "learning_rate": 1.1844005777563795e-05, + "loss": 2.492, + "theoretical_loss": 3.3118997041347775, + "tokens_seen": 3106471936 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1702006, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.321450710296631, + "objective/train/theoretical_loss": 3.311894370116679, + "objective/train/tokens_used": 1476996576, + "theoretical_loss": 3.311894370116679, + "tokens_seen": 3106537472 + }, + { + "epoch": 0.88, + "learning_rate": 1.1835981383405553e-05, + "loss": 2.5888, + "theoretical_loss": 3.311889036242614, + "tokens_seen": 3106603008 + }, + { + "epoch": 0.88, + "learning_rate": 1.1827956989247313e-05, + "loss": 2.439, + "theoretical_loss": 3.3118783689265547, + "tokens_seen": 3106734080 + }, + { + "epoch": 0.88, + "learning_rate": 1.1819932595089071e-05, + "loss": 2.5071, + "theoretical_loss": 3.3118677021865444, + "tokens_seen": 3106865152 + }, + { + "epoch": 0.88, + "learning_rate": 1.181190820093083e-05, + "loss": 2.6544, + "theoretical_loss": 3.3118570360225275, + "tokens_seen": 3106996224 + }, + { + "epoch": 0.88, + "learning_rate": 1.180388380677259e-05, + "loss": 2.6397, + "theoretical_loss": 3.3118463704344485, + "tokens_seen": 3107127296 + }, + { + "epoch": 0.88, + "learning_rate": 1.1795859412614348e-05, + "loss": 2.575, + "theoretical_loss": 3.3118357054222516, + "tokens_seen": 3107258368 + }, + { + "epoch": 0.88, + "learning_rate": 1.1787835018456108e-05, + "loss": 2.5487, + "theoretical_loss": 3.311825040985882, + "tokens_seen": 3107389440 + }, + { + "epoch": 0.88, + "learning_rate": 1.1779810624297866e-05, + "loss": 2.6079, + "theoretical_loss": 3.311814377125285, + "tokens_seen": 3107520512 + }, + { + "epoch": 0.88, + "learning_rate": 1.1771786230139626e-05, + "loss": 2.5289, + "theoretical_loss": 3.311803713840404, + "tokens_seen": 3107651584 + }, + { + "epoch": 0.88, + "learning_rate": 1.1763761835981384e-05, + "loss": 2.5367, + "theoretical_loss": 3.3117930511311835, + "tokens_seen": 3107782656 + }, + { + "epoch": 0.88, + "learning_rate": 1.1755737441823144e-05, + "loss": 2.675, + "theoretical_loss": 3.3117823889975693, + "tokens_seen": 3107913728 + }, + { + "epoch": 0.88, + "learning_rate": 1.1747713047664902e-05, + "loss": 2.551, + "theoretical_loss": 3.3117717274395053, + "tokens_seen": 3108044800 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1702889, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5995166301727295, + "objective/train/theoretical_loss": 3.3117610664569366, + "objective/train/tokens_used": 1478634976, + "theoretical_loss": 3.3117610664569366, + "tokens_seen": 3108175872 + }, + { + "epoch": 0.88, + "learning_rate": 1.1739688653506662e-05, + "loss": 2.5998, + "theoretical_loss": 3.3117610664569366, + "tokens_seen": 3108175872 + }, + { + "epoch": 0.88, + "learning_rate": 1.173166425934842e-05, + "loss": 2.6198, + "theoretical_loss": 3.311750406049807, + "tokens_seen": 3108306944 + }, + { + "epoch": 0.88, + "learning_rate": 1.172363986519018e-05, + "loss": 2.4352, + "theoretical_loss": 3.3117397462180627, + "tokens_seen": 3108438016 + }, + { + "epoch": 0.88, + "learning_rate": 1.1715615471031938e-05, + "loss": 2.5869, + "theoretical_loss": 3.3117290869616465, + "tokens_seen": 3108569088 + }, + { + "epoch": 0.88, + "learning_rate": 1.1707591076873696e-05, + "loss": 2.5138, + "theoretical_loss": 3.3117184282805048, + "tokens_seen": 3108700160 + }, + { + "epoch": 0.88, + "learning_rate": 1.1699566682715456e-05, + "loss": 2.5104, + "theoretical_loss": 3.311707770174581, + "tokens_seen": 3108831232 + }, + { + "epoch": 0.88, + "learning_rate": 1.1691542288557215e-05, + "loss": 2.5703, + "theoretical_loss": 3.3116971126438206, + "tokens_seen": 3108962304 + }, + { + "epoch": 0.88, + "learning_rate": 1.1683517894398974e-05, + "loss": 2.5374, + "theoretical_loss": 3.311686455688168, + "tokens_seen": 3109093376 + }, + { + "epoch": 0.88, + "learning_rate": 1.1675493500240733e-05, + "loss": 2.691, + "theoretical_loss": 3.3116757993075683, + "tokens_seen": 3109224448 + }, + { + "epoch": 0.88, + "learning_rate": 1.1667469106082491e-05, + "loss": 2.4287, + "theoretical_loss": 3.3116651435019655, + "tokens_seen": 3109355520 + }, + { + "epoch": 0.88, + "learning_rate": 1.165944471192425e-05, + "loss": 2.4374, + "theoretical_loss": 3.311654488271305, + "tokens_seen": 3109486592 + }, + { + "epoch": 0.88, + "learning_rate": 1.1651420317766009e-05, + "loss": 2.5583, + "theoretical_loss": 3.3116438336155314, + "tokens_seen": 3109617664 + }, + { + "epoch": 0.88, + "learning_rate": 1.1643395923607767e-05, + "loss": 2.4043, + "theoretical_loss": 3.3116331795345895, + "tokens_seen": 3109748736 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1704007, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.617337465286255, + "objective/train/theoretical_loss": 3.311627852709663, + "objective/train/tokens_used": 1480273376, + "theoretical_loss": 3.311627852709663, + "tokens_seen": 3109814272 + }, + { + "epoch": 0.88, + "learning_rate": 1.1635371529449527e-05, + "loss": 2.6148, + "theoretical_loss": 3.3116225260284238, + "tokens_seen": 3109879808 + }, + { + "epoch": 0.88, + "learning_rate": 1.1627347135291285e-05, + "loss": 2.4958, + "theoretical_loss": 3.311611873096979, + "tokens_seen": 3110010880 + }, + { + "epoch": 0.88, + "learning_rate": 1.1619322741133044e-05, + "loss": 2.4324, + "theoretical_loss": 3.3116012207402, + "tokens_seen": 3110141952 + }, + { + "epoch": 0.89, + "learning_rate": 1.1611298346974803e-05, + "loss": 2.3664, + "theoretical_loss": 3.311590568958032, + "tokens_seen": 3110273024 + }, + { + "epoch": 0.89, + "learning_rate": 1.1603273952816562e-05, + "loss": 2.549, + "theoretical_loss": 3.3115799177504193, + "tokens_seen": 3110404096 + }, + { + "epoch": 0.89, + "learning_rate": 1.1595249558658322e-05, + "loss": 2.4585, + "theoretical_loss": 3.3115692671173065, + "tokens_seen": 3110535168 + }, + { + "epoch": 0.89, + "learning_rate": 1.158722516450008e-05, + "loss": 2.6369, + "theoretical_loss": 3.311558617058639, + "tokens_seen": 3110666240 + }, + { + "epoch": 0.89, + "learning_rate": 1.157920077034184e-05, + "loss": 2.5176, + "theoretical_loss": 3.3115479675743615, + "tokens_seen": 3110797312 + }, + { + "epoch": 0.89, + "learning_rate": 1.1571176376183598e-05, + "loss": 2.6095, + "theoretical_loss": 3.311537318664419, + "tokens_seen": 3110928384 + }, + { + "epoch": 0.89, + "learning_rate": 1.1563151982025358e-05, + "loss": 2.5132, + "theoretical_loss": 3.3115266703287554, + "tokens_seen": 3111059456 + }, + { + "epoch": 0.89, + "learning_rate": 1.1555127587867116e-05, + "loss": 2.6209, + "theoretical_loss": 3.311516022567316, + "tokens_seen": 3111190528 + }, + { + "epoch": 0.89, + "learning_rate": 1.1547103193708876e-05, + "loss": 2.649, + "theoretical_loss": 3.3115053753800465, + "tokens_seen": 3111321600 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1704747, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4558629989624023, + "objective/train/theoretical_loss": 3.311494728766891, + "objective/train/tokens_used": 1481911776, + "theoretical_loss": 3.311494728766891, + "tokens_seen": 3111452672 + }, + { + "epoch": 0.89, + "learning_rate": 1.1539078799550634e-05, + "loss": 2.4704, + "theoretical_loss": 3.311494728766891, + "tokens_seen": 3111452672 + }, + { + "epoch": 0.89, + "learning_rate": 1.1531054405392392e-05, + "loss": 2.5594, + "theoretical_loss": 3.311484082727794, + "tokens_seen": 3111583744 + }, + { + "epoch": 0.89, + "learning_rate": 1.1523030011234152e-05, + "loss": 2.5694, + "theoretical_loss": 3.311473437262701, + "tokens_seen": 3111714816 + }, + { + "epoch": 0.89, + "learning_rate": 1.151500561707591e-05, + "loss": 2.5365, + "theoretical_loss": 3.3114627923715565, + "tokens_seen": 3111845888 + }, + { + "epoch": 0.89, + "learning_rate": 1.150698122291767e-05, + "loss": 2.6519, + "theoretical_loss": 3.3114521480543058, + "tokens_seen": 3111976960 + }, + { + "epoch": 0.89, + "learning_rate": 1.1498956828759429e-05, + "loss": 2.4853, + "theoretical_loss": 3.311441504310893, + "tokens_seen": 3112108032 + }, + { + "epoch": 0.89, + "learning_rate": 1.1490932434601188e-05, + "loss": 2.6344, + "theoretical_loss": 3.3114308611412637, + "tokens_seen": 3112239104 + }, + { + "epoch": 0.89, + "learning_rate": 1.1482908040442947e-05, + "loss": 2.532, + "theoretical_loss": 3.311420218545363, + "tokens_seen": 3112370176 + }, + { + "epoch": 0.89, + "learning_rate": 1.1474883646284707e-05, + "loss": 2.5403, + "theoretical_loss": 3.3114095765231353, + "tokens_seen": 3112501248 + }, + { + "epoch": 0.89, + "learning_rate": 1.1466859252126465e-05, + "loss": 2.6205, + "theoretical_loss": 3.3113989350745254, + "tokens_seen": 3112632320 + }, + { + "epoch": 0.89, + "learning_rate": 1.1458834857968225e-05, + "loss": 2.5936, + "theoretical_loss": 3.311388294199479, + "tokens_seen": 3112763392 + }, + { + "epoch": 0.89, + "learning_rate": 1.1450810463809983e-05, + "loss": 2.6073, + "theoretical_loss": 3.31137765389794, + "tokens_seen": 3112894464 + }, + { + "epoch": 0.89, + "learning_rate": 1.1442786069651743e-05, + "loss": 2.5998, + "theoretical_loss": 3.3113670141698544, + "tokens_seen": 3113025536 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1705767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4811882972717285, + "objective/train/theoretical_loss": 3.311361694520839, + "objective/train/tokens_used": 1483550176, + "theoretical_loss": 3.311361694520839, + "tokens_seen": 3113091072 + }, + { + "epoch": 0.89, + "learning_rate": 1.1434761675493501e-05, + "loss": 2.6302, + "theoretical_loss": 3.311356375015166, + "tokens_seen": 3113156608 + }, + { + "epoch": 0.89, + "learning_rate": 1.142673728133526e-05, + "loss": 2.5773, + "theoretical_loss": 3.311345736433821, + "tokens_seen": 3113287680 + }, + { + "epoch": 0.89, + "learning_rate": 1.141871288717702e-05, + "loss": 2.7006, + "theoretical_loss": 3.3113350984257632, + "tokens_seen": 3113418752 + }, + { + "epoch": 0.89, + "learning_rate": 1.1410688493018777e-05, + "loss": 2.5591, + "theoretical_loss": 3.3113244609909382, + "tokens_seen": 3113549824 + }, + { + "epoch": 0.89, + "learning_rate": 1.1402664098860537e-05, + "loss": 2.5402, + "theoretical_loss": 3.311313824129291, + "tokens_seen": 3113680896 + }, + { + "epoch": 0.89, + "learning_rate": 1.1394639704702296e-05, + "loss": 2.5997, + "theoretical_loss": 3.3113031878407666, + "tokens_seen": 3113811968 + }, + { + "epoch": 0.89, + "learning_rate": 1.1386615310544055e-05, + "loss": 2.633, + "theoretical_loss": 3.31129255212531, + "tokens_seen": 3113943040 + }, + { + "epoch": 0.89, + "learning_rate": 1.1378590916385814e-05, + "loss": 2.6448, + "theoretical_loss": 3.311281916982866, + "tokens_seen": 3114074112 + }, + { + "epoch": 0.89, + "learning_rate": 1.1370566522227574e-05, + "loss": 2.6208, + "theoretical_loss": 3.3112712824133794, + "tokens_seen": 3114205184 + }, + { + "epoch": 0.89, + "learning_rate": 1.1362542128069332e-05, + "loss": 2.578, + "theoretical_loss": 3.311260648416796, + "tokens_seen": 3114336256 + }, + { + "epoch": 0.89, + "learning_rate": 1.1354517733911092e-05, + "loss": 2.4092, + "theoretical_loss": 3.31125001499306, + "tokens_seen": 3114467328 + }, + { + "epoch": 0.89, + "learning_rate": 1.134649333975285e-05, + "loss": 2.5185, + "theoretical_loss": 3.3112393821421167, + "tokens_seen": 3114598400 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1706383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4116265773773193, + "objective/train/theoretical_loss": 3.3112287498639117, + "objective/train/tokens_used": 1485188576, + "theoretical_loss": 3.3112287498639117, + "tokens_seen": 3114729472 + }, + { + "epoch": 0.89, + "learning_rate": 1.1338468945594608e-05, + "loss": 2.4152, + "theoretical_loss": 3.3112287498639117, + "tokens_seen": 3114729472 + }, + { + "epoch": 0.89, + "learning_rate": 1.1330444551436368e-05, + "loss": 2.6435, + "theoretical_loss": 3.311218118158389, + "tokens_seen": 3114860544 + }, + { + "epoch": 0.89, + "learning_rate": 1.1322420157278126e-05, + "loss": 2.521, + "theoretical_loss": 3.311207487025495, + "tokens_seen": 3114991616 + }, + { + "epoch": 0.89, + "learning_rate": 1.1314395763119884e-05, + "loss": 2.5948, + "theoretical_loss": 3.3111968564651733, + "tokens_seen": 3115122688 + }, + { + "epoch": 0.89, + "learning_rate": 1.1306371368961644e-05, + "loss": 2.6796, + "theoretical_loss": 3.31118622647737, + "tokens_seen": 3115253760 + }, + { + "epoch": 0.89, + "learning_rate": 1.1298346974803403e-05, + "loss": 2.6726, + "theoretical_loss": 3.3111755970620296, + "tokens_seen": 3115384832 + }, + { + "epoch": 0.89, + "learning_rate": 1.129032258064516e-05, + "loss": 2.5859, + "theoretical_loss": 3.3111649682190976, + "tokens_seen": 3115515904 + }, + { + "epoch": 0.89, + "learning_rate": 1.128229818648692e-05, + "loss": 2.5131, + "theoretical_loss": 3.3111543399485193, + "tokens_seen": 3115646976 + }, + { + "epoch": 0.89, + "learning_rate": 1.1274273792328679e-05, + "loss": 2.4152, + "theoretical_loss": 3.3111437122502387, + "tokens_seen": 3115778048 + }, + { + "epoch": 0.89, + "learning_rate": 1.1266249398170439e-05, + "loss": 2.6394, + "theoretical_loss": 3.3111330851242022, + "tokens_seen": 3115909120 + }, + { + "epoch": 0.89, + "learning_rate": 1.1258225004012197e-05, + "loss": 2.4835, + "theoretical_loss": 3.3111224585703543, + "tokens_seen": 3116040192 + }, + { + "epoch": 0.89, + "learning_rate": 1.1250200609853957e-05, + "loss": 2.6412, + "theoretical_loss": 3.31111183258864, + "tokens_seen": 3116171264 + }, + { + "epoch": 0.89, + "learning_rate": 1.1242176215695715e-05, + "loss": 2.5045, + "theoretical_loss": 3.311101207179005, + "tokens_seen": 3116302336 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1707531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8599119186401367, + "objective/train/theoretical_loss": 3.3110958946887, + "objective/train/tokens_used": 1486826976, + "theoretical_loss": 3.3110958946887, + "tokens_seen": 3116367872 + }, + { + "epoch": 0.89, + "learning_rate": 1.1234151821537473e-05, + "loss": 2.5797, + "theoretical_loss": 3.311090582341394, + "tokens_seen": 3116433408 + }, + { + "epoch": 0.89, + "learning_rate": 1.1226127427379233e-05, + "loss": 2.6776, + "theoretical_loss": 3.3110799580757515, + "tokens_seen": 3116564480 + }, + { + "epoch": 0.89, + "learning_rate": 1.1218103033220991e-05, + "loss": 2.483, + "theoretical_loss": 3.311069334382024, + "tokens_seen": 3116695552 + }, + { + "epoch": 0.89, + "learning_rate": 1.1210078639062751e-05, + "loss": 2.5885, + "theoretical_loss": 3.3110587112601557, + "tokens_seen": 3116826624 + }, + { + "epoch": 0.89, + "learning_rate": 1.120205424490451e-05, + "loss": 2.4791, + "theoretical_loss": 3.3110480887100926, + "tokens_seen": 3116957696 + }, + { + "epoch": 0.89, + "learning_rate": 1.119402985074627e-05, + "loss": 2.4946, + "theoretical_loss": 3.3110374667317792, + "tokens_seen": 3117088768 + }, + { + "epoch": 0.89, + "learning_rate": 1.1186005456588028e-05, + "loss": 2.3795, + "theoretical_loss": 3.311026845325161, + "tokens_seen": 3117219840 + }, + { + "epoch": 0.89, + "learning_rate": 1.1177981062429788e-05, + "loss": 2.5292, + "theoretical_loss": 3.311016224490183, + "tokens_seen": 3117350912 + }, + { + "epoch": 0.89, + "learning_rate": 1.1169956668271546e-05, + "loss": 2.5459, + "theoretical_loss": 3.31100560422679, + "tokens_seen": 3117481984 + }, + { + "epoch": 0.89, + "learning_rate": 1.1161932274113306e-05, + "loss": 2.6581, + "theoretical_loss": 3.3109949845349282, + "tokens_seen": 3117613056 + }, + { + "epoch": 0.89, + "learning_rate": 1.1153907879955064e-05, + "loss": 2.6456, + "theoretical_loss": 3.3109843654145417, + "tokens_seen": 3117744128 + }, + { + "epoch": 0.89, + "learning_rate": 1.1145883485796822e-05, + "loss": 2.4958, + "theoretical_loss": 3.310973746865577, + "tokens_seen": 3117875200 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1708028, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3633224964141846, + "objective/train/theoretical_loss": 3.310963128887978, + "objective/train/tokens_used": 1488465376, + "theoretical_loss": 3.310963128887978, + "tokens_seen": 3118006272 + }, + { + "epoch": 0.89, + "learning_rate": 1.1137859091638582e-05, + "loss": 2.4777, + "theoretical_loss": 3.310963128887978, + "tokens_seen": 3118006272 + }, + { + "epoch": 0.89, + "learning_rate": 1.112983469748034e-05, + "loss": 2.4824, + "theoretical_loss": 3.310952511481691, + "tokens_seen": 3118137344 + }, + { + "epoch": 0.89, + "learning_rate": 1.11218103033221e-05, + "loss": 2.4891, + "theoretical_loss": 3.310941894646661, + "tokens_seen": 3118268416 + }, + { + "epoch": 0.89, + "learning_rate": 1.1113785909163858e-05, + "loss": 2.5016, + "theoretical_loss": 3.310931278382833, + "tokens_seen": 3118399488 + }, + { + "epoch": 0.89, + "learning_rate": 1.1105761515005618e-05, + "loss": 2.5567, + "theoretical_loss": 3.310920662690152, + "tokens_seen": 3118530560 + }, + { + "epoch": 0.89, + "learning_rate": 1.1097737120847376e-05, + "loss": 2.385, + "theoretical_loss": 3.310910047568564, + "tokens_seen": 3118661632 + }, + { + "epoch": 0.89, + "learning_rate": 1.1089712726689136e-05, + "loss": 2.4724, + "theoretical_loss": 3.3108994330180135, + "tokens_seen": 3118792704 + }, + { + "epoch": 0.89, + "learning_rate": 1.1081688332530895e-05, + "loss": 2.3938, + "theoretical_loss": 3.3108888190384467, + "tokens_seen": 3118923776 + }, + { + "epoch": 0.89, + "learning_rate": 1.1073663938372654e-05, + "loss": 2.6343, + "theoretical_loss": 3.310878205629808, + "tokens_seen": 3119054848 + }, + { + "epoch": 0.89, + "learning_rate": 1.1065639544214413e-05, + "loss": 2.5147, + "theoretical_loss": 3.310867592792043, + "tokens_seen": 3119185920 + }, + { + "epoch": 0.89, + "learning_rate": 1.1057615150056171e-05, + "loss": 2.4894, + "theoretical_loss": 3.3108569805250974, + "tokens_seen": 3119316992 + }, + { + "epoch": 0.89, + "learning_rate": 1.104959075589793e-05, + "loss": 2.556, + "theoretical_loss": 3.310846368828916, + "tokens_seen": 3119448064 + }, + { + "epoch": 0.89, + "learning_rate": 1.1041566361739689e-05, + "loss": 2.5077, + "theoretical_loss": 3.3108357577034444, + "tokens_seen": 3119579136 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1709277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7068288326263428, + "objective/train/theoretical_loss": 3.3108304523547076, + "objective/train/tokens_used": 1490103776, + "theoretical_loss": 3.3108304523547076, + "tokens_seen": 3119644672 + }, + { + "epoch": 0.89, + "learning_rate": 1.1033541967581449e-05, + "loss": 2.4529, + "theoretical_loss": 3.310825147148628, + "tokens_seen": 3119710208 + }, + { + "epoch": 0.89, + "learning_rate": 1.1025517573423207e-05, + "loss": 2.5852, + "theoretical_loss": 3.3108145371644113, + "tokens_seen": 3119841280 + }, + { + "epoch": 0.89, + "learning_rate": 1.1017493179264967e-05, + "loss": 2.4514, + "theoretical_loss": 3.310803927750741, + "tokens_seen": 3119972352 + }, + { + "epoch": 0.89, + "learning_rate": 1.1009468785106725e-05, + "loss": 2.5348, + "theoretical_loss": 3.3107933189075616, + "tokens_seen": 3120103424 + }, + { + "epoch": 0.89, + "learning_rate": 1.1001444390948485e-05, + "loss": 2.4618, + "theoretical_loss": 3.3107827106348187, + "tokens_seen": 3120234496 + }, + { + "epoch": 0.89, + "learning_rate": 1.0993419996790243e-05, + "loss": 2.6701, + "theoretical_loss": 3.3107721029324573, + "tokens_seen": 3120365568 + }, + { + "epoch": 0.89, + "learning_rate": 1.0985395602632002e-05, + "loss": 2.3711, + "theoretical_loss": 3.3107614958004232, + "tokens_seen": 3120496640 + }, + { + "epoch": 0.89, + "learning_rate": 1.0977371208473761e-05, + "loss": 2.5341, + "theoretical_loss": 3.310750889238662, + "tokens_seen": 3120627712 + }, + { + "epoch": 0.89, + "learning_rate": 1.096934681431552e-05, + "loss": 2.6469, + "theoretical_loss": 3.3107402832471187, + "tokens_seen": 3120758784 + }, + { + "epoch": 0.89, + "learning_rate": 1.0961322420157278e-05, + "loss": 2.5528, + "theoretical_loss": 3.3107296778257385, + "tokens_seen": 3120889856 + }, + { + "epoch": 0.89, + "learning_rate": 1.0953298025999038e-05, + "loss": 2.511, + "theoretical_loss": 3.310719072974467, + "tokens_seen": 3121020928 + }, + { + "epoch": 0.89, + "learning_rate": 1.0945273631840796e-05, + "loss": 2.5977, + "theoretical_loss": 3.31070846869325, + "tokens_seen": 3121152000 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1709835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5008368492126465, + "objective/train/theoretical_loss": 3.310697864982032, + "objective/train/tokens_used": 1491742176, + "theoretical_loss": 3.310697864982032, + "tokens_seen": 3121283072 + }, + { + "epoch": 0.89, + "learning_rate": 1.0937249237682554e-05, + "loss": 2.4828, + "theoretical_loss": 3.310697864982032, + "tokens_seen": 3121283072 + }, + { + "epoch": 0.89, + "learning_rate": 1.0929224843524314e-05, + "loss": 2.4369, + "theoretical_loss": 3.3106872618407595, + "tokens_seen": 3121414144 + }, + { + "epoch": 0.89, + "learning_rate": 1.0921200449366072e-05, + "loss": 2.4766, + "theoretical_loss": 3.3106766592693777, + "tokens_seen": 3121545216 + }, + { + "epoch": 0.89, + "learning_rate": 1.0913176055207832e-05, + "loss": 2.5364, + "theoretical_loss": 3.3106660572678313, + "tokens_seen": 3121676288 + }, + { + "epoch": 0.89, + "learning_rate": 1.090515166104959e-05, + "loss": 2.4619, + "theoretical_loss": 3.3106554558360664, + "tokens_seen": 3121807360 + }, + { + "epoch": 0.89, + "learning_rate": 1.089712726689135e-05, + "loss": 2.4558, + "theoretical_loss": 3.3106448549740284, + "tokens_seen": 3121938432 + }, + { + "epoch": 0.89, + "learning_rate": 1.0889102872733109e-05, + "loss": 2.607, + "theoretical_loss": 3.3106342546816623, + "tokens_seen": 3122069504 + }, + { + "epoch": 0.89, + "learning_rate": 1.0881078478574869e-05, + "loss": 2.6487, + "theoretical_loss": 3.3106236549589143, + "tokens_seen": 3122200576 + }, + { + "epoch": 0.89, + "learning_rate": 1.0873054084416627e-05, + "loss": 2.5697, + "theoretical_loss": 3.3106130558057294, + "tokens_seen": 3122331648 + }, + { + "epoch": 0.89, + "learning_rate": 1.0865029690258385e-05, + "loss": 2.463, + "theoretical_loss": 3.310602457222053, + "tokens_seen": 3122462720 + }, + { + "epoch": 0.89, + "learning_rate": 1.0857005296100145e-05, + "loss": 2.6442, + "theoretical_loss": 3.310591859207831, + "tokens_seen": 3122593792 + }, + { + "epoch": 0.89, + "learning_rate": 1.0848980901941903e-05, + "loss": 2.6775, + "theoretical_loss": 3.3105812617630086, + "tokens_seen": 3122724864 + }, + { + "epoch": 0.89, + "learning_rate": 1.0840956507783663e-05, + "loss": 2.5591, + "theoretical_loss": 3.3105706648875315, + "tokens_seen": 3122855936 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1711181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.454939126968384, + "objective/train/theoretical_loss": 3.3105653666632806, + "objective/train/tokens_used": 1493380576, + "theoretical_loss": 3.3105653666632806, + "tokens_seen": 3122921472 + }, + { + "epoch": 0.89, + "learning_rate": 1.0832932113625421e-05, + "loss": 2.4579, + "theoretical_loss": 3.310560068581345, + "tokens_seen": 3122987008 + }, + { + "epoch": 0.89, + "learning_rate": 1.0824907719467181e-05, + "loss": 2.4309, + "theoretical_loss": 3.310549472844395, + "tokens_seen": 3123118080 + }, + { + "epoch": 0.89, + "learning_rate": 1.081688332530894e-05, + "loss": 2.6053, + "theoretical_loss": 3.3105388776766267, + "tokens_seen": 3123249152 + }, + { + "epoch": 0.89, + "learning_rate": 1.08088589311507e-05, + "loss": 2.5574, + "theoretical_loss": 3.3105282830779856, + "tokens_seen": 3123380224 + }, + { + "epoch": 0.89, + "learning_rate": 1.0800834536992457e-05, + "loss": 2.5345, + "theoretical_loss": 3.310517689048418, + "tokens_seen": 3123511296 + }, + { + "epoch": 0.89, + "learning_rate": 1.0792810142834217e-05, + "loss": 2.6029, + "theoretical_loss": 3.3105070955878677, + "tokens_seen": 3123642368 + }, + { + "epoch": 0.89, + "learning_rate": 1.0784785748675976e-05, + "loss": 2.4966, + "theoretical_loss": 3.3104965026962825, + "tokens_seen": 3123773440 + }, + { + "epoch": 0.89, + "learning_rate": 1.0776761354517735e-05, + "loss": 2.3935, + "theoretical_loss": 3.310485910373606, + "tokens_seen": 3123904512 + }, + { + "epoch": 0.89, + "learning_rate": 1.0768736960359494e-05, + "loss": 2.4911, + "theoretical_loss": 3.310475318619785, + "tokens_seen": 3124035584 + }, + { + "epoch": 0.89, + "learning_rate": 1.0760712566201252e-05, + "loss": 2.6674, + "theoretical_loss": 3.310464727434765, + "tokens_seen": 3124166656 + }, + { + "epoch": 0.89, + "learning_rate": 1.0752688172043012e-05, + "loss": 2.5586, + "theoretical_loss": 3.3104541368184908, + "tokens_seen": 3124297728 + }, + { + "epoch": 0.89, + "learning_rate": 1.074466377788477e-05, + "loss": 2.6156, + "theoretical_loss": 3.310443546770909, + "tokens_seen": 3124428800 + }, + { + "debugging/Self-BLEU-5": 0.4395198768972146, + "debugging/distinct-1-grams": 0.7994853841137938, + "debugging/distinct-2-grams": 0.982290668194527, + "debugging/entropy-1-grams": 5.403654133290264, + "debugging/entropy-2-grams": 6.035900558727296, + "debugging/length": 532.0, + "debugging/num_segments": 6, + "debugging/score": 0.0064958087354485446, + "debugging/score_std": 0.004397384398086343, + "epoch": 0.89, + "objective/train/docs_used": 1712507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3347630500793457, + "objective/train/theoretical_loss": 3.3104329572919644, + "objective/train/tokens_used": 1495018976, + "theoretical_loss": 3.3104329572919644, + "tokens_seen": 3124559872 + }, + { + "epoch": 0.89, + "learning_rate": 1.073663938372653e-05, + "loss": 2.4543, + "theoretical_loss": 3.3104329572919644, + "tokens_seen": 3124559872 + }, + { + "epoch": 0.89, + "learning_rate": 1.0728614989568288e-05, + "loss": 2.5159, + "theoretical_loss": 3.310422368381603, + "tokens_seen": 3124690944 + }, + { + "epoch": 0.89, + "learning_rate": 1.0720590595410048e-05, + "loss": 2.5435, + "theoretical_loss": 3.3104117800397703, + "tokens_seen": 3124822016 + }, + { + "epoch": 0.89, + "learning_rate": 1.0712566201251806e-05, + "loss": 2.354, + "theoretical_loss": 3.3104011922664123, + "tokens_seen": 3124953088 + }, + { + "epoch": 0.89, + "learning_rate": 1.0704541807093566e-05, + "loss": 2.5026, + "theoretical_loss": 3.3103906050614738, + "tokens_seen": 3125084160 + }, + { + "epoch": 0.89, + "learning_rate": 1.0696517412935324e-05, + "loss": 2.4954, + "theoretical_loss": 3.3103800184249015, + "tokens_seen": 3125215232 + }, + { + "epoch": 0.89, + "learning_rate": 1.0688493018777084e-05, + "loss": 2.504, + "theoretical_loss": 3.31036943235664, + "tokens_seen": 3125346304 + }, + { + "epoch": 0.89, + "learning_rate": 1.0680468624618842e-05, + "loss": 2.5285, + "theoretical_loss": 3.310358846856636, + "tokens_seen": 3125477376 + }, + { + "epoch": 0.89, + "learning_rate": 1.06724442304606e-05, + "loss": 2.5773, + "theoretical_loss": 3.3103482619248346, + "tokens_seen": 3125608448 + }, + { + "epoch": 0.89, + "learning_rate": 1.066441983630236e-05, + "loss": 2.461, + "theoretical_loss": 3.310337677561181, + "tokens_seen": 3125739520 + }, + { + "epoch": 0.89, + "learning_rate": 1.0656395442144119e-05, + "loss": 2.4877, + "theoretical_loss": 3.310327093765622, + "tokens_seen": 3125870592 + }, + { + "epoch": 0.89, + "learning_rate": 1.0648371047985877e-05, + "loss": 2.4611, + "theoretical_loss": 3.3103165105381023, + "tokens_seen": 3126001664 + }, + { + "epoch": 0.89, + "learning_rate": 1.0640346653827637e-05, + "loss": 2.5011, + "theoretical_loss": 3.3103059278785683, + "tokens_seen": 3126132736 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1713045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.369112730026245, + "objective/train/theoretical_loss": 3.3103006367617787, + "objective/train/tokens_used": 1496657376, + "theoretical_loss": 3.3103006367617787, + "tokens_seen": 3126198272 + }, + { + "epoch": 0.89, + "learning_rate": 1.0632322259669395e-05, + "loss": 2.4738, + "theoretical_loss": 3.3102953457869653, + "tokens_seen": 3126263808 + }, + { + "epoch": 0.89, + "learning_rate": 1.0624297865511153e-05, + "loss": 2.6762, + "theoretical_loss": 3.310284764263239, + "tokens_seen": 3126394880 + }, + { + "epoch": 0.89, + "learning_rate": 1.0616273471352913e-05, + "loss": 2.4956, + "theoretical_loss": 3.3102741833073352, + "tokens_seen": 3126525952 + }, + { + "epoch": 0.89, + "learning_rate": 1.0608249077194671e-05, + "loss": 2.4857, + "theoretical_loss": 3.3102636029192, + "tokens_seen": 3126657024 + }, + { + "epoch": 0.9, + "learning_rate": 1.0600224683036431e-05, + "loss": 2.5066, + "theoretical_loss": 3.3102530230987783, + "tokens_seen": 3126788096 + }, + { + "epoch": 0.9, + "learning_rate": 1.059220028887819e-05, + "loss": 2.5451, + "theoretical_loss": 3.310242443846017, + "tokens_seen": 3126919168 + }, + { + "epoch": 0.9, + "learning_rate": 1.0584175894719948e-05, + "loss": 2.4748, + "theoretical_loss": 3.3102318651608607, + "tokens_seen": 3127050240 + }, + { + "epoch": 0.9, + "learning_rate": 1.0576151500561708e-05, + "loss": 2.6304, + "theoretical_loss": 3.3102212870432557, + "tokens_seen": 3127181312 + }, + { + "epoch": 0.9, + "learning_rate": 1.0568127106403466e-05, + "loss": 2.5437, + "theoretical_loss": 3.3102107094931474, + "tokens_seen": 3127312384 + }, + { + "epoch": 0.9, + "learning_rate": 1.0560102712245226e-05, + "loss": 2.6655, + "theoretical_loss": 3.3102001325104826, + "tokens_seen": 3127443456 + }, + { + "epoch": 0.9, + "learning_rate": 1.0552078318086984e-05, + "loss": 2.6138, + "theoretical_loss": 3.3101895560952057, + "tokens_seen": 3127574528 + }, + { + "epoch": 0.9, + "learning_rate": 1.0544053923928744e-05, + "loss": 2.5056, + "theoretical_loss": 3.3101789802472634, + "tokens_seen": 3127705600 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1713856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.498500108718872, + "objective/train/theoretical_loss": 3.3101684049666016, + "objective/train/tokens_used": 1498295776, + "theoretical_loss": 3.3101684049666016, + "tokens_seen": 3127836672 + }, + { + "epoch": 0.9, + "learning_rate": 1.0536029529770502e-05, + "loss": 2.4946, + "theoretical_loss": 3.3101684049666016, + "tokens_seen": 3127836672 + }, + { + "epoch": 0.9, + "learning_rate": 1.0528005135612262e-05, + "loss": 2.6598, + "theoretical_loss": 3.310157830253165, + "tokens_seen": 3127967744 + }, + { + "epoch": 0.9, + "learning_rate": 1.051998074145402e-05, + "loss": 2.6136, + "theoretical_loss": 3.3101472561069007, + "tokens_seen": 3128098816 + }, + { + "epoch": 0.9, + "learning_rate": 1.051195634729578e-05, + "loss": 2.614, + "theoretical_loss": 3.310136682527754, + "tokens_seen": 3128229888 + }, + { + "epoch": 0.9, + "learning_rate": 1.0503931953137538e-05, + "loss": 2.5287, + "theoretical_loss": 3.3101261095156707, + "tokens_seen": 3128360960 + }, + { + "epoch": 0.9, + "learning_rate": 1.0495907558979298e-05, + "loss": 2.5452, + "theoretical_loss": 3.310115537070596, + "tokens_seen": 3128492032 + }, + { + "epoch": 0.9, + "learning_rate": 1.0487883164821056e-05, + "loss": 2.6275, + "theoretical_loss": 3.310104965192477, + "tokens_seen": 3128623104 + }, + { + "epoch": 0.9, + "learning_rate": 1.0479858770662815e-05, + "loss": 2.6646, + "theoretical_loss": 3.3100943938812586, + "tokens_seen": 3128754176 + }, + { + "epoch": 0.9, + "learning_rate": 1.0471834376504575e-05, + "loss": 2.5697, + "theoretical_loss": 3.310083823136887, + "tokens_seen": 3128885248 + }, + { + "epoch": 0.9, + "learning_rate": 1.0463809982346333e-05, + "loss": 2.6366, + "theoretical_loss": 3.310073252959308, + "tokens_seen": 3129016320 + }, + { + "epoch": 0.9, + "learning_rate": 1.0455785588188093e-05, + "loss": 2.4798, + "theoretical_loss": 3.310062683348468, + "tokens_seen": 3129147392 + }, + { + "epoch": 0.9, + "learning_rate": 1.0447761194029851e-05, + "loss": 2.7337, + "theoretical_loss": 3.3100521143043116, + "tokens_seen": 3129278464 + }, + { + "epoch": 0.9, + "learning_rate": 1.043973679987161e-05, + "loss": 2.5121, + "theoretical_loss": 3.310041545826786, + "tokens_seen": 3129409536 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1714507, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6042678356170654, + "objective/train/theoretical_loss": 3.3100362618004926, + "objective/train/tokens_used": 1499934176, + "theoretical_loss": 3.3100362618004926, + "tokens_seen": 3129475072 + }, + { + "epoch": 0.9, + "learning_rate": 1.0431712405713369e-05, + "loss": 2.4964, + "theoretical_loss": 3.3100309779158366, + "tokens_seen": 3129540608 + }, + { + "epoch": 0.9, + "learning_rate": 1.0423688011555129e-05, + "loss": 2.5658, + "theoretical_loss": 3.3100204105714086, + "tokens_seen": 3129671680 + }, + { + "epoch": 0.9, + "learning_rate": 1.0415663617396887e-05, + "loss": 2.519, + "theoretical_loss": 3.310009843793449, + "tokens_seen": 3129802752 + }, + { + "epoch": 0.9, + "learning_rate": 1.0407639223238647e-05, + "loss": 2.54, + "theoretical_loss": 3.309999277581903, + "tokens_seen": 3129933824 + }, + { + "epoch": 0.9, + "learning_rate": 1.0399614829080405e-05, + "loss": 2.4669, + "theoretical_loss": 3.309988711936717, + "tokens_seen": 3130064896 + }, + { + "epoch": 0.9, + "learning_rate": 1.0391590434922163e-05, + "loss": 2.6126, + "theoretical_loss": 3.3099781468578366, + "tokens_seen": 3130195968 + }, + { + "epoch": 0.9, + "learning_rate": 1.0383566040763923e-05, + "loss": 2.5597, + "theoretical_loss": 3.309967582345208, + "tokens_seen": 3130327040 + }, + { + "epoch": 0.9, + "learning_rate": 1.0375541646605682e-05, + "loss": 2.4559, + "theoretical_loss": 3.3099570183987765, + "tokens_seen": 3130458112 + }, + { + "epoch": 0.9, + "learning_rate": 1.0367517252447442e-05, + "loss": 2.4701, + "theoretical_loss": 3.309946455018489, + "tokens_seen": 3130589184 + }, + { + "epoch": 0.9, + "learning_rate": 1.03594928582892e-05, + "loss": 2.6367, + "theoretical_loss": 3.309935892204291, + "tokens_seen": 3130720256 + }, + { + "epoch": 0.9, + "learning_rate": 1.035146846413096e-05, + "loss": 2.5464, + "theoretical_loss": 3.3099253299561284, + "tokens_seen": 3130851328 + }, + { + "epoch": 0.9, + "learning_rate": 1.0343444069972718e-05, + "loss": 2.4785, + "theoretical_loss": 3.309914768273947, + "tokens_seen": 3130982400 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1715427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5358169078826904, + "objective/train/theoretical_loss": 3.309904207157693, + "objective/train/tokens_used": 1501572576, + "theoretical_loss": 3.309904207157693, + "tokens_seen": 3131113472 + }, + { + "epoch": 0.9, + "learning_rate": 1.0335419675814478e-05, + "loss": 2.4194, + "theoretical_loss": 3.309904207157693, + "tokens_seen": 3131113472 + }, + { + "epoch": 0.9, + "learning_rate": 1.0327395281656236e-05, + "loss": 2.6343, + "theoretical_loss": 3.3098936466073132, + "tokens_seen": 3131244544 + }, + { + "epoch": 0.9, + "learning_rate": 1.0319370887497994e-05, + "loss": 2.6371, + "theoretical_loss": 3.309883086622752, + "tokens_seen": 3131375616 + }, + { + "epoch": 0.9, + "learning_rate": 1.0311346493339754e-05, + "loss": 2.6055, + "theoretical_loss": 3.3098725272039564, + "tokens_seen": 3131506688 + }, + { + "epoch": 0.9, + "learning_rate": 1.0303322099181512e-05, + "loss": 2.6404, + "theoretical_loss": 3.309861968350872, + "tokens_seen": 3131637760 + }, + { + "epoch": 0.9, + "learning_rate": 1.029529770502327e-05, + "loss": 2.5163, + "theoretical_loss": 3.3098514100634455, + "tokens_seen": 3131768832 + }, + { + "epoch": 0.9, + "learning_rate": 1.028727331086503e-05, + "loss": 2.474, + "theoretical_loss": 3.309840852341622, + "tokens_seen": 3131899904 + }, + { + "epoch": 0.9, + "learning_rate": 1.0279248916706789e-05, + "loss": 2.532, + "theoretical_loss": 3.309830295185348, + "tokens_seen": 3132030976 + }, + { + "epoch": 0.9, + "learning_rate": 1.0271224522548547e-05, + "loss": 2.5609, + "theoretical_loss": 3.3098197385945696, + "tokens_seen": 3132162048 + }, + { + "epoch": 0.9, + "learning_rate": 1.0263200128390307e-05, + "loss": 2.5897, + "theoretical_loss": 3.309809182569233, + "tokens_seen": 3132293120 + }, + { + "epoch": 0.9, + "learning_rate": 1.0255175734232065e-05, + "loss": 2.4652, + "theoretical_loss": 3.309798627109284, + "tokens_seen": 3132424192 + }, + { + "epoch": 0.9, + "learning_rate": 1.0247151340073825e-05, + "loss": 2.5453, + "theoretical_loss": 3.309788072214668, + "tokens_seen": 3132555264 + }, + { + "epoch": 0.9, + "learning_rate": 1.0239126945915583e-05, + "loss": 2.5105, + "theoretical_loss": 3.3097775178853324, + "tokens_seen": 3132686336 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1716696, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6400840282440186, + "objective/train/theoretical_loss": 3.309772240932628, + "objective/train/tokens_used": 1503210976, + "theoretical_loss": 3.309772240932628, + "tokens_seen": 3132751872 + }, + { + "epoch": 0.9, + "learning_rate": 1.0231102551757343e-05, + "loss": 2.5004, + "theoretical_loss": 3.3097669641212226, + "tokens_seen": 3132817408 + }, + { + "epoch": 0.9, + "learning_rate": 1.0223078157599101e-05, + "loss": 2.5562, + "theoretical_loss": 3.309756410922285, + "tokens_seen": 3132948480 + }, + { + "epoch": 0.9, + "learning_rate": 1.0215053763440861e-05, + "loss": 2.5891, + "theoretical_loss": 3.3097458582884647, + "tokens_seen": 3133079552 + }, + { + "epoch": 0.9, + "learning_rate": 1.020702936928262e-05, + "loss": 2.6252, + "theoretical_loss": 3.3097353062197086, + "tokens_seen": 3133210624 + }, + { + "epoch": 0.9, + "learning_rate": 1.0199004975124378e-05, + "loss": 2.5392, + "theoretical_loss": 3.3097247547159627, + "tokens_seen": 3133341696 + }, + { + "epoch": 0.9, + "learning_rate": 1.0190980580966137e-05, + "loss": 2.5984, + "theoretical_loss": 3.3097142037771734, + "tokens_seen": 3133472768 + }, + { + "epoch": 0.9, + "learning_rate": 1.0182956186807896e-05, + "loss": 2.5005, + "theoretical_loss": 3.3097036534032864, + "tokens_seen": 3133603840 + }, + { + "epoch": 0.9, + "learning_rate": 1.0174931792649656e-05, + "loss": 2.5525, + "theoretical_loss": 3.309693103594248, + "tokens_seen": 3133734912 + }, + { + "epoch": 0.9, + "learning_rate": 1.0166907398491414e-05, + "loss": 2.6241, + "theoretical_loss": 3.3096825543500046, + "tokens_seen": 3133865984 + }, + { + "epoch": 0.9, + "learning_rate": 1.0158883004333174e-05, + "loss": 2.6563, + "theoretical_loss": 3.3096720056705013, + "tokens_seen": 3133997056 + }, + { + "epoch": 0.9, + "learning_rate": 1.0150858610174932e-05, + "loss": 2.6279, + "theoretical_loss": 3.309661457555686, + "tokens_seen": 3134128128 + }, + { + "epoch": 0.9, + "learning_rate": 1.0142834216016692e-05, + "loss": 2.5493, + "theoretical_loss": 3.309650910005503, + "tokens_seen": 3134259200 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1717402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5755386352539062, + "objective/train/theoretical_loss": 3.3096403630198994, + "objective/train/tokens_used": 1504849376, + "theoretical_loss": 3.3096403630198994, + "tokens_seen": 3134390272 + }, + { + "epoch": 0.9, + "learning_rate": 1.013480982185845e-05, + "loss": 2.4246, + "theoretical_loss": 3.3096403630198994, + "tokens_seen": 3134390272 + }, + { + "epoch": 0.9, + "learning_rate": 1.012678542770021e-05, + "loss": 2.6018, + "theoretical_loss": 3.3096298165988216, + "tokens_seen": 3134521344 + }, + { + "epoch": 0.9, + "learning_rate": 1.0118761033541968e-05, + "loss": 2.5818, + "theoretical_loss": 3.3096192707422154, + "tokens_seen": 3134652416 + }, + { + "epoch": 0.9, + "learning_rate": 1.0110736639383726e-05, + "loss": 2.4255, + "theoretical_loss": 3.309608725450027, + "tokens_seen": 3134783488 + }, + { + "epoch": 0.9, + "learning_rate": 1.0102712245225486e-05, + "loss": 2.6305, + "theoretical_loss": 3.309598180722203, + "tokens_seen": 3134914560 + }, + { + "epoch": 0.9, + "learning_rate": 1.0094687851067244e-05, + "loss": 2.5496, + "theoretical_loss": 3.3095876365586885, + "tokens_seen": 3135045632 + }, + { + "epoch": 0.9, + "learning_rate": 1.0086663456909004e-05, + "loss": 2.5982, + "theoretical_loss": 3.3095770929594313, + "tokens_seen": 3135176704 + }, + { + "epoch": 0.9, + "learning_rate": 1.0078639062750763e-05, + "loss": 2.3782, + "theoretical_loss": 3.309566549924376, + "tokens_seen": 3135307776 + }, + { + "epoch": 0.9, + "learning_rate": 1.0070614668592522e-05, + "loss": 2.5887, + "theoretical_loss": 3.30955600745347, + "tokens_seen": 3135438848 + }, + { + "epoch": 0.9, + "learning_rate": 1.006259027443428e-05, + "loss": 2.489, + "theoretical_loss": 3.309545465546659, + "tokens_seen": 3135569920 + }, + { + "epoch": 0.9, + "learning_rate": 1.005456588027604e-05, + "loss": 2.5477, + "theoretical_loss": 3.3095349242038896, + "tokens_seen": 3135700992 + }, + { + "epoch": 0.9, + "learning_rate": 1.0046541486117799e-05, + "loss": 2.6541, + "theoretical_loss": 3.309524383425108, + "tokens_seen": 3135832064 + }, + { + "epoch": 0.9, + "learning_rate": 1.0038517091959559e-05, + "loss": 2.4973, + "theoretical_loss": 3.3095138432102598, + "tokens_seen": 3135963136 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1718738, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6781880855560303, + "objective/train/theoretical_loss": 3.309508573314294, + "objective/train/tokens_used": 1506487776, + "theoretical_loss": 3.309508573314294, + "tokens_seen": 3136028672 + }, + { + "epoch": 0.9, + "learning_rate": 1.0030492697801317e-05, + "loss": 2.5914, + "theoretical_loss": 3.309503303559292, + "tokens_seen": 3136094208 + }, + { + "epoch": 0.9, + "learning_rate": 1.0022468303643077e-05, + "loss": 2.7047, + "theoretical_loss": 3.30949276447215, + "tokens_seen": 3136225280 + }, + { + "epoch": 0.9, + "learning_rate": 1.0014443909484835e-05, + "loss": 2.6477, + "theoretical_loss": 3.3094822259487815, + "tokens_seen": 3136356352 + }, + { + "epoch": 0.9, + "learning_rate": 1.0006419515326593e-05, + "loss": 2.6313, + "theoretical_loss": 3.3094716879891317, + "tokens_seen": 3136487424 + }, + { + "epoch": 0.9, + "learning_rate": 9.998395121168353e-06, + "loss": 2.436, + "theoretical_loss": 3.309461150593147, + "tokens_seen": 3136618496 + }, + { + "epoch": 0.9, + "learning_rate": 9.990370727010111e-06, + "loss": 2.5183, + "theoretical_loss": 3.3094506137607738, + "tokens_seen": 3136749568 + }, + { + "epoch": 0.9, + "learning_rate": 9.982346332851871e-06, + "loss": 2.6519, + "theoretical_loss": 3.309440077491959, + "tokens_seen": 3136880640 + }, + { + "epoch": 0.9, + "learning_rate": 9.97432193869363e-06, + "loss": 2.5354, + "theoretical_loss": 3.3094295417866477, + "tokens_seen": 3137011712 + }, + { + "epoch": 0.9, + "learning_rate": 9.966297544535388e-06, + "loss": 2.6234, + "theoretical_loss": 3.309419006644787, + "tokens_seen": 3137142784 + }, + { + "epoch": 0.9, + "learning_rate": 9.958273150377148e-06, + "loss": 2.387, + "theoretical_loss": 3.3094084720663233, + "tokens_seen": 3137273856 + }, + { + "epoch": 0.9, + "learning_rate": 9.950248756218906e-06, + "loss": 2.6243, + "theoretical_loss": 3.3093979380512026, + "tokens_seen": 3137404928 + }, + { + "epoch": 0.9, + "learning_rate": 9.942224362060664e-06, + "loss": 2.5063, + "theoretical_loss": 3.309387404599372, + "tokens_seen": 3137536000 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1719408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5275678634643555, + "objective/train/theoretical_loss": 3.3093768717107763, + "objective/train/tokens_used": 1508126176, + "theoretical_loss": 3.3093768717107763, + "tokens_seen": 3137667072 + }, + { + "epoch": 0.9, + "learning_rate": 9.934199967902424e-06, + "loss": 2.545, + "theoretical_loss": 3.3093768717107763, + "tokens_seen": 3137667072 + }, + { + "epoch": 0.9, + "learning_rate": 9.926175573744182e-06, + "loss": 2.5015, + "theoretical_loss": 3.309366339385363, + "tokens_seen": 3137798144 + }, + { + "epoch": 0.9, + "learning_rate": 9.91815117958594e-06, + "loss": 2.6475, + "theoretical_loss": 3.3093558076230787, + "tokens_seen": 3137929216 + }, + { + "epoch": 0.9, + "learning_rate": 9.9101267854277e-06, + "loss": 2.4832, + "theoretical_loss": 3.309345276423869, + "tokens_seen": 3138060288 + }, + { + "epoch": 0.9, + "learning_rate": 9.902102391269458e-06, + "loss": 2.5778, + "theoretical_loss": 3.3093347457876807, + "tokens_seen": 3138191360 + }, + { + "epoch": 0.9, + "learning_rate": 9.894077997111218e-06, + "loss": 2.4673, + "theoretical_loss": 3.3093242157144602, + "tokens_seen": 3138322432 + }, + { + "epoch": 0.9, + "learning_rate": 9.886053602952977e-06, + "loss": 2.6149, + "theoretical_loss": 3.309313686204154, + "tokens_seen": 3138453504 + }, + { + "epoch": 0.9, + "learning_rate": 9.878029208794736e-06, + "loss": 2.5367, + "theoretical_loss": 3.309303157256708, + "tokens_seen": 3138584576 + }, + { + "epoch": 0.9, + "learning_rate": 9.870004814636495e-06, + "loss": 2.4776, + "theoretical_loss": 3.3092926288720688, + "tokens_seen": 3138715648 + }, + { + "epoch": 0.9, + "learning_rate": 9.861980420478255e-06, + "loss": 2.6212, + "theoretical_loss": 3.309282101050183, + "tokens_seen": 3138846720 + }, + { + "epoch": 0.9, + "learning_rate": 9.853956026320013e-06, + "loss": 2.6224, + "theoretical_loss": 3.309271573790997, + "tokens_seen": 3138977792 + }, + { + "epoch": 0.9, + "learning_rate": 9.845931632161773e-06, + "loss": 2.5521, + "theoretical_loss": 3.3092610470944566, + "tokens_seen": 3139108864 + }, + { + "epoch": 0.9, + "learning_rate": 9.837907238003531e-06, + "loss": 2.5669, + "theoretical_loss": 3.3092505209605094, + "tokens_seen": 3139239936 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1720490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7142200469970703, + "objective/train/theoretical_loss": 3.3092452581044913, + "objective/train/tokens_used": 1509764576, + "theoretical_loss": 3.3092452581044913, + "tokens_seen": 3139305472 + }, + { + "epoch": 0.9, + "learning_rate": 9.82988284384529e-06, + "loss": 2.3659, + "theoretical_loss": 3.309239995389101, + "tokens_seen": 3139371008 + }, + { + "epoch": 0.9, + "learning_rate": 9.821858449687049e-06, + "loss": 2.5265, + "theoretical_loss": 3.3092294703801786, + "tokens_seen": 3139502080 + }, + { + "epoch": 0.9, + "learning_rate": 9.813834055528807e-06, + "loss": 2.4745, + "theoretical_loss": 3.309218945933688, + "tokens_seen": 3139633152 + }, + { + "epoch": 0.9, + "learning_rate": 9.805809661370567e-06, + "loss": 2.4884, + "theoretical_loss": 3.309208422049575, + "tokens_seen": 3139764224 + }, + { + "epoch": 0.9, + "learning_rate": 9.797785267212325e-06, + "loss": 2.5276, + "theoretical_loss": 3.309197898727788, + "tokens_seen": 3139895296 + }, + { + "epoch": 0.9, + "learning_rate": 9.789760873054085e-06, + "loss": 2.6418, + "theoretical_loss": 3.3091873759682717, + "tokens_seen": 3140026368 + }, + { + "epoch": 0.9, + "learning_rate": 9.781736478895843e-06, + "loss": 2.5892, + "theoretical_loss": 3.3091768537709734, + "tokens_seen": 3140157440 + }, + { + "epoch": 0.9, + "learning_rate": 9.773712084737603e-06, + "loss": 2.378, + "theoretical_loss": 3.3091663321358395, + "tokens_seen": 3140288512 + }, + { + "epoch": 0.9, + "learning_rate": 9.765687690579362e-06, + "loss": 2.5804, + "theoretical_loss": 3.3091558110628165, + "tokens_seen": 3140419584 + }, + { + "epoch": 0.9, + "learning_rate": 9.757663296421122e-06, + "loss": 2.5533, + "theoretical_loss": 3.309145290551851, + "tokens_seen": 3140550656 + }, + { + "epoch": 0.9, + "learning_rate": 9.74963890226288e-06, + "loss": 2.6254, + "theoretical_loss": 3.3091347706028893, + "tokens_seen": 3140681728 + }, + { + "epoch": 0.9, + "learning_rate": 9.74161450810464e-06, + "loss": 2.5446, + "theoretical_loss": 3.309124251215878, + "tokens_seen": 3140812800 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1720925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1672134399414062, + "objective/train/theoretical_loss": 3.3091137323907636, + "objective/train/tokens_used": 1511402976, + "theoretical_loss": 3.3091137323907636, + "tokens_seen": 3140943872 + }, + { + "epoch": 0.9, + "learning_rate": 9.733590113946398e-06, + "loss": 2.3569, + "theoretical_loss": 3.3091137323907636, + "tokens_seen": 3140943872 + }, + { + "epoch": 0.9, + "learning_rate": 9.725565719788156e-06, + "loss": 2.5222, + "theoretical_loss": 3.309103214127493, + "tokens_seen": 3141074944 + }, + { + "epoch": 0.9, + "learning_rate": 9.717541325629916e-06, + "loss": 2.53, + "theoretical_loss": 3.3090926964260117, + "tokens_seen": 3141206016 + }, + { + "epoch": 0.9, + "learning_rate": 9.709516931471674e-06, + "loss": 2.4724, + "theoretical_loss": 3.3090821792862677, + "tokens_seen": 3141337088 + }, + { + "epoch": 0.9, + "learning_rate": 9.701492537313434e-06, + "loss": 2.534, + "theoretical_loss": 3.309071662708207, + "tokens_seen": 3141468160 + }, + { + "epoch": 0.9, + "learning_rate": 9.693468143155192e-06, + "loss": 2.5598, + "theoretical_loss": 3.3090611466917754, + "tokens_seen": 3141599232 + }, + { + "epoch": 0.9, + "learning_rate": 9.685443748996952e-06, + "loss": 2.647, + "theoretical_loss": 3.30905063123692, + "tokens_seen": 3141730304 + }, + { + "epoch": 0.9, + "learning_rate": 9.67741935483871e-06, + "loss": 2.5861, + "theoretical_loss": 3.309040116343588, + "tokens_seen": 3141861376 + }, + { + "epoch": 0.9, + "learning_rate": 9.66939496068047e-06, + "loss": 2.4945, + "theoretical_loss": 3.3090296020117256, + "tokens_seen": 3141992448 + }, + { + "epoch": 0.9, + "learning_rate": 9.661370566522229e-06, + "loss": 2.4644, + "theoretical_loss": 3.309019088241279, + "tokens_seen": 3142123520 + }, + { + "epoch": 0.9, + "learning_rate": 9.653346172363987e-06, + "loss": 2.3733, + "theoretical_loss": 3.309008575032195, + "tokens_seen": 3142254592 + }, + { + "epoch": 0.9, + "learning_rate": 9.645321778205747e-06, + "loss": 2.5825, + "theoretical_loss": 3.30899806238442, + "tokens_seen": 3142385664 + }, + { + "epoch": 0.9, + "learning_rate": 9.637297384047505e-06, + "loss": 2.5129, + "theoretical_loss": 3.3089875502979016, + "tokens_seen": 3142516736 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1722164, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4004015922546387, + "objective/train/theoretical_loss": 3.3089822944650966, + "objective/train/tokens_used": 1513041376, + "theoretical_loss": 3.3089822944650966, + "tokens_seen": 3142582272 + }, + { + "epoch": 0.9, + "learning_rate": 9.629272989889263e-06, + "loss": 2.5178, + "theoretical_loss": 3.3089770387725856, + "tokens_seen": 3142647808 + }, + { + "epoch": 0.9, + "learning_rate": 9.621248595731023e-06, + "loss": 2.4271, + "theoretical_loss": 3.3089665278084186, + "tokens_seen": 3142778880 + }, + { + "epoch": 0.9, + "learning_rate": 9.613224201572781e-06, + "loss": 2.6075, + "theoretical_loss": 3.3089560174053476, + "tokens_seen": 3142909952 + }, + { + "epoch": 0.9, + "learning_rate": 9.60519980741454e-06, + "loss": 2.526, + "theoretical_loss": 3.308945507563319, + "tokens_seen": 3143041024 + }, + { + "epoch": 0.9, + "learning_rate": 9.5971754132563e-06, + "loss": 2.5601, + "theoretical_loss": 3.3089349982822793, + "tokens_seen": 3143172096 + }, + { + "epoch": 0.91, + "learning_rate": 9.589151019098058e-06, + "loss": 2.4336, + "theoretical_loss": 3.3089244895621754, + "tokens_seen": 3143303168 + }, + { + "epoch": 0.91, + "learning_rate": 9.581126624939817e-06, + "loss": 2.5974, + "theoretical_loss": 3.308913981402954, + "tokens_seen": 3143434240 + }, + { + "epoch": 0.91, + "learning_rate": 9.573102230781576e-06, + "loss": 2.5294, + "theoretical_loss": 3.3089034738045617, + "tokens_seen": 3143565312 + }, + { + "epoch": 0.91, + "learning_rate": 9.565077836623336e-06, + "loss": 2.6321, + "theoretical_loss": 3.3088929667669458, + "tokens_seen": 3143696384 + }, + { + "epoch": 0.91, + "learning_rate": 9.557053442465094e-06, + "loss": 2.635, + "theoretical_loss": 3.308882460290052, + "tokens_seen": 3143827456 + }, + { + "epoch": 0.91, + "learning_rate": 9.549029048306854e-06, + "loss": 2.7054, + "theoretical_loss": 3.308871954373827, + "tokens_seen": 3143958528 + }, + { + "epoch": 0.91, + "learning_rate": 9.541004654148612e-06, + "loss": 2.7193, + "theoretical_loss": 3.3088614490182184, + "tokens_seen": 3144089600 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1722628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5680460929870605, + "objective/train/theoretical_loss": 3.308850944223172, + "objective/train/tokens_used": 1514679776, + "theoretical_loss": 3.308850944223172, + "tokens_seen": 3144220672 + }, + { + "epoch": 0.91, + "learning_rate": 9.53298025999037e-06, + "loss": 2.6032, + "theoretical_loss": 3.308850944223172, + "tokens_seen": 3144220672 + }, + { + "epoch": 0.91, + "learning_rate": 9.52495586583213e-06, + "loss": 2.566, + "theoretical_loss": 3.3088404399886358, + "tokens_seen": 3144351744 + }, + { + "epoch": 0.91, + "learning_rate": 9.516931471673888e-06, + "loss": 2.4986, + "theoretical_loss": 3.308829936314555, + "tokens_seen": 3144482816 + }, + { + "epoch": 0.91, + "learning_rate": 9.508907077515648e-06, + "loss": 2.5214, + "theoretical_loss": 3.3088194332008776, + "tokens_seen": 3144613888 + }, + { + "epoch": 0.91, + "learning_rate": 9.500882683357406e-06, + "loss": 2.5215, + "theoretical_loss": 3.308808930647549, + "tokens_seen": 3144744960 + }, + { + "epoch": 0.91, + "learning_rate": 9.492858289199166e-06, + "loss": 2.5335, + "theoretical_loss": 3.3087984286545176, + "tokens_seen": 3144876032 + }, + { + "epoch": 0.91, + "learning_rate": 9.484833895040924e-06, + "loss": 2.5111, + "theoretical_loss": 3.308787927221729, + "tokens_seen": 3145007104 + }, + { + "epoch": 0.91, + "learning_rate": 9.476809500882684e-06, + "loss": 2.5215, + "theoretical_loss": 3.30877742634913, + "tokens_seen": 3145138176 + }, + { + "epoch": 0.91, + "learning_rate": 9.468785106724443e-06, + "loss": 2.6229, + "theoretical_loss": 3.3087669260366677, + "tokens_seen": 3145269248 + }, + { + "epoch": 0.91, + "learning_rate": 9.460760712566202e-06, + "loss": 2.6987, + "theoretical_loss": 3.3087564262842886, + "tokens_seen": 3145400320 + }, + { + "epoch": 0.91, + "learning_rate": 9.45273631840796e-06, + "loss": 2.5827, + "theoretical_loss": 3.30874592709194, + "tokens_seen": 3145531392 + }, + { + "epoch": 0.91, + "learning_rate": 9.444711924249719e-06, + "loss": 2.5485, + "theoretical_loss": 3.3087354284595687, + "tokens_seen": 3145662464 + }, + { + "epoch": 0.91, + "learning_rate": 9.436687530091479e-06, + "loss": 2.3054, + "theoretical_loss": 3.308724930387121, + "tokens_seen": 3145793536 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1723860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7178304195404053, + "objective/train/theoretical_loss": 3.3087196815608517, + "objective/train/tokens_used": 1516318176, + "theoretical_loss": 3.3087196815608517, + "tokens_seen": 3145859072 + }, + { + "epoch": 0.91, + "learning_rate": 9.428663135933237e-06, + "loss": 2.4766, + "theoretical_loss": 3.3087144328745435, + "tokens_seen": 3145924608 + }, + { + "epoch": 0.91, + "learning_rate": 9.420638741774997e-06, + "loss": 2.393, + "theoretical_loss": 3.3087039359217836, + "tokens_seen": 3146055680 + }, + { + "epoch": 0.91, + "learning_rate": 9.412614347616755e-06, + "loss": 2.5871, + "theoretical_loss": 3.308693439528788, + "tokens_seen": 3146186752 + }, + { + "epoch": 0.91, + "learning_rate": 9.404589953458515e-06, + "loss": 2.549, + "theoretical_loss": 3.308682943695503, + "tokens_seen": 3146317824 + }, + { + "epoch": 0.91, + "learning_rate": 9.396565559300273e-06, + "loss": 2.5632, + "theoretical_loss": 3.308672448421877, + "tokens_seen": 3146448896 + }, + { + "epoch": 0.91, + "learning_rate": 9.388541165142033e-06, + "loss": 2.5747, + "theoretical_loss": 3.3086619537078548, + "tokens_seen": 3146579968 + }, + { + "epoch": 0.91, + "learning_rate": 9.380516770983791e-06, + "loss": 2.4939, + "theoretical_loss": 3.308651459553385, + "tokens_seen": 3146711040 + }, + { + "epoch": 0.91, + "learning_rate": 9.372492376825551e-06, + "loss": 2.3254, + "theoretical_loss": 3.3086409659584133, + "tokens_seen": 3146842112 + }, + { + "epoch": 0.91, + "learning_rate": 9.36446798266731e-06, + "loss": 2.5336, + "theoretical_loss": 3.3086304729228866, + "tokens_seen": 3146973184 + }, + { + "epoch": 0.91, + "learning_rate": 9.356443588509068e-06, + "loss": 2.5239, + "theoretical_loss": 3.3086199804467524, + "tokens_seen": 3147104256 + }, + { + "epoch": 0.91, + "learning_rate": 9.348419194350828e-06, + "loss": 2.5708, + "theoretical_loss": 3.3086094885299575, + "tokens_seen": 3147235328 + }, + { + "epoch": 0.91, + "learning_rate": 9.340394800192586e-06, + "loss": 2.5893, + "theoretical_loss": 3.3085989971724485, + "tokens_seen": 3147366400 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1724798, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.363929271697998, + "objective/train/theoretical_loss": 3.3085885063741722, + "objective/train/tokens_used": 1517956576, + "theoretical_loss": 3.3085885063741722, + "tokens_seen": 3147497472 + }, + { + "epoch": 0.91, + "learning_rate": 9.332370406034346e-06, + "loss": 2.5637, + "theoretical_loss": 3.3085885063741722, + "tokens_seen": 3147497472 + }, + { + "epoch": 0.91, + "learning_rate": 9.324346011876104e-06, + "loss": 2.7041, + "theoretical_loss": 3.3085780161350757, + "tokens_seen": 3147628544 + }, + { + "epoch": 0.91, + "learning_rate": 9.316321617717864e-06, + "loss": 2.466, + "theoretical_loss": 3.308567526455106, + "tokens_seen": 3147759616 + }, + { + "epoch": 0.91, + "learning_rate": 9.308297223559622e-06, + "loss": 2.4759, + "theoretical_loss": 3.30855703733421, + "tokens_seen": 3147890688 + }, + { + "epoch": 0.91, + "learning_rate": 9.30027282940138e-06, + "loss": 2.5232, + "theoretical_loss": 3.3085465487723345, + "tokens_seen": 3148021760 + }, + { + "epoch": 0.91, + "learning_rate": 9.29224843524314e-06, + "loss": 2.4897, + "theoretical_loss": 3.308536060769426, + "tokens_seen": 3148152832 + }, + { + "epoch": 0.91, + "learning_rate": 9.284224041084898e-06, + "loss": 2.552, + "theoretical_loss": 3.308525573325433, + "tokens_seen": 3148283904 + }, + { + "epoch": 0.91, + "learning_rate": 9.276199646926657e-06, + "loss": 2.495, + "theoretical_loss": 3.3085150864403, + "tokens_seen": 3148414976 + }, + { + "epoch": 0.91, + "learning_rate": 9.268175252768416e-06, + "loss": 2.6086, + "theoretical_loss": 3.3085046001139764, + "tokens_seen": 3148546048 + }, + { + "epoch": 0.91, + "learning_rate": 9.260150858610175e-06, + "loss": 2.5034, + "theoretical_loss": 3.3084941143464075, + "tokens_seen": 3148677120 + }, + { + "epoch": 0.91, + "learning_rate": 9.252126464451933e-06, + "loss": 2.535, + "theoretical_loss": 3.308483629137541, + "tokens_seen": 3148808192 + }, + { + "epoch": 0.91, + "learning_rate": 9.244102070293693e-06, + "loss": 2.6302, + "theoretical_loss": 3.308473144487324, + "tokens_seen": 3148939264 + }, + { + "epoch": 0.91, + "learning_rate": 9.236077676135451e-06, + "loss": 2.6235, + "theoretical_loss": 3.308462660395703, + "tokens_seen": 3149070336 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1725490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.356010913848877, + "objective/train/theoretical_loss": 3.3084574185593496, + "objective/train/tokens_used": 1519594976, + "theoretical_loss": 3.3084574185593496, + "tokens_seen": 3149135872 + }, + { + "epoch": 0.91, + "learning_rate": 9.228053281977211e-06, + "loss": 2.6148, + "theoretical_loss": 3.3084521768626254, + "tokens_seen": 3149201408 + }, + { + "epoch": 0.91, + "learning_rate": 9.220028887818969e-06, + "loss": 2.7329, + "theoretical_loss": 3.308441693888038, + "tokens_seen": 3149332480 + }, + { + "epoch": 0.91, + "learning_rate": 9.212004493660729e-06, + "loss": 2.6242, + "theoretical_loss": 3.3084312114718877, + "tokens_seen": 3149463552 + }, + { + "epoch": 0.91, + "learning_rate": 9.203980099502487e-06, + "loss": 2.6423, + "theoretical_loss": 3.3084207296141215, + "tokens_seen": 3149594624 + }, + { + "epoch": 0.91, + "learning_rate": 9.195955705344247e-06, + "loss": 2.5863, + "theoretical_loss": 3.3084102483146864, + "tokens_seen": 3149725696 + }, + { + "epoch": 0.91, + "learning_rate": 9.187931311186005e-06, + "loss": 2.4444, + "theoretical_loss": 3.3083997675735297, + "tokens_seen": 3149856768 + }, + { + "epoch": 0.91, + "learning_rate": 9.179906917027765e-06, + "loss": 2.5798, + "theoretical_loss": 3.3083892873905985, + "tokens_seen": 3149987840 + }, + { + "epoch": 0.91, + "learning_rate": 9.171882522869524e-06, + "loss": 2.5112, + "theoretical_loss": 3.30837880776584, + "tokens_seen": 3150118912 + }, + { + "epoch": 0.91, + "learning_rate": 9.163858128711282e-06, + "loss": 2.5177, + "theoretical_loss": 3.3083683286992, + "tokens_seen": 3150249984 + }, + { + "epoch": 0.91, + "learning_rate": 9.155833734553042e-06, + "loss": 2.3793, + "theoretical_loss": 3.3083578501906272, + "tokens_seen": 3150381056 + }, + { + "epoch": 0.91, + "learning_rate": 9.1478093403948e-06, + "loss": 2.654, + "theoretical_loss": 3.3083473722400676, + "tokens_seen": 3150512128 + }, + { + "epoch": 0.91, + "learning_rate": 9.13978494623656e-06, + "loss": 2.6525, + "theoretical_loss": 3.3083368948474683, + "tokens_seen": 3150643200 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1726849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0358283519744873, + "objective/train/theoretical_loss": 3.308326418012777, + "objective/train/tokens_used": 1521233376, + "theoretical_loss": 3.308326418012777, + "tokens_seen": 3150774272 + }, + { + "epoch": 0.91, + "learning_rate": 9.131760552078318e-06, + "loss": 2.5067, + "theoretical_loss": 3.308326418012777, + "tokens_seen": 3150774272 + }, + { + "epoch": 0.91, + "learning_rate": 9.123736157920078e-06, + "loss": 2.6684, + "theoretical_loss": 3.3083159417359402, + "tokens_seen": 3150905344 + }, + { + "epoch": 0.91, + "learning_rate": 9.115711763761836e-06, + "loss": 2.5225, + "theoretical_loss": 3.308305466016906, + "tokens_seen": 3151036416 + }, + { + "epoch": 0.91, + "learning_rate": 9.107687369603596e-06, + "loss": 2.5059, + "theoretical_loss": 3.30829499085562, + "tokens_seen": 3151167488 + }, + { + "epoch": 0.91, + "learning_rate": 9.099662975445354e-06, + "loss": 2.7393, + "theoretical_loss": 3.30828451625203, + "tokens_seen": 3151298560 + }, + { + "epoch": 0.91, + "learning_rate": 9.091638581287114e-06, + "loss": 2.5419, + "theoretical_loss": 3.3082740422060835, + "tokens_seen": 3151429632 + }, + { + "epoch": 0.91, + "learning_rate": 9.083614187128872e-06, + "loss": 2.5089, + "theoretical_loss": 3.3082635687177273, + "tokens_seen": 3151560704 + }, + { + "epoch": 0.91, + "learning_rate": 9.075589792970632e-06, + "loss": 2.5569, + "theoretical_loss": 3.3082530957869083, + "tokens_seen": 3151691776 + }, + { + "epoch": 0.91, + "learning_rate": 9.06756539881239e-06, + "loss": 2.4563, + "theoretical_loss": 3.308242623413574, + "tokens_seen": 3151822848 + }, + { + "epoch": 0.91, + "learning_rate": 9.059541004654149e-06, + "loss": 2.5578, + "theoretical_loss": 3.308232151597671, + "tokens_seen": 3151953920 + }, + { + "epoch": 0.91, + "learning_rate": 9.051516610495909e-06, + "loss": 2.5878, + "theoretical_loss": 3.3082216803391473, + "tokens_seen": 3152084992 + }, + { + "epoch": 0.91, + "learning_rate": 9.043492216337667e-06, + "loss": 2.5095, + "theoretical_loss": 3.3082112096379497, + "tokens_seen": 3152216064 + }, + { + "epoch": 0.91, + "learning_rate": 9.035467822179427e-06, + "loss": 2.5683, + "theoretical_loss": 3.308200739494025, + "tokens_seen": 3152347136 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1727307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.195695161819458, + "objective/train/theoretical_loss": 3.3081955046310236, + "objective/train/tokens_used": 1522871776, + "theoretical_loss": 3.3081955046310236, + "tokens_seen": 3152412672 + }, + { + "epoch": 0.91, + "learning_rate": 9.027443428021185e-06, + "loss": 2.3953, + "theoretical_loss": 3.3081902699073207, + "tokens_seen": 3152478208 + }, + { + "epoch": 0.91, + "learning_rate": 9.019419033862945e-06, + "loss": 2.4892, + "theoretical_loss": 3.3081798008777836, + "tokens_seen": 3152609280 + }, + { + "epoch": 0.91, + "learning_rate": 9.011394639704703e-06, + "loss": 2.5436, + "theoretical_loss": 3.3081693324053614, + "tokens_seen": 3152740352 + }, + { + "epoch": 0.91, + "learning_rate": 9.003370245546463e-06, + "loss": 2.5276, + "theoretical_loss": 3.308158864490001, + "tokens_seen": 3152871424 + }, + { + "epoch": 0.91, + "learning_rate": 8.995345851388221e-06, + "loss": 2.5391, + "theoretical_loss": 3.3081483971316503, + "tokens_seen": 3153002496 + }, + { + "epoch": 0.91, + "learning_rate": 8.987321457229981e-06, + "loss": 2.5138, + "theoretical_loss": 3.3081379303302554, + "tokens_seen": 3153133568 + }, + { + "epoch": 0.91, + "learning_rate": 8.97929706307174e-06, + "loss": 2.5185, + "theoretical_loss": 3.3081274640857643, + "tokens_seen": 3153264640 + }, + { + "epoch": 0.91, + "learning_rate": 8.971272668913497e-06, + "loss": 2.3524, + "theoretical_loss": 3.3081169983981233, + "tokens_seen": 3153395712 + }, + { + "epoch": 0.91, + "learning_rate": 8.963248274755257e-06, + "loss": 2.6039, + "theoretical_loss": 3.3081065332672805, + "tokens_seen": 3153526784 + }, + { + "epoch": 0.91, + "learning_rate": 8.955223880597016e-06, + "loss": 2.5944, + "theoretical_loss": 3.3080960686931835, + "tokens_seen": 3153657856 + }, + { + "epoch": 0.91, + "learning_rate": 8.947199486438774e-06, + "loss": 2.4836, + "theoretical_loss": 3.3080856046757785, + "tokens_seen": 3153788928 + }, + { + "epoch": 0.91, + "learning_rate": 8.939175092280534e-06, + "loss": 2.5118, + "theoretical_loss": 3.308075141215013, + "tokens_seen": 3153920000 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1728625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.475020170211792, + "objective/train/theoretical_loss": 3.3080646783108345, + "objective/train/tokens_used": 1524510176, + "theoretical_loss": 3.3080646783108345, + "tokens_seen": 3154051072 + }, + { + "epoch": 0.91, + "learning_rate": 8.931150698122292e-06, + "loss": 2.7133, + "theoretical_loss": 3.3080646783108345, + "tokens_seen": 3154051072 + }, + { + "epoch": 0.91, + "learning_rate": 8.92312630396405e-06, + "loss": 2.6705, + "theoretical_loss": 3.3080542159631907, + "tokens_seen": 3154182144 + }, + { + "epoch": 0.91, + "learning_rate": 8.91510190980581e-06, + "loss": 2.4504, + "theoretical_loss": 3.308043754172028, + "tokens_seen": 3154313216 + }, + { + "epoch": 0.91, + "learning_rate": 8.907077515647568e-06, + "loss": 2.5449, + "theoretical_loss": 3.308033292937294, + "tokens_seen": 3154444288 + }, + { + "epoch": 0.91, + "learning_rate": 8.899053121489328e-06, + "loss": 2.5362, + "theoretical_loss": 3.3080228322589362, + "tokens_seen": 3154575360 + }, + { + "epoch": 0.91, + "learning_rate": 8.891028727331086e-06, + "loss": 2.5483, + "theoretical_loss": 3.308012372136902, + "tokens_seen": 3154706432 + }, + { + "epoch": 0.91, + "learning_rate": 8.883004333172845e-06, + "loss": 2.6468, + "theoretical_loss": 3.3080019125711377, + "tokens_seen": 3154837504 + }, + { + "epoch": 0.91, + "learning_rate": 8.874979939014604e-06, + "loss": 2.5622, + "theoretical_loss": 3.307991453561592, + "tokens_seen": 3154968576 + }, + { + "epoch": 0.91, + "learning_rate": 8.866955544856363e-06, + "loss": 2.4754, + "theoretical_loss": 3.3079809951082115, + "tokens_seen": 3155099648 + }, + { + "epoch": 0.91, + "learning_rate": 8.858931150698123e-06, + "loss": 2.6656, + "theoretical_loss": 3.3079705372109434, + "tokens_seen": 3155230720 + }, + { + "epoch": 0.91, + "learning_rate": 8.85090675653988e-06, + "loss": 2.4191, + "theoretical_loss": 3.3079600798697353, + "tokens_seen": 3155361792 + }, + { + "epoch": 0.91, + "learning_rate": 8.84288236238164e-06, + "loss": 2.4757, + "theoretical_loss": 3.307949623084535, + "tokens_seen": 3155492864 + }, + { + "epoch": 0.91, + "learning_rate": 8.834857968223399e-06, + "loss": 2.5734, + "theoretical_loss": 3.307939166855289, + "tokens_seen": 3155623936 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1729266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.635824680328369, + "objective/train/theoretical_loss": 3.307933938949132, + "objective/train/tokens_used": 1526148576, + "theoretical_loss": 3.307933938949132, + "tokens_seen": 3155689472 + }, + { + "epoch": 0.91, + "learning_rate": 8.826833574065159e-06, + "loss": 2.6325, + "theoretical_loss": 3.3079287111819444, + "tokens_seen": 3155755008 + }, + { + "epoch": 0.91, + "learning_rate": 8.818809179906917e-06, + "loss": 2.4154, + "theoretical_loss": 3.3079182560644496, + "tokens_seen": 3155886080 + }, + { + "epoch": 0.91, + "learning_rate": 8.810784785748677e-06, + "loss": 2.4646, + "theoretical_loss": 3.3079078015027514, + "tokens_seen": 3156017152 + }, + { + "epoch": 0.91, + "learning_rate": 8.802760391590435e-06, + "loss": 2.5834, + "theoretical_loss": 3.307897347496797, + "tokens_seen": 3156148224 + }, + { + "epoch": 0.91, + "learning_rate": 8.794735997432195e-06, + "loss": 2.602, + "theoretical_loss": 3.3078868940465345, + "tokens_seen": 3156279296 + }, + { + "epoch": 0.91, + "learning_rate": 8.786711603273953e-06, + "loss": 2.3855, + "theoretical_loss": 3.3078764411519104, + "tokens_seen": 3156410368 + }, + { + "epoch": 0.91, + "learning_rate": 8.778687209115711e-06, + "loss": 2.5673, + "theoretical_loss": 3.307865988812873, + "tokens_seen": 3156541440 + }, + { + "epoch": 0.91, + "learning_rate": 8.770662814957471e-06, + "loss": 2.5488, + "theoretical_loss": 3.307855537029369, + "tokens_seen": 3156672512 + }, + { + "epoch": 0.91, + "learning_rate": 8.76263842079923e-06, + "loss": 2.6169, + "theoretical_loss": 3.307845085801346, + "tokens_seen": 3156803584 + }, + { + "epoch": 0.91, + "learning_rate": 8.75461402664099e-06, + "loss": 2.5839, + "theoretical_loss": 3.307834635128751, + "tokens_seen": 3156934656 + }, + { + "epoch": 0.91, + "learning_rate": 8.746589632482748e-06, + "loss": 2.6098, + "theoretical_loss": 3.3078241850115324, + "tokens_seen": 3157065728 + }, + { + "epoch": 0.91, + "learning_rate": 8.738565238324508e-06, + "loss": 2.519, + "theoretical_loss": 3.307813735449637, + "tokens_seen": 3157196800 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1730350, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6037704944610596, + "objective/train/theoretical_loss": 3.3078032864430122, + "objective/train/tokens_used": 1527786976, + "theoretical_loss": 3.3078032864430122, + "tokens_seen": 3157327872 + }, + { + "epoch": 0.91, + "learning_rate": 8.730540844166266e-06, + "loss": 2.6163, + "theoretical_loss": 3.3078032864430122, + "tokens_seen": 3157327872 + }, + { + "epoch": 0.91, + "learning_rate": 8.722516450008026e-06, + "loss": 2.4449, + "theoretical_loss": 3.307792837991606, + "tokens_seen": 3157458944 + }, + { + "epoch": 0.91, + "learning_rate": 8.714492055849784e-06, + "loss": 2.5806, + "theoretical_loss": 3.3077823900953653, + "tokens_seen": 3157590016 + }, + { + "epoch": 0.91, + "learning_rate": 8.706467661691544e-06, + "loss": 2.512, + "theoretical_loss": 3.307771942754237, + "tokens_seen": 3157721088 + }, + { + "epoch": 0.91, + "learning_rate": 8.698443267533302e-06, + "loss": 2.5633, + "theoretical_loss": 3.3077614959681703, + "tokens_seen": 3157852160 + }, + { + "epoch": 0.91, + "learning_rate": 8.69041887337506e-06, + "loss": 2.3617, + "theoretical_loss": 3.307751049737111, + "tokens_seen": 3157983232 + }, + { + "epoch": 0.91, + "learning_rate": 8.68239447921682e-06, + "loss": 2.542, + "theoretical_loss": 3.3077406040610073, + "tokens_seen": 3158114304 + }, + { + "epoch": 0.91, + "learning_rate": 8.674370085058578e-06, + "loss": 2.5614, + "theoretical_loss": 3.307730158939807, + "tokens_seen": 3158245376 + }, + { + "epoch": 0.91, + "learning_rate": 8.666345690900338e-06, + "loss": 2.4909, + "theoretical_loss": 3.3077197143734565, + "tokens_seen": 3158376448 + }, + { + "epoch": 0.91, + "learning_rate": 8.658321296742097e-06, + "loss": 2.4515, + "theoretical_loss": 3.3077092703619044, + "tokens_seen": 3158507520 + }, + { + "epoch": 0.91, + "learning_rate": 8.650296902583856e-06, + "loss": 2.4966, + "theoretical_loss": 3.3076988269050975, + "tokens_seen": 3158638592 + }, + { + "epoch": 0.91, + "learning_rate": 8.642272508425615e-06, + "loss": 2.5228, + "theoretical_loss": 3.307688384002984, + "tokens_seen": 3158769664 + }, + { + "epoch": 0.91, + "learning_rate": 8.634248114267373e-06, + "loss": 2.696, + "theoretical_loss": 3.307677941655511, + "tokens_seen": 3158900736 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1730945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.13921856880188, + "objective/train/theoretical_loss": 3.307672720689748, + "objective/train/tokens_used": 1529425376, + "theoretical_loss": 3.307672720689748, + "tokens_seen": 3158966272 + }, + { + "epoch": 0.91, + "learning_rate": 8.626223720109133e-06, + "loss": 2.4546, + "theoretical_loss": 3.307667499862626, + "tokens_seen": 3159031808 + }, + { + "epoch": 0.91, + "learning_rate": 8.618199325950891e-06, + "loss": 2.3995, + "theoretical_loss": 3.3076570586242764, + "tokens_seen": 3159162880 + }, + { + "epoch": 0.91, + "learning_rate": 8.61017493179265e-06, + "loss": 2.4701, + "theoretical_loss": 3.30764661794041, + "tokens_seen": 3159293952 + }, + { + "epoch": 0.91, + "learning_rate": 8.602150537634409e-06, + "loss": 2.3651, + "theoretical_loss": 3.3076361778109744, + "tokens_seen": 3159425024 + }, + { + "epoch": 0.91, + "learning_rate": 8.594126143476167e-06, + "loss": 2.5344, + "theoretical_loss": 3.3076257382359175, + "tokens_seen": 3159556096 + }, + { + "epoch": 0.91, + "learning_rate": 8.586101749317925e-06, + "loss": 2.5839, + "theoretical_loss": 3.3076152992151857, + "tokens_seen": 3159687168 + }, + { + "epoch": 0.92, + "learning_rate": 8.578077355159685e-06, + "loss": 2.4412, + "theoretical_loss": 3.307604860748728, + "tokens_seen": 3159818240 + }, + { + "epoch": 0.92, + "learning_rate": 8.570052961001444e-06, + "loss": 2.5618, + "theoretical_loss": 3.3075944228364906, + "tokens_seen": 3159949312 + }, + { + "epoch": 0.92, + "learning_rate": 8.562028566843204e-06, + "loss": 2.4818, + "theoretical_loss": 3.307583985478422, + "tokens_seen": 3160080384 + }, + { + "epoch": 0.92, + "learning_rate": 8.554004172684962e-06, + "loss": 2.4801, + "theoretical_loss": 3.30757354867447, + "tokens_seen": 3160211456 + }, + { + "epoch": 0.92, + "learning_rate": 8.545979778526722e-06, + "loss": 2.4735, + "theoretical_loss": 3.3075631124245812, + "tokens_seen": 3160342528 + }, + { + "epoch": 0.92, + "learning_rate": 8.53795538436848e-06, + "loss": 2.5064, + "theoretical_loss": 3.307552676728704, + "tokens_seen": 3160473600 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1732078, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1340365409851074, + "objective/train/theoretical_loss": 3.307542241586786, + "objective/train/tokens_used": 1531063776, + "theoretical_loss": 3.307542241586786, + "tokens_seen": 3160604672 + }, + { + "epoch": 0.92, + "learning_rate": 8.52993099021024e-06, + "loss": 2.4001, + "theoretical_loss": 3.307542241586786, + "tokens_seen": 3160604672 + }, + { + "epoch": 0.92, + "learning_rate": 8.521906596051998e-06, + "loss": 2.3648, + "theoretical_loss": 3.3075318069987745, + "tokens_seen": 3160735744 + }, + { + "epoch": 0.92, + "learning_rate": 8.513882201893758e-06, + "loss": 2.4305, + "theoretical_loss": 3.307521372964617, + "tokens_seen": 3160866816 + }, + { + "epoch": 0.92, + "learning_rate": 8.505857807735516e-06, + "loss": 2.456, + "theoretical_loss": 3.3075109394842612, + "tokens_seen": 3160997888 + }, + { + "epoch": 0.92, + "learning_rate": 8.497833413577274e-06, + "loss": 2.4587, + "theoretical_loss": 3.3075005065576555, + "tokens_seen": 3161128960 + }, + { + "epoch": 0.92, + "learning_rate": 8.489809019419034e-06, + "loss": 2.5638, + "theoretical_loss": 3.307490074184747, + "tokens_seen": 3161260032 + }, + { + "epoch": 0.92, + "learning_rate": 8.481784625260792e-06, + "loss": 2.3055, + "theoretical_loss": 3.307479642365483, + "tokens_seen": 3161391104 + }, + { + "epoch": 0.92, + "learning_rate": 8.473760231102552e-06, + "loss": 2.4326, + "theoretical_loss": 3.307469211099811, + "tokens_seen": 3161522176 + }, + { + "epoch": 0.92, + "learning_rate": 8.46573583694431e-06, + "loss": 2.2212, + "theoretical_loss": 3.3074587803876803, + "tokens_seen": 3161653248 + }, + { + "epoch": 0.92, + "learning_rate": 8.45771144278607e-06, + "loss": 2.5287, + "theoretical_loss": 3.307448350229037, + "tokens_seen": 3161784320 + }, + { + "epoch": 0.92, + "learning_rate": 8.449687048627829e-06, + "loss": 2.638, + "theoretical_loss": 3.307437920623829, + "tokens_seen": 3161915392 + }, + { + "epoch": 0.92, + "learning_rate": 8.441662654469589e-06, + "loss": 2.4481, + "theoretical_loss": 3.307427491572004, + "tokens_seen": 3162046464 + }, + { + "epoch": 0.92, + "learning_rate": 8.433638260311347e-06, + "loss": 2.3382, + "theoretical_loss": 3.3074170630735105, + "tokens_seen": 3162177536 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1732556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8314449787139893, + "objective/train/theoretical_loss": 3.3074118490317463, + "objective/train/tokens_used": 1532702176, + "theoretical_loss": 3.3074118490317463, + "tokens_seen": 3162243072 + }, + { + "epoch": 0.92, + "learning_rate": 8.425613866153107e-06, + "loss": 2.4956, + "theoretical_loss": 3.3074066351282956, + "tokens_seen": 3162308608 + }, + { + "epoch": 0.92, + "learning_rate": 8.417589471994865e-06, + "loss": 2.4877, + "theoretical_loss": 3.3073962077363066, + "tokens_seen": 3162439680 + }, + { + "epoch": 0.92, + "learning_rate": 8.409565077836623e-06, + "loss": 2.5435, + "theoretical_loss": 3.3073857808974925, + "tokens_seen": 3162570752 + }, + { + "epoch": 0.92, + "learning_rate": 8.401540683678383e-06, + "loss": 2.5161, + "theoretical_loss": 3.3073753546117994, + "tokens_seen": 3162701824 + }, + { + "epoch": 0.92, + "learning_rate": 8.393516289520141e-06, + "loss": 2.4386, + "theoretical_loss": 3.307364928879176, + "tokens_seen": 3162832896 + }, + { + "epoch": 0.92, + "learning_rate": 8.385491895361901e-06, + "loss": 2.5666, + "theoretical_loss": 3.30735450369957, + "tokens_seen": 3162963968 + }, + { + "epoch": 0.92, + "learning_rate": 8.37746750120366e-06, + "loss": 2.6331, + "theoretical_loss": 3.307344079072929, + "tokens_seen": 3163095040 + }, + { + "epoch": 0.92, + "learning_rate": 8.36944310704542e-06, + "loss": 2.4849, + "theoretical_loss": 3.3073336549992014, + "tokens_seen": 3163226112 + }, + { + "epoch": 0.92, + "learning_rate": 8.361418712887177e-06, + "loss": 2.5583, + "theoretical_loss": 3.3073232314783336, + "tokens_seen": 3163357184 + }, + { + "epoch": 0.92, + "learning_rate": 8.353394318728937e-06, + "loss": 2.6027, + "theoretical_loss": 3.3073128085102748, + "tokens_seen": 3163488256 + }, + { + "epoch": 0.92, + "learning_rate": 8.345369924570696e-06, + "loss": 2.6134, + "theoretical_loss": 3.3073023860949715, + "tokens_seen": 3163619328 + }, + { + "epoch": 0.92, + "learning_rate": 8.337345530412455e-06, + "loss": 2.3703, + "theoretical_loss": 3.3072919642323724, + "tokens_seen": 3163750400 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1733728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8408443927764893, + "objective/train/theoretical_loss": 3.307281542922425, + "objective/train/tokens_used": 1534340576, + "theoretical_loss": 3.307281542922425, + "tokens_seen": 3163881472 + }, + { + "epoch": 0.92, + "learning_rate": 8.329321136254214e-06, + "loss": 2.4836, + "theoretical_loss": 3.307281542922425, + "tokens_seen": 3163881472 + }, + { + "epoch": 0.92, + "learning_rate": 8.321296742095974e-06, + "loss": 2.5688, + "theoretical_loss": 3.3072711221650772, + "tokens_seen": 3164012544 + }, + { + "epoch": 0.92, + "learning_rate": 8.313272347937732e-06, + "loss": 2.6511, + "theoretical_loss": 3.3072607019602764, + "tokens_seen": 3164143616 + }, + { + "epoch": 0.92, + "learning_rate": 8.30524795377949e-06, + "loss": 2.5045, + "theoretical_loss": 3.3072502823079706, + "tokens_seen": 3164274688 + }, + { + "epoch": 0.92, + "learning_rate": 8.29722355962125e-06, + "loss": 2.6016, + "theoretical_loss": 3.307239863208108, + "tokens_seen": 3164405760 + }, + { + "epoch": 0.92, + "learning_rate": 8.289199165463008e-06, + "loss": 2.5379, + "theoretical_loss": 3.3072294446606367, + "tokens_seen": 3164536832 + }, + { + "epoch": 0.92, + "learning_rate": 8.281174771304766e-06, + "loss": 2.6477, + "theoretical_loss": 3.3072190266655035, + "tokens_seen": 3164667904 + }, + { + "epoch": 0.92, + "learning_rate": 8.273150377146526e-06, + "loss": 2.7461, + "theoretical_loss": 3.3072086092226565, + "tokens_seen": 3164798976 + }, + { + "epoch": 0.92, + "learning_rate": 8.265125982988284e-06, + "loss": 2.5955, + "theoretical_loss": 3.307198192332044, + "tokens_seen": 3164930048 + }, + { + "epoch": 0.92, + "learning_rate": 8.257101588830043e-06, + "loss": 2.3763, + "theoretical_loss": 3.307187775993614, + "tokens_seen": 3165061120 + }, + { + "epoch": 0.92, + "learning_rate": 8.249077194671803e-06, + "loss": 2.4234, + "theoretical_loss": 3.307177360207314, + "tokens_seen": 3165192192 + }, + { + "epoch": 0.92, + "learning_rate": 8.24105280051356e-06, + "loss": 2.532, + "theoretical_loss": 3.3071669449730914, + "tokens_seen": 3165323264 + }, + { + "epoch": 0.92, + "learning_rate": 8.23302840635532e-06, + "loss": 2.562, + "theoretical_loss": 3.3071565302908947, + "tokens_seen": 3165454336 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1734416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.831005573272705, + "objective/train/theoretical_loss": 3.30715132315679, + "objective/train/tokens_used": 1535978976, + "theoretical_loss": 3.30715132315679, + "tokens_seen": 3165519872 + }, + { + "epoch": 0.92, + "learning_rate": 8.225004012197079e-06, + "loss": 2.6493, + "theoretical_loss": 3.307146116160672, + "tokens_seen": 3165585408 + }, + { + "epoch": 0.92, + "learning_rate": 8.216979618038837e-06, + "loss": 2.557, + "theoretical_loss": 3.3071357025823707, + "tokens_seen": 3165716480 + }, + { + "epoch": 0.92, + "learning_rate": 8.208955223880597e-06, + "loss": 2.4978, + "theoretical_loss": 3.307125289555939, + "tokens_seen": 3165847552 + }, + { + "epoch": 0.92, + "learning_rate": 8.200930829722355e-06, + "loss": 2.4528, + "theoretical_loss": 3.3071148770813243, + "tokens_seen": 3165978624 + }, + { + "epoch": 0.92, + "learning_rate": 8.192906435564115e-06, + "loss": 2.4758, + "theoretical_loss": 3.307104465158475, + "tokens_seen": 3166109696 + }, + { + "epoch": 0.92, + "learning_rate": 8.184882041405873e-06, + "loss": 2.4884, + "theoretical_loss": 3.3070940537873392, + "tokens_seen": 3166240768 + }, + { + "epoch": 0.92, + "learning_rate": 8.176857647247633e-06, + "loss": 2.5294, + "theoretical_loss": 3.3070836429678643, + "tokens_seen": 3166371840 + }, + { + "epoch": 0.92, + "learning_rate": 8.168833253089391e-06, + "loss": 2.4178, + "theoretical_loss": 3.3070732326999983, + "tokens_seen": 3166502912 + }, + { + "epoch": 0.92, + "learning_rate": 8.160808858931151e-06, + "loss": 2.4661, + "theoretical_loss": 3.3070628229836894, + "tokens_seen": 3166633984 + }, + { + "epoch": 0.92, + "learning_rate": 8.15278446477291e-06, + "loss": 2.4418, + "theoretical_loss": 3.3070524138188855, + "tokens_seen": 3166765056 + }, + { + "epoch": 0.92, + "learning_rate": 8.14476007061467e-06, + "loss": 2.4755, + "theoretical_loss": 3.3070420052055347, + "tokens_seen": 3166896128 + }, + { + "epoch": 0.92, + "learning_rate": 8.136735676456428e-06, + "loss": 2.5592, + "theoretical_loss": 3.3070315971435846, + "tokens_seen": 3167027200 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1735648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.578500747680664, + "objective/train/theoretical_loss": 3.307021189632983, + "objective/train/tokens_used": 1537617376, + "theoretical_loss": 3.307021189632983, + "tokens_seen": 3167158272 + }, + { + "epoch": 0.92, + "learning_rate": 8.128711282298188e-06, + "loss": 2.5788, + "theoretical_loss": 3.307021189632983, + "tokens_seen": 3167158272 + }, + { + "epoch": 0.92, + "learning_rate": 8.120686888139946e-06, + "loss": 2.4635, + "theoretical_loss": 3.3070107826736788, + "tokens_seen": 3167289344 + }, + { + "epoch": 0.92, + "learning_rate": 8.112662493981704e-06, + "loss": 2.552, + "theoretical_loss": 3.307000376265619, + "tokens_seen": 3167420416 + }, + { + "epoch": 0.92, + "learning_rate": 8.104638099823464e-06, + "loss": 2.508, + "theoretical_loss": 3.306989970408752, + "tokens_seen": 3167551488 + }, + { + "epoch": 0.92, + "learning_rate": 8.096613705665222e-06, + "loss": 2.6159, + "theoretical_loss": 3.3069795651030263, + "tokens_seen": 3167682560 + }, + { + "epoch": 0.92, + "learning_rate": 8.088589311506982e-06, + "loss": 2.5706, + "theoretical_loss": 3.306969160348389, + "tokens_seen": 3167813632 + }, + { + "epoch": 0.92, + "learning_rate": 8.08056491734874e-06, + "loss": 2.4626, + "theoretical_loss": 3.3069587561447884, + "tokens_seen": 3167944704 + }, + { + "epoch": 0.92, + "learning_rate": 8.0725405231905e-06, + "loss": 2.4196, + "theoretical_loss": 3.306948352492173, + "tokens_seen": 3168075776 + }, + { + "epoch": 0.92, + "learning_rate": 8.064516129032258e-06, + "loss": 2.6157, + "theoretical_loss": 3.3069379493904902, + "tokens_seen": 3168206848 + }, + { + "epoch": 0.92, + "learning_rate": 8.056491734874018e-06, + "loss": 2.5342, + "theoretical_loss": 3.306927546839688, + "tokens_seen": 3168337920 + }, + { + "epoch": 0.92, + "learning_rate": 8.048467340715777e-06, + "loss": 2.5117, + "theoretical_loss": 3.3069171448397157, + "tokens_seen": 3168468992 + }, + { + "epoch": 0.92, + "learning_rate": 8.040442946557536e-06, + "loss": 2.5852, + "theoretical_loss": 3.3069067433905195, + "tokens_seen": 3168600064 + }, + { + "epoch": 0.92, + "learning_rate": 8.032418552399295e-06, + "loss": 2.2442, + "theoretical_loss": 3.3068963424920486, + "tokens_seen": 3168731136 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1736287, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3761351108551025, + "objective/train/theoretical_loss": 3.306891142249319, + "objective/train/tokens_used": 1539255776, + "theoretical_loss": 3.306891142249319, + "tokens_seen": 3168796672 + }, + { + "epoch": 0.92, + "learning_rate": 8.024394158241053e-06, + "loss": 2.435, + "theoretical_loss": 3.306885942144251, + "tokens_seen": 3168862208 + }, + { + "epoch": 0.92, + "learning_rate": 8.016369764082813e-06, + "loss": 2.418, + "theoretical_loss": 3.306875542347074, + "tokens_seen": 3168993280 + }, + { + "epoch": 0.92, + "learning_rate": 8.008345369924571e-06, + "loss": 2.6033, + "theoretical_loss": 3.3068651431004668, + "tokens_seen": 3169124352 + }, + { + "epoch": 0.92, + "learning_rate": 8.000320975766331e-06, + "loss": 2.4605, + "theoretical_loss": 3.3068547444043768, + "tokens_seen": 3169255424 + }, + { + "epoch": 0.92, + "learning_rate": 7.992296581608089e-06, + "loss": 2.2923, + "theoretical_loss": 3.3068443462587522, + "tokens_seen": 3169386496 + }, + { + "epoch": 0.92, + "learning_rate": 7.984272187449849e-06, + "loss": 2.6179, + "theoretical_loss": 3.306833948663541, + "tokens_seen": 3169517568 + }, + { + "epoch": 0.92, + "learning_rate": 7.976247793291607e-06, + "loss": 2.3665, + "theoretical_loss": 3.3068235516186917, + "tokens_seen": 3169648640 + }, + { + "epoch": 0.92, + "learning_rate": 7.968223399133367e-06, + "loss": 2.4516, + "theoretical_loss": 3.306813155124152, + "tokens_seen": 3169779712 + }, + { + "epoch": 0.92, + "learning_rate": 7.960199004975125e-06, + "loss": 2.424, + "theoretical_loss": 3.3068027591798703, + "tokens_seen": 3169910784 + }, + { + "epoch": 0.92, + "learning_rate": 7.952174610816884e-06, + "loss": 2.4465, + "theoretical_loss": 3.3067923637857946, + "tokens_seen": 3170041856 + }, + { + "epoch": 0.92, + "learning_rate": 7.944150216658643e-06, + "loss": 2.5037, + "theoretical_loss": 3.306781968941873, + "tokens_seen": 3170172928 + }, + { + "epoch": 0.92, + "learning_rate": 7.936125822500402e-06, + "loss": 2.492, + "theoretical_loss": 3.306771574648053, + "tokens_seen": 3170304000 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1737490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5130202770233154, + "objective/train/theoretical_loss": 3.3067611809042843, + "objective/train/tokens_used": 1540894176, + "theoretical_loss": 3.3067611809042843, + "tokens_seen": 3170435072 + }, + { + "epoch": 0.92, + "learning_rate": 7.92810142834216e-06, + "loss": 2.39, + "theoretical_loss": 3.3067611809042843, + "tokens_seen": 3170435072 + }, + { + "epoch": 0.92, + "learning_rate": 7.92007703418392e-06, + "loss": 2.594, + "theoretical_loss": 3.306750787710514, + "tokens_seen": 3170566144 + }, + { + "epoch": 0.92, + "learning_rate": 7.912052640025678e-06, + "loss": 2.4343, + "theoretical_loss": 3.30674039506669, + "tokens_seen": 3170697216 + }, + { + "epoch": 0.92, + "learning_rate": 7.904028245867436e-06, + "loss": 2.4446, + "theoretical_loss": 3.3067300029727607, + "tokens_seen": 3170828288 + }, + { + "epoch": 0.92, + "learning_rate": 7.896003851709196e-06, + "loss": 2.4028, + "theoretical_loss": 3.306719611428675, + "tokens_seen": 3170959360 + }, + { + "epoch": 0.92, + "learning_rate": 7.887979457550954e-06, + "loss": 2.5632, + "theoretical_loss": 3.30670922043438, + "tokens_seen": 3171090432 + }, + { + "epoch": 0.92, + "learning_rate": 7.879955063392714e-06, + "loss": 2.4603, + "theoretical_loss": 3.3066988299898252, + "tokens_seen": 3171221504 + }, + { + "epoch": 0.92, + "learning_rate": 7.871930669234472e-06, + "loss": 2.3772, + "theoretical_loss": 3.3066884400949577, + "tokens_seen": 3171352576 + }, + { + "epoch": 0.92, + "learning_rate": 7.863906275076232e-06, + "loss": 2.5526, + "theoretical_loss": 3.306678050749726, + "tokens_seen": 3171483648 + }, + { + "epoch": 0.92, + "learning_rate": 7.85588188091799e-06, + "loss": 2.6861, + "theoretical_loss": 3.3066676619540782, + "tokens_seen": 3171614720 + }, + { + "epoch": 0.92, + "learning_rate": 7.84785748675975e-06, + "loss": 2.3725, + "theoretical_loss": 3.3066572737079625, + "tokens_seen": 3171745792 + }, + { + "epoch": 0.92, + "learning_rate": 7.839833092601509e-06, + "loss": 2.6582, + "theoretical_loss": 3.3066468860113276, + "tokens_seen": 3171876864 + }, + { + "epoch": 0.92, + "learning_rate": 7.831808698443267e-06, + "loss": 2.4486, + "theoretical_loss": 3.3066364988641213, + "tokens_seen": 3172007936 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1738634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3359835147857666, + "objective/train/theoretical_loss": 3.3066313054965377, + "objective/train/tokens_used": 1542532576, + "theoretical_loss": 3.3066313054965377, + "tokens_seen": 3172073472 + }, + { + "epoch": 0.92, + "learning_rate": 7.823784304285027e-06, + "loss": 2.4809, + "theoretical_loss": 3.306626112266292, + "tokens_seen": 3172139008 + }, + { + "epoch": 0.92, + "learning_rate": 7.815759910126785e-06, + "loss": 2.4839, + "theoretical_loss": 3.306615726217788, + "tokens_seen": 3172270080 + }, + { + "epoch": 0.92, + "learning_rate": 7.807735515968545e-06, + "loss": 2.5115, + "theoretical_loss": 3.306605340718557, + "tokens_seen": 3172401152 + }, + { + "epoch": 0.92, + "learning_rate": 7.799711121810303e-06, + "loss": 2.4956, + "theoretical_loss": 3.3065949557685483, + "tokens_seen": 3172532224 + }, + { + "epoch": 0.92, + "learning_rate": 7.791686727652063e-06, + "loss": 2.4395, + "theoretical_loss": 3.306584571367709, + "tokens_seen": 3172663296 + }, + { + "epoch": 0.92, + "learning_rate": 7.783662333493821e-06, + "loss": 2.4025, + "theoretical_loss": 3.3065741875159884, + "tokens_seen": 3172794368 + }, + { + "epoch": 0.92, + "learning_rate": 7.775637939335581e-06, + "loss": 2.4758, + "theoretical_loss": 3.306563804213334, + "tokens_seen": 3172925440 + }, + { + "epoch": 0.92, + "learning_rate": 7.76761354517734e-06, + "loss": 2.545, + "theoretical_loss": 3.3065534214596948, + "tokens_seen": 3173056512 + }, + { + "epoch": 0.92, + "learning_rate": 7.7595891510191e-06, + "loss": 2.5676, + "theoretical_loss": 3.3065430392550184, + "tokens_seen": 3173187584 + }, + { + "epoch": 0.92, + "learning_rate": 7.751564756860857e-06, + "loss": 2.4697, + "theoretical_loss": 3.3065326575992535, + "tokens_seen": 3173318656 + }, + { + "epoch": 0.92, + "learning_rate": 7.743540362702616e-06, + "loss": 2.5622, + "theoretical_loss": 3.3065222764923483, + "tokens_seen": 3173449728 + }, + { + "epoch": 0.92, + "learning_rate": 7.735515968544376e-06, + "loss": 2.4967, + "theoretical_loss": 3.3065118959342508, + "tokens_seen": 3173580800 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1739179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.737079381942749, + "objective/train/theoretical_loss": 3.3065015159249103, + "objective/train/tokens_used": 1544170976, + "theoretical_loss": 3.3065015159249103, + "tokens_seen": 3173711872 + }, + { + "epoch": 0.92, + "learning_rate": 7.727491574386134e-06, + "loss": 2.5882, + "theoretical_loss": 3.3065015159249103, + "tokens_seen": 3173711872 + }, + { + "epoch": 0.92, + "learning_rate": 7.719467180227894e-06, + "loss": 2.5689, + "theoretical_loss": 3.306491136464274, + "tokens_seen": 3173842944 + }, + { + "epoch": 0.92, + "learning_rate": 7.711442786069652e-06, + "loss": 2.5803, + "theoretical_loss": 3.3064807575522908, + "tokens_seen": 3173974016 + }, + { + "epoch": 0.92, + "learning_rate": 7.703418391911412e-06, + "loss": 2.5541, + "theoretical_loss": 3.3064703791889087, + "tokens_seen": 3174105088 + }, + { + "epoch": 0.92, + "learning_rate": 7.69539399775317e-06, + "loss": 2.4514, + "theoretical_loss": 3.306460001374077, + "tokens_seen": 3174236160 + }, + { + "epoch": 0.92, + "learning_rate": 7.68736960359493e-06, + "loss": 2.4676, + "theoretical_loss": 3.306449624107743, + "tokens_seen": 3174367232 + }, + { + "epoch": 0.92, + "learning_rate": 7.679345209436688e-06, + "loss": 2.5378, + "theoretical_loss": 3.3064392473898554, + "tokens_seen": 3174498304 + }, + { + "epoch": 0.92, + "learning_rate": 7.671320815278448e-06, + "loss": 2.3225, + "theoretical_loss": 3.3064288712203624, + "tokens_seen": 3174629376 + }, + { + "epoch": 0.92, + "learning_rate": 7.663296421120206e-06, + "loss": 2.631, + "theoretical_loss": 3.306418495599213, + "tokens_seen": 3174760448 + }, + { + "epoch": 0.92, + "learning_rate": 7.655272026961964e-06, + "loss": 2.4105, + "theoretical_loss": 3.306408120526355, + "tokens_seen": 3174891520 + }, + { + "epoch": 0.92, + "learning_rate": 7.647247632803724e-06, + "loss": 2.5083, + "theoretical_loss": 3.306397746001737, + "tokens_seen": 3175022592 + }, + { + "epoch": 0.92, + "learning_rate": 7.639223238645483e-06, + "loss": 2.508, + "theoretical_loss": 3.3063873720253074, + "tokens_seen": 3175153664 + }, + { + "epoch": 0.92, + "learning_rate": 7.631198844487242e-06, + "loss": 2.5686, + "theoretical_loss": 3.306376998597014, + "tokens_seen": 3175284736 + }, + { + "epoch": 0.92, + "objective/train/docs_used": 1739867, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.094205141067505, + "objective/train/theoretical_loss": 3.306371812088403, + "objective/train/tokens_used": 1545809376, + "theoretical_loss": 3.306371812088403, + "tokens_seen": 3175350272 + }, + { + "epoch": 0.92, + "learning_rate": 7.623174450329e-06, + "loss": 2.5063, + "theoretical_loss": 3.3063666257168065, + "tokens_seen": 3175415808 + }, + { + "epoch": 0.92, + "learning_rate": 7.61515005617076e-06, + "loss": 2.5726, + "theoretical_loss": 3.3063562533846325, + "tokens_seen": 3175546880 + }, + { + "epoch": 0.92, + "learning_rate": 7.607125662012518e-06, + "loss": 2.3783, + "theoretical_loss": 3.30634588160044, + "tokens_seen": 3175677952 + }, + { + "epoch": 0.92, + "learning_rate": 7.599101267854278e-06, + "loss": 2.5254, + "theoretical_loss": 3.3063355103641783, + "tokens_seen": 3175809024 + }, + { + "epoch": 0.92, + "learning_rate": 7.591076873696036e-06, + "loss": 2.6492, + "theoretical_loss": 3.3063251396757956, + "tokens_seen": 3175940096 + }, + { + "epoch": 0.92, + "learning_rate": 7.583052479537796e-06, + "loss": 2.6046, + "theoretical_loss": 3.3063147695352404, + "tokens_seen": 3176071168 + }, + { + "epoch": 0.93, + "learning_rate": 7.575028085379554e-06, + "loss": 2.4721, + "theoretical_loss": 3.3063043999424604, + "tokens_seen": 3176202240 + }, + { + "epoch": 0.93, + "learning_rate": 7.567003691221314e-06, + "loss": 2.5159, + "theoretical_loss": 3.3062940308974054, + "tokens_seen": 3176333312 + }, + { + "epoch": 0.93, + "learning_rate": 7.558979297063072e-06, + "loss": 2.5165, + "theoretical_loss": 3.3062836624000225, + "tokens_seen": 3176464384 + }, + { + "epoch": 0.93, + "learning_rate": 7.5509549029048305e-06, + "loss": 2.4765, + "theoretical_loss": 3.306273294450261, + "tokens_seen": 3176595456 + }, + { + "epoch": 0.93, + "learning_rate": 7.5429305087465905e-06, + "loss": 2.5067, + "theoretical_loss": 3.3062629270480692, + "tokens_seen": 3176726528 + }, + { + "epoch": 0.93, + "learning_rate": 7.534906114588349e-06, + "loss": 2.4678, + "theoretical_loss": 3.306252560193396, + "tokens_seen": 3176857600 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1740434, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.473209857940674, + "objective/train/theoretical_loss": 3.306242193886189, + "objective/train/tokens_used": 1547447776, + "theoretical_loss": 3.306242193886189, + "tokens_seen": 3176988672 + }, + { + "epoch": 0.93, + "learning_rate": 7.526881720430108e-06, + "loss": 2.5717, + "theoretical_loss": 3.306242193886189, + "tokens_seen": 3176988672 + }, + { + "epoch": 0.93, + "learning_rate": 7.518857326271867e-06, + "loss": 2.4552, + "theoretical_loss": 3.306231828126397, + "tokens_seen": 3177119744 + }, + { + "epoch": 0.93, + "learning_rate": 7.510832932113626e-06, + "loss": 2.3446, + "theoretical_loss": 3.3062214629139692, + "tokens_seen": 3177250816 + }, + { + "epoch": 0.93, + "learning_rate": 7.502808537955384e-06, + "loss": 2.5389, + "theoretical_loss": 3.3062110982488533, + "tokens_seen": 3177381888 + }, + { + "epoch": 0.93, + "learning_rate": 7.494784143797144e-06, + "loss": 2.4595, + "theoretical_loss": 3.3062007341309982, + "tokens_seen": 3177512960 + }, + { + "epoch": 0.93, + "learning_rate": 7.486759749638902e-06, + "loss": 2.4885, + "theoretical_loss": 3.3061903705603526, + "tokens_seen": 3177644032 + }, + { + "epoch": 0.93, + "learning_rate": 7.478735355480662e-06, + "loss": 2.3643, + "theoretical_loss": 3.3061800075368644, + "tokens_seen": 3177775104 + }, + { + "epoch": 0.93, + "learning_rate": 7.47071096132242e-06, + "loss": 2.5401, + "theoretical_loss": 3.306169645060483, + "tokens_seen": 3177906176 + }, + { + "epoch": 0.93, + "learning_rate": 7.4626865671641785e-06, + "loss": 2.3287, + "theoretical_loss": 3.306159283131156, + "tokens_seen": 3178037248 + }, + { + "epoch": 0.93, + "learning_rate": 7.454662173005938e-06, + "loss": 2.3959, + "theoretical_loss": 3.306148921748833, + "tokens_seen": 3178168320 + }, + { + "epoch": 0.93, + "learning_rate": 7.446637778847697e-06, + "loss": 2.5607, + "theoretical_loss": 3.3061385609134617, + "tokens_seen": 3178299392 + }, + { + "epoch": 0.93, + "learning_rate": 7.4386133846894565e-06, + "loss": 2.4595, + "theoretical_loss": 3.306128200624991, + "tokens_seen": 3178430464 + }, + { + "epoch": 0.93, + "learning_rate": 7.430588990531215e-06, + "loss": 2.4192, + "theoretical_loss": 3.3061178408833696, + "tokens_seen": 3178561536 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1741553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.764373302459717, + "objective/train/theoretical_loss": 3.306112661217611, + "objective/train/tokens_used": 1549086176, + "theoretical_loss": 3.306112661217611, + "tokens_seen": 3178627072 + }, + { + "epoch": 0.93, + "learning_rate": 7.422564596372975e-06, + "loss": 2.6189, + "theoretical_loss": 3.306107481688546, + "tokens_seen": 3178692608 + }, + { + "epoch": 0.93, + "learning_rate": 7.414540202214733e-06, + "loss": 2.5682, + "theoretical_loss": 3.3060971230404683, + "tokens_seen": 3178823680 + }, + { + "epoch": 0.93, + "learning_rate": 7.406515808056493e-06, + "loss": 2.4072, + "theoretical_loss": 3.306086764939086, + "tokens_seen": 3178954752 + }, + { + "epoch": 0.93, + "learning_rate": 7.398491413898251e-06, + "loss": 2.4747, + "theoretical_loss": 3.306076407384347, + "tokens_seen": 3179085824 + }, + { + "epoch": 0.93, + "learning_rate": 7.390467019740011e-06, + "loss": 2.6307, + "theoretical_loss": 3.3060660503762005, + "tokens_seen": 3179216896 + }, + { + "epoch": 0.93, + "learning_rate": 7.382442625581769e-06, + "loss": 2.5544, + "theoretical_loss": 3.306055693914595, + "tokens_seen": 3179347968 + }, + { + "epoch": 0.93, + "learning_rate": 7.374418231423528e-06, + "loss": 2.6075, + "theoretical_loss": 3.3060453379994783, + "tokens_seen": 3179479040 + }, + { + "epoch": 0.93, + "learning_rate": 7.366393837265287e-06, + "loss": 2.4765, + "theoretical_loss": 3.3060349826308, + "tokens_seen": 3179610112 + }, + { + "epoch": 0.93, + "learning_rate": 7.358369443107045e-06, + "loss": 2.4854, + "theoretical_loss": 3.3060246278085086, + "tokens_seen": 3179741184 + }, + { + "epoch": 0.93, + "learning_rate": 7.3503450489488045e-06, + "loss": 2.48, + "theoretical_loss": 3.3060142735325524, + "tokens_seen": 3179872256 + }, + { + "epoch": 0.93, + "learning_rate": 7.3423206547905635e-06, + "loss": 2.665, + "theoretical_loss": 3.3060039198028806, + "tokens_seen": 3180003328 + }, + { + "epoch": 0.93, + "learning_rate": 7.334296260632323e-06, + "loss": 2.4024, + "theoretical_loss": 3.305993566619441, + "tokens_seen": 3180134400 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1742088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6287500858306885, + "objective/train/theoretical_loss": 3.3059832139821825, + "objective/train/tokens_used": 1550724576, + "theoretical_loss": 3.3059832139821825, + "tokens_seen": 3180265472 + }, + { + "epoch": 0.93, + "learning_rate": 7.326271866474081e-06, + "loss": 2.5138, + "theoretical_loss": 3.3059832139821825, + "tokens_seen": 3180265472 + }, + { + "epoch": 0.93, + "learning_rate": 7.318247472315841e-06, + "loss": 2.6394, + "theoretical_loss": 3.3059728618910547, + "tokens_seen": 3180396544 + }, + { + "epoch": 0.93, + "learning_rate": 7.310223078157599e-06, + "loss": 2.5497, + "theoretical_loss": 3.3059625103460055, + "tokens_seen": 3180527616 + }, + { + "epoch": 0.93, + "learning_rate": 7.302198683999359e-06, + "loss": 2.4313, + "theoretical_loss": 3.305952159346983, + "tokens_seen": 3180658688 + }, + { + "epoch": 0.93, + "learning_rate": 7.294174289841117e-06, + "loss": 2.4278, + "theoretical_loss": 3.3059418088939374, + "tokens_seen": 3180789760 + }, + { + "epoch": 0.93, + "learning_rate": 7.286149895682877e-06, + "loss": 2.4654, + "theoretical_loss": 3.3059314589868167, + "tokens_seen": 3180920832 + }, + { + "epoch": 0.93, + "learning_rate": 7.278125501524635e-06, + "loss": 2.4814, + "theoretical_loss": 3.305921109625569, + "tokens_seen": 3181051904 + }, + { + "epoch": 0.93, + "learning_rate": 7.270101107366393e-06, + "loss": 2.4037, + "theoretical_loss": 3.3059107608101437, + "tokens_seen": 3181182976 + }, + { + "epoch": 0.93, + "learning_rate": 7.262076713208153e-06, + "loss": 2.5482, + "theoretical_loss": 3.3059004125404896, + "tokens_seen": 3181314048 + }, + { + "epoch": 0.93, + "learning_rate": 7.2540523190499115e-06, + "loss": 2.5936, + "theoretical_loss": 3.305890064816555, + "tokens_seen": 3181445120 + }, + { + "epoch": 0.93, + "learning_rate": 7.246027924891671e-06, + "loss": 2.5412, + "theoretical_loss": 3.3058797176382893, + "tokens_seen": 3181576192 + }, + { + "epoch": 0.93, + "learning_rate": 7.23800353073343e-06, + "loss": 2.5971, + "theoretical_loss": 3.30586937100564, + "tokens_seen": 3181707264 + }, + { + "epoch": 0.93, + "learning_rate": 7.2299791365751895e-06, + "loss": 2.645, + "theoretical_loss": 3.305859024918557, + "tokens_seen": 3181838336 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1743107, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3039746284484863, + "objective/train/theoretical_loss": 3.305853852079587, + "objective/train/tokens_used": 1552362976, + "theoretical_loss": 3.305853852079587, + "tokens_seen": 3181903872 + }, + { + "epoch": 0.93, + "learning_rate": 7.221954742416948e-06, + "loss": 2.4068, + "theoretical_loss": 3.305848679376989, + "tokens_seen": 3181969408 + }, + { + "epoch": 0.93, + "learning_rate": 7.213930348258708e-06, + "loss": 2.4657, + "theoretical_loss": 3.3058383343808844, + "tokens_seen": 3182100480 + }, + { + "epoch": 0.93, + "learning_rate": 7.205905954100466e-06, + "loss": 2.6275, + "theoretical_loss": 3.305827989930192, + "tokens_seen": 3182231552 + }, + { + "epoch": 0.93, + "learning_rate": 7.197881559942225e-06, + "loss": 2.5883, + "theoretical_loss": 3.30581764602486, + "tokens_seen": 3182362624 + }, + { + "epoch": 0.93, + "learning_rate": 7.189857165783984e-06, + "loss": 2.5163, + "theoretical_loss": 3.3058073026648387, + "tokens_seen": 3182493696 + }, + { + "epoch": 0.93, + "learning_rate": 7.181832771625742e-06, + "loss": 2.5174, + "theoretical_loss": 3.3057969598500754, + "tokens_seen": 3182624768 + }, + { + "epoch": 0.93, + "learning_rate": 7.173808377467501e-06, + "loss": 2.3785, + "theoretical_loss": 3.30578661758052, + "tokens_seen": 3182755840 + }, + { + "epoch": 0.93, + "learning_rate": 7.16578398330926e-06, + "loss": 2.5698, + "theoretical_loss": 3.3057762758561204, + "tokens_seen": 3182886912 + }, + { + "epoch": 0.93, + "learning_rate": 7.157759589151019e-06, + "loss": 2.5888, + "theoretical_loss": 3.3057659346768262, + "tokens_seen": 3183017984 + }, + { + "epoch": 0.93, + "learning_rate": 7.1497351949927776e-06, + "loss": 2.5154, + "theoretical_loss": 3.305755594042586, + "tokens_seen": 3183149056 + }, + { + "epoch": 0.93, + "learning_rate": 7.1417108008345375e-06, + "loss": 2.285, + "theoretical_loss": 3.305745253953348, + "tokens_seen": 3183280128 + }, + { + "epoch": 0.93, + "learning_rate": 7.133686406676296e-06, + "loss": 2.5197, + "theoretical_loss": 3.3057349144090615, + "tokens_seen": 3183411200 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1744349, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8953969478607178, + "objective/train/theoretical_loss": 3.3057245754096756, + "objective/train/tokens_used": 1554001376, + "theoretical_loss": 3.3057245754096756, + "tokens_seen": 3183542272 + }, + { + "epoch": 0.93, + "learning_rate": 7.125662012518056e-06, + "loss": 2.6313, + "theoretical_loss": 3.3057245754096756, + "tokens_seen": 3183542272 + }, + { + "epoch": 0.93, + "learning_rate": 7.117637618359814e-06, + "loss": 2.6238, + "theoretical_loss": 3.305714236955139, + "tokens_seen": 3183673344 + }, + { + "epoch": 0.93, + "learning_rate": 7.109613224201574e-06, + "loss": 2.5037, + "theoretical_loss": 3.3057038990454006, + "tokens_seen": 3183804416 + }, + { + "epoch": 0.93, + "learning_rate": 7.101588830043332e-06, + "loss": 2.5161, + "theoretical_loss": 3.305693561680409, + "tokens_seen": 3183935488 + }, + { + "epoch": 0.93, + "learning_rate": 7.093564435885092e-06, + "loss": 2.4491, + "theoretical_loss": 3.3056832248601133, + "tokens_seen": 3184066560 + }, + { + "epoch": 0.93, + "learning_rate": 7.08554004172685e-06, + "loss": 2.4083, + "theoretical_loss": 3.3056728885844624, + "tokens_seen": 3184197632 + }, + { + "epoch": 0.93, + "learning_rate": 7.077515647568608e-06, + "loss": 2.5859, + "theoretical_loss": 3.3056625528534047, + "tokens_seen": 3184328704 + }, + { + "epoch": 0.93, + "learning_rate": 7.069491253410368e-06, + "loss": 2.4744, + "theoretical_loss": 3.3056522176668897, + "tokens_seen": 3184459776 + }, + { + "epoch": 0.93, + "learning_rate": 7.061466859252126e-06, + "loss": 2.4549, + "theoretical_loss": 3.305641883024866, + "tokens_seen": 3184590848 + }, + { + "epoch": 0.93, + "learning_rate": 7.053442465093886e-06, + "loss": 2.4274, + "theoretical_loss": 3.3056315489272827, + "tokens_seen": 3184721920 + }, + { + "epoch": 0.93, + "learning_rate": 7.0454180709356445e-06, + "loss": 2.4965, + "theoretical_loss": 3.3056212153740887, + "tokens_seen": 3184852992 + }, + { + "epoch": 0.93, + "learning_rate": 7.037393676777404e-06, + "loss": 2.4751, + "theoretical_loss": 3.3056108823652326, + "tokens_seen": 3184984064 + }, + { + "epoch": 0.93, + "learning_rate": 7.029369282619163e-06, + "loss": 2.5443, + "theoretical_loss": 3.3056005499006638, + "tokens_seen": 3185115136 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1745003, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.459510326385498, + "objective/train/theoretical_loss": 3.3055953838724705, + "objective/train/tokens_used": 1555639776, + "theoretical_loss": 3.3055953838724705, + "tokens_seen": 3185180672 + }, + { + "epoch": 0.93, + "learning_rate": 7.021344888460922e-06, + "loss": 2.6376, + "theoretical_loss": 3.3055902179803303, + "tokens_seen": 3185246208 + }, + { + "epoch": 0.93, + "learning_rate": 7.013320494302681e-06, + "loss": 2.4123, + "theoretical_loss": 3.3055798866041823, + "tokens_seen": 3185377280 + }, + { + "epoch": 0.93, + "learning_rate": 7.00529610014444e-06, + "loss": 2.4748, + "theoretical_loss": 3.305569555772168, + "tokens_seen": 3185508352 + }, + { + "epoch": 0.93, + "learning_rate": 6.997271705986198e-06, + "loss": 2.4134, + "theoretical_loss": 3.3055592254842368, + "tokens_seen": 3185639424 + }, + { + "epoch": 0.93, + "learning_rate": 6.989247311827957e-06, + "loss": 2.6301, + "theoretical_loss": 3.305548895740337, + "tokens_seen": 3185770496 + }, + { + "epoch": 0.93, + "learning_rate": 6.981222917669716e-06, + "loss": 2.5918, + "theoretical_loss": 3.3055385665404184, + "tokens_seen": 3185901568 + }, + { + "epoch": 0.93, + "learning_rate": 6.973198523511474e-06, + "loss": 2.6194, + "theoretical_loss": 3.3055282378844293, + "tokens_seen": 3186032640 + }, + { + "epoch": 0.93, + "learning_rate": 6.965174129353234e-06, + "loss": 2.424, + "theoretical_loss": 3.305517909772319, + "tokens_seen": 3186163712 + }, + { + "epoch": 0.93, + "learning_rate": 6.9571497351949924e-06, + "loss": 2.6066, + "theoretical_loss": 3.3055075822040356, + "tokens_seen": 3186294784 + }, + { + "epoch": 0.93, + "learning_rate": 6.949125341036752e-06, + "loss": 2.5034, + "theoretical_loss": 3.30549725517953, + "tokens_seen": 3186425856 + }, + { + "epoch": 0.93, + "learning_rate": 6.9411009468785106e-06, + "loss": 2.438, + "theoretical_loss": 3.305486928698749, + "tokens_seen": 3186556928 + }, + { + "epoch": 0.93, + "learning_rate": 6.9330765527202705e-06, + "loss": 2.5995, + "theoretical_loss": 3.305476602761644, + "tokens_seen": 3186688000 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1746088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6318657398223877, + "objective/train/theoretical_loss": 3.305466277368162, + "objective/train/tokens_used": 1557278176, + "theoretical_loss": 3.305466277368162, + "tokens_seen": 3186819072 + }, + { + "epoch": 0.93, + "learning_rate": 6.925052158562029e-06, + "loss": 2.5118, + "theoretical_loss": 3.305466277368162, + "tokens_seen": 3186819072 + }, + { + "epoch": 0.93, + "learning_rate": 6.917027764403789e-06, + "loss": 2.6021, + "theoretical_loss": 3.3054559525182525, + "tokens_seen": 3186950144 + }, + { + "epoch": 0.93, + "learning_rate": 6.909003370245547e-06, + "loss": 2.582, + "theoretical_loss": 3.305445628211865, + "tokens_seen": 3187081216 + }, + { + "epoch": 0.93, + "learning_rate": 6.900978976087307e-06, + "loss": 2.4789, + "theoretical_loss": 3.3054353044489484, + "tokens_seen": 3187212288 + }, + { + "epoch": 0.93, + "learning_rate": 6.892954581929065e-06, + "loss": 2.3997, + "theoretical_loss": 3.3054249812294514, + "tokens_seen": 3187343360 + }, + { + "epoch": 0.93, + "learning_rate": 6.884930187770823e-06, + "loss": 2.5599, + "theoretical_loss": 3.3054146585533237, + "tokens_seen": 3187474432 + }, + { + "epoch": 0.93, + "learning_rate": 6.876905793612583e-06, + "loss": 2.611, + "theoretical_loss": 3.3054043364205135, + "tokens_seen": 3187605504 + }, + { + "epoch": 0.93, + "learning_rate": 6.868881399454341e-06, + "loss": 2.3922, + "theoretical_loss": 3.305394014830971, + "tokens_seen": 3187736576 + }, + { + "epoch": 0.93, + "learning_rate": 6.860857005296101e-06, + "loss": 2.4265, + "theoretical_loss": 3.305383693784644, + "tokens_seen": 3187867648 + }, + { + "epoch": 0.93, + "learning_rate": 6.852832611137859e-06, + "loss": 2.5257, + "theoretical_loss": 3.305373373281482, + "tokens_seen": 3187998720 + }, + { + "epoch": 0.93, + "learning_rate": 6.844808216979618e-06, + "loss": 2.6387, + "theoretical_loss": 3.3053630533214347, + "tokens_seen": 3188129792 + }, + { + "epoch": 0.93, + "learning_rate": 6.8367838228213775e-06, + "loss": 2.5257, + "theoretical_loss": 3.3053527339044506, + "tokens_seen": 3188260864 + }, + { + "epoch": 0.93, + "learning_rate": 6.8287594286631365e-06, + "loss": 2.5769, + "theoretical_loss": 3.305342415030479, + "tokens_seen": 3188391936 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1746515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7945430278778076, + "objective/train/theoretical_loss": 3.305337255797107, + "objective/train/tokens_used": 1558916576, + "theoretical_loss": 3.305337255797107, + "tokens_seen": 3188457472 + }, + { + "epoch": 0.93, + "learning_rate": 6.820735034504895e-06, + "loss": 2.5576, + "theoretical_loss": 3.305332096699469, + "tokens_seen": 3188523008 + }, + { + "epoch": 0.93, + "learning_rate": 6.812710640346655e-06, + "loss": 2.4344, + "theoretical_loss": 3.3053217789113694, + "tokens_seen": 3188654080 + }, + { + "epoch": 0.93, + "learning_rate": 6.804686246188413e-06, + "loss": 2.5675, + "theoretical_loss": 3.30531146166613, + "tokens_seen": 3188785152 + }, + { + "epoch": 0.93, + "learning_rate": 6.796661852030171e-06, + "loss": 2.5034, + "theoretical_loss": 3.305301144963699, + "tokens_seen": 3188916224 + }, + { + "epoch": 0.93, + "learning_rate": 6.788637457871931e-06, + "loss": 2.5793, + "theoretical_loss": 3.3052908288040266, + "tokens_seen": 3189047296 + }, + { + "epoch": 0.93, + "learning_rate": 6.780613063713689e-06, + "loss": 2.5884, + "theoretical_loss": 3.3052805131870606, + "tokens_seen": 3189178368 + }, + { + "epoch": 0.93, + "learning_rate": 6.772588669555449e-06, + "loss": 2.5521, + "theoretical_loss": 3.3052701981127517, + "tokens_seen": 3189309440 + }, + { + "epoch": 0.93, + "learning_rate": 6.764564275397207e-06, + "loss": 2.5303, + "theoretical_loss": 3.305259883581048, + "tokens_seen": 3189440512 + }, + { + "epoch": 0.93, + "learning_rate": 6.756539881238967e-06, + "loss": 2.522, + "theoretical_loss": 3.305249569591899, + "tokens_seen": 3189571584 + }, + { + "epoch": 0.93, + "learning_rate": 6.7485154870807254e-06, + "loss": 2.529, + "theoretical_loss": 3.3052392561452537, + "tokens_seen": 3189702656 + }, + { + "epoch": 0.93, + "learning_rate": 6.740491092922485e-06, + "loss": 2.6688, + "theoretical_loss": 3.3052289432410613, + "tokens_seen": 3189833728 + }, + { + "epoch": 0.93, + "learning_rate": 6.7324666987642435e-06, + "loss": 2.3565, + "theoretical_loss": 3.3052186308792715, + "tokens_seen": 3189964800 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1746985, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8048529624938965, + "objective/train/theoretical_loss": 3.3052083190598323, + "objective/train/tokens_used": 1560554976, + "theoretical_loss": 3.3052083190598323, + "tokens_seen": 3190095872 + }, + { + "epoch": 0.93, + "learning_rate": 6.7244423046060035e-06, + "loss": 2.6694, + "theoretical_loss": 3.3052083190598323, + "tokens_seen": 3190095872 + }, + { + "epoch": 0.93, + "learning_rate": 6.716417910447762e-06, + "loss": 2.4893, + "theoretical_loss": 3.3051980077826943, + "tokens_seen": 3190226944 + }, + { + "epoch": 0.93, + "learning_rate": 6.70839351628952e-06, + "loss": 2.6043, + "theoretical_loss": 3.305187697047806, + "tokens_seen": 3190358016 + }, + { + "epoch": 0.93, + "learning_rate": 6.70036912213128e-06, + "loss": 2.4497, + "theoretical_loss": 3.3051773868551164, + "tokens_seen": 3190489088 + }, + { + "epoch": 0.93, + "learning_rate": 6.692344727973038e-06, + "loss": 2.6096, + "theoretical_loss": 3.305167077204575, + "tokens_seen": 3190620160 + }, + { + "epoch": 0.93, + "learning_rate": 6.684320333814798e-06, + "loss": 2.3655, + "theoretical_loss": 3.305156768096131, + "tokens_seen": 3190751232 + }, + { + "epoch": 0.93, + "learning_rate": 6.676295939656556e-06, + "loss": 2.5927, + "theoretical_loss": 3.305146459529734, + "tokens_seen": 3190882304 + }, + { + "epoch": 0.93, + "learning_rate": 6.668271545498315e-06, + "loss": 2.5108, + "theoretical_loss": 3.3051361515053324, + "tokens_seen": 3191013376 + }, + { + "epoch": 0.93, + "learning_rate": 6.660247151340074e-06, + "loss": 2.6162, + "theoretical_loss": 3.305125844022876, + "tokens_seen": 3191144448 + }, + { + "epoch": 0.93, + "learning_rate": 6.652222757181833e-06, + "loss": 2.5424, + "theoretical_loss": 3.3051155370823135, + "tokens_seen": 3191275520 + }, + { + "epoch": 0.93, + "learning_rate": 6.6441983630235915e-06, + "loss": 2.5006, + "theoretical_loss": 3.3051052306835955, + "tokens_seen": 3191406592 + }, + { + "epoch": 0.93, + "learning_rate": 6.636173968865351e-06, + "loss": 2.6523, + "theoretical_loss": 3.30509492482667, + "tokens_seen": 3191537664 + }, + { + "epoch": 0.93, + "learning_rate": 6.62814957470711e-06, + "loss": 2.561, + "theoretical_loss": 3.3050846195114865, + "tokens_seen": 3191668736 + }, + { + "epoch": 0.93, + "objective/train/docs_used": 1748281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7127978801727295, + "objective/train/theoretical_loss": 3.3050794670570323, + "objective/train/tokens_used": 1562193376, + "theoretical_loss": 3.3050794670570323, + "tokens_seen": 3191734272 + }, + { + "epoch": 0.93, + "learning_rate": 6.6201251805488695e-06, + "loss": 2.558, + "theoretical_loss": 3.305074314737994, + "tokens_seen": 3191799808 + }, + { + "epoch": 0.93, + "learning_rate": 6.612100786390628e-06, + "loss": 2.5494, + "theoretical_loss": 3.3050640105061433, + "tokens_seen": 3191930880 + }, + { + "epoch": 0.93, + "learning_rate": 6.604076392232386e-06, + "loss": 2.5678, + "theoretical_loss": 3.305053706815882, + "tokens_seen": 3192061952 + }, + { + "epoch": 0.93, + "learning_rate": 6.596051998074146e-06, + "loss": 2.5509, + "theoretical_loss": 3.3050434036671597, + "tokens_seen": 3192193024 + }, + { + "epoch": 0.93, + "learning_rate": 6.588027603915904e-06, + "loss": 2.514, + "theoretical_loss": 3.305033101059926, + "tokens_seen": 3192324096 + }, + { + "epoch": 0.93, + "learning_rate": 6.580003209757664e-06, + "loss": 2.4149, + "theoretical_loss": 3.3050227989941305, + "tokens_seen": 3192455168 + }, + { + "epoch": 0.93, + "learning_rate": 6.571978815599422e-06, + "loss": 2.579, + "theoretical_loss": 3.3050124974697224, + "tokens_seen": 3192586240 + }, + { + "epoch": 0.94, + "learning_rate": 6.563954421441182e-06, + "loss": 2.5182, + "theoretical_loss": 3.3050021964866505, + "tokens_seen": 3192717312 + }, + { + "epoch": 0.94, + "learning_rate": 6.55593002728294e-06, + "loss": 2.562, + "theoretical_loss": 3.3049918960448643, + "tokens_seen": 3192848384 + }, + { + "epoch": 0.94, + "learning_rate": 6.5479056331247e-06, + "loss": 2.5949, + "theoretical_loss": 3.304981596144314, + "tokens_seen": 3192979456 + }, + { + "epoch": 0.94, + "learning_rate": 6.5398812389664584e-06, + "loss": 2.5232, + "theoretical_loss": 3.304971296784948, + "tokens_seen": 3193110528 + }, + { + "epoch": 0.94, + "learning_rate": 6.531856844808218e-06, + "loss": 2.548, + "theoretical_loss": 3.3049609979667154, + "tokens_seen": 3193241600 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1749492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.783308744430542, + "objective/train/theoretical_loss": 3.3049506996895666, + "objective/train/tokens_used": 1563831776, + "theoretical_loss": 3.3049506996895666, + "tokens_seen": 3193372672 + }, + { + "epoch": 0.94, + "learning_rate": 6.5238324506499765e-06, + "loss": 2.497, + "theoretical_loss": 3.3049506996895666, + "tokens_seen": 3193372672 + }, + { + "epoch": 0.94, + "learning_rate": 6.515808056491735e-06, + "loss": 2.6158, + "theoretical_loss": 3.3049404019534503, + "tokens_seen": 3193503744 + }, + { + "epoch": 0.94, + "learning_rate": 6.507783662333494e-06, + "loss": 2.3978, + "theoretical_loss": 3.304930104758316, + "tokens_seen": 3193634816 + }, + { + "epoch": 0.94, + "learning_rate": 6.499759268175253e-06, + "loss": 2.487, + "theoretical_loss": 3.304919808104113, + "tokens_seen": 3193765888 + }, + { + "epoch": 0.94, + "learning_rate": 6.491734874017012e-06, + "loss": 2.5148, + "theoretical_loss": 3.3049095119907905, + "tokens_seen": 3193896960 + }, + { + "epoch": 0.94, + "learning_rate": 6.48371047985877e-06, + "loss": 2.5413, + "theoretical_loss": 3.3048992164182986, + "tokens_seen": 3194028032 + }, + { + "epoch": 0.94, + "learning_rate": 6.47568608570053e-06, + "loss": 2.5461, + "theoretical_loss": 3.3048889213865857, + "tokens_seen": 3194159104 + }, + { + "epoch": 0.94, + "learning_rate": 6.467661691542288e-06, + "loss": 2.6027, + "theoretical_loss": 3.3048786268956025, + "tokens_seen": 3194290176 + }, + { + "epoch": 0.94, + "learning_rate": 6.459637297384048e-06, + "loss": 2.5028, + "theoretical_loss": 3.3048683329452975, + "tokens_seen": 3194421248 + }, + { + "epoch": 0.94, + "learning_rate": 6.451612903225806e-06, + "loss": 2.5938, + "theoretical_loss": 3.30485803953562, + "tokens_seen": 3194552320 + }, + { + "epoch": 0.94, + "learning_rate": 6.443588509067566e-06, + "loss": 2.6057, + "theoretical_loss": 3.30484774666652, + "tokens_seen": 3194683392 + }, + { + "epoch": 0.94, + "learning_rate": 6.4355641149093245e-06, + "loss": 2.3942, + "theoretical_loss": 3.3048374543379464, + "tokens_seen": 3194814464 + }, + { + "epoch": 0.94, + "learning_rate": 6.427539720751084e-06, + "loss": 2.3556, + "theoretical_loss": 3.304827162549849, + "tokens_seen": 3194945536 + }, + { + "epoch": 0.94, + "objective/train/docs_used": 1749977, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8383281230926514, + "objective/train/theoretical_loss": 3.304822016858463, + "objective/train/tokens_used": 1565470176, + "theoretical_loss": 3.304822016858463, + "tokens_seen": 3195011072 + }, + { + "epoch": 0.94, + "learning_rate": 6.419515326592843e-06, + "loss": 2.4866, + "theoretical_loss": 3.3048168713021773, + "tokens_seen": 3195076608 + }, + { + "epoch": 0.94, + "learning_rate": 6.411490932434601e-06, + "loss": 2.6162, + "theoretical_loss": 3.3048065805948803, + "tokens_seen": 3195207680 + }, + { + "epoch": 0.94, + "learning_rate": 6.403466538276361e-06, + "loss": 2.4537, + "theoretical_loss": 3.304796290427908, + "tokens_seen": 3195338752 + }, + { + "epoch": 0.94, + "learning_rate": 6.395442144118119e-06, + "loss": 2.5882, + "theoretical_loss": 3.3047860008012098, + "tokens_seen": 3195469824 + }, + { + "epoch": 0.94, + "learning_rate": 6.387417749959879e-06, + "loss": 2.4432, + "theoretical_loss": 3.3047757117147345, + "tokens_seen": 3195600896 + }, + { + "epoch": 0.94, + "learning_rate": 6.379393355801637e-06, + "loss": 2.4487, + "theoretical_loss": 3.3047654231684325, + "tokens_seen": 3195731968 + }, + { + "epoch": 0.94, + "learning_rate": 6.371368961643397e-06, + "loss": 2.4922, + "theoretical_loss": 3.3047551351622526, + "tokens_seen": 3195863040 + }, + { + "epoch": 0.94, + "learning_rate": 6.363344567485155e-06, + "loss": 2.5997, + "theoretical_loss": 3.3047448476961447, + "tokens_seen": 3195994112 + }, + { + "epoch": 0.94, + "learning_rate": 6.355320173326914e-06, + "loss": 2.4727, + "theoretical_loss": 3.3047345607700582, + "tokens_seen": 3196125184 + }, + { + "epoch": 0.94, + "learning_rate": 6.347295779168673e-06, + "loss": 2.6507, + "theoretical_loss": 3.3047242743839425, + "tokens_seen": 3196256256 + } + ], + "max_steps": 12588, + "num_train_epochs": 9223372036854775807, + "total_flos": 7.89110297985024e+17, + "trial_name": null, + "trial_params": null +}