diff --git "a/checkpoint-12588/trainer_state.json" "b/checkpoint-12588/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-12588/trainer_state.json" @@ -0,0 +1,91911 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "global_step": 12588, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 1.2954, + "theoretical_loss": 3.515367210984307, + "tokens_seen": 1650130944 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 1.323, + "theoretical_loss": 3.515343236624406, + "tokens_seen": 1650262016 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 1.3026, + "theoretical_loss": 3.5153192647017093, + "tokens_seen": 1650393088 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 1.328, + "theoretical_loss": 3.5152952952157754, + "tokens_seen": 1650524160 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 1.292, + "theoretical_loss": 3.515271328166164, + "tokens_seen": 1650655232 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 1.2743, + "theoretical_loss": 3.515247363552433, + "tokens_seen": 1650786304 + }, + { + "epoch": 0.0, + "learning_rate": 2.7777777777777776e-05, + "loss": 1.2506, + "theoretical_loss": 3.5152234013741417, + "tokens_seen": 1650917376 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 1.293, + "theoretical_loss": 3.51519944163085, + "tokens_seen": 1651048448 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428571e-05, + "loss": 1.3584, + "theoretical_loss": 3.515175484322116, + "tokens_seen": 1651179520 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 1.3155, + "theoretical_loss": 3.5151515294474995, + "tokens_seen": 1651310592 + }, + { + "epoch": 0.0, + "learning_rate": 4.365079365079365e-05, + "loss": 1.2702, + "theoretical_loss": 3.5151275770065604, + "tokens_seen": 1651441664 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 1.2836, + "theoretical_loss": 3.5151036269988585, + "tokens_seen": 1651572736 + }, + { + "epoch": 0.0, + "learning_rate": 5.1587301587301586e-05, + "loss": 1.3128, + "theoretical_loss": 3.5150796794239527, + "tokens_seen": 1651703808 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555555e-05, + "loss": 1.3298, + "theoretical_loss": 3.515055734281404, + "tokens_seen": 1651834880 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809524e-05, + "loss": 1.2828, + "theoretical_loss": 3.5150317915707707, + "tokens_seen": 1651965952 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 1.2641, + "theoretical_loss": 3.5150078512916143, + "tokens_seen": 1652097024 + }, + { + "epoch": 0.0, + "learning_rate": 6.746031746031745e-05, + "loss": 1.3453, + "theoretical_loss": 3.5149839134434946, + "tokens_seen": 1652228096 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857142e-05, + "loss": 1.3684, + "theoretical_loss": 3.5149599780259715, + "tokens_seen": 1652359168 + }, + { + "epoch": 0.0, + "learning_rate": 7.53968253968254e-05, + "loss": 1.3011, + "theoretical_loss": 3.5149360450386062, + "tokens_seen": 1652490240 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 1.2501, + "theoretical_loss": 3.5149121144809587, + "tokens_seen": 1652621312 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 1.2529, + "theoretical_loss": 3.5148881863525903, + "tokens_seen": 1652752384 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-05, + "loss": 1.3354, + "theoretical_loss": 3.5148642606530602, + "tokens_seen": 1652883456 + }, + { + "epoch": 0.0, + "learning_rate": 9.126984126984126e-05, + "loss": 1.3142, + "theoretical_loss": 3.514840337381931, + "tokens_seen": 1653014528 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 1.3052, + "theoretical_loss": 3.514816416538763, + "tokens_seen": 1653145600 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-05, + "loss": 1.3221, + "theoretical_loss": 3.5147924981231173, + "tokens_seen": 1653276672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460317, + "loss": 1.3672, + "theoretical_loss": 3.514768582134555, + "tokens_seen": 1653407744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010714285714285714, + "loss": 1.2662, + "theoretical_loss": 3.514744668572638, + "tokens_seen": 1653538816 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001111111111111111, + "loss": 1.322, + "theoretical_loss": 3.514720757436927, + "tokens_seen": 1653669888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011507936507936508, + "loss": 1.2723, + "theoretical_loss": 3.5146968487269845, + "tokens_seen": 1653800960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 1.2962, + "theoretical_loss": 3.5146729424423713, + "tokens_seen": 1653932032 + }, + { + "epoch": 0.0, + "learning_rate": 0.000123015873015873, + "loss": 1.2676, + "theoretical_loss": 3.5146490385826494, + "tokens_seen": 1654063104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 1.2566, + "theoretical_loss": 3.5146251371473807, + "tokens_seen": 1654194176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013095238095238096, + "loss": 1.3675, + "theoretical_loss": 3.5146012381361276, + "tokens_seen": 1654325248 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001349206349206349, + "loss": 1.3177, + "theoretical_loss": 3.514577341548452, + "tokens_seen": 1654456320 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001388888888888889, + "loss": 1.3195, + "theoretical_loss": 3.514553447383916, + "tokens_seen": 1654587392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714284, + "loss": 1.2776, + "theoretical_loss": 3.514529555642082, + "tokens_seen": 1654718464 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014682539682539685, + "loss": 1.34, + "theoretical_loss": 3.514505666322513, + "tokens_seen": 1654849536 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 1.2555, + "theoretical_loss": 3.514481779424771, + "tokens_seen": 1654980608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015476190476190478, + "loss": 1.3034, + "theoretical_loss": 3.5144578949484186, + "tokens_seen": 1655111680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 1.2644, + "theoretical_loss": 3.5144340128930196, + "tokens_seen": 1655242752 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001626984126984127, + "loss": 1.3046, + "theoretical_loss": 3.5144101332581354, + "tokens_seen": 1655373824 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 1.3482, + "theoretical_loss": 3.5143862560433305, + "tokens_seen": 1655504896 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017063492063492064, + "loss": 1.3105, + "theoretical_loss": 3.514362381248167, + "tokens_seen": 1655635968 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001746031746031746, + "loss": 1.2689, + "theoretical_loss": 3.5143385088722088, + "tokens_seen": 1655767040 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017857142857142857, + "loss": 1.3015, + "theoretical_loss": 3.5143146389150193, + "tokens_seen": 1655898112 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968252, + "loss": 1.289, + "theoretical_loss": 3.514290771376162, + "tokens_seen": 1656029184 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001865079365079365, + "loss": 1.2948, + "theoretical_loss": 3.5142669062552, + "tokens_seen": 1656160256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 1.2804, + "theoretical_loss": 3.5142430435516974, + "tokens_seen": 1656291328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019444444444444446, + "loss": 1.2553, + "theoretical_loss": 3.514219183265218, + "tokens_seen": 1656422400 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.9931244850158691, + "objective/train/avg_token_score": 0.006875487044453621, + "objective/train/docs_used": 604805, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3570479154586792, + "objective/train/original_loss": 1.3666274547576904, + "objective/train/theoretical_loss": 3.5141953253953258, + "objective/train/tokens_used": 27012576, + "objective/train/unlikelihood_loss": 0.019734039902687073, + "theoretical_loss": 3.5141953253953258, + "tokens_seen": 1656553472 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001984126984126984, + "loss": 1.3355, + "theoretical_loss": 3.5141953253953258, + "tokens_seen": 1656553472 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002023809523809524, + "loss": 1.2595, + "theoretical_loss": 3.5141714699415845, + "tokens_seen": 1656684544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920634, + "loss": 1.2891, + "theoretical_loss": 3.514147616903559, + "tokens_seen": 1656815616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021031746031746032, + "loss": 1.2867, + "theoretical_loss": 3.5141237662808127, + "tokens_seen": 1656946688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021428571428571427, + "loss": 1.371, + "theoretical_loss": 3.514099918072911, + "tokens_seen": 1657077760 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021825396825396825, + "loss": 1.2391, + "theoretical_loss": 3.5140760722794178, + "tokens_seen": 1657208832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002222222222222222, + "loss": 1.2492, + "theoretical_loss": 3.5140522288998977, + "tokens_seen": 1657339904 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022619047619047618, + "loss": 1.2711, + "theoretical_loss": 3.514028387933916, + "tokens_seen": 1657470976 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 1.2276, + "theoretical_loss": 3.5140045493810366, + "tokens_seen": 1657602048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023412698412698414, + "loss": 1.3298, + "theoretical_loss": 3.5139807132408256, + "tokens_seen": 1657733120 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 1.2457, + "theoretical_loss": 3.513956879512847, + "tokens_seen": 1657864192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024206349206349207, + "loss": 1.2985, + "theoretical_loss": 3.513933048196667, + "tokens_seen": 1657995264 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246031746031746, + "loss": 1.3261, + "theoretical_loss": 3.5139092192918495, + "tokens_seen": 1658126336 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 1.2236, + "theoretical_loss": 3.5138853927979614, + "tokens_seen": 1658257408 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025396825396825396, + "loss": 1.3278, + "theoretical_loss": 3.513861568714568, + "tokens_seen": 1658388480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025793650793650796, + "loss": 1.3517, + "theoretical_loss": 3.513837747041234, + "tokens_seen": 1658519552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002619047619047619, + "loss": 1.3326, + "theoretical_loss": 3.513813927777526, + "tokens_seen": 1658650624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026587301587301587, + "loss": 1.3122, + "theoretical_loss": 3.5137901109230096, + "tokens_seen": 1658781696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002698412698412698, + "loss": 1.2725, + "theoretical_loss": 3.513766296477251, + "tokens_seen": 1658912768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002738095238095238, + "loss": 1.3628, + "theoretical_loss": 3.5137424844398164, + "tokens_seen": 1659043840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002777777777777778, + "loss": 1.3441, + "theoretical_loss": 3.5137186748102716, + "tokens_seen": 1659174912 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028174603174603173, + "loss": 1.342, + "theoretical_loss": 3.5136948675881827, + "tokens_seen": 1659305984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002857142857142857, + "loss": 1.2716, + "theoretical_loss": 3.5136710627731165, + "tokens_seen": 1659437056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002896825396825397, + "loss": 1.2171, + "theoretical_loss": 3.51364726036464, + "tokens_seen": 1659568128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002936507936507937, + "loss": 1.4285, + "theoretical_loss": 3.513623460362319, + "tokens_seen": 1659699200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029761904761904765, + "loss": 1.3291, + "theoretical_loss": 3.5135996627657207, + "tokens_seen": 1659830272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003015873015873016, + "loss": 1.313, + "theoretical_loss": 3.5135758675744118, + "tokens_seen": 1659961344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003055555555555556, + "loss": 1.2207, + "theoretical_loss": 3.5135520747879596, + "tokens_seen": 1660092416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030952380952380956, + "loss": 1.3769, + "theoretical_loss": 3.5135282844059317, + "tokens_seen": 1660223488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003134920634920635, + "loss": 1.2857, + "theoretical_loss": 3.5135044964278936, + "tokens_seen": 1660354560 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031746031746031746, + "loss": 1.2861, + "theoretical_loss": 3.5134807108534147, + "tokens_seen": 1660485632 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032142857142857147, + "loss": 1.2486, + "theoretical_loss": 3.5134569276820606, + "tokens_seen": 1660616704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003253968253968254, + "loss": 1.2224, + "theoretical_loss": 3.5134331469134006, + "tokens_seen": 1660747776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032936507936507937, + "loss": 1.2796, + "theoretical_loss": 3.5134093685470007, + "tokens_seen": 1660878848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003333333333333333, + "loss": 1.3365, + "theoretical_loss": 3.5133855925824298, + "tokens_seen": 1661009920 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033730158730158733, + "loss": 1.3668, + "theoretical_loss": 3.5133618190192553, + "tokens_seen": 1661140992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003412698412698413, + "loss": 1.3324, + "theoretical_loss": 3.5133380478570455, + "tokens_seen": 1661272064 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034523809523809523, + "loss": 1.2642, + "theoretical_loss": 3.5133142790953684, + "tokens_seen": 1661403136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003492063492063492, + "loss": 1.3294, + "theoretical_loss": 3.513290512733792, + "tokens_seen": 1661534208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003531746031746032, + "loss": 1.3075, + "theoretical_loss": 3.513266748771885, + "tokens_seen": 1661665280 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035714285714285714, + "loss": 1.2986, + "theoretical_loss": 3.5132429872092157, + "tokens_seen": 1661796352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003611111111111111, + "loss": 1.2774, + "theoretical_loss": 3.5132192280453527, + "tokens_seen": 1661927424 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036507936507936505, + "loss": 1.3365, + "theoretical_loss": 3.5131954712798645, + "tokens_seen": 1662058496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036904761904761905, + "loss": 1.3278, + "theoretical_loss": 3.5131717169123196, + "tokens_seen": 1662189568 + }, + { + "epoch": 0.01, + "learning_rate": 0.000373015873015873, + "loss": 1.2726, + "theoretical_loss": 3.513147964942288, + "tokens_seen": 1662320640 + }, + { + "epoch": 0.01, + "learning_rate": 0.000376984126984127, + "loss": 1.2737, + "theoretical_loss": 3.513124215369338, + "tokens_seen": 1662451712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038095238095238096, + "loss": 1.3483, + "theoretical_loss": 3.513100468193038, + "tokens_seen": 1662582784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038492063492063497, + "loss": 1.2867, + "theoretical_loss": 3.5130767234129587, + "tokens_seen": 1662713856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003888888888888889, + "loss": 1.3175, + "theoretical_loss": 3.513052981028668, + "tokens_seen": 1662844928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003928571428571429, + "loss": 1.3579, + "theoretical_loss": 3.513029241039737, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9920444488525391, + "objective/train/avg_token_score": 0.007955576293170452, + "objective/train/docs_used": 607267, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3090605735778809, + "objective/train/original_loss": 1.3201595544815063, + "objective/train/theoretical_loss": 3.5130055034457337, + "objective/train/tokens_used": 33566176, + "objective/train/unlikelihood_loss": 0.02126361057162285, + "theoretical_loss": 3.5130055034457337, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003968253968253968, + "loss": 1.2779, + "theoretical_loss": 3.5130055034457337, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040079365079365083, + "loss": 1.3357, + "theoretical_loss": 3.5129817682462283, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 1.3508, + "theoretical_loss": 3.512958035440791, + "tokens_seen": 1663369216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040873015873015874, + "loss": 1.3005, + "theoretical_loss": 3.512934305028991, + "tokens_seen": 1663500288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004126984126984127, + "loss": 1.3042, + "theoretical_loss": 3.512910577010399, + "tokens_seen": 1663631360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004166666666666667, + "loss": 1.3062, + "theoretical_loss": 3.512886851384585, + "tokens_seen": 1663762432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 1.3094, + "theoretical_loss": 3.5128631281511185, + "tokens_seen": 1663893504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004246031746031746, + "loss": 1.3668, + "theoretical_loss": 3.5128394073095706, + "tokens_seen": 1664024576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042857142857142855, + "loss": 1.2962, + "theoretical_loss": 3.512815688859512, + "tokens_seen": 1664155648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043253968253968256, + "loss": 1.2947, + "theoretical_loss": 3.512791972800512, + "tokens_seen": 1664286720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 1.3039, + "theoretical_loss": 3.5127682591321427, + "tokens_seen": 1664417792 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044047619047619046, + "loss": 1.269, + "theoretical_loss": 3.5127445478539743, + "tokens_seen": 1664548864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004444444444444444, + "loss": 1.2902, + "theoretical_loss": 3.5127208389655777, + "tokens_seen": 1664679936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004484126984126984, + "loss": 1.3061, + "theoretical_loss": 3.5126971324665237, + "tokens_seen": 1664811008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 1.3102, + "theoretical_loss": 3.5126734283563836, + "tokens_seen": 1664942080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004563492063492063, + "loss": 1.3044, + "theoretical_loss": 3.5126497266347285, + "tokens_seen": 1665073152 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 1.2718, + "theoretical_loss": 3.5126260273011303, + "tokens_seen": 1665204224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046428571428571433, + "loss": 1.3504, + "theoretical_loss": 3.5126023303551595, + "tokens_seen": 1665335296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 1.2953, + "theoretical_loss": 3.5125786357963884, + "tokens_seen": 1665466368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047222222222222224, + "loss": 1.3977, + "theoretical_loss": 3.5125549436243886, + "tokens_seen": 1665597440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 1.2928, + "theoretical_loss": 3.5125312538387314, + "tokens_seen": 1665728512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004801587301587302, + "loss": 1.3614, + "theoretical_loss": 3.5125075664389893, + "tokens_seen": 1665859584 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048412698412698415, + "loss": 1.3694, + "theoretical_loss": 3.512483881424734, + "tokens_seen": 1665990656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004880952380952381, + "loss": 1.2858, + "theoretical_loss": 3.5124601987955377, + "tokens_seen": 1666121728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492063492063492, + "loss": 1.363, + "theoretical_loss": 3.5124365185509725, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496031746031746, + "loss": 1.3014, + "theoretical_loss": 3.5124128406906108, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 1.4074, + "theoretical_loss": 3.5123891652140253, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999598780292088, + "loss": 1.3423, + "theoretical_loss": 3.512365492120788, + "tokens_seen": 1666646016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999197560584176, + "loss": 1.3785, + "theoretical_loss": 3.5123418214104722, + "tokens_seen": 1666777088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998796340876264, + "loss": 1.2125, + "theoretical_loss": 3.51231815308265, + "tokens_seen": 1666908160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998395121168352, + "loss": 1.3318, + "theoretical_loss": 3.5122944871368946, + "tokens_seen": 1667039232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997993901460439, + "loss": 1.233, + "theoretical_loss": 3.5122708235727798, + "tokens_seen": 1667170304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997592681752528, + "loss": 1.3325, + "theoretical_loss": 3.512247162389877, + "tokens_seen": 1667301376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997191462044615, + "loss": 1.3193, + "theoretical_loss": 3.512223503587761, + "tokens_seen": 1667432448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996790242336704, + "loss": 1.254, + "theoretical_loss": 3.512199847166004, + "tokens_seen": 1667563520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996389022628792, + "loss": 1.3069, + "theoretical_loss": 3.5121761931241804, + "tokens_seen": 1667694592 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499598780292088, + "loss": 1.2874, + "theoretical_loss": 3.512152541461863, + "tokens_seen": 1667825664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995586583212968, + "loss": 1.3728, + "theoretical_loss": 3.512128892178626, + "tokens_seen": 1667956736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995185363505056, + "loss": 1.29, + "theoretical_loss": 3.5121052452740424, + "tokens_seen": 1668087808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994784143797144, + "loss": 1.282, + "theoretical_loss": 3.512081600747687, + "tokens_seen": 1668218880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994382924089231, + "loss": 1.2923, + "theoretical_loss": 3.512057958599133, + "tokens_seen": 1668349952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993981704381319, + "loss": 1.4211, + "theoretical_loss": 3.512034318827955, + "tokens_seen": 1668481024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993580484673407, + "loss": 1.2612, + "theoretical_loss": 3.5120106814337273, + "tokens_seen": 1668612096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993179264965495, + "loss": 1.3874, + "theoretical_loss": 3.511987046416024, + "tokens_seen": 1668743168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992778045257583, + "loss": 1.3213, + "theoretical_loss": 3.5119634137744193, + "tokens_seen": 1668874240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992376825549671, + "loss": 1.2878, + "theoretical_loss": 3.5119397835084882, + "tokens_seen": 1669005312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991975605841759, + "loss": 1.3387, + "theoretical_loss": 3.5119161556178047, + "tokens_seen": 1669136384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991574386133847, + "loss": 1.3605, + "theoretical_loss": 3.511892530101944, + "tokens_seen": 1669267456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991173166425935, + "loss": 1.3786, + "theoretical_loss": 3.511868906960481, + "tokens_seen": 1669398528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990771946718023, + "loss": 1.3979, + "theoretical_loss": 3.5118452861929907, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9960150122642517, + "objective/train/avg_token_score": 0.003985044080764055, + "objective/train/docs_used": 609762, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4626469612121582, + "objective/train/original_loss": 1.4692569971084595, + "objective/train/theoretical_loss": 3.5118216677990484, + "objective/train/tokens_used": 40119776, + "objective/train/unlikelihood_loss": 0.010992846451699734, + "theoretical_loss": 3.5118216677990484, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499037072701011, + "loss": 1.3894, + "theoretical_loss": 3.5118216677990484, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989969507302198, + "loss": 1.3617, + "theoretical_loss": 3.5117980517782286, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989568287594286, + "loss": 1.2727, + "theoretical_loss": 3.5117744381301064, + "tokens_seen": 1669922816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989167067886375, + "loss": 1.3315, + "theoretical_loss": 3.5117508268542585, + "tokens_seen": 1670053888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988765848178463, + "loss": 1.4162, + "theoretical_loss": 3.511727217950259, + "tokens_seen": 1670184960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988364628470551, + "loss": 1.3527, + "theoretical_loss": 3.511703611417685, + "tokens_seen": 1670316032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987963408762639, + "loss": 1.3206, + "theoretical_loss": 3.5116800072561114, + "tokens_seen": 1670447104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987562189054726, + "loss": 1.2814, + "theoretical_loss": 3.5116564054651134, + "tokens_seen": 1670578176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987160969346815, + "loss": 1.3643, + "theoretical_loss": 3.5116328060442683, + "tokens_seen": 1670709248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986759749638902, + "loss": 1.2985, + "theoretical_loss": 3.511609208993151, + "tokens_seen": 1670840320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986358529930991, + "loss": 1.3197, + "theoretical_loss": 3.511585614311339, + "tokens_seen": 1670971392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985957310223078, + "loss": 1.333, + "theoretical_loss": 3.511562021998407, + "tokens_seen": 1671102464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985556090515166, + "loss": 1.3421, + "theoretical_loss": 3.5115384320539325, + "tokens_seen": 1671233536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985154870807254, + "loss": 1.3619, + "theoretical_loss": 3.5115148444774915, + "tokens_seen": 1671364608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984753651099342, + "loss": 1.3792, + "theoretical_loss": 3.511491259268661, + "tokens_seen": 1671495680 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498435243139143, + "loss": 1.3389, + "theoretical_loss": 3.5114676764270176, + "tokens_seen": 1671626752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983951211683518, + "loss": 1.3388, + "theoretical_loss": 3.511444095952138, + "tokens_seen": 1671757824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983549991975606, + "loss": 1.4075, + "theoretical_loss": 3.5114205178435984, + "tokens_seen": 1671888896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983148772267693, + "loss": 1.3054, + "theoretical_loss": 3.5113969421009776, + "tokens_seen": 1672019968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982747552559782, + "loss": 1.362, + "theoretical_loss": 3.5113733687238513, + "tokens_seen": 1672151040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982346332851869, + "loss": 1.3747, + "theoretical_loss": 3.5113497977117976, + "tokens_seen": 1672282112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981945113143958, + "loss": 1.3145, + "theoretical_loss": 3.5113262290643936, + "tokens_seen": 1672413184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981543893436046, + "loss": 1.3433, + "theoretical_loss": 3.5113026627812163, + "tokens_seen": 1672544256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981142673728134, + "loss": 1.3309, + "theoretical_loss": 3.5112790988618436, + "tokens_seen": 1672675328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980741454020222, + "loss": 1.2986, + "theoretical_loss": 3.5112555373058534, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498034023431231, + "loss": 1.3597, + "theoretical_loss": 3.511231978112823, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979939014604398, + "loss": 1.3624, + "theoretical_loss": 3.5112084212823316, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979537794896486, + "loss": 1.3481, + "theoretical_loss": 3.511184866813956, + "tokens_seen": 1673199616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979136575188573, + "loss": 1.3952, + "theoretical_loss": 3.5111613147072744, + "tokens_seen": 1673330688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978735355480661, + "loss": 1.3286, + "theoretical_loss": 3.5111377649618656, + "tokens_seen": 1673461760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978334135772749, + "loss": 1.3466, + "theoretical_loss": 3.5111142175773074, + "tokens_seen": 1673592832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977932916064837, + "loss": 1.2937, + "theoretical_loss": 3.5110906725531787, + "tokens_seen": 1673723904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977531696356925, + "loss": 1.3231, + "theoretical_loss": 3.5110671298890574, + "tokens_seen": 1673854976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977130476649013, + "loss": 1.3438, + "theoretical_loss": 3.5110435895845233, + "tokens_seen": 1673986048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976729256941101, + "loss": 1.3622, + "theoretical_loss": 3.511020051639154, + "tokens_seen": 1674117120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976328037233188, + "loss": 1.2417, + "theoretical_loss": 3.510996516052529, + "tokens_seen": 1674248192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975926817525277, + "loss": 1.3894, + "theoretical_loss": 3.5109729828242275, + "tokens_seen": 1674379264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975525597817364, + "loss": 1.3257, + "theoretical_loss": 3.5109494519538282, + "tokens_seen": 1674510336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975124378109452, + "loss": 1.4126, + "theoretical_loss": 3.5109259234409107, + "tokens_seen": 1674641408 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497472315840154, + "loss": 1.3665, + "theoretical_loss": 3.5109023972850535, + "tokens_seen": 1674772480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974321938693629, + "loss": 1.305, + "theoretical_loss": 3.510878873485837, + "tokens_seen": 1674903552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973920718985717, + "loss": 1.3188, + "theoretical_loss": 3.51085535204284, + "tokens_seen": 1675034624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973519499277805, + "loss": 1.3684, + "theoretical_loss": 3.5108318329556427, + "tokens_seen": 1675165696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973118279569893, + "loss": 1.3153, + "theoretical_loss": 3.510808316223825, + "tokens_seen": 1675296768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972717059861981, + "loss": 1.2203, + "theoretical_loss": 3.510784801846966, + "tokens_seen": 1675427840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972315840154069, + "loss": 1.275, + "theoretical_loss": 3.5107612898246456, + "tokens_seen": 1675558912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971914620446156, + "loss": 1.2465, + "theoretical_loss": 3.510737780156445, + "tokens_seen": 1675689984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971513400738245, + "loss": 1.304, + "theoretical_loss": 3.5107142728419434, + "tokens_seen": 1675821056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971112181030332, + "loss": 1.3867, + "theoretical_loss": 3.510690767880721, + "tokens_seen": 1675952128 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497071096132242, + "loss": 1.3083, + "theoretical_loss": 3.5106672652723594, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9904825091362, + "objective/train/avg_token_score": 0.009517486207187176, + "objective/train/docs_used": 612232, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3090497255325317, + "objective/train/original_loss": 1.3231884241104126, + "objective/train/theoretical_loss": 3.5106437650164377, + "objective/train/tokens_used": 46673376, + "objective/train/unlikelihood_loss": 0.026900524273514748, + "theoretical_loss": 3.5106437650164377, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970309741614508, + "loss": 1.2807, + "theoretical_loss": 3.5106437650164377, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969908521906596, + "loss": 1.2733, + "theoretical_loss": 3.5106202671125377, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969507302198684, + "loss": 1.3088, + "theoretical_loss": 3.5105967715602393, + "tokens_seen": 1676476416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969106082490772, + "loss": 1.2668, + "theoretical_loss": 3.5105732783591237, + "tokens_seen": 1676607488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496870486278286, + "loss": 1.2891, + "theoretical_loss": 3.5105497875087717, + "tokens_seen": 1676738560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968303643074947, + "loss": 1.2995, + "theoretical_loss": 3.510526299008764, + "tokens_seen": 1676869632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967902423367036, + "loss": 1.3015, + "theoretical_loss": 3.5105028128586824, + "tokens_seen": 1677000704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967501203659123, + "loss": 1.3186, + "theoretical_loss": 3.5104793290581084, + "tokens_seen": 1677131776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967099983951212, + "loss": 1.2787, + "theoretical_loss": 3.510455847606622, + "tokens_seen": 1677262848 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049666987642433, + "loss": 1.3972, + "theoretical_loss": 3.510432368503806, + "tokens_seen": 1677393920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966297544535388, + "loss": 1.222, + "theoretical_loss": 3.5104088917492415, + "tokens_seen": 1677524992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965896324827476, + "loss": 1.2476, + "theoretical_loss": 3.51038541734251, + "tokens_seen": 1677656064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965495105119564, + "loss": 1.2688, + "theoretical_loss": 3.5103619452831936, + "tokens_seen": 1677787136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965093885411652, + "loss": 1.3785, + "theoretical_loss": 3.510338475570874, + "tokens_seen": 1677918208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496469266570374, + "loss": 1.3245, + "theoretical_loss": 3.5103150082051338, + "tokens_seen": 1678049280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964291445995827, + "loss": 1.3758, + "theoretical_loss": 3.5102915431855544, + "tokens_seen": 1678180352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963890226287915, + "loss": 1.3454, + "theoretical_loss": 3.510268080511718, + "tokens_seen": 1678311424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963489006580003, + "loss": 1.369, + "theoretical_loss": 3.510244620183207, + "tokens_seen": 1678442496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963087786872091, + "loss": 1.3466, + "theoretical_loss": 3.510221162199605, + "tokens_seen": 1678573568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962686567164179, + "loss": 1.343, + "theoretical_loss": 3.5101977065604926, + "tokens_seen": 1678704640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962285347456267, + "loss": 1.3307, + "theoretical_loss": 3.510174253265454, + "tokens_seen": 1678835712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961884127748355, + "loss": 1.3065, + "theoretical_loss": 3.510150802314071, + "tokens_seen": 1678966784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961482908040443, + "loss": 1.3551, + "theoretical_loss": 3.5101273537059274, + "tokens_seen": 1679097856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961081688332531, + "loss": 1.3529, + "theoretical_loss": 3.5101039074406053, + "tokens_seen": 1679228928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960680468624618, + "loss": 1.34, + "theoretical_loss": 3.5100804635176877, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960279248916708, + "loss": 1.2438, + "theoretical_loss": 3.510057021936759, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959878029208794, + "loss": 1.2747, + "theoretical_loss": 3.510033582697401, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959476809500883, + "loss": 1.3517, + "theoretical_loss": 3.510010145799198, + "tokens_seen": 1679753216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959075589792971, + "loss": 1.2558, + "theoretical_loss": 3.5099867112417336, + "tokens_seen": 1679884288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958674370085059, + "loss": 1.2991, + "theoretical_loss": 3.5099632790245905, + "tokens_seen": 1680015360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958273150377147, + "loss": 1.3149, + "theoretical_loss": 3.5099398491473535, + "tokens_seen": 1680146432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957871930669235, + "loss": 1.33, + "theoretical_loss": 3.5099164216096055, + "tokens_seen": 1680277504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957470710961323, + "loss": 1.2207, + "theoretical_loss": 3.5098929964109313, + "tokens_seen": 1680408576 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495706949125341, + "loss": 1.259, + "theoretical_loss": 3.5098695735509144, + "tokens_seen": 1680539648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956668271545499, + "loss": 1.2296, + "theoretical_loss": 3.509846153029139, + "tokens_seen": 1680670720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956267051837586, + "loss": 1.3282, + "theoretical_loss": 3.50982273484519, + "tokens_seen": 1680801792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955865832129674, + "loss": 1.4033, + "theoretical_loss": 3.5097993189986507, + "tokens_seen": 1680932864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955464612421762, + "loss": 1.3808, + "theoretical_loss": 3.5097759054891062, + "tokens_seen": 1681063936 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495506339271385, + "loss": 1.3112, + "theoretical_loss": 3.5097524943161407, + "tokens_seen": 1681195008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954662173005938, + "loss": 1.3157, + "theoretical_loss": 3.5097290854793393, + "tokens_seen": 1681326080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954260953298026, + "loss": 1.2927, + "theoretical_loss": 3.5097056789782863, + "tokens_seen": 1681457152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953859733590114, + "loss": 1.3807, + "theoretical_loss": 3.509682274812567, + "tokens_seen": 1681588224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953458513882202, + "loss": 1.3212, + "theoretical_loss": 3.509658872981767, + "tokens_seen": 1681719296 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495305729417429, + "loss": 1.3163, + "theoretical_loss": 3.50963547348547, + "tokens_seen": 1681850368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952656074466377, + "loss": 1.3597, + "theoretical_loss": 3.509612076323261, + "tokens_seen": 1681981440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952254854758466, + "loss": 1.324, + "theoretical_loss": 3.5095886814947277, + "tokens_seen": 1682112512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951853635050554, + "loss": 1.3226, + "theoretical_loss": 3.509565288999453, + "tokens_seen": 1682243584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951452415342642, + "loss": 1.3884, + "theoretical_loss": 3.509541898837024, + "tokens_seen": 1682374656 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495105119563473, + "loss": 1.3022, + "theoretical_loss": 3.509518511007026, + "tokens_seen": 1682505728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950649975926818, + "loss": 1.302, + "theoretical_loss": 3.5094951255090443, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9924430251121521, + "objective/train/avg_token_score": 0.00755696464329958, + "objective/train/docs_used": 614786, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2718364000320435, + "objective/train/original_loss": 1.2817505598068237, + "objective/train/theoretical_loss": 3.5094717423426642, + "objective/train/tokens_used": 53226976, + "objective/train/unlikelihood_loss": 0.023592958226799965, + "theoretical_loss": 3.5094717423426642, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950248756218906, + "loss": 1.2908, + "theoretical_loss": 3.5094717423426642, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949847536510994, + "loss": 1.2867, + "theoretical_loss": 3.5094483615074736, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949446316803081, + "loss": 1.3505, + "theoretical_loss": 3.5094249830030564, + "tokens_seen": 1683030016 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494904509709517, + "loss": 1.336, + "theoretical_loss": 3.509401606829, + "tokens_seen": 1683161088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948643877387257, + "loss": 1.3562, + "theoretical_loss": 3.509378232984891, + "tokens_seen": 1683292160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948242657679345, + "loss": 1.2861, + "theoretical_loss": 3.509354861470314, + "tokens_seen": 1683423232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947841437971433, + "loss": 1.3324, + "theoretical_loss": 3.5093314922848577, + "tokens_seen": 1683554304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947440218263521, + "loss": 1.3405, + "theoretical_loss": 3.509308125428107, + "tokens_seen": 1683685376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947038998555609, + "loss": 1.3763, + "theoretical_loss": 3.509284760899649, + "tokens_seen": 1683816448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946637778847697, + "loss": 1.3439, + "theoretical_loss": 3.5092613986990715, + "tokens_seen": 1683947520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946236559139785, + "loss": 1.4279, + "theoretical_loss": 3.50923803882596, + "tokens_seen": 1684078592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945835339431872, + "loss": 1.3433, + "theoretical_loss": 3.5092146812799023, + "tokens_seen": 1684209664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945434119723962, + "loss": 1.3781, + "theoretical_loss": 3.5091913260604857, + "tokens_seen": 1684340736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945032900016048, + "loss": 1.254, + "theoretical_loss": 3.5091679731672967, + "tokens_seen": 1684471808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944631680308137, + "loss": 1.3782, + "theoretical_loss": 3.5091446225999228, + "tokens_seen": 1684602880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944230460600225, + "loss": 1.3556, + "theoretical_loss": 3.5091212743579514, + "tokens_seen": 1684733952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943829240892313, + "loss": 1.3962, + "theoretical_loss": 3.5090979284409705, + "tokens_seen": 1684865024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943428021184401, + "loss": 1.4912, + "theoretical_loss": 3.509074584848568, + "tokens_seen": 1684996096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943026801476489, + "loss": 1.2943, + "theoretical_loss": 3.50905124358033, + "tokens_seen": 1685127168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942625581768577, + "loss": 1.307, + "theoretical_loss": 3.509027904635846, + "tokens_seen": 1685258240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942224362060665, + "loss": 1.2453, + "theoretical_loss": 3.5090045680147037, + "tokens_seen": 1685389312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941823142352753, + "loss": 1.3605, + "theoretical_loss": 3.50898123371649, + "tokens_seen": 1685520384 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494142192264484, + "loss": 1.391, + "theoretical_loss": 3.5089579017407946, + "tokens_seen": 1685651456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941020702936929, + "loss": 1.3063, + "theoretical_loss": 3.508934572087205, + "tokens_seen": 1685782528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940619483229016, + "loss": 1.3489, + "theoretical_loss": 3.5089112447553097, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940218263521104, + "loss": 1.3887, + "theoretical_loss": 3.5088879197446974, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939817043813192, + "loss": 1.3445, + "theoretical_loss": 3.5088645970549557, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493941582410528, + "loss": 1.3514, + "theoretical_loss": 3.5088412766856747, + "tokens_seen": 1686306816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939014604397368, + "loss": 1.3275, + "theoretical_loss": 3.508817958636442, + "tokens_seen": 1686437888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938613384689456, + "loss": 1.3691, + "theoretical_loss": 3.5087946429068477, + "tokens_seen": 1686568960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938212164981544, + "loss": 1.3898, + "theoretical_loss": 3.5087713294964793, + "tokens_seen": 1686700032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937810945273631, + "loss": 1.2456, + "theoretical_loss": 3.508748018404927, + "tokens_seen": 1686831104 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493740972556572, + "loss": 1.379, + "theoretical_loss": 3.50872470963178, + "tokens_seen": 1686962176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937008505857808, + "loss": 1.2819, + "theoretical_loss": 3.5087014031766266, + "tokens_seen": 1687093248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936607286149896, + "loss": 1.2506, + "theoretical_loss": 3.5086780990390576, + "tokens_seen": 1687224320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936206066441984, + "loss": 1.2445, + "theoretical_loss": 3.5086547972186617, + "tokens_seen": 1687355392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935804846734072, + "loss": 1.359, + "theoretical_loss": 3.5086314977150286, + "tokens_seen": 1687486464 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493540362702616, + "loss": 1.3084, + "theoretical_loss": 3.508608200527749, + "tokens_seen": 1687617536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935002407318248, + "loss": 1.357, + "theoretical_loss": 3.5085849056564107, + "tokens_seen": 1687748608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934601187610335, + "loss": 1.2975, + "theoretical_loss": 3.508561613100605, + "tokens_seen": 1687879680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934199967902424, + "loss": 1.4147, + "theoretical_loss": 3.5085383228599225, + "tokens_seen": 1688010752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933798748194511, + "loss": 1.3952, + "theoretical_loss": 3.5085150349339522, + "tokens_seen": 1688141824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933397528486599, + "loss": 1.3827, + "theoretical_loss": 3.5084917493222845, + "tokens_seen": 1688272896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932996308778687, + "loss": 1.3329, + "theoretical_loss": 3.5084684660245107, + "tokens_seen": 1688403968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932595089070775, + "loss": 1.2748, + "theoretical_loss": 3.5084451850402205, + "tokens_seen": 1688535040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932193869362863, + "loss": 1.248, + "theoretical_loss": 3.508421906369004, + "tokens_seen": 1688666112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931792649654951, + "loss": 1.2905, + "theoretical_loss": 3.5083986300104533, + "tokens_seen": 1688797184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931391429947039, + "loss": 1.3039, + "theoretical_loss": 3.5083753559641577, + "tokens_seen": 1688928256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930990210239126, + "loss": 1.3297, + "theoretical_loss": 3.508352084229709, + "tokens_seen": 1689059328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930588990531215, + "loss": 1.3637, + "theoretical_loss": 3.508328814806698, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9927401542663574, + "objective/train/avg_token_score": 0.00725984014570713, + "objective/train/docs_used": 617011, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4559146165847778, + "objective/train/original_loss": 1.4672493934631348, + "objective/train/theoretical_loss": 3.5083055476947154, + "objective/train/tokens_used": 59780576, + "objective/train/unlikelihood_loss": 0.020174244418740273, + "theoretical_loss": 3.5083055476947154, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930187770823302, + "loss": 1.3931, + "theoretical_loss": 3.5083055476947154, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929786551115392, + "loss": 1.2177, + "theoretical_loss": 3.5082822828933526, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929385331407479, + "loss": 1.2527, + "theoretical_loss": 3.5082590204022015, + "tokens_seen": 1689583616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928984111699567, + "loss": 1.3366, + "theoretical_loss": 3.5082357602208525, + "tokens_seen": 1689714688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928582891991655, + "loss": 1.3216, + "theoretical_loss": 3.508212502348898, + "tokens_seen": 1689845760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928181672283743, + "loss": 1.3039, + "theoretical_loss": 3.5081892467859292, + "tokens_seen": 1689976832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927780452575831, + "loss": 1.3295, + "theoretical_loss": 3.5081659935315375, + "tokens_seen": 1690107904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927379232867919, + "loss": 1.2548, + "theoretical_loss": 3.5081427425853153, + "tokens_seen": 1690238976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926978013160007, + "loss": 1.3197, + "theoretical_loss": 3.508119493946854, + "tokens_seen": 1690370048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926576793452094, + "loss": 1.3737, + "theoretical_loss": 3.5080962476157462, + "tokens_seen": 1690501120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926175573744183, + "loss": 1.4703, + "theoretical_loss": 3.508073003591584, + "tokens_seen": 1690632192 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492577435403627, + "loss": 1.3061, + "theoretical_loss": 3.508049761873959, + "tokens_seen": 1690763264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925373134328358, + "loss": 1.3745, + "theoretical_loss": 3.5080265224624636, + "tokens_seen": 1690894336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924971914620446, + "loss": 1.2994, + "theoretical_loss": 3.508003285356691, + "tokens_seen": 1691025408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924570694912534, + "loss": 1.333, + "theoretical_loss": 3.507980050556233, + "tokens_seen": 1691156480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924169475204622, + "loss": 1.2988, + "theoretical_loss": 3.507956818060683, + "tokens_seen": 1691287552 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492376825549671, + "loss": 1.3587, + "theoretical_loss": 3.507933587869633, + "tokens_seen": 1691418624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923367035788798, + "loss": 1.417, + "theoretical_loss": 3.507910359982676, + "tokens_seen": 1691549696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922965816080887, + "loss": 1.2789, + "theoretical_loss": 3.507887134399405, + "tokens_seen": 1691680768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922564596372974, + "loss": 1.3635, + "theoretical_loss": 3.5078639111194136, + "tokens_seen": 1691811840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922163376665062, + "loss": 1.2802, + "theoretical_loss": 3.5078406901422943, + "tokens_seen": 1691942912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492176215695715, + "loss": 1.3498, + "theoretical_loss": 3.5078174714676407, + "tokens_seen": 1692073984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921360937249238, + "loss": 1.2959, + "theoretical_loss": 3.507794255095046, + "tokens_seen": 1692205056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920959717541326, + "loss": 1.341, + "theoretical_loss": 3.5077710410241036, + "tokens_seen": 1692336128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920558497833414, + "loss": 1.3571, + "theoretical_loss": 3.5077478292544075, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920157278125502, + "loss": 1.3065, + "theoretical_loss": 3.5077246197855505, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919756058417589, + "loss": 1.3676, + "theoretical_loss": 3.507701412617128, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919354838709678, + "loss": 1.3725, + "theoretical_loss": 3.5076782077487323, + "tokens_seen": 1692860416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918953619001765, + "loss": 1.3316, + "theoretical_loss": 3.5076550051799575, + "tokens_seen": 1692991488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918552399293853, + "loss": 1.3464, + "theoretical_loss": 3.5076318049103987, + "tokens_seen": 1693122560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918151179585941, + "loss": 1.3211, + "theoretical_loss": 3.50760860693965, + "tokens_seen": 1693253632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917749959878029, + "loss": 1.3689, + "theoretical_loss": 3.5075854112673044, + "tokens_seen": 1693384704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917348740170117, + "loss": 1.3102, + "theoretical_loss": 3.507562217892958, + "tokens_seen": 1693515776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916947520462205, + "loss": 1.2944, + "theoretical_loss": 3.507539026816204, + "tokens_seen": 1693646848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916546300754293, + "loss": 1.3585, + "theoretical_loss": 3.507515838036637, + "tokens_seen": 1693777920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916145081046381, + "loss": 1.3954, + "theoretical_loss": 3.5074926515538527, + "tokens_seen": 1693908992 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491574386133847, + "loss": 1.3275, + "theoretical_loss": 3.5074694673674456, + "tokens_seen": 1694040064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915342641630556, + "loss": 1.4257, + "theoretical_loss": 3.5074462854770103, + "tokens_seen": 1694171136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914941421922646, + "loss": 1.2713, + "theoretical_loss": 3.507423105882142, + "tokens_seen": 1694302208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914540202214733, + "loss": 1.3922, + "theoretical_loss": 3.5073999285824353, + "tokens_seen": 1694433280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914138982506821, + "loss": 1.3212, + "theoretical_loss": 3.5073767535774865, + "tokens_seen": 1694564352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913737762798909, + "loss": 1.387, + "theoretical_loss": 3.50735358086689, + "tokens_seen": 1694695424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913336543090997, + "loss": 1.3293, + "theoretical_loss": 3.507330410450242, + "tokens_seen": 1694826496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912935323383085, + "loss": 1.3853, + "theoretical_loss": 3.507307242327137, + "tokens_seen": 1694957568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912534103675173, + "loss": 1.3383, + "theoretical_loss": 3.5072840764971716, + "tokens_seen": 1695088640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912132883967261, + "loss": 1.3287, + "theoretical_loss": 3.507260912959941, + "tokens_seen": 1695219712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911731664259348, + "loss": 1.2918, + "theoretical_loss": 3.5072377517150413, + "tokens_seen": 1695350784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911330444551437, + "loss": 1.3318, + "theoretical_loss": 3.507214592762068, + "tokens_seen": 1695481856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910929224843524, + "loss": 1.3019, + "theoretical_loss": 3.507191436100618, + "tokens_seen": 1695612928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910528005135612, + "loss": 1.3449, + "theoretical_loss": 3.507168281730287, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.9919037222862244, + "objective/train/avg_token_score": 0.008096263743937016, + "objective/train/docs_used": 619583, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2747950553894043, + "objective/train/original_loss": 1.2848873138427734, + "objective/train/theoretical_loss": 3.507145129650671, + "objective/train/tokens_used": 66334176, + "objective/train/unlikelihood_loss": 0.02430354803800583, + "theoretical_loss": 3.507145129650671, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049101267854277, + "loss": 1.2822, + "theoretical_loss": 3.507145129650671, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909725565719788, + "loss": 1.4168, + "theoretical_loss": 3.507121979861367, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909324346011876, + "loss": 1.2881, + "theoretical_loss": 3.50709883236197, + "tokens_seen": 1696137216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908923126303964, + "loss": 1.3472, + "theoretical_loss": 3.5070756871520783, + "tokens_seen": 1696268288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908521906596052, + "loss": 1.3693, + "theoretical_loss": 3.5070525442312874, + "tokens_seen": 1696399360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490812068688814, + "loss": 1.3973, + "theoretical_loss": 3.507029403599195, + "tokens_seen": 1696530432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907719467180228, + "loss": 1.3107, + "theoretical_loss": 3.5070062652553977, + "tokens_seen": 1696661504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907318247472316, + "loss": 1.2247, + "theoretical_loss": 3.5069831291994915, + "tokens_seen": 1696792576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906917027764404, + "loss": 1.3352, + "theoretical_loss": 3.5069599954310746, + "tokens_seen": 1696923648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906515808056492, + "loss": 1.2292, + "theoretical_loss": 3.5069368639497434, + "tokens_seen": 1697054720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490611458834858, + "loss": 1.3984, + "theoretical_loss": 3.506913734755096, + "tokens_seen": 1697185792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905713368640668, + "loss": 1.3372, + "theoretical_loss": 3.506890607846729, + "tokens_seen": 1697316864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905312148932756, + "loss": 1.2176, + "theoretical_loss": 3.5068674832242404, + "tokens_seen": 1697447936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904910929224843, + "loss": 1.3393, + "theoretical_loss": 3.5068443608872277, + "tokens_seen": 1697579008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904509709516932, + "loss": 1.3045, + "theoretical_loss": 3.506821240835288, + "tokens_seen": 1697710080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904108489809019, + "loss": 1.3687, + "theoretical_loss": 3.5067981230680196, + "tokens_seen": 1697841152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903707270101108, + "loss": 1.3014, + "theoretical_loss": 3.506775007585021, + "tokens_seen": 1697972224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903306050393195, + "loss": 1.2881, + "theoretical_loss": 3.5067518943858893, + "tokens_seen": 1698103296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902904830685283, + "loss": 1.393, + "theoretical_loss": 3.5067287834702223, + "tokens_seen": 1698234368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902503610977371, + "loss": 1.3459, + "theoretical_loss": 3.506705674837619, + "tokens_seen": 1698365440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902102391269459, + "loss": 1.4298, + "theoretical_loss": 3.5066825684876775, + "tokens_seen": 1698496512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901701171561547, + "loss": 1.306, + "theoretical_loss": 3.5066594644199958, + "tokens_seen": 1698627584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901299951853635, + "loss": 1.4065, + "theoretical_loss": 3.5066363626341728, + "tokens_seen": 1698758656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900898732145723, + "loss": 1.2559, + "theoretical_loss": 3.5066132631298075, + "tokens_seen": 1698889728 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490049751243781, + "loss": 1.3017, + "theoretical_loss": 3.5065901659064975, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049000962927299, + "loss": 1.3606, + "theoretical_loss": 3.5065670709638423, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899695073021987, + "loss": 1.2513, + "theoretical_loss": 3.5065439783014405, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899293853314075, + "loss": 1.2749, + "theoretical_loss": 3.5065208879188914, + "tokens_seen": 1699414016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898892633606163, + "loss": 1.2942, + "theoretical_loss": 3.506497799815794, + "tokens_seen": 1699545088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898491413898251, + "loss": 1.2596, + "theoretical_loss": 3.5064747139917474, + "tokens_seen": 1699676160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898090194190339, + "loss": 1.2887, + "theoretical_loss": 3.5064516304463513, + "tokens_seen": 1699807232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897688974482427, + "loss": 1.3307, + "theoretical_loss": 3.5064285491792044, + "tokens_seen": 1699938304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897287754774515, + "loss": 1.2483, + "theoretical_loss": 3.506405470189907, + "tokens_seen": 1700069376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896886535066603, + "loss": 1.2838, + "theoretical_loss": 3.5063823934780585, + "tokens_seen": 1700200448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896485315358691, + "loss": 1.2836, + "theoretical_loss": 3.5063593190432574, + "tokens_seen": 1700331520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896084095650778, + "loss": 1.2452, + "theoretical_loss": 3.5063362468851054, + "tokens_seen": 1700462592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895682875942866, + "loss": 1.2913, + "theoretical_loss": 3.506313177003202, + "tokens_seen": 1700593664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895281656234954, + "loss": 1.4119, + "theoretical_loss": 3.506290109397146, + "tokens_seen": 1700724736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894880436527042, + "loss": 1.274, + "theoretical_loss": 3.506267044066538, + "tokens_seen": 1700855808 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489447921681913, + "loss": 1.2711, + "theoretical_loss": 3.5062439810109796, + "tokens_seen": 1700986880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894077997111218, + "loss": 1.4431, + "theoretical_loss": 3.5062209202300694, + "tokens_seen": 1701117952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893676777403306, + "loss": 1.3569, + "theoretical_loss": 3.5061978617234084, + "tokens_seen": 1701249024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893275557695395, + "loss": 1.3508, + "theoretical_loss": 3.5061748054905975, + "tokens_seen": 1701380096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892874337987481, + "loss": 1.2837, + "theoretical_loss": 3.506151751531237, + "tokens_seen": 1701511168 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489247311827957, + "loss": 1.3407, + "theoretical_loss": 3.5061286998449273, + "tokens_seen": 1701642240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892071898571658, + "loss": 1.2495, + "theoretical_loss": 3.50610565043127, + "tokens_seen": 1701773312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891670678863746, + "loss": 1.3157, + "theoretical_loss": 3.5060826032898653, + "tokens_seen": 1701904384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891269459155834, + "loss": 1.3888, + "theoretical_loss": 3.506059558420314, + "tokens_seen": 1702035456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890868239447922, + "loss": 1.3394, + "theoretical_loss": 3.5060365158222186, + "tokens_seen": 1702166528 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489046701974001, + "loss": 1.2828, + "theoretical_loss": 3.506013475495179, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.9905288815498352, + "objective/train/avg_token_score": 0.009471178986132145, + "objective/train/docs_used": 621885, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2826157808303833, + "objective/train/original_loss": 1.2962803840637207, + "objective/train/theoretical_loss": 3.5059904374387973, + "objective/train/tokens_used": 72887776, + "objective/train/unlikelihood_loss": 0.025106385350227356, + "theoretical_loss": 3.5059904374387973, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890065800032098, + "loss": 1.2953, + "theoretical_loss": 3.5059904374387973, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889664580324186, + "loss": 1.2458, + "theoretical_loss": 3.5059674016526747, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889263360616273, + "loss": 1.2834, + "theoretical_loss": 3.5059443681364124, + "tokens_seen": 1702690816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888862140908362, + "loss": 1.3445, + "theoretical_loss": 3.505921336889612, + "tokens_seen": 1702821888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888460921200449, + "loss": 1.3565, + "theoretical_loss": 3.5058983079118757, + "tokens_seen": 1702952960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888059701492537, + "loss": 1.3434, + "theoretical_loss": 3.505875281202805, + "tokens_seen": 1703084032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887658481784625, + "loss": 1.3251, + "theoretical_loss": 3.5058522567620023, + "tokens_seen": 1703215104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887257262076713, + "loss": 1.3194, + "theoretical_loss": 3.5058292345890694, + "tokens_seen": 1703346176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886856042368801, + "loss": 1.2759, + "theoretical_loss": 3.505806214683608, + "tokens_seen": 1703477248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886454822660889, + "loss": 1.4206, + "theoretical_loss": 3.5057831970452207, + "tokens_seen": 1703608320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886053602952977, + "loss": 1.382, + "theoretical_loss": 3.5057601816735096, + "tokens_seen": 1703739392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885652383245064, + "loss": 1.2645, + "theoretical_loss": 3.5057371685680776, + "tokens_seen": 1703870464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885251163537154, + "loss": 1.2864, + "theoretical_loss": 3.5057141577285273, + "tokens_seen": 1704001536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884849943829241, + "loss": 1.2759, + "theoretical_loss": 3.5056911491544605, + "tokens_seen": 1704132608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884448724121329, + "loss": 1.3474, + "theoretical_loss": 3.5056681428454803, + "tokens_seen": 1704263680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884047504413417, + "loss": 1.4136, + "theoretical_loss": 3.5056451388011904, + "tokens_seen": 1704394752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883646284705505, + "loss": 1.2914, + "theoretical_loss": 3.5056221370211924, + "tokens_seen": 1704525824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883245064997593, + "loss": 1.2986, + "theoretical_loss": 3.50559913750509, + "tokens_seen": 1704656896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882843845289681, + "loss": 1.4043, + "theoretical_loss": 3.5055761402524865, + "tokens_seen": 1704787968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882442625581769, + "loss": 1.3189, + "theoretical_loss": 3.5055531452629847, + "tokens_seen": 1704919040 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048820414058738565, + "loss": 1.3539, + "theoretical_loss": 3.5055301525361884, + "tokens_seen": 1705050112 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048816401861659446, + "loss": 1.2973, + "theoretical_loss": 3.5055071620717007, + "tokens_seen": 1705181184 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048812389664580326, + "loss": 1.3955, + "theoretical_loss": 3.505484173869125, + "tokens_seen": 1705312256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488083774675012, + "loss": 1.3217, + "theoretical_loss": 3.505461187928065, + "tokens_seen": 1705443328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880436527042209, + "loss": 1.3095, + "theoretical_loss": 3.505438204248125, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880035307334296, + "loss": 1.3086, + "theoretical_loss": 3.5054152228289084, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048796340876263843, + "loss": 1.3322, + "theoretical_loss": 3.505392243670019, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048792328679184724, + "loss": 1.376, + "theoretical_loss": 3.5053692667710608, + "tokens_seen": 1705967616 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048788316482105605, + "loss": 1.3229, + "theoretical_loss": 3.5053462921316383, + "tokens_seen": 1706098688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878430428502648, + "loss": 1.2845, + "theoretical_loss": 3.505323319751356, + "tokens_seen": 1706229760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878029208794736, + "loss": 1.3275, + "theoretical_loss": 3.505300349629817, + "tokens_seen": 1706360832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877627989086824, + "loss": 1.22, + "theoretical_loss": 3.5052773817666267, + "tokens_seen": 1706491904 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048772267693789116, + "loss": 1.3484, + "theoretical_loss": 3.5052544161613897, + "tokens_seen": 1706622976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876825549671, + "loss": 1.3772, + "theoretical_loss": 3.5052314528137103, + "tokens_seen": 1706754048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876424329963088, + "loss": 1.3779, + "theoretical_loss": 3.5052084917231934, + "tokens_seen": 1706885120 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048760231102551764, + "loss": 1.3398, + "theoretical_loss": 3.5051855328894437, + "tokens_seen": 1707016192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875621890547264, + "loss": 1.2427, + "theoretical_loss": 3.5051625763120664, + "tokens_seen": 1707147264 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048752206708393514, + "loss": 1.2631, + "theoretical_loss": 3.505139621990666, + "tokens_seen": 1707278336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487481945113144, + "loss": 1.3114, + "theoretical_loss": 3.505116669924848, + "tokens_seen": 1707409408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048744182314235275, + "loss": 1.3229, + "theoretical_loss": 3.5050937201142176, + "tokens_seen": 1707540480 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048740170117156156, + "loss": 1.3658, + "theoretical_loss": 3.5050707725583803, + "tokens_seen": 1707671552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048736157920077036, + "loss": 1.2521, + "theoretical_loss": 3.5050478272569414, + "tokens_seen": 1707802624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048732145722997917, + "loss": 1.332, + "theoretical_loss": 3.505024884209506, + "tokens_seen": 1707933696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872813352591879, + "loss": 1.2462, + "theoretical_loss": 3.50500194341568, + "tokens_seen": 1708064768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048724121328839673, + "loss": 1.2669, + "theoretical_loss": 3.5049790048750697, + "tokens_seen": 1708195840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048720109131760554, + "loss": 1.3511, + "theoretical_loss": 3.50495606858728, + "tokens_seen": 1708326912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871609693468143, + "loss": 1.3513, + "theoretical_loss": 3.5049331345519175, + "tokens_seen": 1708457984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048712084737602315, + "loss": 1.2544, + "theoretical_loss": 3.504910202768588, + "tokens_seen": 1708589056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870807254052319, + "loss": 1.295, + "theoretical_loss": 3.5048872732368976, + "tokens_seen": 1708720128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870406034344407, + "loss": 1.3038, + "theoretical_loss": 3.5048643459564524, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9949111342430115, + "objective/train/avg_token_score": 0.005088924430310726, + "objective/train/docs_used": 624223, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.353865623474121, + "objective/train/original_loss": 1.3613947629928589, + "objective/train/theoretical_loss": 3.5048414209268586, + "objective/train/tokens_used": 79441376, + "objective/train/unlikelihood_loss": 0.012973498553037643, + "theoretical_loss": 3.5048414209268586, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870004814636495, + "loss": 1.2921, + "theoretical_loss": 3.5048414209268586, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048696035949285827, + "loss": 1.2469, + "theoretical_loss": 3.5048184981477233, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869202375220671, + "loss": 1.2829, + "theoretical_loss": 3.504795577618653, + "tokens_seen": 1709244416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868801155512759, + "loss": 1.2893, + "theoretical_loss": 3.5047726593392534, + "tokens_seen": 1709375488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868399935804847, + "loss": 1.309, + "theoretical_loss": 3.5047497433091315, + "tokens_seen": 1709506560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867998716096935, + "loss": 1.2582, + "theoretical_loss": 3.5047268295278946, + "tokens_seen": 1709637632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867597496389023, + "loss": 1.3368, + "theoretical_loss": 3.5047039179951494, + "tokens_seen": 1709768704 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048671962766811105, + "loss": 1.3275, + "theoretical_loss": 3.5046810087105027, + "tokens_seen": 1709899776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048667950569731986, + "loss": 1.3368, + "theoretical_loss": 3.504658101673562, + "tokens_seen": 1710030848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048663938372652866, + "loss": 1.3871, + "theoretical_loss": 3.504635196883934, + "tokens_seen": 1710161920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865992617557374, + "loss": 1.3305, + "theoretical_loss": 3.504612294341227, + "tokens_seen": 1710292992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865591397849463, + "loss": 1.3595, + "theoretical_loss": 3.5045893940450465, + "tokens_seen": 1710424064 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486519017814155, + "loss": 1.353, + "theoretical_loss": 3.504566495995002, + "tokens_seen": 1710555136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048647889584336383, + "loss": 1.3522, + "theoretical_loss": 3.5045436001907007, + "tokens_seen": 1710686208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048643877387257264, + "loss": 1.3513, + "theoretical_loss": 3.5045207066317494, + "tokens_seen": 1710817280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048639865190178145, + "loss": 1.2866, + "theoretical_loss": 3.5044978153177566, + "tokens_seen": 1710948352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863585299309902, + "loss": 1.348, + "theoretical_loss": 3.5044749262483297, + "tokens_seen": 1711079424 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486318407960199, + "loss": 1.2716, + "theoretical_loss": 3.504452039423078, + "tokens_seen": 1711210496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862782859894078, + "loss": 1.2578, + "theoretical_loss": 3.5044291548416076, + "tokens_seen": 1711341568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862381640186166, + "loss": 1.3189, + "theoretical_loss": 3.504406272503528, + "tokens_seen": 1711472640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861980420478254, + "loss": 1.2867, + "theoretical_loss": 3.5043833924084473, + "tokens_seen": 1711603712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861579200770342, + "loss": 1.3786, + "theoretical_loss": 3.504360514555974, + "tokens_seen": 1711734784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048611779810624304, + "loss": 1.3411, + "theoretical_loss": 3.5043376389457164, + "tokens_seen": 1711865856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860776761354518, + "loss": 1.3734, + "theoretical_loss": 3.504314765577283, + "tokens_seen": 1711996928 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048603755416466054, + "loss": 1.3183, + "theoretical_loss": 3.5042918944502826, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859974321938694, + "loss": 1.3157, + "theoretical_loss": 3.504269025564324, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048595731022307815, + "loss": 1.2968, + "theoretical_loss": 3.504246158919016, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048591718825228696, + "loss": 1.2926, + "theoretical_loss": 3.5042232945139675, + "tokens_seen": 1712521216 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048587706628149576, + "loss": 1.2532, + "theoretical_loss": 3.504200432348788, + "tokens_seen": 1712652288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048583694431070457, + "loss": 1.3013, + "theoretical_loss": 3.504177572423086, + "tokens_seen": 1712783360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857968223399133, + "loss": 1.2876, + "theoretical_loss": 3.504154714736472, + "tokens_seen": 1712914432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048575670036912213, + "loss": 1.2628, + "theoretical_loss": 3.504131859288554, + "tokens_seen": 1713045504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048571657839833094, + "loss": 1.3095, + "theoretical_loss": 3.5041090060789424, + "tokens_seen": 1713176576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856764564275397, + "loss": 1.3621, + "theoretical_loss": 3.504086155107246, + "tokens_seen": 1713307648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563633445674855, + "loss": 1.3425, + "theoretical_loss": 3.504063306373075, + "tokens_seen": 1713438720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855962124859573, + "loss": 1.2687, + "theoretical_loss": 3.5040404598760393, + "tokens_seen": 1713569792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048555609051516616, + "loss": 1.3156, + "theoretical_loss": 3.504017615615748, + "tokens_seen": 1713700864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855159685443749, + "loss": 1.3428, + "theoretical_loss": 3.5039947735918124, + "tokens_seen": 1713831936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048547584657358366, + "loss": 1.3777, + "theoretical_loss": 3.5039719338038413, + "tokens_seen": 1713963008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854357246027925, + "loss": 1.3785, + "theoretical_loss": 3.503949096251445, + "tokens_seen": 1714094080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853956026320013, + "loss": 1.2862, + "theoretical_loss": 3.503926260934234, + "tokens_seen": 1714225152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853554806612101, + "loss": 1.3242, + "theoretical_loss": 3.503903427851819, + "tokens_seen": 1714356224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853153586904189, + "loss": 1.2978, + "theoretical_loss": 3.5038805970038105, + "tokens_seen": 1714487296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852752367196277, + "loss": 1.3363, + "theoretical_loss": 3.5038577683898184, + "tokens_seen": 1714618368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048523511474883645, + "loss": 1.3543, + "theoretical_loss": 3.503834942009454, + "tokens_seen": 1714749440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048519499277804525, + "loss": 1.4012, + "theoretical_loss": 3.5038121178623274, + "tokens_seen": 1714880512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048515487080725406, + "loss": 1.2589, + "theoretical_loss": 3.5037892959480494, + "tokens_seen": 1715011584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851147488364628, + "loss": 1.3258, + "theoretical_loss": 3.503766476266232, + "tokens_seen": 1715142656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850746268656717, + "loss": 1.3467, + "theoretical_loss": 3.503743658816485, + "tokens_seen": 1715273728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850345048948804, + "loss": 1.3974, + "theoretical_loss": 3.5037208435984204, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9915130734443665, + "objective/train/avg_token_score": 0.008486890234053135, + "objective/train/docs_used": 626732, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2568973302841187, + "objective/train/original_loss": 1.2689979076385498, + "objective/train/theoretical_loss": 3.503698030611649, + "objective/train/tokens_used": 85994976, + "objective/train/unlikelihood_loss": 0.022805046290159225, + "theoretical_loss": 3.503698030611649, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849943829240893, + "loss": 1.308, + "theoretical_loss": 3.503698030611649, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048495426095329804, + "loss": 1.3066, + "theoretical_loss": 3.503675219855782, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048491413898250684, + "loss": 1.3611, + "theoretical_loss": 3.5036524113304317, + "tokens_seen": 1715798016 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048487401701171565, + "loss": 1.3885, + "theoretical_loss": 3.5036296050352087, + "tokens_seen": 1715929088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848338950409244, + "loss": 1.3473, + "theoretical_loss": 3.5036068009697248, + "tokens_seen": 1716060160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847937730701332, + "loss": 1.4253, + "theoretical_loss": 3.5035839991335918, + "tokens_seen": 1716191232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484753651099342, + "loss": 1.3111, + "theoretical_loss": 3.5035611995264215, + "tokens_seen": 1716322304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847135291285508, + "loss": 1.3213, + "theoretical_loss": 3.503538402147826, + "tokens_seen": 1716453376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846734071577596, + "loss": 1.3322, + "theoretical_loss": 3.503515606997417, + "tokens_seen": 1716584448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048463328518696843, + "loss": 1.369, + "theoretical_loss": 3.5034928140748067, + "tokens_seen": 1716715520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845931632161772, + "loss": 1.3173, + "theoretical_loss": 3.5034700233796077, + "tokens_seen": 1716846592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048455304124538594, + "loss": 1.3861, + "theoretical_loss": 3.5034472349114316, + "tokens_seen": 1716977664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845129192745948, + "loss": 1.3305, + "theoretical_loss": 3.5034244486698913, + "tokens_seen": 1717108736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048447279730380355, + "loss": 1.3419, + "theoretical_loss": 3.5034016646545996, + "tokens_seen": 1717239808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048443267533301236, + "loss": 1.3848, + "theoretical_loss": 3.503378882865168, + "tokens_seen": 1717370880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048439255336222116, + "loss": 1.2766, + "theoretical_loss": 3.5033561033012104, + "tokens_seen": 1717501952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048435243139142997, + "loss": 1.3318, + "theoretical_loss": 3.5033333259623394, + "tokens_seen": 1717633024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843123094206388, + "loss": 1.3218, + "theoretical_loss": 3.503310550848167, + "tokens_seen": 1717764096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048427218744984753, + "loss": 1.3946, + "theoretical_loss": 3.503287777958307, + "tokens_seen": 1717895168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048423206547905634, + "loss": 1.3216, + "theoretical_loss": 3.503265007292372, + "tokens_seen": 1718026240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048419194350826514, + "loss": 1.2791, + "theoretical_loss": 3.5032422388499755, + "tokens_seen": 1718157312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048415182153747395, + "loss": 1.352, + "theoretical_loss": 3.5032194726307306, + "tokens_seen": 1718288384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841116995666827, + "loss": 1.2666, + "theoretical_loss": 3.5031967086342504, + "tokens_seen": 1718419456 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048407157759589156, + "loss": 1.3448, + "theoretical_loss": 3.503173946860149, + "tokens_seen": 1718550528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840314556251003, + "loss": 1.2976, + "theoretical_loss": 3.5031511873080396, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048399133365430906, + "loss": 1.3592, + "theoretical_loss": 3.503128429977536, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839512116835179, + "loss": 1.3917, + "theoretical_loss": 3.5031056748682516, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839110897127267, + "loss": 1.2916, + "theoretical_loss": 3.503082921979801, + "tokens_seen": 1719074816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838709677419355, + "loss": 1.2445, + "theoretical_loss": 3.5030601713117973, + "tokens_seen": 1719205888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838308457711443, + "loss": 1.3251, + "theoretical_loss": 3.503037422863855, + "tokens_seen": 1719336960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837907238003531, + "loss": 1.3803, + "theoretical_loss": 3.5030146766355887, + "tokens_seen": 1719468032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837506018295619, + "loss": 1.3782, + "theoretical_loss": 3.5029919326266112, + "tokens_seen": 1719599104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048371047985877065, + "loss": 1.3135, + "theoretical_loss": 3.502969190836538, + "tokens_seen": 1719730176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048367035788797946, + "loss": 1.4054, + "theoretical_loss": 3.5029464512649833, + "tokens_seen": 1719861248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048363023591718827, + "loss": 1.3224, + "theoretical_loss": 3.502923713911562, + "tokens_seen": 1719992320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835901139463971, + "loss": 1.3594, + "theoretical_loss": 3.502900978775888, + "tokens_seen": 1720123392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835499919756058, + "loss": 1.3035, + "theoretical_loss": 3.502878245857576, + "tokens_seen": 1720254464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835098700048147, + "loss": 1.3329, + "theoretical_loss": 3.5028555151562415, + "tokens_seen": 1720385536 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048346974803402344, + "loss": 1.3102, + "theoretical_loss": 3.502832786671499, + "tokens_seen": 1720516608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834296260632322, + "loss": 1.2934, + "theoretical_loss": 3.5028100604029637, + "tokens_seen": 1720647680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048338950409244105, + "loss": 1.3766, + "theoretical_loss": 3.5027873363502504, + "tokens_seen": 1720778752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833493821216498, + "loss": 1.3305, + "theoretical_loss": 3.5027646145129747, + "tokens_seen": 1720909824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833092601508586, + "loss": 1.4259, + "theoretical_loss": 3.502741894890751, + "tokens_seen": 1721040896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832691381800674, + "loss": 1.415, + "theoretical_loss": 3.502719177483196, + "tokens_seen": 1721171968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832290162092762, + "loss": 1.4475, + "theoretical_loss": 3.502696462289925, + "tokens_seen": 1721303040 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483188894238485, + "loss": 1.3343, + "theoretical_loss": 3.5026737493105524, + "tokens_seen": 1721434112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048314877226769383, + "loss": 1.2608, + "theoretical_loss": 3.5026510385446947, + "tokens_seen": 1721565184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831086502969026, + "loss": 1.3966, + "theoretical_loss": 3.5026283299919676, + "tokens_seen": 1721696256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830685283261114, + "loss": 1.2644, + "theoretical_loss": 3.5026056236519874, + "tokens_seen": 1721827328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830284063553202, + "loss": 1.3374, + "theoretical_loss": 3.502582919524369, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9925838708877563, + "objective/train/avg_token_score": 0.00741605693474412, + "objective/train/docs_used": 629016, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.32856285572052, + "objective/train/original_loss": 1.3379650115966797, + "objective/train/theoretical_loss": 3.50256021760873, + "objective/train/tokens_used": 92548576, + "objective/train/unlikelihood_loss": 0.021268518641591072, + "theoretical_loss": 3.50256021760873, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048298828438452895, + "loss": 1.3276, + "theoretical_loss": 3.50256021760873, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829481624137378, + "loss": 1.2671, + "theoretical_loss": 3.5025375179046847, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048290804044294656, + "loss": 1.3776, + "theoretical_loss": 3.502514820411851, + "tokens_seen": 1722351616 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048286791847215537, + "loss": 1.288, + "theoretical_loss": 3.5024921251298444, + "tokens_seen": 1722482688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828277965013642, + "loss": 1.4891, + "theoretical_loss": 3.502469432058281, + "tokens_seen": 1722613760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048278767453057293, + "loss": 1.3403, + "theoretical_loss": 3.5024467411967786, + "tokens_seen": 1722744832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048274755255978173, + "loss": 1.4281, + "theoretical_loss": 3.502424052544953, + "tokens_seen": 1722875904 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048270743058899054, + "loss": 1.3247, + "theoretical_loss": 3.502401366102421, + "tokens_seen": 1723006976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266730861819935, + "loss": 1.3263, + "theoretical_loss": 3.502378681868799, + "tokens_seen": 1723138048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826271866474081, + "loss": 1.3509, + "theoretical_loss": 3.5023559998437053, + "tokens_seen": 1723269120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258706467661696, + "loss": 1.337, + "theoretical_loss": 3.5023333200267555, + "tokens_seen": 1723400192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825469427058257, + "loss": 1.3212, + "theoretical_loss": 3.502310642417567, + "tokens_seen": 1723531264 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048250682073503446, + "loss": 1.3427, + "theoretical_loss": 3.5022879670157576, + "tokens_seen": 1723662336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824666987642433, + "loss": 1.3418, + "theoretical_loss": 3.502265293820944, + "tokens_seen": 1723793408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824265767934521, + "loss": 1.3475, + "theoretical_loss": 3.5022426228327443, + "tokens_seen": 1723924480 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048238645482266094, + "loss": 1.2529, + "theoretical_loss": 3.5022199540507755, + "tokens_seen": 1724055552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823463328518697, + "loss": 1.2937, + "theoretical_loss": 3.5021972874746554, + "tokens_seen": 1724186624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823062108810785, + "loss": 1.4055, + "theoretical_loss": 3.5021746231040014, + "tokens_seen": 1724317696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822660889102873, + "loss": 1.3469, + "theoretical_loss": 3.502151960938431, + "tokens_seen": 1724448768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048222596693949605, + "loss": 1.311, + "theoretical_loss": 3.502129300977563, + "tokens_seen": 1724579840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048218584496870486, + "loss": 1.3774, + "theoretical_loss": 3.5021066432210146, + "tokens_seen": 1724710912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048214572299791367, + "loss": 1.3449, + "theoretical_loss": 3.5020839876684047, + "tokens_seen": 1724841984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821056010271225, + "loss": 1.3908, + "theoretical_loss": 3.5020613343193507, + "tokens_seen": 1724973056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820654790563312, + "loss": 1.3572, + "theoretical_loss": 3.502038683173471, + "tokens_seen": 1725104128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820253570855401, + "loss": 1.3577, + "theoretical_loss": 3.502016034230384, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048198523511474884, + "loss": 1.3649, + "theoretical_loss": 3.5019933874897085, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819451131439576, + "loss": 1.3477, + "theoretical_loss": 3.5019707429510625, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048190499117316645, + "loss": 1.3186, + "theoretical_loss": 3.501948100614065, + "tokens_seen": 1725628416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818648692023752, + "loss": 1.3004, + "theoretical_loss": 3.5019254604783345, + "tokens_seen": 1725759488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048182474723158406, + "loss": 1.421, + "theoretical_loss": 3.5019028225434905, + "tokens_seen": 1725890560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817846252607928, + "loss": 1.4126, + "theoretical_loss": 3.5018801868091507, + "tokens_seen": 1726021632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817445032900016, + "loss": 1.3855, + "theoretical_loss": 3.501857553274935, + "tokens_seen": 1726152704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048170438131921043, + "loss": 1.3338, + "theoretical_loss": 3.5018349219404628, + "tokens_seen": 1726283776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048166425934841923, + "loss": 1.3513, + "theoretical_loss": 3.5018122928053526, + "tokens_seen": 1726414848 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481624137377628, + "loss": 1.2866, + "theoretical_loss": 3.5017896658692234, + "tokens_seen": 1726545920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815840154068368, + "loss": 1.4403, + "theoretical_loss": 3.501767041131696, + "tokens_seen": 1726676992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815438934360456, + "loss": 1.3092, + "theoretical_loss": 3.5017444185923887, + "tokens_seen": 1726808064 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048150377146525435, + "loss": 1.264, + "theoretical_loss": 3.5017217982509212, + "tokens_seen": 1726939136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814636494944632, + "loss": 1.3544, + "theoretical_loss": 3.5016991801069137, + "tokens_seen": 1727070208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048142352752367196, + "loss": 1.2527, + "theoretical_loss": 3.501676564159986, + "tokens_seen": 1727201280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048138340555288077, + "loss": 1.3826, + "theoretical_loss": 3.5016539504097572, + "tokens_seen": 1727332352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813432835820896, + "loss": 1.3799, + "theoretical_loss": 3.501631338855848, + "tokens_seen": 1727463424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048130316161129833, + "loss": 1.3129, + "theoretical_loss": 3.501608729497878, + "tokens_seen": 1727594496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048126303964050713, + "loss": 1.3335, + "theoretical_loss": 3.501586122335468, + "tokens_seen": 1727725568 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048122291766971594, + "loss": 1.3275, + "theoretical_loss": 3.5015635173682376, + "tokens_seen": 1727856640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048118279569892475, + "loss": 1.3803, + "theoretical_loss": 3.5015409145958074, + "tokens_seen": 1727987712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048114267372813355, + "loss": 1.3556, + "theoretical_loss": 3.5015183140177975, + "tokens_seen": 1728118784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048110255175734236, + "loss": 1.3165, + "theoretical_loss": 3.5014957156338293, + "tokens_seen": 1728249856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810624297865511, + "loss": 1.2862, + "theoretical_loss": 3.5014731194435225, + "tokens_seen": 1728380928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810223078157599, + "loss": 1.308, + "theoretical_loss": 3.5014505254464985, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9941221475601196, + "objective/train/avg_token_score": 0.005877892952412367, + "objective/train/docs_used": 631392, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3028059005737305, + "objective/train/original_loss": 1.3112616539001465, + "objective/train/theoretical_loss": 3.5014279336423777, + "objective/train/tokens_used": 99102176, + "objective/train/unlikelihood_loss": 0.01588469184935093, + "theoretical_loss": 3.5014279336423777, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809821858449687, + "loss": 1.3188, + "theoretical_loss": 3.5014279336423777, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809420638741775, + "loss": 1.3706, + "theoretical_loss": 3.501405344030781, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048090194190338634, + "loss": 1.3487, + "theoretical_loss": 3.5013827566113296, + "tokens_seen": 1728905216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808618199325951, + "loss": 1.2725, + "theoretical_loss": 3.501360171383645, + "tokens_seen": 1729036288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808216979618039, + "loss": 1.2803, + "theoretical_loss": 3.5013375883473477, + "tokens_seen": 1729167360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807815759910127, + "loss": 1.3557, + "theoretical_loss": 3.5013150075020594, + "tokens_seen": 1729298432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048074145402022145, + "loss": 1.3678, + "theoretical_loss": 3.5012924288474006, + "tokens_seen": 1729429504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048070133204943026, + "loss": 1.3052, + "theoretical_loss": 3.501269852382994, + "tokens_seen": 1729560576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048066121007863907, + "loss": 1.3985, + "theoretical_loss": 3.501247278108461, + "tokens_seen": 1729691648 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048062108810784787, + "loss": 1.3582, + "theoretical_loss": 3.5012247060234225, + "tokens_seen": 1729822720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805809661370567, + "loss": 1.2904, + "theoretical_loss": 3.501202136127501, + "tokens_seen": 1729953792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805408441662655, + "loss": 1.311, + "theoretical_loss": 3.501179568420318, + "tokens_seen": 1730084864 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048050072219547424, + "loss": 1.4398, + "theoretical_loss": 3.501157002901495, + "tokens_seen": 1730215936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048046060022468304, + "loss": 1.2954, + "theoretical_loss": 3.5011344395706554, + "tokens_seen": 1730347008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042047825389185, + "loss": 1.3054, + "theoretical_loss": 3.5011118784274204, + "tokens_seen": 1730478080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803803562831006, + "loss": 1.3077, + "theoretical_loss": 3.501089319471412, + "tokens_seen": 1730609152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034023431230946, + "loss": 1.3656, + "theoretical_loss": 3.501066762702253, + "tokens_seen": 1730740224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803001123415182, + "loss": 1.2645, + "theoretical_loss": 3.5010442081195654, + "tokens_seen": 1730871296 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480259990370727, + "loss": 1.2921, + "theoretical_loss": 3.501021655722972, + "tokens_seen": 1731002368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048021986839993583, + "loss": 1.3498, + "theoretical_loss": 3.5009991055120957, + "tokens_seen": 1731133440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048017974642914463, + "loss": 1.4088, + "theoretical_loss": 3.500976557486559, + "tokens_seen": 1731264512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801396244583534, + "loss": 1.3867, + "theoretical_loss": 3.5009540116459843, + "tokens_seen": 1731395584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800995024875622, + "loss": 1.2665, + "theoretical_loss": 3.5009314679899948, + "tokens_seen": 1731526656 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480059380516771, + "loss": 1.2602, + "theoretical_loss": 3.500908926518213, + "tokens_seen": 1731657728 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048001925854597975, + "loss": 1.3227, + "theoretical_loss": 3.5008863872302634, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799791365751886, + "loss": 1.3607, + "theoretical_loss": 3.500863850125768, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047993901460439736, + "loss": 1.3846, + "theoretical_loss": 3.5008413152043496, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798988926336062, + "loss": 1.3167, + "theoretical_loss": 3.5008187824656325, + "tokens_seen": 1732182016 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479858770662815, + "loss": 1.365, + "theoretical_loss": 3.50079625190924, + "tokens_seen": 1732313088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047981864869202373, + "loss": 1.398, + "theoretical_loss": 3.5007737235347953, + "tokens_seen": 1732444160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797785267212326, + "loss": 1.2531, + "theoretical_loss": 3.5007511973419225, + "tokens_seen": 1732575232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047973840475044134, + "loss": 1.3066, + "theoretical_loss": 3.5007286733302445, + "tokens_seen": 1732706304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047969828277965015, + "loss": 1.3471, + "theoretical_loss": 3.5007061514993865, + "tokens_seen": 1732837376 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047965816080885895, + "loss": 1.3732, + "theoretical_loss": 3.500683631848971, + "tokens_seen": 1732968448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047961803883806776, + "loss": 1.3898, + "theoretical_loss": 3.500661114378623, + "tokens_seen": 1733099520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795779168672765, + "loss": 1.3534, + "theoretical_loss": 3.5006385990879663, + "tokens_seen": 1733230592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795377948964853, + "loss": 1.3584, + "theoretical_loss": 3.5006160859766244, + "tokens_seen": 1733361664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794976729256941, + "loss": 1.3743, + "theoretical_loss": 3.5005935750442223, + "tokens_seen": 1733492736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794575509549029, + "loss": 1.3487, + "theoretical_loss": 3.5005710662903846, + "tokens_seen": 1733623808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047941742898411174, + "loss": 1.3899, + "theoretical_loss": 3.500548559714735, + "tokens_seen": 1733754880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793773070133205, + "loss": 1.316, + "theoretical_loss": 3.5005260553168993, + "tokens_seen": 1733885952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047933718504252935, + "loss": 1.3638, + "theoretical_loss": 3.500503553096501, + "tokens_seen": 1734017024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792970630717381, + "loss": 1.3735, + "theoretical_loss": 3.500481053053165, + "tokens_seen": 1734148096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047925694110094685, + "loss": 1.3058, + "theoretical_loss": 3.500458555186517, + "tokens_seen": 1734279168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792168191301557, + "loss": 1.3281, + "theoretical_loss": 3.500436059496181, + "tokens_seen": 1734410240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047917669715936447, + "loss": 1.3146, + "theoretical_loss": 3.5004135659817823, + "tokens_seen": 1734541312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913657518857327, + "loss": 1.3592, + "theoretical_loss": 3.5003910746429456, + "tokens_seen": 1734672384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790964532177821, + "loss": 1.4318, + "theoretical_loss": 3.500368585479297, + "tokens_seen": 1734803456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790563312469909, + "loss": 1.365, + "theoretical_loss": 3.5003460984904615, + "tokens_seen": 1734934528 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047901620927619964, + "loss": 1.3266, + "theoretical_loss": 3.5003236136760645, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9936129450798035, + "objective/train/avg_token_score": 0.006387016270309687, + "objective/train/docs_used": 633815, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3701329231262207, + "objective/train/original_loss": 1.3798182010650635, + "objective/train/theoretical_loss": 3.500301131035731, + "objective/train/tokens_used": 105655776, + "objective/train/unlikelihood_loss": 0.016626279801130295, + "theoretical_loss": 3.500301131035731, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047897608730540844, + "loss": 1.3595, + "theoretical_loss": 3.500301131035731, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047893596533461725, + "loss": 1.3447, + "theoretical_loss": 3.5002786505690873, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478895843363826, + "loss": 1.2931, + "theoretical_loss": 3.5002561722757592, + "tokens_seen": 1735458816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047885572139303486, + "loss": 1.2968, + "theoretical_loss": 3.5002336961553713, + "tokens_seen": 1735589888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788155994222436, + "loss": 1.393, + "theoretical_loss": 3.5002112222075508, + "tokens_seen": 1735720960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787754774514524, + "loss": 1.2615, + "theoretical_loss": 3.500188750431923, + "tokens_seen": 1735852032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787353554806612, + "loss": 1.3327, + "theoretical_loss": 3.5001662808281138, + "tokens_seen": 1735983104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047869523350987, + "loss": 1.3513, + "theoretical_loss": 3.5001438133957494, + "tokens_seen": 1736114176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047865511153907884, + "loss": 1.2997, + "theoretical_loss": 3.500121348134457, + "tokens_seen": 1736245248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786149895682876, + "loss": 1.431, + "theoretical_loss": 3.5000988850438617, + "tokens_seen": 1736376320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785748675974964, + "loss": 1.3476, + "theoretical_loss": 3.5000764241235904, + "tokens_seen": 1736507392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785347456267052, + "loss": 1.3607, + "theoretical_loss": 3.5000539653732696, + "tokens_seen": 1736638464 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478494623655914, + "loss": 1.3892, + "theoretical_loss": 3.5000315087925253, + "tokens_seen": 1736769536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047845450168512276, + "loss": 1.2743, + "theoretical_loss": 3.5000090543809854, + "tokens_seen": 1736900608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784143797143316, + "loss": 1.3361, + "theoretical_loss": 3.4999866021382764, + "tokens_seen": 1737031680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783742577435404, + "loss": 1.4006, + "theoretical_loss": 3.499964152064025, + "tokens_seen": 1737162752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047833413577274913, + "loss": 1.3458, + "theoretical_loss": 3.499941704157857, + "tokens_seen": 1737293824 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478294013801958, + "loss": 1.3808, + "theoretical_loss": 3.499919258419401, + "tokens_seen": 1737424896 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047825389183116674, + "loss": 1.3007, + "theoretical_loss": 3.4998968148482845, + "tokens_seen": 1737555968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047821376986037555, + "loss": 1.3808, + "theoretical_loss": 3.499874373444133, + "tokens_seen": 1737687040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047817364788958435, + "loss": 1.2832, + "theoretical_loss": 3.4998519342065753, + "tokens_seen": 1737818112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047813352591879316, + "loss": 1.3531, + "theoretical_loss": 3.499829497135238, + "tokens_seen": 1737949184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780934039480019, + "loss": 1.2468, + "theoretical_loss": 3.499807062229749, + "tokens_seen": 1738080256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780532819772107, + "loss": 1.4828, + "theoretical_loss": 3.4997846294897363, + "tokens_seen": 1738211328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780131600064195, + "loss": 1.3196, + "theoretical_loss": 3.4997621989148273, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047797303803562833, + "loss": 1.3259, + "theoretical_loss": 3.4997397705046493, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047793291606483714, + "loss": 1.3553, + "theoretical_loss": 3.4997173442588307, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778927940940459, + "loss": 1.2703, + "theoretical_loss": 3.4996949201769993, + "tokens_seen": 1738735616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047785267212325475, + "loss": 1.3321, + "theoretical_loss": 3.4996724982587835, + "tokens_seen": 1738866688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778125501524635, + "loss": 1.3387, + "theoretical_loss": 3.4996500785038114, + "tokens_seen": 1738997760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047777242818167225, + "loss": 1.3977, + "theoretical_loss": 3.4996276609117105, + "tokens_seen": 1739128832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777323062108811, + "loss": 1.307, + "theoretical_loss": 3.4996052454821105, + "tokens_seen": 1739259904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047769218424008987, + "loss": 1.3231, + "theoretical_loss": 3.4995828322146387, + "tokens_seen": 1739390976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047765206226929867, + "loss": 1.3635, + "theoretical_loss": 3.4995604211089244, + "tokens_seen": 1739522048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776119402985075, + "loss": 1.2777, + "theoretical_loss": 3.499538012164596, + "tokens_seen": 1739653120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775718183277163, + "loss": 1.3704, + "theoretical_loss": 3.4995156053812817, + "tokens_seen": 1739784192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753169635692504, + "loss": 1.3189, + "theoretical_loss": 3.499493200758611, + "tokens_seen": 1739915264 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047749157438613384, + "loss": 1.3675, + "theoretical_loss": 3.4994707982962128, + "tokens_seen": 1740046336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745145241534265, + "loss": 1.4119, + "theoretical_loss": 3.4994483979937154, + "tokens_seen": 1740177408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774113304445514, + "loss": 1.274, + "theoretical_loss": 3.499425999850749, + "tokens_seen": 1740308480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047737120847376026, + "loss": 1.3407, + "theoretical_loss": 3.499403603866942, + "tokens_seen": 1740439552 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477331086502969, + "loss": 1.3359, + "theoretical_loss": 3.4993812100419235, + "tokens_seen": 1740570624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004772909645321779, + "loss": 1.3306, + "theoretical_loss": 3.4993588183753235, + "tokens_seen": 1740701696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772508425613866, + "loss": 1.4142, + "theoretical_loss": 3.499336428866771, + "tokens_seen": 1740832768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772107205905954, + "loss": 1.3641, + "theoretical_loss": 3.4993140415158956, + "tokens_seen": 1740963840 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047717059861980424, + "loss": 1.3342, + "theoretical_loss": 3.4992916563223275, + "tokens_seen": 1741094912 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477130476649013, + "loss": 1.3212, + "theoretical_loss": 3.499269273285696, + "tokens_seen": 1741225984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770903546782218, + "loss": 1.2883, + "theoretical_loss": 3.4992468924056306, + "tokens_seen": 1741357056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770502327074306, + "loss": 1.3165, + "theoretical_loss": 3.499224513681762, + "tokens_seen": 1741488128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770101107366394, + "loss": 1.3332, + "theoretical_loss": 3.499202137113719, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9916802048683167, + "objective/train/avg_token_score": 0.008319806307554245, + "objective/train/docs_used": 636395, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2986737489700317, + "objective/train/original_loss": 1.3091821670532227, + "objective/train/theoretical_loss": 3.499179762701133, + "objective/train/tokens_used": 112209376, + "objective/train/unlikelihood_loss": 0.02823907695710659, + "theoretical_loss": 3.499179762701133, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047696998876584816, + "loss": 1.3375, + "theoretical_loss": 3.499179762701133, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476929866795057, + "loss": 1.2789, + "theoretical_loss": 3.4991573904436333, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768897448242658, + "loss": 1.3439, + "theoretical_loss": 3.4991350203408507, + "tokens_seen": 1742012416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768496228534745, + "loss": 1.3524, + "theoretical_loss": 3.499112652392416, + "tokens_seen": 1742143488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768095008826834, + "loss": 1.2909, + "theoretical_loss": 3.4990902865979585, + "tokens_seen": 1742274560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047676937891189214, + "loss": 1.3029, + "theoretical_loss": 3.499067922957109, + "tokens_seen": 1742405632 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476729256941101, + "loss": 1.3774, + "theoretical_loss": 3.499045561469499, + "tokens_seen": 1742536704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047668913497030975, + "loss": 1.304, + "theoretical_loss": 3.4990232021347585, + "tokens_seen": 1742667776 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047664901299951856, + "loss": 1.3728, + "theoretical_loss": 3.4990008449525183, + "tokens_seen": 1742798848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047660889102872736, + "loss": 1.3357, + "theoretical_loss": 3.49897848992241, + "tokens_seen": 1742929920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765687690579361, + "loss": 1.3676, + "theoretical_loss": 3.498956137044064, + "tokens_seen": 1743060992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765286470871449, + "loss": 1.3915, + "theoretical_loss": 3.4989337863171115, + "tokens_seen": 1743192064 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047648852511635373, + "loss": 1.3831, + "theoretical_loss": 3.498911437741184, + "tokens_seen": 1743323136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047644840314556254, + "loss": 1.4132, + "theoretical_loss": 3.4988890913159123, + "tokens_seen": 1743454208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764082811747713, + "loss": 1.357, + "theoretical_loss": 3.4988667470409283, + "tokens_seen": 1743585280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047636815920398015, + "loss": 1.375, + "theoretical_loss": 3.4988444049158627, + "tokens_seen": 1743716352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763280372331889, + "loss": 1.2958, + "theoretical_loss": 3.498822064940348, + "tokens_seen": 1743847424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047628791526239765, + "loss": 1.3729, + "theoretical_loss": 3.498799727114015, + "tokens_seen": 1743978496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762477932916065, + "loss": 1.3649, + "theoretical_loss": 3.498777391436496, + "tokens_seen": 1744109568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047620767132081527, + "loss": 1.3464, + "theoretical_loss": 3.498755057907423, + "tokens_seen": 1744240640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047616754935002407, + "loss": 1.3461, + "theoretical_loss": 3.498732726526427, + "tokens_seen": 1744371712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761274273792329, + "loss": 1.2978, + "theoretical_loss": 3.498710397293141, + "tokens_seen": 1744502784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760873054084417, + "loss": 1.3494, + "theoretical_loss": 3.498688070207197, + "tokens_seen": 1744633856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760471834376505, + "loss": 1.402, + "theoretical_loss": 3.498665745268226, + "tokens_seen": 1744764928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047600706146685924, + "loss": 1.3687, + "theoretical_loss": 3.498643422475862, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047596693949606805, + "loss": 1.2276, + "theoretical_loss": 3.4986211018297357, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047592681752527686, + "loss": 1.2823, + "theoretical_loss": 3.4985987833294816, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047588669555448566, + "loss": 1.3448, + "theoretical_loss": 3.49857646697473, + "tokens_seen": 1745289216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758465735836944, + "loss": 1.333, + "theoretical_loss": 3.498554152765115, + "tokens_seen": 1745420288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758064516129033, + "loss": 1.3621, + "theoretical_loss": 3.4985318407002692, + "tokens_seen": 1745551360 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475766329642112, + "loss": 1.3441, + "theoretical_loss": 3.498509530779825, + "tokens_seen": 1745682432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757262076713208, + "loss": 1.3723, + "theoretical_loss": 3.498487223003415, + "tokens_seen": 1745813504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047568608570052964, + "loss": 1.3246, + "theoretical_loss": 3.498464917370673, + "tokens_seen": 1745944576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756459637297384, + "loss": 1.4134, + "theoretical_loss": 3.498442613881232, + "tokens_seen": 1746075648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756058417589472, + "loss": 1.3682, + "theoretical_loss": 3.498420312534725, + "tokens_seen": 1746206720 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475565719788156, + "loss": 1.2983, + "theoretical_loss": 3.4983980133307853, + "tokens_seen": 1746337792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755255978173648, + "loss": 1.292, + "theoretical_loss": 3.498375716269046, + "tokens_seen": 1746468864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754854758465736, + "loss": 1.3229, + "theoretical_loss": 3.4983534213491407, + "tokens_seen": 1746599936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754453538757824, + "loss": 1.3657, + "theoretical_loss": 3.4983311285707033, + "tokens_seen": 1746731008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754052319049912, + "loss": 1.2755, + "theoretical_loss": 3.498308837933367, + "tokens_seen": 1746862080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753651099342, + "loss": 1.3426, + "theoretical_loss": 3.4982865494367656, + "tokens_seen": 1746993152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753249879634088, + "loss": 1.3128, + "theoretical_loss": 3.498264263080533, + "tokens_seen": 1747124224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047528486599261754, + "loss": 1.4022, + "theoretical_loss": 3.4982419788643035, + "tokens_seen": 1747255296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752447440218264, + "loss": 1.4244, + "theoretical_loss": 3.498219696787711, + "tokens_seen": 1747386368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047520462205103515, + "loss": 1.3246, + "theoretical_loss": 3.4981974168503887, + "tokens_seen": 1747517440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047516450008024396, + "loss": 1.3273, + "theoretical_loss": 3.4981751390519715, + "tokens_seen": 1747648512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047512437810945276, + "loss": 1.3516, + "theoretical_loss": 3.498152863392094, + "tokens_seen": 1747779584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750842561386615, + "loss": 1.3191, + "theoretical_loss": 3.4981305898703896, + "tokens_seen": 1747910656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750441341678703, + "loss": 1.4029, + "theoretical_loss": 3.4981083184864934, + "tokens_seen": 1748041728 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047500401219707913, + "loss": 1.3437, + "theoretical_loss": 3.49808604924004, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9924666285514832, + "objective/train/avg_token_score": 0.007533431053161621, + "objective/train/docs_used": 638722, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2412388324737549, + "objective/train/original_loss": 1.2503128051757812, + "objective/train/theoretical_loss": 3.4980637821306635, + "objective/train/tokens_used": 118762976, + "objective/train/unlikelihood_loss": 0.02628133073449135, + "theoretical_loss": 3.4980637821306635, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047496389022628794, + "loss": 1.2859, + "theoretical_loss": 3.4980637821306635, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749237682554967, + "loss": 1.3187, + "theoretical_loss": 3.498041517157999, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047488364628470555, + "loss": 1.3049, + "theoretical_loss": 3.4980192543216813, + "tokens_seen": 1748566016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748435243139143, + "loss": 1.2765, + "theoretical_loss": 3.4979969936213458, + "tokens_seen": 1748697088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748034023431231, + "loss": 1.3461, + "theoretical_loss": 3.4979747350566264, + "tokens_seen": 1748828160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747632803723319, + "loss": 1.3348, + "theoretical_loss": 3.497952478627159, + "tokens_seen": 1748959232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047472315840154066, + "loss": 1.3779, + "theoretical_loss": 3.4979302243325785, + "tokens_seen": 1749090304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746830364307495, + "loss": 1.3508, + "theoretical_loss": 3.49790797217252, + "tokens_seen": 1749221376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746429144599583, + "loss": 1.3961, + "theoretical_loss": 3.497885722146619, + "tokens_seen": 1749352448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746027924891671, + "loss": 1.2881, + "theoretical_loss": 3.497863474254511, + "tokens_seen": 1749483520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745626705183759, + "loss": 1.351, + "theoretical_loss": 3.4978412284958313, + "tokens_seen": 1749614592 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047452254854758464, + "loss": 1.3507, + "theoretical_loss": 3.497818984870216, + "tokens_seen": 1749745664 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047448242657679345, + "loss": 1.394, + "theoretical_loss": 3.4977967433773003, + "tokens_seen": 1749876736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444230460600225, + "loss": 1.3943, + "theoretical_loss": 3.4977745040167205, + "tokens_seen": 1750007808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047440218263521106, + "loss": 1.3147, + "theoretical_loss": 3.497752266788112, + "tokens_seen": 1750138880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743620606644198, + "loss": 1.3308, + "theoretical_loss": 3.497730031691111, + "tokens_seen": 1750269952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743219386936287, + "loss": 1.3246, + "theoretical_loss": 3.497707798725353, + "tokens_seen": 1750401024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742818167228374, + "loss": 1.2988, + "theoretical_loss": 3.4976855678904752, + "tokens_seen": 1750532096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742416947520462, + "loss": 1.3883, + "theoretical_loss": 3.497663339186113, + "tokens_seen": 1750663168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047420157278125504, + "loss": 1.3415, + "theoretical_loss": 3.4976411126119036, + "tokens_seen": 1750794240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741614508104638, + "loss": 1.3476, + "theoretical_loss": 3.497618888167482, + "tokens_seen": 1750925312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047412132883967265, + "loss": 1.3284, + "theoretical_loss": 3.497596665852486, + "tokens_seen": 1751056384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740812068688814, + "loss": 1.3949, + "theoretical_loss": 3.497574445666552, + "tokens_seen": 1751187456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740410848980902, + "loss": 1.3768, + "theoretical_loss": 3.497552227609316, + "tokens_seen": 1751318528 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474000962927299, + "loss": 1.3728, + "theoretical_loss": 3.4975300116804156, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047396084095650777, + "loss": 1.3101, + "theoretical_loss": 3.4975077978794875, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739207189857166, + "loss": 1.3539, + "theoretical_loss": 3.497485586206168, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738805970149254, + "loss": 1.4418, + "theoretical_loss": 3.4974633766600944, + "tokens_seen": 1751842816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738404750441342, + "loss": 1.3223, + "theoretical_loss": 3.497441169240904, + "tokens_seen": 1751973888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047380035307334294, + "loss": 1.3566, + "theoretical_loss": 3.4974189639482347, + "tokens_seen": 1752104960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737602311025518, + "loss": 1.3391, + "theoretical_loss": 3.497396760781723, + "tokens_seen": 1752236032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047372010913176055, + "loss": 1.3285, + "theoretical_loss": 3.497374559741006, + "tokens_seen": 1752367104 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047367998716096936, + "loss": 1.4061, + "theoretical_loss": 3.497352360825722, + "tokens_seen": 1752498176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047363986519017816, + "loss": 1.4128, + "theoretical_loss": 3.4973301640355086, + "tokens_seen": 1752629248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735997432193869, + "loss": 1.3014, + "theoretical_loss": 3.497307969370002, + "tokens_seen": 1752760320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735596212485958, + "loss": 1.3091, + "theoretical_loss": 3.497285776828842, + "tokens_seen": 1752891392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351949927780453, + "loss": 1.4481, + "theoretical_loss": 3.4972635864116652, + "tokens_seen": 1753022464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047347937730701334, + "loss": 1.2999, + "theoretical_loss": 3.49724139811811, + "tokens_seen": 1753153536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047343925533622214, + "loss": 1.3694, + "theoretical_loss": 3.4972192119478143, + "tokens_seen": 1753284608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047339913336543095, + "loss": 1.3153, + "theoretical_loss": 3.4971970279004156, + "tokens_seen": 1753415680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733590113946397, + "loss": 1.3558, + "theoretical_loss": 3.497174845975553, + "tokens_seen": 1753546752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733188894238485, + "loss": 1.4094, + "theoretical_loss": 3.4971526661728647, + "tokens_seen": 1753677824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732787674530573, + "loss": 1.3459, + "theoretical_loss": 3.4971304884919885, + "tokens_seen": 1753808896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047323864548226606, + "loss": 1.3797, + "theoretical_loss": 3.4971083129325633, + "tokens_seen": 1753939968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731985235114749, + "loss": 1.3502, + "theoretical_loss": 3.497086139494228, + "tokens_seen": 1754071040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731584015406837, + "loss": 1.3856, + "theoretical_loss": 3.4970639681766205, + "tokens_seen": 1754202112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731182795698925, + "loss": 1.3743, + "theoretical_loss": 3.49704179897938, + "tokens_seen": 1754333184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730781575991013, + "loss": 1.4168, + "theoretical_loss": 3.4970196319021447, + "tokens_seen": 1754464256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303803562831004, + "loss": 1.4233, + "theoretical_loss": 3.496997466944554, + "tokens_seen": 1754595328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047299791365751885, + "loss": 1.3138, + "theoretical_loss": 3.496975304106247, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9905670285224915, + "objective/train/avg_token_score": 0.009432926774024963, + "objective/train/docs_used": 641116, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3313068151474, + "objective/train/original_loss": 1.343801498413086, + "objective/train/theoretical_loss": 3.496953143386863, + "objective/train/tokens_used": 125316576, + "objective/train/unlikelihood_loss": 0.024912001565098763, + "theoretical_loss": 3.496953143386863, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047295779168672765, + "loss": 1.3485, + "theoretical_loss": 3.496953143386863, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047291766971593646, + "loss": 1.3385, + "theoretical_loss": 3.49693098478604, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047287754774514527, + "loss": 1.3454, + "theoretical_loss": 3.496908828303419, + "tokens_seen": 1755119616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728374257743541, + "loss": 1.3306, + "theoretical_loss": 3.496886673938638, + "tokens_seen": 1755250688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727973038035628, + "loss": 1.3639, + "theoretical_loss": 3.4968645216913368, + "tokens_seen": 1755381760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047275718183277163, + "loss": 1.3582, + "theoretical_loss": 3.496842371561155, + "tokens_seen": 1755512832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047271705986198044, + "loss": 1.3157, + "theoretical_loss": 3.4968202235477324, + "tokens_seen": 1755643904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726769378911892, + "loss": 1.3386, + "theoretical_loss": 3.4967980776507086, + "tokens_seen": 1755774976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047263681592039805, + "loss": 1.3151, + "theoretical_loss": 3.4967759338697233, + "tokens_seen": 1755906048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725966939496068, + "loss": 1.3087, + "theoretical_loss": 3.496753792204416, + "tokens_seen": 1756037120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725565719788156, + "loss": 1.2769, + "theoretical_loss": 3.4967316526544274, + "tokens_seen": 1756168192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725164500080244, + "loss": 1.3383, + "theoretical_loss": 3.496709515219397, + "tokens_seen": 1756299264 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047247632803723317, + "loss": 1.3754, + "theoretical_loss": 3.4966873798989653, + "tokens_seen": 1756430336 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472436206066442, + "loss": 1.3735, + "theoretical_loss": 3.4966652466927726, + "tokens_seen": 1756561408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723960840956508, + "loss": 1.3698, + "theoretical_loss": 3.4966431156004587, + "tokens_seen": 1756692480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723559621248596, + "loss": 1.3271, + "theoretical_loss": 3.4966209866216644, + "tokens_seen": 1756823552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723158401540684, + "loss": 1.3969, + "theoretical_loss": 3.4965988597560305, + "tokens_seen": 1756954624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722757181832772, + "loss": 1.4203, + "theoretical_loss": 3.496576735003197, + "tokens_seen": 1757085696 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047223559621248595, + "loss": 1.3928, + "theoretical_loss": 3.4965546123628046, + "tokens_seen": 1757216768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721954742416948, + "loss": 1.2597, + "theoretical_loss": 3.496532491834494, + "tokens_seen": 1757347840 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047215535227090356, + "loss": 1.3412, + "theoretical_loss": 3.496510373417907, + "tokens_seen": 1757478912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721152303001123, + "loss": 1.355, + "theoretical_loss": 3.4964882571126834, + "tokens_seen": 1757609984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720751083293212, + "loss": 1.323, + "theoretical_loss": 3.496466142918465, + "tokens_seen": 1757741056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047203498635852993, + "loss": 1.3893, + "theoretical_loss": 3.4964440308348923, + "tokens_seen": 1757872128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047199486438773873, + "loss": 1.3548, + "theoretical_loss": 3.4964219208616067, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047195474241694754, + "loss": 1.3218, + "theoretical_loss": 3.4963998129982494, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047191462044615635, + "loss": 1.2946, + "theoretical_loss": 3.4963777072444624, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718744984753651, + "loss": 1.3557, + "theoretical_loss": 3.496355603599887, + "tokens_seen": 1758396416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718343765045739, + "loss": 1.2716, + "theoretical_loss": 3.4963335020641635, + "tokens_seen": 1758527488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717942545337827, + "loss": 1.396, + "theoretical_loss": 3.496311402636935, + "tokens_seen": 1758658560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047175413256299146, + "loss": 1.424, + "theoretical_loss": 3.4962893053178425, + "tokens_seen": 1758789632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717140105922003, + "loss": 1.4318, + "theoretical_loss": 3.496267210106528, + "tokens_seen": 1758920704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716738886214091, + "loss": 1.3846, + "theoretical_loss": 3.496245117002634, + "tokens_seen": 1759051776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163376665061794, + "loss": 1.3288, + "theoretical_loss": 3.4962230260058016, + "tokens_seen": 1759182848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715936446798267, + "loss": 1.298, + "theoretical_loss": 3.496200937115673, + "tokens_seen": 1759313920 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047155352270903544, + "loss": 1.3445, + "theoretical_loss": 3.4961788503318907, + "tokens_seen": 1759444992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715134007382443, + "loss": 1.3439, + "theoretical_loss": 3.4961567656540966, + "tokens_seen": 1759576064 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047147327876745305, + "loss": 1.2931, + "theoretical_loss": 3.4961346830819338, + "tokens_seen": 1759707136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047143315679666186, + "loss": 1.2786, + "theoretical_loss": 3.4961126026150433, + "tokens_seen": 1759838208 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139303482587067, + "loss": 1.4213, + "theoretical_loss": 3.496090524253069, + "tokens_seen": 1759969280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713529128550795, + "loss": 1.4392, + "theoretical_loss": 3.496068447995653, + "tokens_seen": 1760100352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713127908842882, + "loss": 1.3813, + "theoretical_loss": 3.4960463738424385, + "tokens_seen": 1760231424 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047127266891349703, + "loss": 1.2864, + "theoretical_loss": 3.4960243017930672, + "tokens_seen": 1760362496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047123254694270584, + "loss": 1.3995, + "theoretical_loss": 3.496002231847182, + "tokens_seen": 1760493568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711924249719146, + "loss": 1.3243, + "theoretical_loss": 3.4959801640044272, + "tokens_seen": 1760624640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115230300112345, + "loss": 1.3149, + "theoretical_loss": 3.4959580982644445, + "tokens_seen": 1760755712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711121810303322, + "loss": 1.3788, + "theoretical_loss": 3.495936034626878, + "tokens_seen": 1760886784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047107205905954106, + "loss": 1.3619, + "theoretical_loss": 3.49591397309137, + "tokens_seen": 1761017856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710319370887498, + "loss": 1.314, + "theoretical_loss": 3.4958919136575646, + "tokens_seen": 1761148928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047099181511795857, + "loss": 1.3531, + "theoretical_loss": 3.495869856325105, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9927342534065247, + "objective/train/avg_token_score": 0.007265761028975248, + "objective/train/docs_used": 643454, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2912812232971191, + "objective/train/original_loss": 1.3019251823425293, + "objective/train/theoretical_loss": 3.4958478010936336, + "objective/train/tokens_used": 131870176, + "objective/train/unlikelihood_loss": 0.01763155683875084, + "theoretical_loss": 3.4958478010936336, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047095169314716743, + "loss": 1.2853, + "theoretical_loss": 3.4958478010936336, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709115711763762, + "loss": 1.4121, + "theoretical_loss": 3.4958257479627957, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470871449205585, + "loss": 1.3671, + "theoretical_loss": 3.4958036969322337, + "tokens_seen": 1761673216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708313272347938, + "loss": 1.3629, + "theoretical_loss": 3.495781648001592, + "tokens_seen": 1761804288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707912052640026, + "loss": 1.3043, + "theoretical_loss": 3.495759601170514, + "tokens_seen": 1761935360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047075108329321135, + "loss": 1.4701, + "theoretical_loss": 3.4957375564386437, + "tokens_seen": 1762066432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707109613224202, + "loss": 1.3295, + "theoretical_loss": 3.495715513805625, + "tokens_seen": 1762197504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047067083935162896, + "loss": 1.3487, + "theoretical_loss": 3.4956934732711025, + "tokens_seen": 1762328576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706307173808377, + "loss": 1.4509, + "theoretical_loss": 3.4956714348347195, + "tokens_seen": 1762459648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705905954100466, + "loss": 1.3504, + "theoretical_loss": 3.4956493984961208, + "tokens_seen": 1762590720 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047055047343925533, + "loss": 1.3303, + "theoretical_loss": 3.4956273642549514, + "tokens_seen": 1762721792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047051035146846413, + "loss": 1.4175, + "theoretical_loss": 3.4956053321108547, + "tokens_seen": 1762852864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047047022949767294, + "loss": 1.3389, + "theoretical_loss": 3.4955833020634755, + "tokens_seen": 1762983936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047043010752688175, + "loss": 1.4131, + "theoretical_loss": 3.495561274112458, + "tokens_seen": 1763115008 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047038998555609055, + "loss": 1.3354, + "theoretical_loss": 3.4955392482574474, + "tokens_seen": 1763246080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703498635852993, + "loss": 1.3814, + "theoretical_loss": 3.495517224498089, + "tokens_seen": 1763377152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703097416145081, + "loss": 1.3759, + "theoretical_loss": 3.495495202834027, + "tokens_seen": 1763508224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702696196437169, + "loss": 1.5348, + "theoretical_loss": 3.495473183264906, + "tokens_seen": 1763639296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702294976729257, + "loss": 1.3553, + "theoretical_loss": 3.4954511657903717, + "tokens_seen": 1763770368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701893757021345, + "loss": 1.2595, + "theoretical_loss": 3.495429150410069, + "tokens_seen": 1763901440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047014925373134334, + "loss": 1.4141, + "theoretical_loss": 3.495407137123643, + "tokens_seen": 1764032512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701091317605521, + "loss": 1.3052, + "theoretical_loss": 3.495385125930739, + "tokens_seen": 1764163584 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006900978976084, + "loss": 1.3709, + "theoretical_loss": 3.4953631168310024, + "tokens_seen": 1764294656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700288878189697, + "loss": 1.3533, + "theoretical_loss": 3.4953411098240785, + "tokens_seen": 1764425728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046998876584817845, + "loss": 1.3476, + "theoretical_loss": 3.4953191049096137, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046994864387738726, + "loss": 1.3405, + "theoretical_loss": 3.4952971020872527, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046990852190659607, + "loss": 1.3955, + "theoretical_loss": 3.4952751013566417, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046986839993580487, + "loss": 1.4065, + "theoretical_loss": 3.495253102717426, + "tokens_seen": 1764950016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698282779650136, + "loss": 1.4243, + "theoretical_loss": 3.495231106169252, + "tokens_seen": 1765081088 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046978815599422243, + "loss": 1.3507, + "theoretical_loss": 3.4952091117117656, + "tokens_seen": 1765212160 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046974803402343124, + "loss": 1.2923, + "theoretical_loss": 3.495187119344613, + "tokens_seen": 1765343232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046970791205264004, + "loss": 1.4286, + "theoretical_loss": 3.4951651290674395, + "tokens_seen": 1765474304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046966779008184885, + "loss": 1.346, + "theoretical_loss": 3.4951431408798928, + "tokens_seen": 1765605376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696276681110576, + "loss": 1.3328, + "theoretical_loss": 3.495121154781618, + "tokens_seen": 1765736448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046958754614026646, + "loss": 1.3088, + "theoretical_loss": 3.495099170772262, + "tokens_seen": 1765867520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695474241694752, + "loss": 1.4185, + "theoretical_loss": 3.4950771888514716, + "tokens_seen": 1765998592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046950730219868397, + "loss": 1.3097, + "theoretical_loss": 3.495055209018892, + "tokens_seen": 1766129664 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046946718022789283, + "loss": 1.2983, + "theoretical_loss": 3.4950332312741716, + "tokens_seen": 1766260736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694270582571016, + "loss": 1.3611, + "theoretical_loss": 3.495011255616957, + "tokens_seen": 1766391808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693869362863104, + "loss": 1.333, + "theoretical_loss": 3.4949892820468937, + "tokens_seen": 1766522880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693468143155192, + "loss": 1.3407, + "theoretical_loss": 3.49496731056363, + "tokens_seen": 1766653952 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469306692344728, + "loss": 1.4589, + "theoretical_loss": 3.4949453411668117, + "tokens_seen": 1766785024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046926657037393675, + "loss": 1.2912, + "theoretical_loss": 3.4949233738560874, + "tokens_seen": 1766916096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046922644840314556, + "loss": 1.3497, + "theoretical_loss": 3.4949014086311028, + "tokens_seen": 1767047168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046918632643235436, + "loss": 1.3543, + "theoretical_loss": 3.494879445491506, + "tokens_seen": 1767178240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691462044615631, + "loss": 1.345, + "theoretical_loss": 3.4948574844369444, + "tokens_seen": 1767309312 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469106082490772, + "loss": 1.3345, + "theoretical_loss": 3.494835525467065, + "tokens_seen": 1767440384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046906596051998073, + "loss": 1.3291, + "theoretical_loss": 3.494813568581516, + "tokens_seen": 1767571456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690258385491896, + "loss": 1.3071, + "theoretical_loss": 3.494791613779944, + "tokens_seen": 1767702528 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046898571657839834, + "loss": 1.4127, + "theoretical_loss": 3.494769661061998, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9928597807884216, + "objective/train/avg_token_score": 0.00714022945612669, + "objective/train/docs_used": 645687, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3491356372833252, + "objective/train/original_loss": 1.3599581718444824, + "objective/train/theoretical_loss": 3.4947477104273243, + "objective/train/tokens_used": 138423776, + "objective/train/unlikelihood_loss": 0.020140038803219795, + "theoretical_loss": 3.4947477104273243, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046894559460760715, + "loss": 1.3241, + "theoretical_loss": 3.4947477104273243, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046890547263681595, + "loss": 1.3496, + "theoretical_loss": 3.494725761875572, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688653506660247, + "loss": 1.2926, + "theoretical_loss": 3.494703815406388, + "tokens_seen": 1768226816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688252286952335, + "loss": 1.3037, + "theoretical_loss": 3.494681871019422, + "tokens_seen": 1768357888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687851067244423, + "loss": 1.3916, + "theoretical_loss": 3.49465992871432, + "tokens_seen": 1768488960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687449847536511, + "loss": 1.3806, + "theoretical_loss": 3.494637988490732, + "tokens_seen": 1768620032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687048627828599, + "loss": 1.4328, + "theoretical_loss": 3.4946160503483057, + "tokens_seen": 1768751104 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046866474081206874, + "loss": 1.3321, + "theoretical_loss": 3.4945941142866888, + "tokens_seen": 1768882176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686246188412775, + "loss": 1.3069, + "theoretical_loss": 3.494572180305531, + "tokens_seen": 1769013248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046858449687048624, + "loss": 1.2832, + "theoretical_loss": 3.4945502484044804, + "tokens_seen": 1769144320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685443748996951, + "loss": 1.3471, + "theoretical_loss": 3.4945283185831846, + "tokens_seen": 1769275392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046850425292890385, + "loss": 1.3395, + "theoretical_loss": 3.494506390841294, + "tokens_seen": 1769406464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684641309581127, + "loss": 1.3946, + "theoretical_loss": 3.4944844651784566, + "tokens_seen": 1769537536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046842400898732147, + "loss": 1.3608, + "theoretical_loss": 3.494462541594321, + "tokens_seen": 1769668608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838388701653027, + "loss": 1.3368, + "theoretical_loss": 3.494440620088536, + "tokens_seen": 1769799680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683437650457391, + "loss": 1.2854, + "theoretical_loss": 3.494418700660752, + "tokens_seen": 1769930752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046830364307494783, + "loss": 1.32, + "theoretical_loss": 3.4943967833106173, + "tokens_seen": 1770061824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046826352110415664, + "loss": 1.3341, + "theoretical_loss": 3.494374868037781, + "tokens_seen": 1770192896 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046822339913336544, + "loss": 1.2492, + "theoretical_loss": 3.4943529548418923, + "tokens_seen": 1770323968 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046818327716257425, + "loss": 1.3439, + "theoretical_loss": 3.4943310437226014, + "tokens_seen": 1770455040 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468143155191783, + "loss": 1.3448, + "theoretical_loss": 3.4943091346795567, + "tokens_seen": 1770586112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810303322099186, + "loss": 1.3592, + "theoretical_loss": 3.4942872277124093, + "tokens_seen": 1770717184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680629112502006, + "loss": 1.345, + "theoretical_loss": 3.494265322820808, + "tokens_seen": 1770848256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802278927940937, + "loss": 1.3318, + "theoretical_loss": 3.4942434200044015, + "tokens_seen": 1770979328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679826673086182, + "loss": 1.3049, + "theoretical_loss": 3.494221519262841, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467942545337827, + "loss": 1.3396, + "theoretical_loss": 3.4941996205957766, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679024233670358, + "loss": 1.3402, + "theoretical_loss": 3.4941777240028573, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678623013962446, + "loss": 1.2791, + "theoretical_loss": 3.494155829483734, + "tokens_seen": 1771503616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678221794254534, + "loss": 1.326, + "theoretical_loss": 3.4941339370380566, + "tokens_seen": 1771634688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677820574546622, + "loss": 1.3263, + "theoretical_loss": 3.4941120466654745, + "tokens_seen": 1771765760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046774193548387096, + "loss": 1.3639, + "theoretical_loss": 3.4940901583656396, + "tokens_seen": 1771896832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046770181351307976, + "loss": 1.3252, + "theoretical_loss": 3.4940682721382017, + "tokens_seen": 1772027904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046766169154228857, + "loss": 1.2636, + "theoretical_loss": 3.494046387982811, + "tokens_seen": 1772158976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676215695714974, + "loss": 1.4415, + "theoretical_loss": 3.494024505899118, + "tokens_seen": 1772290048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046758144760070613, + "loss": 1.3457, + "theoretical_loss": 3.4940026258867745, + "tokens_seen": 1772421120 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467541325629915, + "loss": 1.3635, + "theoretical_loss": 3.4939807479454297, + "tokens_seen": 1772552192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046750120365912374, + "loss": 1.3246, + "theoretical_loss": 3.4939588720747357, + "tokens_seen": 1772683264 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046746108168833255, + "loss": 1.3495, + "theoretical_loss": 3.493936998274343, + "tokens_seen": 1772814336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046742095971754135, + "loss": 1.3118, + "theoretical_loss": 3.493915126543902, + "tokens_seen": 1772945408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673808377467501, + "loss": 1.3353, + "theoretical_loss": 3.493893256883065, + "tokens_seen": 1773076480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673407157759589, + "loss": 1.3257, + "theoretical_loss": 3.493871389291482, + "tokens_seen": 1773207552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673005938051677, + "loss": 1.3189, + "theoretical_loss": 3.4938495237688056, + "tokens_seen": 1773338624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672604718343765, + "loss": 1.3768, + "theoretical_loss": 3.493827660314686, + "tokens_seen": 1773469696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046722034986358533, + "loss": 1.3383, + "theoretical_loss": 3.493805798928775, + "tokens_seen": 1773600768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046718022789279414, + "loss": 1.3507, + "theoretical_loss": 3.493783939610725, + "tokens_seen": 1773731840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671401059220029, + "loss": 1.4747, + "theoretical_loss": 3.493762082360186, + "tokens_seen": 1773862912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670999839512117, + "loss": 1.4175, + "theoretical_loss": 3.4937402271768105, + "tokens_seen": 1773993984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670598619804205, + "loss": 1.2972, + "theoretical_loss": 3.4937183740602507, + "tokens_seen": 1774125056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046701974000962925, + "loss": 1.3516, + "theoretical_loss": 3.4936965230101578, + "tokens_seen": 1774256128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669796180388381, + "loss": 1.32, + "theoretical_loss": 3.4936746740261846, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9899575114250183, + "objective/train/avg_token_score": 0.010042516514658928, + "objective/train/docs_used": 647978, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3564637899398804, + "objective/train/original_loss": 1.369072675704956, + "objective/train/theoretical_loss": 3.493652827107982, + "objective/train/tokens_used": 144977376, + "objective/train/unlikelihood_loss": 0.028294233605265617, + "theoretical_loss": 3.493652827107982, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046693949606804687, + "loss": 1.3664, + "theoretical_loss": 3.493652827107982, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046689937409725567, + "loss": 1.328, + "theoretical_loss": 3.493630982255203, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668592521264645, + "loss": 1.2395, + "theoretical_loss": 3.4936091394674995, + "tokens_seen": 1774780416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046681913015567323, + "loss": 1.404, + "theoretical_loss": 3.493587298744524, + "tokens_seen": 1774911488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046677900818488204, + "loss": 1.3336, + "theoretical_loss": 3.4935654600859283, + "tokens_seen": 1775042560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046673888621409084, + "loss": 1.3335, + "theoretical_loss": 3.4935436234913655, + "tokens_seen": 1775173632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046669876424329965, + "loss": 1.3048, + "theoretical_loss": 3.4935217889604875, + "tokens_seen": 1775304704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666586422725084, + "loss": 1.315, + "theoretical_loss": 3.493499956492948, + "tokens_seen": 1775435776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046661852030171726, + "loss": 1.3422, + "theoretical_loss": 3.493478126088398, + "tokens_seen": 1775566848 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466578398330926, + "loss": 1.3629, + "theoretical_loss": 3.4934562977464925, + "tokens_seen": 1775697920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665382763601348, + "loss": 1.3501, + "theoretical_loss": 3.493434471466882, + "tokens_seen": 1775828992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664981543893436, + "loss": 1.4014, + "theoretical_loss": 3.4934126472492215, + "tokens_seen": 1775960064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664580324185524, + "loss": 1.3483, + "theoretical_loss": 3.4933908250931633, + "tokens_seen": 1776091136 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046641791044776124, + "loss": 1.3578, + "theoretical_loss": 3.49336900499836, + "tokens_seen": 1776222208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046637778847697, + "loss": 1.3341, + "theoretical_loss": 3.4933471869644652, + "tokens_seen": 1776353280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663376665061788, + "loss": 1.3184, + "theoretical_loss": 3.4933253709911325, + "tokens_seen": 1776484352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662975445353876, + "loss": 1.3312, + "theoretical_loss": 3.493303557078015, + "tokens_seen": 1776615424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046625742256459636, + "loss": 1.3907, + "theoretical_loss": 3.4932817452247664, + "tokens_seen": 1776746496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046621730059380516, + "loss": 1.2897, + "theoretical_loss": 3.4932599354310403, + "tokens_seen": 1776877568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046617717862301397, + "loss": 1.2965, + "theoretical_loss": 3.4932381276964897, + "tokens_seen": 1777008640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661370566522228, + "loss": 1.3315, + "theoretical_loss": 3.4932163220207686, + "tokens_seen": 1777139712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046609693468143153, + "loss": 1.3236, + "theoretical_loss": 3.4931945184035307, + "tokens_seen": 1777270784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660568127106404, + "loss": 1.3448, + "theoretical_loss": 3.4931727168444304, + "tokens_seen": 1777401856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046601669073984914, + "loss": 1.3542, + "theoretical_loss": 3.4931509173431214, + "tokens_seen": 1777532928 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465976568769058, + "loss": 1.3863, + "theoretical_loss": 3.4931291198992582, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046593644679826675, + "loss": 1.2698, + "theoretical_loss": 3.493107324512494, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658963248274755, + "loss": 1.3148, + "theoretical_loss": 3.4930855311824835, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046585620285668436, + "loss": 1.3266, + "theoretical_loss": 3.493063739908881, + "tokens_seen": 1778057216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658160808858931, + "loss": 1.3057, + "theoretical_loss": 3.493041950691341, + "tokens_seen": 1778188288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657759589151019, + "loss": 1.307, + "theoretical_loss": 3.4930201635295175, + "tokens_seen": 1778319360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046573583694431073, + "loss": 1.3568, + "theoretical_loss": 3.4929983784230663, + "tokens_seen": 1778450432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046569571497351954, + "loss": 1.2477, + "theoretical_loss": 3.4929765953716405, + "tokens_seen": 1778581504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656555930027283, + "loss": 1.2976, + "theoretical_loss": 3.4929548143748956, + "tokens_seen": 1778712576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656154710319371, + "loss": 1.2601, + "theoretical_loss": 3.492933035432486, + "tokens_seen": 1778843648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655753490611459, + "loss": 1.3118, + "theoretical_loss": 3.4929112585440674, + "tokens_seen": 1778974720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553522709035465, + "loss": 1.27, + "theoretical_loss": 3.492889483709294, + "tokens_seen": 1779105792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654951051195635, + "loss": 1.309, + "theoretical_loss": 3.492867710927821, + "tokens_seen": 1779236864 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046545498314877227, + "loss": 1.258, + "theoretical_loss": 3.4928459401993033, + "tokens_seen": 1779367936 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046541486117798107, + "loss": 1.3024, + "theoretical_loss": 3.492824171523397, + "tokens_seen": 1779499008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653747392071899, + "loss": 1.3195, + "theoretical_loss": 3.4928024048997566, + "tokens_seen": 1779630080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533461723639863, + "loss": 1.3041, + "theoretical_loss": 3.4927806403280384, + "tokens_seen": 1779761152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652944952656075, + "loss": 1.2691, + "theoretical_loss": 3.492758877807897, + "tokens_seen": 1779892224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525437329481624, + "loss": 1.4029, + "theoretical_loss": 3.4927371173389874, + "tokens_seen": 1780023296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046521425132402505, + "loss": 1.3473, + "theoretical_loss": 3.4927153589209667, + "tokens_seen": 1780154368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046517412935323386, + "loss": 1.332, + "theoretical_loss": 3.49269360255349, + "tokens_seen": 1780285440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046513400738244266, + "loss": 1.2936, + "theoretical_loss": 3.492671848236213, + "tokens_seen": 1780416512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650938854116514, + "loss": 1.2564, + "theoretical_loss": 3.4926500959687914, + "tokens_seen": 1780547584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650537634408602, + "loss": 1.3819, + "theoretical_loss": 3.4926283457508815, + "tokens_seen": 1780678656 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465013641470069, + "loss": 1.3876, + "theoretical_loss": 3.4926065975821396, + "tokens_seen": 1780809728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649735194992778, + "loss": 1.2756, + "theoretical_loss": 3.4925848514622215, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9912664294242859, + "objective/train/avg_token_score": 0.008733604103326797, + "objective/train/docs_used": 650314, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3752527236938477, + "objective/train/original_loss": 1.3863849639892578, + "objective/train/theoretical_loss": 3.492563107390783, + "objective/train/tokens_used": 151530976, + "objective/train/unlikelihood_loss": 0.03043108992278576, + "theoretical_loss": 3.492563107390783, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046493339752848664, + "loss": 1.3237, + "theoretical_loss": 3.492563107390783, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648932755576954, + "loss": 1.3352, + "theoretical_loss": 3.4925413653674813, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648531535869042, + "loss": 1.3687, + "theoretical_loss": 3.4925196253919717, + "tokens_seen": 1781334016 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464813031616113, + "loss": 1.414, + "theoretical_loss": 3.4924978874639114, + "tokens_seen": 1781465088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046477290964532176, + "loss": 1.3171, + "theoretical_loss": 3.4924761515829577, + "tokens_seen": 1781596160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046473278767453056, + "loss": 1.3471, + "theoretical_loss": 3.4924544177487657, + "tokens_seen": 1781727232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046469266570373937, + "loss": 1.2691, + "theoretical_loss": 3.492432685960993, + "tokens_seen": 1781858304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646525437329482, + "loss": 1.4372, + "theoretical_loss": 3.492410956219296, + "tokens_seen": 1781989376 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464612421762157, + "loss": 1.3712, + "theoretical_loss": 3.492389228523332, + "tokens_seen": 1782120448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645722997913658, + "loss": 1.3244, + "theoretical_loss": 3.4923675028727574, + "tokens_seen": 1782251520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046453217782057454, + "loss": 1.3318, + "theoretical_loss": 3.49234577926723, + "tokens_seen": 1782382592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046449205584978335, + "loss": 1.3855, + "theoretical_loss": 3.4923240577064067, + "tokens_seen": 1782513664 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046445193387899215, + "loss": 1.3315, + "theoretical_loss": 3.492302338189944, + "tokens_seen": 1782644736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644118119082009, + "loss": 1.3248, + "theoretical_loss": 3.4922806207174997, + "tokens_seen": 1782775808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046437168993740976, + "loss": 1.2788, + "theoretical_loss": 3.4922589052887316, + "tokens_seen": 1782906880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643315679666185, + "loss": 1.3016, + "theoretical_loss": 3.492237191903297, + "tokens_seen": 1783037952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642914459958273, + "loss": 1.261, + "theoretical_loss": 3.492215480560853, + "tokens_seen": 1783169024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046425132402503613, + "loss": 1.3372, + "theoretical_loss": 3.4921937712610576, + "tokens_seen": 1783300096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046421120205424494, + "loss": 1.3058, + "theoretical_loss": 3.492172064003568, + "tokens_seen": 1783431168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641710800834537, + "loss": 1.319, + "theoretical_loss": 3.4921503587880425, + "tokens_seen": 1783562240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641309581126625, + "loss": 1.322, + "theoretical_loss": 3.492128655614139, + "tokens_seen": 1783693312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640908361418713, + "loss": 1.4106, + "theoretical_loss": 3.4921069544815153, + "tokens_seen": 1783824384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046405071417108005, + "loss": 1.3301, + "theoretical_loss": 3.4920852553898296, + "tokens_seen": 1783955456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640105922002889, + "loss": 1.4559, + "theoretical_loss": 3.4920635583387396, + "tokens_seen": 1784086528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046397047022949767, + "loss": 1.3289, + "theoretical_loss": 3.492041863327904, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639303482587065, + "loss": 1.3483, + "theoretical_loss": 3.49202017035698, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638902262879153, + "loss": 1.3896, + "theoretical_loss": 3.491998479425628, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385010431712403, + "loss": 1.4185, + "theoretical_loss": 3.4919767905335046, + "tokens_seen": 1784610816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638099823463329, + "loss": 1.3927, + "theoretical_loss": 3.491955103680269, + "tokens_seen": 1784741888 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046376986037554164, + "loss": 1.4494, + "theoretical_loss": 3.49193341886558, + "tokens_seen": 1784872960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046372973840475045, + "loss": 1.3463, + "theoretical_loss": 3.4919117360890963, + "tokens_seen": 1785004032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046368961643395926, + "loss": 1.4498, + "theoretical_loss": 3.491890055350477, + "tokens_seen": 1785135104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046364949446316806, + "loss": 1.4075, + "theoretical_loss": 3.4918683766493794, + "tokens_seen": 1785266176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636093724923768, + "loss": 1.3041, + "theoretical_loss": 3.491846699985464, + "tokens_seen": 1785397248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635692505215856, + "loss": 1.4024, + "theoretical_loss": 3.491825025358389, + "tokens_seen": 1785528320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635291285507944, + "loss": 1.3873, + "theoretical_loss": 3.4918033527678145, + "tokens_seen": 1785659392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634890065800032, + "loss": 1.2706, + "theoretical_loss": 3.4917816822133982, + "tokens_seen": 1785790464 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046344888460921204, + "loss": 1.3135, + "theoretical_loss": 3.491760013694801, + "tokens_seen": 1785921536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634087626384208, + "loss": 1.3387, + "theoretical_loss": 3.4917383472116814, + "tokens_seen": 1786052608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046336864066762965, + "loss": 1.2568, + "theoretical_loss": 3.4917166827636987, + "tokens_seen": 1786183680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633285186968384, + "loss": 1.3508, + "theoretical_loss": 3.4916950203505124, + "tokens_seen": 1786314752 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046328839672604716, + "loss": 1.332, + "theoretical_loss": 3.4916733599717826, + "tokens_seen": 1786445824 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463248274755256, + "loss": 1.2921, + "theoretical_loss": 3.491651701627169, + "tokens_seen": 1786576896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046320815278446477, + "loss": 1.4225, + "theoretical_loss": 3.491630045316331, + "tokens_seen": 1786707968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631680308136736, + "loss": 1.4491, + "theoretical_loss": 3.491608391038928, + "tokens_seen": 1786839040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631279088428824, + "loss": 1.3177, + "theoretical_loss": 3.491586738794621, + "tokens_seen": 1786970112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630877868720912, + "loss": 1.2836, + "theoretical_loss": 3.4915650885830694, + "tokens_seen": 1787101184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046304766490129994, + "loss": 1.3253, + "theoretical_loss": 3.4915434404039334, + "tokens_seen": 1787232256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046300754293050875, + "loss": 1.3444, + "theoretical_loss": 3.491521794256873, + "tokens_seen": 1787363328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046296742095971755, + "loss": 1.3141, + "theoretical_loss": 3.4915001501415484, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.987833559513092, + "objective/train/avg_token_score": 0.012166428379714489, + "objective/train/docs_used": 652772, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3774573802947998, + "objective/train/original_loss": 1.393503189086914, + "objective/train/theoretical_loss": 3.4914785080576207, + "objective/train/tokens_used": 158084576, + "objective/train/unlikelihood_loss": 0.033605657517910004, + "theoretical_loss": 3.4914785080576207, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629272989889263, + "loss": 1.4099, + "theoretical_loss": 3.4914785080576207, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046288717701813516, + "loss": 1.337, + "theoretical_loss": 3.4914568680047493, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628470550473439, + "loss": 1.3699, + "theoretical_loss": 3.491435229982595, + "tokens_seen": 1787887616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628069330765527, + "loss": 1.3988, + "theoretical_loss": 3.491413593990819, + "tokens_seen": 1788018688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046276681110576153, + "loss": 1.3569, + "theoretical_loss": 3.4913919600290813, + "tokens_seen": 1788149760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046272668913497034, + "loss": 1.2969, + "theoretical_loss": 3.4913703280970427, + "tokens_seen": 1788280832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046268656716417914, + "loss": 1.311, + "theoretical_loss": 3.4913486981943644, + "tokens_seen": 1788411904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626464451933879, + "loss": 1.2637, + "theoretical_loss": 3.491327070320707, + "tokens_seen": 1788542976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626063232225967, + "loss": 1.3369, + "theoretical_loss": 3.4913054444757314, + "tokens_seen": 1788674048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625662012518055, + "loss": 1.2881, + "theoretical_loss": 3.491283820659099, + "tokens_seen": 1788805120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625260792810143, + "loss": 1.3883, + "theoretical_loss": 3.491262198870471, + "tokens_seen": 1788936192 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046248595731022306, + "loss": 1.3132, + "theoretical_loss": 3.4912405791095082, + "tokens_seen": 1789067264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624458353394319, + "loss": 1.3271, + "theoretical_loss": 3.4912189613758726, + "tokens_seen": 1789198336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624057133686407, + "loss": 1.282, + "theoretical_loss": 3.491197345669225, + "tokens_seen": 1789329408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236559139784943, + "loss": 1.285, + "theoretical_loss": 3.4911757319892267, + "tokens_seen": 1789460480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623254694270583, + "loss": 1.3007, + "theoretical_loss": 3.4911541203355396, + "tokens_seen": 1789591552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046228534745626704, + "loss": 1.2762, + "theoretical_loss": 3.4911325107078257, + "tokens_seen": 1789722624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046224522548547585, + "loss": 1.3599, + "theoretical_loss": 3.4911109031057457, + "tokens_seen": 1789853696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046220510351468465, + "loss": 1.2881, + "theoretical_loss": 3.491089297528963, + "tokens_seen": 1789984768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046216498154389346, + "loss": 1.2431, + "theoretical_loss": 3.491067693977138, + "tokens_seen": 1790115840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046212485957310227, + "loss": 1.392, + "theoretical_loss": 3.491046092449933, + "tokens_seen": 1790246912 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462084737602311, + "loss": 1.4352, + "theoretical_loss": 3.4910244929470107, + "tokens_seen": 1790377984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620446156315198, + "loss": 1.2472, + "theoretical_loss": 3.4910028954680326, + "tokens_seen": 1790509056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046200449366072863, + "loss": 1.3911, + "theoretical_loss": 3.490981300012661, + "tokens_seen": 1790640128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046196437168993744, + "loss": 1.2743, + "theoretical_loss": 3.4909597065805587, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619242497191462, + "loss": 1.4064, + "theoretical_loss": 3.490938115171387, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046188412774835505, + "loss": 1.2816, + "theoretical_loss": 3.49091652578481, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618440057775638, + "loss": 1.3497, + "theoretical_loss": 3.4908949384204884, + "tokens_seen": 1791164416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046180388380677256, + "loss": 1.3224, + "theoretical_loss": 3.4908733530780856, + "tokens_seen": 1791295488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617637618359814, + "loss": 1.243, + "theoretical_loss": 3.490851769757265, + "tokens_seen": 1791426560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046172363986519017, + "loss": 1.2244, + "theoretical_loss": 3.4908301884576884, + "tokens_seen": 1791557632 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461683517894399, + "loss": 1.3271, + "theoretical_loss": 3.490808609179019, + "tokens_seen": 1791688704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616433959236078, + "loss": 1.3568, + "theoretical_loss": 3.4907870319209198, + "tokens_seen": 1791819776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616032739528166, + "loss": 1.3093, + "theoretical_loss": 3.4907654566830537, + "tokens_seen": 1791950848 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046156315198202534, + "loss": 1.3885, + "theoretical_loss": 3.4907438834650844, + "tokens_seen": 1792081920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046152303001123415, + "loss": 1.2439, + "theoretical_loss": 3.4907223122666737, + "tokens_seen": 1792212992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046148290804044295, + "loss": 1.3885, + "theoretical_loss": 3.490700743087486, + "tokens_seen": 1792344064 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046144278606965176, + "loss": 1.3345, + "theoretical_loss": 3.4906791759271845, + "tokens_seen": 1792475136 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046140266409886056, + "loss": 1.2428, + "theoretical_loss": 3.4906576107854326, + "tokens_seen": 1792606208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613625421280693, + "loss": 1.3204, + "theoretical_loss": 3.4906360476618934, + "tokens_seen": 1792737280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613224201572782, + "loss": 1.2931, + "theoretical_loss": 3.4906144865562307, + "tokens_seen": 1792868352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046128229818648693, + "loss": 1.302, + "theoretical_loss": 3.490592927468108, + "tokens_seen": 1792999424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612421762156957, + "loss": 1.3268, + "theoretical_loss": 3.49057137039719, + "tokens_seen": 1793130496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046120205424490454, + "loss": 1.2435, + "theoretical_loss": 3.4905498153431394, + "tokens_seen": 1793261568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611619322741133, + "loss": 1.3184, + "theoretical_loss": 3.49052826230562, + "tokens_seen": 1793392640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611218103033221, + "loss": 1.2934, + "theoretical_loss": 3.490506711284297, + "tokens_seen": 1793523712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610816883325309, + "loss": 1.3954, + "theoretical_loss": 3.490485162278834, + "tokens_seen": 1793654784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610415663617397, + "loss": 1.3517, + "theoretical_loss": 3.4904636152888946, + "tokens_seen": 1793785856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046100144439094846, + "loss": 1.3033, + "theoretical_loss": 3.490442070314143, + "tokens_seen": 1793916928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609613224201573, + "loss": 1.2609, + "theoretical_loss": 3.490420527354244, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9925491809844971, + "objective/train/avg_token_score": 0.007450864650309086, + "objective/train/docs_used": 655001, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3207000494003296, + "objective/train/original_loss": 1.3301713466644287, + "objective/train/theoretical_loss": 3.4903989864088625, + "objective/train/tokens_used": 164638176, + "objective/train/unlikelihood_loss": 0.024044645950198174, + "theoretical_loss": 3.4903989864088625, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609212004493661, + "loss": 1.3479, + "theoretical_loss": 3.4903989864088625, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046088107847857483, + "loss": 1.2716, + "theoretical_loss": 3.4903774474776617, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608409565077837, + "loss": 1.2414, + "theoretical_loss": 3.4903559105603073, + "tokens_seen": 1794441216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046080083453699244, + "loss": 1.3093, + "theoretical_loss": 3.4903343756564627, + "tokens_seen": 1794572288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607607125662013, + "loss": 1.3946, + "theoretical_loss": 3.4903128427657935, + "tokens_seen": 1794703360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046072059059541005, + "loss": 1.3179, + "theoretical_loss": 3.4902913118879644, + "tokens_seen": 1794834432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046068046862461886, + "loss": 1.3239, + "theoretical_loss": 3.4902697830226406, + "tokens_seen": 1794965504 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046064034665382767, + "loss": 1.2798, + "theoretical_loss": 3.490248256169486, + "tokens_seen": 1795096576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606002246830364, + "loss": 1.3339, + "theoretical_loss": 3.4902267313281667, + "tokens_seen": 1795227648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605601027122452, + "loss": 1.2375, + "theoretical_loss": 3.490205208498347, + "tokens_seen": 1795358720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046051998074145403, + "loss": 1.2875, + "theoretical_loss": 3.490183687679693, + "tokens_seen": 1795489792 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046047985877066284, + "loss": 1.2024, + "theoretical_loss": 3.490162168871869, + "tokens_seen": 1795620864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604397367998716, + "loss": 1.3063, + "theoretical_loss": 3.490140652074541, + "tokens_seen": 1795751936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046039961482908045, + "loss": 1.4118, + "theoretical_loss": 3.4901191372873743, + "tokens_seen": 1795883008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603594928582892, + "loss": 1.3775, + "theoretical_loss": 3.4900976245100344, + "tokens_seen": 1796014080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046031937088749795, + "loss": 1.3521, + "theoretical_loss": 3.4900761137421865, + "tokens_seen": 1796145152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602792489167068, + "loss": 1.2673, + "theoretical_loss": 3.490054604983497, + "tokens_seen": 1796276224 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046023912694591557, + "loss": 1.2578, + "theoretical_loss": 3.490033098233631, + "tokens_seen": 1796407296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046019900497512443, + "loss": 1.3191, + "theoretical_loss": 3.4900115934922544, + "tokens_seen": 1796538368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601588830043332, + "loss": 1.3895, + "theoretical_loss": 3.4899900907590338, + "tokens_seen": 1796669440 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460118761033542, + "loss": 1.3066, + "theoretical_loss": 3.4899685900336346, + "tokens_seen": 1796800512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600786390627508, + "loss": 1.3058, + "theoretical_loss": 3.489947091315723, + "tokens_seen": 1796931584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046003851709195954, + "loss": 1.3031, + "theoretical_loss": 3.489925594604965, + "tokens_seen": 1797062656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045999839512116835, + "loss": 1.422, + "theoretical_loss": 3.489904099901027, + "tokens_seen": 1797193728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045995827315037716, + "loss": 1.3414, + "theoretical_loss": 3.489882607203576, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045991815117958596, + "loss": 1.3374, + "theoretical_loss": 3.4898611165122766, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598780292087947, + "loss": 1.3691, + "theoretical_loss": 3.489839627826797, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598379072380036, + "loss": 1.2589, + "theoretical_loss": 3.489818141146803, + "tokens_seen": 1797718016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045979778526721233, + "loss": 1.3477, + "theoretical_loss": 3.4897966564719614, + "tokens_seen": 1797849088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597576632964211, + "loss": 1.3806, + "theoretical_loss": 3.489775173801939, + "tokens_seen": 1797980160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045971754132562994, + "loss": 1.3213, + "theoretical_loss": 3.489753693136402, + "tokens_seen": 1798111232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596774193548387, + "loss": 1.2838, + "theoretical_loss": 3.4897322144750182, + "tokens_seen": 1798242304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596372973840475, + "loss": 1.3955, + "theoretical_loss": 3.489710737817454, + "tokens_seen": 1798373376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595971754132563, + "loss": 1.3374, + "theoretical_loss": 3.4896892631633762, + "tokens_seen": 1798504448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595570534424651, + "loss": 1.3805, + "theoretical_loss": 3.4896677905124527, + "tokens_seen": 1798635520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595169314716739, + "loss": 1.2888, + "theoretical_loss": 3.4896463198643497, + "tokens_seen": 1798766592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594768095008827, + "loss": 1.3507, + "theoretical_loss": 3.4896248512187356, + "tokens_seen": 1798897664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594366875300915, + "loss": 1.2929, + "theoretical_loss": 3.4896033845752763, + "tokens_seen": 1799028736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593965655593003, + "loss": 1.2693, + "theoretical_loss": 3.4895819199336406, + "tokens_seen": 1799159808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593564435885091, + "loss": 1.2551, + "theoretical_loss": 3.4895604572934955, + "tokens_seen": 1799290880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931632161771784, + "loss": 1.3144, + "theoretical_loss": 3.4895389966545087, + "tokens_seen": 1799421952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592761996469267, + "loss": 1.3105, + "theoretical_loss": 3.4895175380163472, + "tokens_seen": 1799553024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923607767613545, + "loss": 1.2426, + "theoretical_loss": 3.4894960813786797, + "tokens_seen": 1799684096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045919595570534426, + "loss": 1.3273, + "theoretical_loss": 3.489474626741174, + "tokens_seen": 1799815168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045915583373455307, + "loss": 1.3463, + "theoretical_loss": 3.4894531741034966, + "tokens_seen": 1799946240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591157117637618, + "loss": 1.3098, + "theoretical_loss": 3.489431723465317, + "tokens_seen": 1800077312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590755897929706, + "loss": 1.2709, + "theoretical_loss": 3.489410274826303, + "tokens_seen": 1800208384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045903546782217943, + "loss": 1.4031, + "theoretical_loss": 3.4893888281861223, + "tokens_seen": 1800339456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045899534585138824, + "loss": 1.397, + "theoretical_loss": 3.489367383544443, + "tokens_seen": 1800470528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045895522388059704, + "loss": 1.3317, + "theoretical_loss": 3.489345940900934, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9920512437820435, + "objective/train/avg_token_score": 0.007948792539536953, + "objective/train/docs_used": 657521, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3929649591445923, + "objective/train/original_loss": 1.405166745185852, + "objective/train/theoretical_loss": 3.4893245002552638, + "objective/train/tokens_used": 171191776, + "objective/train/unlikelihood_loss": 0.01814604550600052, + "theoretical_loss": 3.4893245002552638, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045891510190980585, + "loss": 1.4002, + "theoretical_loss": 3.4893245002552638, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588749799390146, + "loss": 1.2963, + "theoretical_loss": 3.4893030616070995, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588348579682234, + "loss": 1.3266, + "theoretical_loss": 3.4892816249561114, + "tokens_seen": 1800994816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587947359974322, + "loss": 1.3081, + "theoretical_loss": 3.489260190301967, + "tokens_seen": 1801125888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045875461402664097, + "loss": 1.3446, + "theoretical_loss": 3.4892387576443356, + "tokens_seen": 1801256960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045871449205584983, + "loss": 1.2776, + "theoretical_loss": 3.4892173269828852, + "tokens_seen": 1801388032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586743700850586, + "loss": 1.3522, + "theoretical_loss": 3.489195898317286, + "tokens_seen": 1801519104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586342481142674, + "loss": 1.3558, + "theoretical_loss": 3.489174471647205, + "tokens_seen": 1801650176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585941261434762, + "loss": 1.3145, + "theoretical_loss": 3.489153046972313, + "tokens_seen": 1801781248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045855400417268494, + "loss": 1.3756, + "theoretical_loss": 3.4891316242922787, + "tokens_seen": 1801912320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045851388220189375, + "loss": 1.3474, + "theoretical_loss": 3.4891102036067703, + "tokens_seen": 1802043392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045847376023110256, + "loss": 1.2961, + "theoretical_loss": 3.4890887849154577, + "tokens_seen": 1802174464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045843363826031136, + "loss": 1.2431, + "theoretical_loss": 3.4890673682180107, + "tokens_seen": 1802305536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583935162895201, + "loss": 1.2229, + "theoretical_loss": 3.489045953514098, + "tokens_seen": 1802436608 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458353394318729, + "loss": 1.407, + "theoretical_loss": 3.4890245408033893, + "tokens_seen": 1802567680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045831327234793773, + "loss": 1.3385, + "theoretical_loss": 3.489003130085554, + "tokens_seen": 1802698752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045827315037714653, + "loss": 1.3534, + "theoretical_loss": 3.4889817213602625, + "tokens_seen": 1802829824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045823302840635534, + "loss": 1.2575, + "theoretical_loss": 3.4889603146271835, + "tokens_seen": 1802960896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581929064355641, + "loss": 1.3798, + "theoretical_loss": 3.488938909885988, + "tokens_seen": 1803091968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045815278446477295, + "loss": 1.3393, + "theoretical_loss": 3.4889175071363443, + "tokens_seen": 1803223040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581126624939817, + "loss": 1.3634, + "theoretical_loss": 3.488896106377924, + "tokens_seen": 1803354112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580725405231905, + "loss": 1.299, + "theoretical_loss": 3.4888747076103956, + "tokens_seen": 1803485184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580324185523993, + "loss": 1.2833, + "theoretical_loss": 3.4888533108334303, + "tokens_seen": 1803616256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579922965816081, + "loss": 1.2666, + "theoretical_loss": 3.4888319160466983, + "tokens_seen": 1803747328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579521746108169, + "loss": 1.3494, + "theoretical_loss": 3.488810523249869, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579120526400257, + "loss": 1.3806, + "theoretical_loss": 3.4887891324426135, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578719306692345, + "loss": 1.297, + "theoretical_loss": 3.488767743624602, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783180869844324, + "loss": 1.3186, + "theoretical_loss": 3.488746356795505, + "tokens_seen": 1804271616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577916867276521, + "loss": 1.2675, + "theoretical_loss": 3.4887249719549933, + "tokens_seen": 1804402688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045775156475686085, + "loss": 1.3065, + "theoretical_loss": 3.4887035891027374, + "tokens_seen": 1804533760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577114427860697, + "loss": 1.3047, + "theoretical_loss": 3.4886822082384077, + "tokens_seen": 1804664832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045767132081527847, + "loss": 1.3627, + "theoretical_loss": 3.4886608293616757, + "tokens_seen": 1804795904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576311988444872, + "loss": 1.2196, + "theoretical_loss": 3.4886394524722117, + "tokens_seen": 1804926976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575910768736961, + "loss": 1.3005, + "theoretical_loss": 3.4886180775696873, + "tokens_seen": 1805058048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045755095490290483, + "loss": 1.2971, + "theoretical_loss": 3.488596704653773, + "tokens_seen": 1805189120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045751083293211364, + "loss": 1.2936, + "theoretical_loss": 3.4885753337241403, + "tokens_seen": 1805320192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045747071096132244, + "loss": 1.3702, + "theoretical_loss": 3.48855396478046, + "tokens_seen": 1805451264 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045743058899053125, + "loss": 1.2863, + "theoretical_loss": 3.4885325978224038, + "tokens_seen": 1805582336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739046701974, + "loss": 1.3517, + "theoretical_loss": 3.4885112328496426, + "tokens_seen": 1805713408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573503450489488, + "loss": 1.3492, + "theoretical_loss": 3.4884898698618487, + "tokens_seen": 1805844480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573102230781576, + "loss": 1.3563, + "theoretical_loss": 3.488468508858693, + "tokens_seen": 1805975552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045727010110736637, + "loss": 1.2877, + "theoretical_loss": 3.488447149839847, + "tokens_seen": 1806106624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045722997913657523, + "loss": 1.3464, + "theoretical_loss": 3.488425792804983, + "tokens_seen": 1806237696 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457189857165784, + "loss": 1.2509, + "theoretical_loss": 3.488404437753772, + "tokens_seen": 1806368768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571497351949928, + "loss": 1.4099, + "theoretical_loss": 3.4883830846858856, + "tokens_seen": 1806499840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571096132242016, + "loss": 1.3417, + "theoretical_loss": 3.488361733600997, + "tokens_seen": 1806630912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045706949125341034, + "loss": 1.371, + "theoretical_loss": 3.4883403844987777, + "tokens_seen": 1806761984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570293692826192, + "loss": 1.3052, + "theoretical_loss": 3.4883190373788997, + "tokens_seen": 1806893056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045698924731182796, + "loss": 1.3049, + "theoretical_loss": 3.4882976922410345, + "tokens_seen": 1807024128 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045694912534103676, + "loss": 1.3081, + "theoretical_loss": 3.488276349084855, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9913113117218018, + "objective/train/avg_token_score": 0.008688726462423801, + "objective/train/docs_used": 659683, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2862976789474487, + "objective/train/original_loss": 1.2978061437606812, + "objective/train/theoretical_loss": 3.4882550079100336, + "objective/train/tokens_used": 177745376, + "objective/train/unlikelihood_loss": 0.026831937953829765, + "theoretical_loss": 3.4882550079100336, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045690900337024557, + "loss": 1.313, + "theoretical_loss": 3.4882550079100336, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568688813994544, + "loss": 1.3756, + "theoretical_loss": 3.4882336687162425, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045682875942866313, + "loss": 1.3513, + "theoretical_loss": 3.4882123315031537, + "tokens_seen": 1807548416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045678863745787193, + "loss": 1.3621, + "theoretical_loss": 3.4881909962704407, + "tokens_seen": 1807679488 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045674851548708074, + "loss": 1.3148, + "theoretical_loss": 3.488169663017775, + "tokens_seen": 1807810560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567083935162895, + "loss": 1.2885, + "theoretical_loss": 3.4881483317448305, + "tokens_seen": 1807941632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045666827154549835, + "loss": 1.3109, + "theoretical_loss": 3.488127002451279, + "tokens_seen": 1808072704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566281495747071, + "loss": 1.3891, + "theoretical_loss": 3.488105675136794, + "tokens_seen": 1808203776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565880276039159, + "loss": 1.3315, + "theoretical_loss": 3.4880843498010483, + "tokens_seen": 1808334848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565479056331247, + "loss": 1.3134, + "theoretical_loss": 3.4880630264437142, + "tokens_seen": 1808465920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045650778366233347, + "loss": 1.386, + "theoretical_loss": 3.488041705064466, + "tokens_seen": 1808596992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564676616915423, + "loss": 1.3349, + "theoretical_loss": 3.488020385662976, + "tokens_seen": 1808728064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564275397207511, + "loss": 1.3167, + "theoretical_loss": 3.4879990682389175, + "tokens_seen": 1808859136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563874177499599, + "loss": 1.3225, + "theoretical_loss": 3.4879777527919638, + "tokens_seen": 1808990208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563472957791687, + "loss": 1.3269, + "theoretical_loss": 3.4879564393217892, + "tokens_seen": 1809121280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563071738083775, + "loss": 1.3412, + "theoretical_loss": 3.4879351278280657, + "tokens_seen": 1809252352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045626705183758625, + "loss": 1.3799, + "theoretical_loss": 3.487913818310468, + "tokens_seen": 1809383424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562269298667951, + "loss": 1.2988, + "theoretical_loss": 3.4878925107686696, + "tokens_seen": 1809514496 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045618680789600387, + "loss": 1.38, + "theoretical_loss": 3.4878712052023433, + "tokens_seen": 1809645568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561466859252126, + "loss": 1.3151, + "theoretical_loss": 3.487849901611164, + "tokens_seen": 1809776640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561065639544215, + "loss": 1.2934, + "theoretical_loss": 3.4878285999948053, + "tokens_seen": 1809907712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045606644198363023, + "loss": 1.3054, + "theoretical_loss": 3.487807300352941, + "tokens_seen": 1810038784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045602632001283904, + "loss": 1.3217, + "theoretical_loss": 3.4877860026852443, + "tokens_seen": 1810169856 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045598619804204784, + "loss": 1.3304, + "theoretical_loss": 3.487764706991391, + "tokens_seen": 1810300928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045594607607125665, + "loss": 1.3125, + "theoretical_loss": 3.487743413271054, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559059541004654, + "loss": 1.3044, + "theoretical_loss": 3.487722121523907, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558658321296742, + "loss": 1.3535, + "theoretical_loss": 3.4877008317496268, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455825710158883, + "loss": 1.3669, + "theoretical_loss": 3.487679543947885, + "tokens_seen": 1810825216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045578558818809177, + "loss": 1.3128, + "theoretical_loss": 3.4876582581183575, + "tokens_seen": 1810956288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557454662173006, + "loss": 1.3606, + "theoretical_loss": 3.4876369742607194, + "tokens_seen": 1811087360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557053442465094, + "loss": 1.3507, + "theoretical_loss": 3.4876156923746437, + "tokens_seen": 1811218432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045566522227571824, + "loss": 1.3542, + "theoretical_loss": 3.4875944124598064, + "tokens_seen": 1811349504 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455625100304927, + "loss": 1.362, + "theoretical_loss": 3.4875731345158822, + "tokens_seen": 1811480576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045558497833413574, + "loss": 1.3498, + "theoretical_loss": 3.4875518585425453, + "tokens_seen": 1811611648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555448563633446, + "loss": 1.3395, + "theoretical_loss": 3.4875305845394715, + "tokens_seen": 1811742720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045550473439255336, + "loss": 1.313, + "theoretical_loss": 3.487509312506335, + "tokens_seen": 1811873792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045546461242176216, + "loss": 1.3502, + "theoretical_loss": 3.487488042442811, + "tokens_seen": 1812004864 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045542449045097097, + "loss": 1.3236, + "theoretical_loss": 3.487466774348575, + "tokens_seen": 1812135936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553843684801798, + "loss": 1.3325, + "theoretical_loss": 3.4874455082233027, + "tokens_seen": 1812267008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045534424650938853, + "loss": 1.2848, + "theoretical_loss": 3.4874242440666685, + "tokens_seen": 1812398080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045530412453859733, + "loss": 1.3093, + "theoretical_loss": 3.4874029818783487, + "tokens_seen": 1812529152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045526400256780614, + "loss": 1.3745, + "theoretical_loss": 3.487381721658018, + "tokens_seen": 1812660224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552238805970149, + "loss": 1.3399, + "theoretical_loss": 3.487360463405352, + "tokens_seen": 1812791296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045518375862622375, + "loss": 1.3089, + "theoretical_loss": 3.4873392071200264, + "tokens_seen": 1812922368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551436366554325, + "loss": 1.2799, + "theoretical_loss": 3.4873179528017175, + "tokens_seen": 1813053440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045510351468464137, + "loss": 1.2936, + "theoretical_loss": 3.487296700450101, + "tokens_seen": 1813184512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550633927138501, + "loss": 1.3549, + "theoretical_loss": 3.4872754500648524, + "tokens_seen": 1813315584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045502327074305887, + "loss": 1.2997, + "theoretical_loss": 3.487254201645648, + "tokens_seen": 1813446656 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045498314877226773, + "loss": 1.3006, + "theoretical_loss": 3.487232955192163, + "tokens_seen": 1813577728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549430268014765, + "loss": 1.3591, + "theoretical_loss": 3.487211710704074, + "tokens_seen": 1813708800 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9918608069419861, + "objective/train/avg_token_score": 0.008139228448271751, + "objective/train/docs_used": 661946, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2484031915664673, + "objective/train/original_loss": 1.2585875988006592, + "objective/train/theoretical_loss": 3.4871904681810575, + "objective/train/tokens_used": 184298976, + "objective/train/unlikelihood_loss": 0.023493703454732895, + "theoretical_loss": 3.4871904681810575, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549029048306853, + "loss": 1.2464, + "theoretical_loss": 3.4871904681810575, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548627828598941, + "loss": 1.4215, + "theoretical_loss": 3.4871692276227897, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548226608891029, + "loss": 1.3631, + "theoretical_loss": 3.487147989028946, + "tokens_seen": 1814102016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045478253891831165, + "loss": 1.3222, + "theoretical_loss": 3.487126752399204, + "tokens_seen": 1814233088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547424169475205, + "loss": 1.2831, + "theoretical_loss": 3.48710551773324, + "tokens_seen": 1814364160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045470229497672927, + "loss": 1.3912, + "theoretical_loss": 3.48708428503073, + "tokens_seen": 1814495232 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454662173005938, + "loss": 1.4139, + "theoretical_loss": 3.487063054291351, + "tokens_seen": 1814626304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546220510351469, + "loss": 1.3046, + "theoretical_loss": 3.4870418255147793, + "tokens_seen": 1814757376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045458192906435563, + "loss": 1.4122, + "theoretical_loss": 3.4870205987006924, + "tokens_seen": 1814888448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045454180709356444, + "loss": 1.3145, + "theoretical_loss": 3.486999373848767, + "tokens_seen": 1815019520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045450168512277324, + "loss": 1.3693, + "theoretical_loss": 3.486978150958679, + "tokens_seen": 1815150592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045446156315198205, + "loss": 1.3553, + "theoretical_loss": 3.486956930030107, + "tokens_seen": 1815281664 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045442144118119086, + "loss": 1.3264, + "theoretical_loss": 3.486935711062727, + "tokens_seen": 1815412736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543813192103996, + "loss": 1.3738, + "theoretical_loss": 3.486914494056217, + "tokens_seen": 1815543808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543411972396084, + "loss": 1.3006, + "theoretical_loss": 3.4868932790102534, + "tokens_seen": 1815674880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543010752688172, + "loss": 1.3193, + "theoretical_loss": 3.486872065924514, + "tokens_seen": 1815805952 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454260953298026, + "loss": 1.3115, + "theoretical_loss": 3.4868508547986754, + "tokens_seen": 1815937024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542208313272348, + "loss": 1.2724, + "theoretical_loss": 3.4868296456324166, + "tokens_seen": 1816068096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045418070935644364, + "loss": 1.2586, + "theoretical_loss": 3.486808438425414, + "tokens_seen": 1816199168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541405873856524, + "loss": 1.3418, + "theoretical_loss": 3.4867872331773455, + "tokens_seen": 1816330240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045410046541486114, + "loss": 1.3914, + "theoretical_loss": 3.486766029887889, + "tokens_seen": 1816461312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045406034344407, + "loss": 1.2518, + "theoretical_loss": 3.4867448285567226, + "tokens_seen": 1816592384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045402022147327876, + "loss": 1.3325, + "theoretical_loss": 3.486723629183523, + "tokens_seen": 1816723456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045398009950248756, + "loss": 1.3315, + "theoretical_loss": 3.486702431767969, + "tokens_seen": 1816854528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045393997753169637, + "loss": 1.2909, + "theoretical_loss": 3.486681236309739, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538998555609052, + "loss": 1.4139, + "theoretical_loss": 3.48666004280851, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453859733590114, + "loss": 1.3175, + "theoretical_loss": 3.486638851263961, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045381961161932273, + "loss": 1.3494, + "theoretical_loss": 3.48661766167577, + "tokens_seen": 1817378816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045377948964853154, + "loss": 1.3082, + "theoretical_loss": 3.486596474043615, + "tokens_seen": 1817509888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045373936767774035, + "loss": 1.2705, + "theoretical_loss": 3.486575288367175, + "tokens_seen": 1817640960 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045369924570694915, + "loss": 1.295, + "theoretical_loss": 3.486554104646128, + "tokens_seen": 1817772032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536591237361579, + "loss": 1.2572, + "theoretical_loss": 3.4865329228801523, + "tokens_seen": 1817903104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045361900176536676, + "loss": 1.302, + "theoretical_loss": 3.4865117430689274, + "tokens_seen": 1818034176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535788797945755, + "loss": 1.2486, + "theoretical_loss": 3.486490565212131, + "tokens_seen": 1818165248 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045353875782378427, + "loss": 1.3923, + "theoretical_loss": 3.486469389309443, + "tokens_seen": 1818296320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045349863585299313, + "loss": 1.3634, + "theoretical_loss": 3.4864482153605416, + "tokens_seen": 1818427392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534585138822019, + "loss": 1.2862, + "theoretical_loss": 3.4864270433651052, + "tokens_seen": 1818558464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534183919114107, + "loss": 1.3844, + "theoretical_loss": 3.486405873322814, + "tokens_seen": 1818689536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533782699406195, + "loss": 1.3391, + "theoretical_loss": 3.486384705233346, + "tokens_seen": 1818820608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533381479698283, + "loss": 1.3687, + "theoretical_loss": 3.486363539096381, + "tokens_seen": 1818951680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045329802599903705, + "loss": 1.3934, + "theoretical_loss": 3.4863423749115983, + "tokens_seen": 1819082752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532579040282459, + "loss": 1.3324, + "theoretical_loss": 3.4863212126786762, + "tokens_seen": 1819213824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045321778205745467, + "loss": 1.294, + "theoretical_loss": 3.4863000523972953, + "tokens_seen": 1819344896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045317766008666347, + "loss": 1.3265, + "theoretical_loss": 3.4862788940671345, + "tokens_seen": 1819475968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531375381158723, + "loss": 1.4165, + "theoretical_loss": 3.486257737687873, + "tokens_seen": 1819607040 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045309741614508103, + "loss": 1.3181, + "theoretical_loss": 3.486236583259191, + "tokens_seen": 1819738112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530572941742899, + "loss": 1.3504, + "theoretical_loss": 3.486215430780769, + "tokens_seen": 1819869184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045301717220349864, + "loss": 1.2983, + "theoretical_loss": 3.4861942802522843, + "tokens_seen": 1820000256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045297705023270745, + "loss": 1.306, + "theoretical_loss": 3.486173131673419, + "tokens_seen": 1820131328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045293692826191626, + "loss": 1.3818, + "theoretical_loss": 3.4861519850438514, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9930785298347473, + "objective/train/avg_token_score": 0.006921479944139719, + "objective/train/docs_used": 664244, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3807071447372437, + "objective/train/original_loss": 1.3911049365997314, + "objective/train/theoretical_loss": 3.486130840363263, + "objective/train/tokens_used": 190852576, + "objective/train/unlikelihood_loss": 0.01864830031991005, + "theoretical_loss": 3.486130840363263, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452896806291125, + "loss": 1.3384, + "theoretical_loss": 3.486130840363263, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528566843203338, + "loss": 1.3981, + "theoretical_loss": 3.486109697631333, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528165623495426, + "loss": 1.3789, + "theoretical_loss": 3.4860885568477413, + "tokens_seen": 1820655616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527764403787514, + "loss": 1.2246, + "theoretical_loss": 3.486067418012169, + "tokens_seen": 1820786688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527363184079602, + "loss": 1.4394, + "theoretical_loss": 3.4860462811242954, + "tokens_seen": 1820917760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045269619643716904, + "loss": 1.3639, + "theoretical_loss": 3.4860251461838017, + "tokens_seen": 1821048832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526560744663778, + "loss": 1.3028, + "theoretical_loss": 3.486004013190368, + "tokens_seen": 1821179904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045261595249558654, + "loss": 1.3829, + "theoretical_loss": 3.4859828821436745, + "tokens_seen": 1821310976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525758305247954, + "loss": 1.3621, + "theoretical_loss": 3.4859617530434024, + "tokens_seen": 1821442048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045253570855400416, + "loss": 1.3283, + "theoretical_loss": 3.4859406258892314, + "tokens_seen": 1821573120 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452495586583213, + "loss": 1.3157, + "theoretical_loss": 3.4859195006808434, + "tokens_seen": 1821704192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045245546461242177, + "loss": 1.3099, + "theoretical_loss": 3.485898377417919, + "tokens_seen": 1821835264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524153426416306, + "loss": 1.3821, + "theoretical_loss": 3.485877256100138, + "tokens_seen": 1821966336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523752206708394, + "loss": 1.2884, + "theoretical_loss": 3.4858561367271825, + "tokens_seen": 1822097408 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045233509870004813, + "loss": 1.4091, + "theoretical_loss": 3.4858350192987335, + "tokens_seen": 1822228480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045229497672925694, + "loss": 1.3485, + "theoretical_loss": 3.4858139038144715, + "tokens_seen": 1822359552 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045225485475846575, + "loss": 1.4074, + "theoretical_loss": 3.485792790274078, + "tokens_seen": 1822490624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045221473278767455, + "loss": 1.4203, + "theoretical_loss": 3.4857716786772346, + "tokens_seen": 1822621696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521746108168833, + "loss": 1.3985, + "theoretical_loss": 3.4857505690236223, + "tokens_seen": 1822752768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045213448884609216, + "loss": 1.2992, + "theoretical_loss": 3.485729461312922, + "tokens_seen": 1822883840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520943668753009, + "loss": 1.3564, + "theoretical_loss": 3.485708355544816, + "tokens_seen": 1823014912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045205424490450967, + "loss": 1.3152, + "theoretical_loss": 3.4856872517189856, + "tokens_seen": 1823145984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045201412293371853, + "loss": 1.3376, + "theoretical_loss": 3.4856661498351125, + "tokens_seen": 1823277056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519740009629273, + "loss": 1.2985, + "theoretical_loss": 3.485645049892878, + "tokens_seen": 1823408128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045193387899213614, + "loss": 1.3249, + "theoretical_loss": 3.4856239518919647, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518937570213449, + "loss": 1.3793, + "theoretical_loss": 3.485602855832054, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518536350505537, + "loss": 1.3429, + "theoretical_loss": 3.4855817617128277, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518135130797625, + "loss": 1.3707, + "theoretical_loss": 3.4855606695339683, + "tokens_seen": 1823932416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045177339110897126, + "loss": 1.3588, + "theoretical_loss": 3.4855395792951573, + "tokens_seen": 1824063488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173326913818006, + "loss": 1.3548, + "theoretical_loss": 3.4855184909960775, + "tokens_seen": 1824194560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045169314716738887, + "loss": 1.285, + "theoretical_loss": 3.4854974046364102, + "tokens_seen": 1824325632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516530251965977, + "loss": 1.3559, + "theoretical_loss": 3.4854763202158385, + "tokens_seen": 1824456704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045161290322580643, + "loss": 1.3037, + "theoretical_loss": 3.485455237734045, + "tokens_seen": 1824587776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515727812550153, + "loss": 1.3052, + "theoretical_loss": 3.485434157190711, + "tokens_seen": 1824718848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045153265928422404, + "loss": 1.2295, + "theoretical_loss": 3.4854130785855206, + "tokens_seen": 1824849920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045149253731343285, + "loss": 1.3492, + "theoretical_loss": 3.485392001918155, + "tokens_seen": 1824980992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045145241534264165, + "loss": 1.28, + "theoretical_loss": 3.485370927188298, + "tokens_seen": 1825112064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514122933718504, + "loss": 1.3629, + "theoretical_loss": 3.485349854395632, + "tokens_seen": 1825243136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513721714010592, + "loss": 1.3149, + "theoretical_loss": 3.4853287835398388, + "tokens_seen": 1825374208 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451332049430268, + "loss": 1.3456, + "theoretical_loss": 3.485307714620603, + "tokens_seen": 1825505280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512919274594768, + "loss": 1.3241, + "theoretical_loss": 3.4852866476376065, + "tokens_seen": 1825636352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125180548868563, + "loss": 1.3479, + "theoretical_loss": 3.4852655825905328, + "tokens_seen": 1825767424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045121168351789444, + "loss": 1.2733, + "theoretical_loss": 3.485244519479065, + "tokens_seen": 1825898496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511715615471032, + "loss": 1.3507, + "theoretical_loss": 3.4852234583028863, + "tokens_seen": 1826029568 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451131439576312, + "loss": 1.4115, + "theoretical_loss": 3.48520239906168, + "tokens_seen": 1826160640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510913176055208, + "loss": 1.3314, + "theoretical_loss": 3.485181341755129, + "tokens_seen": 1826291712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045105119563472956, + "loss": 1.2909, + "theoretical_loss": 3.4851602863829174, + "tokens_seen": 1826422784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510110736639384, + "loss": 1.2824, + "theoretical_loss": 3.4851392329447286, + "tokens_seen": 1826553856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045097095169314717, + "loss": 1.352, + "theoretical_loss": 3.485118181440246, + "tokens_seen": 1826684928 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450930829722356, + "loss": 1.3014, + "theoretical_loss": 3.485097131869153, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9935856461524963, + "objective/train/avg_token_score": 0.006414412520825863, + "objective/train/docs_used": 666029, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.324583649635315, + "objective/train/original_loss": 1.3324058055877686, + "objective/train/theoretical_loss": 3.485076084231134, + "objective/train/tokens_used": 197406176, + "objective/train/unlikelihood_loss": 0.02087901346385479, + "theoretical_loss": 3.485076084231134, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508907077515648, + "loss": 1.2565, + "theoretical_loss": 3.485076084231134, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045085058578077353, + "loss": 1.311, + "theoretical_loss": 3.4850550385258723, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045081046380998234, + "loss": 1.4242, + "theoretical_loss": 3.485033994753052, + "tokens_seen": 1827209216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045077034183919115, + "loss": 1.3127, + "theoretical_loss": 3.4850129529123564, + "tokens_seen": 1827340288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045073021986839995, + "loss": 1.3318, + "theoretical_loss": 3.484991913003471, + "tokens_seen": 1827471360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045069009789760876, + "loss": 1.3354, + "theoretical_loss": 3.484970875026079, + "tokens_seen": 1827602432 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045064997592681756, + "loss": 1.3623, + "theoretical_loss": 3.4849498389798637, + "tokens_seen": 1827733504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506098539560263, + "loss": 1.319, + "theoretical_loss": 3.484928804864511, + "tokens_seen": 1827864576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505697319852351, + "loss": 1.3546, + "theoretical_loss": 3.484907772679705, + "tokens_seen": 1827995648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045052961001444393, + "loss": 1.299, + "theoretical_loss": 3.484886742425129, + "tokens_seen": 1828126720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504894880436527, + "loss": 1.3629, + "theoretical_loss": 3.4848657141004677, + "tokens_seen": 1828257792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045044936607286154, + "loss": 1.3467, + "theoretical_loss": 3.4848446877054062, + "tokens_seen": 1828388864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504092441020703, + "loss": 1.3322, + "theoretical_loss": 3.4848236632396294, + "tokens_seen": 1828519936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503691221312791, + "loss": 1.3926, + "theoretical_loss": 3.484802640702821, + "tokens_seen": 1828651008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503290001604879, + "loss": 1.3219, + "theoretical_loss": 3.484781620094667, + "tokens_seen": 1828782080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045028887818969666, + "loss": 1.2862, + "theoretical_loss": 3.4847606014148518, + "tokens_seen": 1828913152 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045024875621890546, + "loss": 1.2731, + "theoretical_loss": 3.4847395846630596, + "tokens_seen": 1829044224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045020863424811427, + "loss": 1.2407, + "theoretical_loss": 3.484718569838976, + "tokens_seen": 1829175296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501685122773231, + "loss": 1.2901, + "theoretical_loss": 3.484697556942286, + "tokens_seen": 1829306368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045012839030653183, + "loss": 1.3034, + "theoretical_loss": 3.4846765459726745, + "tokens_seen": 1829437440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500882683357407, + "loss": 1.3095, + "theoretical_loss": 3.484655536929827, + "tokens_seen": 1829568512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045004814636494944, + "loss": 1.3062, + "theoretical_loss": 3.484634529813429, + "tokens_seen": 1829699584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500080243941583, + "loss": 1.3315, + "theoretical_loss": 3.4846135246231658, + "tokens_seen": 1829830656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044996790242336705, + "loss": 1.3696, + "theoretical_loss": 3.484592521358722, + "tokens_seen": 1829961728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499277804525758, + "loss": 1.3329, + "theoretical_loss": 3.4845715200197844, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044988765848178467, + "loss": 1.2819, + "theoretical_loss": 3.484550520606038, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498475365109934, + "loss": 1.333, + "theoretical_loss": 3.484529523117168, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498074145402022, + "loss": 1.2566, + "theoretical_loss": 3.484508527552861, + "tokens_seen": 1830486016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044976729256941103, + "loss": 1.248, + "theoretical_loss": 3.4844875339128016, + "tokens_seen": 1830617088 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044972717059861984, + "loss": 1.3498, + "theoretical_loss": 3.484466542196677, + "tokens_seen": 1830748160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496870486278286, + "loss": 1.2872, + "theoretical_loss": 3.484445552404173, + "tokens_seen": 1830879232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496469266570374, + "loss": 1.3195, + "theoretical_loss": 3.484424564534975, + "tokens_seen": 1831010304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496068046862462, + "loss": 1.3804, + "theoretical_loss": 3.4844035785887693, + "tokens_seen": 1831141376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044956668271545495, + "loss": 1.298, + "theoretical_loss": 3.4843825945652425, + "tokens_seen": 1831272448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495265607446638, + "loss": 1.3349, + "theoretical_loss": 3.4843616124640797, + "tokens_seen": 1831403520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044948643877387257, + "loss": 1.2337, + "theoretical_loss": 3.484340632284969, + "tokens_seen": 1831534592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044944631680308143, + "loss": 1.2933, + "theoretical_loss": 3.484319654027595, + "tokens_seen": 1831665664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494061948322902, + "loss": 1.3039, + "theoretical_loss": 3.484298677691645, + "tokens_seen": 1831796736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044936607286149893, + "loss": 1.3209, + "theoretical_loss": 3.484277703276806, + "tokens_seen": 1831927808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493259508907078, + "loss": 1.3457, + "theoretical_loss": 3.4842567307827643, + "tokens_seen": 1832058880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044928582891991654, + "loss": 1.3198, + "theoretical_loss": 3.484235760209206, + "tokens_seen": 1832189952 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044924570694912535, + "loss": 1.384, + "theoretical_loss": 3.4842147915558184, + "tokens_seen": 1832321024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044920558497833416, + "loss": 1.3304, + "theoretical_loss": 3.4841938248222886, + "tokens_seen": 1832452096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044916546300754296, + "loss": 1.3774, + "theoretical_loss": 3.484172860008303, + "tokens_seen": 1832583168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491253410367517, + "loss": 1.3386, + "theoretical_loss": 3.4841518971135486, + "tokens_seen": 1832714240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490852190659605, + "loss": 1.2741, + "theoretical_loss": 3.484130936137713, + "tokens_seen": 1832845312 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044904509709516933, + "loss": 1.3179, + "theoretical_loss": 3.484109977080483, + "tokens_seen": 1832976384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490049751243781, + "loss": 1.2808, + "theoretical_loss": 3.4840890199415453, + "tokens_seen": 1833107456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044896485315358694, + "loss": 1.3395, + "theoretical_loss": 3.484068064720588, + "tokens_seen": 1833238528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489247311827957, + "loss": 1.3991, + "theoretical_loss": 3.484047111417298, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9920766353607178, + "objective/train/avg_token_score": 0.00792335532605648, + "objective/train/docs_used": 668449, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3440781831741333, + "objective/train/original_loss": 1.3555406332015991, + "objective/train/theoretical_loss": 3.4840261600313633, + "objective/train/tokens_used": 203959776, + "objective/train/unlikelihood_loss": 0.022288914769887924, + "theoretical_loss": 3.4840261600313633, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488846092120045, + "loss": 1.3581, + "theoretical_loss": 3.4840261600313633, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488444872412133, + "loss": 1.3847, + "theoretical_loss": 3.4840052105624704, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044880436527042206, + "loss": 1.33, + "theoretical_loss": 3.483984263010308, + "tokens_seen": 1833762816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487642432996309, + "loss": 1.2927, + "theoretical_loss": 3.4839633173745628, + "tokens_seen": 1833893888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044872412132883967, + "loss": 1.3086, + "theoretical_loss": 3.483942373654923, + "tokens_seen": 1834024960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486839993580485, + "loss": 1.3216, + "theoretical_loss": 3.4839214318510763, + "tokens_seen": 1834156032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486438773872573, + "loss": 1.3694, + "theoretical_loss": 3.483900491962711, + "tokens_seen": 1834287104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486037554164661, + "loss": 1.3223, + "theoretical_loss": 3.4838795539895138, + "tokens_seen": 1834418176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044856363344567484, + "loss": 1.4028, + "theoretical_loss": 3.483858617931174, + "tokens_seen": 1834549248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485235114748837, + "loss": 1.3627, + "theoretical_loss": 3.483837683787379, + "tokens_seen": 1834680320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044848338950409245, + "loss": 1.2339, + "theoretical_loss": 3.4838167515578173, + "tokens_seen": 1834811392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484432675333012, + "loss": 1.227, + "theoretical_loss": 3.483795821242177, + "tokens_seen": 1834942464 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044840314556251007, + "loss": 1.3035, + "theoretical_loss": 3.4837748928401466, + "tokens_seen": 1835073536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483630235917188, + "loss": 1.3279, + "theoretical_loss": 3.483753966351414, + "tokens_seen": 1835204608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483229016209276, + "loss": 1.3016, + "theoretical_loss": 3.4837330417756682, + "tokens_seen": 1835335680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828277965013643, + "loss": 1.2476, + "theoretical_loss": 3.4837121191125977, + "tokens_seen": 1835466752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044824265767934524, + "loss": 1.359, + "theoretical_loss": 3.483691198361891, + "tokens_seen": 1835597824 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448202535708554, + "loss": 1.2884, + "theoretical_loss": 3.4836702795232357, + "tokens_seen": 1835728896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481624137377628, + "loss": 1.3283, + "theoretical_loss": 3.483649362596322, + "tokens_seen": 1835859968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481222917669716, + "loss": 1.3845, + "theoretical_loss": 3.4836284475808386, + "tokens_seen": 1835991040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480821697961804, + "loss": 1.3323, + "theoretical_loss": 3.4836075344764734, + "tokens_seen": 1836122112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480420478253892, + "loss": 1.2629, + "theoretical_loss": 3.4835866232829167, + "tokens_seen": 1836253184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044800192585459797, + "loss": 1.2975, + "theoretical_loss": 3.483565713999856, + "tokens_seen": 1836384256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044796180388380683, + "loss": 1.3071, + "theoretical_loss": 3.4835448066269814, + "tokens_seen": 1836515328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479216819130156, + "loss": 1.3046, + "theoretical_loss": 3.4835239011639825, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044788155994222433, + "loss": 1.3532, + "theoretical_loss": 3.4835029976105476, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478414379714332, + "loss": 1.2858, + "theoretical_loss": 3.483482095966366, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044780131600064194, + "loss": 1.3467, + "theoretical_loss": 3.483461196231128, + "tokens_seen": 1837039616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044776119402985075, + "loss": 1.3829, + "theoretical_loss": 3.4834402984045223, + "tokens_seen": 1837170688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044772107205905956, + "loss": 1.3253, + "theoretical_loss": 3.4834194024862386, + "tokens_seen": 1837301760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044768095008826836, + "loss": 1.2427, + "theoretical_loss": 3.483398508475967, + "tokens_seen": 1837432832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476408281174771, + "loss": 1.3848, + "theoretical_loss": 3.4833776163733967, + "tokens_seen": 1837563904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476007061466859, + "loss": 1.4299, + "theoretical_loss": 3.4833567261782177, + "tokens_seen": 1837694976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044756058417589473, + "loss": 1.3344, + "theoretical_loss": 3.4833358378901194, + "tokens_seen": 1837826048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475204622051035, + "loss": 1.3566, + "theoretical_loss": 3.483314951508792, + "tokens_seen": 1837957120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044748034023431234, + "loss": 1.3648, + "theoretical_loss": 3.483294067033926, + "tokens_seen": 1838088192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474402182635211, + "loss": 1.289, + "theoretical_loss": 3.48327318446521, + "tokens_seen": 1838219264 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044740009629272995, + "loss": 1.3155, + "theoretical_loss": 3.4832523038023355, + "tokens_seen": 1838350336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473599743219387, + "loss": 1.2508, + "theoretical_loss": 3.483231425044993, + "tokens_seen": 1838481408 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044731985235114746, + "loss": 1.3094, + "theoretical_loss": 3.4832105481928712, + "tokens_seen": 1838612480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472797303803563, + "loss": 1.323, + "theoretical_loss": 3.4831896732456613, + "tokens_seen": 1838743552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044723960840956507, + "loss": 1.2658, + "theoretical_loss": 3.483168800203054, + "tokens_seen": 1838874624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471994864387739, + "loss": 1.3104, + "theoretical_loss": 3.4831479290647396, + "tokens_seen": 1839005696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471593644679827, + "loss": 1.3494, + "theoretical_loss": 3.4831270598304087, + "tokens_seen": 1839136768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471192424971915, + "loss": 1.3642, + "theoretical_loss": 3.4831061924997515, + "tokens_seen": 1839267840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044707912052640024, + "loss": 1.2736, + "theoretical_loss": 3.483085327072459, + "tokens_seen": 1839398912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044703899855560905, + "loss": 1.3452, + "theoretical_loss": 3.4830644635482217, + "tokens_seen": 1839529984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044699887658481785, + "loss": 1.3072, + "theoretical_loss": 3.483043601926731, + "tokens_seen": 1839661056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469587546140266, + "loss": 1.3117, + "theoretical_loss": 3.4830227422076776, + "tokens_seen": 1839792128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044691863264323547, + "loss": 1.2993, + "theoretical_loss": 3.4830018843907524, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9910498261451721, + "objective/train/avg_token_score": 0.008950138464570045, + "objective/train/docs_used": 670853, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.246282696723938, + "objective/train/original_loss": 1.259093165397644, + "objective/train/theoretical_loss": 3.4829810284756464, + "objective/train/tokens_used": 210513376, + "objective/train/unlikelihood_loss": 0.025417782366275787, + "theoretical_loss": 3.4829810284756464, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468785106724442, + "loss": 1.306, + "theoretical_loss": 3.4829810284756464, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468383887016531, + "loss": 1.3202, + "theoretical_loss": 3.4829601744620513, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044679826673086183, + "loss": 1.3506, + "theoretical_loss": 3.482939322349657, + "tokens_seen": 1840316416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044675814476007064, + "loss": 1.2805, + "theoretical_loss": 3.4829184721381568, + "tokens_seen": 1840447488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044671802278927944, + "loss": 1.3024, + "theoretical_loss": 3.48289762382724, + "tokens_seen": 1840578560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466779008184882, + "loss": 1.317, + "theoretical_loss": 3.482876777416599, + "tokens_seen": 1840709632 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446637778847697, + "loss": 1.2783, + "theoretical_loss": 3.4828559329059257, + "tokens_seen": 1840840704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465976568769058, + "loss": 1.3729, + "theoretical_loss": 3.482835090294911, + "tokens_seen": 1840971776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465575349061146, + "loss": 1.2872, + "theoretical_loss": 3.4828142495832464, + "tokens_seen": 1841102848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044651741293532337, + "loss": 1.3107, + "theoretical_loss": 3.482793410770624, + "tokens_seen": 1841233920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044647729096453223, + "loss": 1.3328, + "theoretical_loss": 3.482772573856736, + "tokens_seen": 1841364992 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446437168993741, + "loss": 1.3802, + "theoretical_loss": 3.4827517388412734, + "tokens_seen": 1841496064 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044639704702294973, + "loss": 1.3477, + "theoretical_loss": 3.482730905723929, + "tokens_seen": 1841627136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463569250521586, + "loss": 1.2515, + "theoretical_loss": 3.4827100745043937, + "tokens_seen": 1841758208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044631680308136734, + "loss": 1.3227, + "theoretical_loss": 3.4826892451823603, + "tokens_seen": 1841889280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044627668111057615, + "loss": 1.2656, + "theoretical_loss": 3.4826684177575213, + "tokens_seen": 1842020352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044623655913978496, + "loss": 1.3471, + "theoretical_loss": 3.4826475922295677, + "tokens_seen": 1842151424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044619643716899376, + "loss": 1.3614, + "theoretical_loss": 3.4826267685981933, + "tokens_seen": 1842282496 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044615631519820257, + "loss": 1.3238, + "theoretical_loss": 3.4826059468630897, + "tokens_seen": 1842413568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461161932274113, + "loss": 1.3522, + "theoretical_loss": 3.482585127023949, + "tokens_seen": 1842544640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044607607125662013, + "loss": 1.3885, + "theoretical_loss": 3.4825643090804643, + "tokens_seen": 1842675712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044603594928582893, + "loss": 1.3105, + "theoretical_loss": 3.4825434930323276, + "tokens_seen": 1842806784 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044599582731503774, + "loss": 1.3294, + "theoretical_loss": 3.4825226788792323, + "tokens_seen": 1842937856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459557053442465, + "loss": 1.3915, + "theoretical_loss": 3.4825018666208702, + "tokens_seen": 1843068928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591558337345535, + "loss": 1.2772, + "theoretical_loss": 3.482481056256935, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458754614026641, + "loss": 1.3375, + "theoretical_loss": 3.4824602477871194, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044583533943187286, + "loss": 1.3229, + "theoretical_loss": 3.482439441211116, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457952174610817, + "loss": 1.2761, + "theoretical_loss": 3.4824186365286174, + "tokens_seen": 1843593216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575509549029047, + "loss": 1.347, + "theoretical_loss": 3.4823978337393173, + "tokens_seen": 1843724288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457149735194993, + "loss": 1.3084, + "theoretical_loss": 3.4823770328429093, + "tokens_seen": 1843855360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456748515487081, + "loss": 1.3653, + "theoretical_loss": 3.4823562338390857, + "tokens_seen": 1843986432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456347295779169, + "loss": 1.3678, + "theoretical_loss": 3.4823354367275403, + "tokens_seen": 1844117504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455946076071257, + "loss": 1.3757, + "theoretical_loss": 3.482314641507966, + "tokens_seen": 1844248576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044555448563633445, + "loss": 1.3045, + "theoretical_loss": 3.4822938481800567, + "tokens_seen": 1844379648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044551436366554325, + "loss": 1.306, + "theoretical_loss": 3.482273056743506, + "tokens_seen": 1844510720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044547424169475206, + "loss": 1.3037, + "theoretical_loss": 3.4822522671980067, + "tokens_seen": 1844641792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044543411972396087, + "loss": 1.3783, + "theoretical_loss": 3.4822314795432536, + "tokens_seen": 1844772864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453939977531696, + "loss": 1.2522, + "theoretical_loss": 3.4822106937789394, + "tokens_seen": 1844903936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453538757823785, + "loss": 1.3413, + "theoretical_loss": 3.4821899099047586, + "tokens_seen": 1845035008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531375381158723, + "loss": 1.3433, + "theoretical_loss": 3.482169127920405, + "tokens_seen": 1845166080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527363184079604, + "loss": 1.3672, + "theoretical_loss": 3.482148347825572, + "tokens_seen": 1845297152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044523350987000484, + "loss": 1.2878, + "theoretical_loss": 3.482127569619954, + "tokens_seen": 1845428224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451933878992136, + "loss": 1.2936, + "theoretical_loss": 3.482106793303245, + "tokens_seen": 1845559296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451532659284224, + "loss": 1.3835, + "theoretical_loss": 3.482086018875139, + "tokens_seen": 1845690368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451131439576312, + "loss": 1.2737, + "theoretical_loss": 3.4820652463353303, + "tokens_seen": 1845821440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044507302198684, + "loss": 1.3894, + "theoretical_loss": 3.4820444756835136, + "tokens_seen": 1845952512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044503290001604877, + "loss": 1.3904, + "theoretical_loss": 3.482023706919383, + "tokens_seen": 1846083584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449927780452576, + "loss": 1.3437, + "theoretical_loss": 3.4820029400426327, + "tokens_seen": 1846214656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449526560744664, + "loss": 1.2656, + "theoretical_loss": 3.481982175052957, + "tokens_seen": 1846345728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449125341036752, + "loss": 1.3984, + "theoretical_loss": 3.481961411950051, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9902796149253845, + "objective/train/avg_token_score": 0.009720345959067345, + "objective/train/docs_used": 673286, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4028799533843994, + "objective/train/original_loss": 1.4180868864059448, + "objective/train/theoretical_loss": 3.4819406507336095, + "objective/train/tokens_used": 217066976, + "objective/train/unlikelihood_loss": 0.02417025715112686, + "theoretical_loss": 3.4819406507336095, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444872412132884, + "loss": 1.3734, + "theoretical_loss": 3.4819406507336095, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044483229016209274, + "loss": 1.3172, + "theoretical_loss": 3.4819198914033267, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447921681913016, + "loss": 1.334, + "theoretical_loss": 3.4818991339588976, + "tokens_seen": 1846870016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044475204622051036, + "loss": 1.3461, + "theoretical_loss": 3.481878378400017, + "tokens_seen": 1847001088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044471192424971916, + "loss": 1.3477, + "theoretical_loss": 3.4818576247263806, + "tokens_seen": 1847132160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044467180227892797, + "loss": 1.3366, + "theoretical_loss": 3.4818368729376816, + "tokens_seen": 1847263232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446316803081367, + "loss": 1.3261, + "theoretical_loss": 3.4818161230336173, + "tokens_seen": 1847394304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044459155833734553, + "loss": 1.312, + "theoretical_loss": 3.4817953750138813, + "tokens_seen": 1847525376 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044455143636655433, + "loss": 1.2741, + "theoretical_loss": 3.481774628878169, + "tokens_seen": 1847656448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044451131439576314, + "loss": 1.4064, + "theoretical_loss": 3.4817538846261757, + "tokens_seen": 1847787520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444711924249719, + "loss": 1.3669, + "theoretical_loss": 3.4817331422575974, + "tokens_seen": 1847918592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044443107045418075, + "loss": 1.3683, + "theoretical_loss": 3.4817124017721293, + "tokens_seen": 1848049664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443909484833895, + "loss": 1.3227, + "theoretical_loss": 3.4816916631694665, + "tokens_seen": 1848180736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044435082651259826, + "loss": 1.3226, + "theoretical_loss": 3.4816709264493046, + "tokens_seen": 1848311808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443107045418071, + "loss": 1.3198, + "theoretical_loss": 3.4816501916113394, + "tokens_seen": 1848442880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044427058257101587, + "loss": 1.3512, + "theoretical_loss": 3.481629458655267, + "tokens_seen": 1848573952 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044423046060022473, + "loss": 1.3156, + "theoretical_loss": 3.4816087275807828, + "tokens_seen": 1848705024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441903386294335, + "loss": 1.4159, + "theoretical_loss": 3.4815879983875826, + "tokens_seen": 1848836096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441502166586423, + "loss": 1.3054, + "theoretical_loss": 3.4815672710753622, + "tokens_seen": 1848967168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441100946878511, + "loss": 1.3202, + "theoretical_loss": 3.481546545643818, + "tokens_seen": 1849098240 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044406997271705985, + "loss": 1.2976, + "theoretical_loss": 3.481525822092646, + "tokens_seen": 1849229312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044402985074626865, + "loss": 1.3585, + "theoretical_loss": 3.4815051004215416, + "tokens_seen": 1849360384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044398972877547746, + "loss": 1.3013, + "theoretical_loss": 3.481484380630202, + "tokens_seen": 1849491456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044394960680468627, + "loss": 1.3743, + "theoretical_loss": 3.4814636627183235, + "tokens_seen": 1849622528 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443909484833895, + "loss": 1.3375, + "theoretical_loss": 3.481442946685602, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438693628631039, + "loss": 1.326, + "theoretical_loss": 3.481422232531733, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044382924089231263, + "loss": 1.3432, + "theoretical_loss": 3.481401520256415, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044378911892152144, + "loss": 1.3785, + "theoretical_loss": 3.481380809859343, + "tokens_seen": 1850146816 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044374899695073024, + "loss": 1.4497, + "theoretical_loss": 3.481360101340214, + "tokens_seen": 1850277888 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443708874979939, + "loss": 1.3069, + "theoretical_loss": 3.4813393946987254, + "tokens_seen": 1850408960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044366875300914786, + "loss": 1.3047, + "theoretical_loss": 3.481318689934573, + "tokens_seen": 1850540032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436286310383566, + "loss": 1.2379, + "theoretical_loss": 3.481297987047454, + "tokens_seen": 1850671104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435885090675654, + "loss": 1.3032, + "theoretical_loss": 3.4812772860370655, + "tokens_seen": 1850802176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435483870967742, + "loss": 1.37, + "theoretical_loss": 3.4812565869031045, + "tokens_seen": 1850933248 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443508265125983, + "loss": 1.3146, + "theoretical_loss": 3.481235889645268, + "tokens_seen": 1851064320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434681431551918, + "loss": 1.3436, + "theoretical_loss": 3.4812151942632523, + "tokens_seen": 1851195392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434280211844006, + "loss": 1.2821, + "theoretical_loss": 3.4811945007567564, + "tokens_seen": 1851326464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433878992136094, + "loss": 1.2924, + "theoretical_loss": 3.4811738091254756, + "tokens_seen": 1851457536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044334777724281814, + "loss": 1.2778, + "theoretical_loss": 3.4811531193691083, + "tokens_seen": 1851588608 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443307655272027, + "loss": 1.3001, + "theoretical_loss": 3.481132431487352, + "tokens_seen": 1851719680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044326753330123576, + "loss": 1.3268, + "theoretical_loss": 3.481111745479904, + "tokens_seen": 1851850752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044322741133044456, + "loss": 1.2969, + "theoretical_loss": 3.4810910613464614, + "tokens_seen": 1851981824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044318728935965337, + "loss": 1.3376, + "theoretical_loss": 3.481070379086722, + "tokens_seen": 1852112896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431471673888621, + "loss": 1.3496, + "theoretical_loss": 3.4810496987003843, + "tokens_seen": 1852243968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431070454180709, + "loss": 1.2954, + "theoretical_loss": 3.4810290201871448, + "tokens_seen": 1852375040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044306692344727973, + "loss": 1.3618, + "theoretical_loss": 3.4810083435467023, + "tokens_seen": 1852506112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044302680147648854, + "loss": 1.3705, + "theoretical_loss": 3.480987668778754, + "tokens_seen": 1852637184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044298667950569735, + "loss": 1.2835, + "theoretical_loss": 3.4809669958829983, + "tokens_seen": 1852768256 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294655753490615, + "loss": 1.2701, + "theoretical_loss": 3.4809463248591332, + "tokens_seen": 1852899328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429064355641149, + "loss": 1.3525, + "theoretical_loss": 3.480925655706857, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9897891879081726, + "objective/train/avg_token_score": 0.010210780426859856, + "objective/train/docs_used": 675642, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.357620358467102, + "objective/train/original_loss": 1.3720636367797852, + "objective/train/theoretical_loss": 3.480904988425867, + "objective/train/tokens_used": 223620576, + "objective/train/unlikelihood_loss": 0.027902085334062576, + "theoretical_loss": 3.480904988425867, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428663135933237, + "loss": 1.3675, + "theoretical_loss": 3.480904988425867, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428261916225325, + "loss": 1.3484, + "theoretical_loss": 3.4808843230158626, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044278606965174127, + "loss": 1.3358, + "theoretical_loss": 3.480863659476541, + "tokens_seen": 1853423616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044274594768095013, + "loss": 1.3104, + "theoretical_loss": 3.480842997807602, + "tokens_seen": 1853554688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427058257101589, + "loss": 1.3117, + "theoretical_loss": 3.4808223380087426, + "tokens_seen": 1853685760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426657037393677, + "loss": 1.2699, + "theoretical_loss": 3.4808016800796624, + "tokens_seen": 1853816832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426255817685765, + "loss": 1.3631, + "theoretical_loss": 3.480781024020059, + "tokens_seen": 1853947904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044258545979778525, + "loss": 1.3275, + "theoretical_loss": 3.480760369829632, + "tokens_seen": 1854078976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044254533782699405, + "loss": 1.3044, + "theoretical_loss": 3.4807397175080794, + "tokens_seen": 1854210048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044250521585620286, + "loss": 1.3387, + "theoretical_loss": 3.4807190670551007, + "tokens_seen": 1854341120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246509388541167, + "loss": 1.3022, + "theoretical_loss": 3.480698418470394, + "tokens_seen": 1854472192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424249719146204, + "loss": 1.2688, + "theoretical_loss": 3.480677771753659, + "tokens_seen": 1854603264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423848499438293, + "loss": 1.2813, + "theoretical_loss": 3.4806571269045943, + "tokens_seen": 1854734336 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044234472797303803, + "loss": 1.2996, + "theoretical_loss": 3.480636483922899, + "tokens_seen": 1854865408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230460600224684, + "loss": 1.3555, + "theoretical_loss": 3.4806158428082723, + "tokens_seen": 1854996480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044226448403145564, + "loss": 1.2892, + "theoretical_loss": 3.480595203560414, + "tokens_seen": 1855127552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422243620606644, + "loss": 1.3522, + "theoretical_loss": 3.480574566179022, + "tokens_seen": 1855258624 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044218424008987326, + "loss": 1.2841, + "theoretical_loss": 3.480553930663797, + "tokens_seen": 1855389696 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442144118119082, + "loss": 1.3592, + "theoretical_loss": 3.4805332970144383, + "tokens_seen": 1855520768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421039961482908, + "loss": 1.3194, + "theoretical_loss": 3.4805126652306444, + "tokens_seen": 1855651840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420638741774996, + "loss": 1.3431, + "theoretical_loss": 3.480492035312116, + "tokens_seen": 1855782912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420237522067084, + "loss": 1.2301, + "theoretical_loss": 3.480471407258552, + "tokens_seen": 1855913984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419836302359172, + "loss": 1.3163, + "theoretical_loss": 3.480450781069652, + "tokens_seen": 1856045056 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441943508265126, + "loss": 1.2696, + "theoretical_loss": 3.4804301567451166, + "tokens_seen": 1856176128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419033862943348, + "loss": 1.2862, + "theoretical_loss": 3.480409534284645, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044186326432354354, + "loss": 1.274, + "theoretical_loss": 3.4803889136879373, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418231423527524, + "loss": 1.381, + "theoretical_loss": 3.480368294954694, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044178302038196116, + "loss": 1.3075, + "theoretical_loss": 3.4803476780846143, + "tokens_seen": 1856700416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044174289841117, + "loss": 1.314, + "theoretical_loss": 3.4803270630773984, + "tokens_seen": 1856831488 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044170277644037877, + "loss": 1.3068, + "theoretical_loss": 3.4803064499327476, + "tokens_seen": 1856962560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416626544695875, + "loss": 1.3436, + "theoretical_loss": 3.480285838650361, + "tokens_seen": 1857093632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416225324987964, + "loss": 1.3548, + "theoretical_loss": 3.480265229229939, + "tokens_seen": 1857224704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044158241052800513, + "loss": 1.2849, + "theoretical_loss": 3.4802446216711824, + "tokens_seen": 1857355776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044154228855721394, + "loss": 1.3516, + "theoretical_loss": 3.480224015973792, + "tokens_seen": 1857486848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044150216658642275, + "loss": 1.358, + "theoretical_loss": 3.4802034121374676, + "tokens_seen": 1857617920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044146204461563155, + "loss": 1.388, + "theoretical_loss": 3.48018281016191, + "tokens_seen": 1857748992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414219226448403, + "loss": 1.2588, + "theoretical_loss": 3.480162210046821, + "tokens_seen": 1857880064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413818006740491, + "loss": 1.3149, + "theoretical_loss": 3.480141611791899, + "tokens_seen": 1858011136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413416787032579, + "loss": 1.3256, + "theoretical_loss": 3.480121015396847, + "tokens_seen": 1858142208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044130155673246667, + "loss": 1.3769, + "theoretical_loss": 3.480100420861365, + "tokens_seen": 1858273280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044126143476167553, + "loss": 1.3462, + "theoretical_loss": 3.480079828185154, + "tokens_seen": 1858404352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412213127908843, + "loss": 1.3524, + "theoretical_loss": 3.4800592373679153, + "tokens_seen": 1858535424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411811908200931, + "loss": 1.2781, + "theoretical_loss": 3.4800386484093497, + "tokens_seen": 1858666496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411410688493019, + "loss": 1.2729, + "theoretical_loss": 3.480018061309158, + "tokens_seen": 1858797568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044110094687851065, + "loss": 1.3465, + "theoretical_loss": 3.4799974760670427, + "tokens_seen": 1858928640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410608249077195, + "loss": 1.2369, + "theoretical_loss": 3.4799768926827035, + "tokens_seen": 1859059712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044102070293692826, + "loss": 1.3962, + "theoretical_loss": 3.479956311155843, + "tokens_seen": 1859190784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044098058096613706, + "loss": 1.2145, + "theoretical_loss": 3.4799357314861625, + "tokens_seen": 1859321856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094045899534587, + "loss": 1.3259, + "theoretical_loss": 3.479915153673363, + "tokens_seen": 1859452928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409003370245547, + "loss": 1.2646, + "theoretical_loss": 3.479894577717146, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9955404996871948, + "objective/train/avg_token_score": 0.0044595031067729, + "objective/train/docs_used": 678160, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2758374214172363, + "objective/train/original_loss": 1.2825698852539062, + "objective/train/theoretical_loss": 3.4798740036172138, + "objective/train/tokens_used": 230174176, + "objective/train/unlikelihood_loss": 0.012204045429825783, + "theoretical_loss": 3.4798740036172138, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086021505376343, + "loss": 1.323, + "theoretical_loss": 3.4798740036172138, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044082009308297224, + "loss": 1.3438, + "theoretical_loss": 3.4798534313732676, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044077997111218104, + "loss": 1.3097, + "theoretical_loss": 3.4798328609850095, + "tokens_seen": 1859977216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407398491413898, + "loss": 1.3135, + "theoretical_loss": 3.4798122924521415, + "tokens_seen": 1860108288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044069972717059865, + "loss": 1.3481, + "theoretical_loss": 3.4797917257743647, + "tokens_seen": 1860239360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406596051998074, + "loss": 1.318, + "theoretical_loss": 3.479771160951382, + "tokens_seen": 1860370432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406194832290162, + "loss": 1.2773, + "theoretical_loss": 3.4797505979828953, + "tokens_seen": 1860501504 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440579361258225, + "loss": 1.3633, + "theoretical_loss": 3.479730036868606, + "tokens_seen": 1860632576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405392392874338, + "loss": 1.3046, + "theoretical_loss": 3.4797094776082176, + "tokens_seen": 1860763648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044049911731664263, + "loss": 1.2962, + "theoretical_loss": 3.479688920201431, + "tokens_seen": 1860894720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404589953458514, + "loss": 1.2736, + "theoretical_loss": 3.4796683646479494, + "tokens_seen": 1861025792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404188733750602, + "loss": 1.286, + "theoretical_loss": 3.4796478109474753, + "tokens_seen": 1861156864 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440378751404269, + "loss": 1.3451, + "theoretical_loss": 3.4796272590997104, + "tokens_seen": 1861287936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403386294334778, + "loss": 1.307, + "theoretical_loss": 3.479606709104358, + "tokens_seen": 1861419008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044029850746268656, + "loss": 1.3697, + "theoretical_loss": 3.4795861609611203, + "tokens_seen": 1861550080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402583854918954, + "loss": 1.337, + "theoretical_loss": 3.4795656146697, + "tokens_seen": 1861681152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044021826352110417, + "loss": 1.3728, + "theoretical_loss": 3.4795450702298005, + "tokens_seen": 1861812224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401781415503129, + "loss": 1.2986, + "theoretical_loss": 3.4795245276411233, + "tokens_seen": 1861943296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401380195795218, + "loss": 1.4369, + "theoretical_loss": 3.4795039869033726, + "tokens_seen": 1862074368 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044009789760873053, + "loss": 1.2989, + "theoretical_loss": 3.479483448016251, + "tokens_seen": 1862205440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044005777563793934, + "loss": 1.339, + "theoretical_loss": 3.4794629109794606, + "tokens_seen": 1862336512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044001765366714815, + "loss": 1.3155, + "theoretical_loss": 3.479442375792706, + "tokens_seen": 1862467584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043997753169635695, + "loss": 1.2916, + "theoretical_loss": 3.4794218424556895, + "tokens_seen": 1862598656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399374097255657, + "loss": 1.3137, + "theoretical_loss": 3.479401310968114, + "tokens_seen": 1862729728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398972877547745, + "loss": 1.3082, + "theoretical_loss": 3.4793807813296835, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398571657839833, + "loss": 1.2319, + "theoretical_loss": 3.4793602535401016, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398170438131921, + "loss": 1.3042, + "theoretical_loss": 3.4793397275990703, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043977692184240093, + "loss": 1.2776, + "theoretical_loss": 3.4793192035062948, + "tokens_seen": 1863254016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397367998716097, + "loss": 1.301, + "theoretical_loss": 3.4792986812614775, + "tokens_seen": 1863385088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043969667790081854, + "loss": 1.2833, + "theoretical_loss": 3.479278160864323, + "tokens_seen": 1863516160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396565559300273, + "loss": 1.36, + "theoretical_loss": 3.4792576423145345, + "tokens_seen": 1863647232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043961643395923605, + "loss": 1.342, + "theoretical_loss": 3.4792371256118155, + "tokens_seen": 1863778304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395763119884449, + "loss": 1.3458, + "theoretical_loss": 3.47921661075587, + "tokens_seen": 1863909376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043953619001765366, + "loss": 1.3246, + "theoretical_loss": 3.479196097746402, + "tokens_seen": 1864040448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043949606804686246, + "loss": 1.3452, + "theoretical_loss": 3.479175586583116, + "tokens_seen": 1864171520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043945594607607127, + "loss": 1.3543, + "theoretical_loss": 3.479155077265715, + "tokens_seen": 1864302592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394158241052801, + "loss": 1.3403, + "theoretical_loss": 3.479134569793904, + "tokens_seen": 1864433664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043937570213448883, + "loss": 1.2501, + "theoretical_loss": 3.479114064167387, + "tokens_seen": 1864564736 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933558016369764, + "loss": 1.2968, + "theoretical_loss": 3.4790935603858677, + "tokens_seen": 1864695808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043929545819290644, + "loss": 1.375, + "theoretical_loss": 3.4790730584490515, + "tokens_seen": 1864826880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392553362221152, + "loss": 1.2596, + "theoretical_loss": 3.479052558356642, + "tokens_seen": 1864957952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921521425132405, + "loss": 1.3359, + "theoretical_loss": 3.4790320601083433, + "tokens_seen": 1865089024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391750922805328, + "loss": 1.2769, + "theoretical_loss": 3.4790115637038608, + "tokens_seen": 1865220096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043913497030974167, + "loss": 1.2847, + "theoretical_loss": 3.478991069142899, + "tokens_seen": 1865351168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390948483389504, + "loss": 1.3428, + "theoretical_loss": 3.478970576425162, + "tokens_seen": 1865482240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390547263681592, + "loss": 1.3498, + "theoretical_loss": 3.478950085550355, + "tokens_seen": 1865613312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043901460439736803, + "loss": 1.3205, + "theoretical_loss": 3.478929596518183, + "tokens_seen": 1865744384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389744824265768, + "loss": 1.3065, + "theoretical_loss": 3.4789091093283497, + "tokens_seen": 1865875456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389343604557856, + "loss": 1.3044, + "theoretical_loss": 3.478888623980562, + "tokens_seen": 1866006528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388942384849944, + "loss": 1.3826, + "theoretical_loss": 3.478868140474523, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9927670359611511, + "objective/train/avg_token_score": 0.007232929579913616, + "objective/train/docs_used": 681054, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3499912023544312, + "objective/train/original_loss": 1.3610955476760864, + "objective/train/theoretical_loss": 3.478847658809938, + "objective/train/tokens_used": 236727776, + "objective/train/unlikelihood_loss": 0.01791020855307579, + "theoretical_loss": 3.478847658809938, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388541165142032, + "loss": 1.349, + "theoretical_loss": 3.478847658809938, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043881399454341195, + "loss": 1.2474, + "theoretical_loss": 3.4788271789865135, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387738725726208, + "loss": 1.3468, + "theoretical_loss": 3.478806701003954, + "tokens_seen": 1866530816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043873375060182957, + "loss": 1.2575, + "theoretical_loss": 3.4787862248619645, + "tokens_seen": 1866661888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386936286310383, + "loss": 1.3329, + "theoretical_loss": 3.4787657505602505, + "tokens_seen": 1866792960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386535066602472, + "loss": 1.3684, + "theoretical_loss": 3.4787452780985175, + "tokens_seen": 1866924032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043861338468945593, + "loss": 1.3168, + "theoretical_loss": 3.478724807476471, + "tokens_seen": 1867055104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385732627186648, + "loss": 1.3137, + "theoretical_loss": 3.4787043386938166, + "tokens_seen": 1867186176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043853314074787354, + "loss": 1.4244, + "theoretical_loss": 3.4786838717502593, + "tokens_seen": 1867317248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043849301877708235, + "loss": 1.3349, + "theoretical_loss": 3.4786634066455058, + "tokens_seen": 1867448320 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043845289680629116, + "loss": 1.3348, + "theoretical_loss": 3.4786429433792607, + "tokens_seen": 1867579392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384127748354999, + "loss": 1.3545, + "theoretical_loss": 3.4786224819512306, + "tokens_seen": 1867710464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383726528647087, + "loss": 1.3321, + "theoretical_loss": 3.478602022361122, + "tokens_seen": 1867841536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383325308939175, + "loss": 1.3668, + "theoretical_loss": 3.478581564608639, + "tokens_seen": 1867972608 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043829240892312633, + "loss": 1.3789, + "theoretical_loss": 3.47856110869349, + "tokens_seen": 1868103680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382522869523351, + "loss": 1.3786, + "theoretical_loss": 3.478540654615379, + "tokens_seen": 1868234752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043821216498154394, + "loss": 1.2807, + "theoretical_loss": 3.478520202374013, + "tokens_seen": 1868365824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381720430107527, + "loss": 1.3634, + "theoretical_loss": 3.478499751969098, + "tokens_seen": 1868496896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043813192103996145, + "loss": 1.3916, + "theoretical_loss": 3.478479303400341, + "tokens_seen": 1868627968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380917990691703, + "loss": 1.3602, + "theoretical_loss": 3.478458856667448, + "tokens_seen": 1868759040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043805167709837906, + "loss": 1.4672, + "theoretical_loss": 3.4784384117701244, + "tokens_seen": 1868890112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043801155512758786, + "loss": 1.356, + "theoretical_loss": 3.4784179687080785, + "tokens_seen": 1869021184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043797143315679667, + "loss": 1.3155, + "theoretical_loss": 3.4783975274810155, + "tokens_seen": 1869152256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379313111860055, + "loss": 1.3039, + "theoretical_loss": 3.478377088088642, + "tokens_seen": 1869283328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378911892152143, + "loss": 1.3229, + "theoretical_loss": 3.478356650530666, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785106724442304, + "loss": 1.3842, + "theoretical_loss": 3.4783362148067933, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781094527363184, + "loss": 1.3545, + "theoretical_loss": 3.4783157809167307, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777082330284065, + "loss": 1.4547, + "theoretical_loss": 3.4782953488601853, + "tokens_seen": 1869807616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043773070133204945, + "loss": 1.3596, + "theoretical_loss": 3.478274918636864, + "tokens_seen": 1869938688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376905793612582, + "loss": 1.2237, + "theoretical_loss": 3.4782544902464743, + "tokens_seen": 1870069760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043765045739046707, + "loss": 1.3369, + "theoretical_loss": 3.4782340636887223, + "tokens_seen": 1870200832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376103354196758, + "loss": 1.3269, + "theoretical_loss": 3.4782136389633163, + "tokens_seen": 1870331904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043757021344888457, + "loss": 1.2966, + "theoretical_loss": 3.4781932160699625, + "tokens_seen": 1870462976 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043753009147809343, + "loss": 1.2592, + "theoretical_loss": 3.478172795008369, + "tokens_seen": 1870594048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374899695073022, + "loss": 1.27, + "theoretical_loss": 3.478152375778243, + "tokens_seen": 1870725120 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437449847536511, + "loss": 1.3863, + "theoretical_loss": 3.4781319583792913, + "tokens_seen": 1870856192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374097255657198, + "loss": 1.2965, + "theoretical_loss": 3.478111542811222, + "tokens_seen": 1870987264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373696035949286, + "loss": 1.4181, + "theoretical_loss": 3.4780911290737433, + "tokens_seen": 1871118336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373294816241374, + "loss": 1.398, + "theoretical_loss": 3.4780707171665615, + "tokens_seen": 1871249408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372893596533462, + "loss": 1.387, + "theoretical_loss": 3.478050307089385, + "tokens_seen": 1871380480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043724923768255497, + "loss": 1.3781, + "theoretical_loss": 3.478029898841922, + "tokens_seen": 1871511552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372091157117638, + "loss": 1.3421, + "theoretical_loss": 3.4780094924238796, + "tokens_seen": 1871642624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371689937409726, + "loss": 1.4013, + "theoretical_loss": 3.477989087834966, + "tokens_seen": 1871773696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043712887177018133, + "loss": 1.3007, + "theoretical_loss": 3.477968685074889, + "tokens_seen": 1871904768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370887497993902, + "loss": 1.3722, + "theoretical_loss": 3.477948284143357, + "tokens_seen": 1872035840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043704862782859894, + "loss": 1.3596, + "theoretical_loss": 3.477927885040078, + "tokens_seen": 1872166912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043700850585780775, + "loss": 1.2882, + "theoretical_loss": 3.4779074877647593, + "tokens_seen": 1872297984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043696838388701656, + "loss": 1.3015, + "theoretical_loss": 3.477887092317111, + "tokens_seen": 1872429056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369282619162253, + "loss": 1.3185, + "theoretical_loss": 3.4778666986968396, + "tokens_seen": 1872560128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004368881399454341, + "loss": 1.373, + "theoretical_loss": 3.477846306903655, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9942508935928345, + "objective/train/avg_token_score": 0.005749125499278307, + "objective/train/docs_used": 683408, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.524458408355713, + "objective/train/original_loss": 1.5322853326797485, + "objective/train/theoretical_loss": 3.477825916937264, + "objective/train/tokens_used": 243281376, + "objective/train/unlikelihood_loss": 0.01687990128993988, + "theoretical_loss": 3.477825916937264, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368480179746429, + "loss": 1.4051, + "theoretical_loss": 3.477825916937264, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043680789600385173, + "loss": 1.4219, + "theoretical_loss": 3.4778055287973766, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367677740330605, + "loss": 1.3527, + "theoretical_loss": 3.4777851424837007, + "tokens_seen": 1873084416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043672765206226934, + "loss": 1.4077, + "theoretical_loss": 3.4777647579959448, + "tokens_seen": 1873215488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366875300914781, + "loss": 1.3657, + "theoretical_loss": 3.4777443753338178, + "tokens_seen": 1873346560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366474081206869, + "loss": 1.3643, + "theoretical_loss": 3.477723994497029, + "tokens_seen": 1873477632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366072861498957, + "loss": 1.434, + "theoretical_loss": 3.4777036154852863, + "tokens_seen": 1873608704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656716417910446, + "loss": 1.3632, + "theoretical_loss": 3.477683238298299, + "tokens_seen": 1873739776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365270422083133, + "loss": 1.2949, + "theoretical_loss": 3.4776628629357766, + "tokens_seen": 1873870848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648692023752207, + "loss": 1.2411, + "theoretical_loss": 3.477642489397428, + "tokens_seen": 1874001920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364467982667309, + "loss": 1.3624, + "theoretical_loss": 3.4776221176829614, + "tokens_seen": 1874132992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364066762959397, + "loss": 1.3901, + "theoretical_loss": 3.4776017477920864, + "tokens_seen": 1874264064 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043636655432514843, + "loss": 1.3312, + "theoretical_loss": 3.477581379724513, + "tokens_seen": 1874395136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632643235435724, + "loss": 1.3859, + "theoretical_loss": 3.4775610134799497, + "tokens_seen": 1874526208 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043628631038356605, + "loss": 1.3374, + "theoretical_loss": 3.4775406490581062, + "tokens_seen": 1874657280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043624618841277485, + "loss": 1.377, + "theoretical_loss": 3.4775202864586916, + "tokens_seen": 1874788352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362060664419836, + "loss": 1.4018, + "theoretical_loss": 3.4774999256814163, + "tokens_seen": 1874919424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043616594447119247, + "loss": 1.3141, + "theoretical_loss": 3.477479566725989, + "tokens_seen": 1875050496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361258225004012, + "loss": 1.3697, + "theoretical_loss": 3.4774592095921197, + "tokens_seen": 1875181568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043608570052960997, + "loss": 1.3127, + "theoretical_loss": 3.477438854279518, + "tokens_seen": 1875312640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043604557855881883, + "loss": 1.3488, + "theoretical_loss": 3.477418500787894, + "tokens_seen": 1875443712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360054565880276, + "loss": 1.3528, + "theoretical_loss": 3.477398149116957, + "tokens_seen": 1875574784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043596533461723644, + "loss": 1.3451, + "theoretical_loss": 3.477377799266417, + "tokens_seen": 1875705856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359252126464452, + "loss": 1.2779, + "theoretical_loss": 3.477357451235984, + "tokens_seen": 1875836928 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435885090675654, + "loss": 1.3751, + "theoretical_loss": 3.477337105025369, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358449687048628, + "loss": 1.3221, + "theoretical_loss": 3.477316760634281, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358048467340716, + "loss": 1.3595, + "theoretical_loss": 3.477296418062431, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043576472476328037, + "loss": 1.3384, + "theoretical_loss": 3.477276077309528, + "tokens_seen": 1876361216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357246027924892, + "loss": 1.3545, + "theoretical_loss": 3.477255738375283, + "tokens_seen": 1876492288 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435684480821698, + "loss": 1.3006, + "theoretical_loss": 3.4772354012594073, + "tokens_seen": 1876623360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043564435885090673, + "loss": 1.3609, + "theoretical_loss": 3.4772150659616097, + "tokens_seen": 1876754432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356042368801156, + "loss": 1.3669, + "theoretical_loss": 3.4771947324816024, + "tokens_seen": 1876885504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043556411490932434, + "loss": 1.3773, + "theoretical_loss": 3.477174400819094, + "tokens_seen": 1877016576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043552399293853315, + "loss": 1.3762, + "theoretical_loss": 3.477154070973797, + "tokens_seen": 1877147648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043548387096774196, + "loss": 1.3626, + "theoretical_loss": 3.477133742945421, + "tokens_seen": 1877278720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354437489969507, + "loss": 1.3608, + "theoretical_loss": 3.4771134167336775, + "tokens_seen": 1877409792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043540362702615957, + "loss": 1.3677, + "theoretical_loss": 3.4770930923382766, + "tokens_seen": 1877540864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353635050553683, + "loss": 1.4541, + "theoretical_loss": 3.47707276975893, + "tokens_seen": 1877671936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043532338308457713, + "loss": 1.358, + "theoretical_loss": 3.477052448995348, + "tokens_seen": 1877803008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043528326111378593, + "loss": 1.3801, + "theoretical_loss": 3.4770321300472418, + "tokens_seen": 1877934080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043524313914299474, + "loss": 1.3931, + "theoretical_loss": 3.4770118129143226, + "tokens_seen": 1878065152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352030171722035, + "loss": 1.3918, + "theoretical_loss": 3.476991497596302, + "tokens_seen": 1878196224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351628952014123, + "loss": 1.369, + "theoretical_loss": 3.47697118409289, + "tokens_seen": 1878327296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351227732306211, + "loss": 1.3331, + "theoretical_loss": 3.4769508724038, + "tokens_seen": 1878458368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043508265125982986, + "loss": 1.3841, + "theoretical_loss": 3.476930562528741, + "tokens_seen": 1878589440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350425292890387, + "loss": 1.3118, + "theoretical_loss": 3.476910254467426, + "tokens_seen": 1878720512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043500240731824747, + "loss": 1.2705, + "theoretical_loss": 3.476889948219566, + "tokens_seen": 1878851584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349622853474563, + "loss": 1.3292, + "theoretical_loss": 3.4768696437848723, + "tokens_seen": 1878982656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349221633766651, + "loss": 1.3911, + "theoretical_loss": 3.4768493411630574, + "tokens_seen": 1879113728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043488204140587383, + "loss": 1.3235, + "theoretical_loss": 3.476829040353832, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9950573444366455, + "objective/train/avg_token_score": 0.004942657891660929, + "objective/train/docs_used": 685672, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3110394477844238, + "objective/train/original_loss": 1.3175170421600342, + "objective/train/theoretical_loss": 3.4768087413569084, + "objective/train/tokens_used": 249834976, + "objective/train/unlikelihood_loss": 0.01483804453164339, + "theoretical_loss": 3.4768087413569084, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043484191943508264, + "loss": 1.2963, + "theoretical_loss": 3.4768087413569084, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480179746429145, + "loss": 1.395, + "theoretical_loss": 3.4767884441719987, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476167549350025, + "loss": 1.291, + "theoretical_loss": 3.476768148798814, + "tokens_seen": 1879638016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472155352270906, + "loss": 1.4169, + "theoretical_loss": 3.4767478552370674, + "tokens_seen": 1879769088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043468143155191787, + "loss": 1.3165, + "theoretical_loss": 3.4767275634864694, + "tokens_seen": 1879900160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346413095811266, + "loss": 1.341, + "theoretical_loss": 3.476707273546733, + "tokens_seen": 1880031232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346011876103354, + "loss": 1.3496, + "theoretical_loss": 3.476686985417571, + "tokens_seen": 1880162304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043456106563954423, + "loss": 1.3227, + "theoretical_loss": 3.4766666990986947, + "tokens_seen": 1880293376 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434520943668753, + "loss": 1.3316, + "theoretical_loss": 3.4766464145898164, + "tokens_seen": 1880424448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043448082169796184, + "loss": 1.369, + "theoretical_loss": 3.476626131890649, + "tokens_seen": 1880555520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344406997271706, + "loss": 1.3019, + "theoretical_loss": 3.4766058510009046, + "tokens_seen": 1880686592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344005777563794, + "loss": 1.3447, + "theoretical_loss": 3.476585571920295, + "tokens_seen": 1880817664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343604557855882, + "loss": 1.345, + "theoretical_loss": 3.476565294648535, + "tokens_seen": 1880948736 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434320333814797, + "loss": 1.3929, + "theoretical_loss": 3.476545019185334, + "tokens_seen": 1881079808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428021184400577, + "loss": 1.3535, + "theoretical_loss": 3.4765247455304076, + "tokens_seen": 1881210880 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043424008987321457, + "loss": 1.3161, + "theoretical_loss": 3.476504473683467, + "tokens_seen": 1881341952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341999679024234, + "loss": 1.3276, + "theoretical_loss": 3.476484203644225, + "tokens_seen": 1881473024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043415984593163213, + "loss": 1.3396, + "theoretical_loss": 3.4764639354123954, + "tokens_seen": 1881604096 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434119723960841, + "loss": 1.2734, + "theoretical_loss": 3.4764436689876903, + "tokens_seen": 1881735168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043407960199004974, + "loss": 1.3892, + "theoretical_loss": 3.476423404369823, + "tokens_seen": 1881866240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340394800192586, + "loss": 1.3328, + "theoretical_loss": 3.4764031415585066, + "tokens_seen": 1881997312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043399935804846736, + "loss": 1.3445, + "theoretical_loss": 3.4763828805534547, + "tokens_seen": 1882128384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339592360776761, + "loss": 1.4455, + "theoretical_loss": 3.4763626213543795, + "tokens_seen": 1882259456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043391911410688497, + "loss": 1.3009, + "theoretical_loss": 3.4763423639609954, + "tokens_seen": 1882390528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338789921360937, + "loss": 1.2508, + "theoretical_loss": 3.4763221083730147, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043383887016530253, + "loss": 1.4434, + "theoretical_loss": 3.4763018545901514, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043379874819451133, + "loss": 1.3518, + "theoretical_loss": 3.476281602612119, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043375862622372014, + "loss": 1.4025, + "theoretical_loss": 3.476261352438631, + "tokens_seen": 1882914816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337185042529289, + "loss": 1.3341, + "theoretical_loss": 3.476241104069401, + "tokens_seen": 1883045888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336783822821377, + "loss": 1.382, + "theoretical_loss": 3.4762208575041424, + "tokens_seen": 1883176960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336382603113465, + "loss": 1.4214, + "theoretical_loss": 3.476200612742569, + "tokens_seen": 1883308032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043359813834055526, + "loss": 1.3484, + "theoretical_loss": 3.476180369784395, + "tokens_seen": 1883439104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335580163697641, + "loss": 1.3956, + "theoretical_loss": 3.476160128629334, + "tokens_seen": 1883570176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043351789439897287, + "loss": 1.3552, + "theoretical_loss": 3.4761398892771, + "tokens_seen": 1883701248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043347777242818173, + "loss": 1.285, + "theoretical_loss": 3.476119651727407, + "tokens_seen": 1883832320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334376504573905, + "loss": 1.3659, + "theoretical_loss": 3.4760994159799687, + "tokens_seen": 1883963392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043339752848659923, + "loss": 1.2527, + "theoretical_loss": 3.4760791820345, + "tokens_seen": 1884094464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333574065158081, + "loss": 1.305, + "theoretical_loss": 3.476058949890714, + "tokens_seen": 1884225536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043331728454501685, + "loss": 1.341, + "theoretical_loss": 3.4760387195483258, + "tokens_seen": 1884356608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327716257422565, + "loss": 1.3247, + "theoretical_loss": 3.47601849100705, + "tokens_seen": 1884487680 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323704060343446, + "loss": 1.3108, + "theoretical_loss": 3.4759982642666003, + "tokens_seen": 1884618752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043319691863264327, + "loss": 1.2791, + "theoretical_loss": 3.475978039326691, + "tokens_seen": 1884749824 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433156796661852, + "loss": 1.3891, + "theoretical_loss": 3.4759578161870373, + "tokens_seen": 1884880896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331166746910608, + "loss": 1.3353, + "theoretical_loss": 3.4759375948473537, + "tokens_seen": 1885011968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043307655272026963, + "loss": 1.2937, + "theoretical_loss": 3.475917375307354, + "tokens_seen": 1885143040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330364307494784, + "loss": 1.3171, + "theoretical_loss": 3.475897157566754, + "tokens_seen": 1885274112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043299630877868724, + "loss": 1.3139, + "theoretical_loss": 3.475876941625268, + "tokens_seen": 1885405184 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432956186807896, + "loss": 1.3754, + "theoretical_loss": 3.4758567274826104, + "tokens_seen": 1885536256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329160648371048, + "loss": 1.3913, + "theoretical_loss": 3.4758365151384973, + "tokens_seen": 1885667328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328759428663136, + "loss": 1.3449, + "theoretical_loss": 3.4758163045926427, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9930122494697571, + "objective/train/avg_token_score": 0.006987700238823891, + "objective/train/docs_used": 687569, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3358765840530396, + "objective/train/original_loss": 1.3468788862228394, + "objective/train/theoretical_loss": 3.475796095844762, + "objective/train/tokens_used": 256388576, + "objective/train/unlikelihood_loss": 0.01940946653485298, + "theoretical_loss": 3.475796095844762, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043283582089552236, + "loss": 1.3926, + "theoretical_loss": 3.475796095844762, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327956989247312, + "loss": 1.4115, + "theoretical_loss": 3.47577588889457, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043275557695393997, + "loss": 1.3567, + "theoretical_loss": 3.475755683741782, + "tokens_seen": 1886191616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327154549831488, + "loss": 1.3024, + "theoretical_loss": 3.4757354803861134, + "tokens_seen": 1886322688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326753330123576, + "loss": 1.4367, + "theoretical_loss": 3.475715278827279, + "tokens_seen": 1886453760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326352110415664, + "loss": 1.4573, + "theoretical_loss": 3.4756950790649954, + "tokens_seen": 1886584832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043259508907077514, + "loss": 1.3037, + "theoretical_loss": 3.4756748810989766, + "tokens_seen": 1886715904 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432554967099984, + "loss": 1.3163, + "theoretical_loss": 3.475654684928939, + "tokens_seen": 1886846976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043251484512919276, + "loss": 1.3559, + "theoretical_loss": 3.4756344905545977, + "tokens_seen": 1886978048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324747231584015, + "loss": 1.3983, + "theoretical_loss": 3.4756142979756683, + "tokens_seen": 1887109120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043243460118761037, + "loss": 1.3777, + "theoretical_loss": 3.475594107191867, + "tokens_seen": 1887240192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323944792168191, + "loss": 1.3656, + "theoretical_loss": 3.475573918202909, + "tokens_seen": 1887371264 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043235435724602793, + "loss": 1.2912, + "theoretical_loss": 3.475553731008511, + "tokens_seen": 1887502336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043231423527523673, + "loss": 1.3815, + "theoretical_loss": 3.4755335456083873, + "tokens_seen": 1887633408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043227411330444554, + "loss": 1.2917, + "theoretical_loss": 3.4755133620022556, + "tokens_seen": 1887764480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043223399133365435, + "loss": 1.3808, + "theoretical_loss": 3.4754931801898308, + "tokens_seen": 1887895552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321938693628631, + "loss": 1.2919, + "theoretical_loss": 3.4754730001708287, + "tokens_seen": 1888026624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321537473920719, + "loss": 1.3821, + "theoretical_loss": 3.475452821944967, + "tokens_seen": 1888157696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321136254212807, + "loss": 1.2579, + "theoretical_loss": 3.47543264551196, + "tokens_seen": 1888288768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320735034504895, + "loss": 1.3295, + "theoretical_loss": 3.4754124708715257, + "tokens_seen": 1888419840 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043203338147969827, + "loss": 1.2966, + "theoretical_loss": 3.4753922980233796, + "tokens_seen": 1888550912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043199325950890713, + "loss": 1.4077, + "theoretical_loss": 3.4753721269672373, + "tokens_seen": 1888681984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319531375381159, + "loss": 1.2667, + "theoretical_loss": 3.4753519577028165, + "tokens_seen": 1888813056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043191301556732463, + "loss": 1.3486, + "theoretical_loss": 3.4753317902298337, + "tokens_seen": 1888944128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318728935965335, + "loss": 1.3598, + "theoretical_loss": 3.475311624548005, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183277162574225, + "loss": 1.3403, + "theoretical_loss": 3.475291460657047, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043179264965495105, + "loss": 1.3128, + "theoretical_loss": 3.4752712985566765, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043175252768415986, + "loss": 1.2984, + "theoretical_loss": 3.4752511382466107, + "tokens_seen": 1889468416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043171240571336867, + "loss": 1.2782, + "theoretical_loss": 3.475230979726566, + "tokens_seen": 1889599488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316722837425774, + "loss": 1.3291, + "theoretical_loss": 3.4752108229962593, + "tokens_seen": 1889730560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316321617717862, + "loss": 1.3203, + "theoretical_loss": 3.475190668055408, + "tokens_seen": 1889861632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043159203980099503, + "loss": 1.3035, + "theoretical_loss": 3.475170514903729, + "tokens_seen": 1889992704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043155191783020384, + "loss": 1.3507, + "theoretical_loss": 3.475150363540939, + "tokens_seen": 1890123776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043151179585941264, + "loss": 1.3034, + "theoretical_loss": 3.4751302139667555, + "tokens_seen": 1890254848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314716738886214, + "loss": 1.3572, + "theoretical_loss": 3.4751100661808954, + "tokens_seen": 1890385920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043143155191783026, + "loss": 1.4199, + "theoretical_loss": 3.475089920183077, + "tokens_seen": 1890516992 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431391429947039, + "loss": 1.3004, + "theoretical_loss": 3.4750697759730165, + "tokens_seen": 1890648064 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043135130797624776, + "loss": 1.2886, + "theoretical_loss": 3.475049633550432, + "tokens_seen": 1890779136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313111860054566, + "loss": 1.3118, + "theoretical_loss": 3.4750294929150405, + "tokens_seen": 1890910208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127106403466537, + "loss": 1.3605, + "theoretical_loss": 3.4750093540665605, + "tokens_seen": 1891041280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312309420638742, + "loss": 1.2855, + "theoretical_loss": 3.4749892170047083, + "tokens_seen": 1891172352 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431190820093083, + "loss": 1.3285, + "theoretical_loss": 3.474969081729203, + "tokens_seen": 1891303424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311506981222918, + "loss": 1.3579, + "theoretical_loss": 3.4749489482397613, + "tokens_seen": 1891434496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043111057615150054, + "loss": 1.3372, + "theoretical_loss": 3.474928816536101, + "tokens_seen": 1891565568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310704541807094, + "loss": 1.2397, + "theoretical_loss": 3.4749086866179413, + "tokens_seen": 1891696640 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043103033220991816, + "loss": 1.2961, + "theoretical_loss": 3.4748885584849987, + "tokens_seen": 1891827712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309902102391269, + "loss": 1.3225, + "theoretical_loss": 3.474868432136991, + "tokens_seen": 1891958784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043095008826833577, + "loss": 1.2628, + "theoretical_loss": 3.474848307573638, + "tokens_seen": 1892089856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309099662975445, + "loss": 1.3138, + "theoretical_loss": 3.474828184794656, + "tokens_seen": 1892220928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308698443267534, + "loss": 1.3555, + "theoretical_loss": 3.474808063799764, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9939754605293274, + "objective/train/avg_token_score": 0.006024450995028019, + "objective/train/docs_used": 690244, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4132239818572998, + "objective/train/original_loss": 1.4227776527404785, + "objective/train/theoretical_loss": 3.474787944588681, + "objective/train/tokens_used": 262942176, + "objective/train/unlikelihood_loss": 0.015185552649199963, + "theoretical_loss": 3.474787944588681, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043082972235596213, + "loss": 1.3577, + "theoretical_loss": 3.474787944588681, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043078960038517094, + "loss": 1.4072, + "theoretical_loss": 3.4747678271611244, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043074947841437975, + "loss": 1.314, + "theoretical_loss": 3.4747477115168124, + "tokens_seen": 1892745216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307093564435885, + "loss": 1.3197, + "theoretical_loss": 3.474727597655464, + "tokens_seen": 1892876288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306692344727973, + "loss": 1.3037, + "theoretical_loss": 3.4747074855767974, + "tokens_seen": 1893007360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306291125020061, + "loss": 1.3452, + "theoretical_loss": 3.4746873752805314, + "tokens_seen": 1893138432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305889905312149, + "loss": 1.3426, + "theoretical_loss": 3.474667266766385, + "tokens_seen": 1893269504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043054886856042367, + "loss": 1.3014, + "theoretical_loss": 3.4746471600340763, + "tokens_seen": 1893400576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043050874658963253, + "loss": 1.2859, + "theoretical_loss": 3.474627055083324, + "tokens_seen": 1893531648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304686246188413, + "loss": 1.3872, + "theoretical_loss": 3.474606951913848, + "tokens_seen": 1893662720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043042850264805003, + "loss": 1.3992, + "theoretical_loss": 3.4745868505253656, + "tokens_seen": 1893793792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303883806772589, + "loss": 1.3054, + "theoretical_loss": 3.474566750917597, + "tokens_seen": 1893924864 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043034825870646765, + "loss": 1.334, + "theoretical_loss": 3.4745466530902607, + "tokens_seen": 1894055936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303081367356765, + "loss": 1.37, + "theoretical_loss": 3.4745265570430757, + "tokens_seen": 1894187008 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043026801476488526, + "loss": 1.3269, + "theoretical_loss": 3.4745064627757616, + "tokens_seen": 1894318080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043022789279409407, + "loss": 1.3305, + "theoretical_loss": 3.4744863702880378, + "tokens_seen": 1894449152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043018777082330287, + "loss": 1.3568, + "theoretical_loss": 3.474466279579623, + "tokens_seen": 1894580224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301476488525116, + "loss": 1.306, + "theoretical_loss": 3.4744461906502364, + "tokens_seen": 1894711296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043010752688172043, + "loss": 1.2973, + "theoretical_loss": 3.474426103499598, + "tokens_seen": 1894842368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043006740491092924, + "loss": 1.3498, + "theoretical_loss": 3.4744060181274268, + "tokens_seen": 1894973440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043002728294013804, + "loss": 1.3615, + "theoretical_loss": 3.474385934533443, + "tokens_seen": 1895104512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299871609693468, + "loss": 1.3714, + "theoretical_loss": 3.4743658527173658, + "tokens_seen": 1895235584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042994703899855565, + "loss": 1.3394, + "theoretical_loss": 3.4743457726789146, + "tokens_seen": 1895366656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299069170277644, + "loss": 1.386, + "theoretical_loss": 3.4743256944178094, + "tokens_seen": 1895497728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042986679505697316, + "loss": 1.3825, + "theoretical_loss": 3.47430561793377, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429826673086182, + "loss": 1.3359, + "theoretical_loss": 3.4742855432265163, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042978655111539077, + "loss": 1.3599, + "theoretical_loss": 3.4742654702957685, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297464291445996, + "loss": 1.3302, + "theoretical_loss": 3.474245399141246, + "tokens_seen": 1896022016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297063071738084, + "loss": 1.3604, + "theoretical_loss": 3.474225329762669, + "tokens_seen": 1896153088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296661852030172, + "loss": 1.2936, + "theoretical_loss": 3.4742052621597574, + "tokens_seen": 1896284160 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429626063232226, + "loss": 1.2628, + "theoretical_loss": 3.4741851963322326, + "tokens_seen": 1896415232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295859412614348, + "loss": 1.3317, + "theoretical_loss": 3.474165132279813, + "tokens_seen": 1896546304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042954581929064356, + "loss": 1.3467, + "theoretical_loss": 3.47414507000222, + "tokens_seen": 1896677376 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042950569731985236, + "loss": 1.3554, + "theoretical_loss": 3.4741250094991747, + "tokens_seen": 1896808448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042946557534906117, + "loss": 1.2726, + "theoretical_loss": 3.474104950770396, + "tokens_seen": 1896939520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294254533782699, + "loss": 1.3273, + "theoretical_loss": 3.474084893815605, + "tokens_seen": 1897070592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293853314074788, + "loss": 1.3425, + "theoretical_loss": 3.474064838634522, + "tokens_seen": 1897201664 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042934520943668753, + "loss": 1.3667, + "theoretical_loss": 3.4740447852268685, + "tokens_seen": 1897332736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042930508746589634, + "loss": 1.3366, + "theoretical_loss": 3.4740247335923646, + "tokens_seen": 1897463808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042926496549510515, + "loss": 1.3006, + "theoretical_loss": 3.4740046837307306, + "tokens_seen": 1897594880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292248435243139, + "loss": 1.3136, + "theoretical_loss": 3.4739846356416875, + "tokens_seen": 1897725952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291847215535227, + "loss": 1.2829, + "theoretical_loss": 3.4739645893249573, + "tokens_seen": 1897857024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291445995827315, + "loss": 1.3043, + "theoretical_loss": 3.4739445447802595, + "tokens_seen": 1897988096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291044776119403, + "loss": 1.305, + "theoretical_loss": 3.473924502007316, + "tokens_seen": 1898119168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290643556411491, + "loss": 1.3644, + "theoretical_loss": 3.473904461005847, + "tokens_seen": 1898250240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042902423367035793, + "loss": 1.3228, + "theoretical_loss": 3.4738844217755744, + "tokens_seen": 1898381312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289841116995667, + "loss": 1.3288, + "theoretical_loss": 3.4738643843162196, + "tokens_seen": 1898512384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289439897287755, + "loss": 1.3247, + "theoretical_loss": 3.473844348627503, + "tokens_seen": 1898643456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289038677579843, + "loss": 1.3364, + "theoretical_loss": 3.4738243147091463, + "tokens_seen": 1898774528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042886374578719305, + "loss": 1.3069, + "theoretical_loss": 3.473804282560871, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.991893470287323, + "objective/train/avg_token_score": 0.008106522262096405, + "objective/train/docs_used": 692658, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3310139179229736, + "objective/train/original_loss": 1.3423906564712524, + "objective/train/theoretical_loss": 3.4737842521823983, + "objective/train/tokens_used": 269495776, + "objective/train/unlikelihood_loss": 0.021742379292845726, + "theoretical_loss": 3.4737842521823983, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288236238164019, + "loss": 1.3428, + "theoretical_loss": 3.4737842521823983, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878350184561066, + "loss": 1.3744, + "theoretical_loss": 3.47376422357345, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042874337987481946, + "loss": 1.3232, + "theoretical_loss": 3.4737441967337475, + "tokens_seen": 1899298816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042870325790402827, + "loss": 1.3436, + "theoretical_loss": 3.4737241716630125, + "tokens_seen": 1899429888 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428663135933237, + "loss": 1.3351, + "theoretical_loss": 3.4737041483609663, + "tokens_seen": 1899560960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042862301396244583, + "loss": 1.2543, + "theoretical_loss": 3.4736841268273313, + "tokens_seen": 1899692032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042858289199165464, + "loss": 1.322, + "theoretical_loss": 3.473664107061829, + "tokens_seen": 1899823104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854277002086344, + "loss": 1.2866, + "theoretical_loss": 3.473644089064181, + "tokens_seen": 1899954176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285026480500722, + "loss": 1.2732, + "theoretical_loss": 3.47362407283411, + "tokens_seen": 1900085248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846252607928105, + "loss": 1.3489, + "theoretical_loss": 3.473604058371337, + "tokens_seen": 1900216320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284224041084898, + "loss": 1.2992, + "theoretical_loss": 3.473584045675585, + "tokens_seen": 1900347392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283822821376986, + "loss": 1.3293, + "theoretical_loss": 3.473564034746576, + "tokens_seen": 1900478464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283421601669074, + "loss": 1.2893, + "theoretical_loss": 3.4735440255840313, + "tokens_seen": 1900609536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042830203819611617, + "loss": 1.3821, + "theoretical_loss": 3.4735240181876743, + "tokens_seen": 1900740608 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042826191622532503, + "loss": 1.3021, + "theoretical_loss": 3.4735040125572265, + "tokens_seen": 1900871680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282217942545338, + "loss": 1.2832, + "theoretical_loss": 3.473484008692411, + "tokens_seen": 1901002752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281816722837426, + "loss": 1.3115, + "theoretical_loss": 3.4734640065929496, + "tokens_seen": 1901133824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281415503129514, + "loss": 1.3758, + "theoretical_loss": 3.473444006258565, + "tokens_seen": 1901264896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042810142834216015, + "loss": 1.3617, + "theoretical_loss": 3.47342400768898, + "tokens_seen": 1901395968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042806130637136896, + "loss": 1.3797, + "theoretical_loss": 3.4734040108839173, + "tokens_seen": 1901527040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802118440057776, + "loss": 1.3649, + "theoretical_loss": 3.4733840158430995, + "tokens_seen": 1901658112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042798106242978657, + "loss": 1.3336, + "theoretical_loss": 3.4733640225662485, + "tokens_seen": 1901789184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279409404589953, + "loss": 1.3619, + "theoretical_loss": 3.473344031053089, + "tokens_seen": 1901920256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279008184882042, + "loss": 1.4049, + "theoretical_loss": 3.473324041303342, + "tokens_seen": 1902051328 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042786069651741293, + "loss": 1.361, + "theoretical_loss": 3.473304053316731, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278205745466218, + "loss": 1.3129, + "theoretical_loss": 3.4732840670929797, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042778045257583054, + "loss": 1.3246, + "theoretical_loss": 3.4732640826318106, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277403306050393, + "loss": 1.3441, + "theoretical_loss": 3.473244099932947, + "tokens_seen": 1902575616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042770020863424816, + "loss": 1.2094, + "theoretical_loss": 3.4732241189961117, + "tokens_seen": 1902706688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276600866634569, + "loss": 1.3911, + "theoretical_loss": 3.473204139821029, + "tokens_seen": 1902837760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276199646926657, + "loss": 1.3108, + "theoretical_loss": 3.473184162407421, + "tokens_seen": 1902968832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275798427218745, + "loss": 1.2507, + "theoretical_loss": 3.473164186755012, + "tokens_seen": 1903099904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042753972075108333, + "loss": 1.3326, + "theoretical_loss": 3.4731442128635246, + "tokens_seen": 1903230976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274995987802921, + "loss": 1.3261, + "theoretical_loss": 3.473124240732683, + "tokens_seen": 1903362048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274594768095009, + "loss": 1.4035, + "theoretical_loss": 3.473104270362211, + "tokens_seen": 1903493120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274193548387097, + "loss": 1.3245, + "theoretical_loss": 3.473084301751831, + "tokens_seen": 1903624192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042737923286791845, + "loss": 1.269, + "theoretical_loss": 3.4730643349012684, + "tokens_seen": 1903755264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273391108971273, + "loss": 1.2671, + "theoretical_loss": 3.4730443698102453, + "tokens_seen": 1903886336 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042729898892633606, + "loss": 1.3101, + "theoretical_loss": 3.473024406478487, + "tokens_seen": 1904017408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042725886695554486, + "loss": 1.3544, + "theoretical_loss": 3.4730044449057162, + "tokens_seen": 1904148480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042721874498475367, + "loss": 1.2684, + "theoretical_loss": 3.472984485091657, + "tokens_seen": 1904279552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271786230139624, + "loss": 1.2899, + "theoretical_loss": 3.4729645270360345, + "tokens_seen": 1904410624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271385010431713, + "loss": 1.3109, + "theoretical_loss": 3.472944570738571, + "tokens_seen": 1904541696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042709837907238004, + "loss": 1.237, + "theoretical_loss": 3.472924616198993, + "tokens_seen": 1904672768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042705825710158884, + "loss": 1.2487, + "theoretical_loss": 3.4729046634170224, + "tokens_seen": 1904803840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042701813513079765, + "loss": 1.2879, + "theoretical_loss": 3.4728847123923847, + "tokens_seen": 1904934912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042697801316000645, + "loss": 1.3218, + "theoretical_loss": 3.4728647631248037, + "tokens_seen": 1905065984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269378911892152, + "loss": 1.3957, + "theoretical_loss": 3.4728448156140046, + "tokens_seen": 1905197056 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426897769218424, + "loss": 1.3505, + "theoretical_loss": 3.4728248698597106, + "tokens_seen": 1905328128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268576472476328, + "loss": 1.3067, + "theoretical_loss": 3.472804925861648, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9944093227386475, + "objective/train/avg_token_score": 0.005590598098933697, + "objective/train/docs_used": 694687, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2996364831924438, + "objective/train/original_loss": 1.3071355819702148, + "objective/train/theoretical_loss": 3.472784983619539, + "objective/train/tokens_used": 276049376, + "objective/train/unlikelihood_loss": 0.015953294932842255, + "theoretical_loss": 3.472784983619539, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042681752527684157, + "loss": 1.2785, + "theoretical_loss": 3.472784983619539, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042677740330605043, + "loss": 1.3383, + "theoretical_loss": 3.47276504313311, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267372813352592, + "loss": 1.3638, + "theoretical_loss": 3.472745104402085, + "tokens_seen": 1905852416 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426697159364468, + "loss": 1.2815, + "theoretical_loss": 3.472725167426189, + "tokens_seen": 1905983488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266570373936768, + "loss": 1.4039, + "theoretical_loss": 3.472705232205147, + "tokens_seen": 1906114560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042661691542288555, + "loss": 1.3067, + "theoretical_loss": 3.472685298738684, + "tokens_seen": 1906245632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042657679345209435, + "loss": 1.292, + "theoretical_loss": 3.472665367026524, + "tokens_seen": 1906376704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042653667148130316, + "loss": 1.3003, + "theoretical_loss": 3.4726454370683926, + "tokens_seen": 1906507776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042649654951051197, + "loss": 1.3306, + "theoretical_loss": 3.4726255088640148, + "tokens_seen": 1906638848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264564275397208, + "loss": 1.4208, + "theoretical_loss": 3.472605582413116, + "tokens_seen": 1906769920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264163055689296, + "loss": 1.3402, + "theoretical_loss": 3.4725856577154213, + "tokens_seen": 1906900992 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042637618359813833, + "loss": 1.4138, + "theoretical_loss": 3.4725657347706558, + "tokens_seen": 1907032064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263360616273472, + "loss": 1.3524, + "theoretical_loss": 3.4725458135785447, + "tokens_seen": 1907163136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042629593965655594, + "loss": 1.2899, + "theoretical_loss": 3.4725258941388137, + "tokens_seen": 1907294208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262558176857647, + "loss": 1.3102, + "theoretical_loss": 3.472505976451188, + "tokens_seen": 1907425280 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042621569571497356, + "loss": 1.3112, + "theoretical_loss": 3.4724860605153935, + "tokens_seen": 1907556352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261755737441823, + "loss": 1.2952, + "theoretical_loss": 3.472466146331155, + "tokens_seen": 1907687424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261354517733911, + "loss": 1.3669, + "theoretical_loss": 3.4724462338981983, + "tokens_seen": 1907818496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260953298025999, + "loss": 1.378, + "theoretical_loss": 3.47242632321625, + "tokens_seen": 1907949568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042605520783180873, + "loss": 1.3138, + "theoretical_loss": 3.472406414285035, + "tokens_seen": 1908080640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260150858610175, + "loss": 1.3119, + "theoretical_loss": 3.4723865071042797, + "tokens_seen": 1908211712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259749638902263, + "loss": 1.2805, + "theoretical_loss": 3.472366601673709, + "tokens_seen": 1908342784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259348419194351, + "loss": 1.268, + "theoretical_loss": 3.47234669799305, + "tokens_seen": 1908473856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042589471994864385, + "loss": 1.4159, + "theoretical_loss": 3.4723267960620277, + "tokens_seen": 1908604928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258545979778527, + "loss": 1.3061, + "theoretical_loss": 3.4723068958803687, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042581447600706146, + "loss": 1.3154, + "theoretical_loss": 3.472286997447799, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257743540362703, + "loss": 1.2809, + "theoretical_loss": 3.4722671007640447, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042573423206547907, + "loss": 1.367, + "theoretical_loss": 3.4722472058288325, + "tokens_seen": 1909129216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256941100946878, + "loss": 1.3606, + "theoretical_loss": 3.472227312641888, + "tokens_seen": 1909260288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256539881238967, + "loss": 1.2769, + "theoretical_loss": 3.4722074212029383, + "tokens_seen": 1909391360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042561386615310543, + "loss": 1.3173, + "theoretical_loss": 3.4721875315117092, + "tokens_seen": 1909522432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042557374418231424, + "loss": 1.3202, + "theoretical_loss": 3.472167643567927, + "tokens_seen": 1909653504 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042553362221152305, + "loss": 1.3188, + "theoretical_loss": 3.472147757371319, + "tokens_seen": 1909784576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549350024073185, + "loss": 1.2144, + "theoretical_loss": 3.472127872921612, + "tokens_seen": 1909915648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254533782699406, + "loss": 1.365, + "theoretical_loss": 3.4721079902185314, + "tokens_seen": 1910046720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254132562991494, + "loss": 1.4012, + "theoretical_loss": 3.4720881092618052, + "tokens_seen": 1910177792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253731343283582, + "loss": 1.3157, + "theoretical_loss": 3.4720682300511596, + "tokens_seen": 1910308864 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533301235756697, + "loss": 1.2584, + "theoretical_loss": 3.4720483525863215, + "tokens_seen": 1910439936 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042529289038677583, + "loss": 1.3359, + "theoretical_loss": 3.4720284768670178, + "tokens_seen": 1910571008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252527684159846, + "loss": 1.2955, + "theoretical_loss": 3.472008602892976, + "tokens_seen": 1910702080 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042521264644519344, + "loss": 1.308, + "theoretical_loss": 3.471988730663922, + "tokens_seen": 1910833152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251725244744022, + "loss": 1.3117, + "theoretical_loss": 3.471968860179584, + "tokens_seen": 1910964224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042513240250361095, + "loss": 1.3498, + "theoretical_loss": 3.471948991439689, + "tokens_seen": 1911095296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250922805328198, + "loss": 1.2988, + "theoretical_loss": 3.471929124443964, + "tokens_seen": 1911226368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042505215856202856, + "loss": 1.2408, + "theoretical_loss": 3.471909259192136, + "tokens_seen": 1911357440 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042501203659123737, + "loss": 1.3281, + "theoretical_loss": 3.4718893956839327, + "tokens_seen": 1911488512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249719146204462, + "loss": 1.3577, + "theoretical_loss": 3.4718695339190813, + "tokens_seen": 1911619584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424931792649655, + "loss": 1.2682, + "theoretical_loss": 3.4718496738973097, + "tokens_seen": 1911750656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042489167067886373, + "loss": 1.3601, + "theoretical_loss": 3.471829815618345, + "tokens_seen": 1911881728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042485154870807254, + "loss": 1.3913, + "theoretical_loss": 3.471809959081915, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9946847558021545, + "objective/train/avg_token_score": 0.005315225105732679, + "objective/train/docs_used": 697086, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3462016582489014, + "objective/train/original_loss": 1.3530187606811523, + "objective/train/theoretical_loss": 3.4717901042877477, + "objective/train/tokens_used": 282602976, + "objective/train/unlikelihood_loss": 0.016289401799440384, + "theoretical_loss": 3.4717901042877477, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042481142673728134, + "loss": 1.3452, + "theoretical_loss": 3.4717901042877477, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247713047664901, + "loss": 1.3767, + "theoretical_loss": 3.47177025123557, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042473118279569896, + "loss": 1.3127, + "theoretical_loss": 3.47175039992511, + "tokens_seen": 1912406016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246910608249077, + "loss": 1.3043, + "theoretical_loss": 3.471730550356096, + "tokens_seen": 1912537088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246509388541165, + "loss": 1.2636, + "theoretical_loss": 3.471710702528256, + "tokens_seen": 1912668160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246108168833253, + "loss": 1.3559, + "theoretical_loss": 3.4716908564413167, + "tokens_seen": 1912799232 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042457069491253413, + "loss": 1.3217, + "theoretical_loss": 3.4716710120950074, + "tokens_seen": 1912930304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042453057294174293, + "loss": 1.3132, + "theoretical_loss": 3.4716511694890553, + "tokens_seen": 1913061376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244904509709517, + "loss": 1.2913, + "theoretical_loss": 3.4716313286231895, + "tokens_seen": 1913192448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244503290001605, + "loss": 1.3655, + "theoretical_loss": 3.4716114894971373, + "tokens_seen": 1913323520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244102070293693, + "loss": 1.291, + "theoretical_loss": 3.471591652110627, + "tokens_seen": 1913454592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243700850585781, + "loss": 1.3178, + "theoretical_loss": 3.471571816463388, + "tokens_seen": 1913585664 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042432996308778686, + "loss": 1.2916, + "theoretical_loss": 3.4715519825551477, + "tokens_seen": 1913716736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242898411169957, + "loss": 1.2686, + "theoretical_loss": 3.471532150385635, + "tokens_seen": 1913847808 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042424971914620447, + "loss": 1.363, + "theoretical_loss": 3.471512319954578, + "tokens_seen": 1913978880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242095971754132, + "loss": 1.2488, + "theoretical_loss": 3.4714924912617056, + "tokens_seen": 1914109952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241694752046221, + "loss": 1.2323, + "theoretical_loss": 3.4714726643067464, + "tokens_seen": 1914241024 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042412935323383083, + "loss": 1.3009, + "theoretical_loss": 3.4714528390894293, + "tokens_seen": 1914372096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042408923126303964, + "loss": 1.3957, + "theoretical_loss": 3.471433015609482, + "tokens_seen": 1914503168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042404910929224845, + "loss": 1.3257, + "theoretical_loss": 3.471413193866635, + "tokens_seen": 1914634240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042400898732145725, + "loss": 1.2311, + "theoretical_loss": 3.4713933738606157, + "tokens_seen": 1914765312 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042396886535066606, + "loss": 1.356, + "theoretical_loss": 3.471373555591154, + "tokens_seen": 1914896384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239287433798748, + "loss": 1.3614, + "theoretical_loss": 3.471353739057978, + "tokens_seen": 1915027456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238886214090836, + "loss": 1.3979, + "theoretical_loss": 3.4713339242608177, + "tokens_seen": 1915158528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238484994382924, + "loss": 1.3612, + "theoretical_loss": 3.471314111199402, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042380837746750123, + "loss": 1.3341, + "theoretical_loss": 3.4712942998734597, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042376825549671, + "loss": 1.3448, + "theoretical_loss": 3.47127449028272, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042372813352591884, + "loss": 1.3177, + "theoretical_loss": 3.4712546824269124, + "tokens_seen": 1915682816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236880115551276, + "loss": 1.3698, + "theoretical_loss": 3.4712348763057665, + "tokens_seen": 1915813888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042364788958433635, + "loss": 1.3896, + "theoretical_loss": 3.471215071919012, + "tokens_seen": 1915944960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236077676135452, + "loss": 1.2916, + "theoretical_loss": 3.4711952692663774, + "tokens_seen": 1916076032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042356764564275396, + "loss": 1.3777, + "theoretical_loss": 3.4711754683475933, + "tokens_seen": 1916207104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042352752367196277, + "loss": 1.3658, + "theoretical_loss": 3.471155669162388, + "tokens_seen": 1916338176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234874017011716, + "loss": 1.2588, + "theoretical_loss": 3.4711358717104925, + "tokens_seen": 1916469248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234472797303804, + "loss": 1.3224, + "theoretical_loss": 3.4711160759916355, + "tokens_seen": 1916600320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042340715775958913, + "loss": 1.3248, + "theoretical_loss": 3.4710962820055475, + "tokens_seen": 1916731392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042336703578879794, + "loss": 1.2725, + "theoretical_loss": 3.471076489751958, + "tokens_seen": 1916862464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042332691381800674, + "loss": 1.359, + "theoretical_loss": 3.4710566992305973, + "tokens_seen": 1916993536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042328679184721555, + "loss": 1.2807, + "theoretical_loss": 3.471036910441195, + "tokens_seen": 1917124608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042324666987642436, + "loss": 1.3329, + "theoretical_loss": 3.471017123383481, + "tokens_seen": 1917255680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232065479056331, + "loss": 1.2901, + "theoretical_loss": 3.470997338057186, + "tokens_seen": 1917386752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042316642593484197, + "loss": 1.3688, + "theoretical_loss": 3.4709775544620394, + "tokens_seen": 1917517824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231263039640507, + "loss": 1.3242, + "theoretical_loss": 3.470957772597772, + "tokens_seen": 1917648896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042308618199325953, + "loss": 1.3728, + "theoretical_loss": 3.4709379924641137, + "tokens_seen": 1917779968 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042304606002246833, + "loss": 1.2953, + "theoretical_loss": 3.470918214060795, + "tokens_seen": 1917911040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230059380516771, + "loss": 1.2875, + "theoretical_loss": 3.4708984373875467, + "tokens_seen": 1918042112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229658160808859, + "loss": 1.3516, + "theoretical_loss": 3.4708786624440986, + "tokens_seen": 1918173184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229256941100947, + "loss": 1.3275, + "theoretical_loss": 3.470858889230181, + "tokens_seen": 1918304256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228855721393035, + "loss": 1.3412, + "theoretical_loss": 3.470839117745525, + "tokens_seen": 1918435328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284545016851226, + "loss": 1.3145, + "theoretical_loss": 3.4708193479898615, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9923063516616821, + "objective/train/avg_token_score": 0.007693628314882517, + "objective/train/docs_used": 699453, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3433722257614136, + "objective/train/original_loss": 1.3538925647735596, + "objective/train/theoretical_loss": 3.4707995799629208, + "objective/train/tokens_used": 289156576, + "objective/train/unlikelihood_loss": 0.020991943776607513, + "theoretical_loss": 3.4707995799629208, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228053281977211, + "loss": 1.3563, + "theoretical_loss": 3.4707995799629208, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042276520622692987, + "loss": 1.3004, + "theoretical_loss": 3.470779813664434, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227250842561386, + "loss": 1.3214, + "theoretical_loss": 3.4707600490941313, + "tokens_seen": 1918959616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226849622853475, + "loss": 1.3416, + "theoretical_loss": 3.4707402862517442, + "tokens_seen": 1919090688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042264484031455623, + "loss": 1.3377, + "theoretical_loss": 3.470720525137003, + "tokens_seen": 1919221760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226047183437651, + "loss": 1.3331, + "theoretical_loss": 3.4707007657496387, + "tokens_seen": 1919352832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042256459637297385, + "loss": 1.318, + "theoretical_loss": 3.4706810080893833, + "tokens_seen": 1919483904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252447440218265, + "loss": 1.2592, + "theoretical_loss": 3.4706612521559674, + "tokens_seen": 1919614976 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248435243139146, + "loss": 1.2762, + "theoretical_loss": 3.4706414979491216, + "tokens_seen": 1919746048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224442304606002, + "loss": 1.3122, + "theoretical_loss": 3.470621745468578, + "tokens_seen": 1919877120 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422404108489809, + "loss": 1.3069, + "theoretical_loss": 3.470601994714068, + "tokens_seen": 1920008192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223639865190178, + "loss": 1.3672, + "theoretical_loss": 3.4705822456853213, + "tokens_seen": 1920139264 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042232386454822663, + "loss": 1.2814, + "theoretical_loss": 3.4705624983820718, + "tokens_seen": 1920270336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222837425774354, + "loss": 1.266, + "theoretical_loss": 3.470542752804049, + "tokens_seen": 1920401408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042224362060664424, + "loss": 1.1932, + "theoretical_loss": 3.470523008950985, + "tokens_seen": 1920532480 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422203498635853, + "loss": 1.3433, + "theoretical_loss": 3.470503266822612, + "tokens_seen": 1920663552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042216337666506175, + "loss": 1.3819, + "theoretical_loss": 3.470483526418661, + "tokens_seen": 1920794624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221232546942706, + "loss": 1.2782, + "theoretical_loss": 3.470463787738864, + "tokens_seen": 1920925696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042208313272347936, + "loss": 1.2889, + "theoretical_loss": 3.4704440507829526, + "tokens_seen": 1921056768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220430107526882, + "loss": 1.3286, + "theoretical_loss": 3.4704243155506584, + "tokens_seen": 1921187840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042200288878189697, + "loss": 1.3148, + "theoretical_loss": 3.4704045820417138, + "tokens_seen": 1921318912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219627668111058, + "loss": 1.3438, + "theoretical_loss": 3.4703848502558508, + "tokens_seen": 1921449984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219226448403146, + "loss": 1.355, + "theoretical_loss": 3.4703651201928016, + "tokens_seen": 1921581056 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042188252286952334, + "loss": 1.28, + "theoretical_loss": 3.4703453918522973, + "tokens_seen": 1921712128 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042184240089873214, + "loss": 1.2665, + "theoretical_loss": 3.470325665234071, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042180227892794095, + "loss": 1.3132, + "theoretical_loss": 3.4703059403378544, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042176215695714976, + "loss": 1.3465, + "theoretical_loss": 3.4702862171633795, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217220349863585, + "loss": 1.3287, + "theoretical_loss": 3.47026649571038, + "tokens_seen": 1922236416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042168191301556737, + "loss": 1.321, + "theoretical_loss": 3.470246775978586, + "tokens_seen": 1922367488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216417910447761, + "loss": 1.3002, + "theoretical_loss": 3.4702270579677323, + "tokens_seen": 1922498560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042160166907398493, + "loss": 1.3263, + "theoretical_loss": 3.4702073416775496, + "tokens_seen": 1922629632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042156154710319373, + "loss": 1.3698, + "theoretical_loss": 3.470187627107772, + "tokens_seen": 1922760704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215214251324025, + "loss": 1.3131, + "theoretical_loss": 3.4701679142581305, + "tokens_seen": 1922891776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214813031616113, + "loss": 1.1869, + "theoretical_loss": 3.4701482031283595, + "tokens_seen": 1923022848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214411811908201, + "loss": 1.4298, + "theoretical_loss": 3.4701284937181898, + "tokens_seen": 1923153920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214010592200289, + "loss": 1.3376, + "theoretical_loss": 3.470108786027356, + "tokens_seen": 1923284992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213609372492377, + "loss": 1.3018, + "theoretical_loss": 3.4700890800555895, + "tokens_seen": 1923416064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213208152784465, + "loss": 1.2134, + "theoretical_loss": 3.4700693758026246, + "tokens_seen": 1923547136 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042128069330765527, + "loss": 1.3038, + "theoretical_loss": 3.4700496732681936, + "tokens_seen": 1923678208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212405713368641, + "loss": 1.2494, + "theoretical_loss": 3.470029972452029, + "tokens_seen": 1923809280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212004493660729, + "loss": 1.2669, + "theoretical_loss": 3.4700102733538642, + "tokens_seen": 1923940352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042116032739528163, + "loss": 1.3027, + "theoretical_loss": 3.4699905759734335, + "tokens_seen": 1924071424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211202054244905, + "loss": 1.324, + "theoretical_loss": 3.4699708803104685, + "tokens_seen": 1924202496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042108008345369925, + "loss": 1.3007, + "theoretical_loss": 3.4699511863647032, + "tokens_seen": 1924333568 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042103996148290805, + "loss": 1.3406, + "theoretical_loss": 3.4699314941358708, + "tokens_seen": 1924464640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042099983951211686, + "loss": 1.3127, + "theoretical_loss": 3.469911803623705, + "tokens_seen": 1924595712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209597175413256, + "loss": 1.3292, + "theoretical_loss": 3.4698921148279394, + "tokens_seen": 1924726784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209195955705344, + "loss": 1.3452, + "theoretical_loss": 3.4698724277483066, + "tokens_seen": 1924857856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208794735997432, + "loss": 1.2956, + "theoretical_loss": 3.469852742384541, + "tokens_seen": 1924988928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042083935162895203, + "loss": 1.3205, + "theoretical_loss": 3.469833058736376, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9933785796165466, + "objective/train/avg_token_score": 0.006621368229389191, + "objective/train/docs_used": 701973, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.39803946018219, + "objective/train/original_loss": 1.4075064659118652, + "objective/train/theoretical_loss": 3.469813376803545, + "objective/train/tokens_used": 295710176, + "objective/train/unlikelihood_loss": 0.019379692152142525, + "theoretical_loss": 3.469813376803545, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207992296581608, + "loss": 1.391, + "theoretical_loss": 3.469813376803545, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042075910768736964, + "loss": 1.2483, + "theoretical_loss": 3.469793696585782, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207189857165784, + "loss": 1.3258, + "theoretical_loss": 3.469774018082821, + "tokens_seen": 1925513216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206788637457872, + "loss": 1.3548, + "theoretical_loss": 3.469754341294396, + "tokens_seen": 1925644288 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420638741774996, + "loss": 1.4057, + "theoretical_loss": 3.46973466622024, + "tokens_seen": 1925775360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042059861980420476, + "loss": 1.3965, + "theoretical_loss": 3.469714992860088, + "tokens_seen": 1925906432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205584978334136, + "loss": 1.2898, + "theoretical_loss": 3.4696953212136736, + "tokens_seen": 1926037504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042051837586262237, + "loss": 1.3301, + "theoretical_loss": 3.469675651280731, + "tokens_seen": 1926168576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204782538918312, + "loss": 1.2665, + "theoretical_loss": 3.469655983060994, + "tokens_seen": 1926299648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042043813192104, + "loss": 1.3637, + "theoretical_loss": 3.4696363165541975, + "tokens_seen": 1926430720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042039800995024874, + "loss": 1.2615, + "theoretical_loss": 3.4696166517600755, + "tokens_seen": 1926561792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042035788797945754, + "loss": 1.3945, + "theoretical_loss": 3.469596988678362, + "tokens_seen": 1926692864 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042031776600866635, + "loss": 1.3111, + "theoretical_loss": 3.4695773273087918, + "tokens_seen": 1926823936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042027764403787516, + "loss": 1.3433, + "theoretical_loss": 3.469557667651099, + "tokens_seen": 1926955008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202375220670839, + "loss": 1.3637, + "theoretical_loss": 3.469538009705019, + "tokens_seen": 1927086080 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042019740009629277, + "loss": 1.3439, + "theoretical_loss": 3.4695183534702854, + "tokens_seen": 1927217152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201572781255015, + "loss": 1.295, + "theoretical_loss": 3.4694986989466328, + "tokens_seen": 1927348224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201171561547103, + "loss": 1.3021, + "theoretical_loss": 3.469479046133797, + "tokens_seen": 1927479296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042007703418391913, + "loss": 1.3878, + "theoretical_loss": 3.4694593950315116, + "tokens_seen": 1927610368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200369122131279, + "loss": 1.3676, + "theoretical_loss": 3.4694397456395123, + "tokens_seen": 1927741440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041999679024233675, + "loss": 1.4015, + "theoretical_loss": 3.469420097957533, + "tokens_seen": 1927872512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199566682715455, + "loss": 1.2377, + "theoretical_loss": 3.46940045198531, + "tokens_seen": 1928003584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199165463007543, + "loss": 1.3534, + "theoretical_loss": 3.469380807722577, + "tokens_seen": 1928134656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198764243299631, + "loss": 1.3397, + "theoretical_loss": 3.4693611651690692, + "tokens_seen": 1928265728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198363023591719, + "loss": 1.3186, + "theoretical_loss": 3.4693415243245225, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041979618038838067, + "loss": 1.3329, + "theoretical_loss": 3.4693218851886716, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197560584175895, + "loss": 1.2765, + "theoretical_loss": 3.4693022477612514, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197159364467983, + "loss": 1.3566, + "theoretical_loss": 3.469282612041998, + "tokens_seen": 1928790016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041967581447600703, + "loss": 1.2957, + "theoretical_loss": 3.4692629780306463, + "tokens_seen": 1928921088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196356925052159, + "loss": 1.392, + "theoretical_loss": 3.469243345726931, + "tokens_seen": 1929052160 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041959557053442465, + "loss": 1.2768, + "theoretical_loss": 3.4692237151305894, + "tokens_seen": 1929183232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041955544856363345, + "loss": 1.2759, + "theoretical_loss": 3.469204086241355, + "tokens_seen": 1929314304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041951532659284226, + "loss": 1.3277, + "theoretical_loss": 3.469184459058965, + "tokens_seen": 1929445376 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419475204622051, + "loss": 1.4305, + "theoretical_loss": 3.4691648335831538, + "tokens_seen": 1929576448 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943508265125987, + "loss": 1.3628, + "theoretical_loss": 3.469145209813658, + "tokens_seen": 1929707520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193949606804686, + "loss": 1.342, + "theoretical_loss": 3.469125587750213, + "tokens_seen": 1929838592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041935483870967743, + "loss": 1.3836, + "theoretical_loss": 3.4691059673925544, + "tokens_seen": 1929969664 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041931471673888624, + "loss": 1.3458, + "theoretical_loss": 3.4690863487404187, + "tokens_seen": 1930100736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927459476809504, + "loss": 1.368, + "theoretical_loss": 3.469066731793541, + "tokens_seen": 1930231808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192344727973038, + "loss": 1.3072, + "theoretical_loss": 3.469047116551658, + "tokens_seen": 1930362880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191943508265126, + "loss": 1.349, + "theoretical_loss": 3.4690275030145052, + "tokens_seen": 1930493952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191542288557214, + "loss": 1.3838, + "theoretical_loss": 3.4690078911818194, + "tokens_seen": 1930625024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041911410688493016, + "loss": 1.3408, + "theoretical_loss": 3.4689882810533366, + "tokens_seen": 1930756096 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419073984914139, + "loss": 1.3778, + "theoretical_loss": 3.4689686726287925, + "tokens_seen": 1930887168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041903386294334777, + "loss": 1.36, + "theoretical_loss": 3.468949065907924, + "tokens_seen": 1931018240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189937409725566, + "loss": 1.312, + "theoretical_loss": 3.4689294608904664, + "tokens_seen": 1931149312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189536190017654, + "loss": 1.2691, + "theoretical_loss": 3.4689098575761577, + "tokens_seen": 1931280384 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041891349703097414, + "loss": 1.3409, + "theoretical_loss": 3.468890255964733, + "tokens_seen": 1931411456 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418873375060183, + "loss": 1.324, + "theoretical_loss": 3.46887065605593, + "tokens_seen": 1931542528 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041883325308939175, + "loss": 1.3459, + "theoretical_loss": 3.468851057849484, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9932452440261841, + "objective/train/avg_token_score": 0.006754749454557896, + "objective/train/docs_used": 704289, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.455161690711975, + "objective/train/original_loss": 1.4666714668273926, + "objective/train/theoretical_loss": 3.4688314613451325, + "objective/train/tokens_used": 302263776, + "objective/train/unlikelihood_loss": 0.015548823401331902, + "theoretical_loss": 3.4688314613451325, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041879313111860056, + "loss": 1.4385, + "theoretical_loss": 3.4688314613451325, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041875300914780936, + "loss": 1.3775, + "theoretical_loss": 3.4688118665426124, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041871288717701817, + "loss": 1.3009, + "theoretical_loss": 3.46879227344166, + "tokens_seen": 1932066816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186727652062269, + "loss": 1.3236, + "theoretical_loss": 3.4687726820420117, + "tokens_seen": 1932197888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186326432354357, + "loss": 1.2215, + "theoretical_loss": 3.4687530923434053, + "tokens_seen": 1932328960 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041859252126464453, + "loss": 1.3768, + "theoretical_loss": 3.468733504345577, + "tokens_seen": 1932460032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185523992938533, + "loss": 1.3125, + "theoretical_loss": 3.4687139180482647, + "tokens_seen": 1932591104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041851227732306215, + "loss": 1.2726, + "theoretical_loss": 3.4686943334512046, + "tokens_seen": 1932722176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184721553522709, + "loss": 1.3541, + "theoretical_loss": 3.4686747505541335, + "tokens_seen": 1932853248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184320333814797, + "loss": 1.3239, + "theoretical_loss": 3.46865516935679, + "tokens_seen": 1932984320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183919114106885, + "loss": 1.3197, + "theoretical_loss": 3.46863558985891, + "tokens_seen": 1933115392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183517894398973, + "loss": 1.3521, + "theoretical_loss": 3.4686160120602314, + "tokens_seen": 1933246464 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041831166746910607, + "loss": 1.3656, + "theoretical_loss": 3.468596435960491, + "tokens_seen": 1933377536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182715454983149, + "loss": 1.2679, + "theoretical_loss": 3.468576861559427, + "tokens_seen": 1933508608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182314235275237, + "loss": 1.2822, + "theoretical_loss": 3.4685572888567764, + "tokens_seen": 1933639680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181913015567325, + "loss": 1.3397, + "theoretical_loss": 3.468537717852277, + "tokens_seen": 1933770752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181511795859413, + "loss": 1.3367, + "theoretical_loss": 3.468518148545666, + "tokens_seen": 1933901824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041811105761515005, + "loss": 1.3899, + "theoretical_loss": 3.4684985809366813, + "tokens_seen": 1934032896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180709356443589, + "loss": 1.396, + "theoretical_loss": 3.4684790150250606, + "tokens_seen": 1934163968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041803081367356766, + "loss": 1.2989, + "theoretical_loss": 3.468459450810541, + "tokens_seen": 1934295040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179906917027764, + "loss": 1.3248, + "theoretical_loss": 3.4684398882928615, + "tokens_seen": 1934426112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041795056973198527, + "loss": 1.2803, + "theoretical_loss": 3.468420327471759, + "tokens_seen": 1934557184 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417910447761194, + "loss": 1.374, + "theoretical_loss": 3.4684007683469718, + "tokens_seen": 1934688256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041787032579040283, + "loss": 1.3341, + "theoretical_loss": 3.4683812109182384, + "tokens_seen": 1934819328 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783020381961164, + "loss": 1.2583, + "theoretical_loss": 3.4683616551852956, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041779008184882044, + "loss": 1.3563, + "theoretical_loss": 3.4683421011478828, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177499598780292, + "loss": 1.2836, + "theoretical_loss": 3.4683225488057374, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417709837907238, + "loss": 1.3018, + "theoretical_loss": 3.4683029981585975, + "tokens_seen": 1935343616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176697159364468, + "loss": 1.3091, + "theoretical_loss": 3.4682834492062016, + "tokens_seen": 1935474688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041762959396565556, + "loss": 1.2516, + "theoretical_loss": 3.468263901948289, + "tokens_seen": 1935605760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175894719948644, + "loss": 1.3567, + "theoretical_loss": 3.468244356384596, + "tokens_seen": 1935736832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041754935002407317, + "loss": 1.2726, + "theoretical_loss": 3.468224812514863, + "tokens_seen": 1935867904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041750922805328203, + "loss": 1.2692, + "theoretical_loss": 3.468205270338828, + "tokens_seen": 1935998976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174691060824908, + "loss": 1.3078, + "theoretical_loss": 3.468185729856229, + "tokens_seen": 1936130048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041742898411169954, + "loss": 1.3566, + "theoretical_loss": 3.4681661910668047, + "tokens_seen": 1936261120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173888621409084, + "loss": 1.2998, + "theoretical_loss": 3.4681466539702948, + "tokens_seen": 1936392192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041734874017011715, + "loss": 1.2762, + "theoretical_loss": 3.4681271185664366, + "tokens_seen": 1936523264 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041730861819932596, + "loss": 1.3554, + "theoretical_loss": 3.4681075848549696, + "tokens_seen": 1936654336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041726849622853476, + "loss": 1.3268, + "theoretical_loss": 3.4680880528356326, + "tokens_seen": 1936785408 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041722837425774357, + "loss": 1.3241, + "theoretical_loss": 3.4680685225081653, + "tokens_seen": 1936916480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171882522869523, + "loss": 1.3246, + "theoretical_loss": 3.468048993872306, + "tokens_seen": 1937047552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171481303161611, + "loss": 1.3636, + "theoretical_loss": 3.468029466927793, + "tokens_seen": 1937178624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041710800834536993, + "loss": 1.2741, + "theoretical_loss": 3.468009941674366, + "tokens_seen": 1937309696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170678863745787, + "loss": 1.3448, + "theoretical_loss": 3.4679904181117642, + "tokens_seen": 1937440768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041702776440378755, + "loss": 1.3414, + "theoretical_loss": 3.467970896239727, + "tokens_seen": 1937571840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169876424329963, + "loss": 1.3351, + "theoretical_loss": 3.467951376057994, + "tokens_seen": 1937702912 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041694752046220516, + "loss": 1.3619, + "theoretical_loss": 3.467931857566304, + "tokens_seen": 1937833984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169073984914139, + "loss": 1.3112, + "theoretical_loss": 3.4679123407643955, + "tokens_seen": 1937965056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168672765206227, + "loss": 1.337, + "theoretical_loss": 3.46789282565201, + "tokens_seen": 1938096128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168271545498315, + "loss": 1.3156, + "theoretical_loss": 3.467873312228885, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9916789531707764, + "objective/train/avg_token_score": 0.0083210663869977, + "objective/train/docs_used": 706849, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2909971475601196, + "objective/train/original_loss": 1.3043739795684814, + "objective/train/theoretical_loss": 3.467853800494761, + "objective/train/tokens_used": 308817376, + "objective/train/unlikelihood_loss": 0.02125922590494156, + "theoretical_loss": 3.467853800494761, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167870325790403, + "loss": 1.252, + "theoretical_loss": 3.467853800494761, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167469106082491, + "loss": 1.348, + "theoretical_loss": 3.4678342904493777, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167067886374579, + "loss": 1.2547, + "theoretical_loss": 3.4678147820924745, + "tokens_seen": 1938620416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166666666666667, + "loss": 1.3676, + "theoretical_loss": 3.467795275423791, + "tokens_seen": 1938751488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041662654469587545, + "loss": 1.3307, + "theoretical_loss": 3.4677757704430676, + "tokens_seen": 1938882560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165864227250843, + "loss": 1.2879, + "theoretical_loss": 3.467756267150044, + "tokens_seen": 1939013632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654630075429306, + "loss": 1.3512, + "theoretical_loss": 3.467736765544459, + "tokens_seen": 1939144704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165061787835018, + "loss": 1.2889, + "theoretical_loss": 3.467717265626054, + "tokens_seen": 1939275776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041646605681271067, + "loss": 1.2307, + "theoretical_loss": 3.4676977673945677, + "tokens_seen": 1939406848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164259348419194, + "loss": 1.3126, + "theoretical_loss": 3.467678270849741, + "tokens_seen": 1939537920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638581287112823, + "loss": 1.3639, + "theoretical_loss": 3.467658775991315, + "tokens_seen": 1939668992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041634569090033704, + "loss": 1.3357, + "theoretical_loss": 3.4676392828190274, + "tokens_seen": 1939800064 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041630556892954584, + "loss": 1.338, + "theoretical_loss": 3.4676197913326208, + "tokens_seen": 1939931136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041626544695875465, + "loss": 1.3097, + "theoretical_loss": 3.467600301531834, + "tokens_seen": 1940062208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162253249879634, + "loss": 1.3177, + "theoretical_loss": 3.467580813416408, + "tokens_seen": 1940193280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161852030171722, + "loss": 1.3419, + "theoretical_loss": 3.4675613269860834, + "tokens_seen": 1940324352 + }, + { + "epoch": 0.18, + "learning_rate": 0.000416145081046381, + "loss": 1.3644, + "theoretical_loss": 3.4675418422405997, + "tokens_seen": 1940455424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161049590755898, + "loss": 1.3973, + "theoretical_loss": 3.4675223591796986, + "tokens_seen": 1940586496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041606483710479857, + "loss": 1.3243, + "theoretical_loss": 3.4675028778031205, + "tokens_seen": 1940717568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041602471513400743, + "loss": 1.3734, + "theoretical_loss": 3.4674833981106055, + "tokens_seen": 1940848640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159845931632162, + "loss": 1.3448, + "theoretical_loss": 3.4674639201018946, + "tokens_seen": 1940979712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041594447119242494, + "loss": 1.2804, + "theoretical_loss": 3.4674444437767282, + "tokens_seen": 1941110784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159043492216338, + "loss": 1.3052, + "theoretical_loss": 3.467424969134848, + "tokens_seen": 1941241856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041586422725084255, + "loss": 1.341, + "theoretical_loss": 3.467405496175994, + "tokens_seen": 1941372928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041582410528005135, + "loss": 1.2733, + "theoretical_loss": 3.4673860248999073, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041578398330926016, + "loss": 1.3016, + "theoretical_loss": 3.4673665553063295, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041574386133846897, + "loss": 1.3094, + "theoretical_loss": 3.4673470873950007, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157037393676778, + "loss": 1.2912, + "theoretical_loss": 3.467327621165663, + "tokens_seen": 1941897216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156636173968865, + "loss": 1.2813, + "theoretical_loss": 3.4673081566180572, + "tokens_seen": 1942028288 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041562349542609533, + "loss": 1.3125, + "theoretical_loss": 3.4672886937519243, + "tokens_seen": 1942159360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041558337345530414, + "loss": 1.2671, + "theoretical_loss": 3.4672692325670056, + "tokens_seen": 1942290432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041554325148451294, + "loss": 1.2156, + "theoretical_loss": 3.467249773063042, + "tokens_seen": 1942421504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155031295137217, + "loss": 1.3015, + "theoretical_loss": 3.4672303152397763, + "tokens_seen": 1942552576 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041546300754293056, + "loss": 1.3837, + "theoretical_loss": 3.467210859096948, + "tokens_seen": 1942683648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154228855721393, + "loss": 1.345, + "theoretical_loss": 3.467191404634301, + "tokens_seen": 1942814720 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041538276360134806, + "loss": 1.3282, + "theoretical_loss": 3.4671719518515745, + "tokens_seen": 1942945792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153426416305569, + "loss": 1.2656, + "theoretical_loss": 3.4671525007485116, + "tokens_seen": 1943076864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153025196597657, + "loss": 1.3412, + "theoretical_loss": 3.4671330513248533, + "tokens_seen": 1943207936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152623976889745, + "loss": 1.29, + "theoretical_loss": 3.4671136035803416, + "tokens_seen": 1943339008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152222757181833, + "loss": 1.3198, + "theoretical_loss": 3.4670941575147185, + "tokens_seen": 1943470080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151821537473921, + "loss": 1.2943, + "theoretical_loss": 3.4670747131277255, + "tokens_seen": 1943601152 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041514203177660085, + "loss": 1.378, + "theoretical_loss": 3.4670552704191047, + "tokens_seen": 1943732224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151019098058097, + "loss": 1.2797, + "theoretical_loss": 3.467035829388598, + "tokens_seen": 1943863296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041506178783501846, + "loss": 1.3695, + "theoretical_loss": 3.4670163900359476, + "tokens_seen": 1943994368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041502166586422726, + "loss": 1.3513, + "theoretical_loss": 3.466996952360895, + "tokens_seen": 1944125440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498154389343607, + "loss": 1.2888, + "theoretical_loss": 3.4669775163631833, + "tokens_seen": 1944256512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149414219226448, + "loss": 1.2717, + "theoretical_loss": 3.466958082042553, + "tokens_seen": 1944387584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149012999518537, + "loss": 1.306, + "theoretical_loss": 3.466938649398749, + "tokens_seen": 1944518656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041486117798106244, + "loss": 1.3171, + "theoretical_loss": 3.466919218431511, + "tokens_seen": 1944649728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041482105601027124, + "loss": 1.3289, + "theoretical_loss": 3.466899789140583, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9927396774291992, + "objective/train/avg_token_score": 0.007260307669639587, + "objective/train/docs_used": 709206, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3977434635162354, + "objective/train/original_loss": 1.4082019329071045, + "objective/train/theoretical_loss": 3.466880361525707, + "objective/train/tokens_used": 315370976, + "objective/train/unlikelihood_loss": 0.01832396164536476, + "theoretical_loss": 3.466880361525707, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041478093403948005, + "loss": 1.3896, + "theoretical_loss": 3.466880361525707, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147408120686888, + "loss": 1.4296, + "theoretical_loss": 3.4668609355866247, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147006900978976, + "loss": 1.3437, + "theoretical_loss": 3.4668415113230795, + "tokens_seen": 1945174016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146605681271064, + "loss": 1.3863, + "theoretical_loss": 3.466822088734814, + "tokens_seen": 1945305088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146204461563152, + "loss": 1.3215, + "theoretical_loss": 3.4668026678215704, + "tokens_seen": 1945436160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041458032418552397, + "loss": 1.2821, + "theoretical_loss": 3.466783248583092, + "tokens_seen": 1945567232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454020221473283, + "loss": 1.3878, + "theoretical_loss": 3.4667638310191204, + "tokens_seen": 1945698304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145000802439416, + "loss": 1.2511, + "theoretical_loss": 3.4667444151294005, + "tokens_seen": 1945829376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041445995827315034, + "loss": 1.3386, + "theoretical_loss": 3.466725000913673, + "tokens_seen": 1945960448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144198363023592, + "loss": 1.2793, + "theoretical_loss": 3.466705588371682, + "tokens_seen": 1946091520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041437971433156795, + "loss": 1.3352, + "theoretical_loss": 3.46668617750317, + "tokens_seen": 1946222592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143395923607768, + "loss": 1.3474, + "theoretical_loss": 3.4666667683078805, + "tokens_seen": 1946353664 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041429947038998556, + "loss": 1.3719, + "theoretical_loss": 3.466647360785556, + "tokens_seen": 1946484736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041425934841919437, + "loss": 1.3277, + "theoretical_loss": 3.4666279549359404, + "tokens_seen": 1946615808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142192264484032, + "loss": 1.3358, + "theoretical_loss": 3.4666085507587767, + "tokens_seen": 1946746880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141791044776119, + "loss": 1.3666, + "theoretical_loss": 3.466589148253808, + "tokens_seen": 1946877952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041413898250682073, + "loss": 1.3299, + "theoretical_loss": 3.4665697474207775, + "tokens_seen": 1947009024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041409886053602954, + "loss": 1.2941, + "theoretical_loss": 3.466550348259428, + "tokens_seen": 1947140096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041405873856523834, + "loss": 1.3241, + "theoretical_loss": 3.4665309507695046, + "tokens_seen": 1947271168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140186165944471, + "loss": 1.3275, + "theoretical_loss": 3.46651155495075, + "tokens_seen": 1947402240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041397849462365596, + "loss": 1.4136, + "theoretical_loss": 3.466492160802907, + "tokens_seen": 1947533312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139383726528647, + "loss": 1.3479, + "theoretical_loss": 3.4664727683257204, + "tokens_seen": 1947664384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041389825068207346, + "loss": 1.3427, + "theoretical_loss": 3.466453377518933, + "tokens_seen": 1947795456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138581287112823, + "loss": 1.4095, + "theoretical_loss": 3.466433988382289, + "tokens_seen": 1947926528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138180067404911, + "loss": 1.3766, + "theoretical_loss": 3.466414600915532, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041377788476969993, + "loss": 1.2902, + "theoretical_loss": 3.466395215118406, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137377627989087, + "loss": 1.3272, + "theoretical_loss": 3.4663758309906543, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136976408281175, + "loss": 1.3408, + "theoretical_loss": 3.4663564485320215, + "tokens_seen": 1948450816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136575188573263, + "loss": 1.3721, + "theoretical_loss": 3.466337067742252, + "tokens_seen": 1948581888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136173968865351, + "loss": 1.409, + "theoretical_loss": 3.4663176886210887, + "tokens_seen": 1948712960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041357727491574386, + "loss": 1.3158, + "theoretical_loss": 3.4662983111682766, + "tokens_seen": 1948844032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041353715294495266, + "loss": 1.2627, + "theoretical_loss": 3.4662789353835595, + "tokens_seen": 1948975104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041349703097416147, + "loss": 1.3641, + "theoretical_loss": 3.466259561266682, + "tokens_seen": 1949106176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134569090033702, + "loss": 1.3048, + "theoretical_loss": 3.4662401888173875, + "tokens_seen": 1949237248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134167870325791, + "loss": 1.3501, + "theoretical_loss": 3.4662208180354215, + "tokens_seen": 1949368320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041337666506178783, + "loss": 1.3474, + "theoretical_loss": 3.4662014489205277, + "tokens_seen": 1949499392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041333654309099664, + "loss": 1.304, + "theoretical_loss": 3.466182081472451, + "tokens_seen": 1949630464 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041329642112020545, + "loss": 1.2913, + "theoretical_loss": 3.4661627156909356, + "tokens_seen": 1949761536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132562991494142, + "loss": 1.2583, + "theoretical_loss": 3.4661433515757256, + "tokens_seen": 1949892608 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413216177178623, + "loss": 1.2863, + "theoretical_loss": 3.4661239891265665, + "tokens_seen": 1950023680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131760552078318, + "loss": 1.3268, + "theoretical_loss": 3.466104628343203, + "tokens_seen": 1950154752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131359332370406, + "loss": 1.4032, + "theoretical_loss": 3.466085269225379, + "tokens_seen": 1950285824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130958112662494, + "loss": 1.3658, + "theoretical_loss": 3.46606591177284, + "tokens_seen": 1950416896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041305568929545823, + "loss": 1.3068, + "theoretical_loss": 3.4660465559853306, + "tokens_seen": 1950547968 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413015567324667, + "loss": 1.3578, + "theoretical_loss": 3.4660272018625955, + "tokens_seen": 1950679040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129754453538758, + "loss": 1.404, + "theoretical_loss": 3.4660078494043804, + "tokens_seen": 1950810112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129353233830846, + "loss": 1.2982, + "theoretical_loss": 3.4659884986104297, + "tokens_seen": 1950941184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041289520141229335, + "loss": 1.384, + "theoretical_loss": 3.4659691494804887, + "tokens_seen": 1951072256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128550794415022, + "loss": 1.2605, + "theoretical_loss": 3.4659498020143023, + "tokens_seen": 1951203328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041281495747071096, + "loss": 1.3831, + "theoretical_loss": 3.4659304562116158, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9904803037643433, + "objective/train/avg_token_score": 0.009519672021269798, + "objective/train/docs_used": 711575, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2682979106903076, + "objective/train/original_loss": 1.279581069946289, + "objective/train/theoretical_loss": 3.465911112072175, + "objective/train/tokens_used": 321924576, + "objective/train/unlikelihood_loss": 0.02809436246752739, + "theoretical_loss": 3.465911112072175, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041277483549991977, + "loss": 1.3013, + "theoretical_loss": 3.465911112072175, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127347135291286, + "loss": 1.3312, + "theoretical_loss": 3.465891769595724, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126945915583373, + "loss": 1.3498, + "theoretical_loss": 3.465872428782009, + "tokens_seen": 1951727616 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041265446958754613, + "loss": 1.2928, + "theoretical_loss": 3.4658530896307758, + "tokens_seen": 1951858688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041261434761675494, + "loss": 1.263, + "theoretical_loss": 3.465833752141769, + "tokens_seen": 1951989760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041257422564596374, + "loss": 1.3996, + "theoretical_loss": 3.465814416314734, + "tokens_seen": 1952120832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125341036751725, + "loss": 1.3561, + "theoretical_loss": 3.465795082149418, + "tokens_seen": 1952251904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041249398170438136, + "loss": 1.3582, + "theoretical_loss": 3.4657757496455646, + "tokens_seen": 1952382976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124538597335901, + "loss": 1.3554, + "theoretical_loss": 3.4657564188029206, + "tokens_seen": 1952514048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124137377627989, + "loss": 1.3194, + "theoretical_loss": 3.465737089621232, + "tokens_seen": 1952645120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123736157920077, + "loss": 1.3572, + "theoretical_loss": 3.465717762100244, + "tokens_seen": 1952776192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123334938212165, + "loss": 1.3686, + "theoretical_loss": 3.465698436239702, + "tokens_seen": 1952907264 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041229337185042533, + "loss": 1.3249, + "theoretical_loss": 3.465679112039353, + "tokens_seen": 1953038336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122532498796341, + "loss": 1.3526, + "theoretical_loss": 3.4656597894989427, + "tokens_seen": 1953169408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122131279088429, + "loss": 1.331, + "theoretical_loss": 3.4656404686182167, + "tokens_seen": 1953300480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121730059380517, + "loss": 1.3739, + "theoretical_loss": 3.4656211493969216, + "tokens_seen": 1953431552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121328839672605, + "loss": 1.3666, + "theoretical_loss": 3.465601831834803, + "tokens_seen": 1953562624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041209276199646926, + "loss": 1.3488, + "theoretical_loss": 3.465582515931608, + "tokens_seen": 1953693696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041205264002567806, + "loss": 1.3061, + "theoretical_loss": 3.4655632016870817, + "tokens_seen": 1953824768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041201251805488687, + "loss": 1.2845, + "theoretical_loss": 3.465543889100971, + "tokens_seen": 1953955840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119723960840956, + "loss": 1.3705, + "theoretical_loss": 3.4655245781730226, + "tokens_seen": 1954086912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119322741133045, + "loss": 1.313, + "theoretical_loss": 3.465505268902982, + "tokens_seen": 1954217984 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041189215214251323, + "loss": 1.3501, + "theoretical_loss": 3.4654859612905966, + "tokens_seen": 1954349056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118520301717221, + "loss": 1.3748, + "theoretical_loss": 3.4654666553356126, + "tokens_seen": 1954480128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041181190820093085, + "loss": 1.3574, + "theoretical_loss": 3.465447351037776, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117717862301396, + "loss": 1.2914, + "theoretical_loss": 3.4654280483968347, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041173166425934846, + "loss": 1.4677, + "theoretical_loss": 3.4654087474125346, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116915422885572, + "loss": 1.3408, + "theoretical_loss": 3.4653894480846223, + "tokens_seen": 1955004416 + }, + { + "epoch": 0.18, + "learning_rate": 0.000411651420317766, + "loss": 1.4118, + "theoretical_loss": 3.465370150412845, + "tokens_seen": 1955135488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116112983469748, + "loss": 1.3365, + "theoretical_loss": 3.4653508543969496, + "tokens_seen": 1955266560 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041157117637618363, + "loss": 1.3152, + "theoretical_loss": 3.4653315600366827, + "tokens_seen": 1955397632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115310544053924, + "loss": 1.2872, + "theoretical_loss": 3.4653122673317913, + "tokens_seen": 1955528704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114909324346012, + "loss": 1.3461, + "theoretical_loss": 3.465292976282023, + "tokens_seen": 1955659776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041145081046381, + "loss": 1.3421, + "theoretical_loss": 3.465273686887124, + "tokens_seen": 1955790848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041141068849301875, + "loss": 1.3201, + "theoretical_loss": 3.4652543991468425, + "tokens_seen": 1955921920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113705665222276, + "loss": 1.3006, + "theoretical_loss": 3.465235113060925, + "tokens_seen": 1956052992 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041133044455143636, + "loss": 1.2942, + "theoretical_loss": 3.4652158286291184, + "tokens_seen": 1956184064 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041129032258064517, + "loss": 1.2828, + "theoretical_loss": 3.465196545851171, + "tokens_seen": 1956315136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125020060985397, + "loss": 1.3409, + "theoretical_loss": 3.4651772647268295, + "tokens_seen": 1956446208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112100786390627, + "loss": 1.2901, + "theoretical_loss": 3.4651579852558414, + "tokens_seen": 1956577280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111699566682716, + "loss": 1.3418, + "theoretical_loss": 3.4651387074379545, + "tokens_seen": 1956708352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041112983469748034, + "loss": 1.3196, + "theoretical_loss": 3.4651194312729157, + "tokens_seen": 1956839424 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041108971272668914, + "loss": 1.3742, + "theoretical_loss": 3.4651001567604736, + "tokens_seen": 1956970496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041104959075589795, + "loss": 1.3387, + "theoretical_loss": 3.465080883900375, + "tokens_seen": 1957101568 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041100946878510676, + "loss": 1.3615, + "theoretical_loss": 3.4650616126923675, + "tokens_seen": 1957232640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109693468143155, + "loss": 1.3753, + "theoretical_loss": 3.4650423431362, + "tokens_seen": 1957363712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109292248435243, + "loss": 1.3885, + "theoretical_loss": 3.4650230752316187, + "tokens_seen": 1957494784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108891028727331, + "loss": 1.316, + "theoretical_loss": 3.4650038089783726, + "tokens_seen": 1957625856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108489809019419, + "loss": 1.3022, + "theoretical_loss": 3.4649845443762093, + "tokens_seen": 1957756928 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041080885893115073, + "loss": 1.3487, + "theoretical_loss": 3.4649652814248775, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9926722049713135, + "objective/train/avg_token_score": 0.007327770348638296, + "objective/train/docs_used": 713999, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3812774419784546, + "objective/train/original_loss": 1.3924243450164795, + "objective/train/theoretical_loss": 3.4649460201241236, + "objective/train/tokens_used": 328478176, + "objective/train/unlikelihood_loss": 0.01838264986872673, + "theoretical_loss": 3.4649460201241236, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107687369603595, + "loss": 1.3648, + "theoretical_loss": 3.4649460201241236, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107286149895683, + "loss": 1.3206, + "theoretical_loss": 3.4649267604736966, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106884930187771, + "loss": 1.3793, + "theoretical_loss": 3.4649075024733453, + "tokens_seen": 1958281216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041064837104798585, + "loss": 1.3931, + "theoretical_loss": 3.4648882461228174, + "tokens_seen": 1958412288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106082490771947, + "loss": 1.3333, + "theoretical_loss": 3.4648689914218602, + "tokens_seen": 1958543360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041056812710640346, + "loss": 1.2875, + "theoretical_loss": 3.4648497383702237, + "tokens_seen": 1958674432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041052800513561227, + "loss": 1.3996, + "theoretical_loss": 3.4648304869676547, + "tokens_seen": 1958805504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104878831648211, + "loss": 1.445, + "theoretical_loss": 3.464811237213903, + "tokens_seen": 1958936576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104477611940299, + "loss": 1.3302, + "theoretical_loss": 3.4647919891087158, + "tokens_seen": 1959067648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041040763922323863, + "loss": 1.3719, + "theoretical_loss": 3.464772742651842, + "tokens_seen": 1959198720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103675172524475, + "loss": 1.3954, + "theoretical_loss": 3.4647534978430308, + "tokens_seen": 1959329792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041032739528165625, + "loss": 1.3688, + "theoretical_loss": 3.464734254682031, + "tokens_seen": 1959460864 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410287273310865, + "loss": 1.3101, + "theoretical_loss": 3.4647150131685898, + "tokens_seen": 1959591936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041024715134007386, + "loss": 1.3073, + "theoretical_loss": 3.4646957733024575, + "tokens_seen": 1959723008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102070293692826, + "loss": 1.4082, + "theoretical_loss": 3.4646765350833824, + "tokens_seen": 1959854080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101669073984914, + "loss": 1.3186, + "theoretical_loss": 3.4646572985111126, + "tokens_seen": 1959985152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101267854277002, + "loss": 1.2753, + "theoretical_loss": 3.464638063585398, + "tokens_seen": 1960116224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041008666345690903, + "loss": 1.2744, + "theoretical_loss": 3.464618830305988, + "tokens_seen": 1960247296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100465414861178, + "loss": 1.3409, + "theoretical_loss": 3.4645995986726303, + "tokens_seen": 1960378368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100064195153266, + "loss": 1.3492, + "theoretical_loss": 3.4645803686850747, + "tokens_seen": 1960509440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099662975445354, + "loss": 1.3409, + "theoretical_loss": 3.4645611403430703, + "tokens_seen": 1960640512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099261755737442, + "loss": 1.328, + "theoretical_loss": 3.464541913646366, + "tokens_seen": 1960771584 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409886053602953, + "loss": 1.2824, + "theoretical_loss": 3.4645226885947116, + "tokens_seen": 1960902656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040984593163216176, + "loss": 1.3908, + "theoretical_loss": 3.464503465187856, + "tokens_seen": 1961033728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098058096613706, + "loss": 1.3466, + "theoretical_loss": 3.464484243425548, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040976568769057937, + "loss": 1.3013, + "theoretical_loss": 3.464465023307538, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097255657197881, + "loss": 1.3481, + "theoretical_loss": 3.4644458048335753, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409685443748997, + "loss": 1.3018, + "theoretical_loss": 3.464426588003409, + "tokens_seen": 1961558016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040964532177820574, + "loss": 1.3377, + "theoretical_loss": 3.4644073728167895, + "tokens_seen": 1961689088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040960519980741454, + "loss": 1.3105, + "theoretical_loss": 3.464388159273465, + "tokens_seen": 1961820160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040956507783662335, + "loss": 1.378, + "theoretical_loss": 3.4643689473731865, + "tokens_seen": 1961951232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040952495586583216, + "loss": 1.3665, + "theoretical_loss": 3.464349737115703, + "tokens_seen": 1962082304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094848338950409, + "loss": 1.3669, + "theoretical_loss": 3.464330528500765, + "tokens_seen": 1962213376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094447119242497, + "loss": 1.3218, + "theoretical_loss": 3.464311321528122, + "tokens_seen": 1962344448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094045899534585, + "loss": 1.2965, + "theoretical_loss": 3.464292116197523, + "tokens_seen": 1962475520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040936446798266727, + "loss": 1.3223, + "theoretical_loss": 3.4642729125087195, + "tokens_seen": 1962606592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040932434601187613, + "loss": 1.2611, + "theoretical_loss": 3.4642537104614606, + "tokens_seen": 1962737664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092842240410849, + "loss": 1.3232, + "theoretical_loss": 3.4642345100554963, + "tokens_seen": 1962868736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040924410207029375, + "loss": 1.3135, + "theoretical_loss": 3.464215311290577, + "tokens_seen": 1962999808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092039800995025, + "loss": 1.3702, + "theoretical_loss": 3.4641961141664535, + "tokens_seen": 1963130880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040916385812871125, + "loss": 1.3453, + "theoretical_loss": 3.464176918682875, + "tokens_seen": 1963261952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091237361579201, + "loss": 1.361, + "theoretical_loss": 3.464157724839592, + "tokens_seen": 1963393024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040908361418712886, + "loss": 1.3435, + "theoretical_loss": 3.4641385326363556, + "tokens_seen": 1963524096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040904349221633767, + "loss": 1.3629, + "theoretical_loss": 3.464119342072915, + "tokens_seen": 1963655168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090033702455465, + "loss": 1.2607, + "theoretical_loss": 3.4641001531490216, + "tokens_seen": 1963786240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089632482747553, + "loss": 1.2939, + "theoretical_loss": 3.464080965864426, + "tokens_seen": 1963917312 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040892312630396403, + "loss": 1.3586, + "theoretical_loss": 3.464061780218878, + "tokens_seen": 1964048384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088830043331729, + "loss": 1.3456, + "theoretical_loss": 3.4640425962121286, + "tokens_seen": 1964179456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040884288236238165, + "loss": 1.3379, + "theoretical_loss": 3.464023413843928, + "tokens_seen": 1964310528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088027603915904, + "loss": 1.3494, + "theoretical_loss": 3.464004233114028, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9915092587471008, + "objective/train/avg_token_score": 0.008490774780511856, + "objective/train/docs_used": 716326, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3092796802520752, + "objective/train/original_loss": 1.3212692737579346, + "objective/train/theoretical_loss": 3.463985054022179, + "objective/train/tokens_used": 335031776, + "objective/train/unlikelihood_loss": 0.02470581792294979, + "theoretical_loss": 3.463985054022179, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040876263842079926, + "loss": 1.3444, + "theoretical_loss": 3.463985054022179, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408722516450008, + "loss": 1.3296, + "theoretical_loss": 3.463965876568131, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040868239447921687, + "loss": 1.3136, + "theoretical_loss": 3.463946700751636, + "tokens_seen": 1964834816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086422725084256, + "loss": 1.3265, + "theoretical_loss": 3.463927526572444, + "tokens_seen": 1964965888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040860215053763443, + "loss": 1.3961, + "theoretical_loss": 3.4639083540303064, + "tokens_seen": 1965096960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040856202856684324, + "loss": 1.3427, + "theoretical_loss": 3.4638891831249747, + "tokens_seen": 1965228032 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408521906596052, + "loss": 1.3191, + "theoretical_loss": 3.4638700138561993, + "tokens_seen": 1965359104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084817846252608, + "loss": 1.3711, + "theoretical_loss": 3.463850846223732, + "tokens_seen": 1965490176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084416626544696, + "loss": 1.2468, + "theoretical_loss": 3.4638316802273232, + "tokens_seen": 1965621248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084015406836784, + "loss": 1.2762, + "theoretical_loss": 3.4638125158667252, + "tokens_seen": 1965752320 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040836141871288716, + "loss": 1.3392, + "theoretical_loss": 3.4637933531416882, + "tokens_seen": 1965883392 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408321296742096, + "loss": 1.3401, + "theoretical_loss": 3.4637741920519645, + "tokens_seen": 1966014464 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040828117477130477, + "loss": 1.2611, + "theoretical_loss": 3.463755032597305, + "tokens_seen": 1966145536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082410528005135, + "loss": 1.3636, + "theoretical_loss": 3.463735874777462, + "tokens_seen": 1966276608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082009308297224, + "loss": 1.345, + "theoretical_loss": 3.4637167185921856, + "tokens_seen": 1966407680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040816080885893114, + "loss": 1.3441, + "theoretical_loss": 3.4636975640412286, + "tokens_seen": 1966538752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040812068688813994, + "loss": 1.2818, + "theoretical_loss": 3.4636784111243424, + "tokens_seen": 1966669824 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040808056491734875, + "loss": 1.3032, + "theoretical_loss": 3.463659259841279, + "tokens_seen": 1966800896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804044294655756, + "loss": 1.3479, + "theoretical_loss": 3.463640110191789, + "tokens_seen": 1966931968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040800032097576636, + "loss": 1.3563, + "theoretical_loss": 3.463620962175625, + "tokens_seen": 1967063040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079601990049751, + "loss": 1.3532, + "theoretical_loss": 3.463601815792539, + "tokens_seen": 1967194112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079200770341839, + "loss": 1.3133, + "theoretical_loss": 3.463582671042283, + "tokens_seen": 1967325184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078799550633927, + "loss": 1.2705, + "theoretical_loss": 3.463563527924608, + "tokens_seen": 1967456256 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040783983309260153, + "loss": 1.4414, + "theoretical_loss": 3.463544386439268, + "tokens_seen": 1967587328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077997111218103, + "loss": 1.3389, + "theoretical_loss": 3.4635252465860127, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040775958915101915, + "loss": 1.3684, + "theoretical_loss": 3.4635061083645957, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077194671802279, + "loss": 1.289, + "theoretical_loss": 3.463486971774769, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040767934520943665, + "loss": 1.3962, + "theoretical_loss": 3.463467836816285, + "tokens_seen": 1968111616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076392232386455, + "loss": 1.3077, + "theoretical_loss": 3.463448703488895, + "tokens_seen": 1968242688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040759910126785426, + "loss": 1.3297, + "theoretical_loss": 3.4634295717923527, + "tokens_seen": 1968373760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040755897929706307, + "loss": 1.3422, + "theoretical_loss": 3.4634104417264098, + "tokens_seen": 1968504832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075188573262719, + "loss": 1.3478, + "theoretical_loss": 3.4633913132908187, + "tokens_seen": 1968635904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074787353554807, + "loss": 1.2987, + "theoretical_loss": 3.463372186485332, + "tokens_seen": 1968766976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074386133846895, + "loss": 1.3046, + "theoretical_loss": 3.4633530613097028, + "tokens_seen": 1968898048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073984914138983, + "loss": 1.2749, + "theoretical_loss": 3.463333937763683, + "tokens_seen": 1969029120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040735836944310705, + "loss": 1.3319, + "theoretical_loss": 3.463314815847026, + "tokens_seen": 1969160192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040731824747231585, + "loss": 1.3694, + "theoretical_loss": 3.4632956955594834, + "tokens_seen": 1969291264 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040727812550152466, + "loss": 1.3412, + "theoretical_loss": 3.463276576900809, + "tokens_seen": 1969422336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072380035307334, + "loss": 1.3828, + "theoretical_loss": 3.4632574598707553, + "tokens_seen": 1969553408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040719788155994227, + "loss": 1.3285, + "theoretical_loss": 3.463238344469075, + "tokens_seen": 1969684480 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407157759589151, + "loss": 1.3497, + "theoretical_loss": 3.463219230695522, + "tokens_seen": 1969815552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040711763761835983, + "loss": 1.2816, + "theoretical_loss": 3.4632001185498478, + "tokens_seen": 1969946624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040707751564756864, + "loss": 1.3671, + "theoretical_loss": 3.4631810080318064, + "tokens_seen": 1970077696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070373936767774, + "loss": 1.2941, + "theoretical_loss": 3.4631618991411512, + "tokens_seen": 1970208768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069972717059862, + "loss": 1.3135, + "theoretical_loss": 3.463142791877634, + "tokens_seen": 1970339840 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406957149735195, + "loss": 1.2455, + "theoretical_loss": 3.46312368624101, + "tokens_seen": 1970470912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069170277644038, + "loss": 1.3286, + "theoretical_loss": 3.463104582231031, + "tokens_seen": 1970601984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040687690579361256, + "loss": 1.2686, + "theoretical_loss": 3.4630854798474506, + "tokens_seen": 1970733056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068367838228214, + "loss": 1.3076, + "theoretical_loss": 3.4630663790900225, + "tokens_seen": 1970864128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040679666185203017, + "loss": 1.3577, + "theoretical_loss": 3.4630472799585, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9945093393325806, + "objective/train/avg_token_score": 0.005490629933774471, + "objective/train/docs_used": 718798, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4134763479232788, + "objective/train/original_loss": 1.4221107959747314, + "objective/train/theoretical_loss": 3.4630281824526365, + "objective/train/tokens_used": 341585376, + "objective/train/unlikelihood_loss": 0.013743520714342594, + "theoretical_loss": 3.4630281824526365, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406756539881239, + "loss": 1.3556, + "theoretical_loss": 3.4630281824526365, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067164179104478, + "loss": 1.4102, + "theoretical_loss": 3.4630090865721854, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040667629593965654, + "loss": 1.3707, + "theoretical_loss": 3.462989992316901, + "tokens_seen": 1971388416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066361739688654, + "loss": 1.3028, + "theoretical_loss": 3.462970899686536, + "tokens_seen": 1971519488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040659605199807415, + "loss": 1.3868, + "theoretical_loss": 3.4629518086808444, + "tokens_seen": 1971650560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040655593002728296, + "loss": 1.273, + "theoretical_loss": 3.4629327192995802, + "tokens_seen": 1971781632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040651580805649176, + "loss": 1.3181, + "theoretical_loss": 3.4629136315424973, + "tokens_seen": 1971912704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064756860857005, + "loss": 1.329, + "theoretical_loss": 3.462894545409349, + "tokens_seen": 1972043776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064355641149093, + "loss": 1.3046, + "theoretical_loss": 3.46287546089989, + "tokens_seen": 1972174848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063954421441181, + "loss": 1.3019, + "theoretical_loss": 3.462856378013874, + "tokens_seen": 1972305920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040635532017332693, + "loss": 1.3561, + "theoretical_loss": 3.4628372967510543, + "tokens_seen": 1972436992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063151982025357, + "loss": 1.3717, + "theoretical_loss": 3.4628182171111863, + "tokens_seen": 1972568064 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040627507623174455, + "loss": 1.2809, + "theoretical_loss": 3.462799139094023, + "tokens_seen": 1972699136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062349542609533, + "loss": 1.3829, + "theoretical_loss": 3.462780062699319, + "tokens_seen": 1972830208 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040619483229016205, + "loss": 1.4335, + "theoretical_loss": 3.4627609879268286, + "tokens_seen": 1972961280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061547103193709, + "loss": 1.3139, + "theoretical_loss": 3.4627419147763057, + "tokens_seen": 1973092352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040611458834857966, + "loss": 1.3871, + "theoretical_loss": 3.4627228432475055, + "tokens_seen": 1973223424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060744663777885, + "loss": 1.4025, + "theoretical_loss": 3.4627037733401815, + "tokens_seen": 1973354496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060343444069973, + "loss": 1.2878, + "theoretical_loss": 3.462684705054089, + "tokens_seen": 1973485568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059942224362061, + "loss": 1.3267, + "theoretical_loss": 3.4626656383889816, + "tokens_seen": 1973616640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059541004654149, + "loss": 1.2989, + "theoretical_loss": 3.4626465733446143, + "tokens_seen": 1973747712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040591397849462364, + "loss": 1.299, + "theoretical_loss": 3.462627509920742, + "tokens_seen": 1973878784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040587385652383245, + "loss": 1.4074, + "theoretical_loss": 3.462608448117119, + "tokens_seen": 1974009856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040583373455304125, + "loss": 1.3848, + "theoretical_loss": 3.4625893879335, + "tokens_seen": 1974140928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040579361258225006, + "loss": 1.3364, + "theoretical_loss": 3.4625703293696395, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057534906114588, + "loss": 1.2593, + "theoretical_loss": 3.4625512724252934, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040571336864066767, + "loss": 1.3714, + "theoretical_loss": 3.4625322171002155, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056732466698764, + "loss": 1.4483, + "theoretical_loss": 3.4625131633941617, + "tokens_seen": 1974665216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040563312469908523, + "loss": 1.3368, + "theoretical_loss": 3.462494111306886, + "tokens_seen": 1974796288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040559300272829404, + "loss": 1.3528, + "theoretical_loss": 3.4624750608381434, + "tokens_seen": 1974927360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055528807575028, + "loss": 1.3284, + "theoretical_loss": 3.4624560119876904, + "tokens_seen": 1975058432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040551275878671165, + "loss": 1.355, + "theoretical_loss": 3.4624369647552804, + "tokens_seen": 1975189504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054726368159204, + "loss": 1.3878, + "theoretical_loss": 3.4624179191406697, + "tokens_seen": 1975320576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054325148451292, + "loss": 1.3204, + "theoretical_loss": 3.462398875143613, + "tokens_seen": 1975451648 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405392392874338, + "loss": 1.3671, + "theoretical_loss": 3.462379832763866, + "tokens_seen": 1975582720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053522709035468, + "loss": 1.3534, + "theoretical_loss": 3.462360792001184, + "tokens_seen": 1975713792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040531214893275557, + "loss": 1.3617, + "theoretical_loss": 3.4623417528553224, + "tokens_seen": 1975844864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052720269619644, + "loss": 1.3967, + "theoretical_loss": 3.462322715326037, + "tokens_seen": 1975975936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052319049911732, + "loss": 1.3147, + "theoretical_loss": 3.4623036794130826, + "tokens_seen": 1976107008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040519178302038194, + "loss": 1.3104, + "theoretical_loss": 3.462284645116215, + "tokens_seen": 1976238080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051516610495908, + "loss": 1.2906, + "theoretical_loss": 3.46226561243519, + "tokens_seen": 1976369152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040511153907879955, + "loss": 1.3354, + "theoretical_loss": 3.462246581369763, + "tokens_seen": 1976500224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040507141710800835, + "loss": 1.3361, + "theoretical_loss": 3.46222755191969, + "tokens_seen": 1976631296 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040503129513721716, + "loss": 1.3768, + "theoretical_loss": 3.462208524084727, + "tokens_seen": 1976762368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049911731664259, + "loss": 1.319, + "theoretical_loss": 3.46218949786463, + "tokens_seen": 1976893440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049510511956347, + "loss": 1.2657, + "theoretical_loss": 3.4621704732591536, + "tokens_seen": 1977024512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049109292248435, + "loss": 1.327, + "theoretical_loss": 3.4621514502680553, + "tokens_seen": 1977155584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040487080725405233, + "loss": 1.3017, + "theoretical_loss": 3.4621324288910897, + "tokens_seen": 1977286656 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040483068528326114, + "loss": 1.2442, + "theoretical_loss": 3.4621134091280137, + "tokens_seen": 1977417728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040479056331246994, + "loss": 1.288, + "theoretical_loss": 3.4620943909785837, + "tokens_seen": 1977548800 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.991405189037323, + "objective/train/avg_token_score": 0.00859482679516077, + "objective/train/docs_used": 721216, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.388479471206665, + "objective/train/original_loss": 1.4031291007995605, + "objective/train/theoretical_loss": 3.4620753744425556, + "objective/train/tokens_used": 348138976, + "objective/train/unlikelihood_loss": 0.02061946503818035, + "theoretical_loss": 3.4620753744425556, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047504413416787, + "loss": 1.3365, + "theoretical_loss": 3.4620753744425556, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047103193708875, + "loss": 1.2802, + "theoretical_loss": 3.4620563595196847, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046701974000963, + "loss": 1.2926, + "theoretical_loss": 3.462037346209729, + "tokens_seen": 1977942016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040463007542930506, + "loss": 1.2485, + "theoretical_loss": 3.462018334512443, + "tokens_seen": 1978073088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045899534585139, + "loss": 1.3194, + "theoretical_loss": 3.461999324427584, + "tokens_seen": 1978204160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045498314877227, + "loss": 1.328, + "theoretical_loss": 3.461980315954909, + "tokens_seen": 1978335232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045097095169315, + "loss": 1.3271, + "theoretical_loss": 3.461961309094173, + "tokens_seen": 1978466304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044695875461403, + "loss": 1.43, + "theoretical_loss": 3.4619423038451336, + "tokens_seen": 1978597376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040442946557534904, + "loss": 1.3269, + "theoretical_loss": 3.4619233002075473, + "tokens_seen": 1978728448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040438934360455785, + "loss": 1.3264, + "theoretical_loss": 3.46190429818117, + "tokens_seen": 1978859520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040434922163376665, + "loss": 1.2734, + "theoretical_loss": 3.4618852977657593, + "tokens_seen": 1978990592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040430909966297546, + "loss": 1.2748, + "theoretical_loss": 3.461866298961072, + "tokens_seen": 1979121664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042689776921842, + "loss": 1.3868, + "theoretical_loss": 3.461847301766864, + "tokens_seen": 1979252736 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040422885572139307, + "loss": 1.2729, + "theoretical_loss": 3.461828306182893, + "tokens_seen": 1979383808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041887337506018, + "loss": 1.324, + "theoretical_loss": 3.461809312208915, + "tokens_seen": 1979514880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041486117798107, + "loss": 1.2862, + "theoretical_loss": 3.4617903198446878, + "tokens_seen": 1979645952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040410848980901944, + "loss": 1.2575, + "theoretical_loss": 3.461771329089968, + "tokens_seen": 1979777024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040683678382282, + "loss": 1.3659, + "theoretical_loss": 3.4617523399445123, + "tokens_seen": 1979908096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040402824586743705, + "loss": 1.2884, + "theoretical_loss": 3.4617333524080784, + "tokens_seen": 1980039168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039881238966458, + "loss": 1.3197, + "theoretical_loss": 3.461714366480424, + "tokens_seen": 1980170240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039480019258546, + "loss": 1.3054, + "theoretical_loss": 3.461695382161305, + "tokens_seen": 1980301312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039078799550634, + "loss": 1.3032, + "theoretical_loss": 3.4616763994504796, + "tokens_seen": 1980432384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038677579842722, + "loss": 1.3104, + "theoretical_loss": 3.4616574183477047, + "tokens_seen": 1980563456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040382763601348097, + "loss": 1.2847, + "theoretical_loss": 3.461638438852738, + "tokens_seen": 1980694528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037875140426898, + "loss": 1.2816, + "theoretical_loss": 3.461619460965336, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037473920718986, + "loss": 1.3786, + "theoretical_loss": 3.4616004846852575, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040370727010110734, + "loss": 1.3549, + "theoretical_loss": 3.4615815100122593, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036671481303162, + "loss": 1.365, + "theoretical_loss": 3.461562536946099, + "tokens_seen": 1981218816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040362702615952495, + "loss": 1.3451, + "theoretical_loss": 3.4615435654865347, + "tokens_seen": 1981349888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035869041887338, + "loss": 1.2819, + "theoretical_loss": 3.4615245956333234, + "tokens_seen": 1981480960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040354678221794256, + "loss": 1.3094, + "theoretical_loss": 3.4615056273862232, + "tokens_seen": 1981612032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035066602471513, + "loss": 1.286, + "theoretical_loss": 3.461486660744992, + "tokens_seen": 1981743104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034665382763602, + "loss": 1.3219, + "theoretical_loss": 3.4614676957093873, + "tokens_seen": 1981874176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034264163055689, + "loss": 1.3521, + "theoretical_loss": 3.4614487322791674, + "tokens_seen": 1982005248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040338629433477773, + "loss": 1.2754, + "theoretical_loss": 3.46142977045409, + "tokens_seen": 1982136320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040334617236398654, + "loss": 1.266, + "theoretical_loss": 3.461410810233913, + "tokens_seen": 1982267392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040330605039319534, + "loss": 1.3293, + "theoretical_loss": 3.4613918516183952, + "tokens_seen": 1982398464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032659284224041, + "loss": 1.338, + "theoretical_loss": 3.461372894607294, + "tokens_seen": 1982529536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032258064516129, + "loss": 1.2946, + "theoretical_loss": 3.461353939200367, + "tokens_seen": 1982660608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031856844808217, + "loss": 1.3562, + "theoretical_loss": 3.461334985397374, + "tokens_seen": 1982791680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040314556251003046, + "loss": 1.3167, + "theoretical_loss": 3.461316033198072, + "tokens_seen": 1982922752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031054405392393, + "loss": 1.3275, + "theoretical_loss": 3.4612970826022194, + "tokens_seen": 1983053824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030653185684481, + "loss": 1.2805, + "theoretical_loss": 3.461278133609575, + "tokens_seen": 1983184896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030251965976569, + "loss": 1.2966, + "theoretical_loss": 3.4612591862198974, + "tokens_seen": 1983315968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029850746268657, + "loss": 1.301, + "theoretical_loss": 3.4612402404329448, + "tokens_seen": 1983447040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294495265607444, + "loss": 1.3754, + "theoretical_loss": 3.4612212962484756, + "tokens_seen": 1983578112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029048306852833, + "loss": 1.3356, + "theoretical_loss": 3.4612023536662484, + "tokens_seen": 1983709184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040286470871449205, + "loss": 1.2979, + "theoretical_loss": 3.4611834126860224, + "tokens_seen": 1983840256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040282458674370086, + "loss": 1.2488, + "theoretical_loss": 3.461164473307555, + "tokens_seen": 1983971328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040278446477290966, + "loss": 1.3258, + "theoretical_loss": 3.4611455355306067, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.9923150539398193, + "objective/train/avg_token_score": 0.0076849013566970825, + "objective/train/docs_used": 723589, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2174218893051147, + "objective/train/original_loss": 1.2307264804840088, + "objective/train/theoretical_loss": 3.4611265993549347, + "objective/train/tokens_used": 354692576, + "objective/train/unlikelihood_loss": 0.019078988581895828, + "theoretical_loss": 3.4611265993549347, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040274434280211847, + "loss": 1.241, + "theoretical_loss": 3.4611265993549347, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027042208313272, + "loss": 1.3736, + "theoretical_loss": 3.4611076647802985, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026640988605361, + "loss": 1.3578, + "theoretical_loss": 3.461088731806457, + "tokens_seen": 1984495616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040262397688974483, + "loss": 1.3234, + "theoretical_loss": 3.4610698004331693, + "tokens_seen": 1984626688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025838549189536, + "loss": 1.3643, + "theoretical_loss": 3.461050870660194, + "tokens_seen": 1984757760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040254373294816245, + "loss": 1.3403, + "theoretical_loss": 3.46103194248729, + "tokens_seen": 1984888832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025036109773712, + "loss": 1.3212, + "theoretical_loss": 3.461013015914218, + "tokens_seen": 1985019904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040246348900658, + "loss": 1.4097, + "theoretical_loss": 3.4609940909407353, + "tokens_seen": 1985150976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024233670357888, + "loss": 1.3432, + "theoretical_loss": 3.4609751675666014, + "tokens_seen": 1985282048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023832450649976, + "loss": 1.298, + "theoretical_loss": 3.460956245791576, + "tokens_seen": 1985413120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023431230942064, + "loss": 1.2965, + "theoretical_loss": 3.4609373256154186, + "tokens_seen": 1985544192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023030011234152, + "loss": 1.2927, + "theoretical_loss": 3.4609184070378882, + "tokens_seen": 1985675264 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402262879152624, + "loss": 1.3692, + "theoretical_loss": 3.460899490058744, + "tokens_seen": 1985806336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022227571818328, + "loss": 1.3352, + "theoretical_loss": 3.4608805746777467, + "tokens_seen": 1985937408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021826352110416, + "loss": 1.3917, + "theoretical_loss": 3.460861660894654, + "tokens_seen": 1986068480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214251324025035, + "loss": 1.3519, + "theoretical_loss": 3.4608427487092266, + "tokens_seen": 1986199552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021023912694592, + "loss": 1.3029, + "theoretical_loss": 3.4608238381212235, + "tokens_seen": 1986330624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040206226929866796, + "loss": 1.344, + "theoretical_loss": 3.4608049291304055, + "tokens_seen": 1986461696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020221473278767, + "loss": 1.3722, + "theoretical_loss": 3.4607860217365314, + "tokens_seen": 1986592768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019820253570856, + "loss": 1.2405, + "theoretical_loss": 3.460767115939361, + "tokens_seen": 1986723840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019419033862943, + "loss": 1.3169, + "theoretical_loss": 3.460748211738654, + "tokens_seen": 1986854912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040190178141550313, + "loss": 1.305, + "theoretical_loss": 3.4607293091341713, + "tokens_seen": 1986985984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040186165944471194, + "loss": 1.3369, + "theoretical_loss": 3.4607104081256717, + "tokens_seen": 1987117056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040182153747392074, + "loss": 1.3558, + "theoretical_loss": 3.4606915087129155, + "tokens_seen": 1987248128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017814155031295, + "loss": 1.3848, + "theoretical_loss": 3.4606726108956627, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017412935323383, + "loss": 1.2956, + "theoretical_loss": 3.460653714673674, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017011715615471, + "loss": 1.2566, + "theoretical_loss": 3.4606348200467094, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016610495907559, + "loss": 1.316, + "theoretical_loss": 3.460615927014528, + "tokens_seen": 1987772416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016209276199647, + "loss": 1.2641, + "theoretical_loss": 3.4605970355768916, + "tokens_seen": 1987903488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015808056491735, + "loss": 1.3737, + "theoretical_loss": 3.4605781457335594, + "tokens_seen": 1988034560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040154068367838233, + "loss": 1.2876, + "theoretical_loss": 3.4605592574842916, + "tokens_seen": 1988165632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015005617075911, + "loss": 1.3892, + "theoretical_loss": 3.46054037082885, + "tokens_seen": 1988296704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040146043973679984, + "loss": 1.3145, + "theoretical_loss": 3.4605214857669937, + "tokens_seen": 1988427776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014203177660087, + "loss": 1.3353, + "theoretical_loss": 3.4605026022984835, + "tokens_seen": 1988558848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040138019579521745, + "loss": 1.3062, + "theoretical_loss": 3.4604837204230803, + "tokens_seen": 1988689920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040134007382442626, + "loss": 1.3056, + "theoretical_loss": 3.4604648401405447, + "tokens_seen": 1988820992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040129995185363506, + "loss": 1.2945, + "theoretical_loss": 3.460445961450637, + "tokens_seen": 1988952064 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040125982988284387, + "loss": 1.2653, + "theoretical_loss": 3.460427084353118, + "tokens_seen": 1989083136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012197079120526, + "loss": 1.3564, + "theoretical_loss": 3.4604082088477486, + "tokens_seen": 1989214208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040117958594126143, + "loss": 1.3089, + "theoretical_loss": 3.4603893349342894, + "tokens_seen": 1989345280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040113946397047023, + "loss": 1.3456, + "theoretical_loss": 3.4603704626125023, + "tokens_seen": 1989476352 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401099341999679, + "loss": 1.325, + "theoretical_loss": 3.4603515918821466, + "tokens_seen": 1989607424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040105922002888785, + "loss": 1.2996, + "theoretical_loss": 3.4603327227429843, + "tokens_seen": 1989738496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010190980580966, + "loss": 1.2735, + "theoretical_loss": 3.460313855194776, + "tokens_seen": 1989869568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040097897608730546, + "loss": 1.3086, + "theoretical_loss": 3.4602949892372834, + "tokens_seen": 1990000640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009388541165142, + "loss": 1.3301, + "theoretical_loss": 3.4602761248702665, + "tokens_seen": 1990131712 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400898732145723, + "loss": 1.3612, + "theoretical_loss": 3.4602572620934877, + "tokens_seen": 1990262784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008586101749318, + "loss": 1.3464, + "theoretical_loss": 3.4602384009067073, + "tokens_seen": 1990393856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008184882041406, + "loss": 1.3427, + "theoretical_loss": 3.4602195413096877, + "tokens_seen": 1990524928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007783662333494, + "loss": 1.3995, + "theoretical_loss": 3.4602006833021886, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9909166693687439, + "objective/train/avg_token_score": 0.009083281271159649, + "objective/train/docs_used": 726036, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3514455556869507, + "objective/train/original_loss": 1.3641645908355713, + "objective/train/theoretical_loss": 3.4601818268839732, + "objective/train/tokens_used": 361246176, + "objective/train/unlikelihood_loss": 0.025408154353499413, + "theoretical_loss": 3.4601818268839732, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007382442625582, + "loss": 1.3662, + "theoretical_loss": 3.4601818268839732, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400698122291767, + "loss": 1.3077, + "theoretical_loss": 3.4601629720548015, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040065800032097575, + "loss": 1.413, + "theoretical_loss": 3.4601441188144353, + "tokens_seen": 1991049216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006178783501846, + "loss": 1.3629, + "theoretical_loss": 3.4601252671626366, + "tokens_seen": 1991180288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040057775637939336, + "loss": 1.2533, + "theoretical_loss": 3.4601064170991673, + "tokens_seen": 1991311360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005376344086021, + "loss": 1.43, + "theoretical_loss": 3.4600875686237877, + "tokens_seen": 1991442432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040049751243781097, + "loss": 1.3453, + "theoretical_loss": 3.4600687217362607, + "tokens_seen": 1991573504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004573904670197, + "loss": 1.3626, + "theoretical_loss": 3.4600498764363476, + "tokens_seen": 1991704576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004172684962286, + "loss": 1.3071, + "theoretical_loss": 3.46003103272381, + "tokens_seen": 1991835648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040037714652543734, + "loss": 1.3712, + "theoretical_loss": 3.4600121905984103, + "tokens_seen": 1991966720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040033702455464614, + "loss": 1.3225, + "theoretical_loss": 3.45999335005991, + "tokens_seen": 1992097792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040029690258385495, + "loss": 1.3319, + "theoretical_loss": 3.4599745111080713, + "tokens_seen": 1992228864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002567806130637, + "loss": 1.328, + "theoretical_loss": 3.4599556737426553, + "tokens_seen": 1992359936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002166586422725, + "loss": 1.3333, + "theoretical_loss": 3.4599368379634257, + "tokens_seen": 1992491008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001765366714813, + "loss": 1.3612, + "theoretical_loss": 3.4599180037701434, + "tokens_seen": 1992622080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001364147006901, + "loss": 1.3608, + "theoretical_loss": 3.4598991711625704, + "tokens_seen": 1992753152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000962927298989, + "loss": 1.3618, + "theoretical_loss": 3.45988034014047, + "tokens_seen": 1992884224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040005617075910773, + "loss": 1.3263, + "theoretical_loss": 3.4598615107036035, + "tokens_seen": 1993015296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000160487883165, + "loss": 1.3505, + "theoretical_loss": 3.4598426828517335, + "tokens_seen": 1993146368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039997592681752524, + "loss": 1.3316, + "theoretical_loss": 3.459823856584623, + "tokens_seen": 1993277440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999358048467341, + "loss": 1.3221, + "theoretical_loss": 3.459805031902033, + "tokens_seen": 1993408512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039989568287594285, + "loss": 1.3283, + "theoretical_loss": 3.4597862088037266, + "tokens_seen": 1993539584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039985556090515166, + "loss": 1.294, + "theoretical_loss": 3.4597673872894674, + "tokens_seen": 1993670656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039981543893436046, + "loss": 1.3486, + "theoretical_loss": 3.4597485673590165, + "tokens_seen": 1993801728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039977531696356927, + "loss": 1.3356, + "theoretical_loss": 3.459729749012137, + "tokens_seen": 1993932800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997351949927781, + "loss": 1.2495, + "theoretical_loss": 3.459710932248592, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969507302198683, + "loss": 1.2657, + "theoretical_loss": 3.459692117068144, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039965495105119563, + "loss": 1.3675, + "theoretical_loss": 3.459673303470555, + "tokens_seen": 1994326016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039961482908040444, + "loss": 1.3296, + "theoretical_loss": 3.4596544914555887, + "tokens_seen": 1994457088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039957470710961325, + "loss": 1.3613, + "theoretical_loss": 3.4596356810230082, + "tokens_seen": 1994588160 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399534585138822, + "loss": 1.356, + "theoretical_loss": 3.4596168721725755, + "tokens_seen": 1994719232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039949446316803086, + "loss": 1.2609, + "theoretical_loss": 3.4595980649040543, + "tokens_seen": 1994850304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994543411972396, + "loss": 1.33, + "theoretical_loss": 3.459579259217207, + "tokens_seen": 1994981376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994142192264484, + "loss": 1.3261, + "theoretical_loss": 3.4595604551117973, + "tokens_seen": 1995112448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993740972556572, + "loss": 1.2818, + "theoretical_loss": 3.459541652587588, + "tokens_seen": 1995243520 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399333975284866, + "loss": 1.4284, + "theoretical_loss": 3.4595228516443424, + "tokens_seen": 1995374592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992938533140748, + "loss": 1.3359, + "theoretical_loss": 3.4595040522818232, + "tokens_seen": 1995505664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992537313432836, + "loss": 1.2911, + "theoretical_loss": 3.459485254499795, + "tokens_seen": 1995636736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992136093724924, + "loss": 1.2879, + "theoretical_loss": 3.45946645829802, + "tokens_seen": 1995767808 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917348740170115, + "loss": 1.2583, + "theoretical_loss": 3.4594476636762614, + "tokens_seen": 1995898880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913336543091, + "loss": 1.2663, + "theoretical_loss": 3.4594288706342837, + "tokens_seen": 1996029952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039909324346011876, + "loss": 1.189, + "theoretical_loss": 3.4594100791718496, + "tokens_seen": 1996161024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039905312148932757, + "loss": 1.3648, + "theoretical_loss": 3.459391289288723, + "tokens_seen": 1996292096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901299951853637, + "loss": 1.3249, + "theoretical_loss": 3.459372500984667, + "tokens_seen": 1996423168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989728775477451, + "loss": 1.3266, + "theoretical_loss": 3.459353714259446, + "tokens_seen": 1996554240 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398932755576954, + "loss": 1.3528, + "theoretical_loss": 3.4593349291128233, + "tokens_seen": 1996685312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039889263360616274, + "loss": 1.3108, + "theoretical_loss": 3.459316145544562, + "tokens_seen": 1996816384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039885251163537154, + "loss": 1.3843, + "theoretical_loss": 3.459297363554427, + "tokens_seen": 1996947456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039881238966458035, + "loss": 1.3392, + "theoretical_loss": 3.459278583142181, + "tokens_seen": 1997078528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987722676937891, + "loss": 1.371, + "theoretical_loss": 3.4592598043075893, + "tokens_seen": 1997209600 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9915937185287476, + "objective/train/avg_token_score": 0.008406254462897778, + "objective/train/docs_used": 728467, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2814857959747314, + "objective/train/original_loss": 1.2932515144348145, + "objective/train/theoretical_loss": 3.459241027050415, + "objective/train/tokens_used": 367799776, + "objective/train/unlikelihood_loss": 0.02449808269739151, + "theoretical_loss": 3.459241027050415, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987321457229979, + "loss": 1.3056, + "theoretical_loss": 3.459241027050415, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986920237522067, + "loss": 1.3024, + "theoretical_loss": 3.459222251370422, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986519017814155, + "loss": 1.3792, + "theoretical_loss": 3.459203477267374, + "tokens_seen": 1997602816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039861177981062427, + "loss": 1.3708, + "theoretical_loss": 3.4591847047410362, + "tokens_seen": 1997733888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039857165783983313, + "loss": 1.2628, + "theoretical_loss": 3.459165933791172, + "tokens_seen": 1997864960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985315358690419, + "loss": 1.2797, + "theoretical_loss": 3.4591471644175456, + "tokens_seen": 1997996032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984914138982507, + "loss": 1.2526, + "theoretical_loss": 3.4591283966199216, + "tokens_seen": 1998127104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984512919274595, + "loss": 1.2963, + "theoretical_loss": 3.459109630398064, + "tokens_seen": 1998258176 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039841116995666825, + "loss": 1.3326, + "theoretical_loss": 3.4590908657517376, + "tokens_seen": 1998389248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983710479858771, + "loss": 1.367, + "theoretical_loss": 3.459072102680706, + "tokens_seen": 1998520320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039833092601508586, + "loss": 1.3683, + "theoretical_loss": 3.4590533411847346, + "tokens_seen": 1998651392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039829080404429467, + "loss": 1.3101, + "theoretical_loss": 3.4590345812635865, + "tokens_seen": 1998782464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982506820735035, + "loss": 1.2709, + "theoretical_loss": 3.459015822917028, + "tokens_seen": 1998913536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039821056010271223, + "loss": 1.3457, + "theoretical_loss": 3.4589970661448226, + "tokens_seen": 1999044608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039817043813192103, + "loss": 1.3171, + "theoretical_loss": 3.458978310946735, + "tokens_seen": 1999175680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039813031616112984, + "loss": 1.3088, + "theoretical_loss": 3.4589595573225305, + "tokens_seen": 1999306752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039809019419033865, + "loss": 1.3607, + "theoretical_loss": 3.4589408052719737, + "tokens_seen": 1999437824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980500722195474, + "loss": 1.2411, + "theoretical_loss": 3.4589220547948285, + "tokens_seen": 1999568896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039800995024875626, + "loss": 1.3175, + "theoretical_loss": 3.458903305890861, + "tokens_seen": 1999699968 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397969828277965, + "loss": 1.4305, + "theoretical_loss": 3.458884558559835, + "tokens_seen": 1999831040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979297063071738, + "loss": 1.3916, + "theoretical_loss": 3.4588658128015166, + "tokens_seen": 1999962112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978895843363826, + "loss": 1.2539, + "theoretical_loss": 3.45884706861567, + "tokens_seen": 2000093184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978494623655914, + "loss": 1.2925, + "theoretical_loss": 3.4588283260020605, + "tokens_seen": 2000224256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039780934039480024, + "loss": 1.3136, + "theoretical_loss": 3.458809584960453, + "tokens_seen": 2000355328 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397769218424009, + "loss": 1.3682, + "theoretical_loss": 3.458790845490613, + "tokens_seen": 2000486400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977290964532178, + "loss": 1.2987, + "theoretical_loss": 3.4587721075923055, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976889744824266, + "loss": 1.3402, + "theoretical_loss": 3.458753371265296, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976488525116354, + "loss": 1.316, + "theoretical_loss": 3.4587346365093494, + "tokens_seen": 2000879616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039760873054084416, + "loss": 1.3143, + "theoretical_loss": 3.458715903324231, + "tokens_seen": 2001010688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039756860857005297, + "loss": 1.2603, + "theoretical_loss": 3.4586971717097077, + "tokens_seen": 2001141760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039752848659926177, + "loss": 1.3302, + "theoretical_loss": 3.4586784416655423, + "tokens_seen": 2001272832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974883646284705, + "loss": 1.3782, + "theoretical_loss": 3.4586597131915027, + "tokens_seen": 2001403904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974482426576794, + "loss": 1.3696, + "theoretical_loss": 3.458640986287353, + "tokens_seen": 2001534976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039740812068688814, + "loss": 1.3577, + "theoretical_loss": 3.45862226095286, + "tokens_seen": 2001666048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039736799871609694, + "loss": 1.3109, + "theoretical_loss": 3.458603537187788, + "tokens_seen": 2001797120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039732787674530575, + "loss": 1.342, + "theoretical_loss": 3.458584814991904, + "tokens_seen": 2001928192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972877547745145, + "loss": 1.2804, + "theoretical_loss": 3.458566094364973, + "tokens_seen": 2002059264 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039724763280372336, + "loss": 1.2842, + "theoretical_loss": 3.458547375306761, + "tokens_seen": 2002190336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972075108329321, + "loss": 1.3224, + "theoretical_loss": 3.4585286578170336, + "tokens_seen": 2002321408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971673888621409, + "loss": 1.3419, + "theoretical_loss": 3.4585099418955574, + "tokens_seen": 2002452480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971272668913497, + "loss": 1.3226, + "theoretical_loss": 3.4584912275420976, + "tokens_seen": 2002583552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039708714492055853, + "loss": 1.368, + "theoretical_loss": 3.4584725147564206, + "tokens_seen": 2002714624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970470229497673, + "loss": 1.3389, + "theoretical_loss": 3.4584538035382923, + "tokens_seen": 2002845696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970069009789761, + "loss": 1.258, + "theoretical_loss": 3.4584350938874797, + "tokens_seen": 2002976768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969667790081849, + "loss": 1.3608, + "theoretical_loss": 3.4584163858037478, + "tokens_seen": 2003107840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039692665703739365, + "loss": 1.3103, + "theoretical_loss": 3.458397679286863, + "tokens_seen": 2003238912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968865350666025, + "loss": 1.3556, + "theoretical_loss": 3.458378974336592, + "tokens_seen": 2003369984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039684641309581126, + "loss": 1.3722, + "theoretical_loss": 3.458360270952701, + "tokens_seen": 2003501056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039680629112502007, + "loss": 1.3207, + "theoretical_loss": 3.458341569134956, + "tokens_seen": 2003632128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967661691542289, + "loss": 1.3131, + "theoretical_loss": 3.458322868883124, + "tokens_seen": 2003763200 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9897269606590271, + "objective/train/avg_token_score": 0.01027307566255331, + "objective/train/docs_used": 730811, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3902190923690796, + "objective/train/original_loss": 1.406241774559021, + "objective/train/theoretical_loss": 3.4583041701969712, + "objective/train/tokens_used": 374353376, + "objective/train/unlikelihood_loss": 0.025743091478943825, + "theoretical_loss": 3.4583041701969712, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039672604718343763, + "loss": 1.352, + "theoretical_loss": 3.4583041701969712, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039668592521264643, + "loss": 1.4062, + "theoretical_loss": 3.4582854730762635, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039664580324185524, + "loss": 1.3258, + "theoretical_loss": 3.4582667775207687, + "tokens_seen": 2004156416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039660568127106405, + "loss": 1.3124, + "theoretical_loss": 3.4582480835302527, + "tokens_seen": 2004287488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039656555930027285, + "loss": 1.3593, + "theoretical_loss": 3.4582293911044824, + "tokens_seen": 2004418560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039652543732948166, + "loss": 1.314, + "theoretical_loss": 3.4582107002432236, + "tokens_seen": 2004549632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003964853153586904, + "loss": 1.3409, + "theoretical_loss": 3.4581920109462443, + "tokens_seen": 2004680704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964451933878992, + "loss": 1.2901, + "theoretical_loss": 3.4581733232133107, + "tokens_seen": 2004811776 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396405071417108, + "loss": 1.3639, + "theoretical_loss": 3.4581546370441902, + "tokens_seen": 2004942848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963649494463168, + "loss": 1.2916, + "theoretical_loss": 3.4581359524386492, + "tokens_seen": 2005073920 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039632482747552564, + "loss": 1.4215, + "theoretical_loss": 3.458117269396455, + "tokens_seen": 2005204992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962847055047344, + "loss": 1.2895, + "theoretical_loss": 3.458098587917374, + "tokens_seen": 2005336064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962445835339432, + "loss": 1.3175, + "theoretical_loss": 3.4580799080011735, + "tokens_seen": 2005467136 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396204461563152, + "loss": 1.3032, + "theoretical_loss": 3.4580612296476207, + "tokens_seen": 2005598208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961643395923608, + "loss": 1.352, + "theoretical_loss": 3.458042552856484, + "tokens_seen": 2005729280 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039612421762156956, + "loss": 1.2972, + "theoretical_loss": 3.4580238776275283, + "tokens_seen": 2005860352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039608409565077837, + "loss": 1.3325, + "theoretical_loss": 3.4580052039605222, + "tokens_seen": 2005991424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039604397367998717, + "loss": 1.3561, + "theoretical_loss": 3.4579865318552336, + "tokens_seen": 2006122496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960038517091959, + "loss": 1.2897, + "theoretical_loss": 3.4579678613114284, + "tokens_seen": 2006253568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959637297384048, + "loss": 1.2827, + "theoretical_loss": 3.457949192328875, + "tokens_seen": 2006384640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039592360776761354, + "loss": 1.3417, + "theoretical_loss": 3.457930524907341, + "tokens_seen": 2006515712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958834857968224, + "loss": 1.3447, + "theoretical_loss": 3.457911859046593, + "tokens_seen": 2006646784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039584336382603115, + "loss": 1.3325, + "theoretical_loss": 3.4578931947464, + "tokens_seen": 2006777856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958032418552399, + "loss": 1.2431, + "theoretical_loss": 3.457874532006527, + "tokens_seen": 2006908928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039576311988444876, + "loss": 1.2671, + "theoretical_loss": 3.457855870826745, + "tokens_seen": 2007040000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957229979136575, + "loss": 1.351, + "theoretical_loss": 3.4578372112068196, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956828759428663, + "loss": 1.3777, + "theoretical_loss": 3.4578185531465184, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956427539720751, + "loss": 1.3386, + "theoretical_loss": 3.4577998966456107, + "tokens_seen": 2007433216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039560263200128393, + "loss": 1.3329, + "theoretical_loss": 3.4577812417038634, + "tokens_seen": 2007564288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955625100304927, + "loss": 1.3324, + "theoretical_loss": 3.457762588321044, + "tokens_seen": 2007695360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955223880597015, + "loss": 1.3134, + "theoretical_loss": 3.4577439364969216, + "tokens_seen": 2007826432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954822660889103, + "loss": 1.259, + "theoretical_loss": 3.4577252862312635, + "tokens_seen": 2007957504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039544214411811905, + "loss": 1.2109, + "theoretical_loss": 3.457706637523838, + "tokens_seen": 2008088576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954020221473279, + "loss": 1.372, + "theoretical_loss": 3.457687990374413, + "tokens_seen": 2008219648 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039536190017653666, + "loss": 1.2687, + "theoretical_loss": 3.457669344782757, + "tokens_seen": 2008350720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953217782057455, + "loss": 1.3585, + "theoretical_loss": 3.457650700748638, + "tokens_seen": 2008481792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952816562349543, + "loss": 1.3235, + "theoretical_loss": 3.457632058271824, + "tokens_seen": 2008612864 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395241534264163, + "loss": 1.3278, + "theoretical_loss": 3.4576134173520834, + "tokens_seen": 2008743936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952014122933719, + "loss": 1.2945, + "theoretical_loss": 3.457594777989185, + "tokens_seen": 2008875008 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039516129032258064, + "loss": 1.3235, + "theoretical_loss": 3.457576140182897, + "tokens_seen": 2009006080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039512116835178945, + "loss": 1.326, + "theoretical_loss": 3.4575575039329873, + "tokens_seen": 2009137152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039508104638099825, + "loss": 1.3161, + "theoretical_loss": 3.457538869239225, + "tokens_seen": 2009268224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039504092441020706, + "loss": 1.3321, + "theoretical_loss": 3.457520236101379, + "tokens_seen": 2009399296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950008024394158, + "loss": 1.3957, + "theoretical_loss": 3.4575016045192175, + "tokens_seen": 2009530368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949606804686246, + "loss": 1.3625, + "theoretical_loss": 3.4574829744925086, + "tokens_seen": 2009661440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949205584978334, + "loss": 1.3876, + "theoretical_loss": 3.457464346021022, + "tokens_seen": 2009792512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948804365270422, + "loss": 1.3576, + "theoretical_loss": 3.457445719104526, + "tokens_seen": 2009923584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039484031455625104, + "loss": 1.295, + "theoretical_loss": 3.457427093742789, + "tokens_seen": 2010054656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948001925854598, + "loss": 1.2947, + "theoretical_loss": 3.4574084699355807, + "tokens_seen": 2010185728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947600706146686, + "loss": 1.3645, + "theoretical_loss": 3.4573898476826694, + "tokens_seen": 2010316800 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9927845001220703, + "objective/train/avg_token_score": 0.007215526886284351, + "objective/train/docs_used": 733267, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.288818597793579, + "objective/train/original_loss": 1.298001766204834, + "objective/train/theoretical_loss": 3.457371226983824, + "objective/train/tokens_used": 380906976, + "objective/train/unlikelihood_loss": 0.026874808594584465, + "theoretical_loss": 3.457371226983824, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947199486438774, + "loss": 1.2976, + "theoretical_loss": 3.457371226983824, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946798266730862, + "loss": 1.3079, + "theoretical_loss": 3.457352607838814, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394639704702295, + "loss": 1.2903, + "theoretical_loss": 3.4573339902474083, + "tokens_seen": 2010710016 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039459958273150377, + "loss": 1.2684, + "theoretical_loss": 3.457315374209376, + "tokens_seen": 2010841088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039455946076071257, + "loss": 1.3538, + "theoretical_loss": 3.4572967597244855, + "tokens_seen": 2010972160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945193387899214, + "loss": 1.3679, + "theoretical_loss": 3.4572781467925067, + "tokens_seen": 2011103232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944792168191302, + "loss": 1.3102, + "theoretical_loss": 3.4572595354132094, + "tokens_seen": 2011234304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039443909484833894, + "loss": 1.3213, + "theoretical_loss": 3.457240925586362, + "tokens_seen": 2011365376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943989728775478, + "loss": 1.2965, + "theoretical_loss": 3.457222317311734, + "tokens_seen": 2011496448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039435885090675655, + "loss": 1.3405, + "theoretical_loss": 3.457203710589095, + "tokens_seen": 2011627520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943187289359653, + "loss": 1.2313, + "theoretical_loss": 3.4571851054182146, + "tokens_seen": 2011758592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039427860696517416, + "loss": 1.332, + "theoretical_loss": 3.457166501798862, + "tokens_seen": 2011889664 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942384849943829, + "loss": 1.3167, + "theoretical_loss": 3.4571478997308063, + "tokens_seen": 2012020736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941983630235917, + "loss": 1.351, + "theoretical_loss": 3.4571292992138183, + "tokens_seen": 2012151808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941582410528005, + "loss": 1.2639, + "theoretical_loss": 3.4571107002476666, + "tokens_seen": 2012282880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039411811908200933, + "loss": 1.3473, + "theoretical_loss": 3.4570921028321218, + "tokens_seen": 2012413952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039407799711121814, + "loss": 1.2631, + "theoretical_loss": 3.4570735069669523, + "tokens_seen": 2012545024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940378751404269, + "loss": 1.3528, + "theoretical_loss": 3.4570549126519294, + "tokens_seen": 2012676096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939977531696357, + "loss": 1.3357, + "theoretical_loss": 3.457036319886822, + "tokens_seen": 2012807168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939576311988445, + "loss": 1.3093, + "theoretical_loss": 3.4570177286714, + "tokens_seen": 2012938240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939175092280533, + "loss": 1.2711, + "theoretical_loss": 3.4569991390054335, + "tokens_seen": 2013069312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039387738725726206, + "loss": 1.2954, + "theoretical_loss": 3.4569805508886935, + "tokens_seen": 2013200384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938372652864709, + "loss": 1.3105, + "theoretical_loss": 3.456961964320948, + "tokens_seen": 2013331456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937971433156797, + "loss": 1.3629, + "theoretical_loss": 3.4569433793019693, + "tokens_seen": 2013462528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937570213448884, + "loss": 1.2438, + "theoretical_loss": 3.4569247958315255, + "tokens_seen": 2013593600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937168993740973, + "loss": 1.3636, + "theoretical_loss": 3.456906213909388, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039367677740330604, + "loss": 1.3323, + "theoretical_loss": 3.456887633535327, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039363665543251485, + "loss": 1.295, + "theoretical_loss": 3.456869054709112, + "tokens_seen": 2013986816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039359653346172365, + "loss": 1.2975, + "theoretical_loss": 3.4568504774305144, + "tokens_seen": 2014117888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039355641149093246, + "loss": 1.3767, + "theoretical_loss": 3.456831901699304, + "tokens_seen": 2014248960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935162895201412, + "loss": 1.2481, + "theoretical_loss": 3.4568133275152513, + "tokens_seen": 2014380032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039347616754935, + "loss": 1.3205, + "theoretical_loss": 3.4567947548781266, + "tokens_seen": 2014511104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934360455785588, + "loss": 1.3648, + "theoretical_loss": 3.4567761837877002, + "tokens_seen": 2014642176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039339592360776763, + "loss": 1.2934, + "theoretical_loss": 3.456757614243744, + "tokens_seen": 2014773248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039335580163697644, + "loss": 1.2949, + "theoretical_loss": 3.4567390462460272, + "tokens_seen": 2014904320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933156796661852, + "loss": 1.2173, + "theoretical_loss": 3.456720479794321, + "tokens_seen": 2015035392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039327555769539405, + "loss": 1.3355, + "theoretical_loss": 3.456701914888396, + "tokens_seen": 2015166464 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932354357246028, + "loss": 1.2815, + "theoretical_loss": 3.4566833515280235, + "tokens_seen": 2015297536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931953137538116, + "loss": 1.3094, + "theoretical_loss": 3.456664789712973, + "tokens_seen": 2015428608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931551917830204, + "loss": 1.3058, + "theoretical_loss": 3.456646229443017, + "tokens_seen": 2015559680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311506981222916, + "loss": 1.3006, + "theoretical_loss": 3.4566276707179258, + "tokens_seen": 2015690752 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039307494784143797, + "loss": 1.291, + "theoretical_loss": 3.45660911353747, + "tokens_seen": 2015821824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930348258706468, + "loss": 1.3292, + "theoretical_loss": 3.456590557901421, + "tokens_seen": 2015952896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929947038998556, + "loss": 1.3454, + "theoretical_loss": 3.4565720038095495, + "tokens_seen": 2016083968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039295458192906434, + "loss": 1.296, + "theoretical_loss": 3.4565534512616267, + "tokens_seen": 2016215040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929144599582732, + "loss": 1.3151, + "theoretical_loss": 3.4565349002574246, + "tokens_seen": 2016346112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039287433798748195, + "loss": 1.2872, + "theoretical_loss": 3.456516350796713, + "tokens_seen": 2016477184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928342160166907, + "loss": 1.3199, + "theoretical_loss": 3.4564978028792646, + "tokens_seen": 2016608256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039279409404589956, + "loss": 1.3431, + "theoretical_loss": 3.4564792565048497, + "tokens_seen": 2016739328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927539720751083, + "loss": 1.3497, + "theoretical_loss": 3.45646071167324, + "tokens_seen": 2016870400 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9912194013595581, + "objective/train/avg_token_score": 0.00878061167895794, + "objective/train/docs_used": 735776, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4485646486282349, + "objective/train/original_loss": 1.4633667469024658, + "objective/train/theoretical_loss": 3.456442168384207, + "objective/train/tokens_used": 387460576, + "objective/train/unlikelihood_loss": 0.021836766973137856, + "theoretical_loss": 3.456442168384207, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927138501043172, + "loss": 1.4426, + "theoretical_loss": 3.456442168384207, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926737281335259, + "loss": 1.2742, + "theoretical_loss": 3.456423626637522, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039263360616273473, + "loss": 1.343, + "theoretical_loss": 3.456405086432956, + "tokens_seen": 2017263616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039259348419194354, + "loss": 1.3125, + "theoretical_loss": 3.4563865477702818, + "tokens_seen": 2017394688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925533622211523, + "loss": 1.3751, + "theoretical_loss": 3.4563680106492702, + "tokens_seen": 2017525760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925132402503611, + "loss": 1.2504, + "theoretical_loss": 3.456349475069693, + "tokens_seen": 2017656832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924731182795699, + "loss": 1.3151, + "theoretical_loss": 3.4563309410313217, + "tokens_seen": 2017787904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924329963087787, + "loss": 1.278, + "theoretical_loss": 3.456312408533928, + "tokens_seen": 2017918976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039239287433798746, + "loss": 1.2606, + "theoretical_loss": 3.4562938775772842, + "tokens_seen": 2018050048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923527523671963, + "loss": 1.2859, + "theoretical_loss": 3.456275348161162, + "tokens_seen": 2018181120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923126303964051, + "loss": 1.3164, + "theoretical_loss": 3.4562568202853328, + "tokens_seen": 2018312192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922725084256138, + "loss": 1.3725, + "theoretical_loss": 3.456238293949569, + "tokens_seen": 2018443264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922323864548227, + "loss": 1.3387, + "theoretical_loss": 3.4562197691536425, + "tokens_seen": 2018574336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039219226448403144, + "loss": 1.3445, + "theoretical_loss": 3.4562012458973257, + "tokens_seen": 2018705408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921521425132403, + "loss": 1.2817, + "theoretical_loss": 3.4561827241803895, + "tokens_seen": 2018836480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039211202054244905, + "loss": 1.3534, + "theoretical_loss": 3.4561642040026075, + "tokens_seen": 2018967552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039207189857165786, + "loss": 1.3047, + "theoretical_loss": 3.456145685363751, + "tokens_seen": 2019098624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039203177660086666, + "loss": 1.2687, + "theoretical_loss": 3.456127168263593, + "tokens_seen": 2019229696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919916546300754, + "loss": 1.3346, + "theoretical_loss": 3.4561086527019045, + "tokens_seen": 2019360768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919515326592842, + "loss": 1.3259, + "theoretical_loss": 3.4560901386784586, + "tokens_seen": 2019491840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039191141068849303, + "loss": 1.3022, + "theoretical_loss": 3.456071626193028, + "tokens_seen": 2019622912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039187128871770183, + "loss": 1.3547, + "theoretical_loss": 3.4560531152453846, + "tokens_seen": 2019753984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918311667469106, + "loss": 1.301, + "theoretical_loss": 3.4560346058353018, + "tokens_seen": 2019885056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039179104477611945, + "loss": 1.3256, + "theoretical_loss": 3.4560160979625505, + "tokens_seen": 2020016128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917509228053282, + "loss": 1.3037, + "theoretical_loss": 3.455997591626905, + "tokens_seen": 2020147200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171080083453695, + "loss": 1.2808, + "theoretical_loss": 3.455979086828136, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916706788637458, + "loss": 1.2542, + "theoretical_loss": 3.455960583566018, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163055689295456, + "loss": 1.2977, + "theoretical_loss": 3.4559420818403233, + "tokens_seen": 2020540416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039159043492216337, + "loss": 1.3378, + "theoretical_loss": 3.455923581650824, + "tokens_seen": 2020671488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915503129513722, + "loss": 1.3453, + "theoretical_loss": 3.455905082997293, + "tokens_seen": 2020802560 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391510190980581, + "loss": 1.3677, + "theoretical_loss": 3.4558865858795036, + "tokens_seen": 2020933632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914700690097898, + "loss": 1.3456, + "theoretical_loss": 3.4558680902972285, + "tokens_seen": 2021064704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914299470389986, + "loss": 1.3414, + "theoretical_loss": 3.455849596250241, + "tokens_seen": 2021195776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039138982506820735, + "loss": 1.271, + "theoretical_loss": 3.455831103738314, + "tokens_seen": 2021326848 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039134970309741615, + "loss": 1.2867, + "theoretical_loss": 3.4558126127612194, + "tokens_seen": 2021457920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039130958112662496, + "loss": 1.326, + "theoretical_loss": 3.455794123318732, + "tokens_seen": 2021588992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912694591558337, + "loss": 1.2729, + "theoretical_loss": 3.4557756354106246, + "tokens_seen": 2021720064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912293371850426, + "loss": 1.3441, + "theoretical_loss": 3.4557571490366694, + "tokens_seen": 2021851136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911892152142513, + "loss": 1.3426, + "theoretical_loss": 3.45573866419664, + "tokens_seen": 2021982208 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039114909324346013, + "loss": 1.3045, + "theoretical_loss": 3.455720180890311, + "tokens_seen": 2022113280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039110897127266894, + "loss": 1.4178, + "theoretical_loss": 3.455701699117454, + "tokens_seen": 2022244352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910688493018777, + "loss": 1.3286, + "theoretical_loss": 3.4556832188778435, + "tokens_seen": 2022375424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910287273310865, + "loss": 1.3422, + "theoretical_loss": 3.455664740171253, + "tokens_seen": 2022506496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909886053602953, + "loss": 1.3373, + "theoretical_loss": 3.4556462629974547, + "tokens_seen": 2022637568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909484833895041, + "loss": 1.2911, + "theoretical_loss": 3.455627787356223, + "tokens_seen": 2022768640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039090836141871286, + "loss": 1.343, + "theoretical_loss": 3.455609313247332, + "tokens_seen": 2022899712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908682394479217, + "loss": 1.3428, + "theoretical_loss": 3.4555908406705544, + "tokens_seen": 2023030784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908281174771305, + "loss": 1.2728, + "theoretical_loss": 3.455572369625665, + "tokens_seen": 2023161856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907879955063393, + "loss": 1.3479, + "theoretical_loss": 3.4555539001124362, + "tokens_seen": 2023292928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907478735355481, + "loss": 1.2433, + "theoretical_loss": 3.4555354321306426, + "tokens_seen": 2023424000 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9941638112068176, + "objective/train/avg_token_score": 0.005836206488311291, + "objective/train/docs_used": 738179, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.329341173171997, + "objective/train/original_loss": 1.3373048305511475, + "objective/train/theoretical_loss": 3.455516965680058, + "objective/train/tokens_used": 394014176, + "objective/train/unlikelihood_loss": 0.01851975917816162, + "theoretical_loss": 3.455516965680058, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039070775156475684, + "loss": 1.2341, + "theoretical_loss": 3.455516965680058, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906676295939657, + "loss": 1.2983, + "theoretical_loss": 3.455498500760456, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039062750762317445, + "loss": 1.3182, + "theoretical_loss": 3.455480037371611, + "tokens_seen": 2023817216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039058738565238326, + "loss": 1.293, + "theoretical_loss": 3.4554615755132962, + "tokens_seen": 2023948288 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039054726368159206, + "loss": 1.3563, + "theoretical_loss": 3.455443115185286, + "tokens_seen": 2024079360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905071417108008, + "loss": 1.3054, + "theoretical_loss": 3.455424656387355, + "tokens_seen": 2024210432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904670197400096, + "loss": 1.2883, + "theoretical_loss": 3.4554061991192766, + "tokens_seen": 2024341504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039042689776921843, + "loss": 1.2446, + "theoretical_loss": 3.4553877433808253, + "tokens_seen": 2024472576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039038677579842723, + "loss": 1.2747, + "theoretical_loss": 3.4553692891717747, + "tokens_seen": 2024603648 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390346653827636, + "loss": 1.3056, + "theoretical_loss": 3.4553508364919, + "tokens_seen": 2024734720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039030653185684485, + "loss": 1.4222, + "theoretical_loss": 3.455332385340975, + "tokens_seen": 2024865792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902664098860536, + "loss": 1.3414, + "theoretical_loss": 3.455313935718774, + "tokens_seen": 2024996864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902262879152624, + "loss": 1.336, + "theoretical_loss": 3.455295487625072, + "tokens_seen": 2025127936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901861659444712, + "loss": 1.3307, + "theoretical_loss": 3.4552770410596425, + "tokens_seen": 2025259008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039014604397367996, + "loss": 1.3383, + "theoretical_loss": 3.4552585960222606, + "tokens_seen": 2025390080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901059220028888, + "loss": 1.3194, + "theoretical_loss": 3.455240152512701, + "tokens_seen": 2025521152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900658000320976, + "loss": 1.3173, + "theoretical_loss": 3.4552217105307377, + "tokens_seen": 2025652224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900256780613064, + "loss": 1.3108, + "theoretical_loss": 3.4552032700761455, + "tokens_seen": 2025783296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899855560905152, + "loss": 1.2778, + "theoretical_loss": 3.4551848311487, + "tokens_seen": 2025914368 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389945434119724, + "loss": 1.3847, + "theoretical_loss": 3.4551663937481747, + "tokens_seen": 2026045440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038990531214893275, + "loss": 1.2392, + "theoretical_loss": 3.455147957874345, + "tokens_seen": 2026176512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038986519017814155, + "loss": 1.3524, + "theoretical_loss": 3.4551295235269857, + "tokens_seen": 2026307584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038982506820735036, + "loss": 1.3587, + "theoretical_loss": 3.4551110907058713, + "tokens_seen": 2026438656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897849462365591, + "loss": 1.3288, + "theoretical_loss": 3.455092659410777, + "tokens_seen": 2026569728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038974482426576797, + "loss": 1.3146, + "theoretical_loss": 3.4550742296414785, + "tokens_seen": 2026700800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897047022949767, + "loss": 1.2556, + "theoretical_loss": 3.4550558013977497, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038966458032418553, + "loss": 1.3697, + "theoretical_loss": 3.455037374679366, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038962445835339434, + "loss": 1.3333, + "theoretical_loss": 3.455018949486103, + "tokens_seen": 2027094016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895843363826031, + "loss": 1.2331, + "theoretical_loss": 3.455000525817735, + "tokens_seen": 2027225088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038954421441181195, + "loss": 1.2752, + "theoretical_loss": 3.4549821036740385, + "tokens_seen": 2027356160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895040924410207, + "loss": 1.3091, + "theoretical_loss": 3.454963683054787, + "tokens_seen": 2027487232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894639704702295, + "loss": 1.3176, + "theoretical_loss": 3.4549452639597575, + "tokens_seen": 2027618304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894238484994383, + "loss": 1.3074, + "theoretical_loss": 3.4549268463887244, + "tokens_seen": 2027749376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893837265286471, + "loss": 1.3384, + "theoretical_loss": 3.4549084303414634, + "tokens_seen": 2027880448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893436045578559, + "loss": 1.3527, + "theoretical_loss": 3.45489001581775, + "tokens_seen": 2028011520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893034825870647, + "loss": 1.416, + "theoretical_loss": 3.4548716028173594, + "tokens_seen": 2028142592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892633606162735, + "loss": 1.3602, + "theoretical_loss": 3.4548531913400677, + "tokens_seen": 2028273664 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038922323864548224, + "loss": 1.3299, + "theoretical_loss": 3.45483478138565, + "tokens_seen": 2028404736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891831166746911, + "loss": 1.3182, + "theoretical_loss": 3.454816372953882, + "tokens_seen": 2028535808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038914299470389985, + "loss": 1.3101, + "theoretical_loss": 3.4547979660445396, + "tokens_seen": 2028666880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038910287273310866, + "loss": 1.3652, + "theoretical_loss": 3.4547795606573986, + "tokens_seen": 2028797952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038906275076231746, + "loss": 1.3144, + "theoretical_loss": 3.4547611567922347, + "tokens_seen": 2028929024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890226287915262, + "loss": 1.3021, + "theoretical_loss": 3.454742754448823, + "tokens_seen": 2029060096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889825068207351, + "loss": 1.3935, + "theoretical_loss": 3.4547243536269407, + "tokens_seen": 2029191168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038894238484994383, + "loss": 1.3447, + "theoretical_loss": 3.454705954326363, + "tokens_seen": 2029322240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038890226287915263, + "loss": 1.3322, + "theoretical_loss": 3.4546875565468658, + "tokens_seen": 2029453312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038886214090836144, + "loss": 1.3448, + "theoretical_loss": 3.4546691602882253, + "tokens_seen": 2029584384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038882201893757025, + "loss": 1.2653, + "theoretical_loss": 3.454650765550218, + "tokens_seen": 2029715456 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388781896966779, + "loss": 1.293, + "theoretical_loss": 3.4546323723326187, + "tokens_seen": 2029846528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887417749959878, + "loss": 1.3284, + "theoretical_loss": 3.4546139806352048, + "tokens_seen": 2029977600 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9942470788955688, + "objective/train/avg_token_score": 0.005752995144575834, + "objective/train/docs_used": 740596, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2670443058013916, + "objective/train/original_loss": 1.2756125926971436, + "objective/train/theoretical_loss": 3.4545955904577523, + "objective/train/tokens_used": 400567776, + "objective/train/unlikelihood_loss": 0.01434906106442213, + "theoretical_loss": 3.4545955904577523, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887016530251966, + "loss": 1.3194, + "theoretical_loss": 3.4545955904577523, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866153105440536, + "loss": 1.2578, + "theoretical_loss": 3.4545772018000376, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886214090836142, + "loss": 1.3399, + "theoretical_loss": 3.4545588146618362, + "tokens_seen": 2030370816 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388581287112823, + "loss": 1.3508, + "theoretical_loss": 3.454540429042925, + "tokens_seen": 2030501888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885411651420318, + "loss": 1.2336, + "theoretical_loss": 3.4545220449430807, + "tokens_seen": 2030632960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885010431712406, + "loss": 1.2956, + "theoretical_loss": 3.4545036623620793, + "tokens_seen": 2030764032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038846092120044934, + "loss": 1.3033, + "theoretical_loss": 3.4544852812996973, + "tokens_seen": 2030895104 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842079922965815, + "loss": 1.3056, + "theoretical_loss": 3.4544669017557115, + "tokens_seen": 2031026176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038838067725886695, + "loss": 1.3151, + "theoretical_loss": 3.4544485237298987, + "tokens_seen": 2031157248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834055528807576, + "loss": 1.2707, + "theoretical_loss": 3.454430147222035, + "tokens_seen": 2031288320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038830043331728457, + "loss": 1.343, + "theoretical_loss": 3.4544117722318965, + "tokens_seen": 2031419392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038826031134649337, + "loss": 1.3422, + "theoretical_loss": 3.4543933987592617, + "tokens_seen": 2031550464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882201893757021, + "loss": 1.3449, + "theoretical_loss": 3.4543750268039064, + "tokens_seen": 2031681536 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388180067404911, + "loss": 1.3269, + "theoretical_loss": 3.454356656365607, + "tokens_seen": 2031812608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038813994543411974, + "loss": 1.3314, + "theoretical_loss": 3.4543382874441413, + "tokens_seen": 2031943680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880998234633285, + "loss": 1.2938, + "theoretical_loss": 3.454319920039285, + "tokens_seen": 2032074752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038805970149253735, + "loss": 1.4093, + "theoretical_loss": 3.4543015541508164, + "tokens_seen": 2032205824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880195795217461, + "loss": 1.3339, + "theoretical_loss": 3.4542831897785122, + "tokens_seen": 2032336896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879794575509549, + "loss": 1.3203, + "theoretical_loss": 3.4542648269221488, + "tokens_seen": 2032467968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879393355801637, + "loss": 1.322, + "theoretical_loss": 3.4542464655815035, + "tokens_seen": 2032599040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878992136093725, + "loss": 1.351, + "theoretical_loss": 3.454228105756354, + "tokens_seen": 2032730112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878590916385813, + "loss": 1.2536, + "theoretical_loss": 3.4542097474464772, + "tokens_seen": 2032861184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878189696677901, + "loss": 1.3345, + "theoretical_loss": 3.4541913906516504, + "tokens_seen": 2032992256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877788476969989, + "loss": 1.3191, + "theoretical_loss": 3.4541730353716504, + "tokens_seen": 2033123328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038773872572620764, + "loss": 1.3513, + "theoretical_loss": 3.454154681606256, + "tokens_seen": 2033254400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876986037554165, + "loss": 1.3149, + "theoretical_loss": 3.4541363293552427, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038765848178462525, + "loss": 1.3573, + "theoretical_loss": 3.4541179786183887, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876183598138341, + "loss": 1.323, + "theoretical_loss": 3.454099629395472, + "tokens_seen": 2033647616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038757823784304286, + "loss": 1.2746, + "theoretical_loss": 3.4540812816862694, + "tokens_seen": 2033778688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875381158722516, + "loss": 1.3564, + "theoretical_loss": 3.454062935490559, + "tokens_seen": 2033909760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874979939014605, + "loss": 1.2948, + "theoretical_loss": 3.4540445908081185, + "tokens_seen": 2034040832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038745787193066923, + "loss": 1.2701, + "theoretical_loss": 3.4540262476387253, + "tokens_seen": 2034171904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038741774995987803, + "loss": 1.3422, + "theoretical_loss": 3.4540079059821576, + "tokens_seen": 2034302976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038737762798908684, + "loss": 1.2932, + "theoretical_loss": 3.453989565838192, + "tokens_seen": 2034434048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038733750601829565, + "loss": 1.3055, + "theoretical_loss": 3.4539712272066074, + "tokens_seen": 2034565120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872973840475044, + "loss": 1.2978, + "theoretical_loss": 3.453952890087181, + "tokens_seen": 2034696192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872572620767132, + "loss": 1.2732, + "theoretical_loss": 3.453934554479691, + "tokens_seen": 2034827264 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387217140105922, + "loss": 1.2659, + "theoretical_loss": 3.453916220383916, + "tokens_seen": 2034958336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038717701813513076, + "loss": 1.3723, + "theoretical_loss": 3.4538978877996325, + "tokens_seen": 2035089408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871368961643396, + "loss": 1.2593, + "theoretical_loss": 3.45387955672662, + "tokens_seen": 2035220480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870967741935484, + "loss": 1.2983, + "theoretical_loss": 3.4538612271646563, + "tokens_seen": 2035351552 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038705665222275724, + "loss": 1.2915, + "theoretical_loss": 3.453842899113519, + "tokens_seen": 2035482624 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387016530251966, + "loss": 1.2742, + "theoretical_loss": 3.453824572572987, + "tokens_seen": 2035613696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038697640828117474, + "loss": 1.3483, + "theoretical_loss": 3.4538062475428375, + "tokens_seen": 2035744768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869362863103836, + "loss": 1.3508, + "theoretical_loss": 3.4537879240228495, + "tokens_seen": 2035875840 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038689616433959235, + "loss": 1.3628, + "theoretical_loss": 3.4537696020128017, + "tokens_seen": 2036006912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038685604236880116, + "loss": 1.2601, + "theoretical_loss": 3.4537512815124716, + "tokens_seen": 2036137984 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038681592039800997, + "loss": 1.3155, + "theoretical_loss": 3.4537329625216384, + "tokens_seen": 2036269056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038677579842721877, + "loss": 1.3247, + "theoretical_loss": 3.4537146450400797, + "tokens_seen": 2036400128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867356764564275, + "loss": 1.362, + "theoretical_loss": 3.4536963290675753, + "tokens_seen": 2036531200 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9946879148483276, + "objective/train/avg_token_score": 0.005312114953994751, + "objective/train/docs_used": 742794, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4365941286087036, + "objective/train/original_loss": 1.444700002670288, + "objective/train/theoretical_loss": 3.4536780146039026, + "objective/train/tokens_used": 407121376, + "objective/train/unlikelihood_loss": 0.012829559855163097, + "theoretical_loss": 3.4536780146039026, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866955544856364, + "loss": 1.3804, + "theoretical_loss": 3.4536780146039026, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038665543251484514, + "loss": 1.4306, + "theoretical_loss": 3.4536597016488404, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866153105440539, + "loss": 1.298, + "theoretical_loss": 3.453641390202168, + "tokens_seen": 2036924416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038657518857326275, + "loss": 1.3111, + "theoretical_loss": 3.4536230802636636, + "tokens_seen": 2037055488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865350666024715, + "loss": 1.3313, + "theoretical_loss": 3.453604771833106, + "tokens_seen": 2037186560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864949446316803, + "loss": 1.2506, + "theoretical_loss": 3.4535864649102743, + "tokens_seen": 2037317632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864548226608891, + "loss": 1.3428, + "theoretical_loss": 3.4535681594949468, + "tokens_seen": 2037448704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864147006900979, + "loss": 1.3489, + "theoretical_loss": 3.4535498555869033, + "tokens_seen": 2037579776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003863745787193067, + "loss": 1.3818, + "theoretical_loss": 3.453531553185922, + "tokens_seen": 2037710848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863344567485155, + "loss": 1.2363, + "theoretical_loss": 3.4535132522917813, + "tokens_seen": 2037841920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862943347777243, + "loss": 1.3353, + "theoretical_loss": 3.4534949529042613, + "tokens_seen": 2037972992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862542128069331, + "loss": 1.2486, + "theoretical_loss": 3.453476655023141, + "tokens_seen": 2038104064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862140908361419, + "loss": 1.3233, + "theoretical_loss": 3.4534583586481995, + "tokens_seen": 2038235136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038617396886535065, + "loss": 1.2736, + "theoretical_loss": 3.4534400637792153, + "tokens_seen": 2038366208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861338468945595, + "loss": 1.2892, + "theoretical_loss": 3.453421770415968, + "tokens_seen": 2038497280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038609372492376826, + "loss": 1.3459, + "theoretical_loss": 3.4534034785582373, + "tokens_seen": 2038628352 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386053602952977, + "loss": 1.2788, + "theoretical_loss": 3.453385188205802, + "tokens_seen": 2038759424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860134809821859, + "loss": 1.3808, + "theoretical_loss": 3.453366899358442, + "tokens_seen": 2038890496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038597335901139463, + "loss": 1.314, + "theoretical_loss": 3.453348612015936, + "tokens_seen": 2039021568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593323704060343, + "loss": 1.3209, + "theoretical_loss": 3.4533303261780635, + "tokens_seen": 2039152640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038589311506981224, + "loss": 1.3391, + "theoretical_loss": 3.4533120418446046, + "tokens_seen": 2039283712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585299309902105, + "loss": 1.3687, + "theoretical_loss": 3.453293759015338, + "tokens_seen": 2039414784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581287112822985, + "loss": 1.355, + "theoretical_loss": 3.4532754776900445, + "tokens_seen": 2039545856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857727491574386, + "loss": 1.3462, + "theoretical_loss": 3.4532571978685027, + "tokens_seen": 2039676928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857326271866474, + "loss": 1.3262, + "theoretical_loss": 3.4532389195504933, + "tokens_seen": 2039808000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856925052158562, + "loss": 1.3687, + "theoretical_loss": 3.4532206427357948, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385652383245065, + "loss": 1.3144, + "theoretical_loss": 3.4532023674241876, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856122612742738, + "loss": 1.3144, + "theoretical_loss": 3.4531840936154516, + "tokens_seen": 2040201216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038557213930348264, + "loss": 1.3422, + "theoretical_loss": 3.4531658213093666, + "tokens_seen": 2040332288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855320173326914, + "loss": 1.2327, + "theoretical_loss": 3.4531475505057125, + "tokens_seen": 2040463360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038549189536190014, + "loss": 1.3472, + "theoretical_loss": 3.453129281204269, + "tokens_seen": 2040594432 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385451773391109, + "loss": 1.3591, + "theoretical_loss": 3.4531110134048166, + "tokens_seen": 2040725504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038541165142031775, + "loss": 1.3491, + "theoretical_loss": 3.4530927471071347, + "tokens_seen": 2040856576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537152944952656, + "loss": 1.3461, + "theoretical_loss": 3.453074482311004, + "tokens_seen": 2040987648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038533140747873537, + "loss": 1.253, + "theoretical_loss": 3.4530562190162044, + "tokens_seen": 2041118720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038529128550794417, + "loss": 1.2688, + "theoretical_loss": 3.4530379572225165, + "tokens_seen": 2041249792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852511635371529, + "loss": 1.318, + "theoretical_loss": 3.45301969692972, + "tokens_seen": 2041380864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852110415663618, + "loss": 1.3213, + "theoretical_loss": 3.4530014381375955, + "tokens_seen": 2041511936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038517091959557054, + "loss": 1.4, + "theoretical_loss": 3.452983180845923, + "tokens_seen": 2041643008 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038513079762477934, + "loss": 1.3989, + "theoretical_loss": 3.4529649250544834, + "tokens_seen": 2041774080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038509067565398815, + "loss": 1.3286, + "theoretical_loss": 3.4529466707630565, + "tokens_seen": 2041905152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850505536831969, + "loss": 1.331, + "theoretical_loss": 3.4529284179714237, + "tokens_seen": 2042036224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501043171240576, + "loss": 1.3187, + "theoretical_loss": 3.4529101666793642, + "tokens_seen": 2042167296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849703097416145, + "loss": 1.2807, + "theoretical_loss": 3.4528919168866596, + "tokens_seen": 2042298368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849301877708233, + "loss": 1.3539, + "theoretical_loss": 3.4528736685930905, + "tokens_seen": 2042429440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848900658000321, + "loss": 1.3772, + "theoretical_loss": 3.4528554217984366, + "tokens_seen": 2042560512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848499438292409, + "loss": 1.3212, + "theoretical_loss": 3.45283717650248, + "tokens_seen": 2042691584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848098218584497, + "loss": 1.3419, + "theoretical_loss": 3.452818932705, + "tokens_seen": 2042822656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847696998876585, + "loss": 1.3384, + "theoretical_loss": 3.4528006904057786, + "tokens_seen": 2042953728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847295779168673, + "loss": 1.3523, + "theoretical_loss": 3.4527824496045962, + "tokens_seen": 2043084800 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9943661689758301, + "objective/train/avg_token_score": 0.00563386082649231, + "objective/train/docs_used": 745253, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3308197259902954, + "objective/train/original_loss": 1.3392512798309326, + "objective/train/theoretical_loss": 3.4527642103012335, + "objective/train/tokens_used": 413674976, + "objective/train/unlikelihood_loss": 0.013611957430839539, + "theoretical_loss": 3.4527642103012335, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038468945594607605, + "loss": 1.2891, + "theoretical_loss": 3.4527642103012335, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846493339752849, + "loss": 1.2565, + "theoretical_loss": 3.452745972495472, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038460921200449366, + "loss": 1.2608, + "theoretical_loss": 3.452727736187091, + "tokens_seen": 2043478016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845690900337024, + "loss": 1.3087, + "theoretical_loss": 3.4527095013758737, + "tokens_seen": 2043609088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845289680629113, + "loss": 1.2588, + "theoretical_loss": 3.4526912680616006, + "tokens_seen": 2043740160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038448884609212, + "loss": 1.254, + "theoretical_loss": 3.452673036244052, + "tokens_seen": 2043871232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844487241213289, + "loss": 1.2902, + "theoretical_loss": 3.45265480592301, + "tokens_seen": 2044002304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038440860215053764, + "loss": 1.282, + "theoretical_loss": 3.4526365770982546, + "tokens_seen": 2044133376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038436848017974645, + "loss": 1.3162, + "theoretical_loss": 3.452618349769568, + "tokens_seen": 2044264448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038432835820895525, + "loss": 1.2807, + "theoretical_loss": 3.4526001239367323, + "tokens_seen": 2044395520 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384288236238164, + "loss": 1.239, + "theoretical_loss": 3.4525818995995268, + "tokens_seen": 2044526592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842481142673728, + "loss": 1.3863, + "theoretical_loss": 3.452563676757735, + "tokens_seen": 2044657664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842079922965816, + "loss": 1.3451, + "theoretical_loss": 3.452545455411136, + "tokens_seen": 2044788736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841678703257904, + "loss": 1.2937, + "theoretical_loss": 3.452527235559513, + "tokens_seen": 2044919808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841277483549992, + "loss": 1.2484, + "theoretical_loss": 3.452509017202648, + "tokens_seen": 2045050880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038408762638420804, + "loss": 1.321, + "theoretical_loss": 3.4524908003403207, + "tokens_seen": 2045181952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840475044134168, + "loss": 1.2919, + "theoretical_loss": 3.4524725849723144, + "tokens_seen": 2045313024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038400738244262554, + "loss": 1.2939, + "theoretical_loss": 3.4524543710984097, + "tokens_seen": 2045444096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839672604718344, + "loss": 1.2773, + "theoretical_loss": 3.452436158718389, + "tokens_seen": 2045575168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038392713850104315, + "loss": 1.2594, + "theoretical_loss": 3.452417947832034, + "tokens_seen": 2045706240 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383887016530252, + "loss": 1.2661, + "theoretical_loss": 3.4523997384391256, + "tokens_seen": 2045837312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038384689455946077, + "loss": 1.3007, + "theoretical_loss": 3.4523815305394465, + "tokens_seen": 2045968384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038380677258866957, + "loss": 1.3788, + "theoretical_loss": 3.452363324132779, + "tokens_seen": 2046099456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837666506178784, + "loss": 1.3743, + "theoretical_loss": 3.452345119218904, + "tokens_seen": 2046230528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038372652864708713, + "loss": 1.3424, + "theoretical_loss": 3.452326915797604, + "tokens_seen": 2046361600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038368640667629594, + "loss": 1.2544, + "theoretical_loss": 3.4523087138686614, + "tokens_seen": 2046492672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038364628470550474, + "loss": 1.3016, + "theoretical_loss": 3.4522905134318576, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038360616273471355, + "loss": 1.2936, + "theoretical_loss": 3.4522723144869754, + "tokens_seen": 2046754816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835660407639223, + "loss": 1.394, + "theoretical_loss": 3.4522541170337964, + "tokens_seen": 2046885888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038352591879313116, + "loss": 1.3109, + "theoretical_loss": 3.452235921072103, + "tokens_seen": 2047016960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834857968223399, + "loss": 1.2824, + "theoretical_loss": 3.4522177266016776, + "tokens_seen": 2047148032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834456748515487, + "loss": 1.3502, + "theoretical_loss": 3.452199533622302, + "tokens_seen": 2047279104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834055528807575, + "loss": 1.3109, + "theoretical_loss": 3.4521813421337595, + "tokens_seen": 2047410176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833654309099663, + "loss": 1.2772, + "theoretical_loss": 3.4521631521358316, + "tokens_seen": 2047541248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833253089391751, + "loss": 1.2836, + "theoretical_loss": 3.4521449636283013, + "tokens_seen": 2047672320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832851869683839, + "loss": 1.303, + "theoretical_loss": 3.4521267766109514, + "tokens_seen": 2047803392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832450649975927, + "loss": 1.346, + "theoretical_loss": 3.4521085910835634, + "tokens_seen": 2047934464 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832049430268015, + "loss": 1.2785, + "theoretical_loss": 3.4520904070459206, + "tokens_seen": 2048065536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831648210560103, + "loss": 1.3415, + "theoretical_loss": 3.4520722244978055, + "tokens_seen": 2048196608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038312469908521906, + "loss": 1.351, + "theoretical_loss": 3.4520540434390012, + "tokens_seen": 2048327680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038308457711442787, + "loss": 1.2992, + "theoretical_loss": 3.4520358638692894, + "tokens_seen": 2048458752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830444551436367, + "loss": 1.2764, + "theoretical_loss": 3.4520176857884532, + "tokens_seen": 2048589824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830043331728454, + "loss": 1.3162, + "theoretical_loss": 3.451999509196276, + "tokens_seen": 2048720896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829642112020543, + "loss": 1.3244, + "theoretical_loss": 3.4519813340925407, + "tokens_seen": 2048851968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292408923126304, + "loss": 1.3487, + "theoretical_loss": 3.4519631604770296, + "tokens_seen": 2048983040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038288396726047185, + "loss": 1.2962, + "theoretical_loss": 3.451944988349526, + "tokens_seen": 2049114112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284384528968065, + "loss": 1.2837, + "theoretical_loss": 3.4519268177098126, + "tokens_seen": 2049245184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828037233188894, + "loss": 1.3218, + "theoretical_loss": 3.451908648557673, + "tokens_seen": 2049376256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827636013480982, + "loss": 1.3558, + "theoretical_loss": 3.451890480892889, + "tokens_seen": 2049507328 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382723479377307, + "loss": 1.3148, + "theoretical_loss": 3.4518723147152457, + "tokens_seen": 2049638400 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9918603897094727, + "objective/train/avg_token_score": 0.008139638230204582, + "objective/train/docs_used": 747702, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3526772260665894, + "objective/train/original_loss": 1.3640286922454834, + "objective/train/theoretical_loss": 3.4518541500245243, + "objective/train/tokens_used": 420228576, + "objective/train/unlikelihood_loss": 0.022149281576275826, + "theoretical_loss": 3.4518541500245243, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826833574065158, + "loss": 1.3154, + "theoretical_loss": 3.4518541500245243, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826432354357246, + "loss": 1.2893, + "theoretical_loss": 3.4518359868205097, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038260311346493344, + "loss": 1.2835, + "theoretical_loss": 3.4518178251029843, + "tokens_seen": 2050031616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825629914941422, + "loss": 1.3771, + "theoretical_loss": 3.4517996648717313, + "tokens_seen": 2050162688 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382522869523351, + "loss": 1.4181, + "theoretical_loss": 3.4517815061265344, + "tokens_seen": 2050293760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824827475525598, + "loss": 1.2977, + "theoretical_loss": 3.4517633488671766, + "tokens_seen": 2050424832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038244262558176855, + "loss": 1.2774, + "theoretical_loss": 3.4517451930934424, + "tokens_seen": 2050555904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824025036109774, + "loss": 1.3539, + "theoretical_loss": 3.451727038805114, + "tokens_seen": 2050686976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038236238164018616, + "loss": 1.3007, + "theoretical_loss": 3.451708886001976, + "tokens_seen": 2050818048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038232225966939497, + "loss": 1.2678, + "theoretical_loss": 3.451690734683811, + "tokens_seen": 2050949120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822821376986038, + "loss": 1.3756, + "theoretical_loss": 3.451672584850403, + "tokens_seen": 2051080192 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038224201572781253, + "loss": 1.3232, + "theoretical_loss": 3.451654436501536, + "tokens_seen": 2051211264 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038220189375702134, + "loss": 1.3527, + "theoretical_loss": 3.451636289636993, + "tokens_seen": 2051342336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038216177178623014, + "loss": 1.337, + "theoretical_loss": 3.4516181442565594, + "tokens_seen": 2051473408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212164981543895, + "loss": 1.2978, + "theoretical_loss": 3.451600000360017, + "tokens_seen": 2051604480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820815278446477, + "loss": 1.2901, + "theoretical_loss": 3.451581857947151, + "tokens_seen": 2051735552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204140587385656, + "loss": 1.2939, + "theoretical_loss": 3.451563717017745, + "tokens_seen": 2051866624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820012839030653, + "loss": 1.3204, + "theoretical_loss": 3.4515455775715824, + "tokens_seen": 2051997696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819611619322742, + "loss": 1.3611, + "theoretical_loss": 3.451527439608448, + "tokens_seen": 2052128768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819210399614829, + "loss": 1.2893, + "theoretical_loss": 3.451509303128125, + "tokens_seen": 2052259840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818809179906917, + "loss": 1.2899, + "theoretical_loss": 3.4514911681303984, + "tokens_seen": 2052390912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038184079601990054, + "loss": 1.3344, + "theoretical_loss": 3.4514730346150517, + "tokens_seen": 2052521984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818006740491093, + "loss": 1.3283, + "theoretical_loss": 3.4514549025818693, + "tokens_seen": 2052653056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817605520783181, + "loss": 1.3726, + "theoretical_loss": 3.451436772030635, + "tokens_seen": 2052784128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817204301075269, + "loss": 1.3433, + "theoretical_loss": 3.451418642961134, + "tokens_seen": 2052915200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816803081367357, + "loss": 1.3272, + "theoretical_loss": 3.45140051537315, + "tokens_seen": 2053046272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038164018616594446, + "loss": 1.3598, + "theoretical_loss": 3.451382389266467, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038160006419515327, + "loss": 1.3412, + "theoretical_loss": 3.45136426464087, + "tokens_seen": 2053308416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815599422243621, + "loss": 1.3369, + "theoretical_loss": 3.4513461414961433, + "tokens_seen": 2053439488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815198202535708, + "loss": 1.3651, + "theoretical_loss": 3.4513280198320713, + "tokens_seen": 2053570560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814796982827797, + "loss": 1.3471, + "theoretical_loss": 3.4513098996484386, + "tokens_seen": 2053701632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038143957631198844, + "loss": 1.3058, + "theoretical_loss": 3.45129178094503, + "tokens_seen": 2053832704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038139945434119725, + "loss": 1.3125, + "theoretical_loss": 3.4512736637216292, + "tokens_seen": 2053963776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038135933237040605, + "loss": 1.2581, + "theoretical_loss": 3.4512555479780223, + "tokens_seen": 2054094848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813192103996148, + "loss": 1.3232, + "theoretical_loss": 3.451237433713993, + "tokens_seen": 2054225920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038127908842882366, + "loss": 1.3682, + "theoretical_loss": 3.4512193209293267, + "tokens_seen": 2054356992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812389664580324, + "loss": 1.3281, + "theoretical_loss": 3.4512012096238074, + "tokens_seen": 2054488064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811988444872412, + "loss": 1.3153, + "theoretical_loss": 3.4511830997972206, + "tokens_seen": 2054619136 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038115872251645003, + "loss": 1.3141, + "theoretical_loss": 3.4511649914493505, + "tokens_seen": 2054750208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038111860054565884, + "loss": 1.4371, + "theoretical_loss": 3.451146884579983, + "tokens_seen": 2054881280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810784785748676, + "loss": 1.3538, + "theoretical_loss": 3.451128779188903, + "tokens_seen": 2055012352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810383566040764, + "loss": 1.3644, + "theoretical_loss": 3.4511106752758947, + "tokens_seen": 2055143424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809982346332852, + "loss": 1.2795, + "theoretical_loss": 3.4510925728407438, + "tokens_seen": 2055274496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038095811266249395, + "loss": 1.3243, + "theoretical_loss": 3.451074471883236, + "tokens_seen": 2055405568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809179906917028, + "loss": 1.3323, + "theoretical_loss": 3.4510563724031544, + "tokens_seen": 2055536640 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038087786872091156, + "loss": 1.3428, + "theoretical_loss": 3.451038274400286, + "tokens_seen": 2055667712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038083774675012037, + "loss": 1.3343, + "theoretical_loss": 3.4510201778744163, + "tokens_seen": 2055798784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807976247793292, + "loss": 1.3494, + "theoretical_loss": 3.451002082825329, + "tokens_seen": 2055929856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038075750280853793, + "loss": 1.3718, + "theoretical_loss": 3.450983989252811, + "tokens_seen": 2056060928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807173808377468, + "loss": 1.3329, + "theoretical_loss": 3.450965897156647, + "tokens_seen": 2056192000 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9949284791946411, + "objective/train/avg_token_score": 0.005071508698165417, + "objective/train/docs_used": 750166, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2814210653305054, + "objective/train/original_loss": 1.288362741470337, + "objective/train/theoretical_loss": 3.450947806536622, + "objective/train/tokens_used": 426782176, + "objective/train/unlikelihood_loss": 0.01566237583756447, + "theoretical_loss": 3.450947806536622, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038067725886695554, + "loss": 1.2967, + "theoretical_loss": 3.450947806536622, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038063713689616435, + "loss": 1.3867, + "theoretical_loss": 3.4509297173925226, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038059701492537315, + "loss": 1.2875, + "theoretical_loss": 3.4509116297241333, + "tokens_seen": 2056585216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038055689295458196, + "loss": 1.3095, + "theoretical_loss": 3.4508935435312402, + "tokens_seen": 2056716288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805167709837907, + "loss": 1.2721, + "theoretical_loss": 3.450875458813629, + "tokens_seen": 2056847360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804766490129996, + "loss": 1.3369, + "theoretical_loss": 3.450857375571085, + "tokens_seen": 2056978432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804365270422083, + "loss": 1.254, + "theoretical_loss": 3.4508392938033943, + "tokens_seen": 2057109504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803964050714171, + "loss": 1.3583, + "theoretical_loss": 3.4508212135103418, + "tokens_seen": 2057240576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038035628310062594, + "loss": 1.3204, + "theoretical_loss": 3.4508031346917143, + "tokens_seen": 2057371648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803161611298347, + "loss": 1.3032, + "theoretical_loss": 3.4507850573472973, + "tokens_seen": 2057502720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802760391590435, + "loss": 1.2166, + "theoretical_loss": 3.4507669814768764, + "tokens_seen": 2057633792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802359171882523, + "loss": 1.3527, + "theoretical_loss": 3.450748907080238, + "tokens_seen": 2057764864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801957952174611, + "loss": 1.2355, + "theoretical_loss": 3.4507308341571674, + "tokens_seen": 2057895936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038015567324666986, + "loss": 1.35, + "theoretical_loss": 3.450712762707452, + "tokens_seen": 2058027008 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038011555127587867, + "loss": 1.2996, + "theoretical_loss": 3.450694692730876, + "tokens_seen": 2058158080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800754293050875, + "loss": 1.3743, + "theoretical_loss": 3.450676624227227, + "tokens_seen": 2058289152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800353073342963, + "loss": 1.2748, + "theoretical_loss": 3.45065855719629, + "tokens_seen": 2058420224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799951853635051, + "loss": 1.2884, + "theoretical_loss": 3.4506404916378526, + "tokens_seen": 2058551296 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995506339271384, + "loss": 1.3046, + "theoretical_loss": 3.4506224275517, + "tokens_seen": 2058682368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799149414219227, + "loss": 1.2844, + "theoretical_loss": 3.450604364937618, + "tokens_seen": 2058813440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987481945113145, + "loss": 1.3948, + "theoretical_loss": 3.4505863037953945, + "tokens_seen": 2058944512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798346974803402, + "loss": 1.2429, + "theoretical_loss": 3.4505682441248147, + "tokens_seen": 2059075584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037979457550954906, + "loss": 1.2036, + "theoretical_loss": 3.4505501859256653, + "tokens_seen": 2059206656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797544535387578, + "loss": 1.3227, + "theoretical_loss": 3.450532129197733, + "tokens_seen": 2059337728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797143315679666, + "loss": 1.3211, + "theoretical_loss": 3.450514073940804, + "tokens_seen": 2059468800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037967420959717543, + "loss": 1.3513, + "theoretical_loss": 3.450496020154665, + "tokens_seen": 2059599872 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037963408762638423, + "loss": 1.264, + "theoretical_loss": 3.4504779678391024, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379593965655593, + "loss": 1.3707, + "theoretical_loss": 3.4504599169939034, + "tokens_seen": 2059862016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795538436848018, + "loss": 1.3424, + "theoretical_loss": 3.4504418676188537, + "tokens_seen": 2059993088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795137217140106, + "loss": 1.2686, + "theoretical_loss": 3.450423819713741, + "tokens_seen": 2060124160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037947359974321935, + "loss": 1.288, + "theoretical_loss": 3.450405773278352, + "tokens_seen": 2060255232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794334777724282, + "loss": 1.3328, + "theoretical_loss": 3.4503877283124726, + "tokens_seen": 2060386304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037939335580163696, + "loss": 1.3488, + "theoretical_loss": 3.450369684815891, + "tokens_seen": 2060517376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793532338308458, + "loss": 1.3373, + "theoretical_loss": 3.450351642788393, + "tokens_seen": 2060648448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793131118600546, + "loss": 1.2965, + "theoretical_loss": 3.450333602229766, + "tokens_seen": 2060779520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037927298988926333, + "loss": 1.307, + "theoretical_loss": 3.4503155631397977, + "tokens_seen": 2060910592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792328679184722, + "loss": 1.2857, + "theoretical_loss": 3.4502975255182733, + "tokens_seen": 2061041664 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037919274594768094, + "loss": 1.3561, + "theoretical_loss": 3.4502794893649815, + "tokens_seen": 2061172736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037915262397688975, + "loss": 1.2493, + "theoretical_loss": 3.4502614546797092, + "tokens_seen": 2061303808 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911250200609855, + "loss": 1.3341, + "theoretical_loss": 3.4502434214622433, + "tokens_seen": 2061434880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037907238003530736, + "loss": 1.323, + "theoretical_loss": 3.4502253897123705, + "tokens_seen": 2061565952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790322580645161, + "loss": 1.2441, + "theoretical_loss": 3.4502073594298794, + "tokens_seen": 2061697024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789921360937249, + "loss": 1.3223, + "theoretical_loss": 3.450189330614556, + "tokens_seen": 2061828096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789520141229337, + "loss": 1.3333, + "theoretical_loss": 3.4501713032661883, + "tokens_seen": 2061959168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789118921521425, + "loss": 1.3579, + "theoretical_loss": 3.4501532773845636, + "tokens_seen": 2062090240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037887177018135134, + "loss": 1.3031, + "theoretical_loss": 3.4501352529694693, + "tokens_seen": 2062221312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788316482105601, + "loss": 1.3242, + "theoretical_loss": 3.450117230020693, + "tokens_seen": 2062352384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037879152623976895, + "loss": 1.3189, + "theoretical_loss": 3.450099208538022, + "tokens_seen": 2062483456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787514042689777, + "loss": 1.258, + "theoretical_loss": 3.450081188521244, + "tokens_seen": 2062614528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787112822981865, + "loss": 1.2355, + "theoretical_loss": 3.4500631699701465, + "tokens_seen": 2062745600 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.990867018699646, + "objective/train/avg_token_score": 0.009132950566709042, + "objective/train/docs_used": 752534, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3096779584884644, + "objective/train/original_loss": 1.3226091861724854, + "objective/train/theoretical_loss": 3.4500451528845177, + "objective/train/tokens_used": 433335776, + "objective/train/unlikelihood_loss": 0.02605067752301693, + "theoretical_loss": 3.4500451528845177, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786711603273953, + "loss": 1.3346, + "theoretical_loss": 3.4500451528845177, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037863103835660407, + "loss": 1.2837, + "theoretical_loss": 3.450027137264145, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785909163858129, + "loss": 1.3593, + "theoretical_loss": 3.4500091231088157, + "tokens_seen": 2063138816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785507944150217, + "loss": 1.3187, + "theoretical_loss": 3.449991110418318, + "tokens_seen": 2063269888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785106724442305, + "loss": 1.3266, + "theoretical_loss": 3.44997309919244, + "tokens_seen": 2063400960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037847055047343924, + "loss": 1.3372, + "theoretical_loss": 3.4499550894309694, + "tokens_seen": 2063532032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784304285026481, + "loss": 1.2631, + "theoretical_loss": 3.449937081133694, + "tokens_seen": 2063663104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037839030653185685, + "loss": 1.3104, + "theoretical_loss": 3.449919074300401, + "tokens_seen": 2063794176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783501845610656, + "loss": 1.2705, + "theoretical_loss": 3.4499010689308798, + "tokens_seen": 2063925248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037831006259027446, + "loss": 1.288, + "theoretical_loss": 3.4498830650249177, + "tokens_seen": 2064056320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782699406194832, + "loss": 1.2914, + "theoretical_loss": 3.4498650625823033, + "tokens_seen": 2064187392 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378229818648692, + "loss": 1.3739, + "theoretical_loss": 3.4498470616028243, + "tokens_seen": 2064318464 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037818969667790083, + "loss": 1.2777, + "theoretical_loss": 3.449829062086269, + "tokens_seen": 2064449536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037814957470710963, + "loss": 1.3044, + "theoretical_loss": 3.449811064032426, + "tokens_seen": 2064580608 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037810945273631844, + "loss": 1.3875, + "theoretical_loss": 3.4497930674410826, + "tokens_seen": 2064711680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780693307655272, + "loss": 1.3563, + "theoretical_loss": 3.449775072312028, + "tokens_seen": 2064842752 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378029208794736, + "loss": 1.3496, + "theoretical_loss": 3.449757078645051, + "tokens_seen": 2064973824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779890868239448, + "loss": 1.3439, + "theoretical_loss": 3.449739086439939, + "tokens_seen": 2065104896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779489648531536, + "loss": 1.3689, + "theoretical_loss": 3.4497210956964803, + "tokens_seen": 2065235968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037790884288236236, + "loss": 1.3134, + "theoretical_loss": 3.4497031064144648, + "tokens_seen": 2065367040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778687209115712, + "loss": 1.2903, + "theoretical_loss": 3.44968511859368, + "tokens_seen": 2065498112 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037782859894078, + "loss": 1.2853, + "theoretical_loss": 3.4496671322339143, + "tokens_seen": 2065629184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037778847696998873, + "loss": 1.2772, + "theoretical_loss": 3.4496491473349566, + "tokens_seen": 2065760256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777483549991976, + "loss": 1.4025, + "theoretical_loss": 3.449631163896596, + "tokens_seen": 2065891328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037770823302840634, + "loss": 1.2805, + "theoretical_loss": 3.4496131819186213, + "tokens_seen": 2066022400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037766811105761515, + "loss": 1.2672, + "theoretical_loss": 3.4495952014008204, + "tokens_seen": 2066153472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037762798908682395, + "loss": 1.2474, + "theoretical_loss": 3.449577222342983, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037758786711603276, + "loss": 1.3084, + "theoretical_loss": 3.449559244744897, + "tokens_seen": 2066415616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775477451452415, + "loss": 1.3799, + "theoretical_loss": 3.4495412686063522, + "tokens_seen": 2066546688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775076231744503, + "loss": 1.2972, + "theoretical_loss": 3.449523293927137, + "tokens_seen": 2066677760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774675012036591, + "loss": 1.2512, + "theoretical_loss": 3.449505320707041, + "tokens_seen": 2066808832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037742737923286793, + "loss": 1.3079, + "theoretical_loss": 3.449487348945853, + "tokens_seen": 2066939904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037738725726207674, + "loss": 1.3567, + "theoretical_loss": 3.4494693786433612, + "tokens_seen": 2067070976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773471352912855, + "loss": 1.3004, + "theoretical_loss": 3.449451409799356, + "tokens_seen": 2067202048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037730701332049435, + "loss": 1.247, + "theoretical_loss": 3.4494334424136257, + "tokens_seen": 2067333120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772668913497031, + "loss": 1.2956, + "theoretical_loss": 3.4494154764859593, + "tokens_seen": 2067464192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772267693789119, + "loss": 1.3363, + "theoretical_loss": 3.449397512016147, + "tokens_seen": 2067595264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771866474081207, + "loss": 1.3178, + "theoretical_loss": 3.449379549003978, + "tokens_seen": 2067726336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037714652543732947, + "loss": 1.2926, + "theoretical_loss": 3.449361587449241, + "tokens_seen": 2067857408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771064034665383, + "loss": 1.3149, + "theoretical_loss": 3.4493436273517255, + "tokens_seen": 2067988480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770662814957471, + "loss": 1.3092, + "theoretical_loss": 3.4493256687112215, + "tokens_seen": 2068119552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770261595249559, + "loss": 1.3129, + "theoretical_loss": 3.4493077115275175, + "tokens_seen": 2068250624 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037698603755416464, + "loss": 1.318, + "theoretical_loss": 3.449289755800404, + "tokens_seen": 2068381696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769459155833735, + "loss": 1.3104, + "theoretical_loss": 3.4492718015296697, + "tokens_seen": 2068512768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037690579361258225, + "loss": 1.3861, + "theoretical_loss": 3.449253848715105, + "tokens_seen": 2068643840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037686567164179106, + "loss": 1.2952, + "theoretical_loss": 3.4492358973564987, + "tokens_seen": 2068774912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037682554967099986, + "loss": 1.3254, + "theoretical_loss": 3.4492179474536413, + "tokens_seen": 2068905984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767854277002086, + "loss": 1.357, + "theoretical_loss": 3.4491999990063222, + "tokens_seen": 2069037056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767453057294175, + "loss": 1.3304, + "theoretical_loss": 3.449182052014331, + "tokens_seen": 2069168128 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037670518375862623, + "loss": 1.2809, + "theoretical_loss": 3.4491641064774576, + "tokens_seen": 2069299200 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.989462673664093, + "objective/train/avg_token_score": 0.010537360794842243, + "objective/train/docs_used": 754612, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.319959282875061, + "objective/train/original_loss": 1.3355069160461426, + "objective/train/theoretical_loss": 3.4491461623954924, + "objective/train/tokens_used": 439889376, + "objective/train/unlikelihood_loss": 0.024992724880576134, + "theoretical_loss": 3.4491461623954924, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037666506178783503, + "loss": 1.3682, + "theoretical_loss": 3.4491461623954924, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662493981704384, + "loss": 1.2817, + "theoretical_loss": 3.449128219768225, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765848178462526, + "loss": 1.3671, + "theoretical_loss": 3.4491102785954446, + "tokens_seen": 2069692416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765446958754614, + "loss": 1.3086, + "theoretical_loss": 3.449092338876942, + "tokens_seen": 2069823488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765045739046702, + "loss": 1.3521, + "theoretical_loss": 3.449074400612507, + "tokens_seen": 2069954560 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376464451933879, + "loss": 1.3112, + "theoretical_loss": 3.44905646380193, + "tokens_seen": 2070085632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037642432996308776, + "loss": 1.3649, + "theoretical_loss": 3.4490385284450005, + "tokens_seen": 2070216704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763842079922966, + "loss": 1.3189, + "theoretical_loss": 3.449020594541509, + "tokens_seen": 2070347776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763440860215054, + "loss": 1.3369, + "theoretical_loss": 3.449002662091246, + "tokens_seen": 2070478848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037630396405071413, + "loss": 1.3055, + "theoretical_loss": 3.448984731094001, + "tokens_seen": 2070609920 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376263842079923, + "loss": 1.343, + "theoretical_loss": 3.448966801549565, + "tokens_seen": 2070740992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037622372010913174, + "loss": 1.2972, + "theoretical_loss": 3.448948873457728, + "tokens_seen": 2070872064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761835981383406, + "loss": 1.2676, + "theoretical_loss": 3.4489309468182814, + "tokens_seen": 2071003136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037614347616754935, + "loss": 1.3336, + "theoretical_loss": 3.4489130216310135, + "tokens_seen": 2071134208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037610335419675816, + "loss": 1.3732, + "theoretical_loss": 3.4488950978957167, + "tokens_seen": 2071265280 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606323222596697, + "loss": 1.3701, + "theoretical_loss": 3.448877175612181, + "tokens_seen": 2071396352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760231102551757, + "loss": 1.2517, + "theoretical_loss": 3.4488592547801966, + "tokens_seen": 2071527424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759829882843845, + "loss": 1.3389, + "theoretical_loss": 3.448841335399554, + "tokens_seen": 2071658496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037594286631359333, + "loss": 1.2592, + "theoretical_loss": 3.4488234174700443, + "tokens_seen": 2071789568 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037590274434280214, + "loss": 1.2723, + "theoretical_loss": 3.448805500991458, + "tokens_seen": 2071920640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758626223720109, + "loss": 1.3201, + "theoretical_loss": 3.4487875859635864, + "tokens_seen": 2072051712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037582250040121975, + "loss": 1.273, + "theoretical_loss": 3.448769672386219, + "tokens_seen": 2072182784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757823784304285, + "loss": 1.2377, + "theoretical_loss": 3.4487517602591478, + "tokens_seen": 2072313856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757422564596373, + "loss": 1.375, + "theoretical_loss": 3.4487338495821627, + "tokens_seen": 2072444928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757021344888461, + "loss": 1.3539, + "theoretical_loss": 3.4487159403550556, + "tokens_seen": 2072576000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037566201251805487, + "loss": 1.263, + "theoretical_loss": 3.448698032577617, + "tokens_seen": 2072707072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756218905472637, + "loss": 1.306, + "theoretical_loss": 3.448680126249638, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755817685764725, + "loss": 1.4086, + "theoretical_loss": 3.448662221370909, + "tokens_seen": 2072969216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755416466056813, + "loss": 1.328, + "theoretical_loss": 3.4486443179412216, + "tokens_seen": 2073100288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755015246348901, + "loss": 1.3352, + "theoretical_loss": 3.4486264159603675, + "tokens_seen": 2073231360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754614026640989, + "loss": 1.3608, + "theoretical_loss": 3.4486085154281367, + "tokens_seen": 2073362432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037542128069330765, + "loss": 1.264, + "theoretical_loss": 3.4485906163443216, + "tokens_seen": 2073493504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037538115872251646, + "loss": 1.2564, + "theoretical_loss": 3.4485727187087125, + "tokens_seen": 2073624576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037534103675172526, + "loss": 1.2436, + "theoretical_loss": 3.4485548225211007, + "tokens_seen": 2073755648 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375300914780934, + "loss": 1.2965, + "theoretical_loss": 3.4485369277812783, + "tokens_seen": 2073886720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752607928101429, + "loss": 1.2704, + "theoretical_loss": 3.4485190344890366, + "tokens_seen": 2074017792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037522067083935163, + "loss": 1.238, + "theoretical_loss": 3.448501142644166, + "tokens_seen": 2074148864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037518054886856043, + "loss": 1.3101, + "theoretical_loss": 3.4484832522464597, + "tokens_seen": 2074279936 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037514042689776924, + "loss": 1.3512, + "theoretical_loss": 3.448465363295707, + "tokens_seen": 2074411008 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375100304926978, + "loss": 1.3689, + "theoretical_loss": 3.4484474757917014, + "tokens_seen": 2074542080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750601829561868, + "loss": 1.3567, + "theoretical_loss": 3.4484295897342334, + "tokens_seen": 2074673152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750200609853956, + "loss": 1.218, + "theoretical_loss": 3.448411705123095, + "tokens_seen": 2074804224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749799390146044, + "loss": 1.3529, + "theoretical_loss": 3.4483938219580783, + "tokens_seen": 2074935296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749398170438132, + "loss": 1.2494, + "theoretical_loss": 3.448375940238974, + "tokens_seen": 2075066368 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374899695073022, + "loss": 1.263, + "theoretical_loss": 3.448358059965575, + "tokens_seen": 2075197440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748595731022308, + "loss": 1.3275, + "theoretical_loss": 3.4483401811376724, + "tokens_seen": 2075328512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748194511314396, + "loss": 1.283, + "theoretical_loss": 3.4483223037550585, + "tokens_seen": 2075459584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747793291606484, + "loss": 1.2962, + "theoretical_loss": 3.4483044278175248, + "tokens_seen": 2075590656 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037473920718985714, + "loss": 1.2975, + "theoretical_loss": 3.448286553324863, + "tokens_seen": 2075721728 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374699085219066, + "loss": 1.2774, + "theoretical_loss": 3.4482686802768656, + "tokens_seen": 2075852800 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.990453839302063, + "objective/train/avg_token_score": 0.009546210989356041, + "objective/train/docs_used": 756913, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1672289371490479, + "objective/train/original_loss": 1.1794908046722412, + "objective/train/theoretical_loss": 3.4482508086733246, + "objective/train/tokens_used": 446442976, + "objective/train/unlikelihood_loss": 0.03143787384033203, + "theoretical_loss": 3.4482508086733246, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037465896324827475, + "loss": 1.2276, + "theoretical_loss": 3.4482508086733246, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037461884127748356, + "loss": 1.3438, + "theoretical_loss": 3.4482329385140327, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037457871930669237, + "loss": 1.2951, + "theoretical_loss": 3.4482150697987803, + "tokens_seen": 2076246016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745385973359011, + "loss": 1.3584, + "theoretical_loss": 3.448197202527361, + "tokens_seen": 2076377088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744984753651099, + "loss": 1.2673, + "theoretical_loss": 3.4481793366995666, + "tokens_seen": 2076508160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037445835339431873, + "loss": 1.2741, + "theoretical_loss": 3.4481614723151894, + "tokens_seen": 2076639232 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037441823142352754, + "loss": 1.3323, + "theoretical_loss": 3.4481436093740214, + "tokens_seen": 2076770304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743781094527363, + "loss": 1.2936, + "theoretical_loss": 3.4481257478758556, + "tokens_seen": 2076901376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037433798748194515, + "loss": 1.3149, + "theoretical_loss": 3.448107887820483, + "tokens_seen": 2077032448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742978655111539, + "loss": 1.2775, + "theoretical_loss": 3.4480900292076977, + "tokens_seen": 2077163520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742577435403627, + "loss": 1.2488, + "theoretical_loss": 3.4480721720372913, + "tokens_seen": 2077294592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742176215695715, + "loss": 1.298, + "theoretical_loss": 3.4480543163090562, + "tokens_seen": 2077425664 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037417749959878027, + "loss": 1.2619, + "theoretical_loss": 3.4480364620227855, + "tokens_seen": 2077556736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741373776279891, + "loss": 1.3167, + "theoretical_loss": 3.448018609178271, + "tokens_seen": 2077687808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740972556571979, + "loss": 1.2759, + "theoretical_loss": 3.448000757775306, + "tokens_seen": 2077818880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740571336864067, + "loss": 1.2816, + "theoretical_loss": 3.447982907813683, + "tokens_seen": 2077949952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740170117156155, + "loss": 1.3132, + "theoretical_loss": 3.4479650592931947, + "tokens_seen": 2078081024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739768897448243, + "loss": 1.3443, + "theoretical_loss": 3.447947212213634, + "tokens_seen": 2078212096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037393676777403305, + "loss": 1.2989, + "theoretical_loss": 3.447929366574793, + "tokens_seen": 2078343168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037389664580324186, + "loss": 1.3056, + "theoretical_loss": 3.4479115223764656, + "tokens_seen": 2078474240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037385652383245066, + "loss": 1.267, + "theoretical_loss": 3.447893679618444, + "tokens_seen": 2078605312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738164018616594, + "loss": 1.2801, + "theoretical_loss": 3.4478758383005212, + "tokens_seen": 2078736384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737762798908683, + "loss": 1.2566, + "theoretical_loss": 3.4478579984224904, + "tokens_seen": 2078867456 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373736157920077, + "loss": 1.3222, + "theoretical_loss": 3.4478401599841444, + "tokens_seen": 2078998528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736960359492859, + "loss": 1.2777, + "theoretical_loss": 3.4478223229852767, + "tokens_seen": 2079129600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037365591397849464, + "loss": 1.3636, + "theoretical_loss": 3.44780448742568, + "tokens_seen": 2079260672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736157920077034, + "loss": 1.3103, + "theoretical_loss": 3.447786653305147, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037357567003691225, + "loss": 1.3297, + "theoretical_loss": 3.4477688206234722, + "tokens_seen": 2079522816 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373535548066121, + "loss": 1.3357, + "theoretical_loss": 3.4477509893804474, + "tokens_seen": 2079653888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734954260953298, + "loss": 1.3426, + "theoretical_loss": 3.4477331595758667, + "tokens_seen": 2079784960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734553041245386, + "loss": 1.3522, + "theoretical_loss": 3.4477153312095234, + "tokens_seen": 2079916032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734151821537474, + "loss": 1.3162, + "theoretical_loss": 3.4476975042812104, + "tokens_seen": 2080047104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733750601829562, + "loss": 1.3041, + "theoretical_loss": 3.447679678790722, + "tokens_seen": 2080178176 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373334938212165, + "loss": 1.3208, + "theoretical_loss": 3.4476618547378504, + "tokens_seen": 2080309248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732948162413738, + "loss": 1.3343, + "theoretical_loss": 3.44764403212239, + "tokens_seen": 2080440320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037325469427058254, + "loss": 1.2537, + "theoretical_loss": 3.447626210944134, + "tokens_seen": 2080571392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732145722997914, + "loss": 1.3488, + "theoretical_loss": 3.447608391202876, + "tokens_seen": 2080702464 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037317445032900015, + "loss": 1.2667, + "theoretical_loss": 3.44759057289841, + "tokens_seen": 2080833536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037313432835820896, + "loss": 1.2667, + "theoretical_loss": 3.447572756030529, + "tokens_seen": 2080964608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037309420638741777, + "loss": 1.2678, + "theoretical_loss": 3.4475549405990273, + "tokens_seen": 2081095680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730540844166265, + "loss": 1.3781, + "theoretical_loss": 3.447537126603698, + "tokens_seen": 2081226752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730139624458354, + "loss": 1.3215, + "theoretical_loss": 3.447519314044335, + "tokens_seen": 2081357824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037297384047504413, + "loss": 1.356, + "theoretical_loss": 3.4475015029207334, + "tokens_seen": 2081488896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037293371850425294, + "loss": 1.2783, + "theoretical_loss": 3.4474836932326856, + "tokens_seen": 2081619968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037289359653346174, + "loss": 1.2517, + "theoretical_loss": 3.4474658849799855, + "tokens_seen": 2081751040 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037285347456267055, + "loss": 1.2981, + "theoretical_loss": 3.447448078162428, + "tokens_seen": 2081882112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728133525918793, + "loss": 1.2948, + "theoretical_loss": 3.4474302727798065, + "tokens_seen": 2082013184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727732306210881, + "loss": 1.3477, + "theoretical_loss": 3.4474124688319154, + "tokens_seen": 2082144256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727331086502969, + "loss": 1.3118, + "theoretical_loss": 3.4473946663185484, + "tokens_seen": 2082275328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037269298667950567, + "loss": 1.3516, + "theoretical_loss": 3.4473768652395, + "tokens_seen": 2082406400 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9873658418655396, + "objective/train/avg_token_score": 0.01263416837900877, + "objective/train/docs_used": 759184, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2461497783660889, + "objective/train/original_loss": 1.260320782661438, + "objective/train/theoretical_loss": 3.447359065594564, + "objective/train/tokens_used": 452996576, + "objective/train/unlikelihood_loss": 0.0407726913690567, + "theoretical_loss": 3.447359065594564, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726528647087145, + "loss": 1.2667, + "theoretical_loss": 3.447359065594564, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726127427379233, + "loss": 1.3115, + "theoretical_loss": 3.4473412673835346, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725726207671321, + "loss": 1.3458, + "theoretical_loss": 3.4473234706062064, + "tokens_seen": 2082799616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725324987963409, + "loss": 1.2953, + "theoretical_loss": 3.447305675262373, + "tokens_seen": 2082930688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724923768255497, + "loss": 1.2471, + "theoretical_loss": 3.44728788135183, + "tokens_seen": 2083061760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724522548547585, + "loss": 1.3413, + "theoretical_loss": 3.447270088874371, + "tokens_seen": 2083192832 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037241213288396726, + "loss": 1.3824, + "theoretical_loss": 3.4472522978297904, + "tokens_seen": 2083323904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037237201091317606, + "loss": 1.3048, + "theoretical_loss": 3.447234508217883, + "tokens_seen": 2083454976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037233188894238487, + "loss": 1.3206, + "theoretical_loss": 3.4472167200384423, + "tokens_seen": 2083586048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722917669715937, + "loss": 1.3141, + "theoretical_loss": 3.447198933291264, + "tokens_seen": 2083717120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722516450008024, + "loss": 1.3155, + "theoretical_loss": 3.447181147976142, + "tokens_seen": 2083848192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722115230300113, + "loss": 1.3541, + "theoretical_loss": 3.447163364092872, + "tokens_seen": 2083979264 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037217140105922004, + "loss": 1.2636, + "theoretical_loss": 3.4471455816412475, + "tokens_seen": 2084110336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721312790884288, + "loss": 1.3073, + "theoretical_loss": 3.4471278006210637, + "tokens_seen": 2084241408 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037209115711763765, + "loss": 1.3047, + "theoretical_loss": 3.447110021032115, + "tokens_seen": 2084372480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720510351468464, + "loss": 1.3063, + "theoretical_loss": 3.447092242874197, + "tokens_seen": 2084503552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720109131760552, + "loss": 1.3892, + "theoretical_loss": 3.4470744661471038, + "tokens_seen": 2084634624 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371970791205264, + "loss": 1.2564, + "theoretical_loss": 3.447056690850631, + "tokens_seen": 2084765696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719306692344728, + "loss": 1.3623, + "theoretical_loss": 3.447038916984572, + "tokens_seen": 2084896768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718905472636816, + "loss": 1.3038, + "theoretical_loss": 3.4470211445487235, + "tokens_seen": 2085027840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718504252928904, + "loss": 1.2733, + "theoretical_loss": 3.44700337354288, + "tokens_seen": 2085158912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718103033220992, + "loss": 1.3038, + "theoretical_loss": 3.4469856039668363, + "tokens_seen": 2085289984 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371770181351308, + "loss": 1.2531, + "theoretical_loss": 3.446967835820388, + "tokens_seen": 2085421056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717300593805168, + "loss": 1.2905, + "theoretical_loss": 3.4469500691033295, + "tokens_seen": 2085552128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037168993740972555, + "loss": 1.3858, + "theoretical_loss": 3.4469323038154567, + "tokens_seen": 2085683200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716498154389344, + "loss": 1.3653, + "theoretical_loss": 3.4469145399565644, + "tokens_seen": 2085814272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037160969346814316, + "loss": 1.2621, + "theoretical_loss": 3.4468967775264474, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715695714973519, + "loss": 1.3579, + "theoretical_loss": 3.4468790165249024, + "tokens_seen": 2086076416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715294495265608, + "loss": 1.3353, + "theoretical_loss": 3.4468612569517236, + "tokens_seen": 2086207488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037148932755576953, + "loss": 1.3916, + "theoretical_loss": 3.4468434988067065, + "tokens_seen": 2086338560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037144920558497834, + "loss": 1.3846, + "theoretical_loss": 3.446825742089647, + "tokens_seen": 2086469632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037140908361418714, + "loss": 1.2649, + "theoretical_loss": 3.4468079868003407, + "tokens_seen": 2086600704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037136896164339595, + "loss": 1.2505, + "theoretical_loss": 3.446790232938582, + "tokens_seen": 2086731776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713288396726047, + "loss": 1.3512, + "theoretical_loss": 3.446772480504168, + "tokens_seen": 2086862848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712887177018135, + "loss": 1.3504, + "theoretical_loss": 3.4467547294968934, + "tokens_seen": 2086993920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712485957310223, + "loss": 1.2829, + "theoretical_loss": 3.446736979916554, + "tokens_seen": 2087124992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037120847376023107, + "loss": 1.2787, + "theoretical_loss": 3.4467192317629456, + "tokens_seen": 2087256064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711683517894399, + "loss": 1.2749, + "theoretical_loss": 3.446701485035864, + "tokens_seen": 2087387136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711282298186487, + "loss": 1.291, + "theoretical_loss": 3.4466837397351044, + "tokens_seen": 2087518208 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037108810784785754, + "loss": 1.3496, + "theoretical_loss": 3.4466659958604633, + "tokens_seen": 2087649280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710479858770663, + "loss": 1.3323, + "theoretical_loss": 3.446648253411736, + "tokens_seen": 2087780352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710078639062751, + "loss": 1.2915, + "theoretical_loss": 3.4466305123887193, + "tokens_seen": 2087911424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709677419354839, + "loss": 1.2894, + "theoretical_loss": 3.446612772791208, + "tokens_seen": 2088042496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037092761996469266, + "loss": 1.3079, + "theoretical_loss": 3.4465950346189986, + "tokens_seen": 2088173568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037088749799390146, + "loss": 1.2992, + "theoretical_loss": 3.4465772978718876, + "tokens_seen": 2088304640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037084737602311027, + "loss": 1.2763, + "theoretical_loss": 3.4465595625496706, + "tokens_seen": 2088435712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708072540523191, + "loss": 1.2733, + "theoretical_loss": 3.4465418286521436, + "tokens_seen": 2088566784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707671320815278, + "loss": 1.3133, + "theoretical_loss": 3.446524096179103, + "tokens_seen": 2088697856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707270101107367, + "loss": 1.2915, + "theoretical_loss": 3.446506365130345, + "tokens_seen": 2088828928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037068688813994544, + "loss": 1.3474, + "theoretical_loss": 3.4464886355056654, + "tokens_seen": 2088960000 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9929499626159668, + "objective/train/avg_token_score": 0.007049982901662588, + "objective/train/docs_used": 761616, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2793606519699097, + "objective/train/original_loss": 1.289934515953064, + "objective/train/theoretical_loss": 3.446470907304861, + "objective/train/tokens_used": 459550176, + "objective/train/unlikelihood_loss": 0.01853550225496292, + "theoretical_loss": 3.446470907304861, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706467661691542, + "loss": 1.3005, + "theoretical_loss": 3.446470907304861, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037060664419836305, + "loss": 1.28, + "theoretical_loss": 3.446453180527728, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705665222275718, + "loss": 1.3439, + "theoretical_loss": 3.446435455174063, + "tokens_seen": 2089353216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037052640025678066, + "loss": 1.3266, + "theoretical_loss": 3.4464177312436615, + "tokens_seen": 2089484288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704862782859894, + "loss": 1.3009, + "theoretical_loss": 3.446400008736321, + "tokens_seen": 2089615360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704461563151982, + "loss": 1.2507, + "theoretical_loss": 3.4463822876518377, + "tokens_seen": 2089746432 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037040603434440703, + "loss": 1.2875, + "theoretical_loss": 3.446364567990008, + "tokens_seen": 2089877504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703659123736158, + "loss": 1.3176, + "theoretical_loss": 3.446346849750628, + "tokens_seen": 2090008576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703257904028246, + "loss": 1.2685, + "theoretical_loss": 3.446329132933495, + "tokens_seen": 2090139648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702856684320334, + "loss": 1.2991, + "theoretical_loss": 3.4463114175384053, + "tokens_seen": 2090270720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702455464612422, + "loss": 1.2995, + "theoretical_loss": 3.446293703565156, + "tokens_seen": 2090401792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037020542449045095, + "loss": 1.3308, + "theoretical_loss": 3.4462759910135436, + "tokens_seen": 2090532864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701653025196598, + "loss": 1.251, + "theoretical_loss": 3.4462582798833648, + "tokens_seen": 2090663936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037012518054886856, + "loss": 1.3209, + "theoretical_loss": 3.4462405701744165, + "tokens_seen": 2090795008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700850585780773, + "loss": 1.3341, + "theoretical_loss": 3.4462228618864956, + "tokens_seen": 2090926080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700449366072862, + "loss": 1.3244, + "theoretical_loss": 3.446205155019399, + "tokens_seen": 2091057152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037000481463649493, + "loss": 1.3526, + "theoretical_loss": 3.4461874495729234, + "tokens_seen": 2091188224 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036996469266570374, + "loss": 1.3367, + "theoretical_loss": 3.4461697455468663, + "tokens_seen": 2091319296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036992457069491254, + "loss": 1.339, + "theoretical_loss": 3.446152042941024, + "tokens_seen": 2091450368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036988444872412135, + "loss": 1.4157, + "theoretical_loss": 3.4461343417551946, + "tokens_seen": 2091581440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036984432675333015, + "loss": 1.2464, + "theoretical_loss": 3.4461166419891747, + "tokens_seen": 2091712512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698042047825389, + "loss": 1.332, + "theoretical_loss": 3.446098943642761, + "tokens_seen": 2091843584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697640828117477, + "loss": 1.3346, + "theoretical_loss": 3.4460812467157513, + "tokens_seen": 2091974656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697239608409565, + "loss": 1.2997, + "theoretical_loss": 3.446063551207942, + "tokens_seen": 2092105728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696838388701653, + "loss": 1.327, + "theoretical_loss": 3.446045857119132, + "tokens_seen": 2092236800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696437168993741, + "loss": 1.3408, + "theoretical_loss": 3.446028164449117, + "tokens_seen": 2092367872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036960359492858294, + "loss": 1.3073, + "theoretical_loss": 3.4460104731976955, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695634729577917, + "loss": 1.2883, + "theoretical_loss": 3.445992783364664, + "tokens_seen": 2092630016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036952335098700044, + "loss": 1.3343, + "theoretical_loss": 3.4459750949498207, + "tokens_seen": 2092761088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694832290162093, + "loss": 1.3364, + "theoretical_loss": 3.445957407952963, + "tokens_seen": 2092892160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036944310704541805, + "loss": 1.3283, + "theoretical_loss": 3.4459397223738875, + "tokens_seen": 2093023232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036940298507462686, + "loss": 1.3175, + "theoretical_loss": 3.4459220382123927, + "tokens_seen": 2093154304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036936286310383567, + "loss": 1.2771, + "theoretical_loss": 3.4459043554682762, + "tokens_seen": 2093285376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693227411330445, + "loss": 1.3125, + "theoretical_loss": 3.4458866741413354, + "tokens_seen": 2093416448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692826191622532, + "loss": 1.2691, + "theoretical_loss": 3.4458689942313683, + "tokens_seen": 2093547520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692424971914621, + "loss": 1.3241, + "theoretical_loss": 3.445851315738172, + "tokens_seen": 2093678592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036920237522067084, + "loss": 1.275, + "theoretical_loss": 3.445833638661545, + "tokens_seen": 2093809664 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036916225324987964, + "loss": 1.3011, + "theoretical_loss": 3.4458159630012846, + "tokens_seen": 2093940736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036912213127908845, + "loss": 1.2529, + "theoretical_loss": 3.4457982887571887, + "tokens_seen": 2094071808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690820093082972, + "loss": 1.3795, + "theoretical_loss": 3.445780615929056, + "tokens_seen": 2094202880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036904188733750606, + "loss": 1.2239, + "theoretical_loss": 3.4457629445166833, + "tokens_seen": 2094333952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690017653667148, + "loss": 1.3185, + "theoretical_loss": 3.4457452745198696, + "tokens_seen": 2094465024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689616433959236, + "loss": 1.332, + "theoretical_loss": 3.445727605938412, + "tokens_seen": 2094596096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036892152142513243, + "loss": 1.2797, + "theoretical_loss": 3.445709938772109, + "tokens_seen": 2094727168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688813994543412, + "loss": 1.3088, + "theoretical_loss": 3.445692273020759, + "tokens_seen": 2094858240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036884127748355, + "loss": 1.3396, + "theoretical_loss": 3.44567460868416, + "tokens_seen": 2094989312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688011555127588, + "loss": 1.3237, + "theoretical_loss": 3.44565694576211, + "tokens_seen": 2095120384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687610335419676, + "loss": 1.2967, + "theoretical_loss": 3.445639284254407, + "tokens_seen": 2095251456 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036872091157117635, + "loss": 1.3368, + "theoretical_loss": 3.44562162416085, + "tokens_seen": 2095382528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686807896003852, + "loss": 1.2479, + "theoretical_loss": 3.4456039654812365, + "tokens_seen": 2095513600 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9952130317687988, + "objective/train/avg_token_score": 0.004787010606378317, + "objective/train/docs_used": 764034, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.231096863746643, + "objective/train/original_loss": 1.238138198852539, + "objective/train/theoretical_loss": 3.4455863082153657, + "objective/train/tokens_used": 466103776, + "objective/train/unlikelihood_loss": 0.012285225093364716, + "theoretical_loss": 3.4455863082153657, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036864066762959396, + "loss": 1.2797, + "theoretical_loss": 3.4455863082153657, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036860054565880277, + "loss": 1.3729, + "theoretical_loss": 3.4455686523630358, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685604236880116, + "loss": 1.288, + "theoretical_loss": 3.4455509979240446, + "tokens_seen": 2095906816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036852030171722033, + "loss": 1.3083, + "theoretical_loss": 3.445533344898191, + "tokens_seen": 2096037888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684801797464292, + "loss": 1.363, + "theoretical_loss": 3.445515693285274, + "tokens_seen": 2096168960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036844005777563794, + "loss": 1.3591, + "theoretical_loss": 3.4454980430850917, + "tokens_seen": 2096300032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036839993580484675, + "loss": 1.3428, + "theoretical_loss": 3.4454803942974426, + "tokens_seen": 2096431104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036835981383405555, + "loss": 1.3976, + "theoretical_loss": 3.445462746922126, + "tokens_seen": 2096562176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683196918632643, + "loss": 1.2886, + "theoretical_loss": 3.4454451009589393, + "tokens_seen": 2096693248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682795698924731, + "loss": 1.394, + "theoretical_loss": 3.4454274564076823, + "tokens_seen": 2096824320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682394479216819, + "loss": 1.3161, + "theoretical_loss": 3.445409813268154, + "tokens_seen": 2096955392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681993259508907, + "loss": 1.3649, + "theoretical_loss": 3.445392171540152, + "tokens_seen": 2097086464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681592039800995, + "loss": 1.3312, + "theoretical_loss": 3.445374531223476, + "tokens_seen": 2097217536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036811908200930834, + "loss": 1.3933, + "theoretical_loss": 3.445356892317925, + "tokens_seen": 2097348608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680789600385171, + "loss": 1.2932, + "theoretical_loss": 3.445339254823298, + "tokens_seen": 2097479680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036803883806772584, + "loss": 1.2167, + "theoretical_loss": 3.4453216187393934, + "tokens_seen": 2097610752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679987160969347, + "loss": 1.3238, + "theoretical_loss": 3.445303984066011, + "tokens_seen": 2097741824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036795859412614345, + "loss": 1.2552, + "theoretical_loss": 3.4452863508029488, + "tokens_seen": 2097872896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679184721553523, + "loss": 1.3191, + "theoretical_loss": 3.4452687189500066, + "tokens_seen": 2098003968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036787835018456107, + "loss": 1.3304, + "theoretical_loss": 3.445251088506984, + "tokens_seen": 2098135040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678382282137699, + "loss": 1.2811, + "theoretical_loss": 3.4452334594736786, + "tokens_seen": 2098266112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677981062429787, + "loss": 1.413, + "theoretical_loss": 3.4452158318498913, + "tokens_seen": 2098397184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677579842721875, + "loss": 1.2466, + "theoretical_loss": 3.445198205635421, + "tokens_seen": 2098528256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036771786230139624, + "loss": 1.305, + "theoretical_loss": 3.4451805808300664, + "tokens_seen": 2098659328 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036767774033060504, + "loss": 1.3215, + "theoretical_loss": 3.4451629574336273, + "tokens_seen": 2098790400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036763761835981385, + "loss": 1.3392, + "theoretical_loss": 3.445145335445903, + "tokens_seen": 2098921472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675974963890226, + "loss": 1.2679, + "theoretical_loss": 3.4451277148666932, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036755737441823146, + "loss": 1.3152, + "theoretical_loss": 3.4451100956957967, + "tokens_seen": 2099183616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675172524474402, + "loss": 1.3097, + "theoretical_loss": 3.445092477933014, + "tokens_seen": 2099314688 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367477130476649, + "loss": 1.3376, + "theoretical_loss": 3.4450748615781435, + "tokens_seen": 2099445760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036743700850585783, + "loss": 1.279, + "theoretical_loss": 3.4450572466309852, + "tokens_seen": 2099576832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673968865350666, + "loss": 1.2749, + "theoretical_loss": 3.4450396330913398, + "tokens_seen": 2099707904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036735676456427544, + "loss": 1.341, + "theoretical_loss": 3.4450220209590054, + "tokens_seen": 2099838976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673166425934842, + "loss": 1.2645, + "theoretical_loss": 3.445004410233783, + "tokens_seen": 2099970048 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367276520622693, + "loss": 1.3017, + "theoretical_loss": 3.4449868009154714, + "tokens_seen": 2100101120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672363986519018, + "loss": 1.269, + "theoretical_loss": 3.444969193003871, + "tokens_seen": 2100232192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671962766811106, + "loss": 1.2827, + "theoretical_loss": 3.4449515864987816, + "tokens_seen": 2100363264 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036715615471031936, + "loss": 1.3671, + "theoretical_loss": 3.4449339814000024, + "tokens_seen": 2100494336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036711603273952817, + "loss": 1.3252, + "theoretical_loss": 3.4449163777073344, + "tokens_seen": 2100625408 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367075910768737, + "loss": 1.2861, + "theoretical_loss": 3.444898775420577, + "tokens_seen": 2100756480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036703578879794573, + "loss": 1.2897, + "theoretical_loss": 3.44488117453953, + "tokens_seen": 2100887552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669956668271546, + "loss": 1.2964, + "theoretical_loss": 3.4448635750639935, + "tokens_seen": 2101018624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036695554485636334, + "loss": 1.2652, + "theoretical_loss": 3.4448459769937685, + "tokens_seen": 2101149696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036691542288557215, + "loss": 1.2659, + "theoretical_loss": 3.4448283803286537, + "tokens_seen": 2101280768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036687530091478095, + "loss": 1.2983, + "theoretical_loss": 3.4448107850684506, + "tokens_seen": 2101411840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668351789439897, + "loss": 1.2778, + "theoretical_loss": 3.444793191212958, + "tokens_seen": 2101542912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667950569731985, + "loss": 1.3261, + "theoretical_loss": 3.4447755987619777, + "tokens_seen": 2101673984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667549350024073, + "loss": 1.3641, + "theoretical_loss": 3.4447580077153095, + "tokens_seen": 2101805056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667148130316161, + "loss": 1.2999, + "theoretical_loss": 3.4447404180727528, + "tokens_seen": 2101936128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036667469106082493, + "loss": 1.2276, + "theoretical_loss": 3.4447228298341086, + "tokens_seen": 2102067200 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9941034913063049, + "objective/train/avg_token_score": 0.005896490067243576, + "objective/train/docs_used": 765966, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2963688373565674, + "objective/train/original_loss": 1.3047157526016235, + "objective/train/theoretical_loss": 3.444705242999178, + "objective/train/tokens_used": 472657376, + "objective/train/unlikelihood_loss": 0.01829429529607296, + "theoretical_loss": 3.444705242999178, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036663456909003374, + "loss": 1.2642, + "theoretical_loss": 3.444705242999178, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665944471192425, + "loss": 1.2598, + "theoretical_loss": 3.4446876575677603, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665543251484513, + "loss": 1.2949, + "theoretical_loss": 3.444670073539657, + "tokens_seen": 2102460416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665142031776601, + "loss": 1.2738, + "theoretical_loss": 3.4446524909146676, + "tokens_seen": 2102591488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036647408120686885, + "loss": 1.2397, + "theoretical_loss": 3.444634909692594, + "tokens_seen": 2102722560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664339592360777, + "loss": 1.2977, + "theoretical_loss": 3.4446173298732354, + "tokens_seen": 2102853632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036639383726528647, + "loss": 1.3189, + "theoretical_loss": 3.4445997514563937, + "tokens_seen": 2102984704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663537152944953, + "loss": 1.3264, + "theoretical_loss": 3.444582174441869, + "tokens_seen": 2103115776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663135933237041, + "loss": 1.2588, + "theoretical_loss": 3.444564598829462, + "tokens_seen": 2103246848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662734713529129, + "loss": 1.368, + "theoretical_loss": 3.4445470246189736, + "tokens_seen": 2103377920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036623334938212164, + "loss": 1.2515, + "theoretical_loss": 3.4445294518102045, + "tokens_seen": 2103508992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036619322741133044, + "loss": 1.2889, + "theoretical_loss": 3.4445118804029558, + "tokens_seen": 2103640064 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036615310544053925, + "loss": 1.3503, + "theoretical_loss": 3.4444943103970282, + "tokens_seen": 2103771136 + }, + { + "epoch": 0.28, + "learning_rate": 0.000366112983469748, + "loss": 1.2887, + "theoretical_loss": 3.444476741792223, + "tokens_seen": 2103902208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036607286149895686, + "loss": 1.2911, + "theoretical_loss": 3.4444591745883404, + "tokens_seen": 2104033280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660327395281656, + "loss": 1.2888, + "theoretical_loss": 3.4444416087851826, + "tokens_seen": 2104164352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659926175573745, + "loss": 1.3367, + "theoretical_loss": 3.44442404438255, + "tokens_seen": 2104295424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036595249558658323, + "loss": 1.2719, + "theoretical_loss": 3.444406481380243, + "tokens_seen": 2104426496 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365912373615792, + "loss": 1.2033, + "theoretical_loss": 3.444388919778064, + "tokens_seen": 2104557568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036587225164500084, + "loss": 1.3018, + "theoretical_loss": 3.4443713595758134, + "tokens_seen": 2104688640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658321296742096, + "loss": 1.2807, + "theoretical_loss": 3.444353800773293, + "tokens_seen": 2104819712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657920077034184, + "loss": 1.3109, + "theoretical_loss": 3.4443362433703038, + "tokens_seen": 2104950784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657518857326272, + "loss": 1.2714, + "theoretical_loss": 3.444318687366647, + "tokens_seen": 2105081856 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365711763761836, + "loss": 1.27, + "theoretical_loss": 3.4443011327621242, + "tokens_seen": 2105212928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036567164179104476, + "loss": 1.2913, + "theoretical_loss": 3.4442835795565365, + "tokens_seen": 2105344000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563151982025357, + "loss": 1.2735, + "theoretical_loss": 3.444266027749685, + "tokens_seen": 2105475072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655913978494624, + "loss": 1.3226, + "theoretical_loss": 3.4442484773413726, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036555127587867113, + "loss": 1.2867, + "theoretical_loss": 3.4442309283313994, + "tokens_seen": 2105737216 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036551115390788, + "loss": 1.2745, + "theoretical_loss": 3.4442133807195674, + "tokens_seen": 2105868288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036547103193708874, + "loss": 1.3187, + "theoretical_loss": 3.4441958345056785, + "tokens_seen": 2105999360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654309099662976, + "loss": 1.2911, + "theoretical_loss": 3.4441782896895337, + "tokens_seen": 2106130432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036539078799550635, + "loss": 1.322, + "theoretical_loss": 3.444160746270935, + "tokens_seen": 2106261504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653506660247151, + "loss": 1.2632, + "theoretical_loss": 3.4441432042496842, + "tokens_seen": 2106392576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036531054405392397, + "loss": 1.3197, + "theoretical_loss": 3.4441256636255835, + "tokens_seen": 2106523648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652704220831327, + "loss": 1.3215, + "theoretical_loss": 3.4441081243984337, + "tokens_seen": 2106654720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652303001123415, + "loss": 1.252, + "theoretical_loss": 3.444090586568038, + "tokens_seen": 2106785792 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036519017814155033, + "loss": 1.2802, + "theoretical_loss": 3.4440730501341967, + "tokens_seen": 2106916864 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036515005617075914, + "loss": 1.2394, + "theoretical_loss": 3.4440555150967125, + "tokens_seen": 2107047936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651099341999679, + "loss": 1.2344, + "theoretical_loss": 3.444037981455387, + "tokens_seen": 2107179008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650698122291767, + "loss": 1.2025, + "theoretical_loss": 3.444020449210023, + "tokens_seen": 2107310080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650296902583855, + "loss": 1.2342, + "theoretical_loss": 3.444002918360422, + "tokens_seen": 2107441152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036498956828759425, + "loss": 1.3344, + "theoretical_loss": 3.4439853889063863, + "tokens_seen": 2107572224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649494463168031, + "loss": 1.302, + "theoretical_loss": 3.4439678608477178, + "tokens_seen": 2107703296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036490932434601187, + "loss": 1.2853, + "theoretical_loss": 3.4439503341842186, + "tokens_seen": 2107834368 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036486920237522067, + "loss": 1.2684, + "theoretical_loss": 3.443932808915691, + "tokens_seen": 2107965440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648290804044295, + "loss": 1.2705, + "theoretical_loss": 3.4439152850419377, + "tokens_seen": 2108096512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036478895843363823, + "loss": 1.2865, + "theoretical_loss": 3.4438977625627603, + "tokens_seen": 2108227584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647488364628471, + "loss": 1.3121, + "theoretical_loss": 3.4438802414779612, + "tokens_seen": 2108358656 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036470871449205584, + "loss": 1.2527, + "theoretical_loss": 3.4438627217873434, + "tokens_seen": 2108489728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036466859252126465, + "loss": 1.3525, + "theoretical_loss": 3.4438452034907083, + "tokens_seen": 2108620800 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9933674931526184, + "objective/train/avg_token_score": 0.0066325124353170395, + "objective/train/docs_used": 768317, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1591057777404785, + "objective/train/original_loss": 1.1681549549102783, + "objective/train/theoretical_loss": 3.443827686587859, + "objective/train/tokens_used": 479210976, + "objective/train/unlikelihood_loss": 0.020595364272594452, + "theoretical_loss": 3.443827686587859, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036462847055047346, + "loss": 1.247, + "theoretical_loss": 3.443827686587859, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036458834857968226, + "loss": 1.2844, + "theoretical_loss": 3.4438101710785984, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364548226608891, + "loss": 1.3104, + "theoretical_loss": 3.4437926569627284, + "tokens_seen": 2109014016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645081046380999, + "loss": 1.3215, + "theoretical_loss": 3.4437751442400515, + "tokens_seen": 2109145088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036446798266730863, + "loss": 1.2618, + "theoretical_loss": 3.443757632910371, + "tokens_seen": 2109276160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644278606965174, + "loss": 1.3415, + "theoretical_loss": 3.4437401229734883, + "tokens_seen": 2109407232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036438773872572624, + "loss": 1.3206, + "theoretical_loss": 3.4437226144292072, + "tokens_seen": 2109538304 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364347616754935, + "loss": 1.2614, + "theoretical_loss": 3.4437051072773297, + "tokens_seen": 2109669376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643074947841438, + "loss": 1.2452, + "theoretical_loss": 3.4436876015176594, + "tokens_seen": 2109800448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642673728133526, + "loss": 1.2689, + "theoretical_loss": 3.4436700971499983, + "tokens_seen": 2109931520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642272508425614, + "loss": 1.2221, + "theoretical_loss": 3.44365259417415, + "tokens_seen": 2110062592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036418712887177016, + "loss": 1.2952, + "theoretical_loss": 3.4436350925899166, + "tokens_seen": 2110193664 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036414700690097897, + "loss": 1.307, + "theoretical_loss": 3.4436175923971017, + "tokens_seen": 2110324736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641068849301878, + "loss": 1.267, + "theoretical_loss": 3.443600093595508, + "tokens_seen": 2110455808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640667629593966, + "loss": 1.3024, + "theoretical_loss": 3.443582596184938, + "tokens_seen": 2110586880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640266409886054, + "loss": 1.2305, + "theoretical_loss": 3.4435651001651957, + "tokens_seen": 2110717952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036398651901781414, + "loss": 1.2708, + "theoretical_loss": 3.443547605536083, + "tokens_seen": 2110849024 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363946397047023, + "loss": 1.2315, + "theoretical_loss": 3.4435301122974047, + "tokens_seen": 2110980096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036390627507623175, + "loss": 1.1538, + "theoretical_loss": 3.4435126204489626, + "tokens_seen": 2111111168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638661531054405, + "loss": 1.3228, + "theoretical_loss": 3.44349512999056, + "tokens_seen": 2111242240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036382603113464937, + "loss": 1.2625, + "theoretical_loss": 3.443477640922001, + "tokens_seen": 2111373312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637859091638581, + "loss": 1.261, + "theoretical_loss": 3.4434601532430884, + "tokens_seen": 2111504384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637457871930669, + "loss": 1.2001, + "theoretical_loss": 3.443442666953625, + "tokens_seen": 2111635456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036370566522227573, + "loss": 1.2866, + "theoretical_loss": 3.4434251820534145, + "tokens_seen": 2111766528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036366554325148454, + "loss": 1.2255, + "theoretical_loss": 3.443407698542261, + "tokens_seen": 2111897600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636254212806933, + "loss": 1.3297, + "theoretical_loss": 3.4433902164199672, + "tokens_seen": 2112028672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635852993099021, + "loss": 1.2915, + "theoretical_loss": 3.443372735686337, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635451773391109, + "loss": 1.2768, + "theoretical_loss": 3.4433552563411736, + "tokens_seen": 2112290816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635050553683197, + "loss": 1.2528, + "theoretical_loss": 3.443337778384281, + "tokens_seen": 2112421888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634649333975285, + "loss": 1.2629, + "theoretical_loss": 3.4433203018154623, + "tokens_seen": 2112552960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036342481142673727, + "loss": 1.2807, + "theoretical_loss": 3.443302826634521, + "tokens_seen": 2112684032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633846894559461, + "loss": 1.3168, + "theoretical_loss": 3.4432853528412615, + "tokens_seen": 2112815104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633445674851549, + "loss": 1.3545, + "theoretical_loss": 3.443267880435487, + "tokens_seen": 2112946176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036330444551436363, + "loss": 1.3178, + "theoretical_loss": 3.4432504094170016, + "tokens_seen": 2113077248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632643235435725, + "loss": 1.2057, + "theoretical_loss": 3.443232939785609, + "tokens_seen": 2113208320 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036322420157278124, + "loss": 1.2307, + "theoretical_loss": 3.443215471541113, + "tokens_seen": 2113339392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036318407960199005, + "loss": 1.321, + "theoretical_loss": 3.4431980046833175, + "tokens_seen": 2113470464 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036314395763119886, + "loss": 1.2871, + "theoretical_loss": 3.443180539212026, + "tokens_seen": 2113601536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036310383566040766, + "loss": 1.2361, + "theoretical_loss": 3.4431630751270434, + "tokens_seen": 2113732608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630637136896164, + "loss": 1.2232, + "theoretical_loss": 3.4431456124281734, + "tokens_seen": 2113863680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630235917188253, + "loss": 1.3833, + "theoretical_loss": 3.443128151115219, + "tokens_seen": 2113994752 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362983469748034, + "loss": 1.3141, + "theoretical_loss": 3.443110691187986, + "tokens_seen": 2114125824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629433477772428, + "loss": 1.2979, + "theoretical_loss": 3.4430932326462775, + "tokens_seen": 2114256896 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036290322580645164, + "loss": 1.3108, + "theoretical_loss": 3.4430757754898975, + "tokens_seen": 2114387968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628631038356604, + "loss": 1.2854, + "theoretical_loss": 3.4430583197186504, + "tokens_seen": 2114519040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036282298186486925, + "loss": 1.3201, + "theoretical_loss": 3.443040865332341, + "tokens_seen": 2114650112 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362782859894078, + "loss": 1.2386, + "theoretical_loss": 3.4430234123307732, + "tokens_seen": 2114781184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627427379232868, + "loss": 1.2785, + "theoretical_loss": 3.4430059607137515, + "tokens_seen": 2114912256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627026159524956, + "loss": 1.2472, + "theoretical_loss": 3.4429885104810793, + "tokens_seen": 2115043328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036266249398170437, + "loss": 1.2151, + "theoretical_loss": 3.442971061632562, + "tokens_seen": 2115174400 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9916481971740723, + "objective/train/avg_token_score": 0.008351797237992287, + "objective/train/docs_used": 770743, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2126520872116089, + "objective/train/original_loss": 1.223576545715332, + "objective/train/theoretical_loss": 3.4429536141680046, + "objective/train/tokens_used": 485764576, + "objective/train/unlikelihood_loss": 0.02738562598824501, + "theoretical_loss": 3.4429536141680046, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626223720109132, + "loss": 1.2803, + "theoretical_loss": 3.4429536141680046, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362582250040122, + "loss": 1.3065, + "theoretical_loss": 3.4429361680872104, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625421280693308, + "loss": 1.2767, + "theoretical_loss": 3.442918723389984, + "tokens_seen": 2115567616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036250200609853954, + "loss": 1.2326, + "theoretical_loss": 3.4429012800761303, + "tokens_seen": 2115698688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624618841277484, + "loss": 1.3012, + "theoretical_loss": 3.4428838381454545, + "tokens_seen": 2115829760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036242176215695715, + "loss": 1.2648, + "theoretical_loss": 3.4428663975977596, + "tokens_seen": 2115960832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623816401861659, + "loss": 1.2957, + "theoretical_loss": 3.4428489584328523, + "tokens_seen": 2116091904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036234151821537477, + "loss": 1.2955, + "theoretical_loss": 3.442831520650536, + "tokens_seen": 2116222976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623013962445835, + "loss": 1.2728, + "theoretical_loss": 3.442814084250616, + "tokens_seen": 2116354048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622612742737924, + "loss": 1.3417, + "theoretical_loss": 3.4427966492328967, + "tokens_seen": 2116485120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036222115230300113, + "loss": 1.2323, + "theoretical_loss": 3.4427792155971835, + "tokens_seen": 2116616192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036218103033220994, + "loss": 1.324, + "theoretical_loss": 3.442761783343281, + "tokens_seen": 2116747264 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214090836141874, + "loss": 1.3281, + "theoretical_loss": 3.4427443524709935, + "tokens_seen": 2116878336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621007863906275, + "loss": 1.2828, + "theoretical_loss": 3.442726922980127, + "tokens_seen": 2117009408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620606644198363, + "loss": 1.2509, + "theoretical_loss": 3.4427094948704857, + "tokens_seen": 2117140480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620205424490451, + "loss": 1.2948, + "theoretical_loss": 3.4426920681418753, + "tokens_seen": 2117271552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619804204782539, + "loss": 1.2421, + "theoretical_loss": 3.4426746427941004, + "tokens_seen": 2117402624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036194029850746267, + "loss": 1.2942, + "theoretical_loss": 3.4426572188269664, + "tokens_seen": 2117533696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619001765366715, + "loss": 1.2348, + "theoretical_loss": 3.442639796240278, + "tokens_seen": 2117664768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618600545658803, + "loss": 1.2661, + "theoretical_loss": 3.4426223750338414, + "tokens_seen": 2117795840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036181993259508903, + "loss": 1.2482, + "theoretical_loss": 3.442604955207461, + "tokens_seen": 2117926912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617798106242979, + "loss": 1.2468, + "theoretical_loss": 3.442587536760942, + "tokens_seen": 2118057984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036173968865350664, + "loss": 1.3191, + "theoretical_loss": 3.44257011969409, + "tokens_seen": 2118189056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036169956668271545, + "loss": 1.297, + "theoretical_loss": 3.44255270400671, + "tokens_seen": 2118320128 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036165944471192426, + "loss": 1.2934, + "theoretical_loss": 3.4425352896986086, + "tokens_seen": 2118451200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036161932274113306, + "loss": 1.3322, + "theoretical_loss": 3.44251787676959, + "tokens_seen": 2118582272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036157920077034187, + "loss": 1.3178, + "theoretical_loss": 3.4425004652194593, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615390787995507, + "loss": 1.2842, + "theoretical_loss": 3.4424830550480237, + "tokens_seen": 2118844416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614989568287594, + "loss": 1.2538, + "theoretical_loss": 3.4424656462550876, + "tokens_seen": 2118975488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036145883485796823, + "loss": 1.3206, + "theoretical_loss": 3.4424482388404565, + "tokens_seen": 2119106560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036141871288717704, + "loss": 1.293, + "theoretical_loss": 3.442430832803937, + "tokens_seen": 2119237632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613785909163858, + "loss": 1.3535, + "theoretical_loss": 3.4424134281453336, + "tokens_seen": 2119368704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036133846894559465, + "loss": 1.2907, + "theoretical_loss": 3.4423960248644523, + "tokens_seen": 2119499776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612983469748034, + "loss": 1.3447, + "theoretical_loss": 3.4423786229610998, + "tokens_seen": 2119630848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612582250040122, + "loss": 1.287, + "theoretical_loss": 3.442361222435081, + "tokens_seen": 2119761920 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361218103033221, + "loss": 1.2972, + "theoretical_loss": 3.442343823286201, + "tokens_seen": 2119892992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036117798106242977, + "loss": 1.2874, + "theoretical_loss": 3.442326425514268, + "tokens_seen": 2120024064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611378590916386, + "loss": 1.3428, + "theoretical_loss": 3.4423090291190857, + "tokens_seen": 2120155136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610977371208474, + "loss": 1.3327, + "theoretical_loss": 3.442291634100461, + "tokens_seen": 2120286208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610576151500562, + "loss": 1.2808, + "theoretical_loss": 3.4422742404581994, + "tokens_seen": 2120417280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036101749317926494, + "loss": 1.2457, + "theoretical_loss": 3.442256848192107, + "tokens_seen": 2120548352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609773712084738, + "loss": 1.2221, + "theoretical_loss": 3.4422394573019908, + "tokens_seen": 2120679424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036093724923768255, + "loss": 1.2807, + "theoretical_loss": 3.4422220677876565, + "tokens_seen": 2120810496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036089712726689136, + "loss": 1.4012, + "theoretical_loss": 3.4422046796489094, + "tokens_seen": 2120941568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036085700529610017, + "loss": 1.225, + "theoretical_loss": 3.4421872928855564, + "tokens_seen": 2121072640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608168833253089, + "loss": 1.2038, + "theoretical_loss": 3.4421699074974033, + "tokens_seen": 2121203712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607767613545178, + "loss": 1.2523, + "theoretical_loss": 3.442152523484257, + "tokens_seen": 2121334784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036073663938372653, + "loss": 1.2921, + "theoretical_loss": 3.4421351408459238, + "tokens_seen": 2121465856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036069651741293534, + "loss": 1.2815, + "theoretical_loss": 3.4421177595822092, + "tokens_seen": 2121596928 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036065639544214414, + "loss": 1.2552, + "theoretical_loss": 3.4421003796929197, + "tokens_seen": 2121728000 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9931816458702087, + "objective/train/avg_token_score": 0.006818348541855812, + "objective/train/docs_used": 773131, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2597379684448242, + "objective/train/original_loss": 1.2690614461898804, + "objective/train/theoretical_loss": 3.442083001177863, + "objective/train/tokens_used": 492318176, + "objective/train/unlikelihood_loss": 0.01890190690755844, + "theoretical_loss": 3.442083001177863, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606162734713529, + "loss": 1.2293, + "theoretical_loss": 3.442083001177863, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605761515005617, + "loss": 1.3174, + "theoretical_loss": 3.442065624036844, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605360295297705, + "loss": 1.3313, + "theoretical_loss": 3.4420482482696704, + "tokens_seen": 2122121216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604959075589793, + "loss": 1.2749, + "theoretical_loss": 3.4420308738761474, + "tokens_seen": 2122252288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036045578558818807, + "loss": 1.2993, + "theoretical_loss": 3.442013500856083, + "tokens_seen": 2122383360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604156636173969, + "loss": 1.3397, + "theoretical_loss": 3.441996129209283, + "tokens_seen": 2122514432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603755416466057, + "loss": 1.3318, + "theoretical_loss": 3.441978758935554, + "tokens_seen": 2122645504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603354196758145, + "loss": 1.3214, + "theoretical_loss": 3.4419613900347032, + "tokens_seen": 2122776576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602952977050233, + "loss": 1.355, + "theoretical_loss": 3.4419440225065374, + "tokens_seen": 2122907648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036025517573423204, + "loss": 1.3455, + "theoretical_loss": 3.441926656350862, + "tokens_seen": 2123038720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602150537634409, + "loss": 1.2494, + "theoretical_loss": 3.441909291567486, + "tokens_seen": 2123169792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036017493179264966, + "loss": 1.3077, + "theoretical_loss": 3.441891928156215, + "tokens_seen": 2123300864 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036013480982185846, + "loss": 1.3152, + "theoretical_loss": 3.441874566116855, + "tokens_seen": 2123431936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036009468785106727, + "loss": 1.314, + "theoretical_loss": 3.4418572054492147, + "tokens_seen": 2123563008 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360054565880276, + "loss": 1.2959, + "theoretical_loss": 3.4418398461531003, + "tokens_seen": 2123694080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600144439094848, + "loss": 1.3045, + "theoretical_loss": 3.441822488228319, + "tokens_seen": 2123825152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035997432193869363, + "loss": 1.2744, + "theoretical_loss": 3.441805131674677, + "tokens_seen": 2123956224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035993419996790244, + "loss": 1.3051, + "theoretical_loss": 3.4417877764919824, + "tokens_seen": 2124087296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598940779971112, + "loss": 1.2853, + "theoretical_loss": 3.441770422680042, + "tokens_seen": 2124218368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035985395602632005, + "loss": 1.2997, + "theoretical_loss": 3.4417530702386627, + "tokens_seen": 2124349440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598138340555288, + "loss": 1.4105, + "theoretical_loss": 3.441735719167652, + "tokens_seen": 2124480512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597737120847376, + "loss": 1.2411, + "theoretical_loss": 3.441718369466817, + "tokens_seen": 2124611584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597335901139464, + "loss": 1.2726, + "theoretical_loss": 3.4417010211359655, + "tokens_seen": 2124742656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035969346814315517, + "loss": 1.2715, + "theoretical_loss": 3.441683674174904, + "tokens_seen": 2124873728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965334617236403, + "loss": 1.3027, + "theoretical_loss": 3.44166632858344, + "tokens_seen": 2125004800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596132242015728, + "loss": 1.3364, + "theoretical_loss": 3.4416489843613816, + "tokens_seen": 2125135872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595731022307816, + "loss": 1.3133, + "theoretical_loss": 3.441631641508536, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595329802599904, + "loss": 1.3846, + "theoretical_loss": 3.44161430002471, + "tokens_seen": 2125398016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594928582891992, + "loss": 1.2315, + "theoretical_loss": 3.4415969599097114, + "tokens_seen": 2125529088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035945273631840795, + "loss": 1.2838, + "theoretical_loss": 3.4415796211633483, + "tokens_seen": 2125660160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035941261434761676, + "loss": 1.2863, + "theoretical_loss": 3.441562283785428, + "tokens_seen": 2125791232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035937249237682556, + "loss": 1.3698, + "theoretical_loss": 3.4415449477757574, + "tokens_seen": 2125922304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593323704060343, + "loss": 1.2965, + "theoretical_loss": 3.441527613134145, + "tokens_seen": 2126053376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592922484352432, + "loss": 1.2802, + "theoretical_loss": 3.441510279860399, + "tokens_seen": 2126184448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035925212646445193, + "loss": 1.302, + "theoretical_loss": 3.441492947954326, + "tokens_seen": 2126315520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035921200449366074, + "loss": 1.2527, + "theoretical_loss": 3.4414756174157337, + "tokens_seen": 2126446592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035917188252286954, + "loss": 1.3316, + "theoretical_loss": 3.441458288244431, + "tokens_seen": 2126577664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591317605520783, + "loss": 1.2752, + "theoretical_loss": 3.441440960440225, + "tokens_seen": 2126708736 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035909163858128715, + "loss": 1.3283, + "theoretical_loss": 3.4414236340029243, + "tokens_seen": 2126839808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590515166104959, + "loss": 1.2916, + "theoretical_loss": 3.4414063089323363, + "tokens_seen": 2126970880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590113946397047, + "loss": 1.2247, + "theoretical_loss": 3.4413889852282686, + "tokens_seen": 2127101952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589712726689135, + "loss": 1.3734, + "theoretical_loss": 3.44137166289053, + "tokens_seen": 2127233024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589311506981223, + "loss": 1.3703, + "theoretical_loss": 3.4413543419189283, + "tokens_seen": 2127364096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588910287273311, + "loss": 1.2763, + "theoretical_loss": 3.441337022313272, + "tokens_seen": 2127495168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588509067565399, + "loss": 1.2849, + "theoretical_loss": 3.441319704073368, + "tokens_seen": 2127626240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588107847857487, + "loss": 1.3551, + "theoretical_loss": 3.4413023871990256, + "tokens_seen": 2127757312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035877066281495744, + "loss": 1.2005, + "theoretical_loss": 3.4412850716900527, + "tokens_seen": 2127888384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587305408441663, + "loss": 1.2449, + "theoretical_loss": 3.4412677575462576, + "tokens_seen": 2128019456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035869041887337506, + "loss": 1.266, + "theoretical_loss": 3.441250444767448, + "tokens_seen": 2128150528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035865029690258386, + "loss": 1.2885, + "theoretical_loss": 3.441233133353433, + "tokens_seen": 2128281600 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9931174516677856, + "objective/train/avg_token_score": 0.006882534362375736, + "objective/train/docs_used": 775219, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2727327346801758, + "objective/train/original_loss": 1.2806460857391357, + "objective/train/theoretical_loss": 3.441215823304021, + "objective/train/tokens_used": 498871776, + "objective/train/unlikelihood_loss": 0.022802915424108505, + "theoretical_loss": 3.441215823304021, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035861017493179267, + "loss": 1.2507, + "theoretical_loss": 3.441215823304021, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585700529610014, + "loss": 1.3561, + "theoretical_loss": 3.4411985146190203, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585299309902102, + "loss": 1.2887, + "theoretical_loss": 3.4411812072982384, + "tokens_seen": 2128674816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035848980901941903, + "loss": 1.2831, + "theoretical_loss": 3.441163901341485, + "tokens_seen": 2128805888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035844968704862784, + "loss": 1.2646, + "theoretical_loss": 3.441146596748568, + "tokens_seen": 2128936960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035840956507783664, + "loss": 1.1992, + "theoretical_loss": 3.4411292935192965, + "tokens_seen": 2129068032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035836944310704545, + "loss": 1.2344, + "theoretical_loss": 3.4411119916534787, + "tokens_seen": 2129199104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583293211362542, + "loss": 1.3597, + "theoretical_loss": 3.441094691150923, + "tokens_seen": 2129330176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035828919916546306, + "loss": 1.336, + "theoretical_loss": 3.4410773920114384, + "tokens_seen": 2129461248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582490771946718, + "loss": 1.3181, + "theoretical_loss": 3.4410600942348335, + "tokens_seen": 2129592320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035820895522388057, + "loss": 1.2699, + "theoretical_loss": 3.4410427978209173, + "tokens_seen": 2129723392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035816883325308943, + "loss": 1.3016, + "theoretical_loss": 3.4410255027694987, + "tokens_seen": 2129854464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581287112822982, + "loss": 1.3848, + "theoretical_loss": 3.441008209080386, + "tokens_seen": 2129985536 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358088589311507, + "loss": 1.4149, + "theoretical_loss": 3.440990916753388, + "tokens_seen": 2130116608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580484673407158, + "loss": 1.2788, + "theoretical_loss": 3.440973625788314, + "tokens_seen": 2130247680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580083453699246, + "loss": 1.2531, + "theoretical_loss": 3.440956336184974, + "tokens_seen": 2130378752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035796822339913335, + "loss": 1.365, + "theoretical_loss": 3.440939047943175, + "tokens_seen": 2130509824 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035792810142834216, + "loss": 1.3276, + "theoretical_loss": 3.4409217610627265, + "tokens_seen": 2130640896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035788797945755096, + "loss": 1.3295, + "theoretical_loss": 3.4409044755434386, + "tokens_seen": 2130771968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578478574867597, + "loss": 1.2706, + "theoretical_loss": 3.4408871913851193, + "tokens_seen": 2130903040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578077355159686, + "loss": 1.2355, + "theoretical_loss": 3.4408699085875787, + "tokens_seen": 2131034112 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035776761354517733, + "loss": 1.2819, + "theoretical_loss": 3.4408526271506252, + "tokens_seen": 2131165184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577274915743862, + "loss": 1.2462, + "theoretical_loss": 3.440835347074068, + "tokens_seen": 2131296256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035768736960359494, + "loss": 1.2119, + "theoretical_loss": 3.4408180683577174, + "tokens_seen": 2131427328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576472476328037, + "loss": 1.2753, + "theoretical_loss": 3.4408007910013816, + "tokens_seen": 2131558400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035760712566201255, + "loss": 1.374, + "theoretical_loss": 3.4407835150048705, + "tokens_seen": 2131689472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575670036912213, + "loss": 1.3089, + "theoretical_loss": 3.440766240367993, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575268817204301, + "loss": 1.2341, + "theoretical_loss": 3.440748967090559, + "tokens_seen": 2131951616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574867597496389, + "loss": 1.2945, + "theoretical_loss": 3.440731695172377, + "tokens_seen": 2132082688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574466377788477, + "loss": 1.2736, + "theoretical_loss": 3.440714424613258, + "tokens_seen": 2132213760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574065158080565, + "loss": 1.3436, + "theoretical_loss": 3.4406971554130106, + "tokens_seen": 2132344832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573663938372653, + "loss": 1.411, + "theoretical_loss": 3.440679887571444, + "tokens_seen": 2132475904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573262718664741, + "loss": 1.3482, + "theoretical_loss": 3.440662621088369, + "tokens_seen": 2132606976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035728614989568284, + "loss": 1.3239, + "theoretical_loss": 3.440645355963594, + "tokens_seen": 2132738048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572460279248917, + "loss": 1.332, + "theoretical_loss": 3.44062809219693, + "tokens_seen": 2132869120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035720590595410045, + "loss": 1.2686, + "theoretical_loss": 3.440610829788185, + "tokens_seen": 2133000192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571657839833093, + "loss": 1.237, + "theoretical_loss": 3.44059356873717, + "tokens_seen": 2133131264 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035712566201251807, + "loss": 1.3277, + "theoretical_loss": 3.4405763090436947, + "tokens_seen": 2133262336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570855400417268, + "loss": 1.2918, + "theoretical_loss": 3.440559050707569, + "tokens_seen": 2133393408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570454180709357, + "loss": 1.3163, + "theoretical_loss": 3.440541793728602, + "tokens_seen": 2133524480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035700529610014443, + "loss": 1.2707, + "theoretical_loss": 3.4405245381066045, + "tokens_seen": 2133655552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035696517412935324, + "loss": 1.3054, + "theoretical_loss": 3.4405072838413853, + "tokens_seen": 2133786624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035692505215856204, + "loss": 1.3491, + "theoretical_loss": 3.440490030932756, + "tokens_seen": 2133917696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035688493018777085, + "loss": 1.2816, + "theoretical_loss": 3.440472779380525, + "tokens_seen": 2134048768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568448082169796, + "loss": 1.3237, + "theoretical_loss": 3.440455529184504, + "tokens_seen": 2134179840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568046862461884, + "loss": 1.2496, + "theoretical_loss": 3.4404382803445017, + "tokens_seen": 2134310912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567645642753972, + "loss": 1.2324, + "theoretical_loss": 3.440421032860329, + "tokens_seen": 2134441984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035672444230460597, + "loss": 1.3946, + "theoretical_loss": 3.440403786731795, + "tokens_seen": 2134573056 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035668432033381483, + "loss": 1.275, + "theoretical_loss": 3.440386541958712, + "tokens_seen": 2134704128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566441983630236, + "loss": 1.2921, + "theoretical_loss": 3.4403692985408885, + "tokens_seen": 2134835200 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.992087721824646, + "objective/train/avg_token_score": 0.00791224092245102, + "objective/train/docs_used": 777595, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3091490268707275, + "objective/train/original_loss": 1.3199875354766846, + "objective/train/theoretical_loss": 3.440352056478135, + "objective/train/tokens_used": 505425376, + "objective/train/unlikelihood_loss": 0.021838970482349396, + "theoretical_loss": 3.440352056478135, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566040763922324, + "loss": 1.3144, + "theoretical_loss": 3.440352056478135, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565639544214412, + "loss": 1.2577, + "theoretical_loss": 3.4403348157702625, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035652383245065, + "loss": 1.3416, + "theoretical_loss": 3.4403175764170806, + "tokens_seen": 2135228416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564837104798588, + "loss": 1.2615, + "theoretical_loss": 3.4403003384184005, + "tokens_seen": 2135359488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035644358850906756, + "loss": 1.2194, + "theoretical_loss": 3.4402831017740323, + "tokens_seen": 2135490560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035640346653827636, + "loss": 1.3016, + "theoretical_loss": 3.440265866483786, + "tokens_seen": 2135621632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035636334456748517, + "loss": 1.2755, + "theoretical_loss": 3.440248632547473, + "tokens_seen": 2135752704 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356323222596694, + "loss": 1.2282, + "theoretical_loss": 3.4402313999649037, + "tokens_seen": 2135883776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035628310062590273, + "loss": 1.2505, + "theoretical_loss": 3.4402141687358876, + "tokens_seen": 2136014848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562429786551116, + "loss": 1.3901, + "theoretical_loss": 3.4401969388602365, + "tokens_seen": 2136145920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035620285668432034, + "loss": 1.2498, + "theoretical_loss": 3.4401797103377607, + "tokens_seen": 2136276992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561627347135291, + "loss": 1.2551, + "theoretical_loss": 3.440162483168271, + "tokens_seen": 2136408064 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612261274273795, + "loss": 1.2867, + "theoretical_loss": 3.4401452573515785, + "tokens_seen": 2136539136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003560824907719467, + "loss": 1.2863, + "theoretical_loss": 3.440128032887493, + "tokens_seen": 2136670208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560423688011555, + "loss": 1.2645, + "theoretical_loss": 3.4401108097758257, + "tokens_seen": 2136801280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560022468303643, + "loss": 1.2694, + "theoretical_loss": 3.440093588016388, + "tokens_seen": 2136932352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559621248595731, + "loss": 1.3116, + "theoretical_loss": 3.4400763676089907, + "tokens_seen": 2137063424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559220028887819, + "loss": 1.2481, + "theoretical_loss": 3.440059148553444, + "tokens_seen": 2137194496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558818809179907, + "loss": 1.2667, + "theoretical_loss": 3.44004193084956, + "tokens_seen": 2137325568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558417589471995, + "loss": 1.409, + "theoretical_loss": 3.4400247144971488, + "tokens_seen": 2137456640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558016369764083, + "loss": 1.296, + "theoretical_loss": 3.440007499496021, + "tokens_seen": 2137587712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557615150056171, + "loss": 1.2758, + "theoretical_loss": 3.4399902858459894, + "tokens_seen": 2137718784 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035572139303482585, + "loss": 1.3051, + "theoretical_loss": 3.439973073546864, + "tokens_seen": 2137849856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556812710640347, + "loss": 1.3227, + "theoretical_loss": 3.439955862598456, + "tokens_seen": 2137980928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035564114909324347, + "loss": 1.2881, + "theoretical_loss": 3.439938653000577, + "tokens_seen": 2138112000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556010271224522, + "loss": 1.3492, + "theoretical_loss": 3.4399214447530375, + "tokens_seen": 2138243072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555609051516611, + "loss": 1.2588, + "theoretical_loss": 3.4399042378556497, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035552078318086983, + "loss": 1.2882, + "theoretical_loss": 3.4398870323082242, + "tokens_seen": 2138505216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035548066121007864, + "loss": 1.2177, + "theoretical_loss": 3.439869828110573, + "tokens_seen": 2138636288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035544053923928744, + "loss": 1.307, + "theoretical_loss": 3.439852625262507, + "tokens_seen": 2138767360 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035540041726849625, + "loss": 1.1533, + "theoretical_loss": 3.4398354237638378, + "tokens_seen": 2138898432 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355360295297705, + "loss": 1.2513, + "theoretical_loss": 3.4398182236143766, + "tokens_seen": 2139029504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553201733269138, + "loss": 1.3722, + "theoretical_loss": 3.439801024813935, + "tokens_seen": 2139160576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552800513561226, + "loss": 1.2819, + "theoretical_loss": 3.439783827362325, + "tokens_seen": 2139291648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552399293853314, + "loss": 1.3026, + "theoretical_loss": 3.439766631259358, + "tokens_seen": 2139422720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035519980741454023, + "loss": 1.2474, + "theoretical_loss": 3.4397494365048455, + "tokens_seen": 2139553792 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355159685443749, + "loss": 1.2886, + "theoretical_loss": 3.4397322430985993, + "tokens_seen": 2139684864 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035511956347295784, + "loss": 1.2907, + "theoretical_loss": 3.43971505104043, + "tokens_seen": 2139815936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550794415021666, + "loss": 1.2484, + "theoretical_loss": 3.439697860330152, + "tokens_seen": 2139947008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550393195313754, + "loss": 1.2938, + "theoretical_loss": 3.439680670967574, + "tokens_seen": 2140078080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549991975605842, + "loss": 1.2805, + "theoretical_loss": 3.4396634829525095, + "tokens_seen": 2140209152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035495907558979296, + "loss": 1.2356, + "theoretical_loss": 3.4396462962847707, + "tokens_seen": 2140340224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035491895361900176, + "loss": 1.2977, + "theoretical_loss": 3.4396291109641677, + "tokens_seen": 2140471296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035487883164821057, + "loss": 1.2574, + "theoretical_loss": 3.4396119269905143, + "tokens_seen": 2140602368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548387096774194, + "loss": 1.2643, + "theoretical_loss": 3.439594744363622, + "tokens_seen": 2140733440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035479858770662813, + "loss": 1.275, + "theoretical_loss": 3.4395775630833016, + "tokens_seen": 2140864512 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354758465735837, + "loss": 1.2577, + "theoretical_loss": 3.439560383149367, + "tokens_seen": 2140995584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035471834376504574, + "loss": 1.2522, + "theoretical_loss": 3.439543204561629, + "tokens_seen": 2141126656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546782217942545, + "loss": 1.2838, + "theoretical_loss": 3.4395260273199, + "tokens_seen": 2141257728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035463809982346335, + "loss": 1.3173, + "theoretical_loss": 3.439508851423992, + "tokens_seen": 2141388800 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.9912110567092896, + "objective/train/avg_token_score": 0.008788884617388248, + "objective/train/docs_used": 779818, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2776613235473633, + "objective/train/original_loss": 1.2891536951065063, + "objective/train/theoretical_loss": 3.4394916768737174, + "objective/train/tokens_used": 511978976, + "objective/train/unlikelihood_loss": 0.026739470660686493, + "theoretical_loss": 3.4394916768737174, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545979778526721, + "loss": 1.2927, + "theoretical_loss": 3.4394916768737174, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035455785588188097, + "loss": 1.2958, + "theoretical_loss": 3.4394745036688885, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545177339110897, + "loss": 1.2749, + "theoretical_loss": 3.439457331809318, + "tokens_seen": 2141782016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544776119402985, + "loss": 1.2923, + "theoretical_loss": 3.4394401612948173, + "tokens_seen": 2141913088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035443748996950733, + "loss": 1.277, + "theoretical_loss": 3.4394229921251993, + "tokens_seen": 2142044160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543973679987161, + "loss": 1.3211, + "theoretical_loss": 3.439405824300276, + "tokens_seen": 2142175232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543572460279249, + "loss": 1.3188, + "theoretical_loss": 3.43938865781986, + "tokens_seen": 2142306304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543171240571337, + "loss": 1.3591, + "theoretical_loss": 3.4393714926837635, + "tokens_seen": 2142437376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542770020863425, + "loss": 1.2423, + "theoretical_loss": 3.4393543288918, + "tokens_seen": 2142568448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035423688011555125, + "loss": 1.2839, + "theoretical_loss": 3.4393371664437806, + "tokens_seen": 2142699520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541967581447601, + "loss": 1.2855, + "theoretical_loss": 3.4393200053395185, + "tokens_seen": 2142830592 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035415663617396887, + "loss": 1.2664, + "theoretical_loss": 3.439302845578827, + "tokens_seen": 2142961664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541165142031776, + "loss": 1.288, + "theoretical_loss": 3.439285687161518, + "tokens_seen": 2143092736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540763922323865, + "loss": 1.2681, + "theoretical_loss": 3.439268530087404, + "tokens_seen": 2143223808 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035403627026159523, + "loss": 1.361, + "theoretical_loss": 3.439251374356298, + "tokens_seen": 2143354880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539961482908041, + "loss": 1.2128, + "theoretical_loss": 3.4392342199680126, + "tokens_seen": 2143485952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035395602632001284, + "loss": 1.2629, + "theoretical_loss": 3.439217066922361, + "tokens_seen": 2143617024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035391590434922165, + "loss": 1.3394, + "theoretical_loss": 3.439199915219156, + "tokens_seen": 2143748096 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035387578237843046, + "loss": 1.2493, + "theoretical_loss": 3.4391827648582103, + "tokens_seen": 2143879168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538356604076392, + "loss": 1.2407, + "theoretical_loss": 3.439165615839336, + "tokens_seen": 2144010240 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353795538436848, + "loss": 1.3356, + "theoretical_loss": 3.4391484681623474, + "tokens_seen": 2144141312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537554164660568, + "loss": 1.2955, + "theoretical_loss": 3.439131321827057, + "tokens_seen": 2144272384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035371529449526563, + "loss": 1.2103, + "theoretical_loss": 3.4391141768332774, + "tokens_seen": 2144403456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536751725244744, + "loss": 1.3003, + "theoretical_loss": 3.439097033180822, + "tokens_seen": 2144534528 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035363505055368324, + "loss": 1.2474, + "theoretical_loss": 3.4390798908695035, + "tokens_seen": 2144665600 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353594928582892, + "loss": 1.3273, + "theoretical_loss": 3.4390627498991355, + "tokens_seen": 2144796672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535548066121008, + "loss": 1.285, + "theoretical_loss": 3.4390456102695315, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535146846413096, + "loss": 1.2945, + "theoretical_loss": 3.439028471980504, + "tokens_seen": 2145058816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035347456267051836, + "loss": 1.3206, + "theoretical_loss": 3.439011335031866, + "tokens_seen": 2145189888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035343444069972716, + "loss": 1.2464, + "theoretical_loss": 3.4389941994234317, + "tokens_seen": 2145320960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035339431872893597, + "loss": 1.2538, + "theoretical_loss": 3.4389770651550142, + "tokens_seen": 2145452032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533541967581448, + "loss": 1.292, + "theoretical_loss": 3.4389599322264264, + "tokens_seen": 2145583104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533140747873536, + "loss": 1.2962, + "theoretical_loss": 3.4389428006374816, + "tokens_seen": 2145714176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532739528165624, + "loss": 1.3476, + "theoretical_loss": 3.4389256703879942, + "tokens_seen": 2145845248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035323383084577114, + "loss": 1.3699, + "theoretical_loss": 3.438908541477776, + "tokens_seen": 2145976320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035319370887497995, + "loss": 1.2247, + "theoretical_loss": 3.438891413906642, + "tokens_seen": 2146107392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315358690418875, + "loss": 1.3072, + "theoretical_loss": 3.438874287674406, + "tokens_seen": 2146238464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531134649333975, + "loss": 1.258, + "theoretical_loss": 3.43885716278088, + "tokens_seen": 2146369536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035307334296260637, + "loss": 1.31, + "theoretical_loss": 3.4388400392258784, + "tokens_seen": 2146500608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530332209918151, + "loss": 1.2654, + "theoretical_loss": 3.4388229170092153, + "tokens_seen": 2146631680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529930990210239, + "loss": 1.3121, + "theoretical_loss": 3.438805796130704, + "tokens_seen": 2146762752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035295297705023273, + "loss": 1.27, + "theoretical_loss": 3.438788676590158, + "tokens_seen": 2146893824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529128550794415, + "loss": 1.2983, + "theoretical_loss": 3.4387715583873915, + "tokens_seen": 2147024896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528727331086503, + "loss": 1.295, + "theoretical_loss": 3.4387544415222178, + "tokens_seen": 2147155968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528326111378591, + "loss": 1.2243, + "theoretical_loss": 3.4387373259944507, + "tokens_seen": 2147287040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527924891670679, + "loss": 1.2976, + "theoretical_loss": 3.438720211803905, + "tokens_seen": 2147418112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035275236719627665, + "loss": 1.2686, + "theoretical_loss": 3.438703098950394, + "tokens_seen": 2147549184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527122452254855, + "loss": 1.3235, + "theoretical_loss": 3.4386859874337317, + "tokens_seen": 2147680256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035267212325469427, + "loss": 1.2578, + "theoretical_loss": 3.4386688772537317, + "tokens_seen": 2147811328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035263200128390307, + "loss": 1.3094, + "theoretical_loss": 3.4386517684102085, + "tokens_seen": 2147942400 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.9943053126335144, + "objective/train/avg_token_score": 0.0056946841068565845, + "objective/train/docs_used": 782164, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.352531909942627, + "objective/train/original_loss": 1.3615965843200684, + "objective/train/theoretical_loss": 3.4386346609029763, + "objective/train/tokens_used": 518532576, + "objective/train/unlikelihood_loss": 0.013868092559278011, + "theoretical_loss": 3.4386346609029763, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525918793131119, + "loss": 1.3318, + "theoretical_loss": 3.4386346609029763, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035255175734232063, + "loss": 1.3228, + "theoretical_loss": 3.4386175547318487, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525116353715295, + "loss": 1.29, + "theoretical_loss": 3.43860044989664, + "tokens_seen": 2148335616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035247151340073824, + "loss": 1.3066, + "theoretical_loss": 3.4385833463971647, + "tokens_seen": 2148466688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035243139142994705, + "loss": 1.2845, + "theoretical_loss": 3.438566244233236, + "tokens_seen": 2148597760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035239126945915586, + "loss": 1.2901, + "theoretical_loss": 3.4385491434046696, + "tokens_seen": 2148728832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523511474883646, + "loss": 1.2849, + "theoretical_loss": 3.4385320439112794, + "tokens_seen": 2148859904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523110255175734, + "loss": 1.2485, + "theoretical_loss": 3.438514945752879, + "tokens_seen": 2148990976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522709035467822, + "loss": 1.2796, + "theoretical_loss": 3.4384978489292832, + "tokens_seen": 2149122048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035223078157599103, + "loss": 1.2929, + "theoretical_loss": 3.4384807534403063, + "tokens_seen": 2149253120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521906596051998, + "loss": 1.2884, + "theoretical_loss": 3.438463659285763, + "tokens_seen": 2149384192 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035215053763440864, + "loss": 1.2944, + "theoretical_loss": 3.4384465664654673, + "tokens_seen": 2149515264 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521104156636174, + "loss": 1.2937, + "theoretical_loss": 3.4384294749792343, + "tokens_seen": 2149646336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520702936928262, + "loss": 1.2518, + "theoretical_loss": 3.4384123848268784, + "tokens_seen": 2149777408 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352030171722035, + "loss": 1.3017, + "theoretical_loss": 3.4383952960082134, + "tokens_seen": 2149908480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035199004975124376, + "loss": 1.3323, + "theoretical_loss": 3.438378208523055, + "tokens_seen": 2150039552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519499277804526, + "loss": 1.2548, + "theoretical_loss": 3.438361122371217, + "tokens_seen": 2150170624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035190980580966137, + "loss": 1.2558, + "theoretical_loss": 3.4383440375525147, + "tokens_seen": 2150301696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518696838388702, + "loss": 1.2746, + "theoretical_loss": 3.4383269540667625, + "tokens_seen": 2150432768 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351829561868079, + "loss": 1.3249, + "theoretical_loss": 3.4383098719137757, + "tokens_seen": 2150563840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517894398972878, + "loss": 1.2993, + "theoretical_loss": 3.4382927910933683, + "tokens_seen": 2150694912 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035174931792649654, + "loss": 1.3136, + "theoretical_loss": 3.438275711605356, + "tokens_seen": 2150825984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035170919595570535, + "loss": 1.3332, + "theoretical_loss": 3.4382586334495526, + "tokens_seen": 2150957056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035166907398491415, + "loss": 1.3178, + "theoretical_loss": 3.4382415566257745, + "tokens_seen": 2151088128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516289520141229, + "loss": 1.4019, + "theoretical_loss": 3.438224481133835, + "tokens_seen": 2151219200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035158883004333177, + "loss": 1.2531, + "theoretical_loss": 3.43820740697355, + "tokens_seen": 2151350272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515487080725405, + "loss": 1.2531, + "theoretical_loss": 3.4381903341447346, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515085861017493, + "loss": 1.2156, + "theoretical_loss": 3.4381732626472035, + "tokens_seen": 2151612416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035146846413095813, + "loss": 1.3085, + "theoretical_loss": 3.438156192480772, + "tokens_seen": 2151743488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514283421601669, + "loss": 1.3712, + "theoretical_loss": 3.4381391236452554, + "tokens_seen": 2151874560 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035138822018937574, + "loss": 1.2487, + "theoretical_loss": 3.4381220561404686, + "tokens_seen": 2152005632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513480982185845, + "loss": 1.3395, + "theoretical_loss": 3.438104989966227, + "tokens_seen": 2152136704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513079762477933, + "loss": 1.2835, + "theoretical_loss": 3.4380879251223453, + "tokens_seen": 2152267776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512678542770021, + "loss": 1.2576, + "theoretical_loss": 3.4380708616086393, + "tokens_seen": 2152398848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512277323062109, + "loss": 1.2377, + "theoretical_loss": 3.4380537994249245, + "tokens_seen": 2152529920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035118761033541967, + "loss": 1.2799, + "theoretical_loss": 3.438036738571016, + "tokens_seen": 2152660992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035114748836462847, + "loss": 1.3038, + "theoretical_loss": 3.4380196790467292, + "tokens_seen": 2152792064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511073663938373, + "loss": 1.3211, + "theoretical_loss": 3.4380026208518792, + "tokens_seen": 2152923136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035106724442304603, + "loss": 1.2968, + "theoretical_loss": 3.4379855639862824, + "tokens_seen": 2153054208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510271224522549, + "loss": 1.2403, + "theoretical_loss": 3.437968508449753, + "tokens_seen": 2153185280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035098700048146364, + "loss": 1.2948, + "theoretical_loss": 3.437951454242108, + "tokens_seen": 2153316352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035094687851067245, + "loss": 1.2961, + "theoretical_loss": 3.4379344013631616, + "tokens_seen": 2153447424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035090675653988126, + "loss": 1.287, + "theoretical_loss": 3.4379173498127304, + "tokens_seen": 2153578496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035086663456909, + "loss": 1.2638, + "theoretical_loss": 3.43790029959063, + "tokens_seen": 2153709568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035082651259829887, + "loss": 1.3006, + "theoretical_loss": 3.437883250696675, + "tokens_seen": 2153840640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507863906275076, + "loss": 1.3383, + "theoretical_loss": 3.4378662031306826, + "tokens_seen": 2153971712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507462686567164, + "loss": 1.2805, + "theoretical_loss": 3.437849156892468, + "tokens_seen": 2154102784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035070614668592523, + "loss": 1.2747, + "theoretical_loss": 3.4378321119818462, + "tokens_seen": 2154233856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035066602471513404, + "loss": 1.3447, + "theoretical_loss": 3.4378150683986344, + "tokens_seen": 2154364928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506259027443428, + "loss": 1.2866, + "theoretical_loss": 3.4377980261426475, + "tokens_seen": 2154496000 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.9932624697685242, + "objective/train/avg_token_score": 0.006737549789249897, + "objective/train/docs_used": 784476, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3462620973587036, + "objective/train/original_loss": 1.3563402891159058, + "objective/train/theoretical_loss": 3.437780985213702, + "objective/train/tokens_used": 525086176, + "objective/train/unlikelihood_loss": 0.017949730157852173, + "theoretical_loss": 3.437780985213702, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505857807735516, + "loss": 1.2732, + "theoretical_loss": 3.437780985213702, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505456588027604, + "loss": 1.2678, + "theoretical_loss": 3.4377639456116134, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035050553683196916, + "loss": 1.334, + "theoretical_loss": 3.4377469073361975, + "tokens_seen": 2154889216 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350465414861178, + "loss": 1.318, + "theoretical_loss": 3.4377298703872707, + "tokens_seen": 2155020288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035042529289038677, + "loss": 1.2308, + "theoretical_loss": 3.4377128347646497, + "tokens_seen": 2155151360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503851709195956, + "loss": 1.2816, + "theoretical_loss": 3.4376958004681493, + "tokens_seen": 2155282432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503450489488044, + "loss": 1.2628, + "theoretical_loss": 3.4376787674975864, + "tokens_seen": 2155413504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503049269780132, + "loss": 1.3743, + "theoretical_loss": 3.4376617358527772, + "tokens_seen": 2155544576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035026480500722194, + "loss": 1.2892, + "theoretical_loss": 3.437644705533537, + "tokens_seen": 2155675648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035022468303643075, + "loss": 1.2672, + "theoretical_loss": 3.437627676539684, + "tokens_seen": 2155806720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035018456106563955, + "loss": 1.2938, + "theoretical_loss": 3.4376106488710327, + "tokens_seen": 2155937792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035014443909484836, + "loss": 1.3061, + "theoretical_loss": 3.4375936225273995, + "tokens_seen": 2156068864 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035010431712405717, + "loss": 1.2487, + "theoretical_loss": 3.4375765975086017, + "tokens_seen": 2156199936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500641951532659, + "loss": 1.2011, + "theoretical_loss": 3.437559573814455, + "tokens_seen": 2156331008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500240731824748, + "loss": 1.2525, + "theoretical_loss": 3.437542551444776, + "tokens_seen": 2156462080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034998395121168353, + "loss": 1.2937, + "theoretical_loss": 3.4375255303993812, + "tokens_seen": 2156593152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499438292408923, + "loss": 1.2781, + "theoretical_loss": 3.4375085106780876, + "tokens_seen": 2156724224 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034990370727010114, + "loss": 1.3299, + "theoretical_loss": 3.437491492280711, + "tokens_seen": 2156855296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498635852993099, + "loss": 1.2317, + "theoretical_loss": 3.437474475207068, + "tokens_seen": 2156986368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498234633285187, + "loss": 1.2532, + "theoretical_loss": 3.4374574594569753, + "tokens_seen": 2157117440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497833413577275, + "loss": 1.3445, + "theoretical_loss": 3.4374404450302496, + "tokens_seen": 2157248512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497432193869363, + "loss": 1.3283, + "theoretical_loss": 3.4374234319267076, + "tokens_seen": 2157379584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034970309741614507, + "loss": 1.365, + "theoretical_loss": 3.437406420146167, + "tokens_seen": 2157510656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034966297544535387, + "loss": 1.269, + "theoretical_loss": 3.4373894096884423, + "tokens_seen": 2157641728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496228534745627, + "loss": 1.2973, + "theoretical_loss": 3.437372400553352, + "tokens_seen": 2157772800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034958273150377143, + "loss": 1.322, + "theoretical_loss": 3.437355392740713, + "tokens_seen": 2157903872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495426095329803, + "loss": 1.2818, + "theoretical_loss": 3.4373383862503415, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034950248756218904, + "loss": 1.2786, + "theoretical_loss": 3.437321381082054, + "tokens_seen": 2158166016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494623655913979, + "loss": 1.2847, + "theoretical_loss": 3.4373043772356686, + "tokens_seen": 2158297088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034942224362060666, + "loss": 1.3309, + "theoretical_loss": 3.4372873747110013, + "tokens_seen": 2158428160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493821216498154, + "loss": 1.3173, + "theoretical_loss": 3.43727037350787, + "tokens_seen": 2158559232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034934199967902427, + "loss": 1.2961, + "theoretical_loss": 3.437253373626091, + "tokens_seen": 2158690304 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349301877708233, + "loss": 1.2363, + "theoretical_loss": 3.4372363750654813, + "tokens_seen": 2158821376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492617557374418, + "loss": 1.2762, + "theoretical_loss": 3.437219377825859, + "tokens_seen": 2158952448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034922163376665063, + "loss": 1.3159, + "theoretical_loss": 3.43720238190704, + "tokens_seen": 2159083520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034918151179585944, + "loss": 1.2849, + "theoretical_loss": 3.4371853873088423, + "tokens_seen": 2159214592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491413898250682, + "loss": 1.305, + "theoretical_loss": 3.437168394031083, + "tokens_seen": 2159345664 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349101267854277, + "loss": 1.338, + "theoretical_loss": 3.4371514020735794, + "tokens_seen": 2159476736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490611458834858, + "loss": 1.2829, + "theoretical_loss": 3.437134411436148, + "tokens_seen": 2159607808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034902102391269456, + "loss": 1.2736, + "theoretical_loss": 3.4371174221186074, + "tokens_seen": 2159738880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489809019419034, + "loss": 1.3178, + "theoretical_loss": 3.4371004341207745, + "tokens_seen": 2159869952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034894077997111217, + "loss": 1.2832, + "theoretical_loss": 3.4370834474424665, + "tokens_seen": 2160001024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034890065800032103, + "loss": 1.3331, + "theoretical_loss": 3.437066462083501, + "tokens_seen": 2160132096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488605360295298, + "loss": 1.349, + "theoretical_loss": 3.4370494780436953, + "tokens_seen": 2160263168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488204140587386, + "loss": 1.31, + "theoretical_loss": 3.437032495322867, + "tokens_seen": 2160394240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487802920879474, + "loss": 1.2827, + "theoretical_loss": 3.4370155139208336, + "tokens_seen": 2160525312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034874017011715615, + "loss": 1.4115, + "theoretical_loss": 3.436998533837413, + "tokens_seen": 2160656384 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034870004814636495, + "loss": 1.3058, + "theoretical_loss": 3.4369815550724225, + "tokens_seen": 2160787456 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034865992617557376, + "loss": 1.3106, + "theoretical_loss": 3.43696457762568, + "tokens_seen": 2160918528 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034861980420478256, + "loss": 1.2145, + "theoretical_loss": 3.436947601497003, + "tokens_seen": 2161049600 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.9920182228088379, + "objective/train/avg_token_score": 0.007981769740581512, + "objective/train/docs_used": 786923, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1542381048202515, + "objective/train/original_loss": 1.1630343198776245, + "objective/train/theoretical_loss": 3.436930626686209, + "objective/train/tokens_used": 531639776, + "objective/train/unlikelihood_loss": 0.029429782181978226, + "theoretical_loss": 3.436930626686209, + "tokens_seen": 2161180672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485796822339913, + "loss": 1.2508, + "theoretical_loss": 3.436930626686209, + "tokens_seen": 2161180672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485395602632002, + "loss": 1.3165, + "theoretical_loss": 3.436913653193117, + "tokens_seen": 2161311744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034849943829240893, + "loss": 1.2013, + "theoretical_loss": 3.436896681017543, + "tokens_seen": 2161442816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484593163216177, + "loss": 1.2257, + "theoretical_loss": 3.4368797101593063, + "tokens_seen": 2161573888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034841919435082654, + "loss": 1.301, + "theoretical_loss": 3.4368627406182237, + "tokens_seen": 2161704960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483790723800353, + "loss": 1.2166, + "theoretical_loss": 3.436845772394114, + "tokens_seen": 2161836032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483389504092441, + "loss": 1.3689, + "theoretical_loss": 3.436828805486795, + "tokens_seen": 2161967104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482988284384529, + "loss": 1.3047, + "theoretical_loss": 3.4368118398960843, + "tokens_seen": 2162098176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482587064676617, + "loss": 1.3034, + "theoretical_loss": 3.4367948756218, + "tokens_seen": 2162229248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482185844968705, + "loss": 1.2614, + "theoretical_loss": 3.4367779126637603, + "tokens_seen": 2162360320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034817846252607927, + "loss": 1.3088, + "theoretical_loss": 3.4367609510217836, + "tokens_seen": 2162491392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481383405552881, + "loss": 1.2569, + "theoretical_loss": 3.436743990695687, + "tokens_seen": 2162622464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480982185844969, + "loss": 1.2879, + "theoretical_loss": 3.4367270316852903, + "tokens_seen": 2162753536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480580966137057, + "loss": 1.3488, + "theoretical_loss": 3.43671007399041, + "tokens_seen": 2162884608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034801797464291444, + "loss": 1.3046, + "theoretical_loss": 3.4366931176108655, + "tokens_seen": 2163015680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479778526721233, + "loss": 1.2464, + "theoretical_loss": 3.436676162546475, + "tokens_seen": 2163146752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034793773070133206, + "loss": 1.2977, + "theoretical_loss": 3.4366592087970558, + "tokens_seen": 2163277824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478976087305408, + "loss": 1.2897, + "theoretical_loss": 3.4366422563624277, + "tokens_seen": 2163408896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034785748675974967, + "loss": 1.2361, + "theoretical_loss": 3.436625305242408, + "tokens_seen": 2163539968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478173647889584, + "loss": 1.2794, + "theoretical_loss": 3.4366083554368156, + "tokens_seen": 2163671040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477772428181672, + "loss": 1.2823, + "theoretical_loss": 3.4365914069454684, + "tokens_seen": 2163802112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034773712084737603, + "loss": 1.3184, + "theoretical_loss": 3.436574459768186, + "tokens_seen": 2163933184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034769699887658484, + "loss": 1.417, + "theoretical_loss": 3.436557513904786, + "tokens_seen": 2164064256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476568769057936, + "loss": 1.3022, + "theoretical_loss": 3.436540569355087, + "tokens_seen": 2164195328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476167549350024, + "loss": 1.3194, + "theoretical_loss": 3.436523626118908, + "tokens_seen": 2164326400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475766329642112, + "loss": 1.3753, + "theoretical_loss": 3.436506684196068, + "tokens_seen": 2164457472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034753651099342, + "loss": 1.2677, + "theoretical_loss": 3.4364897435863844, + "tokens_seen": 2164588544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474963890226288, + "loss": 1.2719, + "theoretical_loss": 3.436472804289677, + "tokens_seen": 2164719616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034745626705183757, + "loss": 1.2745, + "theoretical_loss": 3.4364558663057645, + "tokens_seen": 2164850688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034741614508104643, + "loss": 1.2512, + "theoretical_loss": 3.4364389296344653, + "tokens_seen": 2164981760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473760231102552, + "loss": 1.3241, + "theoretical_loss": 3.436421994275598, + "tokens_seen": 2165112832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034733590113946393, + "loss": 1.3323, + "theoretical_loss": 3.436405060228982, + "tokens_seen": 2165243904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472957791686728, + "loss": 1.3696, + "theoretical_loss": 3.436388127494436, + "tokens_seen": 2165374976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034725565719788155, + "loss": 1.2842, + "theoretical_loss": 3.4363711960717787, + "tokens_seen": 2165506048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034721553522709035, + "loss": 1.233, + "theoretical_loss": 3.4363542659608295, + "tokens_seen": 2165637120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034717541325629916, + "loss": 1.3701, + "theoretical_loss": 3.4363373371614068, + "tokens_seen": 2165768192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034713529128550796, + "loss": 1.2653, + "theoretical_loss": 3.43632040967333, + "tokens_seen": 2165899264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470951693147167, + "loss": 1.3146, + "theoretical_loss": 3.4363034834964186, + "tokens_seen": 2166030336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470550473439256, + "loss": 1.2558, + "theoretical_loss": 3.436286558630491, + "tokens_seen": 2166161408 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034701492537313433, + "loss": 1.314, + "theoretical_loss": 3.436269635075367, + "tokens_seen": 2166292480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034697480340234314, + "loss": 1.2866, + "theoretical_loss": 3.436252712830865, + "tokens_seen": 2166423552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034693468143155194, + "loss": 1.2305, + "theoretical_loss": 3.436235791896805, + "tokens_seen": 2166554624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468945594607607, + "loss": 1.2732, + "theoretical_loss": 3.4362188722730056, + "tokens_seen": 2166685696 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034685443748996955, + "loss": 1.3384, + "theoretical_loss": 3.4362019539592863, + "tokens_seen": 2166816768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468143155191783, + "loss": 1.3069, + "theoretical_loss": 3.4361850369554667, + "tokens_seen": 2166947840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467741935483871, + "loss": 1.3316, + "theoretical_loss": 3.4361681212613657, + "tokens_seen": 2167078912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467340715775959, + "loss": 1.2745, + "theoretical_loss": 3.4361512068768034, + "tokens_seen": 2167209984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034669394960680467, + "loss": 1.2407, + "theoretical_loss": 3.436134293801598, + "tokens_seen": 2167341056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466538276360135, + "loss": 1.308, + "theoretical_loss": 3.4361173820355706, + "tokens_seen": 2167472128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466137056652223, + "loss": 1.3645, + "theoretical_loss": 3.4361004715785395, + "tokens_seen": 2167603200 + }, + { + "epoch": 0.31, + "objective/train/aligned_token_count": 0.9908237457275391, + "objective/train/avg_token_score": 0.009176228195428848, + "objective/train/docs_used": 789244, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2739293575286865, + "objective/train/original_loss": 1.2859995365142822, + "objective/train/theoretical_loss": 3.436083562430324, + "objective/train/tokens_used": 538193376, + "objective/train/unlikelihood_loss": 0.026628762483596802, + "theoretical_loss": 3.436083562430324, + "tokens_seen": 2167734272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465735836944311, + "loss": 1.2847, + "theoretical_loss": 3.436083562430324, + "tokens_seen": 2167734272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034653346172363984, + "loss": 1.2341, + "theoretical_loss": 3.4360666545907446, + "tokens_seen": 2167865344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464933397528487, + "loss": 1.3719, + "theoretical_loss": 3.4360497480596206, + "tokens_seen": 2167996416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034645321778205745, + "loss": 1.3678, + "theoretical_loss": 3.436032842836772, + "tokens_seen": 2168127488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464130958112662, + "loss": 1.3086, + "theoretical_loss": 3.4360159389220177, + "tokens_seen": 2168258560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034637297384047507, + "loss": 1.2759, + "theoretical_loss": 3.4359990363151773, + "tokens_seen": 2168389632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463328518696838, + "loss": 1.2511, + "theoretical_loss": 3.4359821350160717, + "tokens_seen": 2168520704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462927298988927, + "loss": 1.3303, + "theoretical_loss": 3.4359652350245202, + "tokens_seen": 2168651776 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034625260792810143, + "loss": 1.2656, + "theoretical_loss": 3.435948336340342, + "tokens_seen": 2168782848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034621248595731024, + "loss": 1.2916, + "theoretical_loss": 3.4359314389633573, + "tokens_seen": 2168913920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034617236398651904, + "loss": 1.3415, + "theoretical_loss": 3.4359145428933866, + "tokens_seen": 2169044992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461322420157278, + "loss": 1.2386, + "theoretical_loss": 3.435897648130249, + "tokens_seen": 2169176064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460921200449366, + "loss": 1.3136, + "theoretical_loss": 3.435880754673765, + "tokens_seen": 2169307136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460519980741454, + "loss": 1.2859, + "theoretical_loss": 3.435863862523754, + "tokens_seen": 2169438208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460118761033542, + "loss": 1.3422, + "theoretical_loss": 3.435846971680037, + "tokens_seen": 2169569280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034597175413256297, + "loss": 1.2196, + "theoretical_loss": 3.4358300821424335, + "tokens_seen": 2169700352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034593163216177183, + "loss": 1.2479, + "theoretical_loss": 3.435813193910763, + "tokens_seen": 2169831424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458915101909806, + "loss": 1.2675, + "theoretical_loss": 3.4357963069848467, + "tokens_seen": 2169962496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034585138822018933, + "loss": 1.2842, + "theoretical_loss": 3.4357794213645043, + "tokens_seen": 2170093568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458112662493982, + "loss": 1.3248, + "theoretical_loss": 3.4357625370495564, + "tokens_seen": 2170224640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034577114427860695, + "loss": 1.3416, + "theoretical_loss": 3.4357456540398226, + "tokens_seen": 2170355712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457310223078158, + "loss": 1.3168, + "theoretical_loss": 3.4357287723351235, + "tokens_seen": 2170486784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034569090033702456, + "loss": 1.3407, + "theoretical_loss": 3.4357118919352794, + "tokens_seen": 2170617856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034565077836623336, + "loss": 1.3616, + "theoretical_loss": 3.4356950128401107, + "tokens_seen": 2170748928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034561065639544217, + "loss": 1.3589, + "theoretical_loss": 3.435678135049438, + "tokens_seen": 2170880000 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345570534424651, + "loss": 1.3144, + "theoretical_loss": 3.4356612585630812, + "tokens_seen": 2171011072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034553041245385973, + "loss": 1.3227, + "theoretical_loss": 3.435644383380861, + "tokens_seen": 2171142144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034549029048306854, + "loss": 1.3114, + "theoretical_loss": 3.4356275095025985, + "tokens_seen": 2171273216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034545016851227734, + "loss": 1.299, + "theoretical_loss": 3.4356106369281134, + "tokens_seen": 2171404288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454100465414861, + "loss": 1.373, + "theoretical_loss": 3.4355937656572264, + "tokens_seen": 2171535360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034536992457069495, + "loss": 1.3103, + "theoretical_loss": 3.435576895689759, + "tokens_seen": 2171666432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453298025999037, + "loss": 1.342, + "theoretical_loss": 3.4355600270255304, + "tokens_seen": 2171797504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452896806291125, + "loss": 1.3655, + "theoretical_loss": 3.4355431596643626, + "tokens_seen": 2171928576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452495586583213, + "loss": 1.3168, + "theoretical_loss": 3.4355262936060753, + "tokens_seen": 2172059648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034520943668753007, + "loss": 1.2896, + "theoretical_loss": 3.43550942885049, + "tokens_seen": 2172190720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451693147167389, + "loss": 1.2958, + "theoretical_loss": 3.4354925653974266, + "tokens_seen": 2172321792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451291927459477, + "loss": 1.2491, + "theoretical_loss": 3.435475703246707, + "tokens_seen": 2172452864 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450890707751565, + "loss": 1.3174, + "theoretical_loss": 3.4354588423981514, + "tokens_seen": 2172583936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450489488043653, + "loss": 1.2389, + "theoretical_loss": 3.4354419828515805, + "tokens_seen": 2172715008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450088268335741, + "loss": 1.326, + "theoretical_loss": 3.435425124606816, + "tokens_seen": 2172846080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034496870486278285, + "loss": 1.2681, + "theoretical_loss": 3.4354082676636786, + "tokens_seen": 2172977152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034492858289199166, + "loss": 1.1537, + "theoretical_loss": 3.4353914120219886, + "tokens_seen": 2173108224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034488846092120047, + "loss": 1.2177, + "theoretical_loss": 3.4353745576815675, + "tokens_seen": 2173239296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448483389504092, + "loss": 1.3528, + "theoretical_loss": 3.4353577046422368, + "tokens_seen": 2173370368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448082169796181, + "loss": 1.2702, + "theoretical_loss": 3.435340852903817, + "tokens_seen": 2173501440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034476809500882683, + "loss": 1.2873, + "theoretical_loss": 3.43532400246613, + "tokens_seen": 2173632512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034472797303803564, + "loss": 1.2933, + "theoretical_loss": 3.4353071533289956, + "tokens_seen": 2173763584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034468785106724444, + "loss": 1.2577, + "theoretical_loss": 3.435290305492236, + "tokens_seen": 2173894656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446477290964532, + "loss": 1.2627, + "theoretical_loss": 3.435273458955673, + "tokens_seen": 2174025728 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344607607125662, + "loss": 1.279, + "theoretical_loss": 3.4352566137191265, + "tokens_seen": 2174156800 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.9926750063896179, + "objective/train/avg_token_score": 0.0073250047862529755, + "objective/train/docs_used": 791682, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3189934492111206, + "objective/train/original_loss": 1.3283898830413818, + "objective/train/theoretical_loss": 3.435239769782419, + "objective/train/tokens_used": 544746976, + "objective/train/unlikelihood_loss": 0.022602060809731483, + "theoretical_loss": 3.435239769782419, + "tokens_seen": 2174287872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445674851548708, + "loss": 1.3375, + "theoretical_loss": 3.435239769782419, + "tokens_seen": 2174287872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445273631840796, + "loss": 1.2852, + "theoretical_loss": 3.4352229271453707, + "tokens_seen": 2174418944 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034448724121328837, + "loss": 1.32, + "theoretical_loss": 3.435206085807804, + "tokens_seen": 2174550016 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034444711924249723, + "loss": 1.354, + "theoretical_loss": 3.43518924576954, + "tokens_seen": 2174681088 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344406997271706, + "loss": 1.2502, + "theoretical_loss": 3.4351724070304, + "tokens_seen": 2174812160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443668753009148, + "loss": 1.3235, + "theoretical_loss": 3.4351555695902056, + "tokens_seen": 2174943232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443267533301236, + "loss": 1.3079, + "theoretical_loss": 3.4351387334487784, + "tokens_seen": 2175074304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034428663135933234, + "loss": 1.2916, + "theoretical_loss": 3.4351218986059395, + "tokens_seen": 2175205376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442465093885412, + "loss": 1.2277, + "theoretical_loss": 3.4351050650615114, + "tokens_seen": 2175336448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034420638741774996, + "loss": 1.248, + "theoretical_loss": 3.435088232815315, + "tokens_seen": 2175467520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034416626544695876, + "loss": 1.1865, + "theoretical_loss": 3.435071401867172, + "tokens_seen": 2175598592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034412614347616757, + "loss": 1.3079, + "theoretical_loss": 3.4350545722169046, + "tokens_seen": 2175729664 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440860215053764, + "loss": 1.2523, + "theoretical_loss": 3.435037743864334, + "tokens_seen": 2175860736 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034404589953458513, + "loss": 1.3006, + "theoretical_loss": 3.4350209168092825, + "tokens_seen": 2175991808 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034400577756379393, + "loss": 1.2213, + "theoretical_loss": 3.4350040910515713, + "tokens_seen": 2176122880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034396565559300274, + "loss": 1.2068, + "theoretical_loss": 3.4349872665910226, + "tokens_seen": 2176253952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439255336222115, + "loss": 1.2615, + "theoretical_loss": 3.4349704434274586, + "tokens_seen": 2176385024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034388541165142035, + "loss": 1.3456, + "theoretical_loss": 3.4349536215607004, + "tokens_seen": 2176516096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438452896806291, + "loss": 1.2753, + "theoretical_loss": 3.4349368009905707, + "tokens_seen": 2176647168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034380516770983797, + "loss": 1.221, + "theoretical_loss": 3.4349199817168907, + "tokens_seen": 2176778240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437650457390467, + "loss": 1.2856, + "theoretical_loss": 3.4349031637394836, + "tokens_seen": 2176909312 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034372492376825547, + "loss": 1.3642, + "theoretical_loss": 3.4348863470581703, + "tokens_seen": 2177040384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034368480179746433, + "loss": 1.305, + "theoretical_loss": 3.4348695316727733, + "tokens_seen": 2177171456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436446798266731, + "loss": 1.3459, + "theoretical_loss": 3.4348527175831154, + "tokens_seen": 2177302528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436045578558819, + "loss": 1.3503, + "theoretical_loss": 3.4348359047890176, + "tokens_seen": 2177433600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435644358850907, + "loss": 1.2953, + "theoretical_loss": 3.4348190932903027, + "tokens_seen": 2177564672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435243139142995, + "loss": 1.3453, + "theoretical_loss": 3.434802283086793, + "tokens_seen": 2177695744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034348419194350825, + "loss": 1.2519, + "theoretical_loss": 3.4347854741783106, + "tokens_seen": 2177826816 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034344406997271706, + "loss": 1.3092, + "theoretical_loss": 3.4347686665646777, + "tokens_seen": 2177957888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034340394800192587, + "loss": 1.2787, + "theoretical_loss": 3.4347518602457168, + "tokens_seen": 2178088960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433638260311346, + "loss": 1.285, + "theoretical_loss": 3.4347350552212506, + "tokens_seen": 2178220032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433237040603435, + "loss": 1.2713, + "theoretical_loss": 3.4347182514911005, + "tokens_seen": 2178351104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034328358208955223, + "loss": 1.2114, + "theoretical_loss": 3.43470144905509, + "tokens_seen": 2178482176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034324346011876104, + "loss": 1.3268, + "theoretical_loss": 3.4346846479130413, + "tokens_seen": 2178613248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034320333814796984, + "loss": 1.2924, + "theoretical_loss": 3.4346678480647768, + "tokens_seen": 2178744320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431632161771786, + "loss": 1.2235, + "theoretical_loss": 3.4346510495101183, + "tokens_seen": 2178875392 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034312309420638746, + "loss": 1.263, + "theoretical_loss": 3.4346342522488897, + "tokens_seen": 2179006464 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430829722355962, + "loss": 1.3037, + "theoretical_loss": 3.4346174562809133, + "tokens_seen": 2179137536 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343042850264805, + "loss": 1.2434, + "theoretical_loss": 3.434600661606011, + "tokens_seen": 2179268608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430027282940138, + "loss": 1.3189, + "theoretical_loss": 3.4345838682240064, + "tokens_seen": 2179399680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034296260632322263, + "loss": 1.26, + "theoretical_loss": 3.4345670761347216, + "tokens_seen": 2179530752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429224843524314, + "loss": 1.2972, + "theoretical_loss": 3.4345502853379792, + "tokens_seen": 2179661824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428823623816402, + "loss": 1.2819, + "theoretical_loss": 3.4345334958336027, + "tokens_seen": 2179792896 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342842240410849, + "loss": 1.2568, + "theoretical_loss": 3.4345167076214143, + "tokens_seen": 2179923968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034280211844005774, + "loss": 1.3095, + "theoretical_loss": 3.4344999207012377, + "tokens_seen": 2180055040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427619964692666, + "loss": 1.3979, + "theoretical_loss": 3.4344831350728944, + "tokens_seen": 2180186112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034272187449847536, + "loss": 1.3072, + "theoretical_loss": 3.4344663507362085, + "tokens_seen": 2180317184 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034268175252768416, + "loss": 1.3324, + "theoretical_loss": 3.434449567691003, + "tokens_seen": 2180448256 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034264163055689297, + "loss": 1.3421, + "theoretical_loss": 3.4344327859371004, + "tokens_seen": 2180579328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426015085861017, + "loss": 1.3146, + "theoretical_loss": 3.4344160054743234, + "tokens_seen": 2180710400 + }, + { + "epoch": 0.32, + "objective/train/aligned_token_count": 0.9938268065452576, + "objective/train/avg_token_score": 0.006173178553581238, + "objective/train/docs_used": 794081, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.272702932357788, + "objective/train/original_loss": 1.282888412475586, + "objective/train/theoretical_loss": 3.434399226302496, + "objective/train/tokens_used": 551300576, + "objective/train/unlikelihood_loss": 0.016360698267817497, + "theoretical_loss": 3.434399226302496, + "tokens_seen": 2180841472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034256138661531053, + "loss": 1.2875, + "theoretical_loss": 3.434399226302496, + "tokens_seen": 2180841472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034252126464451933, + "loss": 1.2733, + "theoretical_loss": 3.4343824484214407, + "tokens_seen": 2180972544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034248114267372814, + "loss": 1.3055, + "theoretical_loss": 3.4343656718309807, + "tokens_seen": 2181103616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034244102070293695, + "loss": 1.2679, + "theoretical_loss": 3.434348896530939, + "tokens_seen": 2181234688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034240089873214575, + "loss": 1.2286, + "theoretical_loss": 3.4343321225211394, + "tokens_seen": 2181365760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423607767613545, + "loss": 1.3377, + "theoretical_loss": 3.434315349801405, + "tokens_seen": 2181496832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034232065479056337, + "loss": 1.2483, + "theoretical_loss": 3.4342985783715587, + "tokens_seen": 2181627904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422805328197721, + "loss": 1.3677, + "theoretical_loss": 3.434281808231424, + "tokens_seen": 2181758976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034224041084898087, + "loss": 1.2464, + "theoretical_loss": 3.434265039380824, + "tokens_seen": 2181890048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034220028887818973, + "loss": 1.3234, + "theoretical_loss": 3.4342482718195826, + "tokens_seen": 2182021120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421601669073985, + "loss": 1.2296, + "theoretical_loss": 3.434231505547523, + "tokens_seen": 2182152192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421200449366073, + "loss": 1.2533, + "theoretical_loss": 3.4342147405644683, + "tokens_seen": 2182283264 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420799229658161, + "loss": 1.2905, + "theoretical_loss": 3.4341979768702426, + "tokens_seen": 2182414336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420398009950249, + "loss": 1.3184, + "theoretical_loss": 3.434181214464669, + "tokens_seen": 2182545408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034199967902423365, + "loss": 1.3268, + "theoretical_loss": 3.4341644533475715, + "tokens_seen": 2182676480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034195955705344246, + "loss": 1.2646, + "theoretical_loss": 3.434147693518773, + "tokens_seen": 2182807552 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034191943508265127, + "loss": 1.3405, + "theoretical_loss": 3.4341309349780977, + "tokens_seen": 2182938624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034187931311186007, + "loss": 1.2778, + "theoretical_loss": 3.4341141777253688, + "tokens_seen": 2183069696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418391911410689, + "loss": 1.2434, + "theoretical_loss": 3.4340974217604106, + "tokens_seen": 2183200768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034179906917027763, + "loss": 1.2711, + "theoretical_loss": 3.4340806670830464, + "tokens_seen": 2183331840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417589471994865, + "loss": 1.3376, + "theoretical_loss": 3.4340639136931, + "tokens_seen": 2183462912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034171882522869524, + "loss": 1.2803, + "theoretical_loss": 3.4340471615903954, + "tokens_seen": 2183593984 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341678703257904, + "loss": 1.2967, + "theoretical_loss": 3.434030410774756, + "tokens_seen": 2183725056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034163858128711286, + "loss": 1.284, + "theoretical_loss": 3.434013661246006, + "tokens_seen": 2183856128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415984593163216, + "loss": 1.2742, + "theoretical_loss": 3.43399691300397, + "tokens_seen": 2183987200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415583373455304, + "loss": 1.3674, + "theoretical_loss": 3.43398016604847, + "tokens_seen": 2184118272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415182153747392, + "loss": 1.3064, + "theoretical_loss": 3.433963420379332, + "tokens_seen": 2184249344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034147809340394803, + "loss": 1.3513, + "theoretical_loss": 3.433946675996379, + "tokens_seen": 2184380416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414379714331568, + "loss": 1.3162, + "theoretical_loss": 3.4339299328994355, + "tokens_seen": 2184511488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413978494623656, + "loss": 1.3394, + "theoretical_loss": 3.433913191088325, + "tokens_seen": 2184642560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413577274915744, + "loss": 1.2795, + "theoretical_loss": 3.433896450562872, + "tokens_seen": 2184773632 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034131760552078314, + "loss": 1.2903, + "theoretical_loss": 3.4338797113229007, + "tokens_seen": 2184904704 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341277483549992, + "loss": 1.267, + "theoretical_loss": 3.4338629733682353, + "tokens_seen": 2185035776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034123736157920076, + "loss": 1.2615, + "theoretical_loss": 3.4338462366986997, + "tokens_seen": 2185166848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411972396084096, + "loss": 1.3502, + "theoretical_loss": 3.4338295013141185, + "tokens_seen": 2185297920 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034115711763761837, + "loss": 1.3149, + "theoretical_loss": 3.4338127672143157, + "tokens_seen": 2185428992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411169956668271, + "loss": 1.271, + "theoretical_loss": 3.4337960343991156, + "tokens_seen": 2185560064 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341076873696036, + "loss": 1.335, + "theoretical_loss": 3.4337793028683428, + "tokens_seen": 2185691136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034103675172524473, + "loss": 1.3204, + "theoretical_loss": 3.4337625726218217, + "tokens_seen": 2185822208 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034099662975445354, + "loss": 1.3403, + "theoretical_loss": 3.4337458436593766, + "tokens_seen": 2185953280 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034095650778366235, + "loss": 1.3128, + "theoretical_loss": 3.433729115980832, + "tokens_seen": 2186084352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034091638581287115, + "loss": 1.3724, + "theoretical_loss": 3.433712389586012, + "tokens_seen": 2186215424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408762638420799, + "loss": 1.3203, + "theoretical_loss": 3.433695664474742, + "tokens_seen": 2186346496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034083614187128877, + "loss": 1.3507, + "theoretical_loss": 3.433678940646846, + "tokens_seen": 2186477568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407960199004975, + "loss": 1.2601, + "theoretical_loss": 3.4336622181021488, + "tokens_seen": 2186608640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034075589792970627, + "loss": 1.273, + "theoretical_loss": 3.4336454968404744, + "tokens_seen": 2186739712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034071577595891513, + "loss": 1.3507, + "theoretical_loss": 3.4336287768616485, + "tokens_seen": 2186870784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406756539881239, + "loss": 1.2789, + "theoretical_loss": 3.4336120581654948, + "tokens_seen": 2187001856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034063553201733274, + "loss": 1.3733, + "theoretical_loss": 3.4335953407518387, + "tokens_seen": 2187132928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405954100465415, + "loss": 1.2861, + "theoretical_loss": 3.4335786246205044, + "tokens_seen": 2187264000 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.9942664504051208, + "objective/train/avg_token_score": 0.005733543075621128, + "objective/train/docs_used": 796380, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.406565546989441, + "objective/train/original_loss": 1.4159071445465088, + "objective/train/theoretical_loss": 3.4335619097713175, + "objective/train/tokens_used": 557854176, + "objective/train/unlikelihood_loss": 0.012901734560728073, + "theoretical_loss": 3.4335619097713175, + "tokens_seen": 2187395072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405552880757503, + "loss": 1.3484, + "theoretical_loss": 3.4335619097713175, + "tokens_seen": 2187395072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405151661049591, + "loss": 1.2897, + "theoretical_loss": 3.4335451962041024, + "tokens_seen": 2187526144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034047504413416786, + "loss": 1.3366, + "theoretical_loss": 3.4335284839186837, + "tokens_seen": 2187657216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034043492216337667, + "loss": 1.2829, + "theoretical_loss": 3.433511772914887, + "tokens_seen": 2187788288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034039480019258547, + "loss": 1.2968, + "theoretical_loss": 3.4334950631925363, + "tokens_seen": 2187919360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403546782217943, + "loss": 1.3192, + "theoretical_loss": 3.4334783547514576, + "tokens_seen": 2188050432 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034031455625100303, + "loss": 1.3001, + "theoretical_loss": 3.4334616475914754, + "tokens_seen": 2188181504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402744342802119, + "loss": 1.2782, + "theoretical_loss": 3.4334449417124144, + "tokens_seen": 2188312576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034023431230942064, + "loss": 1.3559, + "theoretical_loss": 3.4334282371141, + "tokens_seen": 2188443648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401941903386294, + "loss": 1.3314, + "theoretical_loss": 3.4334115337963578, + "tokens_seen": 2188574720 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034015406836783826, + "loss": 1.3275, + "theoretical_loss": 3.433394831759012, + "tokens_seen": 2188705792 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340113946397047, + "loss": 1.2818, + "theoretical_loss": 3.4333781310018887, + "tokens_seen": 2188836864 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400738244262558, + "loss": 1.3347, + "theoretical_loss": 3.433361431524813, + "tokens_seen": 2188967936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400337024554646, + "loss": 1.3269, + "theoretical_loss": 3.4333447333276093, + "tokens_seen": 2189099008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399935804846734, + "loss": 1.2685, + "theoretical_loss": 3.4333280364101038, + "tokens_seen": 2189230080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033995345851388223, + "loss": 1.2196, + "theoretical_loss": 3.433311340772121, + "tokens_seen": 2189361152 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339913336543091, + "loss": 1.2387, + "theoretical_loss": 3.433294646413487, + "tokens_seen": 2189492224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398732145722998, + "loss": 1.2986, + "theoretical_loss": 3.4332779533340263, + "tokens_seen": 2189623296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398330926015086, + "loss": 1.3268, + "theoretical_loss": 3.433261261533566, + "tokens_seen": 2189754368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397929706307174, + "loss": 1.3176, + "theoretical_loss": 3.4332445710119295, + "tokens_seen": 2189885440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033975284865992616, + "loss": 1.3031, + "theoretical_loss": 3.4332278817689437, + "tokens_seen": 2190016512 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339712726689135, + "loss": 1.3534, + "theoretical_loss": 3.433211193804434, + "tokens_seen": 2190147584 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033967260471834377, + "loss": 1.2843, + "theoretical_loss": 3.433194507118225, + "tokens_seen": 2190278656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396324827475525, + "loss": 1.3137, + "theoretical_loss": 3.433177821710143, + "tokens_seen": 2190409728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395923607767614, + "loss": 1.2665, + "theoretical_loss": 3.433161137580014, + "tokens_seen": 2190540800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033955223880597013, + "loss": 1.2796, + "theoretical_loss": 3.4331444547276635, + "tokens_seen": 2190671872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033951211683517894, + "loss": 1.2622, + "theoretical_loss": 3.433127773152916, + "tokens_seen": 2190802944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033947199486438775, + "loss": 1.2984, + "theoretical_loss": 3.433111092855599, + "tokens_seen": 2190934016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033943187289359655, + "loss": 1.3293, + "theoretical_loss": 3.433094413835537, + "tokens_seen": 2191065088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393917509228053, + "loss": 1.2687, + "theoretical_loss": 3.4330777360925566, + "tokens_seen": 2191196160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033935162895201417, + "loss": 1.2812, + "theoretical_loss": 3.433061059626483, + "tokens_seen": 2191327232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393115069812229, + "loss": 1.2813, + "theoretical_loss": 3.4330443844371423, + "tokens_seen": 2191458304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392713850104317, + "loss": 1.2763, + "theoretical_loss": 3.4330277105243603, + "tokens_seen": 2191589376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033923126303964053, + "loss": 1.2501, + "theoretical_loss": 3.4330110378879635, + "tokens_seen": 2191720448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391911410688493, + "loss": 1.3609, + "theoretical_loss": 3.432994366527777, + "tokens_seen": 2191851520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033915101909805814, + "loss": 1.3073, + "theoretical_loss": 3.4329776964436274, + "tokens_seen": 2191982592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391108971272669, + "loss": 1.2607, + "theoretical_loss": 3.432961027635341, + "tokens_seen": 2192113664 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390707751564757, + "loss": 1.3458, + "theoretical_loss": 3.4329443601027427, + "tokens_seen": 2192244736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390306531856845, + "loss": 1.2069, + "theoretical_loss": 3.43292769384566, + "tokens_seen": 2192375808 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033899053121489326, + "loss": 1.2746, + "theoretical_loss": 3.4329110288639177, + "tokens_seen": 2192506880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033895040924410207, + "loss": 1.2767, + "theoretical_loss": 3.432894365157343, + "tokens_seen": 2192637952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033891028727331087, + "loss": 1.279, + "theoretical_loss": 3.432877702725762, + "tokens_seen": 2192769024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388701653025197, + "loss": 1.3246, + "theoretical_loss": 3.4328610415690006, + "tokens_seen": 2192900096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033883004333172843, + "loss": 1.2365, + "theoretical_loss": 3.432844381686885, + "tokens_seen": 2193031168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387899213609373, + "loss": 1.2969, + "theoretical_loss": 3.4328277230792423, + "tokens_seen": 2193162240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033874979939014604, + "loss": 1.3049, + "theoretical_loss": 3.4328110657458977, + "tokens_seen": 2193293312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033870967741935485, + "loss": 1.2857, + "theoretical_loss": 3.4327944096866783, + "tokens_seen": 2193424384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033866955544856366, + "loss": 1.297, + "theoretical_loss": 3.43277775490141, + "tokens_seen": 2193555456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386294334777724, + "loss": 1.2998, + "theoretical_loss": 3.43276110138992, + "tokens_seen": 2193686528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033858931150698127, + "loss": 1.3313, + "theoretical_loss": 3.4327444491520342, + "tokens_seen": 2193817600 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.9935293197631836, + "objective/train/avg_token_score": 0.006470712833106518, + "objective/train/docs_used": 798764, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3404849767684937, + "objective/train/original_loss": 1.3500967025756836, + "objective/train/theoretical_loss": 3.4327277981875794, + "objective/train/tokens_used": 564407776, + "objective/train/unlikelihood_loss": 0.015877753496170044, + "theoretical_loss": 3.4327277981875794, + "tokens_seen": 2193948672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033854918953619, + "loss": 1.3287, + "theoretical_loss": 3.4327277981875794, + "tokens_seen": 2193948672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385090675653988, + "loss": 1.329, + "theoretical_loss": 3.432711148496382, + "tokens_seen": 2194079744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033846894559460763, + "loss": 1.3039, + "theoretical_loss": 3.4326945000782687, + "tokens_seen": 2194210816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384288236238164, + "loss": 1.2904, + "theoretical_loss": 3.432677852933066, + "tokens_seen": 2194341888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383887016530252, + "loss": 1.3007, + "theoretical_loss": 3.4326612070606006, + "tokens_seen": 2194472960 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338348579682234, + "loss": 1.2912, + "theoretical_loss": 3.432644562460699, + "tokens_seen": 2194604032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383084577114428, + "loss": 1.2551, + "theoretical_loss": 3.4326279191331883, + "tokens_seen": 2194735104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033826833574065156, + "loss": 1.3559, + "theoretical_loss": 3.432611277077895, + "tokens_seen": 2194866176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382282137698604, + "loss": 1.3329, + "theoretical_loss": 3.432594636294646, + "tokens_seen": 2194997248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033818809179906917, + "loss": 1.2732, + "theoretical_loss": 3.4325779967832677, + "tokens_seen": 2195128320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381479698282779, + "loss": 1.2873, + "theoretical_loss": 3.4325613585435875, + "tokens_seen": 2195259392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381078478574868, + "loss": 1.2383, + "theoretical_loss": 3.4325447215754323, + "tokens_seen": 2195390464 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033806772588669553, + "loss": 1.3033, + "theoretical_loss": 3.4325280858786287, + "tokens_seen": 2195521536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380276039159044, + "loss": 1.2494, + "theoretical_loss": 3.4325114514530037, + "tokens_seen": 2195652608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033798748194511315, + "loss": 1.3855, + "theoretical_loss": 3.4324948182983843, + "tokens_seen": 2195783680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033794735997432195, + "loss": 1.3004, + "theoretical_loss": 3.4324781864145977, + "tokens_seen": 2195914752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033790723800353076, + "loss": 1.2755, + "theoretical_loss": 3.432461555801471, + "tokens_seen": 2196045824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378671160327395, + "loss": 1.2761, + "theoretical_loss": 3.432444926458831, + "tokens_seen": 2196176896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378269940619483, + "loss": 1.2484, + "theoretical_loss": 3.4324282983865047, + "tokens_seen": 2196307968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377868720911571, + "loss": 1.2652, + "theoretical_loss": 3.43241167158432, + "tokens_seen": 2196439040 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033774675012036593, + "loss": 1.3341, + "theoretical_loss": 3.432395046052103, + "tokens_seen": 2196570112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377066281495747, + "loss": 1.347, + "theoretical_loss": 3.432378421789682, + "tokens_seen": 2196701184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033766650617878354, + "loss": 1.2868, + "theoretical_loss": 3.432361798796884, + "tokens_seen": 2196832256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376263842079923, + "loss": 1.2841, + "theoretical_loss": 3.4323451770735356, + "tokens_seen": 2196963328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375862622372011, + "loss": 1.3275, + "theoretical_loss": 3.4323285566194652, + "tokens_seen": 2197094400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375461402664099, + "loss": 1.2895, + "theoretical_loss": 3.4323119374344992, + "tokens_seen": 2197225472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033750601829561866, + "loss": 1.2749, + "theoretical_loss": 3.4322953195184653, + "tokens_seen": 2197356544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374658963248275, + "loss": 1.3086, + "theoretical_loss": 3.432278702871191, + "tokens_seen": 2197487616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033742577435403627, + "loss": 1.3203, + "theoretical_loss": 3.432262087492504, + "tokens_seen": 2197618688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373856523832451, + "loss": 1.278, + "theoretical_loss": 3.4322454733822316, + "tokens_seen": 2197749760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373455304124539, + "loss": 1.341, + "theoretical_loss": 3.432228860540201, + "tokens_seen": 2197880832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373054084416627, + "loss": 1.3188, + "theoretical_loss": 3.4322122489662403, + "tokens_seen": 2198011904 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033726528647087144, + "loss": 1.3308, + "theoretical_loss": 3.432195638660177, + "tokens_seen": 2198142976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033722516450008025, + "loss": 1.2788, + "theoretical_loss": 3.432179029621838, + "tokens_seen": 2198274048 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033718504252928906, + "loss": 1.3206, + "theoretical_loss": 3.4321624218510522, + "tokens_seen": 2198405120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371449205584978, + "loss": 1.2589, + "theoretical_loss": 3.4321458153476465, + "tokens_seen": 2198536192 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033710479858770667, + "loss": 1.2892, + "theoretical_loss": 3.432129210111449, + "tokens_seen": 2198667264 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370646766169154, + "loss": 1.3126, + "theoretical_loss": 3.4321126061422866, + "tokens_seen": 2198798336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370245546461242, + "loss": 1.2593, + "theoretical_loss": 3.4320960034399883, + "tokens_seen": 2198929408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033698443267533303, + "loss": 1.2164, + "theoretical_loss": 3.432079402004381, + "tokens_seen": 2199060480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369443107045418, + "loss": 1.3296, + "theoretical_loss": 3.4320628018352934, + "tokens_seen": 2199191552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369041887337506, + "loss": 1.2482, + "theoretical_loss": 3.4320462029325527, + "tokens_seen": 2199322624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368640667629594, + "loss": 1.2812, + "theoretical_loss": 3.432029605295987, + "tokens_seen": 2199453696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368239447921682, + "loss": 1.2925, + "theoretical_loss": 3.432013008925425, + "tokens_seen": 2199584768 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336783822821377, + "loss": 1.3402, + "theoretical_loss": 3.4319964138206935, + "tokens_seen": 2199715840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367437008505858, + "loss": 1.2973, + "theoretical_loss": 3.4319798199816214, + "tokens_seen": 2199846912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033670357887979457, + "loss": 1.2651, + "theoretical_loss": 3.431963227408036, + "tokens_seen": 2199977984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366634569090034, + "loss": 1.2908, + "theoretical_loss": 3.4319466360997666, + "tokens_seen": 2200109056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366233349382122, + "loss": 1.3533, + "theoretical_loss": 3.43193004605664, + "tokens_seen": 2200240128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033658321296742093, + "loss": 1.3152, + "theoretical_loss": 3.4319134572784855, + "tokens_seen": 2200371200 + }, + { + "epoch": 0.33, + "objective/train/aligned_token_count": 0.9917894601821899, + "objective/train/avg_token_score": 0.0082105016335845, + "objective/train/docs_used": 801168, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2393349409103394, + "objective/train/original_loss": 1.2504234313964844, + "objective/train/theoretical_loss": 3.4318968697651306, + "objective/train/tokens_used": 570961376, + "objective/train/unlikelihood_loss": 0.023127233609557152, + "theoretical_loss": 3.4318968697651306, + "tokens_seen": 2200502272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365430909966298, + "loss": 1.2546, + "theoretical_loss": 3.4318968697651306, + "tokens_seen": 2200502272 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033650296902583855, + "loss": 1.3069, + "theoretical_loss": 3.4318802835164037, + "tokens_seen": 2200633344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033646284705504735, + "loss": 1.2292, + "theoretical_loss": 3.431863698532133, + "tokens_seen": 2200764416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033642272508425616, + "loss": 1.2731, + "theoretical_loss": 3.4318471148121468, + "tokens_seen": 2200895488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363826031134649, + "loss": 1.2886, + "theoretical_loss": 3.431830532356274, + "tokens_seen": 2201026560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363424811426737, + "loss": 1.2717, + "theoretical_loss": 3.431813951164342, + "tokens_seen": 2201157632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363023591718825, + "loss": 1.2938, + "theoretical_loss": 3.43179737123618, + "tokens_seen": 2201288704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033626223720109133, + "loss": 1.2446, + "theoretical_loss": 3.431780792571616, + "tokens_seen": 2201419776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362221152303001, + "loss": 1.2921, + "theoretical_loss": 3.431764215170479, + "tokens_seen": 2201550848 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033618199325950894, + "loss": 1.2629, + "theoretical_loss": 3.431747639032597, + "tokens_seen": 2201681920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361418712887177, + "loss": 1.2833, + "theoretical_loss": 3.4317310641577987, + "tokens_seen": 2201812992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033610174931792655, + "loss": 1.247, + "theoretical_loss": 3.4317144905459127, + "tokens_seen": 2201944064 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360616273471353, + "loss": 1.3269, + "theoretical_loss": 3.4316979181967673, + "tokens_seen": 2202075136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033602150537634406, + "loss": 1.2955, + "theoretical_loss": 3.4316813471101915, + "tokens_seen": 2202206208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359813834055529, + "loss": 1.3222, + "theoretical_loss": 3.431664777286014, + "tokens_seen": 2202337280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033594126143476167, + "loss": 1.3009, + "theoretical_loss": 3.431648208724063, + "tokens_seen": 2202468352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359011394639705, + "loss": 1.2547, + "theoretical_loss": 3.431631641424168, + "tokens_seen": 2202599424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358610174931793, + "loss": 1.2808, + "theoretical_loss": 3.431615075386157, + "tokens_seen": 2202730496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358208955223881, + "loss": 1.4582, + "theoretical_loss": 3.4315985106098594, + "tokens_seen": 2202861568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033578077355159684, + "loss": 1.3244, + "theoretical_loss": 3.431581947095104, + "tokens_seen": 2202992640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033574065158080565, + "loss": 1.2917, + "theoretical_loss": 3.43156538484172, + "tokens_seen": 2203123712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033570052961001445, + "loss": 1.3434, + "theoretical_loss": 3.431548823849535, + "tokens_seen": 2203254784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356604076392232, + "loss": 1.285, + "theoretical_loss": 3.431532264118379, + "tokens_seen": 2203385856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033562028566843207, + "loss": 1.2938, + "theoretical_loss": 3.4315157056480805, + "tokens_seen": 2203516928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355801636976408, + "loss": 1.2849, + "theoretical_loss": 3.431499148438469, + "tokens_seen": 2203648000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355400417268497, + "loss": 1.2977, + "theoretical_loss": 3.4314825924893735, + "tokens_seen": 2203779072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033549991975605843, + "loss": 1.3413, + "theoretical_loss": 3.4314660378006225, + "tokens_seen": 2203910144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354597977852672, + "loss": 1.2704, + "theoretical_loss": 3.4314494843720458, + "tokens_seen": 2204041216 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033541967581447604, + "loss": 1.2326, + "theoretical_loss": 3.431432932203472, + "tokens_seen": 2204172288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353795538436848, + "loss": 1.2848, + "theoretical_loss": 3.4314163812947305, + "tokens_seen": 2204303360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353394318728936, + "loss": 1.3686, + "theoretical_loss": 3.4313998316456504, + "tokens_seen": 2204434432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352993099021024, + "loss": 1.3402, + "theoretical_loss": 3.431383283256061, + "tokens_seen": 2204565504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352591879313112, + "loss": 1.3051, + "theoretical_loss": 3.4313667361257916, + "tokens_seen": 2204696576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033521906596051997, + "loss": 1.2926, + "theoretical_loss": 3.4313501902546717, + "tokens_seen": 2204827648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351789439897288, + "loss": 1.3076, + "theoretical_loss": 3.43133364564253, + "tokens_seen": 2204958720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351388220189376, + "loss": 1.2688, + "theoretical_loss": 3.4313171022891966, + "tokens_seen": 2205089792 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033509870004814633, + "loss": 1.2828, + "theoretical_loss": 3.431300560194501, + "tokens_seen": 2205220864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350585780773552, + "loss": 1.4036, + "theoretical_loss": 3.4312840193582717, + "tokens_seen": 2205351936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033501845610656395, + "loss": 1.3063, + "theoretical_loss": 3.431267479780339, + "tokens_seen": 2205483008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033497833413577275, + "loss": 1.3275, + "theoretical_loss": 3.431250941460532, + "tokens_seen": 2205614080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033493821216498156, + "loss": 1.3358, + "theoretical_loss": 3.4312344043986807, + "tokens_seen": 2205745152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348980901941903, + "loss": 1.3122, + "theoretical_loss": 3.431217868594614, + "tokens_seen": 2205876224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033485796822339917, + "loss": 1.3019, + "theoretical_loss": 3.431201334048162, + "tokens_seen": 2206007296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348178462526079, + "loss": 1.284, + "theoretical_loss": 3.4311848007591546, + "tokens_seen": 2206138368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033477772428181673, + "loss": 1.3013, + "theoretical_loss": 3.4311682687274208, + "tokens_seen": 2206269440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033473760231102554, + "loss": 1.2617, + "theoretical_loss": 3.4311517379527903, + "tokens_seen": 2206400512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033469748034023434, + "loss": 1.2856, + "theoretical_loss": 3.4311352084350935, + "tokens_seen": 2206531584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346573583694431, + "loss": 1.2917, + "theoretical_loss": 3.4311186801741598, + "tokens_seen": 2206662656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033461723639865195, + "loss": 1.3355, + "theoretical_loss": 3.4311021531698187, + "tokens_seen": 2206793728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345771144278607, + "loss": 1.2699, + "theoretical_loss": 3.4310856274219006, + "tokens_seen": 2206924800 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.9934345483779907, + "objective/train/avg_token_score": 0.006565408315509558, + "objective/train/docs_used": 803600, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2986207008361816, + "objective/train/original_loss": 1.307766318321228, + "objective/train/theoretical_loss": 3.431069102930235, + "objective/train/tokens_used": 577514976, + "objective/train/unlikelihood_loss": 0.021864943206310272, + "theoretical_loss": 3.431069102930235, + "tokens_seen": 2207055872 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033453699245706946, + "loss": 1.3559, + "theoretical_loss": 3.431069102930235, + "tokens_seen": 2207055872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344968704862783, + "loss": 1.2917, + "theoretical_loss": 3.4310525796946516, + "tokens_seen": 2207186944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033445674851548707, + "loss": 1.2245, + "theoretical_loss": 3.431036057714981, + "tokens_seen": 2207318016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344166265446959, + "loss": 1.3348, + "theoretical_loss": 3.4310195369910526, + "tokens_seen": 2207449088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343765045739047, + "loss": 1.2997, + "theoretical_loss": 3.431003017522697, + "tokens_seen": 2207580160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343363826031135, + "loss": 1.3097, + "theoretical_loss": 3.4309864993097436, + "tokens_seen": 2207711232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033429626063232224, + "loss": 1.2927, + "theoretical_loss": 3.430969982352023, + "tokens_seen": 2207842304 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033425613866153105, + "loss": 1.266, + "theoretical_loss": 3.4309534666493646, + "tokens_seen": 2207973376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033421601669073985, + "loss": 1.3661, + "theoretical_loss": 3.4309369522015993, + "tokens_seen": 2208104448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033417589471994866, + "loss": 1.2473, + "theoretical_loss": 3.430920439008557, + "tokens_seen": 2208235520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033413577274915747, + "loss": 1.303, + "theoretical_loss": 3.4309039270700676, + "tokens_seen": 2208366592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340956507783662, + "loss": 1.2499, + "theoretical_loss": 3.430887416385962, + "tokens_seen": 2208497664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340555288075751, + "loss": 1.2908, + "theoretical_loss": 3.43087090695607, + "tokens_seen": 2208628736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033401540683678383, + "loss": 1.3174, + "theoretical_loss": 3.4308543987802214, + "tokens_seen": 2208759808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339752848659926, + "loss": 1.3345, + "theoretical_loss": 3.430837891858247, + "tokens_seen": 2208890880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033393516289520144, + "loss": 1.3253, + "theoretical_loss": 3.430821386189978, + "tokens_seen": 2209021952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338950409244102, + "loss": 1.2813, + "theoretical_loss": 3.4308048817752437, + "tokens_seen": 2209153024 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333854918953619, + "loss": 1.3924, + "theoretical_loss": 3.430788378613875, + "tokens_seen": 2209284096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338147969828278, + "loss": 1.3568, + "theoretical_loss": 3.4307718767057023, + "tokens_seen": 2209415168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337746750120366, + "loss": 1.3574, + "theoretical_loss": 3.4307553760505556, + "tokens_seen": 2209546240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033373455304124537, + "loss": 1.2417, + "theoretical_loss": 3.4307388766482667, + "tokens_seen": 2209677312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336944310704542, + "loss": 1.3303, + "theoretical_loss": 3.4307223784986647, + "tokens_seen": 2209808384 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333654309099663, + "loss": 1.2906, + "theoretical_loss": 3.430705881601581, + "tokens_seen": 2209939456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336141871288718, + "loss": 1.2673, + "theoretical_loss": 3.430689385956846, + "tokens_seen": 2210070528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335740651580806, + "loss": 1.2948, + "theoretical_loss": 3.430672891564291, + "tokens_seen": 2210201600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033353394318728934, + "loss": 1.2566, + "theoretical_loss": 3.4306563984237455, + "tokens_seen": 2210332672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334938212164982, + "loss": 1.2983, + "theoretical_loss": 3.430639906535041, + "tokens_seen": 2210463744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033345369924570696, + "loss": 1.313, + "theoretical_loss": 3.4306234158980082, + "tokens_seen": 2210594816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334135772749157, + "loss": 1.2817, + "theoretical_loss": 3.4306069265124783, + "tokens_seen": 2210725888 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033337345530412457, + "loss": 1.246, + "theoretical_loss": 3.4305904383782813, + "tokens_seen": 2210856960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333333333333333, + "loss": 1.2919, + "theoretical_loss": 3.4305739514952482, + "tokens_seen": 2210988032 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033329321136254213, + "loss": 1.3618, + "theoretical_loss": 3.4305574658632105, + "tokens_seen": 2211119104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033325308939175093, + "loss": 1.3495, + "theoretical_loss": 3.4305409814819985, + "tokens_seen": 2211250176 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033321296742095974, + "loss": 1.2614, + "theoretical_loss": 3.430524498351444, + "tokens_seen": 2211381248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331728454501685, + "loss": 1.3684, + "theoretical_loss": 3.430508016471377, + "tokens_seen": 2211512320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331327234793773, + "loss": 1.2552, + "theoretical_loss": 3.430491535841629, + "tokens_seen": 2211643392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330926015085861, + "loss": 1.3443, + "theoretical_loss": 3.4304750564620305, + "tokens_seen": 2211774464 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033305247953779486, + "loss": 1.408, + "theoretical_loss": 3.4304585783324133, + "tokens_seen": 2211905536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330123575670037, + "loss": 1.3004, + "theoretical_loss": 3.4304421014526083, + "tokens_seen": 2212036608 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033297223559621247, + "loss": 1.2759, + "theoretical_loss": 3.430425625822447, + "tokens_seen": 2212167680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033293211362542133, + "loss": 1.2849, + "theoretical_loss": 3.4304091514417605, + "tokens_seen": 2212298752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328919916546301, + "loss": 1.2896, + "theoretical_loss": 3.430392678310379, + "tokens_seen": 2212429824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328518696838389, + "loss": 1.3162, + "theoretical_loss": 3.430376206428135, + "tokens_seen": 2212560896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328117477130477, + "loss": 1.341, + "theoretical_loss": 3.4303597357948594, + "tokens_seen": 2212691968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033277162574225645, + "loss": 1.3008, + "theoretical_loss": 3.4303432664103832, + "tokens_seen": 2212823040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033273150377146525, + "loss": 1.3719, + "theoretical_loss": 3.4303267982745385, + "tokens_seen": 2212954112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033269138180067406, + "loss": 1.2473, + "theoretical_loss": 3.4303103313871555, + "tokens_seen": 2213085184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033265125982988287, + "loss": 1.306, + "theoretical_loss": 3.430293865748067, + "tokens_seen": 2213216256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326111378590916, + "loss": 1.2768, + "theoretical_loss": 3.4302774013571034, + "tokens_seen": 2213347328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325710158883005, + "loss": 1.2536, + "theoretical_loss": 3.4302609382140967, + "tokens_seen": 2213478400 + }, + { + "epoch": 0.34, + "objective/train/aligned_token_count": 0.9922282099723816, + "objective/train/avg_token_score": 0.007771804928779602, + "objective/train/docs_used": 805966, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3094998598098755, + "objective/train/original_loss": 1.3202848434448242, + "objective/train/theoretical_loss": 3.430244476318878, + "objective/train/tokens_used": 584068576, + "objective/train/unlikelihood_loss": 0.021373042836785316, + "theoretical_loss": 3.430244476318878, + "tokens_seen": 2213609472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033253089391750923, + "loss": 1.3322, + "theoretical_loss": 3.430244476318878, + "tokens_seen": 2213609472 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332490771946718, + "loss": 1.3093, + "theoretical_loss": 3.4302280156712794, + "tokens_seen": 2213740544 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033245064997592684, + "loss": 1.3262, + "theoretical_loss": 3.430211556271132, + "tokens_seen": 2213871616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324105280051356, + "loss": 1.2976, + "theoretical_loss": 3.4301950981182676, + "tokens_seen": 2214002688 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033237040603434446, + "loss": 1.2688, + "theoretical_loss": 3.4301786412125184, + "tokens_seen": 2214133760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323302840635532, + "loss": 1.3045, + "theoretical_loss": 3.4301621855537148, + "tokens_seen": 2214264832 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332290162092762, + "loss": 1.2891, + "theoretical_loss": 3.43014573114169, + "tokens_seen": 2214395904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322500401219708, + "loss": 1.2229, + "theoretical_loss": 3.4301292779762744, + "tokens_seen": 2214526976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322099181511796, + "loss": 1.335, + "theoretical_loss": 3.4301128260573006, + "tokens_seen": 2214658048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321697961803884, + "loss": 1.3568, + "theoretical_loss": 3.4300963753846005, + "tokens_seen": 2214789120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321296742095972, + "loss": 1.2079, + "theoretical_loss": 3.4300799259580055, + "tokens_seen": 2214920192 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332089552238806, + "loss": 1.2716, + "theoretical_loss": 3.430063477777347, + "tokens_seen": 2215051264 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033204943026801474, + "loss": 1.268, + "theoretical_loss": 3.4300470308424584, + "tokens_seen": 2215182336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320093082972236, + "loss": 1.2254, + "theoretical_loss": 3.4300305851531703, + "tokens_seen": 2215313408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033196918632643236, + "loss": 1.2678, + "theoretical_loss": 3.4300141407093157, + "tokens_seen": 2215444480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319290643556411, + "loss": 1.3116, + "theoretical_loss": 3.429997697510726, + "tokens_seen": 2215575552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033188894238484997, + "loss": 1.2672, + "theoretical_loss": 3.429981255557233, + "tokens_seen": 2215706624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318488204140587, + "loss": 1.4102, + "theoretical_loss": 3.4299648148486686, + "tokens_seen": 2215837696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033180869844326753, + "loss": 1.3475, + "theoretical_loss": 3.429948375384866, + "tokens_seen": 2215968768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033176857647247633, + "loss": 1.2186, + "theoretical_loss": 3.4299319371656565, + "tokens_seen": 2216099840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033172845450168514, + "loss": 1.341, + "theoretical_loss": 3.429915500190873, + "tokens_seen": 2216230912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033168833253089395, + "loss": 1.3354, + "theoretical_loss": 3.4298990644603466, + "tokens_seen": 2216361984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316482105601027, + "loss": 1.2454, + "theoretical_loss": 3.4298826299739105, + "tokens_seen": 2216493056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316080885893115, + "loss": 1.2949, + "theoretical_loss": 3.429866196731396, + "tokens_seen": 2216624128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315679666185203, + "loss": 1.2613, + "theoretical_loss": 3.429849764732637, + "tokens_seen": 2216755200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315278446477291, + "loss": 1.2734, + "theoretical_loss": 3.4298333339774643, + "tokens_seen": 2216886272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033148772267693787, + "loss": 1.2454, + "theoretical_loss": 3.429816904465711, + "tokens_seen": 2217017344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033144760070614673, + "loss": 1.3131, + "theoretical_loss": 3.429800476197209, + "tokens_seen": 2217148416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314074787353555, + "loss": 1.2087, + "theoretical_loss": 3.4297840491717912, + "tokens_seen": 2217279488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313673567645643, + "loss": 1.3451, + "theoretical_loss": 3.42976762338929, + "tokens_seen": 2217410560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313272347937731, + "loss": 1.3116, + "theoretical_loss": 3.4297511988495377, + "tokens_seen": 2217541632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033128711282298185, + "loss": 1.2467, + "theoretical_loss": 3.429734775552367, + "tokens_seen": 2217672704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033124699085219065, + "loss": 1.268, + "theoretical_loss": 3.42971835349761, + "tokens_seen": 2217803776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033120686888139946, + "loss": 1.2261, + "theoretical_loss": 3.4297019326851004, + "tokens_seen": 2217934848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033116674691060827, + "loss": 1.327, + "theoretical_loss": 3.4296855131146695, + "tokens_seen": 2218065920 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331126624939817, + "loss": 1.2708, + "theoretical_loss": 3.429669094786151, + "tokens_seen": 2218196992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310865029690259, + "loss": 1.3063, + "theoretical_loss": 3.429652677699377, + "tokens_seen": 2218328064 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033104638099823463, + "loss": 1.317, + "theoretical_loss": 3.4296362618541805, + "tokens_seen": 2218459136 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033100625902744344, + "loss": 1.2895, + "theoretical_loss": 3.429619847250394, + "tokens_seen": 2218590208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033096613705665224, + "loss": 1.2915, + "theoretical_loss": 3.4296034338878503, + "tokens_seen": 2218721280 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330926015085861, + "loss": 1.2415, + "theoretical_loss": 3.4295870217663826, + "tokens_seen": 2218852352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033088589311506986, + "loss": 1.2818, + "theoretical_loss": 3.4295706108858233, + "tokens_seen": 2218983424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308457711442786, + "loss": 1.2703, + "theoretical_loss": 3.4295542012460056, + "tokens_seen": 2219114496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003308056491734874, + "loss": 1.1961, + "theoretical_loss": 3.4295377928467623, + "tokens_seen": 2219245568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307655272026962, + "loss": 1.3534, + "theoretical_loss": 3.4295213856879263, + "tokens_seen": 2219376640 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330725405231905, + "loss": 1.2587, + "theoretical_loss": 3.4295049797693307, + "tokens_seen": 2219507712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306852832611138, + "loss": 1.2285, + "theoretical_loss": 3.4294885750908084, + "tokens_seen": 2219638784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306451612903226, + "loss": 1.3021, + "theoretical_loss": 3.4294721716521925, + "tokens_seen": 2219769856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306050393195314, + "loss": 1.2277, + "theoretical_loss": 3.429455769453316, + "tokens_seen": 2219900928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033056491734874014, + "loss": 1.2351, + "theoretical_loss": 3.4294393684940125, + "tokens_seen": 2220032000 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.989526629447937, + "objective/train/avg_token_score": 0.010473337024450302, + "objective/train/docs_used": 808099, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2712284326553345, + "objective/train/original_loss": 1.2837419509887695, + "objective/train/theoretical_loss": 3.4294229687741145, + "objective/train/tokens_used": 590622176, + "objective/train/unlikelihood_loss": 0.03499717637896538, + "theoretical_loss": 3.4294229687741145, + "tokens_seen": 2220163072 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330524795377949, + "loss": 1.2277, + "theoretical_loss": 3.4294229687741145, + "tokens_seen": 2220163072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033048467340715776, + "loss": 1.3138, + "theoretical_loss": 3.4294065702934553, + "tokens_seen": 2220294144 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033044455143636656, + "loss": 1.2616, + "theoretical_loss": 3.4293901730518686, + "tokens_seen": 2220425216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033040442946557537, + "loss": 1.3023, + "theoretical_loss": 3.4293737770491868, + "tokens_seen": 2220556288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303643074947841, + "loss": 1.2691, + "theoretical_loss": 3.429357382285244, + "tokens_seen": 2220687360 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330324185523993, + "loss": 1.3564, + "theoretical_loss": 3.429340988759873, + "tokens_seen": 2220818432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033028406355320173, + "loss": 1.3188, + "theoretical_loss": 3.429324596472907, + "tokens_seen": 2220949504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033024394158241054, + "loss": 1.3547, + "theoretical_loss": 3.4293082054241797, + "tokens_seen": 2221080576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033020381961161935, + "loss": 1.2835, + "theoretical_loss": 3.429291815613525, + "tokens_seen": 2221211648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301636976408281, + "loss": 1.2524, + "theoretical_loss": 3.4292754270407753, + "tokens_seen": 2221342720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301235756700369, + "loss": 1.3113, + "theoretical_loss": 3.4292590397057645, + "tokens_seen": 2221473792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300834536992457, + "loss": 1.2693, + "theoretical_loss": 3.4292426536083265, + "tokens_seen": 2221604864 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300433317284545, + "loss": 1.2766, + "theoretical_loss": 3.429226268748294, + "tokens_seen": 2221735936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033000320975766327, + "loss": 1.2541, + "theoretical_loss": 3.429209885125502, + "tokens_seen": 2221867008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032996308778687213, + "loss": 1.2497, + "theoretical_loss": 3.429193502739782, + "tokens_seen": 2221998080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299229658160809, + "loss": 1.2569, + "theoretical_loss": 3.4291771215909694, + "tokens_seen": 2222129152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298828438452897, + "loss": 1.3383, + "theoretical_loss": 3.4291607416788974, + "tokens_seen": 2222260224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298427218744985, + "loss": 1.3106, + "theoretical_loss": 3.4291443630033993, + "tokens_seen": 2222391296 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032980259990370725, + "loss": 1.348, + "theoretical_loss": 3.4291279855643086, + "tokens_seen": 2222522368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297624779329161, + "loss": 1.2709, + "theoretical_loss": 3.42911160936146, + "tokens_seen": 2222653440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032972235596212486, + "loss": 1.285, + "theoretical_loss": 3.429095234394687, + "tokens_seen": 2222784512 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032968223399133367, + "loss": 1.274, + "theoretical_loss": 3.429078860663823, + "tokens_seen": 2222915584 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032964211202054247, + "loss": 1.3239, + "theoretical_loss": 3.429062488168702, + "tokens_seen": 2223046656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296019900497513, + "loss": 1.2511, + "theoretical_loss": 3.429046116909158, + "tokens_seen": 2223177728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032956186807896003, + "loss": 1.23, + "theoretical_loss": 3.429029746885025, + "tokens_seen": 2223308800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032952174610816884, + "loss": 1.345, + "theoretical_loss": 3.4290133780961365, + "tokens_seen": 2223439872 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032948162413737764, + "loss": 1.3242, + "theoretical_loss": 3.428997010542327, + "tokens_seen": 2223570944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294415021665864, + "loss": 1.3066, + "theoretical_loss": 3.4289806442234303, + "tokens_seen": 2223702016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032940138019579526, + "loss": 1.3311, + "theoretical_loss": 3.4289642791392803, + "tokens_seen": 2223833088 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329361258225004, + "loss": 1.4033, + "theoretical_loss": 3.4289479152897115, + "tokens_seen": 2223964160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293211362542128, + "loss": 1.2383, + "theoretical_loss": 3.4289315526745576, + "tokens_seen": 2224095232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292810142834216, + "loss": 1.3179, + "theoretical_loss": 3.428915191293653, + "tokens_seen": 2224226304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032924089231263037, + "loss": 1.4055, + "theoretical_loss": 3.428898831146832, + "tokens_seen": 2224357376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032920077034183923, + "loss": 1.2175, + "theoretical_loss": 3.428882472233928, + "tokens_seen": 2224488448 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329160648371048, + "loss": 1.2856, + "theoretical_loss": 3.4288661145547765, + "tokens_seen": 2224619520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291205264002568, + "loss": 1.2619, + "theoretical_loss": 3.4288497581092106, + "tokens_seen": 2224750592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290804044294656, + "loss": 1.2998, + "theoretical_loss": 3.428833402897065, + "tokens_seen": 2224881664 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290402824586744, + "loss": 1.3727, + "theoretical_loss": 3.4288170489181744, + "tokens_seen": 2225012736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032900016048788316, + "loss": 1.3468, + "theoretical_loss": 3.428800696172373, + "tokens_seen": 2225143808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032896003851709196, + "loss": 1.2983, + "theoretical_loss": 3.428784344659495, + "tokens_seen": 2225274880 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032891991654630077, + "loss": 1.2845, + "theoretical_loss": 3.4287679943793745, + "tokens_seen": 2225405952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288797945755095, + "loss": 1.2689, + "theoretical_loss": 3.4287516453318467, + "tokens_seen": 2225537024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288396726047184, + "loss": 1.2754, + "theoretical_loss": 3.428735297516746, + "tokens_seen": 2225668096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032879955063392713, + "loss": 1.2929, + "theoretical_loss": 3.4287189509339067, + "tokens_seen": 2225799168 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032875942866313594, + "loss": 1.2848, + "theoretical_loss": 3.4287026055831626, + "tokens_seen": 2225930240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032871930669234475, + "loss": 1.2274, + "theoretical_loss": 3.4286862614643496, + "tokens_seen": 2226061312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286791847215535, + "loss": 1.303, + "theoretical_loss": 3.428669918577302, + "tokens_seen": 2226192384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286390627507623, + "loss": 1.2798, + "theoretical_loss": 3.428653576921854, + "tokens_seen": 2226323456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285989407799711, + "loss": 1.3759, + "theoretical_loss": 3.42863723649784, + "tokens_seen": 2226454528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285588188091799, + "loss": 1.1866, + "theoretical_loss": 3.428620897305096, + "tokens_seen": 2226585600 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.9919843077659607, + "objective/train/avg_token_score": 0.008015724830329418, + "objective/train/docs_used": 810670, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2583391666412354, + "objective/train/original_loss": 1.2699575424194336, + "objective/train/theoretical_loss": 3.428604559343456, + "objective/train/tokens_used": 597175776, + "objective/train/unlikelihood_loss": 0.0215164702385664, + "theoretical_loss": 3.428604559343456, + "tokens_seen": 2226716672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285186968383887, + "loss": 1.2935, + "theoretical_loss": 3.428604559343456, + "tokens_seen": 2226716672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032847857486759753, + "loss": 1.2749, + "theoretical_loss": 3.4285882226127544, + "tokens_seen": 2226847744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284384528968063, + "loss": 1.2911, + "theoretical_loss": 3.428571887112826, + "tokens_seen": 2226978816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283983309260151, + "loss": 1.3316, + "theoretical_loss": 3.4285555528435068, + "tokens_seen": 2227109888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283582089552239, + "loss": 1.2578, + "theoretical_loss": 3.4285392198046303, + "tokens_seen": 2227240960 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032831808698443265, + "loss": 1.2866, + "theoretical_loss": 3.4285228879960323, + "tokens_seen": 2227372032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282779650136415, + "loss": 1.3026, + "theoretical_loss": 3.4285065574175473, + "tokens_seen": 2227503104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032823784304285026, + "loss": 1.2668, + "theoretical_loss": 3.428490228069011, + "tokens_seen": 2227634176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032819772107205907, + "loss": 1.3197, + "theoretical_loss": 3.428473899950257, + "tokens_seen": 2227765248 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032815759910126787, + "loss": 1.2287, + "theoretical_loss": 3.4284575730611215, + "tokens_seen": 2227896320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281174771304767, + "loss": 1.3317, + "theoretical_loss": 3.428441247401439, + "tokens_seen": 2228027392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032807735515968543, + "loss": 1.2825, + "theoretical_loss": 3.428424922971045, + "tokens_seen": 2228158464 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032803723318889424, + "loss": 1.2038, + "theoretical_loss": 3.428408599769775, + "tokens_seen": 2228289536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032799711121810304, + "loss": 1.3097, + "theoretical_loss": 3.428392277797463, + "tokens_seen": 2228420608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279569892473118, + "loss": 1.3041, + "theoretical_loss": 3.428375957053945, + "tokens_seen": 2228551680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032791686727652066, + "loss": 1.3267, + "theoretical_loss": 3.4283596375390557, + "tokens_seen": 2228682752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278767453057294, + "loss": 1.231, + "theoretical_loss": 3.4283433192526314, + "tokens_seen": 2228813824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032783662333493827, + "loss": 1.2844, + "theoretical_loss": 3.4283270021945063, + "tokens_seen": 2228944896 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327796501364147, + "loss": 1.2874, + "theoretical_loss": 3.428310686364516, + "tokens_seen": 2229075968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032775637939335577, + "loss": 1.3406, + "theoretical_loss": 3.428294371762496, + "tokens_seen": 2229207040 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032771625742256463, + "loss": 1.299, + "theoretical_loss": 3.4282780583882815, + "tokens_seen": 2229338112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276761354517734, + "loss": 1.3465, + "theoretical_loss": 3.4282617462417084, + "tokens_seen": 2229469184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276360134809822, + "loss": 1.3026, + "theoretical_loss": 3.428245435322612, + "tokens_seen": 2229600256 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327595891510191, + "loss": 1.2462, + "theoretical_loss": 3.4282291256308275, + "tokens_seen": 2229731328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275557695393998, + "loss": 1.3167, + "theoretical_loss": 3.42821281716619, + "tokens_seen": 2229862400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032751564756860856, + "loss": 1.3028, + "theoretical_loss": 3.428196509928536, + "tokens_seen": 2229993472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032747552559781736, + "loss": 1.3002, + "theoretical_loss": 3.4281802039177007, + "tokens_seen": 2230124544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032743540362702617, + "loss": 1.3182, + "theoretical_loss": 3.4281638991335197, + "tokens_seen": 2230255616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273952816562349, + "loss": 1.2558, + "theoretical_loss": 3.4281475955758287, + "tokens_seen": 2230386688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273551596854438, + "loss": 1.2989, + "theoretical_loss": 3.428131293244463, + "tokens_seen": 2230517760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032731503771465253, + "loss": 1.2897, + "theoretical_loss": 3.4281149921392586, + "tokens_seen": 2230648832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272749157438614, + "loss": 1.2942, + "theoretical_loss": 3.428098692260051, + "tokens_seen": 2230779904 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032723479377307015, + "loss": 1.2733, + "theoretical_loss": 3.4280823936066764, + "tokens_seen": 2230910976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271946718022789, + "loss": 1.3055, + "theoretical_loss": 3.4280660961789704, + "tokens_seen": 2231042048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032715454983148776, + "loss": 1.2388, + "theoretical_loss": 3.428049799976769, + "tokens_seen": 2231173120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271144278606965, + "loss": 1.2286, + "theoretical_loss": 3.4280335049999073, + "tokens_seen": 2231304192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270743058899053, + "loss": 1.2233, + "theoretical_loss": 3.428017211248222, + "tokens_seen": 2231435264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270341839191141, + "loss": 1.3195, + "theoretical_loss": 3.428000918721549, + "tokens_seen": 2231566336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032699406194832293, + "loss": 1.3539, + "theoretical_loss": 3.4279846274197237, + "tokens_seen": 2231697408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269539399775317, + "loss": 1.3361, + "theoretical_loss": 3.427968337342583, + "tokens_seen": 2231828480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269138180067405, + "loss": 1.2945, + "theoretical_loss": 3.4279520484899617, + "tokens_seen": 2231959552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268736960359493, + "loss": 1.32, + "theoretical_loss": 3.427935760861697, + "tokens_seen": 2232090624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032683357406515805, + "loss": 1.3253, + "theoretical_loss": 3.427919474457624, + "tokens_seen": 2232221696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267934520943669, + "loss": 1.3086, + "theoretical_loss": 3.4279031892775795, + "tokens_seen": 2232352768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032675333012357566, + "loss": 1.2629, + "theoretical_loss": 3.4278869053213996, + "tokens_seen": 2232483840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032671320815278447, + "loss": 1.2887, + "theoretical_loss": 3.4278706225889204, + "tokens_seen": 2232614912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032667308618199327, + "loss": 1.2742, + "theoretical_loss": 3.4278543410799776, + "tokens_seen": 2232745984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266329642112021, + "loss": 1.2866, + "theoretical_loss": 3.427838060794408, + "tokens_seen": 2232877056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265928422404109, + "loss": 1.28, + "theoretical_loss": 3.4278217817320478, + "tokens_seen": 2233008128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032655272026961964, + "loss": 1.2982, + "theoretical_loss": 3.4278055038927335, + "tokens_seen": 2233139200 + }, + { + "epoch": 0.35, + "objective/train/aligned_token_count": 0.9938318133354187, + "objective/train/avg_token_score": 0.006168222054839134, + "objective/train/docs_used": 813220, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2443104982376099, + "objective/train/original_loss": 1.252387523651123, + "objective/train/theoretical_loss": 3.4277892272763006, + "objective/train/tokens_used": 603729376, + "objective/train/unlikelihood_loss": 0.01799950934946537, + "theoretical_loss": 3.4277892272763006, + "tokens_seen": 2233270272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032651259829882844, + "loss": 1.308, + "theoretical_loss": 3.4277892272763006, + "tokens_seen": 2233270272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032647247632803725, + "loss": 1.3251, + "theoretical_loss": 3.4277729518825866, + "tokens_seen": 2233401344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032643235435724606, + "loss": 1.2388, + "theoretical_loss": 3.427756677711427, + "tokens_seen": 2233532416 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263922323864548, + "loss": 1.3124, + "theoretical_loss": 3.427740404762659, + "tokens_seen": 2233663488 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032635211041566367, + "loss": 1.2273, + "theoretical_loss": 3.427724133036118, + "tokens_seen": 2233794560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263119884448724, + "loss": 1.2449, + "theoretical_loss": 3.4277078625316415, + "tokens_seen": 2233925632 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032627186647408117, + "loss": 1.2668, + "theoretical_loss": 3.4276915932490657, + "tokens_seen": 2234056704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032623174450329003, + "loss": 1.348, + "theoretical_loss": 3.4276753251882273, + "tokens_seen": 2234187776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261916225324988, + "loss": 1.3531, + "theoretical_loss": 3.4276590583489623, + "tokens_seen": 2234318848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261515005617076, + "loss": 1.3397, + "theoretical_loss": 3.4276427927311075, + "tokens_seen": 2234449920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261113785909164, + "loss": 1.2732, + "theoretical_loss": 3.4276265283345, + "tokens_seen": 2234580992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260712566201252, + "loss": 1.2888, + "theoretical_loss": 3.4276102651589766, + "tokens_seen": 2234712064 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032603113464933396, + "loss": 1.2909, + "theoretical_loss": 3.4275940032043732, + "tokens_seen": 2234843136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032599101267854276, + "loss": 1.3857, + "theoretical_loss": 3.427577742470527, + "tokens_seen": 2234974208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032595089070775157, + "loss": 1.2697, + "theoretical_loss": 3.427561482957275, + "tokens_seen": 2235105280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259107687369604, + "loss": 1.2882, + "theoretical_loss": 3.427545224664454, + "tokens_seen": 2235236352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258706467661692, + "loss": 1.3547, + "theoretical_loss": 3.4275289675919, + "tokens_seen": 2235367424 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032583052479537793, + "loss": 1.25, + "theoretical_loss": 3.427512711739451, + "tokens_seen": 2235498496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003257904028245868, + "loss": 1.3111, + "theoretical_loss": 3.427496457106943, + "tokens_seen": 2235629568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032575028085379555, + "loss": 1.3388, + "theoretical_loss": 3.4274802036942136, + "tokens_seen": 2235760640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257101588830043, + "loss": 1.2906, + "theoretical_loss": 3.4274639515010996, + "tokens_seen": 2235891712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032567003691221316, + "loss": 1.2695, + "theoretical_loss": 3.4274477005274377, + "tokens_seen": 2236022784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256299149414219, + "loss": 1.2556, + "theoretical_loss": 3.427431450773065, + "tokens_seen": 2236153856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255897929706307, + "loss": 1.284, + "theoretical_loss": 3.427415202237819, + "tokens_seen": 2236284928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255496709998395, + "loss": 1.3159, + "theoretical_loss": 3.4273989549215362, + "tokens_seen": 2236416000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032550954902904833, + "loss": 1.2735, + "theoretical_loss": 3.4273827088240543, + "tokens_seen": 2236547072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254694270582571, + "loss": 1.3817, + "theoretical_loss": 3.4273664639452095, + "tokens_seen": 2236678144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254293050874659, + "loss": 1.3232, + "theoretical_loss": 3.42735022028484, + "tokens_seen": 2236809216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253891831166747, + "loss": 1.3234, + "theoretical_loss": 3.4273339778427827, + "tokens_seen": 2236940288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253490611458835, + "loss": 1.3125, + "theoretical_loss": 3.4273177366188747, + "tokens_seen": 2237071360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253089391750923, + "loss": 1.3146, + "theoretical_loss": 3.4273014966129534, + "tokens_seen": 2237202432 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032526881720430106, + "loss": 1.3725, + "theoretical_loss": 3.4272852578248556, + "tokens_seen": 2237333504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252286952335099, + "loss": 1.3252, + "theoretical_loss": 3.4272690202544194, + "tokens_seen": 2237464576 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032518857326271867, + "loss": 1.249, + "theoretical_loss": 3.427252783901482, + "tokens_seen": 2237595648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251484512919275, + "loss": 1.2905, + "theoretical_loss": 3.4272365487658805, + "tokens_seen": 2237726720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251083293211363, + "loss": 1.2896, + "theoretical_loss": 3.4272203148474527, + "tokens_seen": 2237857792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032506820735034504, + "loss": 1.3855, + "theoretical_loss": 3.427204082146036, + "tokens_seen": 2237988864 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032502808537955384, + "loss": 1.2897, + "theoretical_loss": 3.4271878506614675, + "tokens_seen": 2238119936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032498796340876265, + "loss": 1.3356, + "theoretical_loss": 3.427171620393585, + "tokens_seen": 2238251008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032494784143797145, + "loss": 1.3633, + "theoretical_loss": 3.427155391342226, + "tokens_seen": 2238382080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249077194671802, + "loss": 1.2857, + "theoretical_loss": 3.427139163507228, + "tokens_seen": 2238513152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032486759749638907, + "loss": 1.325, + "theoretical_loss": 3.427122936888429, + "tokens_seen": 2238644224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248274755255978, + "loss": 1.2923, + "theoretical_loss": 3.427106711485666, + "tokens_seen": 2238775296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032478735355480657, + "loss": 1.3505, + "theoretical_loss": 3.427090487298778, + "tokens_seen": 2238906368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032474723158401543, + "loss": 1.3546, + "theoretical_loss": 3.427074264327601, + "tokens_seen": 2239037440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247071096132242, + "loss": 1.2939, + "theoretical_loss": 3.4270580425719737, + "tokens_seen": 2239168512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032466698764243304, + "loss": 1.2411, + "theoretical_loss": 3.427041822031734, + "tokens_seen": 2239299584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246268656716418, + "loss": 1.3114, + "theoretical_loss": 3.427025602706719, + "tokens_seen": 2239430656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245867437008506, + "loss": 1.2676, + "theoretical_loss": 3.4270093845967673, + "tokens_seen": 2239561728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245466217300594, + "loss": 1.2622, + "theoretical_loss": 3.4269931677017165, + "tokens_seen": 2239692800 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.9927558302879333, + "objective/train/avg_token_score": 0.007244119420647621, + "objective/train/docs_used": 815683, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3123193979263306, + "objective/train/original_loss": 1.3212146759033203, + "objective/train/theoretical_loss": 3.426976952021404, + "objective/train/tokens_used": 610282976, + "objective/train/unlikelihood_loss": 0.02249067649245262, + "theoretical_loss": 3.426976952021404, + "tokens_seen": 2239823872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032450649975926816, + "loss": 1.289, + "theoretical_loss": 3.426976952021404, + "tokens_seen": 2239823872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032446637778847697, + "loss": 1.2655, + "theoretical_loss": 3.4269607375556683, + "tokens_seen": 2239954944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244262558176858, + "loss": 1.3369, + "theoretical_loss": 3.4269445243043477, + "tokens_seen": 2240086016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243861338468946, + "loss": 1.2827, + "theoretical_loss": 3.4269283122672793, + "tokens_seen": 2240217088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032434601187610333, + "loss": 1.3561, + "theoretical_loss": 3.4269121014443016, + "tokens_seen": 2240348160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243058899053122, + "loss": 1.3387, + "theoretical_loss": 3.426895891835253, + "tokens_seen": 2240479232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032426576793452095, + "loss": 1.3193, + "theoretical_loss": 3.426879683439971, + "tokens_seen": 2240610304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242256459637297, + "loss": 1.3037, + "theoretical_loss": 3.4268634762582937, + "tokens_seen": 2240741376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032418552399293856, + "loss": 1.2874, + "theoretical_loss": 3.426847270290059, + "tokens_seen": 2240872448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241454020221473, + "loss": 1.3177, + "theoretical_loss": 3.4268310655351066, + "tokens_seen": 2241003520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032410528005135617, + "loss": 1.286, + "theoretical_loss": 3.426814861993273, + "tokens_seen": 2241134592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240651580805649, + "loss": 1.2636, + "theoretical_loss": 3.4267986596643976, + "tokens_seen": 2241265664 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032402503610977373, + "loss": 1.2736, + "theoretical_loss": 3.4267824585483178, + "tokens_seen": 2241396736 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032398491413898254, + "loss": 1.2481, + "theoretical_loss": 3.4267662586448724, + "tokens_seen": 2241527808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239447921681913, + "loss": 1.2827, + "theoretical_loss": 3.4267500599538994, + "tokens_seen": 2241658880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239046701974001, + "loss": 1.2873, + "theoretical_loss": 3.4267338624752375, + "tokens_seen": 2241789952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238645482266089, + "loss": 1.3112, + "theoretical_loss": 3.426717666208725, + "tokens_seen": 2241921024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238244262558177, + "loss": 1.3539, + "theoretical_loss": 3.4267014711542005, + "tokens_seen": 2242052096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032378430428502646, + "loss": 1.2326, + "theoretical_loss": 3.4266852773115017, + "tokens_seen": 2242183168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237441823142353, + "loss": 1.3005, + "theoretical_loss": 3.4266690846804684, + "tokens_seen": 2242314240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032370406034344407, + "loss": 1.2659, + "theoretical_loss": 3.426652893260938, + "tokens_seen": 2242445312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236639383726528, + "loss": 1.3108, + "theoretical_loss": 3.4266367030527496, + "tokens_seen": 2242576384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236238164018617, + "loss": 1.2563, + "theoretical_loss": 3.4266205140557413, + "tokens_seen": 2242707456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032358369443107044, + "loss": 1.3521, + "theoretical_loss": 3.4266043262697523, + "tokens_seen": 2242838528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032354357246027924, + "loss": 1.2586, + "theoretical_loss": 3.426588139694621, + "tokens_seen": 2242969600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032350345048948805, + "loss": 1.2312, + "theoretical_loss": 3.4265719543301856, + "tokens_seen": 2243100672 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032346332851869685, + "loss": 1.2021, + "theoretical_loss": 3.426555770176286, + "tokens_seen": 2243231744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032342320654790566, + "loss": 1.3109, + "theoretical_loss": 3.4265395872327593, + "tokens_seen": 2243362816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032338308457711447, + "loss": 1.3185, + "theoretical_loss": 3.426523405499446, + "tokens_seen": 2243493888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233429626063232, + "loss": 1.3662, + "theoretical_loss": 3.4265072249761834, + "tokens_seen": 2243624960 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323302840635532, + "loss": 1.3599, + "theoretical_loss": 3.4264910456628117, + "tokens_seen": 2243756032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032326271866474083, + "loss": 1.2539, + "theoretical_loss": 3.4264748675591687, + "tokens_seen": 2243887104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232225966939496, + "loss": 1.3587, + "theoretical_loss": 3.4264586906650933, + "tokens_seen": 2244018176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032318247472315844, + "loss": 1.3002, + "theoretical_loss": 3.426442514980425, + "tokens_seen": 2244149248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231423527523672, + "loss": 1.3891, + "theoretical_loss": 3.426426340505003, + "tokens_seen": 2244280320 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323102230781576, + "loss": 1.3366, + "theoretical_loss": 3.4264101672386653, + "tokens_seen": 2244411392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230621088107848, + "loss": 1.2425, + "theoretical_loss": 3.4263939951812517, + "tokens_seen": 2244542464 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032302198683999356, + "loss": 1.303, + "theoretical_loss": 3.426377824332601, + "tokens_seen": 2244673536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032298186486920237, + "loss": 1.3012, + "theoretical_loss": 3.426361654692552, + "tokens_seen": 2244804608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229417428984112, + "loss": 1.3115, + "theoretical_loss": 3.426345486260944, + "tokens_seen": 2244935680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032290162092762, + "loss": 1.3258, + "theoretical_loss": 3.4263293190376167, + "tokens_seen": 2245066752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032286149895682873, + "loss": 1.3488, + "theoretical_loss": 3.4263131530224085, + "tokens_seen": 2245197824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228213769860376, + "loss": 1.2883, + "theoretical_loss": 3.426296988215159, + "tokens_seen": 2245328896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032278125501524634, + "loss": 1.304, + "theoretical_loss": 3.4262808246157066, + "tokens_seen": 2245459968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032274113304445515, + "loss": 1.3028, + "theoretical_loss": 3.426264662223892, + "tokens_seen": 2245591040 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032270101107366396, + "loss": 1.288, + "theoretical_loss": 3.4262485010395536, + "tokens_seen": 2245722112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226608891028727, + "loss": 1.368, + "theoretical_loss": 3.426232341062531, + "tokens_seen": 2245853184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032262076713208157, + "loss": 1.3445, + "theoretical_loss": 3.4262161822926633, + "tokens_seen": 2245984256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225806451612903, + "loss": 1.2385, + "theoretical_loss": 3.42620002472979, + "tokens_seen": 2246115328 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032254052319049913, + "loss": 1.2948, + "theoretical_loss": 3.4261838683737507, + "tokens_seen": 2246246400 + }, + { + "epoch": 0.36, + "objective/train/aligned_token_count": 0.9919527173042297, + "objective/train/avg_token_score": 0.008047258481383324, + "objective/train/docs_used": 817879, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.295349359512329, + "objective/train/original_loss": 1.3063161373138428, + "objective/train/theoretical_loss": 3.4261677132243844, + "objective/train/tokens_used": 616836576, + "objective/train/unlikelihood_loss": 0.0223594531416893, + "theoretical_loss": 3.4261677132243844, + "tokens_seen": 2246377472 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032250040121970793, + "loss": 1.2933, + "theoretical_loss": 3.4261677132243844, + "tokens_seen": 2246377472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224602792489167, + "loss": 1.328, + "theoretical_loss": 3.426151559281531, + "tokens_seen": 2246508544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224201572781255, + "loss": 1.222, + "theoretical_loss": 3.42613540654503, + "tokens_seen": 2246639616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223800353073343, + "loss": 1.3964, + "theoretical_loss": 3.4261192550147204, + "tokens_seen": 2246770688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223399133365431, + "loss": 1.37, + "theoretical_loss": 3.4261031046904424, + "tokens_seen": 2246901760 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032229979136575186, + "loss": 1.2799, + "theoretical_loss": 3.4260869555720355, + "tokens_seen": 2247032832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222596693949607, + "loss": 1.2722, + "theoretical_loss": 3.4260708076593396, + "tokens_seen": 2247163904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032221954742416947, + "loss": 1.2701, + "theoretical_loss": 3.426054660952193, + "tokens_seen": 2247294976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221794254533782, + "loss": 1.3287, + "theoretical_loss": 3.426038515450437, + "tokens_seen": 2247426048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221393034825871, + "loss": 1.2806, + "theoretical_loss": 3.426022371153911, + "tokens_seen": 2247557120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032209918151179584, + "loss": 1.2203, + "theoretical_loss": 3.426006228062454, + "tokens_seen": 2247688192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220590595410047, + "loss": 1.2777, + "theoretical_loss": 3.425990086175906, + "tokens_seen": 2247819264 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032201893757021345, + "loss": 1.3315, + "theoretical_loss": 3.425973945494108, + "tokens_seen": 2247950336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032197881559942225, + "loss": 1.3436, + "theoretical_loss": 3.425957806016898, + "tokens_seen": 2248081408 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032193869362863106, + "loss": 1.2921, + "theoretical_loss": 3.425941667744117, + "tokens_seen": 2248212480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032189857165783987, + "loss": 1.3071, + "theoretical_loss": 3.4259255306756042, + "tokens_seen": 2248343552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218584496870486, + "loss": 1.3102, + "theoretical_loss": 3.4259093948112005, + "tokens_seen": 2248474624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218183277162574, + "loss": 1.3336, + "theoretical_loss": 3.4258932601507457, + "tokens_seen": 2248605696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032177820574546623, + "loss": 1.3707, + "theoretical_loss": 3.425877126694079, + "tokens_seen": 2248736768 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321738083774675, + "loss": 1.2387, + "theoretical_loss": 3.425860994441041, + "tokens_seen": 2248867840 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032169796180388384, + "loss": 1.3482, + "theoretical_loss": 3.4258448633914718, + "tokens_seen": 2248998912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216578398330926, + "loss": 1.2429, + "theoretical_loss": 3.4258287335452113, + "tokens_seen": 2249129984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216177178623014, + "loss": 1.2949, + "theoretical_loss": 3.4258126049020996, + "tokens_seen": 2249261056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215775958915102, + "loss": 1.2314, + "theoretical_loss": 3.425796477461977, + "tokens_seen": 2249392128 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032153747392071896, + "loss": 1.2944, + "theoretical_loss": 3.4257803512246836, + "tokens_seen": 2249523200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214973519499278, + "loss": 1.2884, + "theoretical_loss": 3.4257642261900596, + "tokens_seen": 2249654272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214572299791366, + "loss": 1.2508, + "theoretical_loss": 3.4257481023579457, + "tokens_seen": 2249785344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214171080083454, + "loss": 1.3227, + "theoretical_loss": 3.425731979728181, + "tokens_seen": 2249916416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213769860375542, + "loss": 1.2917, + "theoretical_loss": 3.425715858300607, + "tokens_seen": 2250047488 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321336864066763, + "loss": 1.3094, + "theoretical_loss": 3.425699738075063, + "tokens_seen": 2250178560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032129674209597174, + "loss": 1.3528, + "theoretical_loss": 3.425683619051391, + "tokens_seen": 2250309632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032125662012518055, + "loss": 1.2243, + "theoretical_loss": 3.4256675012294293, + "tokens_seen": 2250440704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032121649815438936, + "loss": 1.2334, + "theoretical_loss": 3.42565138460902, + "tokens_seen": 2250571776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211763761835981, + "loss": 1.3121, + "theoretical_loss": 3.425635269190003, + "tokens_seen": 2250702848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032113625421280697, + "loss": 1.2354, + "theoretical_loss": 3.4256191549722184, + "tokens_seen": 2250833920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210961322420157, + "loss": 1.3387, + "theoretical_loss": 3.4256030419555072, + "tokens_seen": 2250964992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032105601027122453, + "loss": 1.3572, + "theoretical_loss": 3.4255869301397097, + "tokens_seen": 2251096064 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032101588830043333, + "loss": 1.2917, + "theoretical_loss": 3.4255708195246664, + "tokens_seen": 2251227136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209757663296421, + "loss": 1.3231, + "theoretical_loss": 3.425554710110218, + "tokens_seen": 2251358208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209356443588509, + "loss": 1.2988, + "theoretical_loss": 3.4255386018962057, + "tokens_seen": 2251489280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208955223880597, + "loss": 1.3001, + "theoretical_loss": 3.425522494882469, + "tokens_seen": 2251620352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208554004172685, + "loss": 1.2992, + "theoretical_loss": 3.42550638906885, + "tokens_seen": 2251751424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208152784464773, + "loss": 1.2853, + "theoretical_loss": 3.425490284455188, + "tokens_seen": 2251882496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207751564756861, + "loss": 1.297, + "theoretical_loss": 3.4254741810413254, + "tokens_seen": 2252013568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032073503450489487, + "loss": 1.2784, + "theoretical_loss": 3.4254580788271016, + "tokens_seen": 2252144640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206949125341037, + "loss": 1.314, + "theoretical_loss": 3.4254419778123575, + "tokens_seen": 2252275712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206547905633125, + "loss": 1.279, + "theoretical_loss": 3.425425877996935, + "tokens_seen": 2252406784 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032061466859252123, + "loss": 1.2959, + "theoretical_loss": 3.425409779380674, + "tokens_seen": 2252537856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205745466217301, + "loss": 1.2683, + "theoretical_loss": 3.4253936819634156, + "tokens_seen": 2252668928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032053442465093885, + "loss": 1.2721, + "theoretical_loss": 3.425377585745001, + "tokens_seen": 2252800000 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.9918113946914673, + "objective/train/avg_token_score": 0.008188593201339245, + "objective/train/docs_used": 820362, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3665231466293335, + "objective/train/original_loss": 1.3780059814453125, + "objective/train/theoretical_loss": 3.4253614907252707, + "objective/train/tokens_used": 623390176, + "objective/train/unlikelihood_loss": 0.023288458585739136, + "theoretical_loss": 3.4253614907252707, + "tokens_seen": 2252931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032049430268014765, + "loss": 1.3231, + "theoretical_loss": 3.4253614907252707, + "tokens_seen": 2252931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032045418070935646, + "loss": 1.2753, + "theoretical_loss": 3.4253453969040666, + "tokens_seen": 2253062144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204140587385652, + "loss": 1.2837, + "theoretical_loss": 3.425329304281229, + "tokens_seen": 2253193216 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320373936767774, + "loss": 1.3156, + "theoretical_loss": 3.425313212856599, + "tokens_seen": 2253324288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203338147969828, + "loss": 1.269, + "theoretical_loss": 3.425297122630018, + "tokens_seen": 2253455360 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032029369282619163, + "loss": 1.2708, + "theoretical_loss": 3.425281033601327, + "tokens_seen": 2253586432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032025357085540044, + "loss": 1.3395, + "theoretical_loss": 3.425264945770367, + "tokens_seen": 2253717504 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032021344888460924, + "loss": 1.2998, + "theoretical_loss": 3.4252488591369796, + "tokens_seen": 2253848576 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320173326913818, + "loss": 1.3745, + "theoretical_loss": 3.4252327737010053, + "tokens_seen": 2253979648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032013320494302686, + "loss": 1.3324, + "theoretical_loss": 3.425216689462286, + "tokens_seen": 2254110720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200930829722356, + "loss": 1.2791, + "theoretical_loss": 3.425200606420663, + "tokens_seen": 2254241792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032005296100144436, + "loss": 1.2765, + "theoretical_loss": 3.425184524575977, + "tokens_seen": 2254372864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200128390306532, + "loss": 1.3268, + "theoretical_loss": 3.4251684439280696, + "tokens_seen": 2254503936 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319972717059862, + "loss": 1.2834, + "theoretical_loss": 3.4251523644767827, + "tokens_seen": 2254635008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199325950890708, + "loss": 1.2495, + "theoretical_loss": 3.425136286221957, + "tokens_seen": 2254766080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198924731182796, + "loss": 1.3278, + "theoretical_loss": 3.4251202091634343, + "tokens_seen": 2254897152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198523511474884, + "loss": 1.3388, + "theoretical_loss": 3.425104133301056, + "tokens_seen": 2255028224 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031981222917669714, + "loss": 1.3414, + "theoretical_loss": 3.4250880586346635, + "tokens_seen": 2255159296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031977210720590595, + "loss": 1.2796, + "theoretical_loss": 3.4250719851640983, + "tokens_seen": 2255290368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031973198523511476, + "loss": 1.2803, + "theoretical_loss": 3.425055912889202, + "tokens_seen": 2255421440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196918632643235, + "loss": 1.3249, + "theoretical_loss": 3.4250398418098165, + "tokens_seen": 2255552512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031965174129353237, + "loss": 1.2853, + "theoretical_loss": 3.4250237719257823, + "tokens_seen": 2255683584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196116193227411, + "loss": 1.2968, + "theoretical_loss": 3.4250077032369424, + "tokens_seen": 2255814656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031957149735195, + "loss": 1.2499, + "theoretical_loss": 3.424991635743138, + "tokens_seen": 2255945728 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031953137538115873, + "loss": 1.3222, + "theoretical_loss": 3.4249755694442108, + "tokens_seen": 2256076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194912534103675, + "loss": 1.2613, + "theoretical_loss": 3.424959504340002, + "tokens_seen": 2256207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031945113143957635, + "loss": 1.273, + "theoretical_loss": 3.4249434404303543, + "tokens_seen": 2256338944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194110094687851, + "loss": 1.3106, + "theoretical_loss": 3.4249273777151084, + "tokens_seen": 2256470016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193708874979939, + "loss": 1.3195, + "theoretical_loss": 3.424911316194107, + "tokens_seen": 2256601088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193307655272027, + "loss": 1.2954, + "theoretical_loss": 3.4248952558671917, + "tokens_seen": 2256732160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192906435564115, + "loss": 1.3341, + "theoretical_loss": 3.4248791967342043, + "tokens_seen": 2256863232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031925052158562027, + "loss": 1.3149, + "theoretical_loss": 3.424863138794987, + "tokens_seen": 2256994304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192103996148291, + "loss": 1.267, + "theoretical_loss": 3.4248470820493813, + "tokens_seen": 2257125376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191702776440379, + "loss": 1.3267, + "theoretical_loss": 3.424831026497229, + "tokens_seen": 2257256448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031913015567324663, + "loss": 1.3033, + "theoretical_loss": 3.424814972138373, + "tokens_seen": 2257387520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190900337024555, + "loss": 1.2744, + "theoretical_loss": 3.4247989189726544, + "tokens_seen": 2257518592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031904991173166425, + "loss": 1.2911, + "theoretical_loss": 3.424782866999916, + "tokens_seen": 2257649664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190097897608731, + "loss": 1.3056, + "theoretical_loss": 3.424766816219999, + "tokens_seen": 2257780736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031896966779008186, + "loss": 1.2645, + "theoretical_loss": 3.424750766632747, + "tokens_seen": 2257911808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189295458192906, + "loss": 1.2809, + "theoretical_loss": 3.4247347182380006, + "tokens_seen": 2258042880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031888942384849947, + "loss": 1.3374, + "theoretical_loss": 3.4247186710356026, + "tokens_seen": 2258173952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188493018777082, + "loss": 1.3605, + "theoretical_loss": 3.4247026250253954, + "tokens_seen": 2258305024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031880917990691703, + "loss": 1.3159, + "theoretical_loss": 3.424686580207221, + "tokens_seen": 2258436096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031876905793612584, + "loss": 1.3231, + "theoretical_loss": 3.424670536580922, + "tokens_seen": 2258567168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031872893596533464, + "loss": 1.2867, + "theoretical_loss": 3.42465449414634, + "tokens_seen": 2258698240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186888139945434, + "loss": 1.2818, + "theoretical_loss": 3.424638452903318, + "tokens_seen": 2258829312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031864869202375226, + "loss": 1.2359, + "theoretical_loss": 3.424622412851698, + "tokens_seen": 2258960384 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318608570052961, + "loss": 1.3148, + "theoretical_loss": 3.424606373991322, + "tokens_seen": 2259091456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031856844808216976, + "loss": 1.2245, + "theoretical_loss": 3.424590336322034, + "tokens_seen": 2259222528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185283261113786, + "loss": 1.31, + "theoretical_loss": 3.424574299843675, + "tokens_seen": 2259353600 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.9913434982299805, + "objective/train/avg_token_score": 0.008656485006213188, + "objective/train/docs_used": 822867, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3417028188705444, + "objective/train/original_loss": 1.3554576635360718, + "objective/train/theoretical_loss": 3.4245582645560875, + "objective/train/tokens_used": 629943776, + "objective/train/unlikelihood_loss": 0.02174399234354496, + "theoretical_loss": 3.4245582645560875, + "tokens_seen": 2259484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184882041405874, + "loss": 1.3115, + "theoretical_loss": 3.4245582645560875, + "tokens_seen": 2259484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184480821697962, + "loss": 1.2422, + "theoretical_loss": 3.4245422304591147, + "tokens_seen": 2259615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318407960199005, + "loss": 1.2718, + "theoretical_loss": 3.424526197552599, + "tokens_seen": 2259746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183678382282138, + "loss": 1.317, + "theoretical_loss": 3.4245101658363826, + "tokens_seen": 2259877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183277162574226, + "loss": 1.2476, + "theoretical_loss": 3.424494135310308, + "tokens_seen": 2260008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031828759428663135, + "loss": 1.2715, + "theoretical_loss": 3.4244781059742184, + "tokens_seen": 2260140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031824747231584016, + "loss": 1.2392, + "theoretical_loss": 3.4244620778279566, + "tokens_seen": 2260271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031820735034504896, + "loss": 1.2946, + "theoretical_loss": 3.4244460508713646, + "tokens_seen": 2260402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031816722837425777, + "loss": 1.253, + "theoretical_loss": 3.424430025104285, + "tokens_seen": 2260533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181271064034665, + "loss": 1.2898, + "theoretical_loss": 3.4244140005265615, + "tokens_seen": 2260664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180869844326754, + "loss": 1.3818, + "theoretical_loss": 3.424397977138036, + "tokens_seen": 2260795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031804686246188413, + "loss": 1.3508, + "theoretical_loss": 3.4243819549385517, + "tokens_seen": 2260926464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180067404910929, + "loss": 1.2744, + "theoretical_loss": 3.4243659339279517, + "tokens_seen": 2261057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031796661852030175, + "loss": 1.3255, + "theoretical_loss": 3.4243499141060783, + "tokens_seen": 2261188608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179264965495105, + "loss": 1.2596, + "theoretical_loss": 3.424333895472775, + "tokens_seen": 2261319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178863745787193, + "loss": 1.293, + "theoretical_loss": 3.4243178780278836, + "tokens_seen": 2261450752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178462526079281, + "loss": 1.3053, + "theoretical_loss": 3.424301861771249, + "tokens_seen": 2261581824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178061306371369, + "loss": 1.3241, + "theoretical_loss": 3.424285846702712, + "tokens_seen": 2261712896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031776600866634567, + "loss": 1.3718, + "theoretical_loss": 3.4242698328221173, + "tokens_seen": 2261843968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177258866955545, + "loss": 1.2897, + "theoretical_loss": 3.424253820129307, + "tokens_seen": 2261975040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176857647247633, + "loss": 1.1949, + "theoretical_loss": 3.4242378086241247, + "tokens_seen": 2262106112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176456427539721, + "loss": 1.315, + "theoretical_loss": 3.424221798306413, + "tokens_seen": 2262237184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176055207831809, + "loss": 1.3075, + "theoretical_loss": 3.424205789176016, + "tokens_seen": 2262368256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031756539881238965, + "loss": 1.392, + "theoretical_loss": 3.4241897812327755, + "tokens_seen": 2262499328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175252768415985, + "loss": 1.3018, + "theoretical_loss": 3.424173774476536, + "tokens_seen": 2262630400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031748515487080726, + "loss": 1.296, + "theoretical_loss": 3.4241577689071394, + "tokens_seen": 2262761472 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317445032900016, + "loss": 1.3072, + "theoretical_loss": 3.42414176452443, + "tokens_seen": 2262892544 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031740491092922487, + "loss": 1.2837, + "theoretical_loss": 3.424125761328251, + "tokens_seen": 2263023616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173647889584336, + "loss": 1.225, + "theoretical_loss": 3.4241097593184455, + "tokens_seen": 2263154688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031732466698764243, + "loss": 1.2963, + "theoretical_loss": 3.424093758494857, + "tokens_seen": 2263285760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031728454501685124, + "loss": 1.3004, + "theoretical_loss": 3.4240777588573286, + "tokens_seen": 2263416832 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031724442304606004, + "loss": 1.2894, + "theoretical_loss": 3.4240617604057038, + "tokens_seen": 2263547904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172043010752688, + "loss": 1.3589, + "theoretical_loss": 3.424045763139826, + "tokens_seen": 2263678976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031716417910447766, + "loss": 1.3245, + "theoretical_loss": 3.4240297670595385, + "tokens_seen": 2263810048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171240571336864, + "loss": 1.3451, + "theoretical_loss": 3.4240137721646855, + "tokens_seen": 2263941120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170839351628952, + "loss": 1.3151, + "theoretical_loss": 3.42399777845511, + "tokens_seen": 2264072192 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317043813192104, + "loss": 1.2958, + "theoretical_loss": 3.4239817859306556, + "tokens_seen": 2264203264 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031700369122131277, + "loss": 1.3181, + "theoretical_loss": 3.423965794591166, + "tokens_seen": 2264334336 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031696356925052163, + "loss": 1.2466, + "theoretical_loss": 3.4239498044364844, + "tokens_seen": 2264465408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169234472797304, + "loss": 1.2559, + "theoretical_loss": 3.423933815466455, + "tokens_seen": 2264596480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168833253089392, + "loss": 1.3093, + "theoretical_loss": 3.4239178276809215, + "tokens_seen": 2264727552 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316843203338148, + "loss": 1.3351, + "theoretical_loss": 3.4239018410797266, + "tokens_seen": 2264858624 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031680308136735675, + "loss": 1.3413, + "theoretical_loss": 3.423885855662715, + "tokens_seen": 2264989696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031676295939656556, + "loss": 1.3602, + "theoretical_loss": 3.423869871429731, + "tokens_seen": 2265120768 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031672283742577436, + "loss": 1.3208, + "theoretical_loss": 3.4238538883806173, + "tokens_seen": 2265251840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031668271545498317, + "loss": 1.3189, + "theoretical_loss": 3.423837906515218, + "tokens_seen": 2265382912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166425934841919, + "loss": 1.248, + "theoretical_loss": 3.423821925833377, + "tokens_seen": 2265513984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166024715134008, + "loss": 1.2921, + "theoretical_loss": 3.4238059463349377, + "tokens_seen": 2265645056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031656234954260953, + "loss": 1.2353, + "theoretical_loss": 3.423789968019745, + "tokens_seen": 2265776128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165222275718183, + "loss": 1.3558, + "theoretical_loss": 3.4237739908876423, + "tokens_seen": 2265907200 + }, + { + "epoch": 0.37, + "objective/train/aligned_token_count": 0.9939123392105103, + "objective/train/avg_token_score": 0.006087609101086855, + "objective/train/docs_used": 825438, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2487326860427856, + "objective/train/original_loss": 1.2574810981750488, + "objective/train/theoretical_loss": 3.423758014938474, + "objective/train/tokens_used": 636497376, + "objective/train/unlikelihood_loss": 0.017746634781360626, + "theoretical_loss": 3.423758014938474, + "tokens_seen": 2266038272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031648210560102715, + "loss": 1.2581, + "theoretical_loss": 3.423758014938474, + "tokens_seen": 2266038272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164419836302359, + "loss": 1.2649, + "theoretical_loss": 3.4237420401720833, + "tokens_seen": 2266169344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031640186165944476, + "loss": 1.2578, + "theoretical_loss": 3.4237260665883147, + "tokens_seen": 2266300416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163617396886535, + "loss": 1.4087, + "theoretical_loss": 3.423710094187012, + "tokens_seen": 2266431488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163216177178623, + "loss": 1.2755, + "theoretical_loss": 3.42369412296802, + "tokens_seen": 2266562560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162814957470711, + "loss": 1.358, + "theoretical_loss": 3.423678152931182, + "tokens_seen": 2266693632 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162413737762799, + "loss": 1.342, + "theoretical_loss": 3.423662184076343, + "tokens_seen": 2266824704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162012518054887, + "loss": 1.2873, + "theoretical_loss": 3.4236462164033457, + "tokens_seen": 2266955776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161611298346975, + "loss": 1.3154, + "theoretical_loss": 3.423630249912036, + "tokens_seen": 2267086848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161210078639063, + "loss": 1.3255, + "theoretical_loss": 3.423614284602257, + "tokens_seen": 2267217920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031608088589311505, + "loss": 1.3096, + "theoretical_loss": 3.423598320473854, + "tokens_seen": 2267348992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160407639223239, + "loss": 1.2606, + "theoretical_loss": 3.42358235752667, + "tokens_seen": 2267480064 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031600064195153266, + "loss": 1.2587, + "theoretical_loss": 3.42356639576055, + "tokens_seen": 2267611136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159605199807414, + "loss": 1.2671, + "theoretical_loss": 3.423550435175339, + "tokens_seen": 2267742208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031592039800995027, + "loss": 1.2698, + "theoretical_loss": 3.42353447577088, + "tokens_seen": 2267873280 + }, + { + "epoch": 0.37, + "learning_rate": 0.000315880276039159, + "loss": 1.2922, + "theoretical_loss": 3.4235185175470186, + "tokens_seen": 2268004352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158401540683679, + "loss": 1.3851, + "theoretical_loss": 3.4235025605035987, + "tokens_seen": 2268135424 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031580003209757664, + "loss": 1.3225, + "theoretical_loss": 3.423486604640465, + "tokens_seen": 2268266496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031575991012678544, + "loss": 1.2781, + "theoretical_loss": 3.423470649957461, + "tokens_seen": 2268397568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031571978815599425, + "loss": 1.3061, + "theoretical_loss": 3.4234546964544332, + "tokens_seen": 2268528640 + }, + { + "epoch": 0.37, + "learning_rate": 0.000315679666185203, + "loss": 1.2266, + "theoretical_loss": 3.4234387441312246, + "tokens_seen": 2268659712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156395442144118, + "loss": 1.3104, + "theoretical_loss": 3.4234227929876804, + "tokens_seen": 2268790784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155994222436206, + "loss": 1.3071, + "theoretical_loss": 3.4234068430236446, + "tokens_seen": 2268921856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155593002728294, + "loss": 1.2944, + "theoretical_loss": 3.423390894238963, + "tokens_seen": 2269052928 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031551917830203817, + "loss": 1.2081, + "theoretical_loss": 3.423374946633479, + "tokens_seen": 2269184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031547905633124703, + "loss": 1.2698, + "theoretical_loss": 3.4233590002070384, + "tokens_seen": 2269315072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154389343604558, + "loss": 1.2986, + "theoretical_loss": 3.4233430549594854, + "tokens_seen": 2269446144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153988123896646, + "loss": 1.295, + "theoretical_loss": 3.4233271108906647, + "tokens_seen": 2269577216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153586904188734, + "loss": 1.2544, + "theoretical_loss": 3.423311168000421, + "tokens_seen": 2269708288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031531856844808215, + "loss": 1.2745, + "theoretical_loss": 3.4232952262886, + "tokens_seen": 2269839360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031527844647729096, + "loss": 1.3388, + "theoretical_loss": 3.423279285755045, + "tokens_seen": 2269970432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031523832450649976, + "loss": 1.2169, + "theoretical_loss": 3.4232633463996023, + "tokens_seen": 2270101504 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031519820253570857, + "loss": 1.2312, + "theoretical_loss": 3.4232474082221165, + "tokens_seen": 2270232576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151580805649174, + "loss": 1.245, + "theoretical_loss": 3.4232314712224317, + "tokens_seen": 2270363648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151179585941262, + "loss": 1.2557, + "theoretical_loss": 3.423215535400394, + "tokens_seen": 2270494720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031507783662333493, + "loss": 1.3074, + "theoretical_loss": 3.423199600755848, + "tokens_seen": 2270625792 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031503771465254374, + "loss": 1.2948, + "theoretical_loss": 3.4231836672886384, + "tokens_seen": 2270756864 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031499759268175255, + "loss": 1.2854, + "theoretical_loss": 3.4231677349986107, + "tokens_seen": 2270887936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149574707109613, + "loss": 1.1649, + "theoretical_loss": 3.4231518038856095, + "tokens_seen": 2271019008 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031491734874017016, + "loss": 1.3172, + "theoretical_loss": 3.42313587394948, + "tokens_seen": 2271150080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148772267693789, + "loss": 1.3711, + "theoretical_loss": 3.4231199451900682, + "tokens_seen": 2271281152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148371047985877, + "loss": 1.2287, + "theoretical_loss": 3.4231040176072187, + "tokens_seen": 2271412224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147969828277965, + "loss": 1.3029, + "theoretical_loss": 3.4230880912007766, + "tokens_seen": 2271543296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147568608570053, + "loss": 1.2299, + "theoretical_loss": 3.423072165970587, + "tokens_seen": 2271674368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147167388862141, + "loss": 1.2905, + "theoretical_loss": 3.423056241916495, + "tokens_seen": 2271805440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146766169154229, + "loss": 1.3092, + "theoretical_loss": 3.423040319038347, + "tokens_seen": 2271936512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146364949446317, + "loss": 1.2484, + "theoretical_loss": 3.423024397335987, + "tokens_seen": 2272067584 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031459637297384045, + "loss": 1.322, + "theoretical_loss": 3.4230084768092612, + "tokens_seen": 2272198656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145562510030493, + "loss": 1.2871, + "theoretical_loss": 3.4229925574580147, + "tokens_seen": 2272329728 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031451612903225806, + "loss": 1.2992, + "theoretical_loss": 3.4229766392820933, + "tokens_seen": 2272460800 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.9928696155548096, + "objective/train/avg_token_score": 0.007130359299480915, + "objective/train/docs_used": 827716, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2769938707351685, + "objective/train/original_loss": 1.2871078252792358, + "objective/train/theoretical_loss": 3.4229607222813416, + "objective/train/tokens_used": 643050976, + "objective/train/unlikelihood_loss": 0.017723754048347473, + "theoretical_loss": 3.4229607222813416, + "tokens_seen": 2272591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031447600706146687, + "loss": 1.235, + "theoretical_loss": 3.4229607222813416, + "tokens_seen": 2272591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031443588509067567, + "loss": 1.3185, + "theoretical_loss": 3.4229448064556056, + "tokens_seen": 2272722944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143957631198844, + "loss": 1.3053, + "theoretical_loss": 3.422928891804731, + "tokens_seen": 2272854016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143556411490933, + "loss": 1.2936, + "theoretical_loss": 3.422912978328563, + "tokens_seen": 2272985088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031431551917830204, + "loss": 1.2632, + "theoretical_loss": 3.4228970660269473, + "tokens_seen": 2273116160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031427539720751084, + "loss": 1.2787, + "theoretical_loss": 3.422881154899729, + "tokens_seen": 2273247232 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031423527523671965, + "loss": 1.2922, + "theoretical_loss": 3.4228652449467547, + "tokens_seen": 2273378304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141951532659284, + "loss": 1.3079, + "theoretical_loss": 3.4228493361678693, + "tokens_seen": 2273509376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141550312951372, + "loss": 1.3485, + "theoretical_loss": 3.4228334285629187, + "tokens_seen": 2273640448 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314114909324346, + "loss": 1.3879, + "theoretical_loss": 3.4228175221317483, + "tokens_seen": 2273771520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140747873535548, + "loss": 1.2886, + "theoretical_loss": 3.4228016168742044, + "tokens_seen": 2273902592 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031403466538276357, + "loss": 1.3243, + "theoretical_loss": 3.4227857127901324, + "tokens_seen": 2274033664 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031399454341197243, + "loss": 1.2451, + "theoretical_loss": 3.4227698098793784, + "tokens_seen": 2274164736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139544214411812, + "loss": 1.3511, + "theoretical_loss": 3.4227539081417877, + "tokens_seen": 2274295808 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031391429947039004, + "loss": 1.3288, + "theoretical_loss": 3.4227380075772067, + "tokens_seen": 2274426880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138741774995988, + "loss": 1.3842, + "theoretical_loss": 3.422722108185481, + "tokens_seen": 2274557952 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031383405552880755, + "loss": 1.2737, + "theoretical_loss": 3.4227062099664565, + "tokens_seen": 2274689024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137939335580164, + "loss": 1.2192, + "theoretical_loss": 3.4226903129199786, + "tokens_seen": 2274820096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031375381158722516, + "loss": 1.25, + "theoretical_loss": 3.4226744170458945, + "tokens_seen": 2274951168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031371368961643397, + "loss": 1.3388, + "theoretical_loss": 3.4226585223440495, + "tokens_seen": 2275082240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136735676456428, + "loss": 1.3092, + "theoretical_loss": 3.4226426288142893, + "tokens_seen": 2275213312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136334456748516, + "loss": 1.2744, + "theoretical_loss": 3.422626736456461, + "tokens_seen": 2275344384 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031359332370406033, + "loss": 1.3123, + "theoretical_loss": 3.4226108452704094, + "tokens_seen": 2275475456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031355320173326914, + "loss": 1.2931, + "theoretical_loss": 3.422594955255981, + "tokens_seen": 2275606528 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031351307976247795, + "loss": 1.3365, + "theoretical_loss": 3.422579066413023, + "tokens_seen": 2275737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134729577916867, + "loss": 1.3148, + "theoretical_loss": 3.42256317874138, + "tokens_seen": 2275868672 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031343283582089556, + "loss": 1.2672, + "theoretical_loss": 3.4225472922408993, + "tokens_seen": 2275999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133927138501043, + "loss": 1.3281, + "theoretical_loss": 3.4225314069114265, + "tokens_seen": 2276130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133525918793131, + "loss": 1.2937, + "theoretical_loss": 3.4225155227528083, + "tokens_seen": 2276261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133124699085219, + "loss": 1.3037, + "theoretical_loss": 3.422499639764891, + "tokens_seen": 2276392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132723479377307, + "loss": 1.2574, + "theoretical_loss": 3.4224837579475205, + "tokens_seen": 2276524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031323222596693954, + "loss": 1.3228, + "theoretical_loss": 3.422467877300543, + "tokens_seen": 2276655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131921039961483, + "loss": 1.2971, + "theoretical_loss": 3.4224519978238055, + "tokens_seen": 2276786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131519820253571, + "loss": 1.2787, + "theoretical_loss": 3.4224361195171547, + "tokens_seen": 2276917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131118600545659, + "loss": 1.2558, + "theoretical_loss": 3.422420242380436, + "tokens_seen": 2277048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130717380837747, + "loss": 1.2936, + "theoretical_loss": 3.422404366413496, + "tokens_seen": 2277179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031303161611298346, + "loss": 1.3058, + "theoretical_loss": 3.422388491616182, + "tokens_seen": 2277310464 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031299149414219226, + "loss": 1.2959, + "theoretical_loss": 3.42237261798834, + "tokens_seen": 2277441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031295137217140107, + "loss": 1.3012, + "theoretical_loss": 3.4223567455298167, + "tokens_seen": 2277572608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129112502006098, + "loss": 1.3091, + "theoretical_loss": 3.422340874240458, + "tokens_seen": 2277703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128711282298187, + "loss": 1.3656, + "theoretical_loss": 3.4223250041201116, + "tokens_seen": 2277834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031283100625902744, + "loss": 1.2991, + "theoretical_loss": 3.4223091351686237, + "tokens_seen": 2277965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031279088428823624, + "loss": 1.3151, + "theoretical_loss": 3.4222932673858404, + "tokens_seen": 2278096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031275076231744505, + "loss": 1.2871, + "theoretical_loss": 3.4222774007716095, + "tokens_seen": 2278227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127106403466538, + "loss": 1.2218, + "theoretical_loss": 3.4222615353257764, + "tokens_seen": 2278359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126705183758626, + "loss": 1.3044, + "theoretical_loss": 3.422245671048189, + "tokens_seen": 2278490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126303964050714, + "loss": 1.3739, + "theoretical_loss": 3.422229807938694, + "tokens_seen": 2278621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125902744342802, + "loss": 1.3066, + "theoretical_loss": 3.422213945997137, + "tokens_seen": 2278752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312550152463489, + "loss": 1.2525, + "theoretical_loss": 3.422198085223366, + "tokens_seen": 2278883328 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031251003049269783, + "loss": 1.2331, + "theoretical_loss": 3.422182225617228, + "tokens_seen": 2279014400 + }, + { + "epoch": 0.38, + "objective/train/aligned_token_count": 0.9944167137145996, + "objective/train/avg_token_score": 0.005583287216722965, + "objective/train/docs_used": 830070, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3124359846115112, + "objective/train/original_loss": 1.321122169494629, + "objective/train/theoretical_loss": 3.4221663671785687, + "objective/train/tokens_used": 649604576, + "objective/train/unlikelihood_loss": 0.014046039432287216, + "theoretical_loss": 3.4221663671785687, + "tokens_seen": 2279145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124699085219066, + "loss": 1.2891, + "theoretical_loss": 3.4221663671785687, + "tokens_seen": 2279145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031242978655111544, + "loss": 1.2939, + "theoretical_loss": 3.422150509907236, + "tokens_seen": 2279276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123896645803242, + "loss": 1.2159, + "theoretical_loss": 3.422134653803077, + "tokens_seen": 2279407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031234954260953295, + "loss": 1.3536, + "theoretical_loss": 3.422118798865938, + "tokens_seen": 2279538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123094206387418, + "loss": 1.3074, + "theoretical_loss": 3.422102945095666, + "tokens_seen": 2279669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031226929866795056, + "loss": 1.2959, + "theoretical_loss": 3.4220870924921085, + "tokens_seen": 2279800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031222917669715937, + "loss": 1.3259, + "theoretical_loss": 3.4220712410551126, + "tokens_seen": 2279931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121890547263682, + "loss": 1.2368, + "theoretical_loss": 3.422055390784525, + "tokens_seen": 2280062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312148932755577, + "loss": 1.2742, + "theoretical_loss": 3.4220395416801934, + "tokens_seen": 2280194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031210881078478573, + "loss": 1.2941, + "theoretical_loss": 3.4220236937419646, + "tokens_seen": 2280325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031206868881399454, + "loss": 1.2394, + "theoretical_loss": 3.4220078469696853, + "tokens_seen": 2280456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031202856684320335, + "loss": 1.3556, + "theoretical_loss": 3.4219920013632037, + "tokens_seen": 2280587264 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031198844487241215, + "loss": 1.2652, + "theoretical_loss": 3.421976156922366, + "tokens_seen": 2280718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031194832290162096, + "loss": 1.334, + "theoretical_loss": 3.4219603136470207, + "tokens_seen": 2280849408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003119082009308297, + "loss": 1.2457, + "theoretical_loss": 3.4219444715370138, + "tokens_seen": 2280980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031186807896003857, + "loss": 1.3045, + "theoretical_loss": 3.4219286305921934, + "tokens_seen": 2281111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118279569892473, + "loss": 1.3471, + "theoretical_loss": 3.421912790812407, + "tokens_seen": 2281242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117878350184561, + "loss": 1.2611, + "theoretical_loss": 3.421896952197501, + "tokens_seen": 2281373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031174771304766494, + "loss": 1.202, + "theoretical_loss": 3.421881114747324, + "tokens_seen": 2281504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117075910768737, + "loss": 1.2494, + "theoretical_loss": 3.421865278461723, + "tokens_seen": 2281635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116674691060825, + "loss": 1.3379, + "theoretical_loss": 3.421849443340545, + "tokens_seen": 2281766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116273471352913, + "loss": 1.3178, + "theoretical_loss": 3.421833609383638, + "tokens_seen": 2281897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115872251645001, + "loss": 1.3123, + "theoretical_loss": 3.4218177765908497, + "tokens_seen": 2282029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031154710319370886, + "loss": 1.2781, + "theoretical_loss": 3.421801944962027, + "tokens_seen": 2282160128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031150698122291766, + "loss": 1.1999, + "theoretical_loss": 3.421786114497018, + "tokens_seen": 2282291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031146685925212647, + "loss": 1.2967, + "theoretical_loss": 3.4217702851956706, + "tokens_seen": 2282422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114267372813352, + "loss": 1.2786, + "theoretical_loss": 3.4217544570578315, + "tokens_seen": 2282553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113866153105441, + "loss": 1.2606, + "theoretical_loss": 3.4217386300833494, + "tokens_seen": 2282684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031134649333975284, + "loss": 1.3343, + "theoretical_loss": 3.421722804272071, + "tokens_seen": 2282815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113063713689617, + "loss": 1.3166, + "theoretical_loss": 3.421706979623845, + "tokens_seen": 2282946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031126624939817045, + "loss": 1.2656, + "theoretical_loss": 3.421691156138518, + "tokens_seen": 2283077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112261274273792, + "loss": 1.2468, + "theoretical_loss": 3.4216753338159394, + "tokens_seen": 2283208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031118600545658806, + "loss": 1.2205, + "theoretical_loss": 3.4216595126559555, + "tokens_seen": 2283339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111458834857968, + "loss": 1.3313, + "theoretical_loss": 3.421643692658415, + "tokens_seen": 2283470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111057615150056, + "loss": 1.3573, + "theoretical_loss": 3.4216278738231654, + "tokens_seen": 2283601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110656395442144, + "loss": 1.2834, + "theoretical_loss": 3.421612056150055, + "tokens_seen": 2283732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031102551757342323, + "loss": 1.2861, + "theoretical_loss": 3.421596239638931, + "tokens_seen": 2283864064 + }, + { + "epoch": 0.38, + "learning_rate": 0.000310985395602632, + "loss": 1.3372, + "theoretical_loss": 3.421580424289642, + "tokens_seen": 2283995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109452736318408, + "loss": 1.2613, + "theoretical_loss": 3.4215646101020356, + "tokens_seen": 2284126208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109051516610496, + "loss": 1.2389, + "theoretical_loss": 3.4215487970759604, + "tokens_seen": 2284257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031086502969025835, + "loss": 1.2732, + "theoretical_loss": 3.4215329852112637, + "tokens_seen": 2284388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108249077194672, + "loss": 1.2693, + "theoretical_loss": 3.4215171745077946, + "tokens_seen": 2284519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031078478574867596, + "loss": 1.2691, + "theoretical_loss": 3.4215013649654, + "tokens_seen": 2284650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107446637778848, + "loss": 1.2623, + "theoretical_loss": 3.4214855565839284, + "tokens_seen": 2284781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107045418070936, + "loss": 1.3255, + "theoretical_loss": 3.4214697493632285, + "tokens_seen": 2284912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106644198363024, + "loss": 1.2449, + "theoretical_loss": 3.4214539433031477, + "tokens_seen": 2285043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106242978655112, + "loss": 1.3288, + "theoretical_loss": 3.421438138403535, + "tokens_seen": 2285174784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031058417589471994, + "loss": 1.3275, + "theoretical_loss": 3.4214223346642383, + "tokens_seen": 2285305856 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031054405392392874, + "loss": 1.3202, + "theoretical_loss": 3.4214065320851055, + "tokens_seen": 2285436928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031050393195313755, + "loss": 1.2158, + "theoretical_loss": 3.4213907306659856, + "tokens_seen": 2285568000 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.9909844398498535, + "objective/train/avg_token_score": 0.0090155228972435, + "objective/train/docs_used": 832464, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2635608911514282, + "objective/train/original_loss": 1.2753270864486694, + "objective/train/theoretical_loss": 3.4213749304067265, + "objective/train/tokens_used": 656158176, + "objective/train/unlikelihood_loss": 0.02614385448396206, + "theoretical_loss": 3.4213749304067265, + "tokens_seen": 2285699072 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031046380998234636, + "loss": 1.2519, + "theoretical_loss": 3.4213749304067265, + "tokens_seen": 2285699072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104236880115551, + "loss": 1.2808, + "theoretical_loss": 3.4213591313071765, + "tokens_seen": 2285830144 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031038356604076397, + "loss": 1.2508, + "theoretical_loss": 3.4213433333671843, + "tokens_seen": 2285961216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103434440699727, + "loss": 1.3116, + "theoretical_loss": 3.421327536586598, + "tokens_seen": 2286092288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103033220991815, + "loss": 1.204, + "theoretical_loss": 3.4213117409652662, + "tokens_seen": 2286223360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031026320012839033, + "loss": 1.3394, + "theoretical_loss": 3.4212959465030375, + "tokens_seen": 2286354432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102230781575991, + "loss": 1.2794, + "theoretical_loss": 3.4212801531997603, + "tokens_seen": 2286485504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101829561868079, + "loss": 1.2861, + "theoretical_loss": 3.421264361055283, + "tokens_seen": 2286616576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101428342160167, + "loss": 1.2765, + "theoretical_loss": 3.421248570069454, + "tokens_seen": 2286747648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101027122452255, + "loss": 1.3341, + "theoretical_loss": 3.4212327802421227, + "tokens_seen": 2286878720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100625902744343, + "loss": 1.3513, + "theoretical_loss": 3.4212169915731367, + "tokens_seen": 2287009792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031002246830364306, + "loss": 1.3084, + "theoretical_loss": 3.4212012040623456, + "tokens_seen": 2287140864 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030998234633285187, + "loss": 1.2925, + "theoretical_loss": 3.421185417709597, + "tokens_seen": 2287271936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099422243620607, + "loss": 1.3381, + "theoretical_loss": 3.4211696325147405, + "tokens_seen": 2287403008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003099021023912695, + "loss": 1.231, + "theoretical_loss": 3.4211538484776245, + "tokens_seen": 2287534080 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030986198042047824, + "loss": 1.2903, + "theoretical_loss": 3.4211380655980976, + "tokens_seen": 2287665152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098218584496871, + "loss": 1.3287, + "theoretical_loss": 3.421122283876009, + "tokens_seen": 2287796224 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030978173647889585, + "loss": 1.3362, + "theoretical_loss": 3.421106503311207, + "tokens_seen": 2287927296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097416145081046, + "loss": 1.3452, + "theoretical_loss": 3.421090723903541, + "tokens_seen": 2288058368 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030970149253731346, + "loss": 1.3366, + "theoretical_loss": 3.4210749456528595, + "tokens_seen": 2288189440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096613705665222, + "loss": 1.2253, + "theoretical_loss": 3.4210591685590113, + "tokens_seen": 2288320512 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309621248595731, + "loss": 1.3509, + "theoretical_loss": 3.4210433926218458, + "tokens_seen": 2288451584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095811266249398, + "loss": 1.2549, + "theoretical_loss": 3.4210276178412116, + "tokens_seen": 2288582656 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030954100465414863, + "loss": 1.307, + "theoretical_loss": 3.4210118442169573, + "tokens_seen": 2288713728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095008826833574, + "loss": 1.2427, + "theoretical_loss": 3.4209960717489327, + "tokens_seen": 2288844800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094607607125662, + "loss": 1.2766, + "theoretical_loss": 3.4209803004369865, + "tokens_seen": 2288975872 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309420638741775, + "loss": 1.3233, + "theoretical_loss": 3.420964530280968, + "tokens_seen": 2289106944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093805167709838, + "loss": 1.2412, + "theoretical_loss": 3.4209487612807257, + "tokens_seen": 2289238016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093403948001926, + "loss": 1.2914, + "theoretical_loss": 3.4209329934361095, + "tokens_seen": 2289369088 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030930027282940136, + "loss": 1.3467, + "theoretical_loss": 3.4209172267469676, + "tokens_seen": 2289500160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092601508586102, + "loss": 1.2855, + "theoretical_loss": 3.42090146121315, + "tokens_seen": 2289631232 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309220028887819, + "loss": 1.3126, + "theoretical_loss": 3.420885696834506, + "tokens_seen": 2289762304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091799069170278, + "loss": 1.2358, + "theoretical_loss": 3.4208699336108843, + "tokens_seen": 2289893376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091397849462366, + "loss": 1.3514, + "theoretical_loss": 3.420854171542134, + "tokens_seen": 2290024448 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030909966297544534, + "loss": 1.2582, + "theoretical_loss": 3.420838410628105, + "tokens_seen": 2290155520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030905954100465414, + "loss": 1.2832, + "theoretical_loss": 3.4208226508686463, + "tokens_seen": 2290286592 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030901941903386295, + "loss": 1.2338, + "theoretical_loss": 3.4208068922636077, + "tokens_seen": 2290417664 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030897929706307176, + "loss": 1.2662, + "theoretical_loss": 3.4207911348128377, + "tokens_seen": 2290548736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089391750922805, + "loss": 1.2756, + "theoretical_loss": 3.4207753785161867, + "tokens_seen": 2290679808 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030889905312148937, + "loss": 1.2648, + "theoretical_loss": 3.4207596233735034, + "tokens_seen": 2290810880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088589311506981, + "loss": 1.3162, + "theoretical_loss": 3.420743869384638, + "tokens_seen": 2290941952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030881880917990693, + "loss": 1.2886, + "theoretical_loss": 3.420728116549439, + "tokens_seen": 2291073024 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030877868720911573, + "loss": 1.2811, + "theoretical_loss": 3.4207123648677564, + "tokens_seen": 2291204096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087385652383245, + "loss": 1.288, + "theoretical_loss": 3.42069661433944, + "tokens_seen": 2291335168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030869844326753335, + "loss": 1.3165, + "theoretical_loss": 3.420680864964339, + "tokens_seen": 2291466240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086583212967421, + "loss": 1.2549, + "theoretical_loss": 3.4206651167423034, + "tokens_seen": 2291597312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086181993259509, + "loss": 1.2997, + "theoretical_loss": 3.420649369673183, + "tokens_seen": 2291728384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085780773551597, + "loss": 1.3106, + "theoretical_loss": 3.4206336237568267, + "tokens_seen": 2291859456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030853795538436846, + "loss": 1.3029, + "theoretical_loss": 3.4206178789930846, + "tokens_seen": 2291990528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030849783341357727, + "loss": 1.2716, + "theoretical_loss": 3.420602135381806, + "tokens_seen": 2292121600 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.9938550591468811, + "objective/train/avg_token_score": 0.0061449627391994, + "objective/train/docs_used": 834793, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2386587858200073, + "objective/train/original_loss": 1.2476441860198975, + "objective/train/theoretical_loss": 3.4205863929228415, + "objective/train/tokens_used": 662711776, + "objective/train/unlikelihood_loss": 0.016188381239771843, + "theoretical_loss": 3.4205863929228415, + "tokens_seen": 2292252672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084577114427861, + "loss": 1.2103, + "theoretical_loss": 3.4205863929228415, + "tokens_seen": 2292252672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084175894719949, + "loss": 1.2177, + "theoretical_loss": 3.4205706516160403, + "tokens_seen": 2292383744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030837746750120363, + "loss": 1.261, + "theoretical_loss": 3.4205549114612523, + "tokens_seen": 2292514816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083373455304125, + "loss": 1.2982, + "theoretical_loss": 3.4205391724583274, + "tokens_seen": 2292645888 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030829722355962125, + "loss": 1.3531, + "theoretical_loss": 3.420523434607116, + "tokens_seen": 2292776960 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030825710158883, + "loss": 1.3364, + "theoretical_loss": 3.4205076979074667, + "tokens_seen": 2292908032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030821697961803886, + "loss": 1.2233, + "theoretical_loss": 3.42049196235923, + "tokens_seen": 2293039104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081768576472476, + "loss": 1.1952, + "theoretical_loss": 3.420476227962256, + "tokens_seen": 2293170176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030813673567645647, + "loss": 1.3227, + "theoretical_loss": 3.4204604947163952, + "tokens_seen": 2293301248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080966137056652, + "loss": 1.2587, + "theoretical_loss": 3.420444762621497, + "tokens_seen": 2293432320 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030805649173487403, + "loss": 1.3602, + "theoretical_loss": 3.4204290316774113, + "tokens_seen": 2293563392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030801636976408284, + "loss": 1.2493, + "theoretical_loss": 3.420413301883988, + "tokens_seen": 2293694464 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079762477932916, + "loss": 1.2024, + "theoretical_loss": 3.420397573241078, + "tokens_seen": 2293825536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079361258225004, + "loss": 1.3394, + "theoretical_loss": 3.4203818457485307, + "tokens_seen": 2293956608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078960038517092, + "loss": 1.3323, + "theoretical_loss": 3.4203661194061965, + "tokens_seen": 2294087680 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307855881880918, + "loss": 1.3135, + "theoretical_loss": 3.420350394213926, + "tokens_seen": 2294218752 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030781575991012676, + "loss": 1.2625, + "theoretical_loss": 3.4203346701715684, + "tokens_seen": 2294349824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077756379393356, + "loss": 1.2781, + "theoretical_loss": 3.420318947278975, + "tokens_seen": 2294480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077355159685444, + "loss": 1.3425, + "theoretical_loss": 3.420303225535995, + "tokens_seen": 2294611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076953939977532, + "loss": 1.2232, + "theoretical_loss": 3.4202875049424795, + "tokens_seen": 2294743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307655272026962, + "loss": 1.3514, + "theoretical_loss": 3.4202717854982785, + "tokens_seen": 2294874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030761515005617074, + "loss": 1.3084, + "theoretical_loss": 3.4202560672032423, + "tokens_seen": 2295005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075750280853796, + "loss": 1.2716, + "theoretical_loss": 3.420240350057221, + "tokens_seen": 2295136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030753490611458835, + "loss": 1.2081, + "theoretical_loss": 3.420224634060066, + "tokens_seen": 2295267328 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030749478414379716, + "loss": 1.2896, + "theoretical_loss": 3.420208919211627, + "tokens_seen": 2295398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030745466217300596, + "loss": 1.2, + "theoretical_loss": 3.420193205511754, + "tokens_seen": 2295529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030741454020221477, + "loss": 1.3122, + "theoretical_loss": 3.420177492960298, + "tokens_seen": 2295660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073744182314235, + "loss": 1.2382, + "theoretical_loss": 3.42016178155711, + "tokens_seen": 2295791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030733429626063233, + "loss": 1.3565, + "theoretical_loss": 3.4201460713020397, + "tokens_seen": 2295922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030729417428984113, + "loss": 1.3473, + "theoretical_loss": 3.4201303621949384, + "tokens_seen": 2296053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072540523190499, + "loss": 1.2881, + "theoretical_loss": 3.4201146542356557, + "tokens_seen": 2296184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030721393034825875, + "loss": 1.339, + "theoretical_loss": 3.420098947424043, + "tokens_seen": 2296315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071738083774675, + "loss": 1.2811, + "theoretical_loss": 3.4200832417599507, + "tokens_seen": 2296446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071336864066763, + "loss": 1.4117, + "theoretical_loss": 3.42006753724323, + "tokens_seen": 2296578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070935644358851, + "loss": 1.2998, + "theoretical_loss": 3.4200518338737305, + "tokens_seen": 2296709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030705344246509386, + "loss": 1.2679, + "theoretical_loss": 3.4200361316513037, + "tokens_seen": 2296840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030701332049430267, + "loss": 1.2766, + "theoretical_loss": 3.4200204305758004, + "tokens_seen": 2296971264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069731985235115, + "loss": 1.3134, + "theoretical_loss": 3.420004730647071, + "tokens_seen": 2297102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069330765527203, + "loss": 1.2563, + "theoretical_loss": 3.419989031864967, + "tokens_seen": 2297233408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068929545819291, + "loss": 1.2461, + "theoretical_loss": 3.419973334229338, + "tokens_seen": 2297364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068528326111379, + "loss": 1.283, + "theoretical_loss": 3.419957637740036, + "tokens_seen": 2297495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030681271064034665, + "loss": 1.2953, + "theoretical_loss": 3.4199419423969117, + "tokens_seen": 2297626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030677258866955545, + "loss": 1.2426, + "theoretical_loss": 3.4199262481998156, + "tokens_seen": 2297757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030673246669876426, + "loss": 1.2564, + "theoretical_loss": 3.419910555148599, + "tokens_seen": 2297888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306692344727973, + "loss": 1.3082, + "theoretical_loss": 3.4198948632431128, + "tokens_seen": 2298019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030665222275718187, + "loss": 1.3442, + "theoretical_loss": 3.419879172483208, + "tokens_seen": 2298150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066121007863906, + "loss": 1.3002, + "theoretical_loss": 3.4198634828687355, + "tokens_seen": 2298281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030657197881559943, + "loss": 1.2749, + "theoretical_loss": 3.4198477943995464, + "tokens_seen": 2298413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030653185684480824, + "loss": 1.2545, + "theoretical_loss": 3.419832107075492, + "tokens_seen": 2298544128 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306491734874017, + "loss": 1.3494, + "theoretical_loss": 3.4198164208964235, + "tokens_seen": 2298675200 + }, + { + "epoch": 0.39, + "objective/train/aligned_token_count": 0.9941117167472839, + "objective/train/avg_token_score": 0.005888289771974087, + "objective/train/docs_used": 837138, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1734493970870972, + "objective/train/original_loss": 1.1815060377120972, + "objective/train/theoretical_loss": 3.419800735862192, + "objective/train/tokens_used": 669265376, + "objective/train/unlikelihood_loss": 0.0195917971432209, + "theoretical_loss": 3.419800735862192, + "tokens_seen": 2298806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064516129032258, + "loss": 1.1878, + "theoretical_loss": 3.419800735862192, + "tokens_seen": 2298806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064114909324346, + "loss": 1.3889, + "theoretical_loss": 3.4197850519726485, + "tokens_seen": 2298937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063713689616434, + "loss": 1.2129, + "theoretical_loss": 3.4197693692276436, + "tokens_seen": 2299068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030633124699085216, + "loss": 1.2923, + "theoretical_loss": 3.41975368762703, + "tokens_seen": 2299199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306291125020061, + "loss": 1.271, + "theoretical_loss": 3.4197380071706576, + "tokens_seen": 2299330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030625100304926977, + "loss": 1.2814, + "theoretical_loss": 3.419722327858379, + "tokens_seen": 2299461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062108810784786, + "loss": 1.2865, + "theoretical_loss": 3.4197066496900437, + "tokens_seen": 2299592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061707591076874, + "loss": 1.2478, + "theoretical_loss": 3.4196909726655047, + "tokens_seen": 2299723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030613063713689614, + "loss": 1.2596, + "theoretical_loss": 3.4196752967846127, + "tokens_seen": 2299854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306090515166105, + "loss": 1.3163, + "theoretical_loss": 3.419659622047219, + "tokens_seen": 2299985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030605039319531375, + "loss": 1.3631, + "theoretical_loss": 3.4196439484531753, + "tokens_seen": 2300116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030601027122452256, + "loss": 1.3373, + "theoretical_loss": 3.4196282760023324, + "tokens_seen": 2300248064 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030597014925373136, + "loss": 1.2964, + "theoretical_loss": 3.419612604694543, + "tokens_seen": 2300379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030593002728294017, + "loss": 1.2954, + "theoretical_loss": 3.419596934529658, + "tokens_seen": 2300510208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058899053121489, + "loss": 1.2944, + "theoretical_loss": 3.4195812655075284, + "tokens_seen": 2300641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030584978334135773, + "loss": 1.2835, + "theoretical_loss": 3.4195655976280066, + "tokens_seen": 2300772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030580966137056653, + "loss": 1.3641, + "theoretical_loss": 3.419549930890944, + "tokens_seen": 2300903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057695393997753, + "loss": 1.2988, + "theoretical_loss": 3.4195342652961918, + "tokens_seen": 2301034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030572941742898415, + "loss": 1.3292, + "theoretical_loss": 3.4195186008436016, + "tokens_seen": 2301165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056892954581929, + "loss": 1.2836, + "theoretical_loss": 3.4195029375330255, + "tokens_seen": 2301296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030564917348740176, + "loss": 1.3879, + "theoretical_loss": 3.419487275364315, + "tokens_seen": 2301427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056090515166105, + "loss": 1.2616, + "theoretical_loss": 3.4194716143373225, + "tokens_seen": 2301558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030556892954581926, + "loss": 1.2662, + "theoretical_loss": 3.4194559544518985, + "tokens_seen": 2301689856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055288075750281, + "loss": 1.216, + "theoretical_loss": 3.419440295707896, + "tokens_seen": 2301820928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054886856042369, + "loss": 1.3105, + "theoretical_loss": 3.4194246381051663, + "tokens_seen": 2301952000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054485636334457, + "loss": 1.2928, + "theoretical_loss": 3.419408981643561, + "tokens_seen": 2302083072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054084416626545, + "loss": 1.3081, + "theoretical_loss": 3.4193933263229317, + "tokens_seen": 2302214144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053683196918633, + "loss": 1.3116, + "theoretical_loss": 3.4193776721431313, + "tokens_seen": 2302345216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030532819772107205, + "loss": 1.2977, + "theoretical_loss": 3.4193620191040113, + "tokens_seen": 2302476288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030528807575028085, + "loss": 1.2574, + "theoretical_loss": 3.4193463672054234, + "tokens_seen": 2302607360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030524795377948966, + "loss": 1.2483, + "theoretical_loss": 3.41933071644722, + "tokens_seen": 2302738432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052078318086984, + "loss": 1.3193, + "theoretical_loss": 3.4193150668292525, + "tokens_seen": 2302869504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030516770983790727, + "loss": 1.3424, + "theoretical_loss": 3.4192994183513736, + "tokens_seen": 2303000576 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305127587867116, + "loss": 1.3016, + "theoretical_loss": 3.4192837710134345, + "tokens_seen": 2303131648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030508746589632483, + "loss": 1.327, + "theoretical_loss": 3.4192681248152885, + "tokens_seen": 2303262720 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030504734392553364, + "loss": 1.2462, + "theoretical_loss": 3.419252479756787, + "tokens_seen": 2303393792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050072219547424, + "loss": 1.3413, + "theoretical_loss": 3.419236835837782, + "tokens_seen": 2303524864 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030496709998395125, + "loss": 1.2118, + "theoretical_loss": 3.419221193058126, + "tokens_seen": 2303655936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030492697801316, + "loss": 1.294, + "theoretical_loss": 3.4192055514176714, + "tokens_seen": 2303787008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048868560423688, + "loss": 1.2618, + "theoretical_loss": 3.4191899109162693, + "tokens_seen": 2303918080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048467340715776, + "loss": 1.3696, + "theoretical_loss": 3.419174271553773, + "tokens_seen": 2304049152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048066121007864, + "loss": 1.2768, + "theoretical_loss": 3.4191586333300354, + "tokens_seen": 2304180224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030476649012999517, + "loss": 1.2725, + "theoretical_loss": 3.4191429962449074, + "tokens_seen": 2304311296 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304726368159204, + "loss": 1.3023, + "theoretical_loss": 3.419127360298242, + "tokens_seen": 2304442368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046862461884128, + "loss": 1.3016, + "theoretical_loss": 3.4191117254898913, + "tokens_seen": 2304573440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030464612421762154, + "loss": 1.361, + "theoretical_loss": 3.419096091819708, + "tokens_seen": 2304704512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046060022468304, + "loss": 1.3048, + "theoretical_loss": 3.4190804592875446, + "tokens_seen": 2304835584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030456588027603915, + "loss": 1.2438, + "theoretical_loss": 3.419064827893253, + "tokens_seen": 2304966656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030452575830524796, + "loss": 1.2988, + "theoretical_loss": 3.4190491976366864, + "tokens_seen": 2305097728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030448563633445676, + "loss": 1.2319, + "theoretical_loss": 3.4190335685176967, + "tokens_seen": 2305228800 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7431796110913657, + "debugging/entropy-1-grams": 5.572979440278162, + "debugging/length": 451.2962962962963, + "debugging/num_segments": 27, + "debugging/score": 0.010468038290363413, + "debugging/score_std": 0.015814533663842854, + "epoch": 0.4, + "objective/train/aligned_token_count": 0.9937704801559448, + "objective/train/avg_token_score": 0.0062295314855873585, + "objective/train/docs_used": 839437, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3759263753890991, + "objective/train/original_loss": 1.3859999179840088, + "objective/train/theoretical_loss": 3.4190179405361367, + "objective/train/tokens_used": 675818976, + "objective/train/unlikelihood_loss": 0.013968252576887608, + "theoretical_loss": 3.4190179405361367, + "tokens_seen": 2305359872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030444551436366557, + "loss": 1.3241, + "theoretical_loss": 3.4190179405361367, + "tokens_seen": 2305359872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044053923928743, + "loss": 1.3325, + "theoretical_loss": 3.419002313691859, + "tokens_seen": 2305490944 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043652704220831, + "loss": 1.3219, + "theoretical_loss": 3.4189866879847157, + "tokens_seen": 2305622016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030432514845129193, + "loss": 1.3846, + "theoretical_loss": 3.4189710634145607, + "tokens_seen": 2305753088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030428502648050074, + "loss": 1.324, + "theoretical_loss": 3.418955439981245, + "tokens_seen": 2305884160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030424490450970955, + "loss": 1.2412, + "theoretical_loss": 3.418939817684622, + "tokens_seen": 2306015232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042047825389183, + "loss": 1.3282, + "theoretical_loss": 3.418924196524545, + "tokens_seen": 2306146304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030416466056812716, + "loss": 1.2345, + "theoretical_loss": 3.418908576500866, + "tokens_seen": 2306277376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041245385973359, + "loss": 1.3228, + "theoretical_loss": 3.418892957613438, + "tokens_seen": 2306408448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030408441662654466, + "loss": 1.2682, + "theoretical_loss": 3.4188773398621137, + "tokens_seen": 2306539520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040442946557535, + "loss": 1.2214, + "theoretical_loss": 3.4188617232467458, + "tokens_seen": 2306670592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040041726849623, + "loss": 1.3003, + "theoretical_loss": 3.4188461077671874, + "tokens_seen": 2306801664 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039640507141711, + "loss": 1.2984, + "theoretical_loss": 3.418830493423291, + "tokens_seen": 2306932736 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003039239287433799, + "loss": 1.375, + "theoretical_loss": 3.4188148802149096, + "tokens_seen": 2307063808 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038838067725887, + "loss": 1.3855, + "theoretical_loss": 3.418799268141896, + "tokens_seen": 2307194880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030384368480179745, + "loss": 1.3154, + "theoretical_loss": 3.418783657204104, + "tokens_seen": 2307325952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030380356283100625, + "loss": 1.2687, + "theoretical_loss": 3.418768047401386, + "tokens_seen": 2307457024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030376344086021506, + "loss": 1.2872, + "theoretical_loss": 3.4187524387335944, + "tokens_seen": 2307588096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030372331888942387, + "loss": 1.3483, + "theoretical_loss": 3.418736831200583, + "tokens_seen": 2307719168 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030368319691863267, + "loss": 1.2719, + "theoretical_loss": 3.4187212248022045, + "tokens_seen": 2307850240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036430749478414, + "loss": 1.2981, + "theoretical_loss": 3.418705619538312, + "tokens_seen": 2307981312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036029529770503, + "loss": 1.3259, + "theoretical_loss": 3.418690015408759, + "tokens_seen": 2308112384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030356283100625904, + "loss": 1.3105, + "theoretical_loss": 3.418674412413398, + "tokens_seen": 2308243456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035227090354678, + "loss": 1.2965, + "theoretical_loss": 3.418658810552083, + "tokens_seen": 2308374528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030348258706467665, + "loss": 1.3794, + "theoretical_loss": 3.418643209824666, + "tokens_seen": 2308505600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034424650938854, + "loss": 1.3176, + "theoretical_loss": 3.4186276102310016, + "tokens_seen": 2308636672 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034023431230942, + "loss": 1.2673, + "theoretical_loss": 3.418612011770942, + "tokens_seen": 2308767744 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303362221152303, + "loss": 1.2876, + "theoretical_loss": 3.4185964144443406, + "tokens_seen": 2308898816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033220991815118, + "loss": 1.3103, + "theoretical_loss": 3.4185808182510513, + "tokens_seen": 2309029888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030328197721072057, + "loss": 1.2542, + "theoretical_loss": 3.418565223190927, + "tokens_seen": 2309160960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032418552399294, + "loss": 1.286, + "theoretical_loss": 3.418549629263821, + "tokens_seen": 2309292032 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032017332691382, + "loss": 1.2596, + "theoretical_loss": 3.4185340364695866, + "tokens_seen": 2309423104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030316161129834694, + "loss": 1.3059, + "theoretical_loss": 3.418518444808078, + "tokens_seen": 2309554176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031214893275558, + "loss": 1.3765, + "theoretical_loss": 3.4185028542791476, + "tokens_seen": 2309685248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030308136735676455, + "loss": 1.2991, + "theoretical_loss": 3.4184872648826494, + "tokens_seen": 2309816320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030412453859734, + "loss": 1.3548, + "theoretical_loss": 3.4184716766184366, + "tokens_seen": 2309947392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030300112341518216, + "loss": 1.354, + "theoretical_loss": 3.418456089486363, + "tokens_seen": 2310078464 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030296100144439097, + "loss": 1.3069, + "theoretical_loss": 3.4184405034862824, + "tokens_seen": 2310209536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029208794735998, + "loss": 1.2661, + "theoretical_loss": 3.4184249186180478, + "tokens_seen": 2310340608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028807575028085, + "loss": 1.3024, + "theoretical_loss": 3.4184093348815128, + "tokens_seen": 2310471680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030284063553201733, + "loss": 1.2982, + "theoretical_loss": 3.4183937522765317, + "tokens_seen": 2310602752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030280051356122614, + "loss": 1.3015, + "theoretical_loss": 3.4183781708029572, + "tokens_seen": 2310733824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030276039159043495, + "loss": 1.2541, + "theoretical_loss": 3.418362590460644, + "tokens_seen": 2310864896 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027202696196437, + "loss": 1.3377, + "theoretical_loss": 3.4183470112494447, + "tokens_seen": 2310995968 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030268014764885256, + "loss": 1.3167, + "theoretical_loss": 3.418331433169214, + "tokens_seen": 2311127040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026400256780613, + "loss": 1.323, + "theoretical_loss": 3.4183158562198055, + "tokens_seen": 2311258112 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030259990370727006, + "loss": 1.3009, + "theoretical_loss": 3.4183002804010725, + "tokens_seen": 2311389184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025597817364789, + "loss": 1.313, + "theoretical_loss": 3.4182847057128694, + "tokens_seen": 2311520256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025196597656877, + "loss": 1.2794, + "theoretical_loss": 3.4182691321550496, + "tokens_seen": 2311651328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030247953779489654, + "loss": 1.3124, + "theoretical_loss": 3.418253559727467, + "tokens_seen": 2311782400 + }, + { + "epoch": 0.4, + "objective/train/aligned_token_count": 0.9930840730667114, + "objective/train/avg_token_score": 0.006915949285030365, + "objective/train/docs_used": 841951, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2751069068908691, + "objective/train/original_loss": 1.2841039896011353, + "objective/train/theoretical_loss": 3.4182379884299756, + "objective/train/tokens_used": 682372576, + "objective/train/unlikelihood_loss": 0.018278755247592926, + "theoretical_loss": 3.4182379884299756, + "tokens_seen": 2311913472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024394158241053, + "loss": 1.2322, + "theoretical_loss": 3.4182379884299756, + "tokens_seen": 2311913472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023992938533141, + "loss": 1.2745, + "theoretical_loss": 3.4182224182624297, + "tokens_seen": 2312044544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023591718825229, + "loss": 1.3213, + "theoretical_loss": 3.4182068492246827, + "tokens_seen": 2312175616 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030231904991173165, + "loss": 1.3174, + "theoretical_loss": 3.418191281316589, + "tokens_seen": 2312306688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030227892794094046, + "loss": 1.3084, + "theoretical_loss": 3.418175714538002, + "tokens_seen": 2312437760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030223880597014926, + "loss": 1.2238, + "theoretical_loss": 3.4181601488887763, + "tokens_seen": 2312568832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030219868399935807, + "loss": 1.3247, + "theoretical_loss": 3.4181445843687657, + "tokens_seen": 2312699904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021585620285668, + "loss": 1.1799, + "theoretical_loss": 3.418129020977825, + "tokens_seen": 2312830976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021184400577757, + "loss": 1.2985, + "theoretical_loss": 3.4181134587158066, + "tokens_seen": 2312962048 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030207831808698444, + "loss": 1.3214, + "theoretical_loss": 3.418097897582567, + "tokens_seen": 2313093120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020381961161932, + "loss": 1.3267, + "theoretical_loss": 3.418082337577958, + "tokens_seen": 2313224192 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030199807414540205, + "loss": 1.3538, + "theoretical_loss": 3.4180667787018355, + "tokens_seen": 2313355264 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003019579521746108, + "loss": 1.3215, + "theoretical_loss": 3.418051220954053, + "tokens_seen": 2313486336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003019178302038196, + "loss": 1.3028, + "theoretical_loss": 3.418035664334465, + "tokens_seen": 2313617408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018777082330284, + "loss": 1.3071, + "theoretical_loss": 3.4180201088429256, + "tokens_seen": 2313748480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018375862622372, + "loss": 1.299, + "theoretical_loss": 3.418004554479289, + "tokens_seen": 2313879552 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301797464291446, + "loss": 1.2976, + "theoretical_loss": 3.41798900124341, + "tokens_seen": 2314010624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017573423206548, + "loss": 1.2666, + "theoretical_loss": 3.417973449135143, + "tokens_seen": 2314141696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017172203498636, + "loss": 1.2876, + "theoretical_loss": 3.4179578981543415, + "tokens_seen": 2314272768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016770983790724, + "loss": 1.3617, + "theoretical_loss": 3.41794234830086, + "tokens_seen": 2314403840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016369764082812, + "loss": 1.2916, + "theoretical_loss": 3.4179267995745546, + "tokens_seen": 2314534912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030159685443748995, + "loss": 1.2544, + "theoretical_loss": 3.4179112519752777, + "tokens_seen": 2314665984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015567324666988, + "loss": 1.2912, + "theoretical_loss": 3.417895705502885, + "tokens_seen": 2314797056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030151661049590756, + "loss": 1.3451, + "theoretical_loss": 3.417880160157231, + "tokens_seen": 2314928128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014764885251163, + "loss": 1.287, + "theoretical_loss": 3.4178646159381696, + "tokens_seen": 2315059200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014363665543252, + "loss": 1.3549, + "theoretical_loss": 3.4178490728455557, + "tokens_seen": 2315190272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013962445835339, + "loss": 1.3059, + "theoretical_loss": 3.4178335308792436, + "tokens_seen": 2315321344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030135612261274273, + "loss": 1.2413, + "theoretical_loss": 3.417817990039089, + "tokens_seen": 2315452416 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030131600064195154, + "loss": 1.2761, + "theoretical_loss": 3.4178024503249453, + "tokens_seen": 2315583488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030127587867116035, + "loss": 1.3476, + "theoretical_loss": 3.417786911736668, + "tokens_seen": 2315714560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012357567003691, + "loss": 1.3155, + "theoretical_loss": 3.4177713742741114, + "tokens_seen": 2315845632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030119563472957796, + "loss": 1.2321, + "theoretical_loss": 3.4177558379371304, + "tokens_seen": 2315976704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011555127587867, + "loss": 1.29, + "theoretical_loss": 3.4177403027255795, + "tokens_seen": 2316107776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011153907879955, + "loss": 1.3086, + "theoretical_loss": 3.4177247686393137, + "tokens_seen": 2316238848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010752688172043, + "loss": 1.2828, + "theoretical_loss": 3.4177092356781884, + "tokens_seen": 2316369920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010351468464131, + "loss": 1.2839, + "theoretical_loss": 3.4176937038420574, + "tokens_seen": 2316500992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030099502487562194, + "loss": 1.2816, + "theoretical_loss": 3.417678173130776, + "tokens_seen": 2316632064 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009549029048307, + "loss": 1.3121, + "theoretical_loss": 3.4176626435441992, + "tokens_seen": 2316763136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009147809340395, + "loss": 1.297, + "theoretical_loss": 3.4176471150821826, + "tokens_seen": 2316894208 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008746589632483, + "loss": 1.3029, + "theoretical_loss": 3.4176315877445793, + "tokens_seen": 2317025280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030083453699245705, + "loss": 1.3312, + "theoretical_loss": 3.417616061531246, + "tokens_seen": 2317156352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030079441502166586, + "loss": 1.2911, + "theoretical_loss": 3.417600536442037, + "tokens_seen": 2317287424 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030075429305087466, + "loss": 1.2544, + "theoretical_loss": 3.417585012476808, + "tokens_seen": 2317418496 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030071417108008347, + "loss": 1.3079, + "theoretical_loss": 3.4175694896354125, + "tokens_seen": 2317549568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006740491092922, + "loss": 1.3019, + "theoretical_loss": 3.4175539679177076, + "tokens_seen": 2317680640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006339271385011, + "loss": 1.3247, + "theoretical_loss": 3.4175384473235466, + "tokens_seen": 2317811712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030059380516770984, + "loss": 1.385, + "theoretical_loss": 3.417522927852786, + "tokens_seen": 2317942784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005536831969186, + "loss": 1.249, + "theoretical_loss": 3.4175074095052804, + "tokens_seen": 2318073856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030051356122612745, + "loss": 1.3118, + "theoretical_loss": 3.417491892280885, + "tokens_seen": 2318204928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004734392553362, + "loss": 1.32, + "theoretical_loss": 3.417476376179455, + "tokens_seen": 2318336000 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.9904979467391968, + "objective/train/avg_token_score": 0.009502057917416096, + "objective/train/docs_used": 844398, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3185793161392212, + "objective/train/original_loss": 1.3320597410202026, + "objective/train/theoretical_loss": 3.4174608612008455, + "objective/train/tokens_used": 688926176, + "objective/train/unlikelihood_loss": 0.02490253746509552, + "theoretical_loss": 3.4174608612008455, + "tokens_seen": 2318467072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030043331728454506, + "loss": 1.3275, + "theoretical_loss": 3.4174608612008455, + "tokens_seen": 2318467072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003931953137538, + "loss": 1.3631, + "theoretical_loss": 3.4174453473449122, + "tokens_seen": 2318598144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003530733429626, + "loss": 1.2734, + "theoretical_loss": 3.41742983461151, + "tokens_seen": 2318729216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003129513721714, + "loss": 1.2657, + "theoretical_loss": 3.4174143230004947, + "tokens_seen": 2318860288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002728294013802, + "loss": 1.3043, + "theoretical_loss": 3.4173988125117214, + "tokens_seen": 2318991360 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300232707430589, + "loss": 1.295, + "theoretical_loss": 3.417383303145045, + "tokens_seen": 2319122432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001925854597978, + "loss": 1.3329, + "theoretical_loss": 3.417367794900322, + "tokens_seen": 2319253504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001524634890066, + "loss": 1.2399, + "theoretical_loss": 3.4173522877774065, + "tokens_seen": 2319384576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030011234151821535, + "loss": 1.2897, + "theoretical_loss": 3.417336781776155, + "tokens_seen": 2319515648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000722195474242, + "loss": 1.3268, + "theoretical_loss": 3.417321276896423, + "tokens_seen": 2319646720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030003209757663296, + "loss": 1.3056, + "theoretical_loss": 3.4173057731380654, + "tokens_seen": 2319777792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999919756058417, + "loss": 1.3447, + "theoretical_loss": 3.417290270500938, + "tokens_seen": 2319908864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999518536350506, + "loss": 1.3628, + "theoretical_loss": 3.417274768984896, + "tokens_seen": 2320039936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999117316642593, + "loss": 1.3017, + "theoretical_loss": 3.4172592685897962, + "tokens_seen": 2320171008 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998716096934682, + "loss": 1.3325, + "theoretical_loss": 3.417243769315493, + "tokens_seen": 2320302080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029983148772267694, + "loss": 1.2755, + "theoretical_loss": 3.4172282711618425, + "tokens_seen": 2320433152 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029979136575188574, + "loss": 1.3389, + "theoretical_loss": 3.4172127741287004, + "tokens_seen": 2320564224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029975124378109455, + "loss": 1.2824, + "theoretical_loss": 3.417197278215922, + "tokens_seen": 2320695296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029971112181030336, + "loss": 1.2832, + "theoretical_loss": 3.4171817834233638, + "tokens_seen": 2320826368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996709998395121, + "loss": 1.2294, + "theoretical_loss": 3.417166289750881, + "tokens_seen": 2320957440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996308778687209, + "loss": 1.3634, + "theoretical_loss": 3.4171507971983295, + "tokens_seen": 2321088512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995907558979297, + "loss": 1.293, + "theoretical_loss": 3.4171353057655653, + "tokens_seen": 2321219584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995506339271385, + "loss": 1.3181, + "theoretical_loss": 3.417119815452444, + "tokens_seen": 2321350656 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029951051195634733, + "loss": 1.2336, + "theoretical_loss": 3.4171043262588214, + "tokens_seen": 2321481728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994703899855561, + "loss": 1.3, + "theoretical_loss": 3.417088838184554, + "tokens_seen": 2321612800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994302680147649, + "loss": 1.2813, + "theoretical_loss": 3.4170733512294964, + "tokens_seen": 2321743872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993901460439737, + "loss": 1.2212, + "theoretical_loss": 3.417057865393506, + "tokens_seen": 2321874944 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029935002407318245, + "loss": 1.2824, + "theoretical_loss": 3.4170423806764383, + "tokens_seen": 2322006016 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029930990210239126, + "loss": 1.2544, + "theoretical_loss": 3.417026897078149, + "tokens_seen": 2322137088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029926978013160006, + "loss": 1.3279, + "theoretical_loss": 3.4170114145984942, + "tokens_seen": 2322268160 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029922965816080887, + "loss": 1.3101, + "theoretical_loss": 3.41699593323733, + "tokens_seen": 2322399232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991895361900177, + "loss": 1.231, + "theoretical_loss": 3.4169804529945127, + "tokens_seen": 2322530304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991494142192265, + "loss": 1.341, + "theoretical_loss": 3.416964973869898, + "tokens_seen": 2322661376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029910929224843524, + "loss": 1.2978, + "theoretical_loss": 3.4169494958633426, + "tokens_seen": 2322792448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029906917027764404, + "loss": 1.433, + "theoretical_loss": 3.416934018974702, + "tokens_seen": 2322923520 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029902904830685285, + "loss": 1.3666, + "theoretical_loss": 3.4169185432038334, + "tokens_seen": 2323054592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989889263360616, + "loss": 1.3441, + "theoretical_loss": 3.4169030685505914, + "tokens_seen": 2323185664 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029894880436527046, + "loss": 1.2378, + "theoretical_loss": 3.416887595014834, + "tokens_seen": 2323316736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989086823944792, + "loss": 1.2436, + "theoretical_loss": 3.4168721225964163, + "tokens_seen": 2323447808 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298868560423688, + "loss": 1.3093, + "theoretical_loss": 3.416856651295195, + "tokens_seen": 2323578880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988284384528968, + "loss": 1.2544, + "theoretical_loss": 3.4168411811110255, + "tokens_seen": 2323709952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987883164821056, + "loss": 1.2919, + "theoretical_loss": 3.416825712043766, + "tokens_seen": 2323841024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987481945113144, + "loss": 1.2938, + "theoretical_loss": 3.4168102440932717, + "tokens_seen": 2323972096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987080725405232, + "loss": 1.2918, + "theoretical_loss": 3.4167947772593985, + "tokens_seen": 2324103168 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298667950569732, + "loss": 1.3184, + "theoretical_loss": 3.416779311542004, + "tokens_seen": 2324234240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986278285989408, + "loss": 1.2723, + "theoretical_loss": 3.4167638469409445, + "tokens_seen": 2324365312 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985877066281496, + "loss": 1.2391, + "theoretical_loss": 3.4167483834560755, + "tokens_seen": 2324496384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029854758465735836, + "loss": 1.2776, + "theoretical_loss": 3.416732921087254, + "tokens_seen": 2324627456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029850746268656717, + "loss": 1.314, + "theoretical_loss": 3.416717459834337, + "tokens_seen": 2324758528 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298467340715776, + "loss": 1.2682, + "theoretical_loss": 3.416701999697181, + "tokens_seen": 2324889600 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.9931883811950684, + "objective/train/avg_token_score": 0.006811574567109346, + "objective/train/docs_used": 846164, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2895138263702393, + "objective/train/original_loss": 1.2998569011688232, + "objective/train/theoretical_loss": 3.4166865406756415, + "objective/train/tokens_used": 695479776, + "objective/train/unlikelihood_loss": 0.01965709961950779, + "theoretical_loss": 3.4166865406756415, + "tokens_seen": 2325020672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984272187449847, + "loss": 1.292, + "theoretical_loss": 3.4166865406756415, + "tokens_seen": 2325020672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983870967741936, + "loss": 1.3119, + "theoretical_loss": 3.4166710827695765, + "tokens_seen": 2325151744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029834697480340234, + "loss": 1.2571, + "theoretical_loss": 3.416655625978842, + "tokens_seen": 2325282816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029830685283261114, + "loss": 1.2348, + "theoretical_loss": 3.4166401703032943, + "tokens_seen": 2325413888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029826673086181995, + "loss": 1.2945, + "theoretical_loss": 3.416624715742791, + "tokens_seen": 2325544960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029822660889102876, + "loss": 1.3165, + "theoretical_loss": 3.416609262297188, + "tokens_seen": 2325676032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981864869202375, + "loss": 1.2549, + "theoretical_loss": 3.416593809966342, + "tokens_seen": 2325807104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981463649494463, + "loss": 1.3338, + "theoretical_loss": 3.416578358750111, + "tokens_seen": 2325938176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981062429786551, + "loss": 1.3042, + "theoretical_loss": 3.41656290864835, + "tokens_seen": 2326069248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980661210078639, + "loss": 1.3144, + "theoretical_loss": 3.4165474596609173, + "tokens_seen": 2326200320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029802599903707273, + "loss": 1.2542, + "theoretical_loss": 3.416532011787669, + "tokens_seen": 2326331392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979858770662815, + "loss": 1.2619, + "theoretical_loss": 3.4165165650284623, + "tokens_seen": 2326462464 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029794575509549035, + "loss": 1.3072, + "theoretical_loss": 3.416501119383154, + "tokens_seen": 2326593536 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979056331246991, + "loss": 1.2683, + "theoretical_loss": 3.4164856748516006, + "tokens_seen": 2326724608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029786551115390785, + "loss": 1.2419, + "theoretical_loss": 3.4164702314336597, + "tokens_seen": 2326855680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978253891831167, + "loss": 1.286, + "theoretical_loss": 3.4164547891291877, + "tokens_seen": 2326986752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029778526721232546, + "loss": 1.2959, + "theoretical_loss": 3.4164393479380424, + "tokens_seen": 2327117824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029774514524153427, + "loss": 1.3293, + "theoretical_loss": 3.4164239078600804, + "tokens_seen": 2327248896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977050232707431, + "loss": 1.2862, + "theoretical_loss": 3.4164084688951584, + "tokens_seen": 2327379968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976649012999519, + "loss": 1.2328, + "theoretical_loss": 3.4163930310431336, + "tokens_seen": 2327511040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029762477932916063, + "loss": 1.2829, + "theoretical_loss": 3.4163775943038637, + "tokens_seen": 2327642112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029758465735836944, + "loss": 1.2652, + "theoretical_loss": 3.4163621586772055, + "tokens_seen": 2327773184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029754453538757825, + "loss": 1.2489, + "theoretical_loss": 3.416346724163016, + "tokens_seen": 2327904256 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297504413416787, + "loss": 1.3204, + "theoretical_loss": 3.4163312907611525, + "tokens_seen": 2328035328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029746429144599586, + "loss": 1.3095, + "theoretical_loss": 3.416315858471472, + "tokens_seen": 2328166400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974241694752046, + "loss": 1.247, + "theoretical_loss": 3.4163004272938324, + "tokens_seen": 2328297472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029738404750441347, + "loss": 1.2546, + "theoretical_loss": 3.4162849972280904, + "tokens_seen": 2328428544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973439255336222, + "loss": 1.2992, + "theoretical_loss": 3.4162695682741036, + "tokens_seen": 2328559616 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297303803562831, + "loss": 1.3312, + "theoretical_loss": 3.416254140431729, + "tokens_seen": 2328690688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029726368159203984, + "loss": 1.2848, + "theoretical_loss": 3.416238713700824, + "tokens_seen": 2328821760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972235596212486, + "loss": 1.3398, + "theoretical_loss": 3.4162232880812464, + "tokens_seen": 2328952832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971834376504574, + "loss": 1.354, + "theoretical_loss": 3.416207863572853, + "tokens_seen": 2329083904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971433156796662, + "loss": 1.3348, + "theoretical_loss": 3.416192440175502, + "tokens_seen": 2329214976 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297103193708875, + "loss": 1.2972, + "theoretical_loss": 3.41617701788905, + "tokens_seen": 2329346048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029706307173808376, + "loss": 1.2658, + "theoretical_loss": 3.416161596713355, + "tokens_seen": 2329477120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029702294976729257, + "loss": 1.2512, + "theoretical_loss": 3.416146176648274, + "tokens_seen": 2329608192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969828277965014, + "loss": 1.2209, + "theoretical_loss": 3.416130757693666, + "tokens_seen": 2329739264 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969427058257101, + "loss": 1.2396, + "theoretical_loss": 3.4161153398493864, + "tokens_seen": 2329870336 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296902583854919, + "loss": 1.2714, + "theoretical_loss": 3.4160999231152944, + "tokens_seen": 2330001408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029686246188412774, + "loss": 1.3761, + "theoretical_loss": 3.416084507491247, + "tokens_seen": 2330132480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029682233991333654, + "loss": 1.2423, + "theoretical_loss": 3.416069092977102, + "tokens_seen": 2330263552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029678221794254535, + "loss": 1.2654, + "theoretical_loss": 3.416053679572716, + "tokens_seen": 2330394624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967420959717541, + "loss": 1.3256, + "theoretical_loss": 3.4160382672779486, + "tokens_seen": 2330525696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029670197400096296, + "loss": 1.2555, + "theoretical_loss": 3.4160228560926567, + "tokens_seen": 2330656768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966618520301717, + "loss": 1.3197, + "theoretical_loss": 3.416007446016698, + "tokens_seen": 2330787840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966217300593805, + "loss": 1.2756, + "theoretical_loss": 3.4159920370499295, + "tokens_seen": 2330918912 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029658160808858933, + "loss": 1.3, + "theoretical_loss": 3.41597662919221, + "tokens_seen": 2331049984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029654148611779813, + "loss": 1.2252, + "theoretical_loss": 3.4159612224433973, + "tokens_seen": 2331181056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965013641470069, + "loss": 1.2861, + "theoretical_loss": 3.4159458168033487, + "tokens_seen": 2331312128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029646124217621575, + "loss": 1.3295, + "theoretical_loss": 3.4159304122719227, + "tokens_seen": 2331443200 + }, + { + "epoch": 0.41, + "objective/train/aligned_token_count": 0.9901067018508911, + "objective/train/avg_token_score": 0.009893345646560192, + "objective/train/docs_used": 848727, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2686280012130737, + "objective/train/original_loss": 1.283354640007019, + "objective/train/theoretical_loss": 3.4159150088489767, + "objective/train/tokens_used": 702033376, + "objective/train/unlikelihood_loss": 0.027282942086458206, + "theoretical_loss": 3.4159150088489767, + "tokens_seen": 2331574272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964211202054245, + "loss": 1.2503, + "theoretical_loss": 3.4159150088489767, + "tokens_seen": 2331574272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029638099823463325, + "loss": 1.3579, + "theoretical_loss": 3.4158996065343685, + "tokens_seen": 2331705344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963408762638421, + "loss": 1.2721, + "theoretical_loss": 3.4158842053279566, + "tokens_seen": 2331836416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029630075429305086, + "loss": 1.2299, + "theoretical_loss": 3.4158688052295987, + "tokens_seen": 2331967488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029626063232225967, + "loss": 1.2839, + "theoretical_loss": 3.415853406239153, + "tokens_seen": 2332098560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962205103514685, + "loss": 1.2891, + "theoretical_loss": 3.415838008356477, + "tokens_seen": 2332229632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961803883806773, + "loss": 1.2948, + "theoretical_loss": 3.4158226115814294, + "tokens_seen": 2332360704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029614026640988603, + "loss": 1.3127, + "theoretical_loss": 3.415807215913868, + "tokens_seen": 2332491776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029610014443909484, + "loss": 1.215, + "theoretical_loss": 3.415791821353651, + "tokens_seen": 2332622848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029606002246830365, + "loss": 1.3237, + "theoretical_loss": 3.415776427900636, + "tokens_seen": 2332753920 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029601990049751245, + "loss": 1.2416, + "theoretical_loss": 3.4157610355546817, + "tokens_seen": 2332884992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029597977852672126, + "loss": 1.3225, + "theoretical_loss": 3.4157456443156464, + "tokens_seen": 2333016064 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029593965655593, + "loss": 1.315, + "theoretical_loss": 3.4157302541833885, + "tokens_seen": 2333147136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029589953458513887, + "loss": 1.3163, + "theoretical_loss": 3.4157148651577653, + "tokens_seen": 2333278208 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958594126143476, + "loss": 1.2987, + "theoretical_loss": 3.4156994772386358, + "tokens_seen": 2333409280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958192906435564, + "loss": 1.345, + "theoretical_loss": 3.4156840904258585, + "tokens_seen": 2333540352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029577916867276524, + "loss": 1.3119, + "theoretical_loss": 3.4156687047192906, + "tokens_seen": 2333671424 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295739046701974, + "loss": 1.3144, + "theoretical_loss": 3.4156533201187917, + "tokens_seen": 2333802496 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956989247311828, + "loss": 1.3038, + "theoretical_loss": 3.4156379366242193, + "tokens_seen": 2333933568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956588027603916, + "loss": 1.2491, + "theoretical_loss": 3.4156225542354326, + "tokens_seen": 2334064640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956186807896004, + "loss": 1.2626, + "theoretical_loss": 3.4156071729522894, + "tokens_seen": 2334195712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029557855881880916, + "loss": 1.2333, + "theoretical_loss": 3.4155917927746486, + "tokens_seen": 2334326784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029553843684801797, + "loss": 1.3376, + "theoretical_loss": 3.415576413702368, + "tokens_seen": 2334457856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029549831487722677, + "loss": 1.2972, + "theoretical_loss": 3.415561035735307, + "tokens_seen": 2334588928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002954581929064356, + "loss": 1.3064, + "theoretical_loss": 3.4155456588733233, + "tokens_seen": 2334720000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002954180709356444, + "loss": 1.3091, + "theoretical_loss": 3.4155302831162757, + "tokens_seen": 2334851072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029537794896485314, + "loss": 1.3242, + "theoretical_loss": 3.415514908464023, + "tokens_seen": 2334982144 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295337826994062, + "loss": 1.1782, + "theoretical_loss": 3.4154995349164237, + "tokens_seen": 2335113216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029529770502327075, + "loss": 1.3859, + "theoretical_loss": 3.4154841624733367, + "tokens_seen": 2335244288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952575830524795, + "loss": 1.3568, + "theoretical_loss": 3.41546879113462, + "tokens_seen": 2335375360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029521746108168836, + "loss": 1.2958, + "theoretical_loss": 3.415453420900133, + "tokens_seen": 2335506432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951773391108971, + "loss": 1.242, + "theoretical_loss": 3.4154380517697343, + "tokens_seen": 2335637504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951372171401059, + "loss": 1.2787, + "theoretical_loss": 3.415422683743282, + "tokens_seen": 2335768576 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029509709516931473, + "loss": 1.3285, + "theoretical_loss": 3.415407316820635, + "tokens_seen": 2335899648 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029505697319852353, + "loss": 1.3109, + "theoretical_loss": 3.4153919510016526, + "tokens_seen": 2336030720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950168512277323, + "loss": 1.2621, + "theoretical_loss": 3.415376586286194, + "tokens_seen": 2336161792 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029497672925694115, + "loss": 1.2499, + "theoretical_loss": 3.415361222674117, + "tokens_seen": 2336292864 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949366072861499, + "loss": 1.2369, + "theoretical_loss": 3.4153458601652806, + "tokens_seen": 2336423936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029489648531535865, + "loss": 1.3367, + "theoretical_loss": 3.415330498759544, + "tokens_seen": 2336555008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948563633445675, + "loss": 1.3093, + "theoretical_loss": 3.4153151384567666, + "tokens_seen": 2336686080 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029481624137377626, + "loss": 1.3585, + "theoretical_loss": 3.4152997792568063, + "tokens_seen": 2336817152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947761194029851, + "loss": 1.2631, + "theoretical_loss": 3.415284421159523, + "tokens_seen": 2336948224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947359974321939, + "loss": 1.2835, + "theoretical_loss": 3.415269064164775, + "tokens_seen": 2337079296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946958754614027, + "loss": 1.2796, + "theoretical_loss": 3.415253708272422, + "tokens_seen": 2337210368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946557534906115, + "loss": 1.3368, + "theoretical_loss": 3.415238353482323, + "tokens_seen": 2337341440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029461563151982024, + "loss": 1.344, + "theoretical_loss": 3.415222999794336, + "tokens_seen": 2337472512 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029457550954902905, + "loss": 1.2879, + "theoretical_loss": 3.4152076472083213, + "tokens_seen": 2337603584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029453538757823785, + "loss": 1.2331, + "theoretical_loss": 3.4151922957241374, + "tokens_seen": 2337734656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029449526560744666, + "loss": 1.2383, + "theoretical_loss": 3.415176945341644, + "tokens_seen": 2337865728 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944551436366554, + "loss": 1.3518, + "theoretical_loss": 3.4151615960607, + "tokens_seen": 2337996800 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.9922761917114258, + "objective/train/avg_token_score": 0.007723750080913305, + "objective/train/docs_used": 851124, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3515827655792236, + "objective/train/original_loss": 1.3626155853271484, + "objective/train/theoretical_loss": 3.4151462478811636, + "objective/train/tokens_used": 708586976, + "objective/train/unlikelihood_loss": 0.02006153017282486, + "theoretical_loss": 3.4151462478811636, + "tokens_seen": 2338127872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029441502166586427, + "loss": 1.3318, + "theoretical_loss": 3.4151462478811636, + "tokens_seen": 2338127872 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294374899695073, + "loss": 1.2966, + "theoretical_loss": 3.415130900802896, + "tokens_seen": 2338258944 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943347777242818, + "loss": 1.2765, + "theoretical_loss": 3.415115554825755, + "tokens_seen": 2338390016 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029429465575349064, + "loss": 1.1896, + "theoretical_loss": 3.4151002099496, + "tokens_seen": 2338521088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942545337826994, + "loss": 1.3779, + "theoretical_loss": 3.415084866174291, + "tokens_seen": 2338652160 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029421441181190825, + "loss": 1.319, + "theoretical_loss": 3.415069523499687, + "tokens_seen": 2338783232 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294174289841117, + "loss": 1.2959, + "theoretical_loss": 3.4150541819256475, + "tokens_seen": 2338914304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941341678703258, + "loss": 1.2198, + "theoretical_loss": 3.415038841452031, + "tokens_seen": 2339045376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940940458995346, + "loss": 1.2273, + "theoretical_loss": 3.4150235020786983, + "tokens_seen": 2339176448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029405392392874337, + "loss": 1.2757, + "theoretical_loss": 3.4150081638055076, + "tokens_seen": 2339307520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029401380195795217, + "loss": 1.3085, + "theoretical_loss": 3.414992826632319, + "tokens_seen": 2339438592 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293973679987161, + "loss": 1.4108, + "theoretical_loss": 3.414977490558992, + "tokens_seen": 2339569664 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939335580163698, + "loss": 1.2832, + "theoretical_loss": 3.414962155585386, + "tokens_seen": 2339700736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029389343604557854, + "loss": 1.2358, + "theoretical_loss": 3.4149468217113608, + "tokens_seen": 2339831808 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938533140747874, + "loss": 1.2937, + "theoretical_loss": 3.4149314889367752, + "tokens_seen": 2339962880 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029381319210399615, + "loss": 1.2778, + "theoretical_loss": 3.4149161572614894, + "tokens_seen": 2340093952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937730701332049, + "loss": 1.2897, + "theoretical_loss": 3.414900826685363, + "tokens_seen": 2340225024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029373294816241376, + "loss": 1.325, + "theoretical_loss": 3.414885497208256, + "tokens_seen": 2340356096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936928261916225, + "loss": 1.3168, + "theoretical_loss": 3.414870168830027, + "tokens_seen": 2340487168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936527042208313, + "loss": 1.3074, + "theoretical_loss": 3.414854841550537, + "tokens_seen": 2340618240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936125822500401, + "loss": 1.3256, + "theoretical_loss": 3.4148395153696445, + "tokens_seen": 2340749312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029357246027924893, + "loss": 1.2616, + "theoretical_loss": 3.41482419028721, + "tokens_seen": 2340880384 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029353233830845774, + "loss": 1.339, + "theoretical_loss": 3.414808866303093, + "tokens_seen": 2341011456 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029349221633766655, + "loss": 1.2275, + "theoretical_loss": 3.414793543417153, + "tokens_seen": 2341142528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934520943668753, + "loss": 1.2873, + "theoretical_loss": 3.414778221629251, + "tokens_seen": 2341273600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934119723960841, + "loss": 1.2895, + "theoretical_loss": 3.4147629009392455, + "tokens_seen": 2341404672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933718504252929, + "loss": 1.2804, + "theoretical_loss": 3.414747581346997, + "tokens_seen": 2341535744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029333172845450166, + "loss": 1.3002, + "theoretical_loss": 3.4147322628523655, + "tokens_seen": 2341666816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932916064837105, + "loss": 1.2989, + "theoretical_loss": 3.4147169454552104, + "tokens_seen": 2341797888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932514845129193, + "loss": 1.2556, + "theoretical_loss": 3.4147016291553918, + "tokens_seen": 2341928960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932113625421281, + "loss": 1.2853, + "theoretical_loss": 3.4146863139527706, + "tokens_seen": 2342060032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931712405713369, + "loss": 1.2912, + "theoretical_loss": 3.4146709998472056, + "tokens_seen": 2342191104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029313111860054564, + "loss": 1.3777, + "theoretical_loss": 3.4146556868385574, + "tokens_seen": 2342322176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029309099662975445, + "loss": 1.299, + "theoretical_loss": 3.414640374926686, + "tokens_seen": 2342453248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029305087465896325, + "loss": 1.3014, + "theoretical_loss": 3.4146250641114513, + "tokens_seen": 2342584320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029301075268817206, + "loss": 1.3085, + "theoretical_loss": 3.414609754392714, + "tokens_seen": 2342715392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929706307173808, + "loss": 1.3572, + "theoretical_loss": 3.414594445770333, + "tokens_seen": 2342846464 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029293050874658967, + "loss": 1.273, + "theoretical_loss": 3.41457913824417, + "tokens_seen": 2342977536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928903867757984, + "loss": 1.2628, + "theoretical_loss": 3.414563831814084, + "tokens_seen": 2343108608 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029285026480500723, + "loss": 1.3436, + "theoretical_loss": 3.414548526479936, + "tokens_seen": 2343239680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029281014283421604, + "loss": 1.2582, + "theoretical_loss": 3.4145332222415856, + "tokens_seen": 2343370752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927700208634248, + "loss": 1.2698, + "theoretical_loss": 3.4145179190988935, + "tokens_seen": 2343501824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029272989889263365, + "loss": 1.378, + "theoretical_loss": 3.4145026170517196, + "tokens_seen": 2343632896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926897769218424, + "loss": 1.2986, + "theoretical_loss": 3.4144873160999243, + "tokens_seen": 2343763968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926496549510512, + "loss": 1.282, + "theoretical_loss": 3.4144720162433684, + "tokens_seen": 2343895040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029260953298026, + "loss": 1.3006, + "theoretical_loss": 3.414456717481912, + "tokens_seen": 2344026112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029256941100946877, + "loss": 1.3014, + "theoretical_loss": 3.4144414198154154, + "tokens_seen": 2344157184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029252928903867757, + "loss": 1.2968, + "theoretical_loss": 3.4144261232437385, + "tokens_seen": 2344288256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924891670678864, + "loss": 1.2643, + "theoretical_loss": 3.4144108277667424, + "tokens_seen": 2344419328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924490450970952, + "loss": 1.2404, + "theoretical_loss": 3.4143955333842877, + "tokens_seen": 2344550400 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.9915265440940857, + "objective/train/avg_token_score": 0.008473447524011135, + "objective/train/docs_used": 853299, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2097439765930176, + "objective/train/original_loss": 1.2226927280426025, + "objective/train/theoretical_loss": 3.414380240096235, + "objective/train/tokens_used": 715140576, + "objective/train/unlikelihood_loss": 0.024785520508885384, + "theoretical_loss": 3.414380240096235, + "tokens_seen": 2344681472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029240892312630394, + "loss": 1.2783, + "theoretical_loss": 3.414380240096235, + "tokens_seen": 2344681472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923688011555128, + "loss": 1.2907, + "theoretical_loss": 3.414364947902444, + "tokens_seen": 2344812544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029232867918472155, + "loss": 1.2447, + "theoretical_loss": 3.4143496568027754, + "tokens_seen": 2344943616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922885572139303, + "loss": 1.3851, + "theoretical_loss": 3.4143343667970907, + "tokens_seen": 2345074688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029224843524313916, + "loss": 1.3028, + "theoretical_loss": 3.4143190778852492, + "tokens_seen": 2345205760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922083132723479, + "loss": 1.2078, + "theoretical_loss": 3.4143037900671125, + "tokens_seen": 2345336832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921681913015568, + "loss": 1.3703, + "theoretical_loss": 3.414288503342541, + "tokens_seen": 2345467904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921280693307655, + "loss": 1.2952, + "theoretical_loss": 3.4142732177113952, + "tokens_seen": 2345598976 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029208794735997433, + "loss": 1.1956, + "theoretical_loss": 3.414257933173536, + "tokens_seen": 2345730048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029204782538918314, + "loss": 1.2829, + "theoretical_loss": 3.414242649728824, + "tokens_seen": 2345861120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920077034183919, + "loss": 1.2846, + "theoretical_loss": 3.4142273673771197, + "tokens_seen": 2345992192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919675814476007, + "loss": 1.3007, + "theoretical_loss": 3.414212086118284, + "tokens_seen": 2346123264 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002919274594768095, + "loss": 1.2822, + "theoretical_loss": 3.4141968059521783, + "tokens_seen": 2346254336 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918873375060183, + "loss": 1.2663, + "theoretical_loss": 3.4141815268786626, + "tokens_seen": 2346385408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029184721553522706, + "loss": 1.3106, + "theoretical_loss": 3.4141662488975983, + "tokens_seen": 2346516480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918070935644359, + "loss": 1.2888, + "theoretical_loss": 3.414150972008846, + "tokens_seen": 2346647552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917669715936447, + "loss": 1.3175, + "theoretical_loss": 3.4141356962122664, + "tokens_seen": 2346778624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917268496228535, + "loss": 1.2693, + "theoretical_loss": 3.4141204215077208, + "tokens_seen": 2346909696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916867276520623, + "loss": 1.2933, + "theoretical_loss": 3.4141051478950706, + "tokens_seen": 2347040768 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029164660568127104, + "loss": 1.3233, + "theoretical_loss": 3.4140898753741755, + "tokens_seen": 2347171840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916064837104799, + "loss": 1.3233, + "theoretical_loss": 3.4140746039448975, + "tokens_seen": 2347302912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029156636173968865, + "loss": 1.2796, + "theoretical_loss": 3.4140593336070975, + "tokens_seen": 2347433984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029152623976889746, + "loss": 1.259, + "theoretical_loss": 3.414044064360636, + "tokens_seen": 2347565056 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029148611779810626, + "loss": 1.3472, + "theoretical_loss": 3.4140287962053746, + "tokens_seen": 2347696128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029144599582731507, + "loss": 1.2929, + "theoretical_loss": 3.414013529141174, + "tokens_seen": 2347827200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914058738565238, + "loss": 1.2758, + "theoretical_loss": 3.4139982631678962, + "tokens_seen": 2347958272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029136575188573263, + "loss": 1.316, + "theoretical_loss": 3.4139829982854017, + "tokens_seen": 2348089344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029132562991494144, + "loss": 1.2058, + "theoretical_loss": 3.4139677344935517, + "tokens_seen": 2348220416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912855079441502, + "loss": 1.1952, + "theoretical_loss": 3.413952471792207, + "tokens_seen": 2348351488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029124538597335905, + "loss": 1.2866, + "theoretical_loss": 3.4139372101812295, + "tokens_seen": 2348482560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912052640025678, + "loss": 1.2961, + "theoretical_loss": 3.4139219496604802, + "tokens_seen": 2348613632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911651420317766, + "loss": 1.3697, + "theoretical_loss": 3.4139066902298207, + "tokens_seen": 2348744704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911250200609854, + "loss": 1.1891, + "theoretical_loss": 3.4138914318891116, + "tokens_seen": 2348875776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029108489809019417, + "loss": 1.2874, + "theoretical_loss": 3.413876174638215, + "tokens_seen": 2349006848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029104477611940297, + "loss": 1.3096, + "theoretical_loss": 3.413860918476992, + "tokens_seen": 2349137920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910046541486118, + "loss": 1.3183, + "theoretical_loss": 3.4138456634053034, + "tokens_seen": 2349268992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909645321778206, + "loss": 1.2861, + "theoretical_loss": 3.4138304094230114, + "tokens_seen": 2349400064 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909244102070294, + "loss": 1.3115, + "theoretical_loss": 3.4138151565299775, + "tokens_seen": 2349531136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908842882362382, + "loss": 1.2986, + "theoretical_loss": 3.413799904726062, + "tokens_seen": 2349662208 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029084416626544695, + "loss": 1.2343, + "theoretical_loss": 3.4137846540111276, + "tokens_seen": 2349793280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029080404429465576, + "loss": 1.3232, + "theoretical_loss": 3.4137694043850355, + "tokens_seen": 2349924352 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029076392232386456, + "loss": 1.2847, + "theoretical_loss": 3.413754155847647, + "tokens_seen": 2350055424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907238003530733, + "loss": 1.2852, + "theoretical_loss": 3.4137389083988237, + "tokens_seen": 2350186496 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906836783822822, + "loss": 1.3098, + "theoretical_loss": 3.4137236620384277, + "tokens_seen": 2350317568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906435564114909, + "loss": 1.3299, + "theoretical_loss": 3.4137084167663194, + "tokens_seen": 2350448640 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029060343444069973, + "loss": 1.3193, + "theoretical_loss": 3.4136931725823616, + "tokens_seen": 2350579712 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029056331246990854, + "loss": 1.3625, + "theoretical_loss": 3.4136779294864157, + "tokens_seen": 2350710784 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905231904991173, + "loss": 1.3143, + "theoretical_loss": 3.413662687478343, + "tokens_seen": 2350841856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904830685283261, + "loss": 1.3249, + "theoretical_loss": 3.413647446558006, + "tokens_seen": 2350972928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904429465575349, + "loss": 1.2797, + "theoretical_loss": 3.413632206725265, + "tokens_seen": 2351104000 + }, + { + "epoch": 0.42, + "objective/train/aligned_token_count": 0.991912305355072, + "objective/train/avg_token_score": 0.008087667636573315, + "objective/train/docs_used": 855824, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.304945945739746, + "objective/train/original_loss": 1.317075490951538, + "objective/train/theoretical_loss": 3.4136169679799835, + "objective/train/tokens_used": 721694176, + "objective/train/unlikelihood_loss": 0.022003687918186188, + "theoretical_loss": 3.4136169679799835, + "tokens_seen": 2351235072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002904028245867437, + "loss": 1.2969, + "theoretical_loss": 3.4136169679799835, + "tokens_seen": 2351235072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903627026159525, + "loss": 1.3246, + "theoretical_loss": 3.4136017303220223, + "tokens_seen": 2351366144 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903225806451613, + "loss": 1.3336, + "theoretical_loss": 3.413586493751243, + "tokens_seen": 2351497216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902824586743701, + "loss": 1.2458, + "theoretical_loss": 3.413571258267508, + "tokens_seen": 2351628288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029024233670357894, + "loss": 1.3055, + "theoretical_loss": 3.413556023870679, + "tokens_seen": 2351759360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902022147327877, + "loss": 1.238, + "theoretical_loss": 3.413540790560618, + "tokens_seen": 2351890432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029016209276199644, + "loss": 1.3604, + "theoretical_loss": 3.4135255583371866, + "tokens_seen": 2352021504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901219707912053, + "loss": 1.2819, + "theoretical_loss": 3.413510327200247, + "tokens_seen": 2352152576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029008184882041405, + "loss": 1.3272, + "theoretical_loss": 3.413495097149661, + "tokens_seen": 2352283648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029004172684962286, + "loss": 1.407, + "theoretical_loss": 3.413479868185291, + "tokens_seen": 2352414720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029000160487883166, + "loss": 1.317, + "theoretical_loss": 3.4134646403069984, + "tokens_seen": 2352545792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028996148290804047, + "loss": 1.2831, + "theoretical_loss": 3.4134494135146456, + "tokens_seen": 2352676864 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899213609372492, + "loss": 1.2737, + "theoretical_loss": 3.4134341878080945, + "tokens_seen": 2352807936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028988123896645803, + "loss": 1.2786, + "theoretical_loss": 3.4134189631872074, + "tokens_seen": 2352939008 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028984111699566684, + "loss": 1.2939, + "theoretical_loss": 3.413403739651846, + "tokens_seen": 2353070080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898009950248756, + "loss": 1.2553, + "theoretical_loss": 3.4133885172018728, + "tokens_seen": 2353201152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028976087305408445, + "loss": 1.3129, + "theoretical_loss": 3.4133732958371503, + "tokens_seen": 2353332224 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897207510832932, + "loss": 1.3015, + "theoretical_loss": 3.4133580755575403, + "tokens_seen": 2353463296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028968062911250206, + "loss": 1.3151, + "theoretical_loss": 3.4133428563629047, + "tokens_seen": 2353594368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896405071417108, + "loss": 1.307, + "theoretical_loss": 3.413327638253106, + "tokens_seen": 2353725440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028960038517091957, + "loss": 1.3179, + "theoretical_loss": 3.4133124212280066, + "tokens_seen": 2353856512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895602632001284, + "loss": 1.2588, + "theoretical_loss": 3.413297205287469, + "tokens_seen": 2353987584 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895201412293372, + "loss": 1.2376, + "theoretical_loss": 3.413281990431355, + "tokens_seen": 2354118656 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289480019258546, + "loss": 1.2618, + "theoretical_loss": 3.413266776659527, + "tokens_seen": 2354249728 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894398972877548, + "loss": 1.3096, + "theoretical_loss": 3.413251563971848, + "tokens_seen": 2354380800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893997753169636, + "loss": 1.2941, + "theoretical_loss": 3.4132363523681795, + "tokens_seen": 2354511872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028935965334617235, + "loss": 1.3098, + "theoretical_loss": 3.4132211418483847, + "tokens_seen": 2354642944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028931953137538115, + "loss": 1.2807, + "theoretical_loss": 3.4132059324123256, + "tokens_seen": 2354774016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028927940940458996, + "loss": 1.2249, + "theoretical_loss": 3.4131907240598647, + "tokens_seen": 2354905088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892392874337987, + "loss": 1.295, + "theoretical_loss": 3.4131755167908646, + "tokens_seen": 2355036160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891991654630076, + "loss": 1.3063, + "theoretical_loss": 3.4131603106051878, + "tokens_seen": 2355167232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891590434922163, + "loss": 1.2828, + "theoretical_loss": 3.413145105502697, + "tokens_seen": 2355298304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891189215214252, + "loss": 1.2948, + "theoretical_loss": 3.4131299014832543, + "tokens_seen": 2355429376 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028907879955063394, + "loss": 1.296, + "theoretical_loss": 3.4131146985467224, + "tokens_seen": 2355560448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890386775798427, + "loss": 1.2885, + "theoretical_loss": 3.4130994966929644, + "tokens_seen": 2355691520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028899855560905155, + "loss": 1.2542, + "theoretical_loss": 3.4130842959218426, + "tokens_seen": 2355822592 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889584336382603, + "loss": 1.3125, + "theoretical_loss": 3.4130690962332197, + "tokens_seen": 2355953664 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889183116674691, + "loss": 1.2822, + "theoretical_loss": 3.4130538976269587, + "tokens_seen": 2356084736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888781896966779, + "loss": 1.2906, + "theoretical_loss": 3.4130387001029217, + "tokens_seen": 2356215808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888380677258867, + "loss": 1.2633, + "theoretical_loss": 3.4130235036609715, + "tokens_seen": 2356346880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887979457550955, + "loss": 1.2929, + "theoretical_loss": 3.413008308300972, + "tokens_seen": 2356477952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028875782378430433, + "loss": 1.2534, + "theoretical_loss": 3.4129931140227843, + "tokens_seen": 2356609024 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887177018135131, + "loss": 1.2468, + "theoretical_loss": 3.4129779208262723, + "tokens_seen": 2356740096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028867757984272184, + "loss": 1.332, + "theoretical_loss": 3.4129627287112987, + "tokens_seen": 2356871168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886374578719307, + "loss": 1.2606, + "theoretical_loss": 3.4129475376777263, + "tokens_seen": 2357002240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028859733590113945, + "loss": 1.2811, + "theoretical_loss": 3.412932347725418, + "tokens_seen": 2357133312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028855721393034826, + "loss": 1.2616, + "theoretical_loss": 3.4129171588542366, + "tokens_seen": 2357264384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028851709195955706, + "loss": 1.3495, + "theoretical_loss": 3.412901971064045, + "tokens_seen": 2357395456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028847696998876587, + "loss": 1.3272, + "theoretical_loss": 3.4128867843547064, + "tokens_seen": 2357526528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884368480179747, + "loss": 1.2914, + "theoretical_loss": 3.412871598726084, + "tokens_seen": 2357657600 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.9918233156204224, + "objective/train/avg_token_score": 0.008176704868674278, + "objective/train/docs_used": 858137, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2078763246536255, + "objective/train/original_loss": 1.2210543155670166, + "objective/train/theoretical_loss": 3.41285641417804, + "objective/train/tokens_used": 728247776, + "objective/train/unlikelihood_loss": 0.028085554018616676, + "theoretical_loss": 3.41285641417804, + "tokens_seen": 2357788672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028839672604718343, + "loss": 1.2238, + "theoretical_loss": 3.41285641417804, + "tokens_seen": 2357788672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028835660407639224, + "loss": 1.2519, + "theoretical_loss": 3.4128412307104377, + "tokens_seen": 2357919744 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028831648210560104, + "loss": 1.3051, + "theoretical_loss": 3.412826048323141, + "tokens_seen": 2358050816 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028827636013480985, + "loss": 1.2928, + "theoretical_loss": 3.412810867016012, + "tokens_seen": 2358181888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882362381640186, + "loss": 1.3134, + "theoretical_loss": 3.4127956867889147, + "tokens_seen": 2358312960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028819611619322746, + "loss": 1.2822, + "theoretical_loss": 3.4127805076417115, + "tokens_seen": 2358444032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881559942224362, + "loss": 1.2588, + "theoretical_loss": 3.412765329574266, + "tokens_seen": 2358575104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028811587225164496, + "loss": 1.2431, + "theoretical_loss": 3.412750152586441, + "tokens_seen": 2358706176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880757502808538, + "loss": 1.3587, + "theoretical_loss": 3.4127349766781, + "tokens_seen": 2358837248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880356283100626, + "loss": 1.2964, + "theoretical_loss": 3.4127198018491067, + "tokens_seen": 2358968320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879955063392714, + "loss": 1.3298, + "theoretical_loss": 3.4127046280993234, + "tokens_seen": 2359099392 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879553843684802, + "loss": 1.2858, + "theoretical_loss": 3.412689455428614, + "tokens_seen": 2359230464 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287915262397689, + "loss": 1.2513, + "theoretical_loss": 3.4126742838368416, + "tokens_seen": 2359361536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028787514042689775, + "loss": 1.2628, + "theoretical_loss": 3.4126591133238695, + "tokens_seen": 2359492608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028783501845610655, + "loss": 1.3035, + "theoretical_loss": 3.4126439438895613, + "tokens_seen": 2359623680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028779489648531536, + "loss": 1.2662, + "theoretical_loss": 3.41262877553378, + "tokens_seen": 2359754752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028775477451452417, + "loss": 1.3465, + "theoretical_loss": 3.41261360825639, + "tokens_seen": 2359885824 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287714652543733, + "loss": 1.2723, + "theoretical_loss": 3.412598442057254, + "tokens_seen": 2360016896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876745305729417, + "loss": 1.2683, + "theoretical_loss": 3.4125832769362345, + "tokens_seen": 2360147968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876344086021506, + "loss": 1.2963, + "theoretical_loss": 3.4125681128931973, + "tokens_seen": 2360279040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028759428663135934, + "loss": 1.2255, + "theoretical_loss": 3.4125529499280036, + "tokens_seen": 2360410112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875541646605681, + "loss": 1.281, + "theoretical_loss": 3.4125377880405185, + "tokens_seen": 2360541184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028751404268977695, + "loss": 1.2729, + "theoretical_loss": 3.412522627230605, + "tokens_seen": 2360672256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874739207189857, + "loss": 1.2637, + "theoretical_loss": 3.4125074674981266, + "tokens_seen": 2360803328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874337987481945, + "loss": 1.3322, + "theoretical_loss": 3.412492308842947, + "tokens_seen": 2360934400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873936767774033, + "loss": 1.2845, + "theoretical_loss": 3.41247715126493, + "tokens_seen": 2361065472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873535548066121, + "loss": 1.3235, + "theoretical_loss": 3.412461994763939, + "tokens_seen": 2361196544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873134328358209, + "loss": 1.2698, + "theoretical_loss": 3.412446839339838, + "tokens_seen": 2361327616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872733108650297, + "loss": 1.22, + "theoretical_loss": 3.4124316849924905, + "tokens_seen": 2361458688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872331888942385, + "loss": 1.3058, + "theoretical_loss": 3.4124165317217603, + "tokens_seen": 2361589760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871930669234473, + "loss": 1.2058, + "theoretical_loss": 3.4124013795275117, + "tokens_seen": 2361720832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871529449526561, + "loss": 1.2435, + "theoretical_loss": 3.4123862284096074, + "tokens_seen": 2361851904 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028711282298186485, + "loss": 1.3124, + "theoretical_loss": 3.4123710783679115, + "tokens_seen": 2361982976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870727010110737, + "loss": 1.3095, + "theoretical_loss": 3.4123559294022883, + "tokens_seen": 2362114048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028703257904028246, + "loss": 1.3386, + "theoretical_loss": 3.4123407815126017, + "tokens_seen": 2362245120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028699245706949127, + "loss": 1.2204, + "theoretical_loss": 3.4123256346987154, + "tokens_seen": 2362376192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869523350987001, + "loss": 1.3433, + "theoretical_loss": 3.412310488960493, + "tokens_seen": 2362507264 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028691221312790883, + "loss": 1.2824, + "theoretical_loss": 3.412295344297799, + "tokens_seen": 2362638336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028687209115711763, + "loss": 1.2574, + "theoretical_loss": 3.412280200710497, + "tokens_seen": 2362769408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028683196918632644, + "loss": 1.3475, + "theoretical_loss": 3.412265058198451, + "tokens_seen": 2362900480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028679184721553525, + "loss": 1.336, + "theoretical_loss": 3.412249916761525, + "tokens_seen": 2363031552 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286751725244744, + "loss": 1.2734, + "theoretical_loss": 3.4122347763995835, + "tokens_seen": 2363162624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028671160327395286, + "loss": 1.2754, + "theoretical_loss": 3.4122196371124898, + "tokens_seen": 2363293696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866714813031616, + "loss": 1.2972, + "theoretical_loss": 3.4122044989001084, + "tokens_seen": 2363424768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028663135933237036, + "loss": 1.3535, + "theoretical_loss": 3.4121893617623034, + "tokens_seen": 2363555840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865912373615792, + "loss": 1.2332, + "theoretical_loss": 3.412174225698939, + "tokens_seen": 2363686912 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286551115390788, + "loss": 1.3234, + "theoretical_loss": 3.4121590907098795, + "tokens_seen": 2363817984 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028651099341999684, + "loss": 1.2902, + "theoretical_loss": 3.4121439567949885, + "tokens_seen": 2363949056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864708714492056, + "loss": 1.2701, + "theoretical_loss": 3.4121288239541308, + "tokens_seen": 2364080128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864307494784144, + "loss": 1.2561, + "theoretical_loss": 3.4121136921871704, + "tokens_seen": 2364211200 + }, + { + "epoch": 0.43, + "objective/train/aligned_token_count": 0.9936105012893677, + "objective/train/avg_token_score": 0.0063894521445035934, + "objective/train/docs_used": 860660, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2636066675186157, + "objective/train/original_loss": 1.2717480659484863, + "objective/train/theoretical_loss": 3.4120985614939716, + "objective/train/tokens_used": 734801376, + "objective/train/unlikelihood_loss": 0.017424358054995537, + "theoretical_loss": 3.4120985614939716, + "tokens_seen": 2364342272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863906275076232, + "loss": 1.2664, + "theoretical_loss": 3.4120985614939716, + "tokens_seen": 2364342272 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028635050553683195, + "loss": 1.3809, + "theoretical_loss": 3.4120834318743984, + "tokens_seen": 2364473344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028631038356604076, + "loss": 1.2511, + "theoretical_loss": 3.412068303328316, + "tokens_seen": 2364604416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028627026159524957, + "loss": 1.2927, + "theoretical_loss": 3.4120531758555877, + "tokens_seen": 2364735488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862301396244584, + "loss": 1.3136, + "theoretical_loss": 3.4120380494560782, + "tokens_seen": 2364866560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861900176536671, + "loss": 1.3205, + "theoretical_loss": 3.412022924129652, + "tokens_seen": 2364997632 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286149895682876, + "loss": 1.3019, + "theoretical_loss": 3.4120077998761733, + "tokens_seen": 2365128704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028610977371208474, + "loss": 1.2939, + "theoretical_loss": 3.411992676695507, + "tokens_seen": 2365259776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860696517412935, + "loss": 1.2773, + "theoretical_loss": 3.411977554587517, + "tokens_seen": 2365390848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028602952977050235, + "loss": 1.345, + "theoretical_loss": 3.4119624335520684, + "tokens_seen": 2365521920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859894077997111, + "loss": 1.3408, + "theoretical_loss": 3.411947313589025, + "tokens_seen": 2365652992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028594928582891996, + "loss": 1.1869, + "theoretical_loss": 3.411932194698252, + "tokens_seen": 2365784064 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859091638581287, + "loss": 1.3012, + "theoretical_loss": 3.411917076879613, + "tokens_seen": 2365915136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858690418873375, + "loss": 1.2599, + "theoretical_loss": 3.411901960132974, + "tokens_seen": 2366046208 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028582891991654633, + "loss": 1.2807, + "theoretical_loss": 3.4118868444581985, + "tokens_seen": 2366177280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857887979457551, + "loss": 1.3241, + "theoretical_loss": 3.411871729855151, + "tokens_seen": 2366308352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857486759749639, + "loss": 1.3191, + "theoretical_loss": 3.411856616323697, + "tokens_seen": 2366439424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857085540041727, + "loss": 1.3062, + "theoretical_loss": 3.411841503863701, + "tokens_seen": 2366570496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856684320333815, + "loss": 1.2727, + "theoretical_loss": 3.411826392475027, + "tokens_seen": 2366701568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028562831006259025, + "loss": 1.2807, + "theoretical_loss": 3.411811282157541, + "tokens_seen": 2366832640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855881880917991, + "loss": 1.2964, + "theoretical_loss": 3.411796172911106, + "tokens_seen": 2366963712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028554806612100786, + "loss": 1.2699, + "theoretical_loss": 3.4117810647355884, + "tokens_seen": 2367094784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028550794415021667, + "loss": 1.306, + "theoretical_loss": 3.4117659576308523, + "tokens_seen": 2367225856 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854678221794255, + "loss": 1.2987, + "theoretical_loss": 3.4117508515967625, + "tokens_seen": 2367356928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028542770020863423, + "loss": 1.1903, + "theoretical_loss": 3.411735746633184, + "tokens_seen": 2367488000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028538757823784303, + "loss": 1.324, + "theoretical_loss": 3.411720642739981, + "tokens_seen": 2367619072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028534745626705184, + "loss": 1.2896, + "theoretical_loss": 3.41170553991702, + "tokens_seen": 2367750144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028530733429626065, + "loss": 1.2581, + "theoretical_loss": 3.411690438164164, + "tokens_seen": 2367881216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028526721232546945, + "loss": 1.32, + "theoretical_loss": 3.411675337481279, + "tokens_seen": 2368012288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028522709035467826, + "loss": 1.1887, + "theoretical_loss": 3.4116602378682304, + "tokens_seen": 2368143360 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285186968383887, + "loss": 1.3901, + "theoretical_loss": 3.4116451393248823, + "tokens_seen": 2368274432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851468464130958, + "loss": 1.2591, + "theoretical_loss": 3.4116300418511, + "tokens_seen": 2368405504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851067244423046, + "loss": 1.299, + "theoretical_loss": 3.411614945446749, + "tokens_seen": 2368536576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850666024715134, + "loss": 1.2968, + "theoretical_loss": 3.4115998501116938, + "tokens_seen": 2368667648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028502648050072224, + "loss": 1.3353, + "theoretical_loss": 3.4115847558457997, + "tokens_seen": 2368798720 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284986358529931, + "loss": 1.352, + "theoretical_loss": 3.411569662648932, + "tokens_seen": 2368929792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849462365591398, + "loss": 1.2787, + "theoretical_loss": 3.411554570520955, + "tokens_seen": 2369060864 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849061145883486, + "loss": 1.1823, + "theoretical_loss": 3.411539479461735, + "tokens_seen": 2369191936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028486599261755735, + "loss": 1.328, + "theoretical_loss": 3.4115243894711367, + "tokens_seen": 2369323008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028482587064676616, + "loss": 1.2682, + "theoretical_loss": 3.4115093005490253, + "tokens_seen": 2369454080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028478574867597497, + "loss": 1.311, + "theoretical_loss": 3.411494212695266, + "tokens_seen": 2369585152 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028474562670518377, + "loss": 1.231, + "theoretical_loss": 3.4114791259097244, + "tokens_seen": 2369716224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847055047343925, + "loss": 1.2967, + "theoretical_loss": 3.411464040192265, + "tokens_seen": 2369847296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846653827636014, + "loss": 1.274, + "theoretical_loss": 3.4114489555427543, + "tokens_seen": 2369978368 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028462526079281014, + "loss": 1.3001, + "theoretical_loss": 3.411433871961057, + "tokens_seen": 2370109440 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028458513882201894, + "loss": 1.3174, + "theoretical_loss": 3.411418789447038, + "tokens_seen": 2370240512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028454501685122775, + "loss": 1.3008, + "theoretical_loss": 3.4114037080005635, + "tokens_seen": 2370371584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845048948804365, + "loss": 1.2151, + "theoretical_loss": 3.4113886276214984, + "tokens_seen": 2370502656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028446477290964536, + "loss": 1.304, + "theoretical_loss": 3.411373548309708, + "tokens_seen": 2370633728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844246509388541, + "loss": 1.2669, + "theoretical_loss": 3.4113584700650588, + "tokens_seen": 2370764800 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.9924556016921997, + "objective/train/avg_token_score": 0.0075443582609295845, + "objective/train/docs_used": 863154, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2076748609542847, + "objective/train/original_loss": 1.2176762819290161, + "objective/train/theoretical_loss": 3.4113433928874146, + "objective/train/tokens_used": 741354976, + "objective/train/unlikelihood_loss": 0.022650009021162987, + "theoretical_loss": 3.4113433928874146, + "tokens_seen": 2370895872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843845289680629, + "loss": 1.2435, + "theoretical_loss": 3.4113433928874146, + "tokens_seen": 2370895872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028434440699727173, + "loss": 1.327, + "theoretical_loss": 3.411328316776643, + "tokens_seen": 2371026944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843042850264805, + "loss": 1.365, + "theoretical_loss": 3.4113132417326075, + "tokens_seen": 2371158016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842641630556893, + "loss": 1.3383, + "theoretical_loss": 3.411298167755175, + "tokens_seen": 2371289088 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842240410848981, + "loss": 1.3188, + "theoretical_loss": 3.4112830948442108, + "tokens_seen": 2371420160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841839191141069, + "loss": 1.3557, + "theoretical_loss": 3.4112680229995798, + "tokens_seen": 2371551232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028414379714331565, + "loss": 1.3345, + "theoretical_loss": 3.4112529522211483, + "tokens_seen": 2371682304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841036751725245, + "loss": 1.3026, + "theoretical_loss": 3.4112378825087823, + "tokens_seen": 2371813376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028406355320173326, + "loss": 1.2964, + "theoretical_loss": 3.411222813862347, + "tokens_seen": 2371944448 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284023431230942, + "loss": 1.2773, + "theoretical_loss": 3.4112077462817076, + "tokens_seen": 2372075520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839833092601509, + "loss": 1.2599, + "theoretical_loss": 3.4111926797667307, + "tokens_seen": 2372206592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028394318728935963, + "loss": 1.22, + "theoretical_loss": 3.411177614317282, + "tokens_seen": 2372337664 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839030653185685, + "loss": 1.2133, + "theoretical_loss": 3.4111625499332265, + "tokens_seen": 2372468736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028386294334777724, + "loss": 1.3328, + "theoretical_loss": 3.4111474866144307, + "tokens_seen": 2372599808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028382282137698605, + "loss": 1.3012, + "theoretical_loss": 3.4111324243607606, + "tokens_seen": 2372730880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028378269940619485, + "loss": 1.3508, + "theoretical_loss": 3.4111173631720813, + "tokens_seen": 2372861952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028374257743540366, + "loss": 1.2291, + "theoretical_loss": 3.411102303048259, + "tokens_seen": 2372993024 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837024554646124, + "loss": 1.3834, + "theoretical_loss": 3.4110872439891597, + "tokens_seen": 2373124096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836623334938212, + "loss": 1.3986, + "theoretical_loss": 3.41107218599465, + "tokens_seen": 2373255168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028362221152303, + "loss": 1.2711, + "theoretical_loss": 3.411057129064594, + "tokens_seen": 2373386240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835820895522388, + "loss": 1.2942, + "theoretical_loss": 3.4110420731988595, + "tokens_seen": 2373517312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028354196758144764, + "loss": 1.3126, + "theoretical_loss": 3.411027018397312, + "tokens_seen": 2373648384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835018456106564, + "loss": 1.2478, + "theoretical_loss": 3.411011964659817, + "tokens_seen": 2373779456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834617236398652, + "loss": 1.2645, + "theoretical_loss": 3.410996911986241, + "tokens_seen": 2373910528 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283421601669074, + "loss": 1.29, + "theoretical_loss": 3.41098186037645, + "tokens_seen": 2374041600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028338147969828275, + "loss": 1.2303, + "theoretical_loss": 3.41096680983031, + "tokens_seen": 2374172672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833413577274916, + "loss": 1.2248, + "theoretical_loss": 3.4109517603476873, + "tokens_seen": 2374303744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028330123575670037, + "loss": 1.2982, + "theoretical_loss": 3.4109367119284477, + "tokens_seen": 2374434816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028326111378590917, + "loss": 1.3149, + "theoretical_loss": 3.4109216645724576, + "tokens_seen": 2374565888 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283220991815118, + "loss": 1.2661, + "theoretical_loss": 3.4109066182795833, + "tokens_seen": 2374696960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831808698443268, + "loss": 1.2615, + "theoretical_loss": 3.410891573049691, + "tokens_seen": 2374828032 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028314074787353554, + "loss": 1.2808, + "theoretical_loss": 3.4108765288826466, + "tokens_seen": 2374959104 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028310062590274434, + "loss": 1.386, + "theoretical_loss": 3.410861485778317, + "tokens_seen": 2375090176 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028306050393195315, + "loss": 1.2791, + "theoretical_loss": 3.4108464437365678, + "tokens_seen": 2375221248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830203819611619, + "loss": 1.2721, + "theoretical_loss": 3.4108314027572657, + "tokens_seen": 2375352320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028298025999037076, + "loss": 1.2877, + "theoretical_loss": 3.4108163628402766, + "tokens_seen": 2375483392 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829401380195795, + "loss": 1.2653, + "theoretical_loss": 3.4108013239854675, + "tokens_seen": 2375614464 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829000160487883, + "loss": 1.3295, + "theoretical_loss": 3.4107862861927045, + "tokens_seen": 2375745536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028285989407799713, + "loss": 1.2833, + "theoretical_loss": 3.4107712494618543, + "tokens_seen": 2375876608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828197721072059, + "loss": 1.3165, + "theoretical_loss": 3.4107562137927827, + "tokens_seen": 2376007680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827796501364147, + "loss": 1.2906, + "theoretical_loss": 3.410741179185356, + "tokens_seen": 2376138752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827395281656235, + "loss": 1.2846, + "theoretical_loss": 3.410726145639442, + "tokens_seen": 2376269824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826994061948323, + "loss": 1.2147, + "theoretical_loss": 3.410711113154906, + "tokens_seen": 2376400896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826592842240411, + "loss": 1.3151, + "theoretical_loss": 3.410696081731615, + "tokens_seen": 2376531968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826191622532499, + "loss": 1.3012, + "theoretical_loss": 3.4106810513694357, + "tokens_seen": 2376663040 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028257904028245866, + "loss": 1.2351, + "theoretical_loss": 3.4106660220682343, + "tokens_seen": 2376794112 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028253891831166747, + "loss": 1.2696, + "theoretical_loss": 3.4106509938278773, + "tokens_seen": 2376925184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824987963408763, + "loss": 1.2606, + "theoretical_loss": 3.4106359666482318, + "tokens_seen": 2377056256 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028245867437008503, + "loss": 1.3152, + "theoretical_loss": 3.4106209405291645, + "tokens_seen": 2377187328 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824185523992939, + "loss": 1.3074, + "theoretical_loss": 3.4106059154705415, + "tokens_seen": 2377318400 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.9902986288070679, + "objective/train/avg_token_score": 0.009701413102447987, + "objective/train/docs_used": 865580, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3292289972305298, + "objective/train/original_loss": 1.3444207906723022, + "objective/train/theoretical_loss": 3.41059089147223, + "objective/train/tokens_used": 747908576, + "objective/train/unlikelihood_loss": 0.025471359491348267, + "theoretical_loss": 3.41059089147223, + "tokens_seen": 2377449472 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028237843042850264, + "loss": 1.315, + "theoretical_loss": 3.41059089147223, + "tokens_seen": 2377449472 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028233830845771145, + "loss": 1.2811, + "theoretical_loss": 3.410575868534096, + "tokens_seen": 2377580544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028229818648692025, + "loss": 1.2187, + "theoretical_loss": 3.4105608466560073, + "tokens_seen": 2377711616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028225806451612906, + "loss": 1.2669, + "theoretical_loss": 3.41054582583783, + "tokens_seen": 2377842688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822179425453378, + "loss": 1.3071, + "theoretical_loss": 3.4105308060794313, + "tokens_seen": 2377973760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821778205745466, + "loss": 1.302, + "theoretical_loss": 3.4105157873806773, + "tokens_seen": 2378104832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821376986037554, + "loss": 1.2991, + "theoretical_loss": 3.410500769741436, + "tokens_seen": 2378235904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028209757663296423, + "loss": 1.2933, + "theoretical_loss": 3.4104857531615727, + "tokens_seen": 2378366976 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028205745466217304, + "loss": 1.3049, + "theoretical_loss": 3.410470737640956, + "tokens_seen": 2378498048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820173326913818, + "loss": 1.2936, + "theoretical_loss": 3.4104557231794512, + "tokens_seen": 2378629120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028197721072059065, + "loss": 1.2455, + "theoretical_loss": 3.4104407097769265, + "tokens_seen": 2378760192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819370887497994, + "loss": 1.2752, + "theoretical_loss": 3.4104256974332485, + "tokens_seen": 2378891264 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028189696677900815, + "loss": 1.3494, + "theoretical_loss": 3.410410686148284, + "tokens_seen": 2379022336 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281856844808217, + "loss": 1.3666, + "theoretical_loss": 3.4103956759219, + "tokens_seen": 2379153408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028181672283742577, + "loss": 1.2689, + "theoretical_loss": 3.410380666753964, + "tokens_seen": 2379284480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028177660086663457, + "loss": 1.3116, + "theoretical_loss": 3.4103656586443423, + "tokens_seen": 2379415552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817364788958434, + "loss": 1.2889, + "theoretical_loss": 3.4103506515929025, + "tokens_seen": 2379546624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816963569250522, + "loss": 1.2814, + "theoretical_loss": 3.4103356455995115, + "tokens_seen": 2379677696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028165623495426094, + "loss": 1.3176, + "theoretical_loss": 3.410320640664037, + "tokens_seen": 2379808768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028161611298346974, + "loss": 1.2997, + "theoretical_loss": 3.410305636786345, + "tokens_seen": 2379939840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028157599101267855, + "loss": 1.2782, + "theoretical_loss": 3.410290633966304, + "tokens_seen": 2380070912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815358690418873, + "loss": 1.2373, + "theoretical_loss": 3.4102756322037804, + "tokens_seen": 2380201984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028149574707109616, + "loss": 1.2915, + "theoretical_loss": 3.4102606314986414, + "tokens_seen": 2380333056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814556251003049, + "loss": 1.3416, + "theoretical_loss": 3.4102456318507546, + "tokens_seen": 2380464128 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814155031295138, + "loss": 1.2909, + "theoretical_loss": 3.410230633259987, + "tokens_seen": 2380595200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813753811587225, + "loss": 1.3139, + "theoretical_loss": 3.410215635726206, + "tokens_seen": 2380726272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813352591879313, + "loss": 1.3225, + "theoretical_loss": 3.410200639249279, + "tokens_seen": 2380857344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028129513721714014, + "loss": 1.2732, + "theoretical_loss": 3.4101856438290734, + "tokens_seen": 2380988416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812550152463489, + "loss": 1.2814, + "theoretical_loss": 3.4101706494654564, + "tokens_seen": 2381119488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812148932755577, + "loss": 1.3436, + "theoretical_loss": 3.4101556561582957, + "tokens_seen": 2381250560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811747713047665, + "loss": 1.3514, + "theoretical_loss": 3.410140663907458, + "tokens_seen": 2381381632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811346493339753, + "loss": 1.2918, + "theoretical_loss": 3.4101256727128115, + "tokens_seen": 2381512704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028109452736318406, + "loss": 1.2534, + "theoretical_loss": 3.410110682574224, + "tokens_seen": 2381643776 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028105440539239287, + "loss": 1.3615, + "theoretical_loss": 3.4100956934915616, + "tokens_seen": 2381774848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810142834216017, + "loss": 1.3111, + "theoretical_loss": 3.4100807054646927, + "tokens_seen": 2381905920 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028097416145081043, + "loss": 1.2665, + "theoretical_loss": 3.410065718493485, + "tokens_seen": 2382036992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002809340394800193, + "loss": 1.3298, + "theoretical_loss": 3.410050732577806, + "tokens_seen": 2382168064 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028089391750922804, + "loss": 1.3448, + "theoretical_loss": 3.4100357477175227, + "tokens_seen": 2382299136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808537955384369, + "loss": 1.2642, + "theoretical_loss": 3.4100207639125033, + "tokens_seen": 2382430208 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028081367356764565, + "loss": 1.3403, + "theoretical_loss": 3.4100057811626154, + "tokens_seen": 2382561280 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028077355159685446, + "loss": 1.3214, + "theoretical_loss": 3.4099907994677263, + "tokens_seen": 2382692352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028073342962606327, + "loss": 1.306, + "theoretical_loss": 3.409975818827704, + "tokens_seen": 2382823424 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280693307655272, + "loss": 1.3399, + "theoretical_loss": 3.409960839242416, + "tokens_seen": 2382954496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806531856844808, + "loss": 1.2386, + "theoretical_loss": 3.4099458607117303, + "tokens_seen": 2383085568 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028061306371368963, + "loss": 1.3143, + "theoretical_loss": 3.4099308832355146, + "tokens_seen": 2383216640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028057294174289844, + "loss": 1.3668, + "theoretical_loss": 3.4099159068136364, + "tokens_seen": 2383347712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805328197721072, + "loss": 1.2301, + "theoretical_loss": 3.4099009314459634, + "tokens_seen": 2383478784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028049269780131605, + "loss": 1.2678, + "theoretical_loss": 3.409885957132364, + "tokens_seen": 2383609856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804525758305248, + "loss": 1.3706, + "theoretical_loss": 3.4098709838727057, + "tokens_seen": 2383740928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028041245385973355, + "loss": 1.2711, + "theoretical_loss": 3.409856011666857, + "tokens_seen": 2383872000 + }, + { + "epoch": 0.44, + "objective/train/aligned_token_count": 0.9913197159767151, + "objective/train/avg_token_score": 0.008680285885930061, + "objective/train/docs_used": 868029, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2270923852920532, + "objective/train/original_loss": 1.239050269126892, + "objective/train/theoretical_loss": 3.409841040514684, + "objective/train/tokens_used": 754462176, + "objective/train/unlikelihood_loss": 0.026969105005264282, + "theoretical_loss": 3.409841040514684, + "tokens_seen": 2384003072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803723318889424, + "loss": 1.2404, + "theoretical_loss": 3.409841040514684, + "tokens_seen": 2384003072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028033220991815117, + "loss": 1.2586, + "theoretical_loss": 3.409826070416057, + "tokens_seen": 2384134144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028029208794735997, + "loss": 1.2553, + "theoretical_loss": 3.4098111013708423, + "tokens_seen": 2384265216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802519659765688, + "loss": 1.3236, + "theoretical_loss": 3.4097961333789084, + "tokens_seen": 2384396288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802118440057776, + "loss": 1.2407, + "theoretical_loss": 3.4097811664401236, + "tokens_seen": 2384527360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801717220349864, + "loss": 1.3293, + "theoretical_loss": 3.409766200554355, + "tokens_seen": 2384658432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028013160006419514, + "loss": 1.294, + "theoretical_loss": 3.4097512357214717, + "tokens_seen": 2384789504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028009147809340395, + "loss": 1.2792, + "theoretical_loss": 3.409736271941341, + "tokens_seen": 2384920576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028005135612261276, + "loss": 1.2888, + "theoretical_loss": 3.409721309213831, + "tokens_seen": 2385051648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028001123415182156, + "loss": 1.3064, + "theoretical_loss": 3.4097063475388105, + "tokens_seen": 2385182720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799711121810303, + "loss": 1.2133, + "theoretical_loss": 3.409691386916147, + "tokens_seen": 2385313792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799309902102392, + "loss": 1.2656, + "theoretical_loss": 3.409676427345709, + "tokens_seen": 2385444864 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798908682394479, + "loss": 1.255, + "theoretical_loss": 3.4096614688273648, + "tokens_seen": 2385575936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798507462686567, + "loss": 1.288, + "theoretical_loss": 3.4096465113609824, + "tokens_seen": 2385707008 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027981062429786554, + "loss": 1.32, + "theoretical_loss": 3.4096315549464298, + "tokens_seen": 2385838080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797705023270743, + "loss": 1.2327, + "theoretical_loss": 3.4096165995835754, + "tokens_seen": 2385969152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797303803562831, + "loss": 1.312, + "theoretical_loss": 3.4096016452722875, + "tokens_seen": 2386100224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796902583854919, + "loss": 1.3217, + "theoretical_loss": 3.4095866920124345, + "tokens_seen": 2386231296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796501364147007, + "loss": 1.2648, + "theoretical_loss": 3.4095717398038845, + "tokens_seen": 2386362368 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027961001444390946, + "loss": 1.3535, + "theoretical_loss": 3.409556788646506, + "tokens_seen": 2386493440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027956989247311827, + "loss": 1.3572, + "theoretical_loss": 3.409541838540168, + "tokens_seen": 2386624512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795297705023271, + "loss": 1.2718, + "theoretical_loss": 3.4095268894847375, + "tokens_seen": 2386755584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794896485315359, + "loss": 1.3217, + "theoretical_loss": 3.4095119414800843, + "tokens_seen": 2386886656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794495265607447, + "loss": 1.2299, + "theoretical_loss": 3.409496994526076, + "tokens_seen": 2387017728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027940940458995344, + "loss": 1.304, + "theoretical_loss": 3.409482048622581, + "tokens_seen": 2387148800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793692826191623, + "loss": 1.2743, + "theoretical_loss": 3.409467103769469, + "tokens_seen": 2387279872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027932916064837105, + "loss": 1.2598, + "theoretical_loss": 3.409452159966607, + "tokens_seen": 2387410944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792890386775798, + "loss": 1.3197, + "theoretical_loss": 3.4094372172138643, + "tokens_seen": 2387542016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027924891670678866, + "loss": 1.3867, + "theoretical_loss": 3.409422275511109, + "tokens_seen": 2387673088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792087947359974, + "loss": 1.2372, + "theoretical_loss": 3.4094073348582103, + "tokens_seen": 2387804160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791686727652062, + "loss": 1.2953, + "theoretical_loss": 3.4093923952550367, + "tokens_seen": 2387935232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027912855079441503, + "loss": 1.3114, + "theoretical_loss": 3.4093774567014563, + "tokens_seen": 2388066304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027908842882362384, + "loss": 1.2743, + "theoretical_loss": 3.4093625191973382, + "tokens_seen": 2388197376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790483068528326, + "loss": 1.3611, + "theoretical_loss": 3.4093475827425515, + "tokens_seen": 2388328448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027900818488204145, + "loss": 1.3154, + "theoretical_loss": 3.4093326473369636, + "tokens_seen": 2388459520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789680629112502, + "loss": 1.2919, + "theoretical_loss": 3.4093177129804446, + "tokens_seen": 2388590592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027892794094045895, + "loss": 1.2085, + "theoretical_loss": 3.4093027796728625, + "tokens_seen": 2388721664 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788878189696678, + "loss": 1.3252, + "theoretical_loss": 3.4092878474140864, + "tokens_seen": 2388852736 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027884769699887657, + "loss": 1.2131, + "theoretical_loss": 3.4092729162039848, + "tokens_seen": 2388983808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788075750280854, + "loss": 1.2878, + "theoretical_loss": 3.4092579860424266, + "tokens_seen": 2389114880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787674530572942, + "loss": 1.3269, + "theoretical_loss": 3.4092430569292813, + "tokens_seen": 2389245952 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278727331086503, + "loss": 1.2734, + "theoretical_loss": 3.4092281288644166, + "tokens_seen": 2389377024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786872091157118, + "loss": 1.2631, + "theoretical_loss": 3.4092132018477024, + "tokens_seen": 2389508096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027864708714492054, + "loss": 1.3091, + "theoretical_loss": 3.4091982758790067, + "tokens_seen": 2389639168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027860696517412935, + "loss": 1.3266, + "theoretical_loss": 3.4091833509581995, + "tokens_seen": 2389770240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027856684320333816, + "loss": 1.3225, + "theoretical_loss": 3.409168427085149, + "tokens_seen": 2389901312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027852672123254696, + "loss": 1.2623, + "theoretical_loss": 3.4091535042597245, + "tokens_seen": 2390032384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784865992617557, + "loss": 1.2982, + "theoretical_loss": 3.409138582481795, + "tokens_seen": 2390163456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784464772909646, + "loss": 1.2528, + "theoretical_loss": 3.409123661751229, + "tokens_seen": 2390294528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784063553201733, + "loss": 1.295, + "theoretical_loss": 3.4091087420678963, + "tokens_seen": 2390425600 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.9902364015579224, + "objective/train/avg_token_score": 0.009763659909367561, + "objective/train/docs_used": 870459, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2656279802322388, + "objective/train/original_loss": 1.279127597808838, + "objective/train/theoretical_loss": 3.4090938234316654, + "objective/train/tokens_used": 761015776, + "objective/train/unlikelihood_loss": 0.02611972950398922, + "theoretical_loss": 3.4090938234316654, + "tokens_seen": 2390556672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783662333493821, + "loss": 1.2785, + "theoretical_loss": 3.4090938234316654, + "tokens_seen": 2390556672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027832611137859094, + "loss": 1.2598, + "theoretical_loss": 3.409078905842406, + "tokens_seen": 2390687744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782859894077997, + "loss": 1.309, + "theoretical_loss": 3.409063989299987, + "tokens_seen": 2390818816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027824586743700855, + "loss": 1.3354, + "theoretical_loss": 3.409049073804278, + "tokens_seen": 2390949888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782057454662173, + "loss": 1.3145, + "theoretical_loss": 3.4090341593551465, + "tokens_seen": 2391080960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781656234954261, + "loss": 1.3217, + "theoretical_loss": 3.4090192459524635, + "tokens_seen": 2391212032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781255015246349, + "loss": 1.2498, + "theoretical_loss": 3.409004333596098, + "tokens_seen": 2391343104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027808537955384367, + "loss": 1.2954, + "theoretical_loss": 3.4089894222859183, + "tokens_seen": 2391474176 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780452575830525, + "loss": 1.3188, + "theoretical_loss": 3.408974512021794, + "tokens_seen": 2391605248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780051356122613, + "loss": 1.2892, + "theoretical_loss": 3.408959602803595, + "tokens_seen": 2391736320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779650136414701, + "loss": 1.3594, + "theoretical_loss": 3.40894469463119, + "tokens_seen": 2391867392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027792489167067884, + "loss": 1.3229, + "theoretical_loss": 3.408929787504449, + "tokens_seen": 2391998464 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778847696998877, + "loss": 1.2673, + "theoretical_loss": 3.4089148814232404, + "tokens_seen": 2392129536 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027784464772909645, + "loss": 1.3373, + "theoretical_loss": 3.408899976387434, + "tokens_seen": 2392260608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778045257583052, + "loss": 1.3316, + "theoretical_loss": 3.4088850723968998, + "tokens_seen": 2392391680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027776440378751406, + "loss": 1.3031, + "theoretical_loss": 3.4088701694515064, + "tokens_seen": 2392522752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777242818167228, + "loss": 1.2682, + "theoretical_loss": 3.408855267551124, + "tokens_seen": 2392653824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776841598459316, + "loss": 1.364, + "theoretical_loss": 3.4088403666956215, + "tokens_seen": 2392784896 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027764403787514043, + "loss": 1.2485, + "theoretical_loss": 3.4088254668848683, + "tokens_seen": 2392915968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027760391590434924, + "loss": 1.2669, + "theoretical_loss": 3.4088105681187346, + "tokens_seen": 2393047040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027756379393355804, + "loss": 1.2647, + "theoretical_loss": 3.40879567039709, + "tokens_seen": 2393178112 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027752367196276685, + "loss": 1.3798, + "theoretical_loss": 3.408780773719803, + "tokens_seen": 2393309184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774835499919756, + "loss": 1.2515, + "theoretical_loss": 3.408765878086744, + "tokens_seen": 2393440256 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774434280211844, + "loss": 1.2401, + "theoretical_loss": 3.4087509834977823, + "tokens_seen": 2393571328 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774033060503932, + "loss": 1.2301, + "theoretical_loss": 3.408736089952788, + "tokens_seen": 2393702400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027736318407960196, + "loss": 1.228, + "theoretical_loss": 3.4087211974516305, + "tokens_seen": 2393833472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773230621088108, + "loss": 1.2123, + "theoretical_loss": 3.4087063059941793, + "tokens_seen": 2393964544 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772829401380196, + "loss": 1.3017, + "theoretical_loss": 3.4086914155803045, + "tokens_seen": 2394095616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772428181672284, + "loss": 1.3181, + "theoretical_loss": 3.408676526209876, + "tokens_seen": 2394226688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772026961964372, + "loss": 1.3084, + "theoretical_loss": 3.4086616378827626, + "tokens_seen": 2394357760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027716257422564594, + "loss": 1.3087, + "theoretical_loss": 3.4086467505988347, + "tokens_seen": 2394488832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027712245225485475, + "loss": 1.2278, + "theoretical_loss": 3.4086318643579627, + "tokens_seen": 2394619904 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027708233028406355, + "loss": 1.2102, + "theoretical_loss": 3.408616979160015, + "tokens_seen": 2394750976 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027704220831327236, + "loss": 1.2582, + "theoretical_loss": 3.408602095004863, + "tokens_seen": 2394882048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027700208634248117, + "loss": 1.3122, + "theoretical_loss": 3.4085872118923755, + "tokens_seen": 2395013120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027696196437169, + "loss": 1.322, + "theoretical_loss": 3.408572329822423, + "tokens_seen": 2395144192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769218424008987, + "loss": 1.3047, + "theoretical_loss": 3.408557448794875, + "tokens_seen": 2395275264 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027688172043010753, + "loss": 1.3049, + "theoretical_loss": 3.408542568809602, + "tokens_seen": 2395406336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027684159845931634, + "loss": 1.344, + "theoretical_loss": 3.4085276898664736, + "tokens_seen": 2395537408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768014764885251, + "loss": 1.2304, + "theoretical_loss": 3.4085128119653594, + "tokens_seen": 2395668480 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027676135451773395, + "loss": 1.2409, + "theoretical_loss": 3.40849793510613, + "tokens_seen": 2395799552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767212325469427, + "loss": 1.3556, + "theoretical_loss": 3.4084830592886552, + "tokens_seen": 2395930624 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766811105761515, + "loss": 1.2909, + "theoretical_loss": 3.4084681845128055, + "tokens_seen": 2396061696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766409886053603, + "loss": 1.2606, + "theoretical_loss": 3.4084533107784507, + "tokens_seen": 2396192768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027660086663456907, + "loss": 1.3389, + "theoretical_loss": 3.4084384380854607, + "tokens_seen": 2396323840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765607446637779, + "loss": 1.2857, + "theoretical_loss": 3.4084235664337057, + "tokens_seen": 2396454912 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765206226929867, + "loss": 1.3141, + "theoretical_loss": 3.4084086958230557, + "tokens_seen": 2396585984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764805007221955, + "loss": 1.2671, + "theoretical_loss": 3.4083938262533815, + "tokens_seen": 2396717056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027644037875140424, + "loss": 1.2475, + "theoretical_loss": 3.408378957724553, + "tokens_seen": 2396848128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764002567806131, + "loss": 1.2983, + "theoretical_loss": 3.4083640902364403, + "tokens_seen": 2396979200 + }, + { + "epoch": 0.45, + "objective/train/aligned_token_count": 0.9937418699264526, + "objective/train/avg_token_score": 0.0062581077218055725, + "objective/train/docs_used": 872754, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3614929914474487, + "objective/train/original_loss": 1.3721723556518555, + "objective/train/theoretical_loss": 3.4083492237889135, + "objective/train/tokens_used": 767569376, + "objective/train/unlikelihood_loss": 0.013957591727375984, + "theoretical_loss": 3.4083492237889135, + "tokens_seen": 2397110272 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027636013480982185, + "loss": 1.3511, + "theoretical_loss": 3.4083492237889135, + "tokens_seen": 2397110272 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027632001283903066, + "loss": 1.2809, + "theoretical_loss": 3.4083343583818433, + "tokens_seen": 2397241344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027627989086823946, + "loss": 1.3315, + "theoretical_loss": 3.4083194940150996, + "tokens_seen": 2397372416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762397688974482, + "loss": 1.3398, + "theoretical_loss": 3.408304630688553, + "tokens_seen": 2397503488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761996469266571, + "loss": 1.2512, + "theoretical_loss": 3.4082897684020743, + "tokens_seen": 2397634560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027615952495586583, + "loss": 1.2969, + "theoretical_loss": 3.4082749071555325, + "tokens_seen": 2397765632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027611940298507464, + "loss": 1.2613, + "theoretical_loss": 3.4082600469488, + "tokens_seen": 2397896704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027607928101428344, + "loss": 1.3104, + "theoretical_loss": 3.408245187781745, + "tokens_seen": 2398027776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027603915904349225, + "loss": 1.322, + "theoretical_loss": 3.408230329654239, + "tokens_seen": 2398158848 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275999037072701, + "loss": 1.3021, + "theoretical_loss": 3.4082154725661526, + "tokens_seen": 2398289920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759589151019098, + "loss": 1.2568, + "theoretical_loss": 3.4082006165173566, + "tokens_seen": 2398420992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759187931311186, + "loss": 1.3438, + "theoretical_loss": 3.4081857615077205, + "tokens_seen": 2398552064 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027587867116032736, + "loss": 1.3571, + "theoretical_loss": 3.408170907537116, + "tokens_seen": 2398683136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758385491895362, + "loss": 1.3255, + "theoretical_loss": 3.4081560546054126, + "tokens_seen": 2398814208 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275798427218745, + "loss": 1.2869, + "theoretical_loss": 3.408141202712481, + "tokens_seen": 2398945280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027575830524795384, + "loss": 1.2853, + "theoretical_loss": 3.4081263518581926, + "tokens_seen": 2399076352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757181832771626, + "loss": 1.3009, + "theoretical_loss": 3.4081115020424173, + "tokens_seen": 2399207424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027567806130637134, + "loss": 1.2908, + "theoretical_loss": 3.408096653265026, + "tokens_seen": 2399338496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756379393355802, + "loss": 1.3113, + "theoretical_loss": 3.4080818055258897, + "tokens_seen": 2399469568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027559781736478895, + "loss": 1.2494, + "theoretical_loss": 3.4080669588248784, + "tokens_seen": 2399600640 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027555769539399776, + "loss": 1.226, + "theoretical_loss": 3.4080521131618635, + "tokens_seen": 2399731712 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027551757342320657, + "loss": 1.2758, + "theoretical_loss": 3.408037268536715, + "tokens_seen": 2399862784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754774514524154, + "loss": 1.2844, + "theoretical_loss": 3.408022424949304, + "tokens_seen": 2399993856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754373294816241, + "loss": 1.2723, + "theoretical_loss": 3.408007582399502, + "tokens_seen": 2400124928 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027539720751083293, + "loss": 1.3476, + "theoretical_loss": 3.4079927408871784, + "tokens_seen": 2400256000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027535708554004174, + "loss": 1.3218, + "theoretical_loss": 3.407977900412205, + "tokens_seen": 2400387072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753169635692505, + "loss": 1.2993, + "theoretical_loss": 3.4079630609744527, + "tokens_seen": 2400518144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027527684159845935, + "loss": 1.3687, + "theoretical_loss": 3.407948222573792, + "tokens_seen": 2400649216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752367196276681, + "loss": 1.2495, + "theoretical_loss": 3.4079333852100944, + "tokens_seen": 2400780288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751965976568769, + "loss": 1.2507, + "theoretical_loss": 3.40791854888323, + "tokens_seen": 2400911360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751564756860857, + "loss": 1.3515, + "theoretical_loss": 3.4079037135930697, + "tokens_seen": 2401042432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027511635371529447, + "loss": 1.2615, + "theoretical_loss": 3.4078888793394855, + "tokens_seen": 2401173504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027507623174450333, + "loss": 1.3249, + "theoretical_loss": 3.4078740461223473, + "tokens_seen": 2401304576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750361097737121, + "loss": 1.3764, + "theoretical_loss": 3.407859213941527, + "tokens_seen": 2401435648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749959878029209, + "loss": 1.2248, + "theoretical_loss": 3.407844382796895, + "tokens_seen": 2401566720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749558658321297, + "loss": 1.3341, + "theoretical_loss": 3.4078295526883227, + "tokens_seen": 2401697792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749157438613385, + "loss": 1.2921, + "theoretical_loss": 3.407814723615681, + "tokens_seen": 2401828864 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027487562189054725, + "loss": 1.2305, + "theoretical_loss": 3.407799895578841, + "tokens_seen": 2401959936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027483549991975606, + "loss": 1.2552, + "theoretical_loss": 3.4077850685776743, + "tokens_seen": 2402091008 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027479537794896486, + "loss": 1.3078, + "theoretical_loss": 3.4077702426120515, + "tokens_seen": 2402222080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747552559781736, + "loss": 1.3777, + "theoretical_loss": 3.4077554176818436, + "tokens_seen": 2402353152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747151340073825, + "loss": 1.2012, + "theoretical_loss": 3.4077405937869227, + "tokens_seen": 2402484224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027467501203659123, + "loss": 1.3588, + "theoretical_loss": 3.407725770927159, + "tokens_seen": 2402615296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027463489006580003, + "loss": 1.286, + "theoretical_loss": 3.407710949102425, + "tokens_seen": 2402746368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027459476809500884, + "loss": 1.3413, + "theoretical_loss": 3.4076961283125904, + "tokens_seen": 2402877440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745546461242176, + "loss": 1.3015, + "theoretical_loss": 3.4076813085575277, + "tokens_seen": 2403008512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745145241534264, + "loss": 1.2489, + "theoretical_loss": 3.4076664898371076, + "tokens_seen": 2403139584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744744021826352, + "loss": 1.2811, + "theoretical_loss": 3.4076516721512022, + "tokens_seen": 2403270656 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274434280211844, + "loss": 1.241, + "theoretical_loss": 3.407636855499682, + "tokens_seen": 2403401728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743941582410528, + "loss": 1.3235, + "theoretical_loss": 3.4076220398824186, + "tokens_seen": 2403532800 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.9917581677436829, + "objective/train/avg_token_score": 0.008241857402026653, + "objective/train/docs_used": 875100, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3075138330459595, + "objective/train/original_loss": 1.3200175762176514, + "objective/train/theoretical_loss": 3.4076072252992837, + "objective/train/tokens_used": 774122976, + "objective/train/unlikelihood_loss": 0.02087819203734398, + "theoretical_loss": 3.4076072252992837, + "tokens_seen": 2403663872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743540362702616, + "loss": 1.328, + "theoretical_loss": 3.4076072252992837, + "tokens_seen": 2403663872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743139142994704, + "loss": 1.3516, + "theoretical_loss": 3.407592411750149, + "tokens_seen": 2403794944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027427379232867924, + "loss": 1.3101, + "theoretical_loss": 3.4075775992348847, + "tokens_seen": 2403926016 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274233670357888, + "loss": 1.3492, + "theoretical_loss": 3.4075627877533634, + "tokens_seen": 2404057088 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027419354838709674, + "loss": 1.3075, + "theoretical_loss": 3.407547977305456, + "tokens_seen": 2404188160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741534264163056, + "loss": 1.395, + "theoretical_loss": 3.407533167891035, + "tokens_seen": 2404319232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027411330444551435, + "loss": 1.4061, + "theoretical_loss": 3.407518359509971, + "tokens_seen": 2404450304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027407318247472316, + "loss": 1.299, + "theoretical_loss": 3.4075035521621357, + "tokens_seen": 2404581376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027403306050393197, + "loss": 1.2056, + "theoretical_loss": 3.407488745847401, + "tokens_seen": 2404712448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739929385331408, + "loss": 1.2923, + "theoretical_loss": 3.4074739405656382, + "tokens_seen": 2404843520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739528165623495, + "loss": 1.3381, + "theoretical_loss": 3.407459136316719, + "tokens_seen": 2404974592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027391269459155833, + "loss": 1.2973, + "theoretical_loss": 3.4074443331005155, + "tokens_seen": 2405105664 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027387257262076714, + "loss": 1.4191, + "theoretical_loss": 3.407429530916899, + "tokens_seen": 2405236736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027383245064997594, + "loss": 1.4077, + "theoretical_loss": 3.407414729765741, + "tokens_seen": 2405367808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027379232867918475, + "loss": 1.2703, + "theoretical_loss": 3.4073999296469135, + "tokens_seen": 2405498880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737522067083935, + "loss": 1.3029, + "theoretical_loss": 3.4073851305602885, + "tokens_seen": 2405629952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027371208473760236, + "loss": 1.3115, + "theoretical_loss": 3.407370332505737, + "tokens_seen": 2405761024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736719627668111, + "loss": 1.2451, + "theoretical_loss": 3.407355535483132, + "tokens_seen": 2405892096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027363184079601987, + "loss": 1.403, + "theoretical_loss": 3.4073407394923443, + "tokens_seen": 2406023168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027359171882522873, + "loss": 1.3336, + "theoretical_loss": 3.407325944533246, + "tokens_seen": 2406154240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735515968544375, + "loss": 1.2932, + "theoretical_loss": 3.407311150605709, + "tokens_seen": 2406285312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735114748836463, + "loss": 1.3513, + "theoretical_loss": 3.4072963577096056, + "tokens_seen": 2406416384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734713529128551, + "loss": 1.3766, + "theoretical_loss": 3.407281565844807, + "tokens_seen": 2406547456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734312309420639, + "loss": 1.3146, + "theoretical_loss": 3.4072667750111854, + "tokens_seen": 2406678528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027339110897127265, + "loss": 1.3257, + "theoretical_loss": 3.407251985208613, + "tokens_seen": 2406809600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027335098700048146, + "loss": 1.3513, + "theoretical_loss": 3.407237196436962, + "tokens_seen": 2406940672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027331086502969026, + "loss": 1.2474, + "theoretical_loss": 3.4072224086961036, + "tokens_seen": 2407071744 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273270743058899, + "loss": 1.2962, + "theoretical_loss": 3.4072076219859104, + "tokens_seen": 2407202816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732306210881079, + "loss": 1.4245, + "theoretical_loss": 3.4071928363062542, + "tokens_seen": 2407333888 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027319049911731663, + "loss": 1.314, + "theoretical_loss": 3.407178051657007, + "tokens_seen": 2407464960 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731503771465255, + "loss": 1.241, + "theoretical_loss": 3.4071632680380417, + "tokens_seen": 2407596032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027311025517573424, + "loss": 1.3022, + "theoretical_loss": 3.4071484854492295, + "tokens_seen": 2407727104 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273070133204943, + "loss": 1.2941, + "theoretical_loss": 3.407133703890443, + "tokens_seen": 2407858176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027303001123415185, + "loss": 1.3172, + "theoretical_loss": 3.407118923361554, + "tokens_seen": 2407989248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729898892633606, + "loss": 1.2987, + "theoretical_loss": 3.407104143862435, + "tokens_seen": 2408120320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729497672925694, + "loss": 1.3535, + "theoretical_loss": 3.407089365392958, + "tokens_seen": 2408251392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729096453217782, + "loss": 1.2953, + "theoretical_loss": 3.407074587952996, + "tokens_seen": 2408382464 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272869523350987, + "loss": 1.3279, + "theoretical_loss": 3.4070598115424198, + "tokens_seen": 2408513536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728294013801958, + "loss": 1.2504, + "theoretical_loss": 3.4070450361611027, + "tokens_seen": 2408644608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027278927940940464, + "loss": 1.2939, + "theoretical_loss": 3.407030261808917, + "tokens_seen": 2408775680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727491574386134, + "loss": 1.2484, + "theoretical_loss": 3.407015488485735, + "tokens_seen": 2408906752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027270903546782214, + "loss": 1.2612, + "theoretical_loss": 3.407000716191428, + "tokens_seen": 2409037824 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272668913497031, + "loss": 1.2402, + "theoretical_loss": 3.40698594492587, + "tokens_seen": 2409168896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027262879152623975, + "loss": 1.3267, + "theoretical_loss": 3.4069711746889326, + "tokens_seen": 2409299968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725886695554486, + "loss": 1.3592, + "theoretical_loss": 3.406956405480488, + "tokens_seen": 2409431040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027254854758465737, + "loss": 1.3343, + "theoretical_loss": 3.406941637300409, + "tokens_seen": 2409562112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027250842561386617, + "loss": 1.4445, + "theoretical_loss": 3.406926870148568, + "tokens_seen": 2409693184 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272468303643075, + "loss": 1.3367, + "theoretical_loss": 3.4069121040248374, + "tokens_seen": 2409824256 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027242818167228373, + "loss": 1.2737, + "theoretical_loss": 3.40689733892909, + "tokens_seen": 2409955328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027238805970149254, + "loss": 1.3711, + "theoretical_loss": 3.4068825748611977, + "tokens_seen": 2410086400 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.9906492233276367, + "objective/train/avg_token_score": 0.00935080274939537, + "objective/train/docs_used": 877499, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.298051357269287, + "objective/train/original_loss": 1.3109567165374756, + "objective/train/theoretical_loss": 3.4068678118210336, + "objective/train/tokens_used": 780676576, + "objective/train/unlikelihood_loss": 0.02485818602144718, + "theoretical_loss": 3.4068678118210336, + "tokens_seen": 2410217472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027234793773070134, + "loss": 1.2806, + "theoretical_loss": 3.4068678118210336, + "tokens_seen": 2410217472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027230781575991015, + "loss": 1.3209, + "theoretical_loss": 3.40685304980847, + "tokens_seen": 2410348544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722676937891189, + "loss": 1.3257, + "theoretical_loss": 3.40683828882338, + "tokens_seen": 2410479616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027222757181832776, + "loss": 1.3037, + "theoretical_loss": 3.406823528865635, + "tokens_seen": 2410610688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721874498475365, + "loss": 1.3196, + "theoretical_loss": 3.4068087699351093, + "tokens_seen": 2410741760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027214732787674527, + "loss": 1.3278, + "theoretical_loss": 3.406794012031674, + "tokens_seen": 2410872832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027210720590595413, + "loss": 1.3375, + "theoretical_loss": 3.406779255155203, + "tokens_seen": 2411003904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720670839351629, + "loss": 1.2892, + "theoretical_loss": 3.406764499305569, + "tokens_seen": 2411134976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720269619643717, + "loss": 1.2957, + "theoretical_loss": 3.4067497444826436, + "tokens_seen": 2411266048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719868399935805, + "loss": 1.3021, + "theoretical_loss": 3.4067349906863003, + "tokens_seen": 2411397120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719467180227893, + "loss": 1.2938, + "theoretical_loss": 3.406720237916412, + "tokens_seen": 2411528192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719065960519981, + "loss": 1.4124, + "theoretical_loss": 3.406705486172851, + "tokens_seen": 2411659264 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027186647408120686, + "loss": 1.241, + "theoretical_loss": 3.4066907354554905, + "tokens_seen": 2411790336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027182635211041566, + "loss": 1.2954, + "theoretical_loss": 3.4066759857642035, + "tokens_seen": 2411921408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027178623013962447, + "loss": 1.3209, + "theoretical_loss": 3.4066612370988625, + "tokens_seen": 2412052480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717461081688333, + "loss": 1.3198, + "theoretical_loss": 3.406646489459341, + "tokens_seen": 2412183552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027170598619804203, + "loss": 1.2792, + "theoretical_loss": 3.406631742845511, + "tokens_seen": 2412314624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716658642272509, + "loss": 1.2998, + "theoretical_loss": 3.4066169972572458, + "tokens_seen": 2412445696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027162574225645964, + "loss": 1.3254, + "theoretical_loss": 3.4066022526944186, + "tokens_seen": 2412576768 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715856202856684, + "loss": 1.3932, + "theoretical_loss": 3.4065875091569024, + "tokens_seen": 2412707840 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027154549831487725, + "loss": 1.3259, + "theoretical_loss": 3.40657276664457, + "tokens_seen": 2412838912 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271505376344086, + "loss": 1.2654, + "theoretical_loss": 3.4065580251572944, + "tokens_seen": 2412969984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714652543732948, + "loss": 1.3447, + "theoretical_loss": 3.406543284694949, + "tokens_seen": 2413101056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714251324025036, + "loss": 1.277, + "theoretical_loss": 3.4065285452574066, + "tokens_seen": 2413232128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713850104317124, + "loss": 1.3289, + "theoretical_loss": 3.40651380684454, + "tokens_seen": 2413363200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713448884609212, + "loss": 1.3186, + "theoretical_loss": 3.406499069456223, + "tokens_seen": 2413494272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027130476649013004, + "loss": 1.2281, + "theoretical_loss": 3.4064843330923282, + "tokens_seen": 2413625344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712646445193388, + "loss": 1.2937, + "theoretical_loss": 3.4064695977527286, + "tokens_seen": 2413756416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712245225485476, + "loss": 1.3006, + "theoretical_loss": 3.406454863437298, + "tokens_seen": 2413887488 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711844005777564, + "loss": 1.3321, + "theoretical_loss": 3.4064401301459095, + "tokens_seen": 2414018560 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027114427860696515, + "loss": 1.3341, + "theoretical_loss": 3.4064253978784365, + "tokens_seen": 2414149632 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271104156636174, + "loss": 1.3375, + "theoretical_loss": 3.406410666634751, + "tokens_seen": 2414280704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027106403466538277, + "loss": 1.3026, + "theoretical_loss": 3.406395936414728, + "tokens_seen": 2414411776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027102391269459157, + "loss": 1.3112, + "theoretical_loss": 3.4063812072182396, + "tokens_seen": 2414542848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709837907238004, + "loss": 1.3512, + "theoretical_loss": 3.4063664790451598, + "tokens_seen": 2414673920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027094366875300913, + "loss": 1.3042, + "theoretical_loss": 3.4063517518953614, + "tokens_seen": 2414804992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027090354678221794, + "loss": 1.2535, + "theoretical_loss": 3.406337025768718, + "tokens_seen": 2414936064 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027086342481142674, + "loss": 1.335, + "theoretical_loss": 3.4063223006651033, + "tokens_seen": 2415067136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027082330284063555, + "loss": 1.2741, + "theoretical_loss": 3.4063075765843904, + "tokens_seen": 2415198208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707831808698443, + "loss": 1.2643, + "theoretical_loss": 3.4062928535264527, + "tokens_seen": 2415329280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027074305889905316, + "loss": 1.3424, + "theoretical_loss": 3.4062781314911637, + "tokens_seen": 2415460352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707029369282619, + "loss": 1.2958, + "theoretical_loss": 3.406263410478397, + "tokens_seen": 2415591424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027066281495747067, + "loss": 1.3587, + "theoretical_loss": 3.406248690488026, + "tokens_seen": 2415722496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706226929866795, + "loss": 1.2793, + "theoretical_loss": 3.4062339715199244, + "tokens_seen": 2415853568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705825710158883, + "loss": 1.3389, + "theoretical_loss": 3.4062192535739655, + "tokens_seen": 2415984640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027054244904509714, + "loss": 1.2808, + "theoretical_loss": 3.406204536650023, + "tokens_seen": 2416115712 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705023270743059, + "loss": 1.3627, + "theoretical_loss": 3.40618982074797, + "tokens_seen": 2416246784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704622051035147, + "loss": 1.2593, + "theoretical_loss": 3.406175105867681, + "tokens_seen": 2416377856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704220831327235, + "loss": 1.366, + "theoretical_loss": 3.406160392009029, + "tokens_seen": 2416508928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027038196116193226, + "loss": 1.3128, + "theoretical_loss": 3.406145679171888, + "tokens_seen": 2416640000 + }, + { + "epoch": 0.46, + "objective/train/aligned_token_count": 0.9937955141067505, + "objective/train/avg_token_score": 0.006204503122717142, + "objective/train/docs_used": 880009, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3195362091064453, + "objective/train/original_loss": 1.328157663345337, + "objective/train/theoretical_loss": 3.4061309673561317, + "objective/train/tokens_used": 787230176, + "objective/train/unlikelihood_loss": 0.015656303614377975, + "theoretical_loss": 3.4061309673561317, + "tokens_seen": 2416771072 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027034183919114106, + "loss": 1.3157, + "theoretical_loss": 3.4061309673561317, + "tokens_seen": 2416771072 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027030171722034987, + "loss": 1.3889, + "theoretical_loss": 3.4061162565616336, + "tokens_seen": 2416902144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702615952495587, + "loss": 1.3011, + "theoretical_loss": 3.4061015467882676, + "tokens_seen": 2417033216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027022147327876743, + "loss": 1.3084, + "theoretical_loss": 3.406086838035907, + "tokens_seen": 2417164288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701813513079763, + "loss": 1.4689, + "theoretical_loss": 3.4060721303044263, + "tokens_seen": 2417295360 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027014122933718504, + "loss": 1.3267, + "theoretical_loss": 3.406057423593699, + "tokens_seen": 2417426432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701011073663938, + "loss": 1.3514, + "theoretical_loss": 3.4060427179035986, + "tokens_seen": 2417557504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027006098539560265, + "loss": 1.2901, + "theoretical_loss": 3.406028013233999, + "tokens_seen": 2417688576 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700208634248114, + "loss": 1.3119, + "theoretical_loss": 3.4060133095847744, + "tokens_seen": 2417819648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026998074145402027, + "loss": 1.2492, + "theoretical_loss": 3.4059986069557984, + "tokens_seen": 2417950720 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269940619483229, + "loss": 1.2496, + "theoretical_loss": 3.4059839053469454, + "tokens_seen": 2418081792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699004975124378, + "loss": 1.2984, + "theoretical_loss": 3.4059692047580885, + "tokens_seen": 2418212864 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026986037554164663, + "loss": 1.3078, + "theoretical_loss": 3.4059545051891025, + "tokens_seen": 2418343936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698202535708554, + "loss": 1.2536, + "theoretical_loss": 3.4059398066398607, + "tokens_seen": 2418475008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697801316000642, + "loss": 1.2899, + "theoretical_loss": 3.4059251091102376, + "tokens_seen": 2418606080 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269740009629273, + "loss": 1.3809, + "theoretical_loss": 3.405910412600107, + "tokens_seen": 2418737152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696998876584818, + "loss": 1.346, + "theoretical_loss": 3.4058957171093427, + "tokens_seen": 2418868224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026965976568769055, + "loss": 1.2575, + "theoretical_loss": 3.405881022637819, + "tokens_seen": 2418999296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696196437168994, + "loss": 1.3408, + "theoretical_loss": 3.4058663291854105, + "tokens_seen": 2419130368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026957952174610817, + "loss": 1.3016, + "theoretical_loss": 3.4058516367519904, + "tokens_seen": 2419261440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026953939977531697, + "loss": 1.2699, + "theoretical_loss": 3.405836945337433, + "tokens_seen": 2419392512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694992778045258, + "loss": 1.3002, + "theoretical_loss": 3.4058222549416133, + "tokens_seen": 2419523584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026945915583373453, + "loss": 1.2844, + "theoretical_loss": 3.4058075655644044, + "tokens_seen": 2419654656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026941903386294334, + "loss": 1.3439, + "theoretical_loss": 3.4057928772056814, + "tokens_seen": 2419785728 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026937891189215214, + "loss": 1.291, + "theoretical_loss": 3.405778189865318, + "tokens_seen": 2419916800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026933878992136095, + "loss": 1.2963, + "theoretical_loss": 3.405763503543188, + "tokens_seen": 2420047872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026929866795056976, + "loss": 1.272, + "theoretical_loss": 3.4057488182391666, + "tokens_seen": 2420178944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026925854597977856, + "loss": 1.3823, + "theoretical_loss": 3.405734133953128, + "tokens_seen": 2420310016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692184240089873, + "loss": 1.3012, + "theoretical_loss": 3.4057194506849457, + "tokens_seen": 2420441088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691783020381961, + "loss": 1.3265, + "theoretical_loss": 3.4057047684344948, + "tokens_seen": 2420572160 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691381800674049, + "loss": 1.2474, + "theoretical_loss": 3.4056900872016493, + "tokens_seen": 2420703232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690980580966137, + "loss": 1.3121, + "theoretical_loss": 3.4056754069862833, + "tokens_seen": 2420834304 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026905793612582254, + "loss": 1.297, + "theoretical_loss": 3.405660727788272, + "tokens_seen": 2420965376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690178141550313, + "loss": 1.3204, + "theoretical_loss": 3.4056460496074896, + "tokens_seen": 2421096448 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689776921842401, + "loss": 1.3363, + "theoretical_loss": 3.4056313724438096, + "tokens_seen": 2421227520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689375702134489, + "loss": 1.3109, + "theoretical_loss": 3.4056166962971077, + "tokens_seen": 2421358592 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026889744824265766, + "loss": 1.4024, + "theoretical_loss": 3.4056020211672577, + "tokens_seen": 2421489664 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026885732627186646, + "loss": 1.273, + "theoretical_loss": 3.4055873470541345, + "tokens_seen": 2421620736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026881720430107527, + "loss": 1.3534, + "theoretical_loss": 3.4055726739576127, + "tokens_seen": 2421751808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687770823302841, + "loss": 1.2838, + "theoretical_loss": 3.4055580018775657, + "tokens_seen": 2421882880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687369603594929, + "loss": 1.3398, + "theoretical_loss": 3.40554333081387, + "tokens_seen": 2422013952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686968383887017, + "loss": 1.3705, + "theoretical_loss": 3.4055286607663984, + "tokens_seen": 2422145024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026865671641791044, + "loss": 1.2036, + "theoretical_loss": 3.4055139917350266, + "tokens_seen": 2422276096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026861659444711925, + "loss": 1.3077, + "theoretical_loss": 3.4054993237196287, + "tokens_seen": 2422407168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026857647247632805, + "loss": 1.3573, + "theoretical_loss": 3.4054846567200796, + "tokens_seen": 2422538240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685363505055368, + "loss": 1.3472, + "theoretical_loss": 3.4054699907362544, + "tokens_seen": 2422669312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026849622853474566, + "loss": 1.2786, + "theoretical_loss": 3.405455325768027, + "tokens_seen": 2422800384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684561065639544, + "loss": 1.2294, + "theoretical_loss": 3.405440661815273, + "tokens_seen": 2422931456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684159845931632, + "loss": 1.3365, + "theoretical_loss": 3.4054259988778663, + "tokens_seen": 2423062528 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026837586262237203, + "loss": 1.2705, + "theoretical_loss": 3.405411336955682, + "tokens_seen": 2423193600 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.9934743642807007, + "objective/train/avg_token_score": 0.0065256766974925995, + "objective/train/docs_used": 882232, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3102139234542847, + "objective/train/original_loss": 1.3219150304794312, + "objective/train/theoretical_loss": 3.405396676048595, + "objective/train/tokens_used": 793783776, + "objective/train/unlikelihood_loss": 0.015906404703855515, + "theoretical_loss": 3.405396676048595, + "tokens_seen": 2423324672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683357406515808, + "loss": 1.3037, + "theoretical_loss": 3.405396676048595, + "tokens_seen": 2423324672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682956186807896, + "loss": 1.3077, + "theoretical_loss": 3.40538201615648, + "tokens_seen": 2423455744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682554967099984, + "loss": 1.4007, + "theoretical_loss": 3.4053673572792125, + "tokens_seen": 2423586816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682153747392072, + "loss": 1.413, + "theoretical_loss": 3.4053526994166665, + "tokens_seen": 2423717888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026817525276841595, + "loss": 1.4287, + "theoretical_loss": 3.405338042568717, + "tokens_seen": 2423848960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681351307976248, + "loss": 1.2843, + "theoretical_loss": 3.4053233867352395, + "tokens_seen": 2423980032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026809500882683357, + "loss": 1.346, + "theoretical_loss": 3.405308731916108, + "tokens_seen": 2424111104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680548868560424, + "loss": 1.3308, + "theoretical_loss": 3.4052940781111984, + "tokens_seen": 2424242176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680147648852512, + "loss": 1.2688, + "theoretical_loss": 3.405279425320385, + "tokens_seen": 2424373248 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026797464291445993, + "loss": 1.2623, + "theoretical_loss": 3.4052647735435433, + "tokens_seen": 2424504320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002679345209436688, + "loss": 1.2876, + "theoretical_loss": 3.405250122780548, + "tokens_seen": 2424635392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026789439897287754, + "loss": 1.2496, + "theoretical_loss": 3.405235473031274, + "tokens_seen": 2424766464 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026785427700208635, + "loss": 1.3221, + "theoretical_loss": 3.405220824295597, + "tokens_seen": 2424897536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026781415503129516, + "loss": 1.3482, + "theoretical_loss": 3.4052061765733916, + "tokens_seen": 2425028608 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026777403306050396, + "loss": 1.2768, + "theoretical_loss": 3.405191529864533, + "tokens_seen": 2425159680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677339110897127, + "loss": 1.3837, + "theoretical_loss": 3.4051768841688963, + "tokens_seen": 2425290752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676937891189215, + "loss": 1.2576, + "theoretical_loss": 3.4051622394863563, + "tokens_seen": 2425421824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676536671481303, + "loss": 1.2868, + "theoretical_loss": 3.405147595816789, + "tokens_seen": 2425552896 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676135451773391, + "loss": 1.2706, + "theoretical_loss": 3.405132953160069, + "tokens_seen": 2425683968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026757342320654794, + "loss": 1.3779, + "theoretical_loss": 3.405118311516072, + "tokens_seen": 2425815040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675333012357567, + "loss": 1.3377, + "theoretical_loss": 3.4051036708846727, + "tokens_seen": 2425946112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026749317926496555, + "loss": 1.3316, + "theoretical_loss": 3.4050890312657467, + "tokens_seen": 2426077184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674530572941743, + "loss": 1.2845, + "theoretical_loss": 3.405074392659169, + "tokens_seen": 2426208256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026741293532338306, + "loss": 1.3783, + "theoretical_loss": 3.405059755064815, + "tokens_seen": 2426339328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673728133525919, + "loss": 1.2957, + "theoretical_loss": 3.4050451184825605, + "tokens_seen": 2426470400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026733269138180067, + "loss": 1.2076, + "theoretical_loss": 3.4050304829122804, + "tokens_seen": 2426601472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672925694110095, + "loss": 1.3606, + "theoretical_loss": 3.40501584835385, + "tokens_seen": 2426732544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672524474402183, + "loss": 1.2974, + "theoretical_loss": 3.405001214807145, + "tokens_seen": 2426863616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672123254694271, + "loss": 1.1959, + "theoretical_loss": 3.4049865822720404, + "tokens_seen": 2426994688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026717220349863584, + "loss": 1.2654, + "theoretical_loss": 3.404971950748412, + "tokens_seen": 2427125760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026713208152784465, + "loss": 1.2943, + "theoretical_loss": 3.404957320236136, + "tokens_seen": 2427256832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026709195955705345, + "loss": 1.2267, + "theoretical_loss": 3.404942690735086, + "tokens_seen": 2427387904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670518375862622, + "loss": 1.3798, + "theoretical_loss": 3.404928062245139, + "tokens_seen": 2427518976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026701171561547106, + "loss": 1.2455, + "theoretical_loss": 3.4049134347661703, + "tokens_seen": 2427650048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669715936446798, + "loss": 1.3936, + "theoretical_loss": 3.404898808298055, + "tokens_seen": 2427781120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669314716738886, + "loss": 1.3635, + "theoretical_loss": 3.404884182840669, + "tokens_seen": 2427912192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026689134970309743, + "loss": 1.2677, + "theoretical_loss": 3.4048695583938877, + "tokens_seen": 2428043264 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668512277323062, + "loss": 1.4028, + "theoretical_loss": 3.404854934957587, + "tokens_seen": 2428174336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026681110576151504, + "loss": 1.3641, + "theoretical_loss": 3.4048403125316424, + "tokens_seen": 2428305408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667709837907238, + "loss": 1.2774, + "theoretical_loss": 3.4048256911159296, + "tokens_seen": 2428436480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667308618199326, + "loss": 1.286, + "theoretical_loss": 3.4048110707103243, + "tokens_seen": 2428567552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666907398491414, + "loss": 1.2815, + "theoretical_loss": 3.4047964513147018, + "tokens_seen": 2428698624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666506178783502, + "loss": 1.2436, + "theoretical_loss": 3.4047818329289385, + "tokens_seen": 2428829696 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026661049590755896, + "loss": 1.2786, + "theoretical_loss": 3.40476721555291, + "tokens_seen": 2428960768 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665703739367678, + "loss": 1.2639, + "theoretical_loss": 3.4047525991864913, + "tokens_seen": 2429091840 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665302519659766, + "loss": 1.3537, + "theoretical_loss": 3.404737983829559, + "tokens_seen": 2429222912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026649012999518533, + "loss": 1.2759, + "theoretical_loss": 3.404723369481989, + "tokens_seen": 2429353984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664500080243942, + "loss": 1.1927, + "theoretical_loss": 3.4047087561436564, + "tokens_seen": 2429485056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026640988605360294, + "loss": 1.2891, + "theoretical_loss": 3.4046941438144382, + "tokens_seen": 2429616128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026636976408281175, + "loss": 1.3388, + "theoretical_loss": 3.4046795324942094, + "tokens_seen": 2429747200 + }, + { + "epoch": 0.47, + "objective/train/aligned_token_count": 0.9913833737373352, + "objective/train/avg_token_score": 0.008616648614406586, + "objective/train/docs_used": 884501, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.331422209739685, + "objective/train/original_loss": 1.3445264101028442, + "objective/train/theoretical_loss": 3.4046649221828456, + "objective/train/tokens_used": 800337376, + "objective/train/unlikelihood_loss": 0.025063809007406235, + "theoretical_loss": 3.4046649221828456, + "tokens_seen": 2429878272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026632964211202055, + "loss": 1.2861, + "theoretical_loss": 3.4046649221828456, + "tokens_seen": 2429878272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026628952014122936, + "loss": 1.3183, + "theoretical_loss": 3.4046503128802237, + "tokens_seen": 2430009344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662493981704381, + "loss": 1.3075, + "theoretical_loss": 3.404635704586219, + "tokens_seen": 2430140416 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662092761996469, + "loss": 1.3511, + "theoretical_loss": 3.4046210973007076, + "tokens_seen": 2430271488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661691542288557, + "loss": 1.4054, + "theoretical_loss": 3.4046064910235656, + "tokens_seen": 2430402560 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026612903225806453, + "loss": 1.2227, + "theoretical_loss": 3.4045918857546686, + "tokens_seen": 2430533632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026608891028727334, + "loss": 1.361, + "theoretical_loss": 3.4045772814938937, + "tokens_seen": 2430664704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660487883164821, + "loss": 1.2221, + "theoretical_loss": 3.404562678241116, + "tokens_seen": 2430795776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026600866634569095, + "loss": 1.2543, + "theoretical_loss": 3.4045480759962117, + "tokens_seen": 2430926848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002659685443748997, + "loss": 1.285, + "theoretical_loss": 3.4045334747590568, + "tokens_seen": 2431057920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026592842240410846, + "loss": 1.3303, + "theoretical_loss": 3.4045188745295283, + "tokens_seen": 2431188992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658883004333173, + "loss": 1.337, + "theoretical_loss": 3.404504275307501, + "tokens_seen": 2431320064 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026584817846252607, + "loss": 1.2326, + "theoretical_loss": 3.404489677092852, + "tokens_seen": 2431451136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658080564917349, + "loss": 1.3006, + "theoretical_loss": 3.4044750798854575, + "tokens_seen": 2431582208 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657679345209437, + "loss": 1.3492, + "theoretical_loss": 3.4044604836851935, + "tokens_seen": 2431713280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657278125501525, + "loss": 1.2246, + "theoretical_loss": 3.404445888491936, + "tokens_seen": 2431844352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026568769057936124, + "loss": 1.3051, + "theoretical_loss": 3.404431294305561, + "tokens_seen": 2431975424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026564756860857005, + "loss": 1.3162, + "theoretical_loss": 3.404416701125946, + "tokens_seen": 2432106496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026560744663777885, + "loss": 1.2886, + "theoretical_loss": 3.4044021089529664, + "tokens_seen": 2432237568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026556732466698766, + "loss": 1.2636, + "theoretical_loss": 3.4043875177864984, + "tokens_seen": 2432368640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026552720269619646, + "loss": 1.2915, + "theoretical_loss": 3.4043729276264187, + "tokens_seen": 2432499712 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654870807254052, + "loss": 1.2896, + "theoretical_loss": 3.4043583384726035, + "tokens_seen": 2432630784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654469587546141, + "loss": 1.3527, + "theoretical_loss": 3.404343750324929, + "tokens_seen": 2432761856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026540683678382283, + "loss": 1.279, + "theoretical_loss": 3.4043291631832724, + "tokens_seen": 2432892928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653667148130316, + "loss": 1.2396, + "theoretical_loss": 3.4043145770475096, + "tokens_seen": 2433024000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026532659284224044, + "loss": 1.2651, + "theoretical_loss": 3.4042999919175165, + "tokens_seen": 2433155072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652864708714492, + "loss": 1.3049, + "theoretical_loss": 3.4042854077931706, + "tokens_seen": 2433286144 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265246348900658, + "loss": 1.2726, + "theoretical_loss": 3.404270824674348, + "tokens_seen": 2433417216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652062269298668, + "loss": 1.212, + "theoretical_loss": 3.4042562425609244, + "tokens_seen": 2433548288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002651661049590756, + "loss": 1.3642, + "theoretical_loss": 3.4042416614527777, + "tokens_seen": 2433679360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026512598298828436, + "loss": 1.385, + "theoretical_loss": 3.4042270813497835, + "tokens_seen": 2433810432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026508586101749317, + "loss": 1.2618, + "theoretical_loss": 3.404212502251819, + "tokens_seen": 2433941504 + }, + { + "epoch": 0.48, + "learning_rate": 0.000265045739046702, + "loss": 1.3576, + "theoretical_loss": 3.40419792415876, + "tokens_seen": 2434072576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026500561707591073, + "loss": 1.3094, + "theoretical_loss": 3.404183347070484, + "tokens_seen": 2434203648 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649654951051196, + "loss": 1.2292, + "theoretical_loss": 3.4041687709868675, + "tokens_seen": 2434334720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026492537313432834, + "loss": 1.3355, + "theoretical_loss": 3.404154195907787, + "tokens_seen": 2434465792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648852511635372, + "loss": 1.3256, + "theoretical_loss": 3.404139621833119, + "tokens_seen": 2434596864 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026484512919274595, + "loss": 1.2452, + "theoretical_loss": 3.40412504876274, + "tokens_seen": 2434727936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026480500722195476, + "loss": 1.2768, + "theoretical_loss": 3.4041104766965273, + "tokens_seen": 2434859008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026476488525116357, + "loss": 1.2437, + "theoretical_loss": 3.4040959056343576, + "tokens_seen": 2434990080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647247632803723, + "loss": 1.3111, + "theoretical_loss": 3.4040813355761075, + "tokens_seen": 2435121152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646846413095811, + "loss": 1.32, + "theoretical_loss": 3.404066766521654, + "tokens_seen": 2435252224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026464451933878993, + "loss": 1.2793, + "theoretical_loss": 3.4040521984708736, + "tokens_seen": 2435383296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026460439736799874, + "loss": 1.2234, + "theoretical_loss": 3.404037631423643, + "tokens_seen": 2435514368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645642753972075, + "loss": 1.3366, + "theoretical_loss": 3.40402306537984, + "tokens_seen": 2435645440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026452415342641635, + "loss": 1.2578, + "theoretical_loss": 3.4040085003393403, + "tokens_seen": 2435776512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644840314556251, + "loss": 1.2988, + "theoretical_loss": 3.4039939363020215, + "tokens_seen": 2435907584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026444390948483385, + "loss": 1.2279, + "theoretical_loss": 3.4039793732677603, + "tokens_seen": 2436038656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644037875140427, + "loss": 1.3531, + "theoretical_loss": 3.4039648112364342, + "tokens_seen": 2436169728 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026436366554325147, + "loss": 1.3582, + "theoretical_loss": 3.4039502502079193, + "tokens_seen": 2436300800 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.9900150299072266, + "objective/train/avg_token_score": 0.009984994307160378, + "objective/train/docs_used": 886917, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.292373776435852, + "objective/train/original_loss": 1.3096230030059814, + "objective/train/theoretical_loss": 3.4039356901820934, + "objective/train/tokens_used": 806890976, + "objective/train/unlikelihood_loss": 0.02295640856027603, + "theoretical_loss": 3.4039356901820934, + "tokens_seen": 2436431872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026432354357246033, + "loss": 1.2798, + "theoretical_loss": 3.4039356901820934, + "tokens_seen": 2436431872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642834216016691, + "loss": 1.2899, + "theoretical_loss": 3.4039211311588327, + "tokens_seen": 2436562944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642432996308779, + "loss": 1.2875, + "theoretical_loss": 3.4039065731380154, + "tokens_seen": 2436694016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642031776600867, + "loss": 1.3032, + "theoretical_loss": 3.403892016119517, + "tokens_seen": 2436825088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026416305568929544, + "loss": 1.2256, + "theoretical_loss": 3.403877460103216, + "tokens_seen": 2436956160 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026412293371850425, + "loss": 1.2709, + "theoretical_loss": 3.403862905088989, + "tokens_seen": 2437087232 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026408281174771306, + "loss": 1.2768, + "theoretical_loss": 3.4038483510767126, + "tokens_seen": 2437218304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026404268977692186, + "loss": 1.2856, + "theoretical_loss": 3.4038337980662643, + "tokens_seen": 2437349376 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640025678061306, + "loss": 1.3387, + "theoretical_loss": 3.403819246057522, + "tokens_seen": 2437480448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002639624458353395, + "loss": 1.3405, + "theoretical_loss": 3.4038046950503618, + "tokens_seen": 2437611520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026392232386454823, + "loss": 1.3039, + "theoretical_loss": 3.4037901450446615, + "tokens_seen": 2437742592 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263882201893757, + "loss": 1.3291, + "theoretical_loss": 3.403775596040298, + "tokens_seen": 2437873664 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026384207992296584, + "loss": 1.2996, + "theoretical_loss": 3.4037610480371487, + "tokens_seen": 2438004736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638019579521746, + "loss": 1.3953, + "theoretical_loss": 3.4037465010350916, + "tokens_seen": 2438135808 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637618359813834, + "loss": 1.2543, + "theoretical_loss": 3.4037319550340026, + "tokens_seen": 2438266880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637217140105922, + "loss": 1.2558, + "theoretical_loss": 3.40371741003376, + "tokens_seen": 2438397952 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263681592039801, + "loss": 1.2882, + "theoretical_loss": 3.4037028660342417, + "tokens_seen": 2438529024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636414700690098, + "loss": 1.2982, + "theoretical_loss": 3.403688323035323, + "tokens_seen": 2438660096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026360134809821857, + "loss": 1.2844, + "theoretical_loss": 3.4036737810368836, + "tokens_seen": 2438791168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635612261274274, + "loss": 1.2977, + "theoretical_loss": 3.4036592400387997, + "tokens_seen": 2438922240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635211041566362, + "loss": 1.2867, + "theoretical_loss": 3.403644700040948, + "tokens_seen": 2439053312 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263480982185845, + "loss": 1.3069, + "theoretical_loss": 3.403630161043208, + "tokens_seen": 2439184384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026344086021505374, + "loss": 1.2876, + "theoretical_loss": 3.4036156230454555, + "tokens_seen": 2439315456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634007382442626, + "loss": 1.2828, + "theoretical_loss": 3.4036010860475683, + "tokens_seen": 2439446528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026336061627347135, + "loss": 1.2728, + "theoretical_loss": 3.4035865500494245, + "tokens_seen": 2439577600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026332049430268016, + "loss": 1.3474, + "theoretical_loss": 3.4035720150509015, + "tokens_seen": 2439708672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026328037233188897, + "loss": 1.3009, + "theoretical_loss": 3.403557481051876, + "tokens_seen": 2439839744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632402503610977, + "loss": 1.2753, + "theoretical_loss": 3.4035429480522263, + "tokens_seen": 2439970816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632001283903065, + "loss": 1.2991, + "theoretical_loss": 3.4035284160518304, + "tokens_seen": 2440101888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026316000641951533, + "loss": 1.3251, + "theoretical_loss": 3.403513885050565, + "tokens_seen": 2440232960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026311988444872414, + "loss": 1.2242, + "theoretical_loss": 3.403499355048308, + "tokens_seen": 2440364032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630797624779329, + "loss": 1.3084, + "theoretical_loss": 3.403484826044938, + "tokens_seen": 2440495104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026303964050714175, + "loss": 1.3055, + "theoretical_loss": 3.4034702980403315, + "tokens_seen": 2440626176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629995185363505, + "loss": 1.3136, + "theoretical_loss": 3.4034557710343667, + "tokens_seen": 2440757248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629593965655593, + "loss": 1.2891, + "theoretical_loss": 3.4034412450269214, + "tokens_seen": 2440888320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629192745947681, + "loss": 1.2934, + "theoretical_loss": 3.403426720017873, + "tokens_seen": 2441019392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026287915262397687, + "loss": 1.3327, + "theoretical_loss": 3.4034121960070993, + "tokens_seen": 2441150464 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026283903065318573, + "loss": 1.27, + "theoretical_loss": 3.403397672994479, + "tokens_seen": 2441281536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627989086823945, + "loss": 1.2453, + "theoretical_loss": 3.4033831509798884, + "tokens_seen": 2441412608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627587867116033, + "loss": 1.3642, + "theoretical_loss": 3.4033686299632064, + "tokens_seen": 2441543680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627186647408121, + "loss": 1.3771, + "theoretical_loss": 3.4033541099443108, + "tokens_seen": 2441674752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026267854277002084, + "loss": 1.2583, + "theoretical_loss": 3.4033395909230792, + "tokens_seen": 2441805824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026263842079922965, + "loss": 1.3241, + "theoretical_loss": 3.40332507289939, + "tokens_seen": 2441936896 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026259829882843846, + "loss": 1.3627, + "theoretical_loss": 3.4033105558731203, + "tokens_seen": 2442067968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026255817685764726, + "loss": 1.2302, + "theoretical_loss": 3.4032960398441485, + "tokens_seen": 2442199040 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262518054886856, + "loss": 1.3005, + "theoretical_loss": 3.4032815248123525, + "tokens_seen": 2442330112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624779329160649, + "loss": 1.3282, + "theoretical_loss": 3.4032670107776104, + "tokens_seen": 2442461184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026243781094527363, + "loss": 1.3067, + "theoretical_loss": 3.4032524977398, + "tokens_seen": 2442592256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623976889744824, + "loss": 1.2397, + "theoretical_loss": 3.4032379856988, + "tokens_seen": 2442723328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026235756700369124, + "loss": 1.3254, + "theoretical_loss": 3.403223474654488, + "tokens_seen": 2442854400 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.9921513199806213, + "objective/train/avg_token_score": 0.007848700508475304, + "objective/train/docs_used": 889343, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2812368869781494, + "objective/train/original_loss": 1.2901561260223389, + "objective/train/theoretical_loss": 3.4032089646067414, + "objective/train/tokens_used": 813444576, + "objective/train/unlikelihood_loss": 0.026721792295575142, + "theoretical_loss": 3.4032089646067414, + "tokens_seen": 2442985472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623174450329, + "loss": 1.2312, + "theoretical_loss": 3.4032089646067414, + "tokens_seen": 2442985472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026227732306210885, + "loss": 1.3497, + "theoretical_loss": 3.4031944555554388, + "tokens_seen": 2443116544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622372010913176, + "loss": 1.2949, + "theoretical_loss": 3.403179947500459, + "tokens_seen": 2443247616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621970791205264, + "loss": 1.2618, + "theoretical_loss": 3.403165440441679, + "tokens_seen": 2443378688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621569571497352, + "loss": 1.3763, + "theoretical_loss": 3.403150934378978, + "tokens_seen": 2443509760 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026211683517894397, + "loss": 1.3578, + "theoretical_loss": 3.4031364293122337, + "tokens_seen": 2443640832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620767132081528, + "loss": 1.3168, + "theoretical_loss": 3.403121925241324, + "tokens_seen": 2443771904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620365912373616, + "loss": 1.3077, + "theoretical_loss": 3.4031074221661273, + "tokens_seen": 2443902976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619964692665704, + "loss": 1.293, + "theoretical_loss": 3.4030929200865225, + "tokens_seen": 2444034048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026195634729577914, + "loss": 1.2454, + "theoretical_loss": 3.4030784190023873, + "tokens_seen": 2444165120 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261916225324988, + "loss": 1.3033, + "theoretical_loss": 3.4030639189135994, + "tokens_seen": 2444296192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026187610335419675, + "loss": 1.3056, + "theoretical_loss": 3.4030494198200385, + "tokens_seen": 2444427264 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026183598138340556, + "loss": 1.2722, + "theoretical_loss": 3.4030349217215816, + "tokens_seen": 2444558336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026179585941261437, + "loss": 1.3461, + "theoretical_loss": 3.403020424618108, + "tokens_seen": 2444689408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617557374418231, + "loss": 1.2825, + "theoretical_loss": 3.4030059285094953, + "tokens_seen": 2444820480 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261715615471032, + "loss": 1.2532, + "theoretical_loss": 3.4029914333956226, + "tokens_seen": 2444951552 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026167549350024073, + "loss": 1.2869, + "theoretical_loss": 3.4029769392763685, + "tokens_seen": 2445082624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026163537152944954, + "loss": 1.3642, + "theoretical_loss": 3.4029624461516104, + "tokens_seen": 2445213696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026159524955865834, + "loss": 1.322, + "theoretical_loss": 3.4029479540212275, + "tokens_seen": 2445344768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026155512758786715, + "loss": 1.3636, + "theoretical_loss": 3.402933462885098, + "tokens_seen": 2445475840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615150056170759, + "loss": 1.2922, + "theoretical_loss": 3.4029189727431004, + "tokens_seen": 2445606912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614748836462847, + "loss": 1.2462, + "theoretical_loss": 3.4029044835951137, + "tokens_seen": 2445737984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614347616754935, + "loss": 1.322, + "theoretical_loss": 3.4028899954410154, + "tokens_seen": 2445869056 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026139463970470227, + "loss": 1.2912, + "theoretical_loss": 3.4028755082806854, + "tokens_seen": 2446000128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026135451773391113, + "loss": 1.2496, + "theoretical_loss": 3.4028610221140014, + "tokens_seen": 2446131200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613143957631199, + "loss": 1.2772, + "theoretical_loss": 3.4028465369408423, + "tokens_seen": 2446262272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612742737923287, + "loss": 1.3262, + "theoretical_loss": 3.4028320527610867, + "tokens_seen": 2446393344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612341518215375, + "loss": 1.3246, + "theoretical_loss": 3.402817569574613, + "tokens_seen": 2446524416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026119402985074624, + "loss": 1.2201, + "theoretical_loss": 3.4028030873813004, + "tokens_seen": 2446655488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026115390787995505, + "loss": 1.335, + "theoretical_loss": 3.4027886061810273, + "tokens_seen": 2446786560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026111378590916386, + "loss": 1.2657, + "theoretical_loss": 3.4027741259736723, + "tokens_seen": 2446917632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026107366393837266, + "loss": 1.2956, + "theoretical_loss": 3.4027596467591144, + "tokens_seen": 2447048704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026103354196758147, + "loss": 1.249, + "theoretical_loss": 3.402745168537232, + "tokens_seen": 2447179776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609934199967903, + "loss": 1.2644, + "theoretical_loss": 3.4027306913079043, + "tokens_seen": 2447310848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026095329802599903, + "loss": 1.231, + "theoretical_loss": 3.4027162150710097, + "tokens_seen": 2447441920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026091317605520783, + "loss": 1.2532, + "theoretical_loss": 3.4027017398264277, + "tokens_seen": 2447572992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026087305408441664, + "loss": 1.3241, + "theoretical_loss": 3.4026872655740363, + "tokens_seen": 2447704064 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608329321136254, + "loss": 1.3117, + "theoretical_loss": 3.402672792313715, + "tokens_seen": 2447835136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026079281014283425, + "loss": 1.2995, + "theoretical_loss": 3.402658320045342, + "tokens_seen": 2447966208 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260752688172043, + "loss": 1.2607, + "theoretical_loss": 3.402643848768797, + "tokens_seen": 2448097280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607125662012518, + "loss": 1.2741, + "theoretical_loss": 3.402629378483958, + "tokens_seen": 2448228352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606724442304606, + "loss": 1.3411, + "theoretical_loss": 3.4026149091907048, + "tokens_seen": 2448359424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026063232225966937, + "loss": 1.3033, + "theoretical_loss": 3.402600440888916, + "tokens_seen": 2448490496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605922002888782, + "loss": 1.3605, + "theoretical_loss": 3.4025859735784714, + "tokens_seen": 2448621568 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260552078318087, + "loss": 1.2911, + "theoretical_loss": 3.4025715072592484, + "tokens_seen": 2448752640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605119563472958, + "loss": 1.236, + "theoretical_loss": 3.4025570419311273, + "tokens_seen": 2448883712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604718343765046, + "loss": 1.3152, + "theoretical_loss": 3.402542577593987, + "tokens_seen": 2449014784 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604317124057134, + "loss": 1.2783, + "theoretical_loss": 3.4025281142477057, + "tokens_seen": 2449145856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026039159043492215, + "loss": 1.3406, + "theoretical_loss": 3.4025136518921637, + "tokens_seen": 2449276928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026035146846413096, + "loss": 1.2311, + "theoretical_loss": 3.40249919052724, + "tokens_seen": 2449408000 + }, + { + "epoch": 0.48, + "objective/train/aligned_token_count": 0.9941378831863403, + "objective/train/avg_token_score": 0.00586214754730463, + "objective/train/docs_used": 891543, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2284047603607178, + "objective/train/original_loss": 1.2358827590942383, + "objective/train/theoretical_loss": 3.4024847301528123, + "objective/train/tokens_used": 819998176, + "objective/train/unlikelihood_loss": 0.02044907584786415, + "theoretical_loss": 3.4024847301528123, + "tokens_seen": 2449539072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026031134649333977, + "loss": 1.188, + "theoretical_loss": 3.4024847301528123, + "tokens_seen": 2449539072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602712245225485, + "loss": 1.21, + "theoretical_loss": 3.4024702707687613, + "tokens_seen": 2449670144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602311025517574, + "loss": 1.244, + "theoretical_loss": 3.4024558123749657, + "tokens_seen": 2449801216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026019098058096613, + "loss": 1.209, + "theoretical_loss": 3.4024413549713044, + "tokens_seen": 2449932288 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026015085861017494, + "loss": 1.1944, + "theoretical_loss": 3.4024268985576573, + "tokens_seen": 2450063360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026011073663938374, + "loss": 1.2469, + "theoretical_loss": 3.4024124431339033, + "tokens_seen": 2450194432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026007061466859255, + "loss": 1.1816, + "theoretical_loss": 3.4023979886999216, + "tokens_seen": 2450325504 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600304926978013, + "loss": 1.1713, + "theoretical_loss": 3.402383535255591, + "tokens_seen": 2450456576 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599903707270101, + "loss": 1.311, + "theoretical_loss": 3.4023690828007918, + "tokens_seen": 2450587648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599502487562189, + "loss": 1.2805, + "theoretical_loss": 3.4023546313354025, + "tokens_seen": 2450718720 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025991012678542767, + "loss": 1.2354, + "theoretical_loss": 3.4023401808593032, + "tokens_seen": 2450849792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598700048146365, + "loss": 1.2684, + "theoretical_loss": 3.4023257313723727, + "tokens_seen": 2450980864 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598298828438453, + "loss": 1.2407, + "theoretical_loss": 3.402311282874491, + "tokens_seen": 2451111936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025978976087305414, + "loss": 1.2592, + "theoretical_loss": 3.4022968353655365, + "tokens_seen": 2451243008 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597496389022629, + "loss": 1.1879, + "theoretical_loss": 3.4022823888453892, + "tokens_seen": 2451374080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025970951693147164, + "loss": 1.2603, + "theoretical_loss": 3.4022679433139293, + "tokens_seen": 2451505152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002596693949606805, + "loss": 1.2278, + "theoretical_loss": 3.402253498771035, + "tokens_seen": 2451636224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025962927298988926, + "loss": 1.307, + "theoretical_loss": 3.4022390552165866, + "tokens_seen": 2451767296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025958915101909806, + "loss": 1.1804, + "theoretical_loss": 3.402224612650463, + "tokens_seen": 2451898368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025954902904830687, + "loss": 1.2285, + "theoretical_loss": 3.4022101710725448, + "tokens_seen": 2452029440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595089070775157, + "loss": 1.2021, + "theoretical_loss": 3.4021957304827106, + "tokens_seen": 2452160512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025946878510672443, + "loss": 1.2132, + "theoretical_loss": 3.4021812908808404, + "tokens_seen": 2452291584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025942866313593323, + "loss": 1.2597, + "theoretical_loss": 3.4021668522668134, + "tokens_seen": 2452422656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025938854116514204, + "loss": 1.2978, + "theoretical_loss": 3.40215241464051, + "tokens_seen": 2452553728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593484191943508, + "loss": 1.2027, + "theoretical_loss": 3.402137978001809, + "tokens_seen": 2452684800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025930829722355965, + "loss": 1.2757, + "theoretical_loss": 3.4021235423505907, + "tokens_seen": 2452815872 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592681752527684, + "loss": 1.2522, + "theoretical_loss": 3.4021091076867345, + "tokens_seen": 2452946944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025922805328197727, + "loss": 1.2641, + "theoretical_loss": 3.40209467401012, + "tokens_seen": 2453078016 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259187931311186, + "loss": 1.3268, + "theoretical_loss": 3.4020802413206273, + "tokens_seen": 2453209088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025914780934039477, + "loss": 1.2402, + "theoretical_loss": 3.402065809618136, + "tokens_seen": 2453340160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025910768736960363, + "loss": 1.2157, + "theoretical_loss": 3.4020513789025255, + "tokens_seen": 2453471232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590675653988124, + "loss": 1.2141, + "theoretical_loss": 3.4020369491736764, + "tokens_seen": 2453602304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590274434280212, + "loss": 1.3163, + "theoretical_loss": 3.4020225204314674, + "tokens_seen": 2453733376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025898732145723, + "loss": 1.1884, + "theoretical_loss": 3.4020080926757794, + "tokens_seen": 2453864448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589471994864388, + "loss": 1.2365, + "theoretical_loss": 3.401993665906492, + "tokens_seen": 2453995520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025890707751564755, + "loss": 1.2474, + "theoretical_loss": 3.401979240123485, + "tokens_seen": 2454126592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025886695554485636, + "loss": 1.3335, + "theoretical_loss": 3.4019648153266377, + "tokens_seen": 2454257664 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025882683357406517, + "loss": 1.2103, + "theoretical_loss": 3.401950391515831, + "tokens_seen": 2454388736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587867116032739, + "loss": 1.2323, + "theoretical_loss": 3.4019359686909443, + "tokens_seen": 2454519808 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587465896324828, + "loss": 1.2917, + "theoretical_loss": 3.4019215468518578, + "tokens_seen": 2454650880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025870646766169153, + "loss": 1.2901, + "theoretical_loss": 3.401907125998451, + "tokens_seen": 2454781952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025866634569090034, + "loss": 1.2536, + "theoretical_loss": 3.4018927061306043, + "tokens_seen": 2454913024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025862622372010914, + "loss": 1.2226, + "theoretical_loss": 3.401878287248198, + "tokens_seen": 2455044096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025858610174931795, + "loss": 1.3163, + "theoretical_loss": 3.4018638693511116, + "tokens_seen": 2455175168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025854597977852676, + "loss": 1.2854, + "theoretical_loss": 3.4018494524392255, + "tokens_seen": 2455306240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585058578077355, + "loss": 1.2753, + "theoretical_loss": 3.40183503651242, + "tokens_seen": 2455437312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584657358369443, + "loss": 1.2829, + "theoretical_loss": 3.4018206215705744, + "tokens_seen": 2455568384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584256138661531, + "loss": 1.2467, + "theoretical_loss": 3.4018062076135696, + "tokens_seen": 2455699456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583854918953619, + "loss": 1.2277, + "theoretical_loss": 3.4017917946412854, + "tokens_seen": 2455830528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583453699245707, + "loss": 1.2249, + "theoretical_loss": 3.401777382653602, + "tokens_seen": 2455961600 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.9953801035881042, + "objective/train/avg_token_score": 0.0046198489144444466, + "objective/train/docs_used": 893908, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1292229890823364, + "objective/train/original_loss": 1.1348636150360107, + "objective/train/theoretical_loss": 3.4017629716504, + "objective/train/tokens_used": 826551776, + "objective/train/unlikelihood_loss": 0.01574266329407692, + "theoretical_loss": 3.4017629716504, + "tokens_seen": 2456092672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025830524795377954, + "loss": 1.1796, + "theoretical_loss": 3.4017629716504, + "tokens_seen": 2456092672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582651259829883, + "loss": 1.2248, + "theoretical_loss": 3.401748561631559, + "tokens_seen": 2456223744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025822500401219704, + "loss": 1.2103, + "theoretical_loss": 3.4017341525969598, + "tokens_seen": 2456354816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581848820414059, + "loss": 1.2772, + "theoretical_loss": 3.401719744546482, + "tokens_seen": 2456485888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025814476007061466, + "loss": 1.3054, + "theoretical_loss": 3.4017053374800064, + "tokens_seen": 2456616960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025810463809982346, + "loss": 1.2179, + "theoretical_loss": 3.4016909313974133, + "tokens_seen": 2456748032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025806451612903227, + "loss": 1.2358, + "theoretical_loss": 3.401676526298583, + "tokens_seen": 2456879104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580243941582411, + "loss": 1.2697, + "theoretical_loss": 3.401662122183396, + "tokens_seen": 2457010176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579842721874498, + "loss": 1.2138, + "theoretical_loss": 3.4016477190517316, + "tokens_seen": 2457141248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025794415021665863, + "loss": 1.2883, + "theoretical_loss": 3.4016333169034714, + "tokens_seen": 2457272320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025790402824586744, + "loss": 1.2735, + "theoretical_loss": 3.4016189157384953, + "tokens_seen": 2457403392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025786390627507625, + "loss": 1.3386, + "theoretical_loss": 3.4016045155566843, + "tokens_seen": 2457534464 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025782378430428505, + "loss": 1.2735, + "theoretical_loss": 3.4015901163579176, + "tokens_seen": 2457665536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577836623334938, + "loss": 1.2256, + "theoretical_loss": 3.401575718142077, + "tokens_seen": 2457796608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025774354036270266, + "loss": 1.2325, + "theoretical_loss": 3.4015613209090425, + "tokens_seen": 2457927680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577034183919114, + "loss": 1.2176, + "theoretical_loss": 3.4015469246586942, + "tokens_seen": 2458058752 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025766329642112017, + "loss": 1.2355, + "theoretical_loss": 3.401532529390913, + "tokens_seen": 2458189824 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025762317445032903, + "loss": 1.2576, + "theoretical_loss": 3.4015181351055794, + "tokens_seen": 2458320896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575830524795378, + "loss": 1.2336, + "theoretical_loss": 3.4015037418025744, + "tokens_seen": 2458451968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575429305087466, + "loss": 1.2577, + "theoretical_loss": 3.401489349481778, + "tokens_seen": 2458583040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575028085379554, + "loss": 1.2274, + "theoretical_loss": 3.401474958143071, + "tokens_seen": 2458714112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574626865671642, + "loss": 1.233, + "theoretical_loss": 3.4014605677863337, + "tokens_seen": 2458845184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025742256459637295, + "loss": 1.2594, + "theoretical_loss": 3.4014461784114474, + "tokens_seen": 2458976256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025738244262558176, + "loss": 1.2826, + "theoretical_loss": 3.401431790018292, + "tokens_seen": 2459107328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025734232065479057, + "loss": 1.2899, + "theoretical_loss": 3.4014174026067496, + "tokens_seen": 2459238400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573021986839993, + "loss": 1.1546, + "theoretical_loss": 3.4014030161766993, + "tokens_seen": 2459369472 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572620767132082, + "loss": 1.2534, + "theoretical_loss": 3.401388630728023, + "tokens_seen": 2459500544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025722195474241693, + "loss": 1.2304, + "theoretical_loss": 3.401374246260601, + "tokens_seen": 2459631616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571818327716258, + "loss": 1.1895, + "theoretical_loss": 3.4013598627743136, + "tokens_seen": 2459762688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025714171080083454, + "loss": 1.2239, + "theoretical_loss": 3.4013454802690424, + "tokens_seen": 2459893760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025710158883004335, + "loss": 1.2426, + "theoretical_loss": 3.401331098744668, + "tokens_seen": 2460024832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025706146685925216, + "loss": 1.3128, + "theoretical_loss": 3.4013167182010706, + "tokens_seen": 2460155904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570213448884609, + "loss": 1.169, + "theoretical_loss": 3.401302338638132, + "tokens_seen": 2460286976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569812229176697, + "loss": 1.1827, + "theoretical_loss": 3.4012879600557326, + "tokens_seen": 2460418048 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569411009468785, + "loss": 1.2861, + "theoretical_loss": 3.4012735824537534, + "tokens_seen": 2460549120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569009789760873, + "loss": 1.2018, + "theoretical_loss": 3.401259205832076, + "tokens_seen": 2460680192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568608570052961, + "loss": 1.2542, + "theoretical_loss": 3.40124483019058, + "tokens_seen": 2460811264 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025682073503450494, + "loss": 1.2413, + "theoretical_loss": 3.4012304555291473, + "tokens_seen": 2460942336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567806130637137, + "loss": 1.2961, + "theoretical_loss": 3.4012160818476587, + "tokens_seen": 2461073408 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025674049109292244, + "loss": 1.3196, + "theoretical_loss": 3.4012017091459947, + "tokens_seen": 2461204480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567003691221313, + "loss": 1.2259, + "theoretical_loss": 3.4011873374240373, + "tokens_seen": 2461335552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025666024715134006, + "loss": 1.2069, + "theoretical_loss": 3.4011729666816666, + "tokens_seen": 2461466624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566201251805489, + "loss": 1.27, + "theoretical_loss": 3.4011585969187648, + "tokens_seen": 2461597696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025658000320975767, + "loss": 1.2524, + "theoretical_loss": 3.4011442281352116, + "tokens_seen": 2461728768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565398812389665, + "loss": 1.2132, + "theoretical_loss": 3.4011298603308893, + "tokens_seen": 2461859840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564997592681753, + "loss": 1.2445, + "theoretical_loss": 3.401115493505678, + "tokens_seen": 2461990912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025645963729738403, + "loss": 1.2757, + "theoretical_loss": 3.40110112765946, + "tokens_seen": 2462121984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025641951532659284, + "loss": 1.2245, + "theoretical_loss": 3.4010867627921155, + "tokens_seen": 2462253056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025637939335580165, + "loss": 1.2689, + "theoretical_loss": 3.4010723989035263, + "tokens_seen": 2462384128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025633927138501045, + "loss": 1.2871, + "theoretical_loss": 3.4010580359935734, + "tokens_seen": 2462515200 + }, + { + "epoch": 0.49, + "objective/train/aligned_token_count": 0.9915258288383484, + "objective/train/avg_token_score": 0.008474156260490417, + "objective/train/docs_used": 896263, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3047091960906982, + "objective/train/original_loss": 1.3164501190185547, + "objective/train/theoretical_loss": 3.401043674062138, + "objective/train/tokens_used": 833105376, + "objective/train/unlikelihood_loss": 0.023283755406737328, + "theoretical_loss": 3.401043674062138, + "tokens_seen": 2462646272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562991494142192, + "loss": 1.238, + "theoretical_loss": 3.401043674062138, + "tokens_seen": 2462646272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025625902744342806, + "loss": 1.1799, + "theoretical_loss": 3.4010293131091016, + "tokens_seen": 2462777344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562189054726368, + "loss": 1.2912, + "theoretical_loss": 3.401014953134345, + "tokens_seen": 2462908416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025617878350184557, + "loss": 1.1376, + "theoretical_loss": 3.4010005941377504, + "tokens_seen": 2463039488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025613866153105443, + "loss": 1.3367, + "theoretical_loss": 3.4009862361191976, + "tokens_seen": 2463170560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560985395602632, + "loss": 1.2614, + "theoretical_loss": 3.4009718790785692, + "tokens_seen": 2463301632 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256058417589472, + "loss": 1.2056, + "theoretical_loss": 3.4009575230157467, + "tokens_seen": 2463432704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560182956186808, + "loss": 1.2093, + "theoretical_loss": 3.4009431679306106, + "tokens_seen": 2463563776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559781736478896, + "loss": 1.2894, + "theoretical_loss": 3.4009288138230427, + "tokens_seen": 2463694848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559380516770984, + "loss": 1.2019, + "theoretical_loss": 3.4009144606929245, + "tokens_seen": 2463825920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025589792970630716, + "loss": 1.191, + "theoretical_loss": 3.4009001085401374, + "tokens_seen": 2463956992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025585780773551597, + "loss": 1.3091, + "theoretical_loss": 3.400885757364563, + "tokens_seen": 2464088064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025581768576472477, + "loss": 1.2281, + "theoretical_loss": 3.400871407166082, + "tokens_seen": 2464219136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557775637939336, + "loss": 1.226, + "theoretical_loss": 3.4008570579445774, + "tokens_seen": 2464350208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025573744182314233, + "loss": 1.2471, + "theoretical_loss": 3.4008427096999294, + "tokens_seen": 2464481280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556973198523512, + "loss": 1.2412, + "theoretical_loss": 3.40082836243202, + "tokens_seen": 2464612352 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025565719788155994, + "loss": 1.2243, + "theoretical_loss": 3.400814016140731, + "tokens_seen": 2464743424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556170759107687, + "loss": 1.203, + "theoretical_loss": 3.400799670825944, + "tokens_seen": 2464874496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025557695393997755, + "loss": 1.2785, + "theoretical_loss": 3.40078532648754, + "tokens_seen": 2465005568 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555368319691863, + "loss": 1.2141, + "theoretical_loss": 3.400770983125401, + "tokens_seen": 2465136640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554967099983951, + "loss": 1.2938, + "theoretical_loss": 3.400756640739409, + "tokens_seen": 2465267712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554565880276039, + "loss": 1.2882, + "theoretical_loss": 3.400742299329445, + "tokens_seen": 2465398784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554164660568127, + "loss": 1.1747, + "theoretical_loss": 3.4007279588953914, + "tokens_seen": 2465529856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025537634408602153, + "loss": 1.285, + "theoretical_loss": 3.4007136194371292, + "tokens_seen": 2465660928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025533622211523034, + "loss": 1.2807, + "theoretical_loss": 3.400699280954541, + "tokens_seen": 2465792000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552961001444391, + "loss": 1.256, + "theoretical_loss": 3.4006849434475077, + "tokens_seen": 2465923072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552559781736479, + "loss": 1.2728, + "theoretical_loss": 3.4006706069159116, + "tokens_seen": 2466054144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552158562028567, + "loss": 1.2612, + "theoretical_loss": 3.400656271359634, + "tokens_seen": 2466185216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025517573423206546, + "loss": 1.3436, + "theoretical_loss": 3.4006419367785576, + "tokens_seen": 2466316288 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551356122612743, + "loss": 1.1914, + "theoretical_loss": 3.4006276031725635, + "tokens_seen": 2466447360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025509549029048307, + "loss": 1.2533, + "theoretical_loss": 3.4006132705415335, + "tokens_seen": 2466578432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002550553683196919, + "loss": 1.1957, + "theoretical_loss": 3.40059893888535, + "tokens_seen": 2466709504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002550152463489007, + "loss": 1.2158, + "theoretical_loss": 3.4005846082038946, + "tokens_seen": 2466840576 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025497512437810943, + "loss": 1.3429, + "theoretical_loss": 3.400570278497049, + "tokens_seen": 2466971648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025493500240731824, + "loss": 1.2702, + "theoretical_loss": 3.4005559497646956, + "tokens_seen": 2467102720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025489488043652705, + "loss": 1.2702, + "theoretical_loss": 3.400541622006716, + "tokens_seen": 2467233792 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025485475846573585, + "loss": 1.2369, + "theoretical_loss": 3.4005272952229926, + "tokens_seen": 2467364864 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548146364949446, + "loss": 1.1686, + "theoretical_loss": 3.4005129694134073, + "tokens_seen": 2467495936 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025477451452415346, + "loss": 1.2919, + "theoretical_loss": 3.4004986445778416, + "tokens_seen": 2467627008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547343925533622, + "loss": 1.2346, + "theoretical_loss": 3.4004843207161777, + "tokens_seen": 2467758080 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254694270582571, + "loss": 1.2583, + "theoretical_loss": 3.4004699978282984, + "tokens_seen": 2467889152 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025465414861177983, + "loss": 1.3457, + "theoretical_loss": 3.400455675914085, + "tokens_seen": 2468020224 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546140266409886, + "loss": 1.2439, + "theoretical_loss": 3.40044135497342, + "tokens_seen": 2468151296 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025457390467019744, + "loss": 1.2498, + "theoretical_loss": 3.400427035006185, + "tokens_seen": 2468282368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545337826994062, + "loss": 1.3029, + "theoretical_loss": 3.400412716012263, + "tokens_seen": 2468413440 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254493660728615, + "loss": 1.2538, + "theoretical_loss": 3.4003983979915358, + "tokens_seen": 2468544512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544535387578238, + "loss": 1.2731, + "theoretical_loss": 3.4003840809438852, + "tokens_seen": 2468675584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025441341678703256, + "loss": 1.3016, + "theoretical_loss": 3.4003697648691937, + "tokens_seen": 2468806656 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025437329481624136, + "loss": 1.2767, + "theoretical_loss": 3.4003554497673436, + "tokens_seen": 2468937728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025433317284545017, + "loss": 1.1954, + "theoretical_loss": 3.400341135638217, + "tokens_seen": 2469068800 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.9927660226821899, + "objective/train/avg_token_score": 0.007233934011310339, + "objective/train/docs_used": 898257, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2373894453048706, + "objective/train/original_loss": 1.2478806972503662, + "objective/train/theoretical_loss": 3.4003268224816963, + "objective/train/tokens_used": 839658976, + "objective/train/unlikelihood_loss": 0.01931128464639187, + "theoretical_loss": 3.4003268224816963, + "tokens_seen": 2469199872 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254293050874659, + "loss": 1.292, + "theoretical_loss": 3.4003268224816963, + "tokens_seen": 2469199872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025425292890386773, + "loss": 1.2106, + "theoretical_loss": 3.4003125102976637, + "tokens_seen": 2469330944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542128069330766, + "loss": 1.2414, + "theoretical_loss": 3.4002981990860017, + "tokens_seen": 2469462016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025417268496228534, + "loss": 1.2947, + "theoretical_loss": 3.4002838888465927, + "tokens_seen": 2469593088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541325629914941, + "loss": 1.3055, + "theoretical_loss": 3.4002695795793185, + "tokens_seen": 2469724160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025409244102070295, + "loss": 1.2049, + "theoretical_loss": 3.4002552712840615, + "tokens_seen": 2469855232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540523190499117, + "loss": 1.2732, + "theoretical_loss": 3.400240963960705, + "tokens_seen": 2469986304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025401219707912057, + "loss": 1.3687, + "theoretical_loss": 3.4002266576091307, + "tokens_seen": 2470117376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539720751083293, + "loss": 1.2501, + "theoretical_loss": 3.400212352229221, + "tokens_seen": 2470248448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002539319531375381, + "loss": 1.391, + "theoretical_loss": 3.400198047820859, + "tokens_seen": 2470379520 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025389183116674693, + "loss": 1.3134, + "theoretical_loss": 3.4001837443839262, + "tokens_seen": 2470510592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025385170919595574, + "loss": 1.288, + "theoretical_loss": 3.4001694419183055, + "tokens_seen": 2470641664 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538115872251645, + "loss": 1.3456, + "theoretical_loss": 3.4001551404238795, + "tokens_seen": 2470772736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537714652543733, + "loss": 1.2592, + "theoretical_loss": 3.400140839900531, + "tokens_seen": 2470903808 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537313432835821, + "loss": 1.3233, + "theoretical_loss": 3.4001265403481424, + "tokens_seen": 2471034880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025369122131279086, + "loss": 1.2144, + "theoretical_loss": 3.4001122417665957, + "tokens_seen": 2471165952 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536510993419997, + "loss": 1.3355, + "theoretical_loss": 3.400097944155774, + "tokens_seen": 2471297024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025361097737120847, + "loss": 1.2264, + "theoretical_loss": 3.4000836475155602, + "tokens_seen": 2471428096 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535708554004173, + "loss": 1.2588, + "theoretical_loss": 3.4000693518458367, + "tokens_seen": 2471559168 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535307334296261, + "loss": 1.2729, + "theoretical_loss": 3.400055057146486, + "tokens_seen": 2471690240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025349061145883483, + "loss": 1.3002, + "theoretical_loss": 3.4000407634173904, + "tokens_seen": 2471821312 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534504894880437, + "loss": 1.2161, + "theoretical_loss": 3.4000264706584336, + "tokens_seen": 2471952384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025341036751725244, + "loss": 1.3401, + "theoretical_loss": 3.4000121788694977, + "tokens_seen": 2472083456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025337024554646125, + "loss": 1.1909, + "theoretical_loss": 3.399997888050465, + "tokens_seen": 2472214528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025333012357567006, + "loss": 1.3618, + "theoretical_loss": 3.3999835982012194, + "tokens_seen": 2472345600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025329000160487886, + "loss": 1.2885, + "theoretical_loss": 3.399969309321643, + "tokens_seen": 2472476672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532498796340876, + "loss": 1.1313, + "theoretical_loss": 3.3999550214116185, + "tokens_seen": 2472607744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532097576632964, + "loss": 1.3023, + "theoretical_loss": 3.399940734471029, + "tokens_seen": 2472738816 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025316963569250523, + "loss": 1.3879, + "theoretical_loss": 3.399926448499757, + "tokens_seen": 2472869888 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253129513721714, + "loss": 1.2621, + "theoretical_loss": 3.3999121634976857, + "tokens_seen": 2473000960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025308939175092284, + "loss": 1.2987, + "theoretical_loss": 3.3998978794646977, + "tokens_seen": 2473132032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530492697801316, + "loss": 1.2977, + "theoretical_loss": 3.3998835964006764, + "tokens_seen": 2473263104 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530091478093404, + "loss": 1.2617, + "theoretical_loss": 3.399869314305504, + "tokens_seen": 2473394176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529690258385492, + "loss": 1.2779, + "theoretical_loss": 3.3998550331790645, + "tokens_seen": 2473525248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025292890386775796, + "loss": 1.2496, + "theoretical_loss": 3.39984075302124, + "tokens_seen": 2473656320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025288878189696676, + "loss": 1.2953, + "theoretical_loss": 3.3998264738319133, + "tokens_seen": 2473787392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025284865992617557, + "loss": 1.2537, + "theoretical_loss": 3.399812195610968, + "tokens_seen": 2473918464 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528085379553844, + "loss": 1.2473, + "theoretical_loss": 3.399797918358287, + "tokens_seen": 2474049536 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527684159845932, + "loss": 1.2765, + "theoretical_loss": 3.399783642073753, + "tokens_seen": 2474180608 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252728294013802, + "loss": 1.2568, + "theoretical_loss": 3.39976936675725, + "tokens_seen": 2474311680 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025268817204301074, + "loss": 1.2433, + "theoretical_loss": 3.3997550924086597, + "tokens_seen": 2474442752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025264805007221955, + "loss": 1.3197, + "theoretical_loss": 3.3997408190278664, + "tokens_seen": 2474573824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025260792810142835, + "loss": 1.2865, + "theoretical_loss": 3.3997265466147524, + "tokens_seen": 2474704896 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525678061306371, + "loss": 1.2182, + "theoretical_loss": 3.3997122751692013, + "tokens_seen": 2474835968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025252768415984597, + "loss": 1.2316, + "theoretical_loss": 3.3996980046910963, + "tokens_seen": 2474967040 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524875621890547, + "loss": 1.2413, + "theoretical_loss": 3.3996837351803206, + "tokens_seen": 2475098112 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524474402182635, + "loss": 1.3193, + "theoretical_loss": 3.399669466636757, + "tokens_seen": 2475229184 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025240731824747233, + "loss": 1.3145, + "theoretical_loss": 3.3996551990602892, + "tokens_seen": 2475360256 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025236719627668114, + "loss": 1.3109, + "theoretical_loss": 3.3996409324508, + "tokens_seen": 2475491328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523270743058899, + "loss": 1.257, + "theoretical_loss": 3.399626666808173, + "tokens_seen": 2475622400 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.9961351156234741, + "objective/train/avg_token_score": 0.003864862257614732, + "objective/train/docs_used": 901124, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3685404062271118, + "objective/train/original_loss": 1.3743438720703125, + "objective/train/theoretical_loss": 3.3996124021322918, + "objective/train/tokens_used": 846212576, + "objective/train/unlikelihood_loss": 0.009362123906612396, + "theoretical_loss": 3.3996124021322918, + "tokens_seen": 2475753472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002522869523350987, + "loss": 1.3371, + "theoretical_loss": 3.3996124021322918, + "tokens_seen": 2475753472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002522468303643075, + "loss": 1.2614, + "theoretical_loss": 3.399598138423039, + "tokens_seen": 2475884544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002522067083935163, + "loss": 1.3488, + "theoretical_loss": 3.399583875680298, + "tokens_seen": 2476015616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002521665864227251, + "loss": 1.2805, + "theoretical_loss": 3.3995696139039526, + "tokens_seen": 2476146688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025212646445193387, + "loss": 1.2755, + "theoretical_loss": 3.399555353093886, + "tokens_seen": 2476277760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025208634248114273, + "loss": 1.3251, + "theoretical_loss": 3.399541093249981, + "tokens_seen": 2476408832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002520462205103515, + "loss": 1.2868, + "theoretical_loss": 3.3995268343721223, + "tokens_seen": 2476539904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025200609853956023, + "loss": 1.2355, + "theoretical_loss": 3.3995125764601926, + "tokens_seen": 2476670976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002519659765687691, + "loss": 1.3056, + "theoretical_loss": 3.399498319514075, + "tokens_seen": 2476802048 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025192585459797784, + "loss": 1.2317, + "theoretical_loss": 3.3994840635336536, + "tokens_seen": 2476933120 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025188573262718665, + "loss": 1.2561, + "theoretical_loss": 3.3994698085188118, + "tokens_seen": 2477064192 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025184561065639546, + "loss": 1.2801, + "theoretical_loss": 3.3994555544694323, + "tokens_seen": 2477195264 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025180548868560426, + "loss": 1.2637, + "theoretical_loss": 3.3994413013853997, + "tokens_seen": 2477326336 + }, + { + "epoch": 0.5, + "learning_rate": 0.000251765366714813, + "loss": 1.3556, + "theoretical_loss": 3.3994270492665972, + "tokens_seen": 2477457408 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517252447440218, + "loss": 1.3408, + "theoretical_loss": 3.399412798112908, + "tokens_seen": 2477588480 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025168512277323063, + "loss": 1.2713, + "theoretical_loss": 3.399398547924216, + "tokens_seen": 2477719552 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516450008024394, + "loss": 1.2209, + "theoretical_loss": 3.399384298700405, + "tokens_seen": 2477850624 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025160487883164824, + "loss": 1.3652, + "theoretical_loss": 3.399370050441358, + "tokens_seen": 2477981696 + }, + { + "epoch": 0.5, + "learning_rate": 0.000251564756860857, + "loss": 1.3103, + "theoretical_loss": 3.3993558031469595, + "tokens_seen": 2478112768 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025152463489006585, + "loss": 1.3058, + "theoretical_loss": 3.3993415568170926, + "tokens_seen": 2478243840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514845129192746, + "loss": 1.2857, + "theoretical_loss": 3.399327311451641, + "tokens_seen": 2478374912 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025144439094848336, + "loss": 1.2897, + "theoretical_loss": 3.3993130670504885, + "tokens_seen": 2478505984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514042689776922, + "loss": 1.3556, + "theoretical_loss": 3.399298823613519, + "tokens_seen": 2478637056 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025136414700690097, + "loss": 1.3036, + "theoretical_loss": 3.3992845811406163, + "tokens_seen": 2478768128 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513240250361098, + "loss": 1.3738, + "theoretical_loss": 3.399270339631664, + "tokens_seen": 2478899200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512839030653186, + "loss": 1.276, + "theoretical_loss": 3.399256099086546, + "tokens_seen": 2479030272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512437810945274, + "loss": 1.2361, + "theoretical_loss": 3.3992418595051457, + "tokens_seen": 2479161344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025120365912373614, + "loss": 1.262, + "theoretical_loss": 3.3992276208873475, + "tokens_seen": 2479292416 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025116353715294495, + "loss": 1.249, + "theoretical_loss": 3.399213383233035, + "tokens_seen": 2479423488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025112341518215375, + "loss": 1.2879, + "theoretical_loss": 3.399199146542092, + "tokens_seen": 2479554560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002510832932113625, + "loss": 1.2227, + "theoretical_loss": 3.3991849108144025, + "tokens_seen": 2479685632 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025104317124057137, + "loss": 1.3308, + "theoretical_loss": 3.3991706760498506, + "tokens_seen": 2479816704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002510030492697801, + "loss": 1.2559, + "theoretical_loss": 3.3991564422483203, + "tokens_seen": 2479947776 + }, + { + "epoch": 0.5, + "learning_rate": 0.000250962927298989, + "loss": 1.3464, + "theoretical_loss": 3.3991422094096944, + "tokens_seen": 2480078848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025092280532819773, + "loss": 1.2692, + "theoretical_loss": 3.3991279775338583, + "tokens_seen": 2480209920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508826833574065, + "loss": 1.3005, + "theoretical_loss": 3.3991137466206958, + "tokens_seen": 2480340992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025084256138661534, + "loss": 1.2359, + "theoretical_loss": 3.3990995166700904, + "tokens_seen": 2480472064 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508024394158241, + "loss": 1.2512, + "theoretical_loss": 3.3990852876819257, + "tokens_seen": 2480603136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507623174450329, + "loss": 1.2445, + "theoretical_loss": 3.399071059656087, + "tokens_seen": 2480734208 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507221954742417, + "loss": 1.277, + "theoretical_loss": 3.3990568325924575, + "tokens_seen": 2480865280 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506820735034505, + "loss": 1.3907, + "theoretical_loss": 3.399042606490922, + "tokens_seen": 2480996352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025064195153265927, + "loss": 1.3095, + "theoretical_loss": 3.3990283813513633, + "tokens_seen": 2481127424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025060182956186813, + "loss": 1.2908, + "theoretical_loss": 3.399014157173667, + "tokens_seen": 2481258496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505617075910769, + "loss": 1.2119, + "theoretical_loss": 3.398999933957717, + "tokens_seen": 2481389568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025052158562028563, + "loss": 1.2879, + "theoretical_loss": 3.3989857117033964, + "tokens_seen": 2481520640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002504814636494945, + "loss": 1.3272, + "theoretical_loss": 3.3989714904105908, + "tokens_seen": 2481651712 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025044134167870324, + "loss": 1.2336, + "theoretical_loss": 3.3989572700791832, + "tokens_seen": 2481782784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025040121970791205, + "loss": 1.3006, + "theoretical_loss": 3.3989430507090588, + "tokens_seen": 2481913856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025036109773712086, + "loss": 1.2875, + "theoretical_loss": 3.398928832300101, + "tokens_seen": 2482044928 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025032097576632966, + "loss": 1.3433, + "theoretical_loss": 3.398914614852195, + "tokens_seen": 2482176000 + }, + { + "epoch": 0.5, + "objective/train/aligned_token_count": 0.9909090399742126, + "objective/train/avg_token_score": 0.009090954437851906, + "objective/train/docs_used": 903113, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3234301805496216, + "objective/train/original_loss": 1.33647620677948, + "objective/train/theoretical_loss": 3.3989003983652246, + "objective/train/tokens_used": 852766176, + "objective/train/unlikelihood_loss": 0.02690979652106762, + "theoretical_loss": 3.3989003983652246, + "tokens_seen": 2482307072 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025028085379553847, + "loss": 1.3103, + "theoretical_loss": 3.3989003983652246, + "tokens_seen": 2482307072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502407318247472, + "loss": 1.2256, + "theoretical_loss": 3.3988861828390737, + "tokens_seen": 2482438144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025020060985395603, + "loss": 1.3429, + "theoretical_loss": 3.3988719682736273, + "tokens_seen": 2482569216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025016048788316483, + "loss": 1.302, + "theoretical_loss": 3.3988577546687697, + "tokens_seen": 2482700288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025012036591237364, + "loss": 1.2308, + "theoretical_loss": 3.3988435420243848, + "tokens_seen": 2482831360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002500802439415824, + "loss": 1.2307, + "theoretical_loss": 3.3988293303403574, + "tokens_seen": 2482962432 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025004012197079125, + "loss": 1.2633, + "theoretical_loss": 3.398815119616572, + "tokens_seen": 2483093504 + }, + { + "epoch": 0.51, + "learning_rate": 0.00025, + "loss": 1.3705, + "theoretical_loss": 3.3988009098529126, + "tokens_seen": 2483224576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002499598780292088, + "loss": 1.3052, + "theoretical_loss": 3.3987867010492643, + "tokens_seen": 2483355648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002499197560584176, + "loss": 1.2575, + "theoretical_loss": 3.398772493205511, + "tokens_seen": 2483486720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498796340876264, + "loss": 1.3616, + "theoretical_loss": 3.398758286321537, + "tokens_seen": 2483617792 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498395121168352, + "loss": 1.2715, + "theoretical_loss": 3.3987440803972278, + "tokens_seen": 2483748864 + }, + { + "epoch": 0.51, + "learning_rate": 0.000249799390146044, + "loss": 1.2228, + "theoretical_loss": 3.3987298754324673, + "tokens_seen": 2483879936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497592681752528, + "loss": 1.3789, + "theoretical_loss": 3.39871567142714, + "tokens_seen": 2484011008 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024971914620446154, + "loss": 1.2604, + "theoretical_loss": 3.3987014683811303, + "tokens_seen": 2484142080 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024967902423367035, + "loss": 1.2209, + "theoretical_loss": 3.3986872662943233, + "tokens_seen": 2484273152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024963890226287915, + "loss": 1.3828, + "theoretical_loss": 3.3986730651666033, + "tokens_seen": 2484404224 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024959878029208796, + "loss": 1.2832, + "theoretical_loss": 3.3986588649978553, + "tokens_seen": 2484535296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024955865832129677, + "loss": 1.3001, + "theoretical_loss": 3.3986446657879634, + "tokens_seen": 2484666368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495185363505055, + "loss": 1.3298, + "theoretical_loss": 3.3986304675368126, + "tokens_seen": 2484797440 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002494784143797143, + "loss": 1.2553, + "theoretical_loss": 3.398616270244288, + "tokens_seen": 2484928512 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024943829240892313, + "loss": 1.2292, + "theoretical_loss": 3.3986020739102734, + "tokens_seen": 2485059584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024939817043813194, + "loss": 1.2555, + "theoretical_loss": 3.398587878534654, + "tokens_seen": 2485190656 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024935804846734074, + "loss": 1.2426, + "theoretical_loss": 3.3985736841173146, + "tokens_seen": 2485321728 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024931792649654955, + "loss": 1.2501, + "theoretical_loss": 3.3985594906581404, + "tokens_seen": 2485452800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492778045257583, + "loss": 1.2353, + "theoretical_loss": 3.3985452981570154, + "tokens_seen": 2485583872 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492376825549671, + "loss": 1.3589, + "theoretical_loss": 3.3985311066138246, + "tokens_seen": 2485714944 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491975605841759, + "loss": 1.2724, + "theoretical_loss": 3.3985169160284534, + "tokens_seen": 2485846016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024915743861338467, + "loss": 1.1866, + "theoretical_loss": 3.3985027264007863, + "tokens_seen": 2485977088 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024911731664259347, + "loss": 1.2946, + "theoretical_loss": 3.3984885377307075, + "tokens_seen": 2486108160 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490771946718023, + "loss": 1.2837, + "theoretical_loss": 3.398474350018103, + "tokens_seen": 2486239232 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490370727010111, + "loss": 1.253, + "theoretical_loss": 3.3984601632628575, + "tokens_seen": 2486370304 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002489969507302199, + "loss": 1.2412, + "theoretical_loss": 3.3984459774648554, + "tokens_seen": 2486501376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024895682875942864, + "loss": 1.2497, + "theoretical_loss": 3.398431792623982, + "tokens_seen": 2486632448 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024891670678863745, + "loss": 1.3535, + "theoretical_loss": 3.3984176087401217, + "tokens_seen": 2486763520 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024887658481784626, + "loss": 1.2407, + "theoretical_loss": 3.398403425813161, + "tokens_seen": 2486894592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024883646284705506, + "loss": 1.3114, + "theoretical_loss": 3.398389243842983, + "tokens_seen": 2487025664 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024879634087626387, + "loss": 1.2998, + "theoretical_loss": 3.398375062829474, + "tokens_seen": 2487156736 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487562189054726, + "loss": 1.2721, + "theoretical_loss": 3.398360882772519, + "tokens_seen": 2487287808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024871609693468143, + "loss": 1.2036, + "theoretical_loss": 3.3983467036720025, + "tokens_seen": 2487418880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024867597496389023, + "loss": 1.2812, + "theoretical_loss": 3.3983325255278096, + "tokens_seen": 2487549952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024863585299309904, + "loss": 1.2502, + "theoretical_loss": 3.398318348339826, + "tokens_seen": 2487681024 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485957310223078, + "loss": 1.314, + "theoretical_loss": 3.398304172107937, + "tokens_seen": 2487812096 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485556090515166, + "loss": 1.3136, + "theoretical_loss": 3.398289996832027, + "tokens_seen": 2487943168 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485154870807254, + "loss": 1.2901, + "theoretical_loss": 3.398275822511981, + "tokens_seen": 2488074240 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002484753651099342, + "loss": 1.2619, + "theoretical_loss": 3.398261649147685, + "tokens_seen": 2488205312 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248435243139143, + "loss": 1.2733, + "theoretical_loss": 3.398247476739024, + "tokens_seen": 2488336384 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483951211683518, + "loss": 1.3242, + "theoretical_loss": 3.398233305285883, + "tokens_seen": 2488467456 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483549991975606, + "loss": 1.3109, + "theoretical_loss": 3.3982191347881474, + "tokens_seen": 2488598528 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483148772267694, + "loss": 1.3331, + "theoretical_loss": 3.3982049652457023, + "tokens_seen": 2488729600 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.9946687817573547, + "objective/train/avg_token_score": 0.005331292282789946, + "objective/train/docs_used": 905609, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.176710844039917, + "objective/train/original_loss": 1.1841609477996826, + "objective/train/theoretical_loss": 3.398190796658433, + "objective/train/tokens_used": 859319776, + "objective/train/unlikelihood_loss": 0.01477224100381136, + "theoretical_loss": 3.398190796658433, + "tokens_seen": 2488860672 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482747552559782, + "loss": 1.2503, + "theoretical_loss": 3.398190796658433, + "tokens_seen": 2488860672 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248234633285187, + "loss": 1.2528, + "theoretical_loss": 3.3981766290262256, + "tokens_seen": 2488991744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024819451131439575, + "loss": 1.3149, + "theoretical_loss": 3.398162462348964, + "tokens_seen": 2489122816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024815438934360455, + "loss": 1.3387, + "theoretical_loss": 3.398148296626535, + "tokens_seen": 2489253888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024811426737281336, + "loss": 1.2792, + "theoretical_loss": 3.398134131858823, + "tokens_seen": 2489384960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024807414540202217, + "loss": 1.33, + "theoretical_loss": 3.3981199680457137, + "tokens_seen": 2489516032 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002480340234312309, + "loss": 1.2617, + "theoretical_loss": 3.3981058051870923, + "tokens_seen": 2489647104 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002479939014604397, + "loss": 1.2393, + "theoretical_loss": 3.3980916432828447, + "tokens_seen": 2489778176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024795377948964853, + "loss": 1.3262, + "theoretical_loss": 3.398077482332856, + "tokens_seen": 2489909248 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024791365751885734, + "loss": 1.3554, + "theoretical_loss": 3.398063322337012, + "tokens_seen": 2490040320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024787353554806614, + "loss": 1.2658, + "theoretical_loss": 3.3980491632951977, + "tokens_seen": 2490171392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024783341357727495, + "loss": 1.2823, + "theoretical_loss": 3.398035005207299, + "tokens_seen": 2490302464 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477932916064837, + "loss": 1.2697, + "theoretical_loss": 3.3980208480732013, + "tokens_seen": 2490433536 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477531696356925, + "loss": 1.3035, + "theoretical_loss": 3.39800669189279, + "tokens_seen": 2490564608 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477130476649013, + "loss": 1.2885, + "theoretical_loss": 3.397992536665951, + "tokens_seen": 2490695680 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476729256941101, + "loss": 1.2824, + "theoretical_loss": 3.3979783823925693, + "tokens_seen": 2490826752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024763280372331887, + "loss": 1.2641, + "theoretical_loss": 3.3979642290725316, + "tokens_seen": 2490957824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475926817525277, + "loss": 1.332, + "theoretical_loss": 3.3979500767057225, + "tokens_seen": 2491088896 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475525597817365, + "loss": 1.2828, + "theoretical_loss": 3.397935925292028, + "tokens_seen": 2491219968 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475124378109453, + "loss": 1.2865, + "theoretical_loss": 3.3979217748313335, + "tokens_seen": 2491351040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024747231584015404, + "loss": 1.2715, + "theoretical_loss": 3.3979076253235254, + "tokens_seen": 2491482112 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024743219386936285, + "loss": 1.3657, + "theoretical_loss": 3.3978934767684885, + "tokens_seen": 2491613184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024739207189857166, + "loss": 1.2792, + "theoretical_loss": 3.3978793291661096, + "tokens_seen": 2491744256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024735194992778046, + "loss": 1.2879, + "theoretical_loss": 3.3978651825162736, + "tokens_seen": 2491875328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024731182795698927, + "loss": 1.2704, + "theoretical_loss": 3.3978510368188664, + "tokens_seen": 2492006400 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472717059861981, + "loss": 1.2119, + "theoretical_loss": 3.397836892073774, + "tokens_seen": 2492137472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024723158401540683, + "loss": 1.3884, + "theoretical_loss": 3.397822748280882, + "tokens_seen": 2492268544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024719146204461563, + "loss": 1.2676, + "theoretical_loss": 3.3978086054400762, + "tokens_seen": 2492399616 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024715134007382444, + "loss": 1.2354, + "theoretical_loss": 3.397794463551243, + "tokens_seen": 2492530688 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024711121810303325, + "loss": 1.2434, + "theoretical_loss": 3.3977803226142673, + "tokens_seen": 2492661760 + }, + { + "epoch": 0.51, + "learning_rate": 0.000247071096132242, + "loss": 1.2376, + "theoretical_loss": 3.397766182629036, + "tokens_seen": 2492792832 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470309741614508, + "loss": 1.3467, + "theoretical_loss": 3.3977520435954345, + "tokens_seen": 2492923904 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002469908521906596, + "loss": 1.2864, + "theoretical_loss": 3.3977379055133485, + "tokens_seen": 2493054976 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002469507302198684, + "loss": 1.3105, + "theoretical_loss": 3.397723768382664, + "tokens_seen": 2493186048 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002469106082490772, + "loss": 1.2764, + "theoretical_loss": 3.397709632203268, + "tokens_seen": 2493317120 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246870486278286, + "loss": 1.238, + "theoretical_loss": 3.397695496975045, + "tokens_seen": 2493448192 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468303643074948, + "loss": 1.2446, + "theoretical_loss": 3.397681362697882, + "tokens_seen": 2493579264 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467902423367036, + "loss": 1.2311, + "theoretical_loss": 3.397667229371664, + "tokens_seen": 2493710336 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467501203659124, + "loss": 1.1743, + "theoretical_loss": 3.3976530969962786, + "tokens_seen": 2493841408 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467099983951212, + "loss": 1.279, + "theoretical_loss": 3.3976389655716104, + "tokens_seen": 2493972480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024666987642432995, + "loss": 1.261, + "theoretical_loss": 3.397624835097546, + "tokens_seen": 2494103552 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024662975445353876, + "loss": 1.2817, + "theoretical_loss": 3.3976107055739724, + "tokens_seen": 2494234624 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024658963248274757, + "loss": 1.298, + "theoretical_loss": 3.3975965770007743, + "tokens_seen": 2494365696 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465495105119563, + "loss": 1.2781, + "theoretical_loss": 3.3975824493778384, + "tokens_seen": 2494496768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465093885411651, + "loss": 1.3048, + "theoretical_loss": 3.397568322705051, + "tokens_seen": 2494627840 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024646926657037393, + "loss": 1.2504, + "theoretical_loss": 3.3975541969822984, + "tokens_seen": 2494758912 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024642914459958274, + "loss": 1.2148, + "theoretical_loss": 3.397540072209466, + "tokens_seen": 2494889984 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024638902262879154, + "loss": 1.3479, + "theoretical_loss": 3.3975259483864413, + "tokens_seen": 2495021056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024634890065800035, + "loss": 1.2659, + "theoretical_loss": 3.397511825513109, + "tokens_seen": 2495152128 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024630877868720916, + "loss": 1.209, + "theoretical_loss": 3.397497703589357, + "tokens_seen": 2495283200 + }, + { + "epoch": 0.51, + "objective/train/aligned_token_count": 0.9934381246566772, + "objective/train/avg_token_score": 0.0065618534572422504, + "objective/train/docs_used": 908214, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2447516918182373, + "objective/train/original_loss": 1.2530643939971924, + "objective/train/theoretical_loss": 3.3974835826150707, + "objective/train/tokens_used": 865873376, + "objective/train/unlikelihood_loss": 0.019958049058914185, + "theoretical_loss": 3.3974835826150707, + "tokens_seen": 2495414272 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462686567164179, + "loss": 1.294, + "theoretical_loss": 3.3974835826150707, + "tokens_seen": 2495414272 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462285347456267, + "loss": 1.2385, + "theoretical_loss": 3.397469462590136, + "tokens_seen": 2495545344 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461884127748355, + "loss": 1.2241, + "theoretical_loss": 3.39745534351444, + "tokens_seen": 2495676416 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461482908040443, + "loss": 1.2778, + "theoretical_loss": 3.397441225387869, + "tokens_seen": 2495807488 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461081688332531, + "loss": 1.3278, + "theoretical_loss": 3.3974271082103087, + "tokens_seen": 2495938560 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460680468624619, + "loss": 1.2825, + "theoretical_loss": 3.397412991981646, + "tokens_seen": 2496069632 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460279248916707, + "loss": 1.3452, + "theoretical_loss": 3.397398876701767, + "tokens_seen": 2496200704 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024598780292087944, + "loss": 1.3578, + "theoretical_loss": 3.3973847623705584, + "tokens_seen": 2496331776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024594768095008825, + "loss": 1.26, + "theoretical_loss": 3.3973706489879065, + "tokens_seen": 2496462848 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024590755897929706, + "loss": 1.2638, + "theoretical_loss": 3.3973565365536977, + "tokens_seen": 2496593920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024586743700850586, + "loss": 1.3235, + "theoretical_loss": 3.3973424250678184, + "tokens_seen": 2496724992 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024582731503771467, + "loss": 1.2488, + "theoretical_loss": 3.3973283145301556, + "tokens_seen": 2496856064 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457871930669235, + "loss": 1.3259, + "theoretical_loss": 3.3973142049405953, + "tokens_seen": 2496987136 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457470710961323, + "loss": 1.2931, + "theoretical_loss": 3.3973000962990243, + "tokens_seen": 2497118208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024570694912534103, + "loss": 1.2102, + "theoretical_loss": 3.397285988605329, + "tokens_seen": 2497249280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024566682715454984, + "loss": 1.3513, + "theoretical_loss": 3.397271881859396, + "tokens_seen": 2497380352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024562670518375865, + "loss": 1.3608, + "theoretical_loss": 3.3972577760611116, + "tokens_seen": 2497511424 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455865832129674, + "loss": 1.3284, + "theoretical_loss": 3.397243671210363, + "tokens_seen": 2497642496 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455464612421762, + "loss": 1.254, + "theoretical_loss": 3.3972295673070363, + "tokens_seen": 2497773568 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245506339271385, + "loss": 1.3064, + "theoretical_loss": 3.3972154643510186, + "tokens_seen": 2497904640 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454662173005938, + "loss": 1.2435, + "theoretical_loss": 3.397201362342196, + "tokens_seen": 2498035712 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454260953298026, + "loss": 1.2872, + "theoretical_loss": 3.3971872612804566, + "tokens_seen": 2498166784 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453859733590114, + "loss": 1.2941, + "theoretical_loss": 3.397173161165685, + "tokens_seen": 2498297856 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453458513882202, + "loss": 1.2399, + "theoretical_loss": 3.3971590619977694, + "tokens_seen": 2498428928 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245305729417429, + "loss": 1.2464, + "theoretical_loss": 3.3971449637765962, + "tokens_seen": 2498560000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452656074466378, + "loss": 1.2631, + "theoretical_loss": 3.397130866502052, + "tokens_seen": 2498691072 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452254854758466, + "loss": 1.2981, + "theoretical_loss": 3.3971167701740237, + "tokens_seen": 2498822144 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451853635050554, + "loss": 1.2278, + "theoretical_loss": 3.397102674792398, + "tokens_seen": 2498953216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024514524153426416, + "loss": 1.331, + "theoretical_loss": 3.3970885803570616, + "tokens_seen": 2499084288 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024510511956347297, + "loss": 1.2689, + "theoretical_loss": 3.397074486867902, + "tokens_seen": 2499215360 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024506499759268177, + "loss": 1.349, + "theoretical_loss": 3.3970603943248054, + "tokens_seen": 2499346432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002450248756218905, + "loss": 1.2712, + "theoretical_loss": 3.397046302727659, + "tokens_seen": 2499477504 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024498475365109933, + "loss": 1.2209, + "theoretical_loss": 3.3970322120763496, + "tokens_seen": 2499608576 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024494463168030814, + "loss": 1.3196, + "theoretical_loss": 3.3970181223707643, + "tokens_seen": 2499739648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024490450970951694, + "loss": 1.277, + "theoretical_loss": 3.3970040336107896, + "tokens_seen": 2499870720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024486438773872575, + "loss": 1.2623, + "theoretical_loss": 3.396989945796313, + "tokens_seen": 2500001792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024482426576793456, + "loss": 1.1996, + "theoretical_loss": 3.3969758589272208, + "tokens_seen": 2500132864 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447841437971433, + "loss": 1.2353, + "theoretical_loss": 3.396961773003401, + "tokens_seen": 2500263936 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447440218263521, + "loss": 1.2629, + "theoretical_loss": 3.3969476880247393, + "tokens_seen": 2500395008 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447038998555609, + "loss": 1.2684, + "theoretical_loss": 3.3969336039911244, + "tokens_seen": 2500526080 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446637778847697, + "loss": 1.1774, + "theoretical_loss": 3.3969195209024416, + "tokens_seen": 2500657152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446236559139785, + "loss": 1.3588, + "theoretical_loss": 3.3969054387585795, + "tokens_seen": 2500788224 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445835339431873, + "loss": 1.1876, + "theoretical_loss": 3.396891357559424, + "tokens_seen": 2500919296 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445434119723961, + "loss": 1.3075, + "theoretical_loss": 3.3968772773048634, + "tokens_seen": 2501050368 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445032900016049, + "loss": 1.2955, + "theoretical_loss": 3.396863197994784, + "tokens_seen": 2501181440 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024446316803081365, + "loss": 1.2405, + "theoretical_loss": 3.3968491196290724, + "tokens_seen": 2501312512 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024442304606002246, + "loss": 1.2767, + "theoretical_loss": 3.3968350422076172, + "tokens_seen": 2501443584 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024438292408923126, + "loss": 1.2686, + "theoretical_loss": 3.3968209657303046, + "tokens_seen": 2501574656 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024434280211844007, + "loss": 1.2582, + "theoretical_loss": 3.396806890197022, + "tokens_seen": 2501705728 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443026801476489, + "loss": 1.306, + "theoretical_loss": 3.396792815607657, + "tokens_seen": 2501836800 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.99200838804245, + "objective/train/avg_token_score": 0.007991586811840534, + "objective/train/docs_used": 910572, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.248425841331482, + "objective/train/original_loss": 1.2603906393051147, + "objective/train/theoretical_loss": 3.3967787419620965, + "objective/train/tokens_used": 872426976, + "objective/train/unlikelihood_loss": 0.02171725407242775, + "theoretical_loss": 3.3967787419620965, + "tokens_seen": 2501967872 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442625581768577, + "loss": 1.2354, + "theoretical_loss": 3.3967787419620965, + "tokens_seen": 2501967872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024422243620606643, + "loss": 1.2254, + "theoretical_loss": 3.3967646692602282, + "tokens_seen": 2502098944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024418231423527524, + "loss": 1.1811, + "theoretical_loss": 3.3967505975019385, + "tokens_seen": 2502230016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024414219226448405, + "loss": 1.2901, + "theoretical_loss": 3.396736526687116, + "tokens_seen": 2502361088 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024410207029369282, + "loss": 1.2425, + "theoretical_loss": 3.396722456815647, + "tokens_seen": 2502492160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024406194832290163, + "loss": 1.2278, + "theoretical_loss": 3.396708387887419, + "tokens_seen": 2502623232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024402182635211044, + "loss": 1.2057, + "theoretical_loss": 3.3966943199023194, + "tokens_seen": 2502754304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024398170438131922, + "loss": 1.2404, + "theoretical_loss": 3.396680252860236, + "tokens_seen": 2502885376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024394158241052802, + "loss": 1.3373, + "theoretical_loss": 3.396666186761056, + "tokens_seen": 2503016448 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002439014604397368, + "loss": 1.2206, + "theoretical_loss": 3.3966521216046663, + "tokens_seen": 2503147520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024386133846894558, + "loss": 1.3247, + "theoretical_loss": 3.3966380573909554, + "tokens_seen": 2503278592 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002438212164981544, + "loss": 1.2823, + "theoretical_loss": 3.3966239941198095, + "tokens_seen": 2503409664 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002437810945273632, + "loss": 1.1801, + "theoretical_loss": 3.3966099317911174, + "tokens_seen": 2503540736 + }, + { + "epoch": 0.52, + "learning_rate": 0.000243740972556572, + "loss": 1.3555, + "theoretical_loss": 3.3965958704047656, + "tokens_seen": 2503671808 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024370085058578078, + "loss": 1.3043, + "theoretical_loss": 3.396581809960642, + "tokens_seen": 2503802880 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024366072861498959, + "loss": 1.1279, + "theoretical_loss": 3.396567750458634, + "tokens_seen": 2503933952 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024362060664419836, + "loss": 1.2881, + "theoretical_loss": 3.3965536918986294, + "tokens_seen": 2504065024 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024358048467340714, + "loss": 1.2348, + "theoretical_loss": 3.396539634280516, + "tokens_seen": 2504196096 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024354036270261595, + "loss": 1.2316, + "theoretical_loss": 3.3965255776041805, + "tokens_seen": 2504327168 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024350024073182476, + "loss": 1.2585, + "theoretical_loss": 3.3965115218695114, + "tokens_seen": 2504458240 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024346011876103356, + "loss": 1.2684, + "theoretical_loss": 3.396497467076396, + "tokens_seen": 2504589312 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024341999679024234, + "loss": 1.2446, + "theoretical_loss": 3.396483413224722, + "tokens_seen": 2504720384 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024337987481945115, + "loss": 1.2513, + "theoretical_loss": 3.3964693603143767, + "tokens_seen": 2504851456 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024333975284865993, + "loss": 1.2215, + "theoretical_loss": 3.3964553083452484, + "tokens_seen": 2504982528 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002432996308778687, + "loss": 1.2193, + "theoretical_loss": 3.3964412573172247, + "tokens_seen": 2505113600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002432595089070775, + "loss": 1.1942, + "theoretical_loss": 3.396427207230193, + "tokens_seen": 2505244672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024321938693628632, + "loss": 1.2879, + "theoretical_loss": 3.3964131580840418, + "tokens_seen": 2505375744 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002431792649654951, + "loss": 1.2269, + "theoretical_loss": 3.396399109878658, + "tokens_seen": 2505506816 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002431391429947039, + "loss": 1.2178, + "theoretical_loss": 3.3963850626139296, + "tokens_seen": 2505637888 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002430990210239127, + "loss": 1.348, + "theoretical_loss": 3.3963710162897445, + "tokens_seen": 2505768960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024305889905312152, + "loss": 1.3075, + "theoretical_loss": 3.3963569709059906, + "tokens_seen": 2505900032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024301877708233027, + "loss": 1.3086, + "theoretical_loss": 3.396342926462556, + "tokens_seen": 2506031104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024297865511153908, + "loss": 1.1753, + "theoretical_loss": 3.3963288829593283, + "tokens_seen": 2506162176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024293853314074788, + "loss": 1.2403, + "theoretical_loss": 3.396314840396195, + "tokens_seen": 2506293248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024289841116995666, + "loss": 1.2805, + "theoretical_loss": 3.3963007987730447, + "tokens_seen": 2506424320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024285828919916547, + "loss": 1.2651, + "theoretical_loss": 3.396286758089765, + "tokens_seen": 2506555392 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024281816722837427, + "loss": 1.2013, + "theoretical_loss": 3.396272718346244, + "tokens_seen": 2506686464 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024277804525758308, + "loss": 1.211, + "theoretical_loss": 3.3962586795423695, + "tokens_seen": 2506817536 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024273792328679183, + "loss": 1.2283, + "theoretical_loss": 3.396244641678029, + "tokens_seen": 2506948608 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024269780131600064, + "loss": 1.2339, + "theoretical_loss": 3.3962306047531117, + "tokens_seen": 2507079680 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024265767934520945, + "loss": 1.2706, + "theoretical_loss": 3.3962165687675046, + "tokens_seen": 2507210752 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024261755737441822, + "loss": 1.2487, + "theoretical_loss": 3.3962025337210964, + "tokens_seen": 2507341824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024257743540362703, + "loss": 1.3306, + "theoretical_loss": 3.396188499613775, + "tokens_seen": 2507472896 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024253731343283584, + "loss": 1.2537, + "theoretical_loss": 3.3961744664454274, + "tokens_seen": 2507603968 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024249719146204464, + "loss": 1.2171, + "theoretical_loss": 3.3961604342159437, + "tokens_seen": 2507735040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024245706949125342, + "loss": 1.2636, + "theoretical_loss": 3.3961464029252104, + "tokens_seen": 2507866112 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002424169475204622, + "loss": 1.3022, + "theoretical_loss": 3.3961323725731165, + "tokens_seen": 2507997184 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242376825549671, + "loss": 1.2479, + "theoretical_loss": 3.3961183431595496, + "tokens_seen": 2508128256 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002423367035788798, + "loss": 1.2303, + "theoretical_loss": 3.3961043146843983, + "tokens_seen": 2508259328 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002422965816080886, + "loss": 1.2951, + "theoretical_loss": 3.3960902871475507, + "tokens_seen": 2508390400 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.9928663372993469, + "objective/train/avg_token_score": 0.007133623585104942, + "objective/train/docs_used": 912960, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.289162039756775, + "objective/train/original_loss": 1.2997078895568848, + "objective/train/theoretical_loss": 3.3960762605488948, + "objective/train/tokens_used": 878980576, + "objective/train/unlikelihood_loss": 0.01906536892056465, + "theoretical_loss": 3.3960762605488948, + "tokens_seen": 2508521472 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002422564596372974, + "loss": 1.2746, + "theoretical_loss": 3.3960762605488948, + "tokens_seen": 2508521472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024221633766650618, + "loss": 1.2419, + "theoretical_loss": 3.396062234888319, + "tokens_seen": 2508652544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024217621569571499, + "loss": 1.242, + "theoretical_loss": 3.396048210165712, + "tokens_seen": 2508783616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024213609372492376, + "loss": 1.1823, + "theoretical_loss": 3.396034186380961, + "tokens_seen": 2508914688 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024209597175413257, + "loss": 1.2558, + "theoretical_loss": 3.3960201635339553, + "tokens_seen": 2509045760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024205584978334135, + "loss": 1.236, + "theoretical_loss": 3.3960061416245826, + "tokens_seen": 2509176832 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024201572781255016, + "loss": 1.2415, + "theoretical_loss": 3.3959921206527315, + "tokens_seen": 2509307904 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024197560584175896, + "loss": 1.3507, + "theoretical_loss": 3.3959781006182905, + "tokens_seen": 2509438976 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024193548387096774, + "loss": 1.2511, + "theoretical_loss": 3.395964081521148, + "tokens_seen": 2509570048 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024189536190017655, + "loss": 1.2281, + "theoretical_loss": 3.3959500633611914, + "tokens_seen": 2509701120 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024185523992938533, + "loss": 1.2727, + "theoretical_loss": 3.3959360461383103, + "tokens_seen": 2509832192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024181511795859413, + "loss": 1.1246, + "theoretical_loss": 3.395922029852393, + "tokens_seen": 2509963264 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002417749959878029, + "loss": 1.2857, + "theoretical_loss": 3.3959080145033274, + "tokens_seen": 2510094336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024173487401701172, + "loss": 1.186, + "theoretical_loss": 3.395894000091002, + "tokens_seen": 2510225408 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024169475204622053, + "loss": 1.2893, + "theoretical_loss": 3.3958799866153058, + "tokens_seen": 2510356480 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416546300754293, + "loss": 1.2036, + "theoretical_loss": 3.3958659740761266, + "tokens_seen": 2510487552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416145081046381, + "loss": 1.2339, + "theoretical_loss": 3.395851962473354, + "tokens_seen": 2510618624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024157438613384692, + "loss": 1.2241, + "theoretical_loss": 3.3958379518068753, + "tokens_seen": 2510749696 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002415342641630557, + "loss": 1.2784, + "theoretical_loss": 3.3958239420765794, + "tokens_seen": 2510880768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024149414219226448, + "loss": 1.2501, + "theoretical_loss": 3.3958099332823557, + "tokens_seen": 2511011840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024145402022147328, + "loss": 1.2603, + "theoretical_loss": 3.395795925424092, + "tokens_seen": 2511142912 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002414138982506821, + "loss": 1.2608, + "theoretical_loss": 3.3957819185016773, + "tokens_seen": 2511273984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024137377627989087, + "loss": 1.3092, + "theoretical_loss": 3.3957679125149998, + "tokens_seen": 2511405056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024133365430909967, + "loss": 1.3152, + "theoretical_loss": 3.3957539074639485, + "tokens_seen": 2511536128 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024129353233830848, + "loss": 1.2582, + "theoretical_loss": 3.395739903348412, + "tokens_seen": 2511667200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024125341036751723, + "loss": 1.283, + "theoretical_loss": 3.395725900168279, + "tokens_seen": 2511798272 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024121328839672604, + "loss": 1.3417, + "theoretical_loss": 3.395711897923438, + "tokens_seen": 2511929344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024117316642593484, + "loss": 1.3314, + "theoretical_loss": 3.395697896613778, + "tokens_seen": 2512060416 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024113304445514365, + "loss": 1.3181, + "theoretical_loss": 3.3956838962391878, + "tokens_seen": 2512191488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024109292248435243, + "loss": 1.1911, + "theoretical_loss": 3.395669896799556, + "tokens_seen": 2512322560 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024105280051356124, + "loss": 1.2216, + "theoretical_loss": 3.3956558982947715, + "tokens_seen": 2512453632 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024101267854277004, + "loss": 1.2473, + "theoretical_loss": 3.395641900724723, + "tokens_seen": 2512584704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002409725565719788, + "loss": 1.2704, + "theoretical_loss": 3.3956279040892996, + "tokens_seen": 2512715776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002409324346011876, + "loss": 1.3699, + "theoretical_loss": 3.3956139083883894, + "tokens_seen": 2512846848 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002408923126303964, + "loss": 1.2599, + "theoretical_loss": 3.395599913621882, + "tokens_seen": 2512977920 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024085219065960521, + "loss": 1.2463, + "theoretical_loss": 3.3955859197896663, + "tokens_seen": 2513108992 + }, + { + "epoch": 0.52, + "learning_rate": 0.000240812068688814, + "loss": 1.3421, + "theoretical_loss": 3.395571926891631, + "tokens_seen": 2513240064 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002407719467180228, + "loss": 1.3254, + "theoretical_loss": 3.395557934927665, + "tokens_seen": 2513371136 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002407318247472316, + "loss": 1.2146, + "theoretical_loss": 3.395543943897657, + "tokens_seen": 2513502208 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024069170277644038, + "loss": 1.277, + "theoretical_loss": 3.395529953801496, + "tokens_seen": 2513633280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024065158080564916, + "loss": 1.1989, + "theoretical_loss": 3.3955159646390713, + "tokens_seen": 2513764352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024061145883485797, + "loss": 1.3202, + "theoretical_loss": 3.395501976410272, + "tokens_seen": 2513895424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024057133686406678, + "loss": 1.301, + "theoretical_loss": 3.3954879891149865, + "tokens_seen": 2514026496 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024053121489327556, + "loss": 1.2632, + "theoretical_loss": 3.3954740027531045, + "tokens_seen": 2514157568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024049109292248436, + "loss": 1.2076, + "theoretical_loss": 3.395460017324515, + "tokens_seen": 2514288640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024045097095169317, + "loss": 1.2799, + "theoretical_loss": 3.3954460328291067, + "tokens_seen": 2514419712 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024041084898090195, + "loss": 1.273, + "theoretical_loss": 3.395432049266769, + "tokens_seen": 2514550784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024037072701011073, + "loss": 1.2714, + "theoretical_loss": 3.3954180666373905, + "tokens_seen": 2514681856 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024033060503931953, + "loss": 1.2234, + "theoretical_loss": 3.3954040849408607, + "tokens_seen": 2514812928 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024029048306852834, + "loss": 1.2629, + "theoretical_loss": 3.395390104177069, + "tokens_seen": 2514944000 + }, + { + "epoch": 0.52, + "objective/train/aligned_token_count": 0.9932106137275696, + "objective/train/avg_token_score": 0.006789429113268852, + "objective/train/docs_used": 915361, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.222291350364685, + "objective/train/original_loss": 1.2305092811584473, + "objective/train/theoretical_loss": 3.395376124345904, + "objective/train/tokens_used": 885534176, + "objective/train/unlikelihood_loss": 0.025381222367286682, + "theoretical_loss": 3.395376124345904, + "tokens_seen": 2515075072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024025036109773712, + "loss": 1.231, + "theoretical_loss": 3.395376124345904, + "tokens_seen": 2515075072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024021023912694593, + "loss": 1.2336, + "theoretical_loss": 3.3953621454472556, + "tokens_seen": 2515206144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024017011715615473, + "loss": 1.2116, + "theoretical_loss": 3.3953481674810124, + "tokens_seen": 2515337216 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002401299951853635, + "loss": 1.2275, + "theoretical_loss": 3.3953341904470635, + "tokens_seen": 2515468288 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024008987321457232, + "loss": 1.2417, + "theoretical_loss": 3.395320214345299, + "tokens_seen": 2515599360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002400497512437811, + "loss": 1.3198, + "theoretical_loss": 3.3953062391756075, + "tokens_seen": 2515730432 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024000962927298988, + "loss": 1.2862, + "theoretical_loss": 3.3952922649378783, + "tokens_seen": 2515861504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00023996950730219868, + "loss": 1.2824, + "theoretical_loss": 3.395278291632001, + "tokens_seen": 2515992576 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002399293853314075, + "loss": 1.2582, + "theoretical_loss": 3.3952643192578646, + "tokens_seen": 2516123648 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002398892633606163, + "loss": 1.2758, + "theoretical_loss": 3.3952503478153586, + "tokens_seen": 2516254720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023984914138982507, + "loss": 1.2698, + "theoretical_loss": 3.3952363773043723, + "tokens_seen": 2516385792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023980901941903388, + "loss": 1.2095, + "theoretical_loss": 3.3952224077247957, + "tokens_seen": 2516516864 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023976889744824266, + "loss": 1.2464, + "theoretical_loss": 3.3952084390765167, + "tokens_seen": 2516647936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023972877547745144, + "loss": 1.2813, + "theoretical_loss": 3.395194471359426, + "tokens_seen": 2516779008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023968865350666024, + "loss": 1.2196, + "theoretical_loss": 3.395180504573413, + "tokens_seen": 2516910080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023964853153586905, + "loss": 1.2663, + "theoretical_loss": 3.3951665387183665, + "tokens_seen": 2517041152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023960840956507786, + "loss": 1.2271, + "theoretical_loss": 3.395152573794176, + "tokens_seen": 2517172224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023956828759428664, + "loss": 1.2565, + "theoretical_loss": 3.395138609800732, + "tokens_seen": 2517303296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023952816562349544, + "loss": 1.2766, + "theoretical_loss": 3.395124646737923, + "tokens_seen": 2517434368 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023948804365270422, + "loss": 1.339, + "theoretical_loss": 3.3951106846056383, + "tokens_seen": 2517565440 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239447921681913, + "loss": 1.3307, + "theoretical_loss": 3.395096723403768, + "tokens_seen": 2517696512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394077997111218, + "loss": 1.2786, + "theoretical_loss": 3.3950827631322023, + "tokens_seen": 2517827584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002393676777403306, + "loss": 1.3011, + "theoretical_loss": 3.3950688037908296, + "tokens_seen": 2517958656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023932755576953942, + "loss": 1.2788, + "theoretical_loss": 3.3950548453795406, + "tokens_seen": 2518089728 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002392874337987482, + "loss": 1.2214, + "theoretical_loss": 3.3950408878982237, + "tokens_seen": 2518220800 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239247311827957, + "loss": 1.3117, + "theoretical_loss": 3.395026931346769, + "tokens_seen": 2518351872 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002392071898571658, + "loss": 1.1526, + "theoretical_loss": 3.3950129757250673, + "tokens_seen": 2518482944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023916706788637456, + "loss": 1.2487, + "theoretical_loss": 3.3949990210330068, + "tokens_seen": 2518614016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023912694591558337, + "loss": 1.2074, + "theoretical_loss": 3.3949850672704773, + "tokens_seen": 2518745088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023908682394479218, + "loss": 1.3027, + "theoretical_loss": 3.3949711144373698, + "tokens_seen": 2518876160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023904670197400096, + "loss": 1.2878, + "theoretical_loss": 3.3949571625335726, + "tokens_seen": 2519007232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023900658000320976, + "loss": 1.2323, + "theoretical_loss": 3.394943211558976, + "tokens_seen": 2519138304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023896645803241857, + "loss": 1.2382, + "theoretical_loss": 3.39492926151347, + "tokens_seen": 2519269376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023892633606162737, + "loss": 1.3271, + "theoretical_loss": 3.394915312396944, + "tokens_seen": 2519400448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023888621409083613, + "loss": 1.2544, + "theoretical_loss": 3.394901364209288, + "tokens_seen": 2519531520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023884609212004493, + "loss": 1.3061, + "theoretical_loss": 3.394887416950392, + "tokens_seen": 2519662592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023880597014925374, + "loss": 1.2795, + "theoretical_loss": 3.3948734706201456, + "tokens_seen": 2519793664 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023876584817846252, + "loss": 1.2663, + "theoretical_loss": 3.394859525218439, + "tokens_seen": 2519924736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023872572620767132, + "loss": 1.1885, + "theoretical_loss": 3.3948455807451614, + "tokens_seen": 2520055808 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023868560423688013, + "loss": 1.2619, + "theoretical_loss": 3.3948316372002036, + "tokens_seen": 2520186880 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023864548226608894, + "loss": 1.2619, + "theoretical_loss": 3.3948176945834545, + "tokens_seen": 2520317952 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386053602952977, + "loss": 1.2509, + "theoretical_loss": 3.3948037528948047, + "tokens_seen": 2520449024 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002385652383245065, + "loss": 1.2861, + "theoretical_loss": 3.3947898121341447, + "tokens_seen": 2520580096 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002385251163537153, + "loss": 1.3219, + "theoretical_loss": 3.3947758723013632, + "tokens_seen": 2520711168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023848499438292408, + "loss": 1.2517, + "theoretical_loss": 3.394761933396351, + "tokens_seen": 2520842240 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002384448724121329, + "loss": 1.2687, + "theoretical_loss": 3.3947479954189976, + "tokens_seen": 2520973312 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002384047504413417, + "loss": 1.2948, + "theoretical_loss": 3.394734058369194, + "tokens_seen": 2521104384 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002383646284705505, + "loss": 1.3043, + "theoretical_loss": 3.3947201222468295, + "tokens_seen": 2521235456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023832450649975928, + "loss": 1.2277, + "theoretical_loss": 3.3947061870517943, + "tokens_seen": 2521366528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023828438452896806, + "loss": 1.3219, + "theoretical_loss": 3.3946922527839782, + "tokens_seen": 2521497600 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.9940690994262695, + "objective/train/avg_token_score": 0.005930920131504536, + "objective/train/docs_used": 917863, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2224217653274536, + "objective/train/original_loss": 1.2297152280807495, + "objective/train/theoretical_loss": 3.394678319443272, + "objective/train/tokens_used": 892087776, + "objective/train/unlikelihood_loss": 0.021188722923398018, + "theoretical_loss": 3.394678319443272, + "tokens_seen": 2521628672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023824426255817686, + "loss": 1.2497, + "theoretical_loss": 3.394678319443272, + "tokens_seen": 2521628672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023820414058738564, + "loss": 1.2889, + "theoretical_loss": 3.3946643870295654, + "tokens_seen": 2521759744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023816401861659445, + "loss": 1.2717, + "theoretical_loss": 3.3946504555427484, + "tokens_seen": 2521890816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023812389664580326, + "loss": 1.2561, + "theoretical_loss": 3.3946365249827117, + "tokens_seen": 2522021888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023808377467501204, + "loss": 1.2711, + "theoretical_loss": 3.394622595349345, + "tokens_seen": 2522152960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023804365270422084, + "loss": 1.3267, + "theoretical_loss": 3.3946086666425384, + "tokens_seen": 2522284032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023800353073342962, + "loss": 1.2323, + "theoretical_loss": 3.3945947388621827, + "tokens_seen": 2522415104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023796340876263843, + "loss": 1.3085, + "theoretical_loss": 3.3945808120081677, + "tokens_seen": 2522546176 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002379232867918472, + "loss": 1.2817, + "theoretical_loss": 3.3945668860803835, + "tokens_seen": 2522677248 + }, + { + "epoch": 0.53, + "learning_rate": 0.000237883164821056, + "loss": 1.3166, + "theoretical_loss": 3.3945529610787215, + "tokens_seen": 2522808320 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023784304285026482, + "loss": 1.2324, + "theoretical_loss": 3.3945390370030704, + "tokens_seen": 2522939392 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378029208794736, + "loss": 1.2921, + "theoretical_loss": 3.3945251138533212, + "tokens_seen": 2523070464 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002377627989086824, + "loss": 1.2278, + "theoretical_loss": 3.394511191629365, + "tokens_seen": 2523201536 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002377226769378912, + "loss": 1.2466, + "theoretical_loss": 3.394497270331091, + "tokens_seen": 2523332608 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002376825549671, + "loss": 1.2605, + "theoretical_loss": 3.39448334995839, + "tokens_seen": 2523463680 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023764243299630877, + "loss": 1.3203, + "theoretical_loss": 3.394469430511153, + "tokens_seen": 2523594752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023760231102551758, + "loss": 1.2842, + "theoretical_loss": 3.394455511989269, + "tokens_seen": 2523725824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023756218905472638, + "loss": 1.2901, + "theoretical_loss": 3.39444159439263, + "tokens_seen": 2523856896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023752206708393516, + "loss": 1.3484, + "theoretical_loss": 3.3944276777211253, + "tokens_seen": 2523987968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023748194511314397, + "loss": 1.2919, + "theoretical_loss": 3.3944137619746457, + "tokens_seen": 2524119040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023744182314235277, + "loss": 1.2575, + "theoretical_loss": 3.394399847153082, + "tokens_seen": 2524250112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023740170117156155, + "loss": 1.3146, + "theoretical_loss": 3.3943859332563244, + "tokens_seen": 2524381184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023736157920077033, + "loss": 1.2924, + "theoretical_loss": 3.3943720202842638, + "tokens_seen": 2524512256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023732145722997914, + "loss": 1.2579, + "theoretical_loss": 3.3943581082367897, + "tokens_seen": 2524643328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023728133525918795, + "loss": 1.417, + "theoretical_loss": 3.394344197113794, + "tokens_seen": 2524774400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023724121328839672, + "loss": 1.2959, + "theoretical_loss": 3.394330286915167, + "tokens_seen": 2524905472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023720109131760553, + "loss": 1.3342, + "theoretical_loss": 3.3943163776407985, + "tokens_seen": 2525036544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023716096934681434, + "loss": 1.2985, + "theoretical_loss": 3.3943024692905794, + "tokens_seen": 2525167616 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002371208473760231, + "loss": 1.3644, + "theoretical_loss": 3.3942885618644008, + "tokens_seen": 2525298688 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002370807254052319, + "loss": 1.2922, + "theoretical_loss": 3.394274655362153, + "tokens_seen": 2525429760 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002370406034344407, + "loss": 1.2828, + "theoretical_loss": 3.3942607497837267, + "tokens_seen": 2525560832 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002370004814636495, + "loss": 1.222, + "theoretical_loss": 3.394246845129013, + "tokens_seen": 2525691904 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002369603594928583, + "loss": 1.329, + "theoretical_loss": 3.3942329413979015, + "tokens_seen": 2525822976 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002369202375220671, + "loss": 1.2896, + "theoretical_loss": 3.3942190385902844, + "tokens_seen": 2525954048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002368801155512759, + "loss": 1.3174, + "theoretical_loss": 3.3942051367060513, + "tokens_seen": 2526085120 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023683999358048468, + "loss": 1.224, + "theoretical_loss": 3.3941912357450934, + "tokens_seen": 2526216192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023679987160969346, + "loss": 1.2777, + "theoretical_loss": 3.3941773357073015, + "tokens_seen": 2526347264 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023675974963890226, + "loss": 1.2461, + "theoretical_loss": 3.3941634365925664, + "tokens_seen": 2526478336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023671962766811107, + "loss": 1.3024, + "theoretical_loss": 3.394149538400779, + "tokens_seen": 2526609408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023667950569731985, + "loss": 1.2771, + "theoretical_loss": 3.3941356411318298, + "tokens_seen": 2526740480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023663938372652866, + "loss": 1.2957, + "theoretical_loss": 3.3941217447856102, + "tokens_seen": 2526871552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023659926175573746, + "loss": 1.2562, + "theoretical_loss": 3.3941078493620105, + "tokens_seen": 2527002624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023655913978494624, + "loss": 1.2712, + "theoretical_loss": 3.394093954860922, + "tokens_seen": 2527133696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023651901781415502, + "loss": 1.3029, + "theoretical_loss": 3.3940800612822355, + "tokens_seen": 2527264768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023647889584336383, + "loss": 1.2242, + "theoretical_loss": 3.3940661686258418, + "tokens_seen": 2527395840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023643877387257263, + "loss": 1.252, + "theoretical_loss": 3.3940522768916317, + "tokens_seen": 2527526912 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002363986519017814, + "loss": 1.2359, + "theoretical_loss": 3.394038386079497, + "tokens_seen": 2527657984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023635852993099022, + "loss": 1.2678, + "theoretical_loss": 3.3940244961893278, + "tokens_seen": 2527789056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023631840796019903, + "loss": 1.2564, + "theoretical_loss": 3.3940106072210154, + "tokens_seen": 2527920128 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002362782859894078, + "loss": 1.2318, + "theoretical_loss": 3.3939967191744507, + "tokens_seen": 2528051200 + }, + { + "epoch": 0.53, + "objective/train/aligned_token_count": 0.9934902787208557, + "objective/train/avg_token_score": 0.00650973804295063, + "objective/train/docs_used": 920165, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1675903797149658, + "objective/train/original_loss": 1.1749894618988037, + "objective/train/theoretical_loss": 3.3939828320495247, + "objective/train/tokens_used": 898641376, + "objective/train/unlikelihood_loss": 0.02220461145043373, + "theoretical_loss": 3.3939828320495247, + "tokens_seen": 2528182272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023623816401861658, + "loss": 1.2237, + "theoretical_loss": 3.3939828320495247, + "tokens_seen": 2528182272 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361980420478254, + "loss": 1.3227, + "theoretical_loss": 3.393968945846129, + "tokens_seen": 2528313344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361579200770342, + "loss": 1.2296, + "theoretical_loss": 3.393955060564154, + "tokens_seen": 2528444416 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023611779810624298, + "loss": 1.2932, + "theoretical_loss": 3.3939411762034917, + "tokens_seen": 2528575488 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023607767613545178, + "loss": 1.2827, + "theoretical_loss": 3.3939272927640323, + "tokens_seen": 2528706560 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002360375541646606, + "loss": 1.2634, + "theoretical_loss": 3.393913410245667, + "tokens_seen": 2528837632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023599743219386937, + "loss": 1.2696, + "theoretical_loss": 3.393899528648287, + "tokens_seen": 2528968704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023595731022307817, + "loss": 1.2311, + "theoretical_loss": 3.3938856479717847, + "tokens_seen": 2529099776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023591718825228695, + "loss": 1.2598, + "theoretical_loss": 3.3938717682160497, + "tokens_seen": 2529230848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023587706628149573, + "loss": 1.3416, + "theoretical_loss": 3.393857889380974, + "tokens_seen": 2529361920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023583694431070454, + "loss": 1.3016, + "theoretical_loss": 3.393844011466448, + "tokens_seen": 2529492992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023579682233991334, + "loss": 1.2878, + "theoretical_loss": 3.3938301344723643, + "tokens_seen": 2529624064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023575670036912215, + "loss": 1.2721, + "theoretical_loss": 3.393816258398613, + "tokens_seen": 2529755136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023571657839833093, + "loss": 1.2471, + "theoretical_loss": 3.3938023832450854, + "tokens_seen": 2529886208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023567645642753974, + "loss": 1.2732, + "theoretical_loss": 3.393788509011674, + "tokens_seen": 2530017280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023563633445674852, + "loss": 1.3028, + "theoretical_loss": 3.393774635698269, + "tokens_seen": 2530148352 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002355962124859573, + "loss": 1.2845, + "theoretical_loss": 3.3937607633047624, + "tokens_seen": 2530279424 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002355560905151661, + "loss": 1.2641, + "theoretical_loss": 3.3937468918310447, + "tokens_seen": 2530410496 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002355159685443749, + "loss": 1.2624, + "theoretical_loss": 3.3937330212770083, + "tokens_seen": 2530541568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023547584657358371, + "loss": 1.2944, + "theoretical_loss": 3.393719151642544, + "tokens_seen": 2530672640 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002354357246027925, + "loss": 1.297, + "theoretical_loss": 3.393705282927543, + "tokens_seen": 2530803712 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002353956026320013, + "loss": 1.2742, + "theoretical_loss": 3.3936914151318973, + "tokens_seen": 2530934784 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002353554806612101, + "loss": 1.3457, + "theoretical_loss": 3.3936775482554977, + "tokens_seen": 2531065856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023531535869041886, + "loss": 1.2637, + "theoretical_loss": 3.3936636822982367, + "tokens_seen": 2531196928 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023527523671962766, + "loss": 1.2356, + "theoretical_loss": 3.3936498172600045, + "tokens_seen": 2531328000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023523511474883647, + "loss": 1.249, + "theoretical_loss": 3.3936359531406937, + "tokens_seen": 2531459072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023519499277804528, + "loss": 1.329, + "theoretical_loss": 3.3936220899401954, + "tokens_seen": 2531590144 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023515487080725406, + "loss": 1.2649, + "theoretical_loss": 3.393608227658401, + "tokens_seen": 2531721216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023511474883646286, + "loss": 1.3369, + "theoretical_loss": 3.393594366295202, + "tokens_seen": 2531852288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023507462686567167, + "loss": 1.3153, + "theoretical_loss": 3.39358050585049, + "tokens_seen": 2531983360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023503450489488042, + "loss": 1.2867, + "theoretical_loss": 3.393566646324157, + "tokens_seen": 2532114432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023499438292408923, + "loss": 1.304, + "theoretical_loss": 3.3935527877160947, + "tokens_seen": 2532245504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023495426095329803, + "loss": 1.3351, + "theoretical_loss": 3.3935389300261942, + "tokens_seen": 2532376576 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002349141389825068, + "loss": 1.226, + "theoretical_loss": 3.393525073254347, + "tokens_seen": 2532507648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023487401701171562, + "loss": 1.2627, + "theoretical_loss": 3.3935112174004454, + "tokens_seen": 2532638720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023483389504092443, + "loss": 1.2519, + "theoretical_loss": 3.3934973624643807, + "tokens_seen": 2532769792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023479377307013323, + "loss": 1.2479, + "theoretical_loss": 3.3934835084460446, + "tokens_seen": 2532900864 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023475365109934198, + "loss": 1.3585, + "theoretical_loss": 3.393469655345329, + "tokens_seen": 2533031936 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002347135291285508, + "loss": 1.2323, + "theoretical_loss": 3.393455803162126, + "tokens_seen": 2533163008 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002346734071577596, + "loss": 1.2144, + "theoretical_loss": 3.393441951896327, + "tokens_seen": 2533294080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023463328518696838, + "loss": 1.2821, + "theoretical_loss": 3.3934281015478227, + "tokens_seen": 2533425152 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023459316321617718, + "loss": 1.274, + "theoretical_loss": 3.3934142521165067, + "tokens_seen": 2533556224 + }, + { + "epoch": 0.54, + "learning_rate": 0.000234553041245386, + "loss": 1.3028, + "theoretical_loss": 3.39340040360227, + "tokens_seen": 2533687296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002345129192745948, + "loss": 1.3214, + "theoretical_loss": 3.3933865560050043, + "tokens_seen": 2533818368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023447279730380357, + "loss": 1.268, + "theoretical_loss": 3.393372709324602, + "tokens_seen": 2533949440 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023443267533301235, + "loss": 1.2657, + "theoretical_loss": 3.3933588635609544, + "tokens_seen": 2534080512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023439255336222116, + "loss": 1.2655, + "theoretical_loss": 3.3933450187139536, + "tokens_seen": 2534211584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023435243139142994, + "loss": 1.2603, + "theoretical_loss": 3.3933311747834916, + "tokens_seen": 2534342656 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023431230942063874, + "loss": 1.303, + "theoretical_loss": 3.3933173317694596, + "tokens_seen": 2534473728 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023427218744984755, + "loss": 1.2457, + "theoretical_loss": 3.3933034896717507, + "tokens_seen": 2534604800 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.9930554628372192, + "objective/train/avg_token_score": 0.006944500375539064, + "objective/train/docs_used": 922493, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.307681679725647, + "objective/train/original_loss": 1.317766785621643, + "objective/train/theoretical_loss": 3.3932896484902564, + "objective/train/tokens_used": 905194976, + "objective/train/unlikelihood_loss": 0.017835600301623344, + "theoretical_loss": 3.3932896484902564, + "tokens_seen": 2534735872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023423206547905636, + "loss": 1.3285, + "theoretical_loss": 3.3932896484902564, + "tokens_seen": 2534735872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023419194350826514, + "loss": 1.2595, + "theoretical_loss": 3.3932758082248684, + "tokens_seen": 2534866944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023415182153747392, + "loss": 1.2388, + "theoretical_loss": 3.3932619688754793, + "tokens_seen": 2534998016 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023411169956668272, + "loss": 1.2931, + "theoretical_loss": 3.3932481304419806, + "tokens_seen": 2535129088 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002340715775958915, + "loss": 1.2426, + "theoretical_loss": 3.393234292924264, + "tokens_seen": 2535260160 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002340314556251003, + "loss": 1.2907, + "theoretical_loss": 3.3932204563222226, + "tokens_seen": 2535391232 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002339913336543091, + "loss": 1.2452, + "theoretical_loss": 3.393206620635748, + "tokens_seen": 2535522304 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002339512116835179, + "loss": 1.2776, + "theoretical_loss": 3.3931927858647315, + "tokens_seen": 2535653376 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002339110897127267, + "loss": 1.1912, + "theoretical_loss": 3.3931789520090665, + "tokens_seen": 2535784448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023387096774193548, + "loss": 1.2309, + "theoretical_loss": 3.3931651190686445, + "tokens_seen": 2535915520 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023383084577114428, + "loss": 1.264, + "theoretical_loss": 3.3931512870433576, + "tokens_seen": 2536046592 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023379072380035306, + "loss": 1.2624, + "theoretical_loss": 3.393137455933098, + "tokens_seen": 2536177664 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023375060182956187, + "loss": 1.3379, + "theoretical_loss": 3.3931236257377577, + "tokens_seen": 2536308736 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023371047985877068, + "loss": 1.2541, + "theoretical_loss": 3.3931097964572294, + "tokens_seen": 2536439808 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023367035788797946, + "loss": 1.2767, + "theoretical_loss": 3.393095968091405, + "tokens_seen": 2536570880 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023363023591718826, + "loss": 1.1884, + "theoretical_loss": 3.393082140640177, + "tokens_seen": 2536701952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023359011394639707, + "loss": 1.2729, + "theoretical_loss": 3.3930683141034375, + "tokens_seen": 2536833024 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023354999197560585, + "loss": 1.2891, + "theoretical_loss": 3.3930544884810785, + "tokens_seen": 2536964096 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023350987000481463, + "loss": 1.186, + "theoretical_loss": 3.393040663772992, + "tokens_seen": 2537095168 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023346974803402343, + "loss": 1.2474, + "theoretical_loss": 3.393026839979072, + "tokens_seen": 2537226240 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023342962606323224, + "loss": 1.2258, + "theoretical_loss": 3.393013017099209, + "tokens_seen": 2537357312 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023338950409244102, + "loss": 1.2746, + "theoretical_loss": 3.392999195133296, + "tokens_seen": 2537488384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023334938212164982, + "loss": 1.2001, + "theoretical_loss": 3.3929853740812255, + "tokens_seen": 2537619456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023330926015085863, + "loss": 1.293, + "theoretical_loss": 3.39297155394289, + "tokens_seen": 2537750528 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002332691381800674, + "loss": 1.3349, + "theoretical_loss": 3.392957734718181, + "tokens_seen": 2537881600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002332290162092762, + "loss": 1.2407, + "theoretical_loss": 3.392943916406992, + "tokens_seen": 2538012672 + }, + { + "epoch": 0.54, + "learning_rate": 0.000233188894238485, + "loss": 1.2094, + "theoretical_loss": 3.392930099009215, + "tokens_seen": 2538143744 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002331487722676938, + "loss": 1.2566, + "theoretical_loss": 3.3929162825247423, + "tokens_seen": 2538274816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023310865029690258, + "loss": 1.222, + "theoretical_loss": 3.392902466953467, + "tokens_seen": 2538405888 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002330685283261114, + "loss": 1.2004, + "theoretical_loss": 3.392888652295281, + "tokens_seen": 2538536960 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002330284063553202, + "loss": 1.1771, + "theoretical_loss": 3.3928748385500764, + "tokens_seen": 2538668032 + }, + { + "epoch": 0.54, + "learning_rate": 0.000232988284384529, + "loss": 1.2336, + "theoretical_loss": 3.392861025717747, + "tokens_seen": 2538799104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023294816241373775, + "loss": 1.2602, + "theoretical_loss": 3.392847213798184, + "tokens_seen": 2538930176 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023290804044294656, + "loss": 1.2659, + "theoretical_loss": 3.3928334027912816, + "tokens_seen": 2539061248 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023286791847215536, + "loss": 1.2969, + "theoretical_loss": 3.3928195926969305, + "tokens_seen": 2539192320 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023282779650136414, + "loss": 1.3498, + "theoretical_loss": 3.3928057835150245, + "tokens_seen": 2539323392 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023278767453057295, + "loss": 1.2573, + "theoretical_loss": 3.392791975245456, + "tokens_seen": 2539454464 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023274755255978176, + "loss": 1.2611, + "theoretical_loss": 3.3927781678881175, + "tokens_seen": 2539585536 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023270743058899054, + "loss": 1.1665, + "theoretical_loss": 3.3927643614429015, + "tokens_seen": 2539716608 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023266730861819932, + "loss": 1.2552, + "theoretical_loss": 3.392750555909701, + "tokens_seen": 2539847680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023262718664740812, + "loss": 1.3272, + "theoretical_loss": 3.3927367512884086, + "tokens_seen": 2539978752 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023258706467661693, + "loss": 1.2084, + "theoretical_loss": 3.392722947578917, + "tokens_seen": 2540109824 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002325469427058257, + "loss": 1.3618, + "theoretical_loss": 3.392709144781119, + "tokens_seen": 2540240896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002325068207350345, + "loss": 1.2686, + "theoretical_loss": 3.3926953428949074, + "tokens_seen": 2540371968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023246669876424332, + "loss": 1.3291, + "theoretical_loss": 3.3926815419201746, + "tokens_seen": 2540503040 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002324265767934521, + "loss": 1.2533, + "theoretical_loss": 3.3926677418568136, + "tokens_seen": 2540634112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023238645482266088, + "loss": 1.2239, + "theoretical_loss": 3.3926539427047175, + "tokens_seen": 2540765184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023234633285186968, + "loss": 1.221, + "theoretical_loss": 3.3926401444637784, + "tokens_seen": 2540896256 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002323062108810785, + "loss": 1.2474, + "theoretical_loss": 3.39262634713389, + "tokens_seen": 2541027328 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023226608891028727, + "loss": 1.3052, + "theoretical_loss": 3.3926125507149445, + "tokens_seen": 2541158400 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.9934854507446289, + "objective/train/avg_token_score": 0.006514542270451784, + "objective/train/docs_used": 924855, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2180243730545044, + "objective/train/original_loss": 1.2264975309371948, + "objective/train/theoretical_loss": 3.3925987552068353, + "objective/train/tokens_used": 911748576, + "objective/train/unlikelihood_loss": 0.02003411203622818, + "theoretical_loss": 3.3925987552068353, + "tokens_seen": 2541289472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023222596693949608, + "loss": 1.2916, + "theoretical_loss": 3.3925987552068353, + "tokens_seen": 2541289472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023218584496870488, + "loss": 1.2916, + "theoretical_loss": 3.392584960609455, + "tokens_seen": 2541420544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023214572299791366, + "loss": 1.1943, + "theoretical_loss": 3.3925711669226963, + "tokens_seen": 2541551616 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023210560102712247, + "loss": 1.3402, + "theoretical_loss": 3.392557374146452, + "tokens_seen": 2541682688 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023206547905633125, + "loss": 1.2751, + "theoretical_loss": 3.392543582280616, + "tokens_seen": 2541813760 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023202535708554003, + "loss": 1.2883, + "theoretical_loss": 3.3925297913250807, + "tokens_seen": 2541944832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023198523511474883, + "loss": 1.2647, + "theoretical_loss": 3.3925160012797386, + "tokens_seen": 2542075904 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023194511314395764, + "loss": 1.2104, + "theoretical_loss": 3.392502212144483, + "tokens_seen": 2542206976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023190499117316645, + "loss": 1.3162, + "theoretical_loss": 3.392488423919208, + "tokens_seen": 2542338048 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023186486920237522, + "loss": 1.2886, + "theoretical_loss": 3.392474636603805, + "tokens_seen": 2542469120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023182474723158403, + "loss": 1.2786, + "theoretical_loss": 3.392460850198168, + "tokens_seen": 2542600192 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002317846252607928, + "loss": 1.2949, + "theoretical_loss": 3.3924470647021896, + "tokens_seen": 2542731264 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002317445032900016, + "loss": 1.3054, + "theoretical_loss": 3.3924332801157635, + "tokens_seen": 2542862336 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002317043813192104, + "loss": 1.3197, + "theoretical_loss": 3.392419496438782, + "tokens_seen": 2542993408 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002316642593484192, + "loss": 1.2485, + "theoretical_loss": 3.392405713671139, + "tokens_seen": 2543124480 + }, + { + "epoch": 0.54, + "learning_rate": 0.000231624137377628, + "loss": 1.2046, + "theoretical_loss": 3.3923919318127274, + "tokens_seen": 2543255552 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002315840154068368, + "loss": 1.2578, + "theoretical_loss": 3.3923781508634403, + "tokens_seen": 2543386624 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002315438934360456, + "loss": 1.2921, + "theoretical_loss": 3.3923643708231705, + "tokens_seen": 2543517696 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023150377146525437, + "loss": 1.3018, + "theoretical_loss": 3.3923505916918115, + "tokens_seen": 2543648768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023146364949446315, + "loss": 1.3032, + "theoretical_loss": 3.392336813469257, + "tokens_seen": 2543779840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023142352752367196, + "loss": 1.2716, + "theoretical_loss": 3.3923230361553998, + "tokens_seen": 2543910912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023138340555288076, + "loss": 1.2777, + "theoretical_loss": 3.392309259750133, + "tokens_seen": 2544041984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023134328358208957, + "loss": 1.2638, + "theoretical_loss": 3.39229548425335, + "tokens_seen": 2544173056 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023130316161129835, + "loss": 1.256, + "theoretical_loss": 3.3922817096649442, + "tokens_seen": 2544304128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023126303964050716, + "loss": 1.2799, + "theoretical_loss": 3.392267935984809, + "tokens_seen": 2544435200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023122291766971596, + "loss": 1.2959, + "theoretical_loss": 3.3922541632128373, + "tokens_seen": 2544566272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023118279569892471, + "loss": 1.2345, + "theoretical_loss": 3.392240391348923, + "tokens_seen": 2544697344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023114267372813352, + "loss": 1.2364, + "theoretical_loss": 3.392226620392959, + "tokens_seen": 2544828416 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023110255175734233, + "loss": 1.3121, + "theoretical_loss": 3.392212850344839, + "tokens_seen": 2544959488 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023106242978655113, + "loss": 1.2231, + "theoretical_loss": 3.3921990812044562, + "tokens_seen": 2545090560 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002310223078157599, + "loss": 1.1722, + "theoretical_loss": 3.392185312971704, + "tokens_seen": 2545221632 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023098218584496872, + "loss": 1.1645, + "theoretical_loss": 3.3921715456464763, + "tokens_seen": 2545352704 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023094206387417753, + "loss": 1.258, + "theoretical_loss": 3.392157779228666, + "tokens_seen": 2545483776 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023090194190338628, + "loss": 1.2952, + "theoretical_loss": 3.3921440137181667, + "tokens_seen": 2545614848 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023086181993259508, + "loss": 1.2115, + "theoretical_loss": 3.3921302491148717, + "tokens_seen": 2545745920 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002308216979618039, + "loss": 1.2426, + "theoretical_loss": 3.392116485418675, + "tokens_seen": 2545876992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023078157599101267, + "loss": 1.2339, + "theoretical_loss": 3.3921027226294695, + "tokens_seen": 2546008064 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023074145402022148, + "loss": 1.2546, + "theoretical_loss": 3.3920889607471496, + "tokens_seen": 2546139136 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023070133204943028, + "loss": 1.308, + "theoretical_loss": 3.3920751997716083, + "tokens_seen": 2546270208 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002306612100786391, + "loss": 1.2106, + "theoretical_loss": 3.392061439702739, + "tokens_seen": 2546401280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023062108810784784, + "loss": 1.2612, + "theoretical_loss": 3.392047680540436, + "tokens_seen": 2546532352 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023058096613705665, + "loss": 1.3294, + "theoretical_loss": 3.392033922284592, + "tokens_seen": 2546663424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023054084416626545, + "loss": 1.2589, + "theoretical_loss": 3.392020164935101, + "tokens_seen": 2546794496 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023050072219547423, + "loss": 1.1677, + "theoretical_loss": 3.392006408491857, + "tokens_seen": 2546925568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023046060022468304, + "loss": 1.2918, + "theoretical_loss": 3.3919926529547535, + "tokens_seen": 2547056640 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023042047825389184, + "loss": 1.2787, + "theoretical_loss": 3.391978898323684, + "tokens_seen": 2547187712 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023038035628310065, + "loss": 1.2691, + "theoretical_loss": 3.391965144598542, + "tokens_seen": 2547318784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023034023431230943, + "loss": 1.2492, + "theoretical_loss": 3.391951391779222, + "tokens_seen": 2547449856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002303001123415182, + "loss": 1.2935, + "theoretical_loss": 3.391937639865617, + "tokens_seen": 2547580928 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023025999037072702, + "loss": 1.19, + "theoretical_loss": 3.391923888857621, + "tokens_seen": 2547712000 + }, + { + "epoch": 0.54, + "objective/train/aligned_token_count": 0.9907525777816772, + "objective/train/avg_token_score": 0.009247391484677792, + "objective/train/docs_used": 927107, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1759027242660522, + "objective/train/original_loss": 1.187464714050293, + "objective/train/theoretical_loss": 3.391910138755128, + "objective/train/tokens_used": 918302176, + "objective/train/unlikelihood_loss": 0.029312141239643097, + "theoretical_loss": 3.391910138755128, + "tokens_seen": 2547843072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002302198683999358, + "loss": 1.2031, + "theoretical_loss": 3.391910138755128, + "tokens_seen": 2547843072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002301797464291446, + "loss": 1.2535, + "theoretical_loss": 3.391896389558031, + "tokens_seen": 2547974144 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002301396244583534, + "loss": 1.1711, + "theoretical_loss": 3.3918826412662253, + "tokens_seen": 2548105216 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023009950248756221, + "loss": 1.2487, + "theoretical_loss": 3.3918688938796033, + "tokens_seen": 2548236288 + }, + { + "epoch": 0.54, + "learning_rate": 0.000230059380516771, + "loss": 1.2451, + "theoretical_loss": 3.3918551473980596, + "tokens_seen": 2548367360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023001925854597977, + "loss": 1.2543, + "theoretical_loss": 3.391841401821488, + "tokens_seen": 2548498432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022997913657518858, + "loss": 1.3165, + "theoretical_loss": 3.391827657149782, + "tokens_seen": 2548629504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022993901460439736, + "loss": 1.2618, + "theoretical_loss": 3.3918139133828356, + "tokens_seen": 2548760576 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022989889263360616, + "loss": 1.3137, + "theoretical_loss": 3.391800170520543, + "tokens_seen": 2548891648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022985877066281497, + "loss": 1.2448, + "theoretical_loss": 3.391786428562798, + "tokens_seen": 2549022720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022981864869202375, + "loss": 1.1983, + "theoretical_loss": 3.3917726875094947, + "tokens_seen": 2549153792 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022977852672123256, + "loss": 1.2321, + "theoretical_loss": 3.3917589473605267, + "tokens_seen": 2549284864 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022973840475044136, + "loss": 1.2881, + "theoretical_loss": 3.3917452081157884, + "tokens_seen": 2549415936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022969828277965014, + "loss": 1.2502, + "theoretical_loss": 3.3917314697751735, + "tokens_seen": 2549547008 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022965816080885892, + "loss": 1.228, + "theoretical_loss": 3.391717732338576, + "tokens_seen": 2549678080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022961803883806773, + "loss": 1.2371, + "theoretical_loss": 3.3917039958058903, + "tokens_seen": 2549809152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022957791686727653, + "loss": 1.2548, + "theoretical_loss": 3.3916902601770103, + "tokens_seen": 2549940224 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002295377948964853, + "loss": 1.237, + "theoretical_loss": 3.39167652545183, + "tokens_seen": 2550071296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022949767292569412, + "loss": 1.2875, + "theoretical_loss": 3.3916627916302433, + "tokens_seen": 2550202368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022945755095490293, + "loss": 1.2923, + "theoretical_loss": 3.391649058712145, + "tokens_seen": 2550333440 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002294174289841117, + "loss": 1.2367, + "theoretical_loss": 3.3916353266974286, + "tokens_seen": 2550464512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022937730701332048, + "loss": 1.2057, + "theoretical_loss": 3.3916215955859883, + "tokens_seen": 2550595584 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002293371850425293, + "loss": 1.2792, + "theoretical_loss": 3.3916078653777184, + "tokens_seen": 2550726656 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002292970630717381, + "loss": 1.1708, + "theoretical_loss": 3.3915941360725133, + "tokens_seen": 2550857728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022925694110094688, + "loss": 1.2717, + "theoretical_loss": 3.3915804076702667, + "tokens_seen": 2550988800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022921681913015568, + "loss": 1.2713, + "theoretical_loss": 3.3915666801708735, + "tokens_seen": 2551119872 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002291766971593645, + "loss": 1.271, + "theoretical_loss": 3.3915529535742275, + "tokens_seen": 2551250944 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022913657518857327, + "loss": 1.2312, + "theoretical_loss": 3.391539227880223, + "tokens_seen": 2551382016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022909645321778205, + "loss": 1.2473, + "theoretical_loss": 3.3915255030887543, + "tokens_seen": 2551513088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022905633124699085, + "loss": 1.2668, + "theoretical_loss": 3.3915117791997154, + "tokens_seen": 2551644160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022901620927619966, + "loss": 1.2612, + "theoretical_loss": 3.3914980562130013, + "tokens_seen": 2551775232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022897608730540844, + "loss": 1.2255, + "theoretical_loss": 3.3914843341285055, + "tokens_seen": 2551906304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022893596533461724, + "loss": 1.1624, + "theoretical_loss": 3.391470612946123, + "tokens_seen": 2552037376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022889584336382605, + "loss": 1.1557, + "theoretical_loss": 3.3914568926657482, + "tokens_seen": 2552168448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022885572139303486, + "loss": 1.2472, + "theoretical_loss": 3.3914431732872754, + "tokens_seen": 2552299520 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002288155994222436, + "loss": 1.2565, + "theoretical_loss": 3.3914294548105985, + "tokens_seen": 2552430592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022877547745145242, + "loss": 1.2545, + "theoretical_loss": 3.3914157372356124, + "tokens_seen": 2552561664 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022873535548066122, + "loss": 1.2516, + "theoretical_loss": 3.391402020562211, + "tokens_seen": 2552692736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022869523350987, + "loss": 1.2401, + "theoretical_loss": 3.3913883047902895, + "tokens_seen": 2552823808 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002286551115390788, + "loss": 1.2468, + "theoretical_loss": 3.3913745899197423, + "tokens_seen": 2552954880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022861498956828761, + "loss": 1.1818, + "theoretical_loss": 3.391360875950463, + "tokens_seen": 2553085952 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002285748675974964, + "loss": 1.1957, + "theoretical_loss": 3.391347162882347, + "tokens_seen": 2553217024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022853474562670517, + "loss": 1.1868, + "theoretical_loss": 3.3913334507152886, + "tokens_seen": 2553348096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022849462365591398, + "loss": 1.2107, + "theoretical_loss": 3.391319739449182, + "tokens_seen": 2553479168 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022845450168512278, + "loss": 1.1689, + "theoretical_loss": 3.3913060290839225, + "tokens_seen": 2553610240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022841437971433156, + "loss": 1.2791, + "theoretical_loss": 3.391292319619404, + "tokens_seen": 2553741312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022837425774354037, + "loss": 1.2424, + "theoretical_loss": 3.391278611055521, + "tokens_seen": 2553872384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022833413577274918, + "loss": 1.2294, + "theoretical_loss": 3.391264903392169, + "tokens_seen": 2554003456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022829401380195796, + "loss": 1.2854, + "theoretical_loss": 3.3912511966292422, + "tokens_seen": 2554134528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022825389183116673, + "loss": 1.2198, + "theoretical_loss": 3.3912374907666347, + "tokens_seen": 2554265600 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.9925299882888794, + "objective/train/avg_token_score": 0.007469974923878908, + "objective/train/docs_used": 929369, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2671695947647095, + "objective/train/original_loss": 1.278935432434082, + "objective/train/theoretical_loss": 3.3912237858042413, + "objective/train/tokens_used": 924855776, + "objective/train/unlikelihood_loss": 0.020693901926279068, + "theoretical_loss": 3.3912237858042413, + "tokens_seen": 2554396672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022821376986037554, + "loss": 1.2115, + "theoretical_loss": 3.3912237858042413, + "tokens_seen": 2554396672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022817364788958435, + "loss": 1.2255, + "theoretical_loss": 3.3912100817419577, + "tokens_seen": 2554527744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022813352591879313, + "loss": 1.2507, + "theoretical_loss": 3.3911963785796777, + "tokens_seen": 2554658816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022809340394800193, + "loss": 1.2394, + "theoretical_loss": 3.391182676317296, + "tokens_seen": 2554789888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022805328197721074, + "loss": 1.1905, + "theoretical_loss": 3.3911689749547076, + "tokens_seen": 2554920960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022801316000641952, + "loss": 1.2972, + "theoretical_loss": 3.391155274491807, + "tokens_seen": 2555052032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022797303803562832, + "loss": 1.2357, + "theoretical_loss": 3.3911415749284894, + "tokens_seen": 2555183104 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002279329160648371, + "loss": 1.234, + "theoretical_loss": 3.3911278762646493, + "tokens_seen": 2555314176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022789279409404588, + "loss": 1.2398, + "theoretical_loss": 3.3911141785001817, + "tokens_seen": 2555445248 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002278526721232547, + "loss": 1.2126, + "theoretical_loss": 3.391100481634981, + "tokens_seen": 2555576320 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002278125501524635, + "loss": 1.2515, + "theoretical_loss": 3.391086785668943, + "tokens_seen": 2555707392 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002277724281816723, + "loss": 1.2544, + "theoretical_loss": 3.391073090601961, + "tokens_seen": 2555838464 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022773230621088108, + "loss": 1.2919, + "theoretical_loss": 3.3910593964339313, + "tokens_seen": 2555969536 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002276921842400899, + "loss": 1.2286, + "theoretical_loss": 3.3910457031647483, + "tokens_seen": 2556100608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022765206226929867, + "loss": 1.2278, + "theoretical_loss": 3.391032010794307, + "tokens_seen": 2556231680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022761194029850745, + "loss": 1.2329, + "theoretical_loss": 3.391018319322502, + "tokens_seen": 2556362752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022757181832771625, + "loss": 1.2654, + "theoretical_loss": 3.391004628749229, + "tokens_seen": 2556493824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022753169635692506, + "loss": 1.2928, + "theoretical_loss": 3.3909909390743818, + "tokens_seen": 2556624896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022749157438613386, + "loss": 1.266, + "theoretical_loss": 3.390977250297856, + "tokens_seen": 2556755968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022745145241534264, + "loss": 1.2576, + "theoretical_loss": 3.390963562419547, + "tokens_seen": 2556887040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022741133044455145, + "loss": 1.2147, + "theoretical_loss": 3.3909498754393494, + "tokens_seen": 2557018112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022737120847376026, + "loss": 1.257, + "theoretical_loss": 3.3909361893571583, + "tokens_seen": 2557149184 + }, + { + "epoch": 0.55, + "learning_rate": 0.000227331086502969, + "loss": 1.2658, + "theoretical_loss": 3.3909225041728686, + "tokens_seen": 2557280256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022729096453217782, + "loss": 1.3581, + "theoretical_loss": 3.3909088198863757, + "tokens_seen": 2557411328 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022725084256138662, + "loss": 1.2466, + "theoretical_loss": 3.3908951364975746, + "tokens_seen": 2557542400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022721072059059543, + "loss": 1.1823, + "theoretical_loss": 3.3908814540063603, + "tokens_seen": 2557673472 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002271705986198042, + "loss": 1.2426, + "theoretical_loss": 3.390867772412628, + "tokens_seen": 2557804544 + }, + { + "epoch": 0.55, + "learning_rate": 0.000227130476649013, + "loss": 1.1752, + "theoretical_loss": 3.390854091716273, + "tokens_seen": 2557935616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022709035467822182, + "loss": 1.2397, + "theoretical_loss": 3.3908404119171895, + "tokens_seen": 2558066688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022705023270743057, + "loss": 1.2947, + "theoretical_loss": 3.3908267330152744, + "tokens_seen": 2558197760 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022701011073663938, + "loss": 1.1656, + "theoretical_loss": 3.390813055010421, + "tokens_seen": 2558328832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022696998876584818, + "loss": 1.2681, + "theoretical_loss": 3.3907993779025265, + "tokens_seen": 2558459904 + }, + { + "epoch": 0.55, + "learning_rate": 0.000226929866795057, + "loss": 1.2625, + "theoretical_loss": 3.3907857016914846, + "tokens_seen": 2558590976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022688974482426577, + "loss": 1.215, + "theoretical_loss": 3.390772026377191, + "tokens_seen": 2558722048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022684962285347458, + "loss": 1.2422, + "theoretical_loss": 3.390758351959541, + "tokens_seen": 2558853120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022680950088268338, + "loss": 1.2472, + "theoretical_loss": 3.39074467843843, + "tokens_seen": 2558984192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022676937891189213, + "loss": 1.1851, + "theoretical_loss": 3.390731005813753, + "tokens_seen": 2559115264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022672925694110094, + "loss": 1.2666, + "theoretical_loss": 3.390717334085406, + "tokens_seen": 2559246336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022668913497030975, + "loss": 1.2573, + "theoretical_loss": 3.3907036632532837, + "tokens_seen": 2559377408 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022664901299951853, + "loss": 1.2142, + "theoretical_loss": 3.3906899933172814, + "tokens_seen": 2559508480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022660889102872733, + "loss": 1.2204, + "theoretical_loss": 3.390676324277295, + "tokens_seen": 2559639552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022656876905793614, + "loss": 1.2201, + "theoretical_loss": 3.390662656133219, + "tokens_seen": 2559770624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022652864708714495, + "loss": 1.211, + "theoretical_loss": 3.3906489888849496, + "tokens_seen": 2559901696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022648852511635372, + "loss": 1.2319, + "theoretical_loss": 3.3906353225323826, + "tokens_seen": 2560032768 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002264484031455625, + "loss": 1.2557, + "theoretical_loss": 3.390621657075412, + "tokens_seen": 2560163840 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002264082811747713, + "loss": 1.2278, + "theoretical_loss": 3.390607992513935, + "tokens_seen": 2560294912 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002263681592039801, + "loss": 1.25, + "theoretical_loss": 3.390594328847846, + "tokens_seen": 2560425984 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002263280372331889, + "loss": 1.2432, + "theoretical_loss": 3.3905806660770406, + "tokens_seen": 2560557056 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002262879152623977, + "loss": 1.2582, + "theoretical_loss": 3.390567004201414, + "tokens_seen": 2560688128 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002262477932916065, + "loss": 1.2305, + "theoretical_loss": 3.390553343220863, + "tokens_seen": 2560819200 + }, + { + "epoch": 0.55, + "objective/train/aligned_token_count": 0.9932734966278076, + "objective/train/avg_token_score": 0.0067265271209180355, + "objective/train/docs_used": 931622, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.066168189048767, + "objective/train/original_loss": 1.0750670433044434, + "objective/train/theoretical_loss": 3.3905396831352816, + "objective/train/tokens_used": 931409376, + "objective/train/unlikelihood_loss": 0.02512192167341709, + "theoretical_loss": 3.3905396831352816, + "tokens_seen": 2560950272 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002262076713208153, + "loss": 1.1538, + "theoretical_loss": 3.3905396831352816, + "tokens_seen": 2560950272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022616754935002407, + "loss": 1.2331, + "theoretical_loss": 3.3905260239445663, + "tokens_seen": 2561081344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022612742737923287, + "loss": 1.1643, + "theoretical_loss": 3.3905123656486125, + "tokens_seen": 2561212416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022608730540844165, + "loss": 1.2517, + "theoretical_loss": 3.390498708247316, + "tokens_seen": 2561343488 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022604718343765046, + "loss": 1.2865, + "theoretical_loss": 3.390485051740572, + "tokens_seen": 2561474560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022600706146685926, + "loss": 1.2954, + "theoretical_loss": 3.3904713961282757, + "tokens_seen": 2561605632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022596693949606807, + "loss": 1.1579, + "theoretical_loss": 3.390457741410324, + "tokens_seen": 2561736704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022592681752527685, + "loss": 1.2156, + "theoretical_loss": 3.390444087586612, + "tokens_seen": 2561867776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022588669555448563, + "loss": 1.271, + "theoretical_loss": 3.390430434657035, + "tokens_seen": 2561998848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022584657358369444, + "loss": 1.2461, + "theoretical_loss": 3.3904167826214895, + "tokens_seen": 2562129920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022580645161290321, + "loss": 1.1783, + "theoretical_loss": 3.3904031314798706, + "tokens_seen": 2562260992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022576632964211202, + "loss": 1.246, + "theoretical_loss": 3.3903894812320736, + "tokens_seen": 2562392064 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022572620767132083, + "loss": 1.2528, + "theoretical_loss": 3.3903758318779955, + "tokens_seen": 2562523136 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002256860857005296, + "loss": 1.3318, + "theoretical_loss": 3.3903621834175315, + "tokens_seen": 2562654208 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002256459637297384, + "loss": 1.2132, + "theoretical_loss": 3.3903485358505767, + "tokens_seen": 2562785280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022560584175894722, + "loss": 1.1761, + "theoretical_loss": 3.390334889177028, + "tokens_seen": 2562916352 + }, + { + "epoch": 0.55, + "learning_rate": 0.000225565719788156, + "loss": 1.2823, + "theoretical_loss": 3.390321243396781, + "tokens_seen": 2563047424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022552559781736478, + "loss": 1.3461, + "theoretical_loss": 3.390307598509731, + "tokens_seen": 2563178496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022548547584657358, + "loss": 1.2983, + "theoretical_loss": 3.390293954515774, + "tokens_seen": 2563309568 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002254453538757824, + "loss": 1.2567, + "theoretical_loss": 3.3902803114148066, + "tokens_seen": 2563440640 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022540523190499117, + "loss": 1.2858, + "theoretical_loss": 3.390266669206724, + "tokens_seen": 2563571712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022536510993419998, + "loss": 1.2916, + "theoretical_loss": 3.390253027891422, + "tokens_seen": 2563702784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022532498796340878, + "loss": 1.2372, + "theoretical_loss": 3.390239387468797, + "tokens_seen": 2563833856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022528486599261756, + "loss": 1.2573, + "theoretical_loss": 3.3902257479387448, + "tokens_seen": 2563964928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022524474402182634, + "loss": 1.2114, + "theoretical_loss": 3.3902121093011615, + "tokens_seen": 2564096000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022520462205103515, + "loss": 1.26, + "theoretical_loss": 3.3901984715559426, + "tokens_seen": 2564227072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022516450008024395, + "loss": 1.2637, + "theoretical_loss": 3.3901848347029846, + "tokens_seen": 2564358144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022512437810945273, + "loss": 1.367, + "theoretical_loss": 3.390171198742183, + "tokens_seen": 2564489216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022508425613866154, + "loss": 1.2315, + "theoretical_loss": 3.390157563673435, + "tokens_seen": 2564620288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022504413416787034, + "loss": 1.2721, + "theoretical_loss": 3.3901439294966353, + "tokens_seen": 2564751360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022500401219707915, + "loss": 1.1875, + "theoretical_loss": 3.3901302962116806, + "tokens_seen": 2564882432 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002249638902262879, + "loss": 1.2917, + "theoretical_loss": 3.3901166638184668, + "tokens_seen": 2565013504 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002249237682554967, + "loss": 1.3625, + "theoretical_loss": 3.3901030323168904, + "tokens_seen": 2565144576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022488364628470552, + "loss": 1.268, + "theoretical_loss": 3.390089401706847, + "tokens_seen": 2565275648 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002248435243139143, + "loss": 1.2949, + "theoretical_loss": 3.390075771988233, + "tokens_seen": 2565406720 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002248034023431231, + "loss": 1.2898, + "theoretical_loss": 3.390062143160945, + "tokens_seen": 2565537792 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002247632803723319, + "loss": 1.3074, + "theoretical_loss": 3.390048515224878, + "tokens_seen": 2565668864 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022472315840154071, + "loss": 1.2522, + "theoretical_loss": 3.390034888179929, + "tokens_seen": 2565799936 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022468303643074947, + "loss": 1.1632, + "theoretical_loss": 3.3900212620259946, + "tokens_seen": 2565931008 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022464291445995827, + "loss": 1.2606, + "theoretical_loss": 3.3900076367629706, + "tokens_seen": 2566062080 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022460279248916708, + "loss": 1.2102, + "theoretical_loss": 3.389994012390753, + "tokens_seen": 2566193152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022456267051837586, + "loss": 1.2877, + "theoretical_loss": 3.389980388909238, + "tokens_seen": 2566324224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022452254854758466, + "loss": 1.2756, + "theoretical_loss": 3.389966766318323, + "tokens_seen": 2566455296 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022448242657679347, + "loss": 1.2136, + "theoretical_loss": 3.3899531446179028, + "tokens_seen": 2566586368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022444230460600225, + "loss": 1.2748, + "theoretical_loss": 3.3899395238078744, + "tokens_seen": 2566717440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022440218263521103, + "loss": 1.2975, + "theoretical_loss": 3.389925903888134, + "tokens_seen": 2566848512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022436206066441984, + "loss": 1.3345, + "theoretical_loss": 3.3899122848585783, + "tokens_seen": 2566979584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022432193869362864, + "loss": 1.2775, + "theoretical_loss": 3.3898986667191036, + "tokens_seen": 2567110656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022428181672283742, + "loss": 1.2999, + "theoretical_loss": 3.389885049469606, + "tokens_seen": 2567241728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022424169475204623, + "loss": 1.2425, + "theoretical_loss": 3.389871433109982, + "tokens_seen": 2567372800 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.990434467792511, + "objective/train/avg_token_score": 0.009565500542521477, + "objective/train/docs_used": 933971, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.209800124168396, + "objective/train/original_loss": 1.223205804824829, + "objective/train/theoretical_loss": 3.389857817640128, + "objective/train/tokens_used": 937962976, + "objective/train/unlikelihood_loss": 0.025624198839068413, + "theoretical_loss": 3.389857817640128, + "tokens_seen": 2567503872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022420157278125503, + "loss": 1.2148, + "theoretical_loss": 3.389857817640128, + "tokens_seen": 2567503872 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002241614508104638, + "loss": 1.3634, + "theoretical_loss": 3.3898442030599405, + "tokens_seen": 2567634944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022412132883967262, + "loss": 1.3023, + "theoretical_loss": 3.389830589369316, + "tokens_seen": 2567766016 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002240812068688814, + "loss": 1.3357, + "theoretical_loss": 3.389816976568151, + "tokens_seen": 2567897088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002240410848980902, + "loss": 1.217, + "theoretical_loss": 3.3898033646563417, + "tokens_seen": 2568028160 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022400096292729898, + "loss": 1.2601, + "theoretical_loss": 3.389789753633785, + "tokens_seen": 2568159232 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002239608409565078, + "loss": 1.2912, + "theoretical_loss": 3.3897761435003773, + "tokens_seen": 2568290304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002239207189857166, + "loss": 1.3349, + "theoretical_loss": 3.3897625342560147, + "tokens_seen": 2568421376 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022388059701492538, + "loss": 1.2941, + "theoretical_loss": 3.3897489259005944, + "tokens_seen": 2568552448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022384047504413418, + "loss": 1.3293, + "theoretical_loss": 3.389735318434013, + "tokens_seen": 2568683520 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022380035307334296, + "loss": 1.2626, + "theoretical_loss": 3.389721711856166, + "tokens_seen": 2568814592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022376023110255174, + "loss": 1.2865, + "theoretical_loss": 3.3897081061669514, + "tokens_seen": 2568945664 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022372010913176055, + "loss": 1.3066, + "theoretical_loss": 3.3896945013662654, + "tokens_seen": 2569076736 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022367998716096935, + "loss": 1.2942, + "theoretical_loss": 3.3896808974540042, + "tokens_seen": 2569207808 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022363986519017816, + "loss": 1.231, + "theoretical_loss": 3.389667294430065, + "tokens_seen": 2569338880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022359974321938694, + "loss": 1.2117, + "theoretical_loss": 3.389653692294344, + "tokens_seen": 2569469952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022355962124859574, + "loss": 1.2953, + "theoretical_loss": 3.3896400910467386, + "tokens_seen": 2569601024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022351949927780452, + "loss": 1.2892, + "theoretical_loss": 3.389626490687145, + "tokens_seen": 2569732096 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002234793773070133, + "loss": 1.3907, + "theoretical_loss": 3.3896128912154593, + "tokens_seen": 2569863168 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002234392553362221, + "loss": 1.2989, + "theoretical_loss": 3.38959929263158, + "tokens_seen": 2569994240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022339913336543092, + "loss": 1.3088, + "theoretical_loss": 3.389585694935402, + "tokens_seen": 2570125312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022335901139463972, + "loss": 1.2576, + "theoretical_loss": 3.389572098126823, + "tokens_seen": 2570256384 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002233188894238485, + "loss": 1.3242, + "theoretical_loss": 3.3895585022057397, + "tokens_seen": 2570387456 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002232787674530573, + "loss": 1.3202, + "theoretical_loss": 3.3895449071720494, + "tokens_seen": 2570518528 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022323864548226611, + "loss": 1.2685, + "theoretical_loss": 3.389531313025648, + "tokens_seen": 2570649600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022319852351147487, + "loss": 1.2317, + "theoretical_loss": 3.389517719766433, + "tokens_seen": 2570780672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022315840154068367, + "loss": 1.4207, + "theoretical_loss": 3.389504127394301, + "tokens_seen": 2570911744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022311827956989248, + "loss": 1.2553, + "theoretical_loss": 3.389490535909149, + "tokens_seen": 2571042816 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022307815759910128, + "loss": 1.2803, + "theoretical_loss": 3.3894769453108737, + "tokens_seen": 2571173888 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022303803562831006, + "loss": 1.2258, + "theoretical_loss": 3.3894633555993723, + "tokens_seen": 2571304960 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022299791365751887, + "loss": 1.3182, + "theoretical_loss": 3.3894497667745416, + "tokens_seen": 2571436032 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022295779168672768, + "loss": 1.317, + "theoretical_loss": 3.389436178836279, + "tokens_seen": 2571567104 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022291766971593643, + "loss": 1.3378, + "theoretical_loss": 3.3894225917844807, + "tokens_seen": 2571698176 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022287754774514523, + "loss": 1.2661, + "theoretical_loss": 3.389409005619044, + "tokens_seen": 2571829248 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022283742577435404, + "loss": 1.339, + "theoretical_loss": 3.3893954203398664, + "tokens_seen": 2571960320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022279730380356285, + "loss": 1.3223, + "theoretical_loss": 3.3893818359468444, + "tokens_seen": 2572091392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022275718183277163, + "loss": 1.2739, + "theoretical_loss": 3.3893682524398745, + "tokens_seen": 2572222464 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022271705986198043, + "loss": 1.2822, + "theoretical_loss": 3.389354669818855, + "tokens_seen": 2572353536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022267693789118924, + "loss": 1.3064, + "theoretical_loss": 3.3893410880836825, + "tokens_seen": 2572484608 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022263681592039802, + "loss": 1.3063, + "theoretical_loss": 3.3893275072342535, + "tokens_seen": 2572615680 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002225966939496068, + "loss": 1.2969, + "theoretical_loss": 3.3893139272704658, + "tokens_seen": 2572746752 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002225565719788156, + "loss": 1.2712, + "theoretical_loss": 3.3893003481922164, + "tokens_seen": 2572877824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022251645000802438, + "loss": 1.2182, + "theoretical_loss": 3.3892867699994023, + "tokens_seen": 2573008896 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002224763280372332, + "loss": 1.3012, + "theoretical_loss": 3.3892731926919204, + "tokens_seen": 2573139968 + }, + { + "epoch": 0.56, + "learning_rate": 0.000222436206066442, + "loss": 1.3093, + "theoretical_loss": 3.3892596162696687, + "tokens_seen": 2573271040 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002223960840956508, + "loss": 1.2914, + "theoretical_loss": 3.3892460407325435, + "tokens_seen": 2573402112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022235596212485958, + "loss": 1.2713, + "theoretical_loss": 3.389232466080443, + "tokens_seen": 2573533184 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022231584015406836, + "loss": 1.2771, + "theoretical_loss": 3.3892188923132633, + "tokens_seen": 2573664256 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022227571818327717, + "loss": 1.3003, + "theoretical_loss": 3.3892053194309026, + "tokens_seen": 2573795328 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022223559621248595, + "loss": 1.3434, + "theoretical_loss": 3.389191747433258, + "tokens_seen": 2573926400 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.9935245513916016, + "objective/train/avg_token_score": 0.006475494243204594, + "objective/train/docs_used": 936262, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2404253482818604, + "objective/train/original_loss": 1.2486048936843872, + "objective/train/theoretical_loss": 3.3891781763202258, + "objective/train/tokens_used": 944516576, + "objective/train/unlikelihood_loss": 0.018979711458086967, + "theoretical_loss": 3.3891781763202258, + "tokens_seen": 2574057472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022219547424169475, + "loss": 1.2492, + "theoretical_loss": 3.3891781763202258, + "tokens_seen": 2574057472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022215535227090356, + "loss": 1.242, + "theoretical_loss": 3.3891646060917044, + "tokens_seen": 2574188544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022211523030011236, + "loss": 1.3526, + "theoretical_loss": 3.3891510367475908, + "tokens_seen": 2574319616 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022207510832932114, + "loss": 1.2544, + "theoretical_loss": 3.389137468287782, + "tokens_seen": 2574450688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022203498635852992, + "loss": 1.2445, + "theoretical_loss": 3.389123900712176, + "tokens_seen": 2574581760 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022199486438773873, + "loss": 1.2375, + "theoretical_loss": 3.38911033402067, + "tokens_seen": 2574712832 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002219547424169475, + "loss": 1.3518, + "theoretical_loss": 3.389096768213161, + "tokens_seen": 2574843904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022191462044615632, + "loss": 1.2795, + "theoretical_loss": 3.3890832032895464, + "tokens_seen": 2574974976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022187449847536512, + "loss": 1.3072, + "theoretical_loss": 3.3890696392497244, + "tokens_seen": 2575106048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022183437650457393, + "loss": 1.2206, + "theoretical_loss": 3.3890560760935915, + "tokens_seen": 2575237120 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002217942545337827, + "loss": 1.2581, + "theoretical_loss": 3.3890425138210456, + "tokens_seen": 2575368192 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002217541325629915, + "loss": 1.3305, + "theoretical_loss": 3.3890289524319837, + "tokens_seen": 2575499264 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002217140105922003, + "loss": 1.3355, + "theoretical_loss": 3.3890153919263044, + "tokens_seen": 2575630336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022167388862140907, + "loss": 1.2612, + "theoretical_loss": 3.389001832303904, + "tokens_seen": 2575761408 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022163376665061788, + "loss": 1.2916, + "theoretical_loss": 3.3889882735646806, + "tokens_seen": 2575892480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022159364467982668, + "loss": 1.2315, + "theoretical_loss": 3.388974715708532, + "tokens_seen": 2576023552 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022155352270903546, + "loss": 1.2648, + "theoretical_loss": 3.3889611587353556, + "tokens_seen": 2576154624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022151340073824427, + "loss": 1.2775, + "theoretical_loss": 3.388947602645048, + "tokens_seen": 2576285696 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022147327876745308, + "loss": 1.2866, + "theoretical_loss": 3.388934047437508, + "tokens_seen": 2576416768 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022143315679666186, + "loss": 1.2893, + "theoretical_loss": 3.388920493112633, + "tokens_seen": 2576547840 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022139303482587063, + "loss": 1.2787, + "theoretical_loss": 3.3889069396703206, + "tokens_seen": 2576678912 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022135291285507944, + "loss": 1.2966, + "theoretical_loss": 3.3888933871104676, + "tokens_seen": 2576809984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022131279088428825, + "loss": 1.3154, + "theoretical_loss": 3.3888798354329728, + "tokens_seen": 2576941056 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022127266891349703, + "loss": 1.3018, + "theoretical_loss": 3.388866284637733, + "tokens_seen": 2577072128 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022123254694270583, + "loss": 1.273, + "theoretical_loss": 3.388852734724647, + "tokens_seen": 2577203200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022119242497191464, + "loss": 1.3144, + "theoretical_loss": 3.3888391856936115, + "tokens_seen": 2577334272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022115230300112342, + "loss": 1.1411, + "theoretical_loss": 3.3888256375445245, + "tokens_seen": 2577465344 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002211121810303322, + "loss": 1.212, + "theoretical_loss": 3.3888120902772836, + "tokens_seen": 2577596416 + }, + { + "epoch": 0.56, + "learning_rate": 0.000221072059059541, + "loss": 1.2484, + "theoretical_loss": 3.388798543891787, + "tokens_seen": 2577727488 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002210319370887498, + "loss": 1.3324, + "theoretical_loss": 3.388784998387932, + "tokens_seen": 2577858560 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002209918151179586, + "loss": 1.2938, + "theoretical_loss": 3.388771453765617, + "tokens_seen": 2577989632 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002209516931471674, + "loss": 1.3031, + "theoretical_loss": 3.388757910024739, + "tokens_seen": 2578120704 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002209115711763762, + "loss": 1.3661, + "theoretical_loss": 3.388744367165197, + "tokens_seen": 2578251776 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220871449205585, + "loss": 1.3356, + "theoretical_loss": 3.388730825186887, + "tokens_seen": 2578382848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022083132723479376, + "loss": 1.3242, + "theoretical_loss": 3.388717284089709, + "tokens_seen": 2578513920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022079120526400257, + "loss": 1.2711, + "theoretical_loss": 3.3887037438735597, + "tokens_seen": 2578644992 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022075108329321137, + "loss": 1.2952, + "theoretical_loss": 3.388690204538337, + "tokens_seen": 2578776064 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022071096132242015, + "loss": 1.254, + "theoretical_loss": 3.3886766660839385, + "tokens_seen": 2578907136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022067083935162896, + "loss": 1.2297, + "theoretical_loss": 3.388663128510263, + "tokens_seen": 2579038208 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022063071738083776, + "loss": 1.2382, + "theoretical_loss": 3.3886495918172086, + "tokens_seen": 2579169280 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022059059541004654, + "loss": 1.2943, + "theoretical_loss": 3.3886360560046724, + "tokens_seen": 2579300352 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022055047343925532, + "loss": 1.2716, + "theoretical_loss": 3.3886225210725525, + "tokens_seen": 2579431424 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022051035146846413, + "loss": 1.318, + "theoretical_loss": 3.3886089870207474, + "tokens_seen": 2579562496 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022047022949767294, + "loss": 1.2581, + "theoretical_loss": 3.3885954538491543, + "tokens_seen": 2579693568 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022043010752688171, + "loss": 1.223, + "theoretical_loss": 3.3885819215576722, + "tokens_seen": 2579824640 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022038998555609052, + "loss": 1.2051, + "theoretical_loss": 3.388568390146199, + "tokens_seen": 2579955712 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022034986358529933, + "loss": 1.1698, + "theoretical_loss": 3.388554859614632, + "tokens_seen": 2580086784 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002203097416145081, + "loss": 1.2893, + "theoretical_loss": 3.3885413299628704, + "tokens_seen": 2580217856 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002202696196437169, + "loss": 1.2065, + "theoretical_loss": 3.3885278011908113, + "tokens_seen": 2580348928 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002202294976729257, + "loss": 1.3736, + "theoretical_loss": 3.388514273298353, + "tokens_seen": 2580480000 + }, + { + "epoch": 0.56, + "objective/train/aligned_token_count": 0.9910797476768494, + "objective/train/avg_token_score": 0.008920260705053806, + "objective/train/docs_used": 938550, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.191901683807373, + "objective/train/original_loss": 1.2042245864868164, + "objective/train/theoretical_loss": 3.388500746285394, + "objective/train/tokens_used": 951070176, + "objective/train/unlikelihood_loss": 0.024832339957356453, + "theoretical_loss": 3.388500746285394, + "tokens_seen": 2580611072 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002201893757021345, + "loss": 1.2049, + "theoretical_loss": 3.388500746285394, + "tokens_seen": 2580611072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022014925373134328, + "loss": 1.2838, + "theoretical_loss": 3.3884872201518323, + "tokens_seen": 2580742144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022010913176055208, + "loss": 1.274, + "theoretical_loss": 3.3884736948975664, + "tokens_seen": 2580873216 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002200690097897609, + "loss": 1.2778, + "theoretical_loss": 3.3884601705224937, + "tokens_seen": 2581004288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022002888781896967, + "loss": 1.2672, + "theoretical_loss": 3.3884466470265133, + "tokens_seen": 2581135360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021998876584817848, + "loss": 1.2719, + "theoretical_loss": 3.3884331244095227, + "tokens_seen": 2581266432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021994864387738725, + "loss": 1.3199, + "theoretical_loss": 3.38841960267142, + "tokens_seen": 2581397504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021990852190659606, + "loss": 1.1846, + "theoretical_loss": 3.388406081812105, + "tokens_seen": 2581528576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021986839993580484, + "loss": 1.3187, + "theoretical_loss": 3.3883925618314743, + "tokens_seen": 2581659648 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021982827796501365, + "loss": 1.3773, + "theoretical_loss": 3.3883790427294267, + "tokens_seen": 2581790720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021978815599422245, + "loss": 1.3153, + "theoretical_loss": 3.3883655245058604, + "tokens_seen": 2581921792 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021974803402343123, + "loss": 1.3126, + "theoretical_loss": 3.388352007160674, + "tokens_seen": 2582052864 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021970791205264004, + "loss": 1.2646, + "theoretical_loss": 3.3883384906937657, + "tokens_seen": 2582183936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021966779008184882, + "loss": 1.2931, + "theoretical_loss": 3.388324975105034, + "tokens_seen": 2582315008 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002196276681110576, + "loss": 1.2287, + "theoretical_loss": 3.3883114603943767, + "tokens_seen": 2582446080 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002195875461402664, + "loss": 1.3179, + "theoretical_loss": 3.388297946561693, + "tokens_seen": 2582577152 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002195474241694752, + "loss": 1.2518, + "theoretical_loss": 3.388284433606881, + "tokens_seen": 2582708224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021950730219868402, + "loss": 1.2685, + "theoretical_loss": 3.388270921529839, + "tokens_seen": 2582839296 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194671802278928, + "loss": 1.1893, + "theoretical_loss": 3.3882574103304655, + "tokens_seen": 2582970368 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194270582571016, + "loss": 1.212, + "theoretical_loss": 3.388243900008659, + "tokens_seen": 2583101440 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002193869362863104, + "loss": 1.2604, + "theoretical_loss": 3.388230390564318, + "tokens_seen": 2583232512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021934681431551916, + "loss": 1.2313, + "theoretical_loss": 3.388216881997341, + "tokens_seen": 2583363584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021930669234472797, + "loss": 1.2445, + "theoretical_loss": 3.388203374307626, + "tokens_seen": 2583494656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021926657037393677, + "loss": 1.2435, + "theoretical_loss": 3.3881898674950723, + "tokens_seen": 2583625728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021922644840314558, + "loss": 1.2619, + "theoretical_loss": 3.3881763615595784, + "tokens_seen": 2583756800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021918632643235436, + "loss": 1.2204, + "theoretical_loss": 3.388162856501042, + "tokens_seen": 2583887872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021914620446156316, + "loss": 1.327, + "theoretical_loss": 3.3881493523193624, + "tokens_seen": 2584018944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021910608249077197, + "loss": 1.2731, + "theoretical_loss": 3.3881358490144384, + "tokens_seen": 2584150016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021906596051998072, + "loss": 1.2576, + "theoretical_loss": 3.3881223465861683, + "tokens_seen": 2584281088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021902583854918953, + "loss": 1.2806, + "theoretical_loss": 3.3881088450344503, + "tokens_seen": 2584412160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021898571657839834, + "loss": 1.2861, + "theoretical_loss": 3.3880953443591837, + "tokens_seen": 2584543232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021894559460760714, + "loss": 1.2035, + "theoretical_loss": 3.3880818445602667, + "tokens_seen": 2584674304 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021890547263681592, + "loss": 1.1997, + "theoretical_loss": 3.388068345637598, + "tokens_seen": 2584805376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021886535066602473, + "loss": 1.3373, + "theoretical_loss": 3.388054847591077, + "tokens_seen": 2584936448 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021882522869523353, + "loss": 1.1955, + "theoretical_loss": 3.3880413504206017, + "tokens_seen": 2585067520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021878510672444229, + "loss": 1.2636, + "theoretical_loss": 3.388027854126071, + "tokens_seen": 2585198592 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002187449847536511, + "loss": 1.2893, + "theoretical_loss": 3.3880143587073834, + "tokens_seen": 2585329664 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002187048627828599, + "loss": 1.3388, + "theoretical_loss": 3.3880008641644386, + "tokens_seen": 2585460736 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186647408120687, + "loss": 1.3102, + "theoretical_loss": 3.3879873704971337, + "tokens_seen": 2585591808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021862461884127748, + "loss": 1.2449, + "theoretical_loss": 3.3879738777053694, + "tokens_seen": 2585722880 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002185844968704863, + "loss": 1.2334, + "theoretical_loss": 3.387960385789043, + "tokens_seen": 2585853952 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002185443748996951, + "loss": 1.2203, + "theoretical_loss": 3.387946894748054, + "tokens_seen": 2585985024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021850425292890388, + "loss": 1.2757, + "theoretical_loss": 3.3879334045823017, + "tokens_seen": 2586116096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021846413095811265, + "loss": 1.2422, + "theoretical_loss": 3.387919915291684, + "tokens_seen": 2586247168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021842400898732146, + "loss": 1.2629, + "theoretical_loss": 3.3879064268761003, + "tokens_seen": 2586378240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021838388701653024, + "loss": 1.1979, + "theoretical_loss": 3.3878929393354493, + "tokens_seen": 2586509312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021834376504573905, + "loss": 1.2605, + "theoretical_loss": 3.3878794526696305, + "tokens_seen": 2586640384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021830364307494785, + "loss": 1.3968, + "theoretical_loss": 3.387865966878542, + "tokens_seen": 2586771456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021826352110415666, + "loss": 1.2588, + "theoretical_loss": 3.387852481962083, + "tokens_seen": 2586902528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021822339913336544, + "loss": 1.3426, + "theoretical_loss": 3.387838997920153, + "tokens_seen": 2587033600 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.9919148683547974, + "objective/train/avg_token_score": 0.008085177280008793, + "objective/train/docs_used": 940829, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2807477712631226, + "objective/train/original_loss": 1.292649745941162, + "objective/train/theoretical_loss": 3.38782551475265, + "objective/train/tokens_used": 957623776, + "objective/train/unlikelihood_loss": 0.020424211397767067, + "theoretical_loss": 3.38782551475265, + "tokens_seen": 2587164672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021818327716257422, + "loss": 1.2448, + "theoretical_loss": 3.38782551475265, + "tokens_seen": 2587164672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021814315519178302, + "loss": 1.2838, + "theoretical_loss": 3.387812032459474, + "tokens_seen": 2587295744 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002181030332209918, + "loss": 1.2409, + "theoretical_loss": 3.3877985510405235, + "tokens_seen": 2587426816 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002180629112502006, + "loss": 1.2259, + "theoretical_loss": 3.3877850704956978, + "tokens_seen": 2587557888 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021802278927940942, + "loss": 1.2667, + "theoretical_loss": 3.3877715908248955, + "tokens_seen": 2587688960 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021798266730861822, + "loss": 1.291, + "theoretical_loss": 3.387758112028016, + "tokens_seen": 2587820032 + }, + { + "epoch": 0.57, + "learning_rate": 0.000217942545337827, + "loss": 1.2136, + "theoretical_loss": 3.3877446341049584, + "tokens_seen": 2587951104 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002179024233670358, + "loss": 1.2858, + "theoretical_loss": 3.3877311570556214, + "tokens_seen": 2588082176 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002178623013962446, + "loss": 1.2406, + "theoretical_loss": 3.3877176808799043, + "tokens_seen": 2588213248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021782217942545337, + "loss": 1.2588, + "theoretical_loss": 3.3877042055777067, + "tokens_seen": 2588344320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021778205745466217, + "loss": 1.3123, + "theoretical_loss": 3.3876907311489273, + "tokens_seen": 2588475392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021774193548387098, + "loss": 1.2621, + "theoretical_loss": 3.3876772575934657, + "tokens_seen": 2588606464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021770181351307978, + "loss": 1.1938, + "theoretical_loss": 3.3876637849112203, + "tokens_seen": 2588737536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021766169154228856, + "loss": 1.361, + "theoretical_loss": 3.3876503131020907, + "tokens_seen": 2588868608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021762156957149737, + "loss": 1.2787, + "theoretical_loss": 3.3876368421659766, + "tokens_seen": 2588999680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021758144760070615, + "loss": 1.2789, + "theoretical_loss": 3.3876233721027766, + "tokens_seen": 2589130752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021754132562991493, + "loss": 1.2509, + "theoretical_loss": 3.3876099029123896, + "tokens_seen": 2589261824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021750120365912373, + "loss": 1.3261, + "theoretical_loss": 3.387596434594716, + "tokens_seen": 2589392896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021746108168833254, + "loss": 1.2812, + "theoretical_loss": 3.3875829671496547, + "tokens_seen": 2589523968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021742095971754132, + "loss": 1.3247, + "theoretical_loss": 3.3875695005771043, + "tokens_seen": 2589655040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021738083774675013, + "loss": 1.3308, + "theoretical_loss": 3.387556034876965, + "tokens_seen": 2589786112 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021734071577595893, + "loss": 1.2583, + "theoretical_loss": 3.3875425700491353, + "tokens_seen": 2589917184 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002173005938051677, + "loss": 1.273, + "theoretical_loss": 3.387529106093515, + "tokens_seen": 2590048256 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002172604718343765, + "loss": 1.3116, + "theoretical_loss": 3.387515643010004, + "tokens_seen": 2590179328 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002172203498635853, + "loss": 1.2705, + "theoretical_loss": 3.3875021807985006, + "tokens_seen": 2590310400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002171802278927941, + "loss": 1.2509, + "theoretical_loss": 3.387488719458905, + "tokens_seen": 2590441472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021714010592200288, + "loss": 1.2411, + "theoretical_loss": 3.387475258991116, + "tokens_seen": 2590572544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002170999839512117, + "loss": 1.2293, + "theoretical_loss": 3.387461799395034, + "tokens_seen": 2590703616 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002170598619804205, + "loss": 1.2301, + "theoretical_loss": 3.3874483406705576, + "tokens_seen": 2590834688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002170197400096293, + "loss": 1.2687, + "theoretical_loss": 3.387434882817586, + "tokens_seen": 2590965760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021697961803883805, + "loss": 1.2464, + "theoretical_loss": 3.3874214258360196, + "tokens_seen": 2591096832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021693949606804686, + "loss": 1.2416, + "theoretical_loss": 3.3874079697257575, + "tokens_seen": 2591227904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021689937409725567, + "loss": 1.2203, + "theoretical_loss": 3.387394514486699, + "tokens_seen": 2591358976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021685925212646445, + "loss": 1.2877, + "theoretical_loss": 3.387381060118744, + "tokens_seen": 2591490048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021681913015567325, + "loss": 1.3332, + "theoretical_loss": 3.387367606621792, + "tokens_seen": 2591621120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021677900818488206, + "loss": 1.2376, + "theoretical_loss": 3.3873541539957426, + "tokens_seen": 2591752192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021673888621409087, + "loss": 1.2439, + "theoretical_loss": 3.387340702240495, + "tokens_seen": 2591883264 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021669876424329962, + "loss": 1.2923, + "theoretical_loss": 3.387327251355949, + "tokens_seen": 2592014336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021665864227250842, + "loss": 1.2609, + "theoretical_loss": 3.387313801342004, + "tokens_seen": 2592145408 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021661852030171723, + "loss": 1.2985, + "theoretical_loss": 3.3873003521985603, + "tokens_seen": 2592276480 + }, + { + "epoch": 0.57, + "learning_rate": 0.000216578398330926, + "loss": 1.2737, + "theoretical_loss": 3.387286903925517, + "tokens_seen": 2592407552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021653827636013482, + "loss": 1.1881, + "theoretical_loss": 3.387273456522774, + "tokens_seen": 2592538624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021649815438934362, + "loss": 1.2605, + "theoretical_loss": 3.387260009990231, + "tokens_seen": 2592669696 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002164580324185524, + "loss": 1.2418, + "theoretical_loss": 3.387246564327787, + "tokens_seen": 2592800768 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021641791044776118, + "loss": 1.2546, + "theoretical_loss": 3.3872331195353427, + "tokens_seen": 2592931840 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021637778847696999, + "loss": 1.3081, + "theoretical_loss": 3.3872196756127972, + "tokens_seen": 2593062912 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002163376665061788, + "loss": 1.2641, + "theoretical_loss": 3.387206232560051, + "tokens_seen": 2593193984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021629754453538757, + "loss": 1.2543, + "theoretical_loss": 3.3871927903770027, + "tokens_seen": 2593325056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021625742256459638, + "loss": 1.2239, + "theoretical_loss": 3.3871793490635533, + "tokens_seen": 2593456128 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021621730059380518, + "loss": 1.2492, + "theoretical_loss": 3.3871659086196018, + "tokens_seen": 2593587200 + }, + { + "epoch": 0.57, + "objective/train/aligned_token_count": 0.9943293929100037, + "objective/train/avg_token_score": 0.005670545622706413, + "objective/train/docs_used": 943233, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2449102401733398, + "objective/train/original_loss": 1.2524820566177368, + "objective/train/theoretical_loss": 3.387152469045048, + "objective/train/tokens_used": 964177376, + "objective/train/unlikelihood_loss": 0.017013994976878166, + "theoretical_loss": 3.387152469045048, + "tokens_seen": 2593718272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021617717862301396, + "loss": 1.2148, + "theoretical_loss": 3.387152469045048, + "tokens_seen": 2593718272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021613705665222277, + "loss": 1.2723, + "theoretical_loss": 3.3871390303397924, + "tokens_seen": 2593849344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021609693468143155, + "loss": 1.2624, + "theoretical_loss": 3.3871255925037342, + "tokens_seen": 2593980416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021605681271064036, + "loss": 1.3462, + "theoretical_loss": 3.3871121555367734, + "tokens_seen": 2594111488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021601669073984913, + "loss": 1.2121, + "theoretical_loss": 3.3870987194388102, + "tokens_seen": 2594242560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021597656876905794, + "loss": 1.3157, + "theoretical_loss": 3.387085284209744, + "tokens_seen": 2594373632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021593644679826675, + "loss": 1.3019, + "theoretical_loss": 3.3870718498494754, + "tokens_seen": 2594504704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021589632482747553, + "loss": 1.2529, + "theoretical_loss": 3.3870584163579034, + "tokens_seen": 2594635776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021585620285668433, + "loss": 1.1681, + "theoretical_loss": 3.387044983734929, + "tokens_seen": 2594766848 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002158160808858931, + "loss": 1.2097, + "theoretical_loss": 3.3870315519804515, + "tokens_seen": 2594897920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021577595891510192, + "loss": 1.2076, + "theoretical_loss": 3.387018121094371, + "tokens_seen": 2595028992 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157358369443107, + "loss": 1.3107, + "theoretical_loss": 3.387004691076587, + "tokens_seen": 2595160064 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002156957149735195, + "loss": 1.2707, + "theoretical_loss": 3.3869912619270006, + "tokens_seen": 2595291136 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002156555930027283, + "loss": 1.3303, + "theoretical_loss": 3.386977833645511, + "tokens_seen": 2595422208 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002156154710319371, + "loss": 1.2718, + "theoretical_loss": 3.386964406232019, + "tokens_seen": 2595553280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155753490611459, + "loss": 1.1964, + "theoretical_loss": 3.3869509796864237, + "tokens_seen": 2595684352 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155352270903547, + "loss": 1.235, + "theoretical_loss": 3.3869375540086257, + "tokens_seen": 2595815424 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021549510511956345, + "loss": 1.3099, + "theoretical_loss": 3.3869241291985253, + "tokens_seen": 2595946496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021545498314877226, + "loss": 1.3659, + "theoretical_loss": 3.386910705256022, + "tokens_seen": 2596077568 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021541486117798107, + "loss": 1.2312, + "theoretical_loss": 3.3868972821810166, + "tokens_seen": 2596208640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021537473920718987, + "loss": 1.2868, + "theoretical_loss": 3.3868838599734086, + "tokens_seen": 2596339712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021533461723639865, + "loss": 1.1827, + "theoretical_loss": 3.386870438633099, + "tokens_seen": 2596470784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021529449526560746, + "loss": 1.2449, + "theoretical_loss": 3.3868570181599873, + "tokens_seen": 2596601856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021525437329481626, + "loss": 1.3509, + "theoretical_loss": 3.3868435985539738, + "tokens_seen": 2596732928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021521425132402502, + "loss": 1.3391, + "theoretical_loss": 3.3868301798149587, + "tokens_seen": 2596864000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021517412935323382, + "loss": 1.3206, + "theoretical_loss": 3.3868167619428426, + "tokens_seen": 2596995072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021513400738244263, + "loss": 1.2449, + "theoretical_loss": 3.386803344937525, + "tokens_seen": 2597126144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021509388541165144, + "loss": 1.3008, + "theoretical_loss": 3.3867899287989074, + "tokens_seen": 2597257216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021505376344086021, + "loss": 1.2493, + "theoretical_loss": 3.386776513526889, + "tokens_seen": 2597388288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021501364147006902, + "loss": 1.3728, + "theoretical_loss": 3.38676309912137, + "tokens_seen": 2597519360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021497351949927783, + "loss": 1.3338, + "theoretical_loss": 3.3867496855822514, + "tokens_seen": 2597650432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021493339752848658, + "loss": 1.3303, + "theoretical_loss": 3.3867362729094332, + "tokens_seen": 2597781504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021489327555769539, + "loss": 1.2949, + "theoretical_loss": 3.386722861102816, + "tokens_seen": 2597912576 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002148531535869042, + "loss": 1.3475, + "theoretical_loss": 3.3867094501623, + "tokens_seen": 2598043648 + }, + { + "epoch": 0.57, + "learning_rate": 0.000214813031616113, + "loss": 1.3166, + "theoretical_loss": 3.3866960400877852, + "tokens_seen": 2598174720 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021477290964532178, + "loss": 1.3517, + "theoretical_loss": 3.3866826308791724, + "tokens_seen": 2598305792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021473278767453058, + "loss": 1.2398, + "theoretical_loss": 3.386669222536362, + "tokens_seen": 2598436864 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002146926657037394, + "loss": 1.3374, + "theoretical_loss": 3.386655815059255, + "tokens_seen": 2598567936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021465254373294817, + "loss": 1.2807, + "theoretical_loss": 3.3866424084477504, + "tokens_seen": 2598699008 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021461242176215695, + "loss": 1.2462, + "theoretical_loss": 3.38662900270175, + "tokens_seen": 2598830080 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021457229979136576, + "loss": 1.2433, + "theoretical_loss": 3.3866155978211534, + "tokens_seen": 2598961152 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021453217782057456, + "loss": 1.2835, + "theoretical_loss": 3.386602193805862, + "tokens_seen": 2599092224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021449205584978334, + "loss": 1.2116, + "theoretical_loss": 3.3865887906557752, + "tokens_seen": 2599223296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021445193387899215, + "loss": 1.2596, + "theoretical_loss": 3.3865753883707943, + "tokens_seen": 2599354368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021441181190820095, + "loss": 1.2551, + "theoretical_loss": 3.3865619869508197, + "tokens_seen": 2599485440 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021437168993740973, + "loss": 1.2364, + "theoretical_loss": 3.386548586395752, + "tokens_seen": 2599616512 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002143315679666185, + "loss": 1.3055, + "theoretical_loss": 3.3865351867054914, + "tokens_seen": 2599747584 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021429144599582732, + "loss": 1.2318, + "theoretical_loss": 3.3865217878799387, + "tokens_seen": 2599878656 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002142513240250361, + "loss": 1.3223, + "theoretical_loss": 3.386508389918995, + "tokens_seen": 2600009728 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002142112020542449, + "loss": 1.3379, + "theoretical_loss": 3.38649499282256, + "tokens_seen": 2600140800 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.9934045076370239, + "objective/train/avg_token_score": 0.006595509126782417, + "objective/train/docs_used": 945762, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2482595443725586, + "objective/train/original_loss": 1.2565879821777344, + "objective/train/theoretical_loss": 3.386481596590535, + "objective/train/tokens_used": 970730976, + "objective/train/unlikelihood_loss": 0.019667841494083405, + "theoretical_loss": 3.386481596590535, + "tokens_seen": 2600271872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141710800834537, + "loss": 1.2457, + "theoretical_loss": 3.386481596590535, + "tokens_seen": 2600271872 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021413095811266252, + "loss": 1.2866, + "theoretical_loss": 3.3864682012228204, + "tokens_seen": 2600402944 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002140908361418713, + "loss": 1.3179, + "theoretical_loss": 3.386454806719317, + "tokens_seen": 2600534016 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021405071417108007, + "loss": 1.3748, + "theoretical_loss": 3.386441413079926, + "tokens_seen": 2600665088 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021401059220028888, + "loss": 1.2682, + "theoretical_loss": 3.386428020304547, + "tokens_seen": 2600796160 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021397047022949766, + "loss": 1.2429, + "theoretical_loss": 3.3864146283930814, + "tokens_seen": 2600927232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021393034825870647, + "loss": 1.3163, + "theoretical_loss": 3.38640123734543, + "tokens_seen": 2601058304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021389022628791527, + "loss": 1.2644, + "theoretical_loss": 3.3863878471614934, + "tokens_seen": 2601189376 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021385010431712408, + "loss": 1.3519, + "theoretical_loss": 3.386374457841172, + "tokens_seen": 2601320448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021380998234633286, + "loss": 1.2955, + "theoretical_loss": 3.3863610693843675, + "tokens_seen": 2601451520 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021376986037554166, + "loss": 1.2099, + "theoretical_loss": 3.38634768179098, + "tokens_seen": 2601582592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021372973840475044, + "loss": 1.3585, + "theoretical_loss": 3.38633429506091, + "tokens_seen": 2601713664 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021368961643395922, + "loss": 1.2615, + "theoretical_loss": 3.386320909194059, + "tokens_seen": 2601844736 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021364949446316803, + "loss": 1.267, + "theoretical_loss": 3.386307524190328, + "tokens_seen": 2601975808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021360937249237684, + "loss": 1.2734, + "theoretical_loss": 3.3862941400496176, + "tokens_seen": 2602106880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021356925052158564, + "loss": 1.2868, + "theoretical_loss": 3.3862807567718285, + "tokens_seen": 2602237952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021352912855079442, + "loss": 1.2026, + "theoretical_loss": 3.3862673743568616, + "tokens_seen": 2602369024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021348900658000323, + "loss": 1.2634, + "theoretical_loss": 3.386253992804618, + "tokens_seen": 2602500096 + }, + { + "epoch": 0.58, + "learning_rate": 0.000213448884609212, + "loss": 1.3167, + "theoretical_loss": 3.3862406121149986, + "tokens_seen": 2602631168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021340876263842079, + "loss": 1.3242, + "theoretical_loss": 3.386227232287905, + "tokens_seen": 2602762240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002133686406676296, + "loss": 1.27, + "theoretical_loss": 3.3862138533232367, + "tokens_seen": 2602893312 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002133285186968384, + "loss": 1.2629, + "theoretical_loss": 3.3862004752208956, + "tokens_seen": 2603024384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021328839672604718, + "loss": 1.2903, + "theoretical_loss": 3.386187097980783, + "tokens_seen": 2603155456 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021324827475525598, + "loss": 1.3505, + "theoretical_loss": 3.386173721602799, + "tokens_seen": 2603286528 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002132081527844648, + "loss": 1.2657, + "theoretical_loss": 3.3861603460868457, + "tokens_seen": 2603417600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002131680308136736, + "loss": 1.3077, + "theoretical_loss": 3.386146971432823, + "tokens_seen": 2603548672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021312790884288235, + "loss": 1.3845, + "theoretical_loss": 3.386133597640633, + "tokens_seen": 2603679744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021308778687209115, + "loss": 1.3402, + "theoretical_loss": 3.3861202247101767, + "tokens_seen": 2603810816 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021304766490129996, + "loss": 1.2201, + "theoretical_loss": 3.3861068526413542, + "tokens_seen": 2603941888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021300754293050874, + "loss": 1.4809, + "theoretical_loss": 3.3860934814340675, + "tokens_seen": 2604072960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021296742095971755, + "loss": 1.2558, + "theoretical_loss": 3.386080111088218, + "tokens_seen": 2604204032 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021292729898892635, + "loss": 1.29, + "theoretical_loss": 3.386066741603706, + "tokens_seen": 2604335104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021288717701813516, + "loss": 1.2139, + "theoretical_loss": 3.3860533729804327, + "tokens_seen": 2604466176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002128470550473439, + "loss": 1.3373, + "theoretical_loss": 3.3860400052183, + "tokens_seen": 2604597248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021280693307655272, + "loss": 1.3166, + "theoretical_loss": 3.3860266383172086, + "tokens_seen": 2604728320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021276681110576152, + "loss": 1.2613, + "theoretical_loss": 3.3860132722770597, + "tokens_seen": 2604859392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127266891349703, + "loss": 1.2302, + "theoretical_loss": 3.385999907097755, + "tokens_seen": 2604990464 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002126865671641791, + "loss": 1.3782, + "theoretical_loss": 3.3859865427791953, + "tokens_seen": 2605121536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021264644519338792, + "loss": 1.2568, + "theoretical_loss": 3.385973179321282, + "tokens_seen": 2605252608 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021260632322259672, + "loss": 1.2401, + "theoretical_loss": 3.385959816723916, + "tokens_seen": 2605383680 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021256620125180547, + "loss": 1.2812, + "theoretical_loss": 3.3859464549869993, + "tokens_seen": 2605514752 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021252607928101428, + "loss": 1.2953, + "theoretical_loss": 3.385933094110433, + "tokens_seen": 2605645824 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002124859573102231, + "loss": 1.219, + "theoretical_loss": 3.385919734094118, + "tokens_seen": 2605776896 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021244583533943187, + "loss": 1.2012, + "theoretical_loss": 3.3859063749379557, + "tokens_seen": 2605907968 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021240571336864067, + "loss": 1.301, + "theoretical_loss": 3.385893016641848, + "tokens_seen": 2606039040 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021236559139784948, + "loss": 1.2876, + "theoretical_loss": 3.385879659205696, + "tokens_seen": 2606170112 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021232546942705826, + "loss": 1.3539, + "theoretical_loss": 3.385866302629401, + "tokens_seen": 2606301184 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021228534745626706, + "loss": 1.3454, + "theoretical_loss": 3.3858529469128644, + "tokens_seen": 2606432256 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021224522548547584, + "loss": 1.3261, + "theoretical_loss": 3.385839592055988, + "tokens_seen": 2606563328 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021220510351468465, + "loss": 1.3269, + "theoretical_loss": 3.3858262380586726, + "tokens_seen": 2606694400 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.9930676221847534, + "objective/train/avg_token_score": 0.006932341493666172, + "objective/train/docs_used": 948007, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.343998908996582, + "objective/train/original_loss": 1.3547778129577637, + "objective/train/theoretical_loss": 3.38581288492082, + "objective/train/tokens_used": 977284576, + "objective/train/unlikelihood_loss": 0.017440542578697205, + "theoretical_loss": 3.38581288492082, + "tokens_seen": 2606825472 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021216498154389343, + "loss": 1.3192, + "theoretical_loss": 3.38581288492082, + "tokens_seen": 2606825472 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021212485957310223, + "loss": 1.2407, + "theoretical_loss": 3.385799532642332, + "tokens_seen": 2606956544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021208473760231104, + "loss": 1.2234, + "theoretical_loss": 3.38578618122311, + "tokens_seen": 2607087616 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021204461563151982, + "loss": 1.3099, + "theoretical_loss": 3.3857728306630546, + "tokens_seen": 2607218688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021200449366072863, + "loss": 1.3118, + "theoretical_loss": 3.385759480962068, + "tokens_seen": 2607349760 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002119643716899374, + "loss": 1.1899, + "theoretical_loss": 3.385746132120052, + "tokens_seen": 2607480832 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002119242497191462, + "loss": 1.3386, + "theoretical_loss": 3.3857327841369083, + "tokens_seen": 2607611904 + }, + { + "epoch": 0.58, + "learning_rate": 0.000211884127748355, + "loss": 1.2384, + "theoretical_loss": 3.3857194370125376, + "tokens_seen": 2607742976 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002118440057775638, + "loss": 1.3687, + "theoretical_loss": 3.385706090746842, + "tokens_seen": 2607874048 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002118038838067726, + "loss": 1.2765, + "theoretical_loss": 3.385692745339723, + "tokens_seen": 2608005120 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021176376183598138, + "loss": 1.2427, + "theoretical_loss": 3.3856794007910826, + "tokens_seen": 2608136192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002117236398651902, + "loss": 1.3234, + "theoretical_loss": 3.385666057100822, + "tokens_seen": 2608267264 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021168351789439897, + "loss": 1.2633, + "theoretical_loss": 3.3856527142688426, + "tokens_seen": 2608398336 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021164339592360778, + "loss": 1.2846, + "theoretical_loss": 3.3856393722950466, + "tokens_seen": 2608529408 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021160327395281655, + "loss": 1.2183, + "theoretical_loss": 3.385626031179336, + "tokens_seen": 2608660480 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021156315198202536, + "loss": 1.2785, + "theoretical_loss": 3.3856126909216115, + "tokens_seen": 2608791552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021152303001123417, + "loss": 1.2939, + "theoretical_loss": 3.3855993515217757, + "tokens_seen": 2608922624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021148290804044295, + "loss": 1.1988, + "theoretical_loss": 3.3855860129797297, + "tokens_seen": 2609053696 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021144278606965175, + "loss": 1.2702, + "theoretical_loss": 3.385572675295376, + "tokens_seen": 2609184768 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021140266409886056, + "loss": 1.2647, + "theoretical_loss": 3.3855593384686156, + "tokens_seen": 2609315840 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002113625421280693, + "loss": 1.2757, + "theoretical_loss": 3.3855460024993502, + "tokens_seen": 2609446912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021132242015727812, + "loss": 1.3324, + "theoretical_loss": 3.3855326673874826, + "tokens_seen": 2609577984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021128229818648692, + "loss": 1.2464, + "theoretical_loss": 3.3855193331329136, + "tokens_seen": 2609709056 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021124217621569573, + "loss": 1.3005, + "theoretical_loss": 3.3855059997355457, + "tokens_seen": 2609840128 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002112020542449045, + "loss": 1.3177, + "theoretical_loss": 3.3854926671952805, + "tokens_seen": 2609971200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021116193227411332, + "loss": 1.3534, + "theoretical_loss": 3.3854793355120196, + "tokens_seen": 2610102272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021112181030332212, + "loss": 1.2899, + "theoretical_loss": 3.3854660046856653, + "tokens_seen": 2610233344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021108168833253087, + "loss": 1.2341, + "theoretical_loss": 3.3854526747161193, + "tokens_seen": 2610364416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021104156636173968, + "loss": 1.2729, + "theoretical_loss": 3.3854393456032836, + "tokens_seen": 2610495488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021100144439094849, + "loss": 1.2258, + "theoretical_loss": 3.38542601734706, + "tokens_seen": 2610626560 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002109613224201573, + "loss": 1.3046, + "theoretical_loss": 3.3854126899473505, + "tokens_seen": 2610757632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021092120044936607, + "loss": 1.2757, + "theoretical_loss": 3.385399363404057, + "tokens_seen": 2610888704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021088107847857488, + "loss": 1.2608, + "theoretical_loss": 3.3853860377170815, + "tokens_seen": 2611019776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021084095650778368, + "loss": 1.2124, + "theoretical_loss": 3.3853727128863262, + "tokens_seen": 2611150848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021080083453699246, + "loss": 1.3222, + "theoretical_loss": 3.3853593889116933, + "tokens_seen": 2611281920 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021076071256620124, + "loss": 1.3001, + "theoretical_loss": 3.385346065793084, + "tokens_seen": 2611412992 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021072059059541005, + "loss": 1.2457, + "theoretical_loss": 3.385332743530401, + "tokens_seen": 2611544064 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021068046862461886, + "loss": 1.1746, + "theoretical_loss": 3.385319422123546, + "tokens_seen": 2611675136 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021064034665382763, + "loss": 1.2919, + "theoretical_loss": 3.385306101572421, + "tokens_seen": 2611806208 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021060022468303644, + "loss": 1.2948, + "theoretical_loss": 3.385292781876929, + "tokens_seen": 2611937280 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021056010271224525, + "loss": 1.3127, + "theoretical_loss": 3.385279463036971, + "tokens_seen": 2612068352 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021051998074145403, + "loss": 1.2875, + "theoretical_loss": 3.3852661450524497, + "tokens_seen": 2612199424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104798587706628, + "loss": 1.2232, + "theoretical_loss": 3.385252827923267, + "tokens_seen": 2612330496 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104397367998716, + "loss": 1.2119, + "theoretical_loss": 3.3852395116493255, + "tokens_seen": 2612461568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103996148290804, + "loss": 1.2617, + "theoretical_loss": 3.385226196230527, + "tokens_seen": 2612592640 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103594928582892, + "loss": 1.3524, + "theoretical_loss": 3.3852128816667735, + "tokens_seen": 2612723712 + }, + { + "epoch": 0.58, + "learning_rate": 0.000210319370887498, + "loss": 1.264, + "theoretical_loss": 3.3851995679579674, + "tokens_seen": 2612854784 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002102792489167068, + "loss": 1.3327, + "theoretical_loss": 3.385186255104011, + "tokens_seen": 2612985856 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002102391269459156, + "loss": 1.3328, + "theoretical_loss": 3.3851729431048065, + "tokens_seen": 2613116928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021019900497512437, + "loss": 1.2129, + "theoretical_loss": 3.385159631960256, + "tokens_seen": 2613248000 + }, + { + "epoch": 0.58, + "objective/train/aligned_token_count": 0.9929156303405762, + "objective/train/avg_token_score": 0.007084388751536608, + "objective/train/docs_used": 950332, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2145977020263672, + "objective/train/original_loss": 1.2242205142974854, + "objective/train/theoretical_loss": 3.3851463216702617, + "objective/train/tokens_used": 983838176, + "objective/train/unlikelihood_loss": 0.02005230262875557, + "theoretical_loss": 3.3851463216702617, + "tokens_seen": 2613379072 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021015888300433317, + "loss": 1.2268, + "theoretical_loss": 3.3851463216702617, + "tokens_seen": 2613379072 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021011876103354195, + "loss": 1.2638, + "theoretical_loss": 3.3851330122347267, + "tokens_seen": 2613510144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021007863906275076, + "loss": 1.3016, + "theoretical_loss": 3.3851197036535523, + "tokens_seen": 2613641216 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021003851709195957, + "loss": 1.287, + "theoretical_loss": 3.385106395926641, + "tokens_seen": 2613772288 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020999839512116837, + "loss": 1.3166, + "theoretical_loss": 3.3850930890538957, + "tokens_seen": 2613903360 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020995827315037715, + "loss": 1.3298, + "theoretical_loss": 3.385079783035218, + "tokens_seen": 2614034432 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020991815117958596, + "loss": 1.3607, + "theoretical_loss": 3.3850664778705113, + "tokens_seen": 2614165504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020987802920879474, + "loss": 1.2337, + "theoretical_loss": 3.385053173559677, + "tokens_seen": 2614296576 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020983790723800352, + "loss": 1.2758, + "theoretical_loss": 3.3850398701026174, + "tokens_seen": 2614427648 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020979778526721232, + "loss": 1.2406, + "theoretical_loss": 3.385026567499236, + "tokens_seen": 2614558720 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020975766329642113, + "loss": 1.2717, + "theoretical_loss": 3.385013265749434, + "tokens_seen": 2614689792 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020971754132562994, + "loss": 1.2651, + "theoretical_loss": 3.3849999648531144, + "tokens_seen": 2614820864 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020967741935483871, + "loss": 1.2992, + "theoretical_loss": 3.38498666481018, + "tokens_seen": 2614951936 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020963729738404752, + "loss": 1.2991, + "theoretical_loss": 3.3849733656205325, + "tokens_seen": 2615083008 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002095971754132563, + "loss": 1.2675, + "theoretical_loss": 3.384960067284075, + "tokens_seen": 2615214080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020955705344246508, + "loss": 1.2721, + "theoretical_loss": 3.38494676980071, + "tokens_seen": 2615345152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020951693147167389, + "loss": 1.1709, + "theoretical_loss": 3.3849334731703395, + "tokens_seen": 2615476224 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002094768095008827, + "loss": 1.2712, + "theoretical_loss": 3.3849201773928668, + "tokens_seen": 2615607296 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002094366875300915, + "loss": 1.2875, + "theoretical_loss": 3.384906882468194, + "tokens_seen": 2615738368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020939656555930028, + "loss": 1.2857, + "theoretical_loss": 3.3848935883962237, + "tokens_seen": 2615869440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020935644358850908, + "loss": 1.2546, + "theoretical_loss": 3.3848802951768584, + "tokens_seen": 2616000512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020931632161771786, + "loss": 1.2328, + "theoretical_loss": 3.384867002810001, + "tokens_seen": 2616131584 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020927619964692664, + "loss": 1.2665, + "theoretical_loss": 3.3848537112955537, + "tokens_seen": 2616262656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020923607767613545, + "loss": 1.3057, + "theoretical_loss": 3.3848404206334193, + "tokens_seen": 2616393728 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020919595570534426, + "loss": 1.3098, + "theoretical_loss": 3.3848271308235005, + "tokens_seen": 2616524800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020915583373455303, + "loss": 1.2017, + "theoretical_loss": 3.3848138418657, + "tokens_seen": 2616655872 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020911571176376184, + "loss": 1.2371, + "theoretical_loss": 3.384800553759921, + "tokens_seen": 2616786944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020907558979297065, + "loss": 1.2683, + "theoretical_loss": 3.384787266506065, + "tokens_seen": 2616918016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020903546782217945, + "loss": 1.2748, + "theoretical_loss": 3.3847739801040357, + "tokens_seen": 2617049088 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002089953458513882, + "loss": 1.2799, + "theoretical_loss": 3.3847606945537354, + "tokens_seen": 2617180160 + }, + { + "epoch": 0.59, + "learning_rate": 0.000208955223880597, + "loss": 1.1765, + "theoretical_loss": 3.384747409855067, + "tokens_seen": 2617311232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020891510190980582, + "loss": 1.3466, + "theoretical_loss": 3.3847341260079338, + "tokens_seen": 2617442304 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002088749799390146, + "loss": 1.2804, + "theoretical_loss": 3.3847208430122375, + "tokens_seen": 2617573376 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002088348579682234, + "loss": 1.2181, + "theoretical_loss": 3.384707560867881, + "tokens_seen": 2617704448 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002087947359974322, + "loss": 1.2496, + "theoretical_loss": 3.3846942795747683, + "tokens_seen": 2617835520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020875461402664102, + "loss": 1.2942, + "theoretical_loss": 3.384680999132801, + "tokens_seen": 2617966592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020871449205584977, + "loss": 1.2759, + "theoretical_loss": 3.3846677195418824, + "tokens_seen": 2618097664 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020867437008505857, + "loss": 1.2092, + "theoretical_loss": 3.3846544408019152, + "tokens_seen": 2618228736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020863424811426738, + "loss": 1.3236, + "theoretical_loss": 3.384641162912803, + "tokens_seen": 2618359808 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020859412614347616, + "loss": 1.2044, + "theoretical_loss": 3.3846278858744476, + "tokens_seen": 2618490880 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020855400417268497, + "loss": 1.3382, + "theoretical_loss": 3.3846146096867527, + "tokens_seen": 2618621952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020851388220189377, + "loss": 1.2749, + "theoretical_loss": 3.384601334349621, + "tokens_seen": 2618753024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020847376023110258, + "loss": 1.2843, + "theoretical_loss": 3.384588059862955, + "tokens_seen": 2618884096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020843363826031136, + "loss": 1.3303, + "theoretical_loss": 3.384574786226658, + "tokens_seen": 2619015168 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020839351628952014, + "loss": 1.3064, + "theoretical_loss": 3.3845615134406337, + "tokens_seen": 2619146240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020835339431872894, + "loss": 1.3031, + "theoretical_loss": 3.384548241504784, + "tokens_seen": 2619277312 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020831327234793772, + "loss": 1.3383, + "theoretical_loss": 3.384534970419012, + "tokens_seen": 2619408384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020827315037714653, + "loss": 1.2784, + "theoretical_loss": 3.3845217001832215, + "tokens_seen": 2619539456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020823302840635534, + "loss": 1.2842, + "theoretical_loss": 3.384508430797315, + "tokens_seen": 2619670528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020819290643556411, + "loss": 1.2925, + "theoretical_loss": 3.3844951622611954, + "tokens_seen": 2619801600 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.99135422706604, + "objective/train/avg_token_score": 0.008645761758089066, + "objective/train/docs_used": 952663, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.218103051185608, + "objective/train/original_loss": 1.2308731079101562, + "objective/train/theoretical_loss": 3.3844818945747663, + "objective/train/tokens_used": 990391776, + "objective/train/unlikelihood_loss": 0.022719206288456917, + "theoretical_loss": 3.3844818945747663, + "tokens_seen": 2619932672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020815278446477292, + "loss": 1.2606, + "theoretical_loss": 3.3844818945747663, + "tokens_seen": 2619932672 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002081126624939817, + "loss": 1.2357, + "theoretical_loss": 3.38446862773793, + "tokens_seen": 2620063744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002080725405231905, + "loss": 1.2942, + "theoretical_loss": 3.3844553617505904, + "tokens_seen": 2620194816 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020803241855239929, + "loss": 1.3299, + "theoretical_loss": 3.3844420966126503, + "tokens_seen": 2620325888 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002079922965816081, + "loss": 1.2045, + "theoretical_loss": 3.3844288323240126, + "tokens_seen": 2620456960 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002079521746108169, + "loss": 1.2452, + "theoretical_loss": 3.384415568884581, + "tokens_seen": 2620588032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020791205264002568, + "loss": 1.3721, + "theoretical_loss": 3.384402306294258, + "tokens_seen": 2620719104 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020787193066923448, + "loss": 1.2767, + "theoretical_loss": 3.384389044552947, + "tokens_seen": 2620850176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020783180869844326, + "loss": 1.3169, + "theoretical_loss": 3.3843757836605515, + "tokens_seen": 2620981248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020779168672765207, + "loss": 1.2492, + "theoretical_loss": 3.384362523616974, + "tokens_seen": 2621112320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020775156475686085, + "loss": 1.2606, + "theoretical_loss": 3.384349264422119, + "tokens_seen": 2621243392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020771144278606965, + "loss": 1.2293, + "theoretical_loss": 3.3843360060758885, + "tokens_seen": 2621374464 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020767132081527846, + "loss": 1.2604, + "theoretical_loss": 3.3843227485781866, + "tokens_seen": 2621505536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020763119884448724, + "loss": 1.2617, + "theoretical_loss": 3.384309491928916, + "tokens_seen": 2621636608 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020759107687369605, + "loss": 1.3286, + "theoretical_loss": 3.38429623612798, + "tokens_seen": 2621767680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020755095490290485, + "loss": 1.2595, + "theoretical_loss": 3.384282981175282, + "tokens_seen": 2621898752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020751083293211363, + "loss": 1.2237, + "theoretical_loss": 3.3842697270707256, + "tokens_seen": 2622029824 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002074707109613224, + "loss": 1.3004, + "theoretical_loss": 3.3842564738142142, + "tokens_seen": 2622160896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020743058899053122, + "loss": 1.3254, + "theoretical_loss": 3.3842432214056504, + "tokens_seen": 2622291968 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020739046701974002, + "loss": 1.3065, + "theoretical_loss": 3.3842299698449385, + "tokens_seen": 2622423040 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002073503450489488, + "loss": 1.2312, + "theoretical_loss": 3.384216719131981, + "tokens_seen": 2622554112 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002073102230781576, + "loss": 1.2318, + "theoretical_loss": 3.3842034692666823, + "tokens_seen": 2622685184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020727010110736642, + "loss": 1.3307, + "theoretical_loss": 3.3841902202489447, + "tokens_seen": 2622816256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020722997913657517, + "loss": 1.2384, + "theoretical_loss": 3.3841769720786727, + "tokens_seen": 2622947328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020718985716578397, + "loss": 1.277, + "theoretical_loss": 3.3841637247557688, + "tokens_seen": 2623078400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020714973519499278, + "loss": 1.1666, + "theoretical_loss": 3.3841504782801373, + "tokens_seen": 2623209472 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002071096132242016, + "loss": 1.2713, + "theoretical_loss": 3.384137232651681, + "tokens_seen": 2623340544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020706949125341037, + "loss": 1.2919, + "theoretical_loss": 3.3841239878703036, + "tokens_seen": 2623471616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020702936928261917, + "loss": 1.2247, + "theoretical_loss": 3.384110743935909, + "tokens_seen": 2623602688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020698924731182798, + "loss": 1.332, + "theoretical_loss": 3.3840975008484, + "tokens_seen": 2623733760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020694912534103673, + "loss": 1.3205, + "theoretical_loss": 3.384084258607681, + "tokens_seen": 2623864832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020690900337024554, + "loss": 1.3761, + "theoretical_loss": 3.384071017213655, + "tokens_seen": 2623995904 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020686888139945434, + "loss": 1.2781, + "theoretical_loss": 3.384057776666226, + "tokens_seen": 2624126976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020682875942866315, + "loss": 1.2511, + "theoretical_loss": 3.384044536965297, + "tokens_seen": 2624258048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020678863745787193, + "loss": 1.2513, + "theoretical_loss": 3.384031298110772, + "tokens_seen": 2624389120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020674851548708074, + "loss": 1.2476, + "theoretical_loss": 3.3840180601025542, + "tokens_seen": 2624520192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020670839351628954, + "loss": 1.2254, + "theoretical_loss": 3.3840048229405477, + "tokens_seen": 2624651264 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020666827154549832, + "loss": 1.2521, + "theoretical_loss": 3.383991586624656, + "tokens_seen": 2624782336 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002066281495747071, + "loss": 1.3294, + "theoretical_loss": 3.3839783511547825, + "tokens_seen": 2624913408 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002065880276039159, + "loss": 1.2589, + "theoretical_loss": 3.3839651165308315, + "tokens_seen": 2625044480 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002065479056331247, + "loss": 1.333, + "theoretical_loss": 3.383951882752706, + "tokens_seen": 2625175552 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002065077836623335, + "loss": 1.3567, + "theoretical_loss": 3.383938649820311, + "tokens_seen": 2625306624 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002064676616915423, + "loss": 1.3128, + "theoretical_loss": 3.3839254177335487, + "tokens_seen": 2625437696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002064275397207511, + "loss": 1.3286, + "theoretical_loss": 3.3839121864923234, + "tokens_seen": 2625568768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020638741774995988, + "loss": 1.2671, + "theoretical_loss": 3.383898956096539, + "tokens_seen": 2625699840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020634729577916866, + "loss": 1.2613, + "theoretical_loss": 3.383885726546099, + "tokens_seen": 2625830912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020630717380837747, + "loss": 1.3575, + "theoretical_loss": 3.3838724978409074, + "tokens_seen": 2625961984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020626705183758625, + "loss": 1.3019, + "theoretical_loss": 3.383859269980868, + "tokens_seen": 2626093056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020622692986679505, + "loss": 1.3086, + "theoretical_loss": 3.383846042965885, + "tokens_seen": 2626224128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020618680789600386, + "loss": 1.2889, + "theoretical_loss": 3.3838328167958616, + "tokens_seen": 2626355200 + }, + { + "epoch": 0.59, + "objective/train/aligned_token_count": 0.9925761818885803, + "objective/train/avg_token_score": 0.007423733361065388, + "objective/train/docs_used": 954702, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1278666257858276, + "objective/train/original_loss": 1.137892484664917, + "objective/train/theoretical_loss": 3.383819591470702, + "objective/train/tokens_used": 996945376, + "objective/train/unlikelihood_loss": 0.020187895745038986, + "theoretical_loss": 3.383819591470702, + "tokens_seen": 2626486272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020614668592521267, + "loss": 1.2287, + "theoretical_loss": 3.383819591470702, + "tokens_seen": 2626486272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020610656395442145, + "loss": 1.2557, + "theoretical_loss": 3.38380636699031, + "tokens_seen": 2626617344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020606644198363025, + "loss": 1.2819, + "theoretical_loss": 3.3837931433545894, + "tokens_seen": 2626748416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020602632001283903, + "loss": 1.3011, + "theoretical_loss": 3.3837799205634442, + "tokens_seen": 2626879488 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002059861980420478, + "loss": 1.2587, + "theoretical_loss": 3.3837666986167783, + "tokens_seen": 2627010560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020594607607125662, + "loss": 1.2727, + "theoretical_loss": 3.383753477514496, + "tokens_seen": 2627141632 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020590595410046542, + "loss": 1.3094, + "theoretical_loss": 3.3837402572565005, + "tokens_seen": 2627272704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020586583212967423, + "loss": 1.2661, + "theoretical_loss": 3.383727037842696, + "tokens_seen": 2627403776 + }, + { + "epoch": 0.59, + "learning_rate": 0.000205825710158883, + "loss": 1.189, + "theoretical_loss": 3.383713819272987, + "tokens_seen": 2627534848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020578558818809182, + "loss": 1.3205, + "theoretical_loss": 3.383700601547277, + "tokens_seen": 2627665920 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002057454662173006, + "loss": 1.2218, + "theoretical_loss": 3.3836873846654703, + "tokens_seen": 2627796992 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020570534424650937, + "loss": 1.2403, + "theoretical_loss": 3.383674168627471, + "tokens_seen": 2627928064 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020566522227571818, + "loss": 1.3701, + "theoretical_loss": 3.3836609534331825, + "tokens_seen": 2628059136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020562510030492699, + "loss": 1.2533, + "theoretical_loss": 3.3836477390825097, + "tokens_seen": 2628190208 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002055849783341358, + "loss": 1.3267, + "theoretical_loss": 3.383634525575356, + "tokens_seen": 2628321280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020554485636334457, + "loss": 1.2775, + "theoretical_loss": 3.383621312911626, + "tokens_seen": 2628452352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020550473439255338, + "loss": 1.2377, + "theoretical_loss": 3.3836081010912236, + "tokens_seen": 2628583424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020546461242176216, + "loss": 1.2633, + "theoretical_loss": 3.3835948901140527, + "tokens_seen": 2628714496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020542449045097094, + "loss": 1.3269, + "theoretical_loss": 3.3835816799800176, + "tokens_seen": 2628845568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020538436848017974, + "loss": 1.3113, + "theoretical_loss": 3.383568470689023, + "tokens_seen": 2628976640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020534424650938855, + "loss": 1.2878, + "theoretical_loss": 3.383555262240972, + "tokens_seen": 2629107712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020530412453859736, + "loss": 1.2869, + "theoretical_loss": 3.38354205463577, + "tokens_seen": 2629238784 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020526400256780613, + "loss": 1.3223, + "theoretical_loss": 3.38352884787332, + "tokens_seen": 2629369856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020522388059701494, + "loss": 1.3082, + "theoretical_loss": 3.383515641953527, + "tokens_seen": 2629500928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020518375862622375, + "loss": 1.2775, + "theoretical_loss": 3.383502436876295, + "tokens_seen": 2629632000 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002051436366554325, + "loss": 1.2274, + "theoretical_loss": 3.3834892326415282, + "tokens_seen": 2629763072 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002051035146846413, + "loss": 1.1917, + "theoretical_loss": 3.383476029249131, + "tokens_seen": 2629894144 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002050633927138501, + "loss": 1.2788, + "theoretical_loss": 3.3834628266990077, + "tokens_seen": 2630025216 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002050232707430589, + "loss": 1.2532, + "theoretical_loss": 3.3834496249910626, + "tokens_seen": 2630156288 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002049831487722677, + "loss": 1.236, + "theoretical_loss": 3.3834364241252, + "tokens_seen": 2630287360 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002049430268014765, + "loss": 1.1914, + "theoretical_loss": 3.383423224101324, + "tokens_seen": 2630418432 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002049029048306853, + "loss": 1.3361, + "theoretical_loss": 3.383410024919339, + "tokens_seen": 2630549504 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020486278285989406, + "loss": 1.2896, + "theoretical_loss": 3.3833968265791494, + "tokens_seen": 2630680576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020482266088910287, + "loss": 1.2649, + "theoretical_loss": 3.38338362908066, + "tokens_seen": 2630811648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020478253891831167, + "loss": 1.2888, + "theoretical_loss": 3.3833704324237748, + "tokens_seen": 2630942720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020474241694752045, + "loss": 1.3045, + "theoretical_loss": 3.383357236608398, + "tokens_seen": 2631073792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020470229497672926, + "loss": 1.2957, + "theoretical_loss": 3.3833440416344347, + "tokens_seen": 2631204864 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020466217300593807, + "loss": 1.2023, + "theoretical_loss": 3.3833308475017883, + "tokens_seen": 2631335936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020462205103514687, + "loss": 1.1791, + "theoretical_loss": 3.383317654210364, + "tokens_seen": 2631467008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020458192906435563, + "loss": 1.2231, + "theoretical_loss": 3.3833044617600665, + "tokens_seen": 2631598080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020454180709356443, + "loss": 1.2366, + "theoretical_loss": 3.3832912701508, + "tokens_seen": 2631729152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020450168512277324, + "loss": 1.2304, + "theoretical_loss": 3.3832780793824684, + "tokens_seen": 2631860224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020446156315198202, + "loss": 1.2318, + "theoretical_loss": 3.383264889454977, + "tokens_seen": 2631991296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020442144118119082, + "loss": 1.2046, + "theoretical_loss": 3.3832517003682296, + "tokens_seen": 2632122368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020438131921039963, + "loss": 1.2334, + "theoretical_loss": 3.383238512122132, + "tokens_seen": 2632253440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020434119723960844, + "loss": 1.3437, + "theoretical_loss": 3.3832253247165873, + "tokens_seen": 2632384512 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020430107526881721, + "loss": 1.2519, + "theoretical_loss": 3.3832121381515012, + "tokens_seen": 2632515584 + }, + { + "epoch": 0.6, + "learning_rate": 0.000204260953298026, + "loss": 1.2273, + "theoretical_loss": 3.3831989524267776, + "tokens_seen": 2632646656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002042208313272348, + "loss": 1.1871, + "theoretical_loss": 3.3831857675423214, + "tokens_seen": 2632777728 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020418070935644358, + "loss": 1.2309, + "theoretical_loss": 3.383172583498037, + "tokens_seen": 2632908800 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.9916519522666931, + "objective/train/avg_token_score": 0.008348029106855392, + "objective/train/docs_used": 956895, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2517743110656738, + "objective/train/original_loss": 1.2650357484817505, + "objective/train/theoretical_loss": 3.3831594002938297, + "objective/train/tokens_used": 1003498976, + "objective/train/unlikelihood_loss": 0.020226135849952698, + "theoretical_loss": 3.3831594002938297, + "tokens_seen": 2633039872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020414058738565239, + "loss": 1.2473, + "theoretical_loss": 3.3831594002938297, + "tokens_seen": 2633039872 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002041004654148612, + "loss": 1.2668, + "theoretical_loss": 3.3831462179296032, + "tokens_seen": 2633170944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020406034344406997, + "loss": 1.2184, + "theoretical_loss": 3.3831330364052628, + "tokens_seen": 2633302016 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020402022147327878, + "loss": 1.2791, + "theoretical_loss": 3.383119855720713, + "tokens_seen": 2633433088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020398009950248756, + "loss": 1.2684, + "theoretical_loss": 3.3831066758758586, + "tokens_seen": 2633564160 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020393997753169636, + "loss": 1.2239, + "theoretical_loss": 3.3830934968706043, + "tokens_seen": 2633695232 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020389985556090514, + "loss": 1.2863, + "theoretical_loss": 3.383080318704855, + "tokens_seen": 2633826304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020385973359011395, + "loss": 1.2761, + "theoretical_loss": 3.3830671413785147, + "tokens_seen": 2633957376 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020381961161932276, + "loss": 1.3063, + "theoretical_loss": 3.383053964891489, + "tokens_seen": 2634088448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020377948964853153, + "loss": 1.287, + "theoretical_loss": 3.3830407892436822, + "tokens_seen": 2634219520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020373936767774034, + "loss": 1.2735, + "theoretical_loss": 3.3830276144349996, + "tokens_seen": 2634350592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020369924570694915, + "loss": 1.2543, + "theoretical_loss": 3.3830144404653453, + "tokens_seen": 2634481664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020365912373615793, + "loss": 1.2733, + "theoretical_loss": 3.383001267334625, + "tokens_seen": 2634612736 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002036190017653667, + "loss": 1.2735, + "theoretical_loss": 3.3829880950427427, + "tokens_seen": 2634743808 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002035788797945755, + "loss": 1.2405, + "theoretical_loss": 3.382974923589604, + "tokens_seen": 2634874880 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020353875782378432, + "loss": 1.2525, + "theoretical_loss": 3.382961752975113, + "tokens_seen": 2635005952 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002034986358529931, + "loss": 1.3193, + "theoretical_loss": 3.382948583199175, + "tokens_seen": 2635137024 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002034585138822019, + "loss": 1.3285, + "theoretical_loss": 3.382935414261695, + "tokens_seen": 2635268096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002034183919114107, + "loss": 1.2685, + "theoretical_loss": 3.3829222461625776, + "tokens_seen": 2635399168 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002033782699406195, + "loss": 1.2283, + "theoretical_loss": 3.3829090789017284, + "tokens_seen": 2635530240 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020333814796982827, + "loss": 1.2221, + "theoretical_loss": 3.3828959124790514, + "tokens_seen": 2635661312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020329802599903707, + "loss": 1.2687, + "theoretical_loss": 3.3828827468944525, + "tokens_seen": 2635792384 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020325790402824588, + "loss": 1.2323, + "theoretical_loss": 3.382869582147836, + "tokens_seen": 2635923456 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020321778205745466, + "loss": 1.2611, + "theoretical_loss": 3.382856418239107, + "tokens_seen": 2636054528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020317766008666347, + "loss": 1.1587, + "theoretical_loss": 3.382843255168171, + "tokens_seen": 2636185600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020313753811587227, + "loss": 1.1967, + "theoretical_loss": 3.3828300929349324, + "tokens_seen": 2636316672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020309741614508102, + "loss": 1.2718, + "theoretical_loss": 3.3828169315392964, + "tokens_seen": 2636447744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020305729417428983, + "loss": 1.2729, + "theoretical_loss": 3.3828037709811687, + "tokens_seen": 2636578816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020301717220349864, + "loss": 1.2495, + "theoretical_loss": 3.3827906112604538, + "tokens_seen": 2636709888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020297705023270744, + "loss": 1.266, + "theoretical_loss": 3.3827774523770566, + "tokens_seen": 2636840960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020293692826191622, + "loss": 1.2613, + "theoretical_loss": 3.382764294330882, + "tokens_seen": 2636972032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020289680629112503, + "loss": 1.2371, + "theoretical_loss": 3.3827511371218364, + "tokens_seen": 2637103104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020285668432033384, + "loss": 1.2345, + "theoretical_loss": 3.382737980749824, + "tokens_seen": 2637234176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020281656234954261, + "loss": 1.305, + "theoretical_loss": 3.38272482521475, + "tokens_seen": 2637365248 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002027764403787514, + "loss": 1.2245, + "theoretical_loss": 3.382711670516519, + "tokens_seen": 2637496320 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002027363184079602, + "loss": 1.2049, + "theoretical_loss": 3.3826985166550374, + "tokens_seen": 2637627392 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202696196437169, + "loss": 1.2614, + "theoretical_loss": 3.38268536363021, + "tokens_seen": 2637758464 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020265607446637779, + "loss": 1.2693, + "theoretical_loss": 3.3826722114419416, + "tokens_seen": 2637889536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002026159524955866, + "loss": 1.2302, + "theoretical_loss": 3.3826590600901376, + "tokens_seen": 2638020608 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002025758305247954, + "loss": 1.198, + "theoretical_loss": 3.3826459095747037, + "tokens_seen": 2638151680 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020253570855400418, + "loss": 1.2774, + "theoretical_loss": 3.3826327598955443, + "tokens_seen": 2638282752 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020249558658321296, + "loss": 1.2284, + "theoretical_loss": 3.382619611052565, + "tokens_seen": 2638413824 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020245546461242176, + "loss": 1.2607, + "theoretical_loss": 3.382606463045672, + "tokens_seen": 2638544896 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020241534264163057, + "loss": 1.2745, + "theoretical_loss": 3.382593315874769, + "tokens_seen": 2638675968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020237522067083935, + "loss": 1.2272, + "theoretical_loss": 3.3825801695397626, + "tokens_seen": 2638807040 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020233509870004815, + "loss": 1.2343, + "theoretical_loss": 3.382567024040558, + "tokens_seen": 2638938112 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020229497672925696, + "loss": 1.2143, + "theoretical_loss": 3.3825538793770598, + "tokens_seen": 2639069184 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020225485475846574, + "loss": 1.2681, + "theoretical_loss": 3.382540735549174, + "tokens_seen": 2639200256 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020221473278767452, + "loss": 1.2171, + "theoretical_loss": 3.382527592556806, + "tokens_seen": 2639331328 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020217461081688333, + "loss": 1.222, + "theoretical_loss": 3.382514450399861, + "tokens_seen": 2639462400 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.9917089343070984, + "objective/train/avg_token_score": 0.008291028439998627, + "objective/train/docs_used": 959162, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2214337587356567, + "objective/train/original_loss": 1.2323882579803467, + "objective/train/theoretical_loss": 3.382501309078244, + "objective/train/tokens_used": 1010052576, + "objective/train/unlikelihood_loss": 0.02299768105149269, + "theoretical_loss": 3.382501309078244, + "tokens_seen": 2639593472 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002021344888460921, + "loss": 1.2396, + "theoretical_loss": 3.382501309078244, + "tokens_seen": 2639593472 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002020943668753009, + "loss": 1.2026, + "theoretical_loss": 3.382488168591861, + "tokens_seen": 2639724544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020205424490450972, + "loss": 1.2311, + "theoretical_loss": 3.3824750289406174, + "tokens_seen": 2639855616 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020201412293371852, + "loss": 1.3134, + "theoretical_loss": 3.382461890124419, + "tokens_seen": 2639986688 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002019740009629273, + "loss": 1.227, + "theoretical_loss": 3.38244875214317, + "tokens_seen": 2640117760 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002019338789921361, + "loss": 1.1797, + "theoretical_loss": 3.382435614996777, + "tokens_seen": 2640248832 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002018937570213449, + "loss": 1.2083, + "theoretical_loss": 3.382422478685146, + "tokens_seen": 2640379904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020185363505055367, + "loss": 1.2627, + "theoretical_loss": 3.382409343208181, + "tokens_seen": 2640510976 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020181351307976247, + "loss": 1.2426, + "theoretical_loss": 3.3823962085657886, + "tokens_seen": 2640642048 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020177339110897128, + "loss": 1.1815, + "theoretical_loss": 3.3823830747578736, + "tokens_seen": 2640773120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002017332691381801, + "loss": 1.2124, + "theoretical_loss": 3.3823699417843427, + "tokens_seen": 2640904192 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020169314716738887, + "loss": 1.3325, + "theoretical_loss": 3.3823568096451004, + "tokens_seen": 2641035264 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020165302519659767, + "loss": 1.2458, + "theoretical_loss": 3.3823436783400527, + "tokens_seen": 2641166336 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020161290322580645, + "loss": 1.2036, + "theoretical_loss": 3.3823305478691053, + "tokens_seen": 2641297408 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020157278125501523, + "loss": 1.2426, + "theoretical_loss": 3.382317418232164, + "tokens_seen": 2641428480 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020153265928422404, + "loss": 1.2557, + "theoretical_loss": 3.382304289429134, + "tokens_seen": 2641559552 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020149253731343284, + "loss": 1.2629, + "theoretical_loss": 3.382291161459921, + "tokens_seen": 2641690624 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020145241534264165, + "loss": 1.2698, + "theoretical_loss": 3.382278034324431, + "tokens_seen": 2641821696 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020141229337185043, + "loss": 1.2965, + "theoretical_loss": 3.3822649080225693, + "tokens_seen": 2641952768 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020137217140105924, + "loss": 1.1705, + "theoretical_loss": 3.3822517825542424, + "tokens_seen": 2642083840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020133204943026804, + "loss": 1.1561, + "theoretical_loss": 3.382238657919355, + "tokens_seen": 2642214912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002012919274594768, + "loss": 1.2108, + "theoretical_loss": 3.3822255341178136, + "tokens_seen": 2642345984 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002012518054886856, + "loss": 1.2627, + "theoretical_loss": 3.3822124111495233, + "tokens_seen": 2642477056 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002012116835178944, + "loss": 1.2064, + "theoretical_loss": 3.3821992890143906, + "tokens_seen": 2642608128 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002011715615471032, + "loss": 1.2288, + "theoretical_loss": 3.3821861677123204, + "tokens_seen": 2642739200 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201131439576312, + "loss": 1.2224, + "theoretical_loss": 3.38217304724322, + "tokens_seen": 2642870272 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002010913176055208, + "loss": 1.2653, + "theoretical_loss": 3.382159927606993, + "tokens_seen": 2643001344 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002010511956347296, + "loss": 1.2422, + "theoretical_loss": 3.3821468088035473, + "tokens_seen": 2643132416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020101107366393836, + "loss": 1.2054, + "theoretical_loss": 3.3821336908327875, + "tokens_seen": 2643263488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020097095169314716, + "loss": 1.2808, + "theoretical_loss": 3.38212057369462, + "tokens_seen": 2643394560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020093082972235597, + "loss": 1.2785, + "theoretical_loss": 3.3821074573889502, + "tokens_seen": 2643525632 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020089070775156475, + "loss": 1.214, + "theoretical_loss": 3.382094341915685, + "tokens_seen": 2643656704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020085058578077355, + "loss": 1.2805, + "theoretical_loss": 3.382081227274729, + "tokens_seen": 2643787776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020081046380998236, + "loss": 1.257, + "theoretical_loss": 3.3820681134659893, + "tokens_seen": 2643918848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020077034183919117, + "loss": 1.217, + "theoretical_loss": 3.382055000489371, + "tokens_seen": 2644049920 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020073021986839992, + "loss": 1.2936, + "theoretical_loss": 3.38204188834478, + "tokens_seen": 2644180992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020069009789760873, + "loss": 1.2486, + "theoretical_loss": 3.382028777032123, + "tokens_seen": 2644312064 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020064997592681753, + "loss": 1.2574, + "theoretical_loss": 3.3820156665513057, + "tokens_seen": 2644443136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002006098539560263, + "loss": 1.2999, + "theoretical_loss": 3.3820025569022336, + "tokens_seen": 2644574208 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020056973198523512, + "loss": 1.2253, + "theoretical_loss": 3.3819894480848136, + "tokens_seen": 2644705280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020052961001444392, + "loss": 1.2969, + "theoretical_loss": 3.3819763400989507, + "tokens_seen": 2644836352 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020048948804365273, + "loss": 1.2317, + "theoretical_loss": 3.381963232944552, + "tokens_seen": 2644967424 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002004493660728615, + "loss": 1.2311, + "theoretical_loss": 3.381950126621523, + "tokens_seen": 2645098496 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002004092441020703, + "loss": 1.2832, + "theoretical_loss": 3.3819370211297692, + "tokens_seen": 2645229568 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002003691221312791, + "loss": 1.2496, + "theoretical_loss": 3.381923916469198, + "tokens_seen": 2645360640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020032900016048787, + "loss": 1.2218, + "theoretical_loss": 3.381910812639714, + "tokens_seen": 2645491712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020028887818969668, + "loss": 1.1751, + "theoretical_loss": 3.3818977096412244, + "tokens_seen": 2645622784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020024875621890549, + "loss": 1.2274, + "theoretical_loss": 3.3818846074736357, + "tokens_seen": 2645753856 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002002086342481143, + "loss": 1.2013, + "theoretical_loss": 3.3818715061368527, + "tokens_seen": 2645884928 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020016851227732307, + "loss": 1.1791, + "theoretical_loss": 3.3818584056307825, + "tokens_seen": 2646016000 + }, + { + "epoch": 0.6, + "objective/train/aligned_token_count": 0.9936308860778809, + "objective/train/avg_token_score": 0.006369124632328749, + "objective/train/docs_used": 961281, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1946879625320435, + "objective/train/original_loss": 1.2036001682281494, + "objective/train/theoretical_loss": 3.3818453059553315, + "objective/train/tokens_used": 1016606176, + "objective/train/unlikelihood_loss": 0.018956858664751053, + "theoretical_loss": 3.3818453059553315, + "tokens_seen": 2646147072 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020012839030653185, + "loss": 1.1984, + "theoretical_loss": 3.3818453059553315, + "tokens_seen": 2646147072 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020008826833574066, + "loss": 1.2021, + "theoretical_loss": 3.3818322071104046, + "tokens_seen": 2646278144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020004814636494944, + "loss": 1.2826, + "theoretical_loss": 3.3818191090959093, + "tokens_seen": 2646409216 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020000802439415824, + "loss": 1.2233, + "theoretical_loss": 3.3818060119117517, + "tokens_seen": 2646540288 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019996790242336705, + "loss": 1.2354, + "theoretical_loss": 3.3817929155578375, + "tokens_seen": 2646671360 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019992778045257583, + "loss": 1.2357, + "theoretical_loss": 3.381779820034073, + "tokens_seen": 2646802432 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019988765848178463, + "loss": 1.2349, + "theoretical_loss": 3.381766725340365, + "tokens_seen": 2646933504 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019984753651099341, + "loss": 1.2316, + "theoretical_loss": 3.3817536314766192, + "tokens_seen": 2647064576 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019980741454020222, + "loss": 1.2684, + "theoretical_loss": 3.381740538442742, + "tokens_seen": 2647195648 + }, + { + "epoch": 0.6, + "learning_rate": 0.000199767292569411, + "loss": 1.2178, + "theoretical_loss": 3.3817274462386404, + "tokens_seen": 2647326720 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001997271705986198, + "loss": 1.2539, + "theoretical_loss": 3.3817143548642195, + "tokens_seen": 2647457792 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001996870486278286, + "loss": 1.23, + "theoretical_loss": 3.381701264319387, + "tokens_seen": 2647588864 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001996469266570374, + "loss": 1.2482, + "theoretical_loss": 3.381688174604048, + "tokens_seen": 2647719936 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001996068046862462, + "loss": 1.2078, + "theoretical_loss": 3.3816750857181104, + "tokens_seen": 2647851008 + }, + { + "epoch": 0.6, + "learning_rate": 0.000199566682715455, + "loss": 1.2459, + "theoretical_loss": 3.381661997661479, + "tokens_seen": 2647982080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019952656074466378, + "loss": 1.3154, + "theoretical_loss": 3.3816489104340612, + "tokens_seen": 2648113152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019948643877387256, + "loss": 1.2174, + "theoretical_loss": 3.381635824035763, + "tokens_seen": 2648244224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019944631680308137, + "loss": 1.2135, + "theoretical_loss": 3.3816227384664908, + "tokens_seen": 2648375296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019940619483229017, + "loss": 1.1552, + "theoretical_loss": 3.3816096537261515, + "tokens_seen": 2648506368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019936607286149895, + "loss": 1.2721, + "theoretical_loss": 3.381596569814651, + "tokens_seen": 2648637440 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019932595089070776, + "loss": 1.1424, + "theoretical_loss": 3.3815834867318966, + "tokens_seen": 2648768512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019928582891991657, + "loss": 1.1961, + "theoretical_loss": 3.381570404477794, + "tokens_seen": 2648899584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019924570694912535, + "loss": 1.233, + "theoretical_loss": 3.3815573230522498, + "tokens_seen": 2649030656 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019920558497833413, + "loss": 1.2339, + "theoretical_loss": 3.381544242455171, + "tokens_seen": 2649161728 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019916546300754293, + "loss": 1.1812, + "theoretical_loss": 3.3815311626864637, + "tokens_seen": 2649292800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019912534103675174, + "loss": 1.2075, + "theoretical_loss": 3.381518083746035, + "tokens_seen": 2649423872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019908521906596052, + "loss": 1.2578, + "theoretical_loss": 3.3815050056337905, + "tokens_seen": 2649554944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019904509709516932, + "loss": 1.1697, + "theoretical_loss": 3.3814919283496376, + "tokens_seen": 2649686016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019900497512437813, + "loss": 1.2376, + "theoretical_loss": 3.3814788518934833, + "tokens_seen": 2649817088 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001989648531535869, + "loss": 1.2299, + "theoretical_loss": 3.381465776265233, + "tokens_seen": 2649948160 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001989247311827957, + "loss": 1.2529, + "theoretical_loss": 3.381452701464794, + "tokens_seen": 2650079232 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001988846092120045, + "loss": 1.2507, + "theoretical_loss": 3.381439627492073, + "tokens_seen": 2650210304 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001988444872412133, + "loss": 1.1985, + "theoretical_loss": 3.3814265543469766, + "tokens_seen": 2650341376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019880436527042208, + "loss": 1.2747, + "theoretical_loss": 3.3814134820294117, + "tokens_seen": 2650472448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019876424329963089, + "loss": 1.2033, + "theoretical_loss": 3.3814004105392845, + "tokens_seen": 2650603520 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001987241213288397, + "loss": 1.2863, + "theoretical_loss": 3.381387339876502, + "tokens_seen": 2650734592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019868399935804847, + "loss": 1.2487, + "theoretical_loss": 3.381374270040971, + "tokens_seen": 2650865664 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019864387738725725, + "loss": 1.2036, + "theoretical_loss": 3.381361201032598, + "tokens_seen": 2650996736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019860375541646606, + "loss": 1.2778, + "theoretical_loss": 3.3813481328512904, + "tokens_seen": 2651127808 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019856363344567486, + "loss": 1.23, + "theoretical_loss": 3.381335065496954, + "tokens_seen": 2651258880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019852351147488364, + "loss": 1.2614, + "theoretical_loss": 3.3813219989694963, + "tokens_seen": 2651389952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019848338950409245, + "loss": 1.1869, + "theoretical_loss": 3.3813089332688238, + "tokens_seen": 2651521024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019844326753330126, + "loss": 1.2498, + "theoretical_loss": 3.3812958683948438, + "tokens_seen": 2651652096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019840314556251003, + "loss": 1.2615, + "theoretical_loss": 3.381282804347462, + "tokens_seen": 2651783168 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019836302359171881, + "loss": 1.2769, + "theoretical_loss": 3.3812697411265864, + "tokens_seen": 2651914240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019832290162092762, + "loss": 1.2097, + "theoretical_loss": 3.3812566787321234, + "tokens_seen": 2652045312 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019828277965013643, + "loss": 1.2917, + "theoretical_loss": 3.38124361716398, + "tokens_seen": 2652176384 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001982426576793452, + "loss": 1.2018, + "theoretical_loss": 3.3812305564220626, + "tokens_seen": 2652307456 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198202535708554, + "loss": 1.2552, + "theoretical_loss": 3.3812174965062787, + "tokens_seen": 2652438528 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019816241373776282, + "loss": 1.1841, + "theoretical_loss": 3.3812044374165353, + "tokens_seen": 2652569600 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.9926072955131531, + "objective/train/avg_token_score": 0.007392706349492073, + "objective/train/docs_used": 963694, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2919871807098389, + "objective/train/original_loss": 1.3012454509735107, + "objective/train/theoretical_loss": 3.381191379152739, + "objective/train/tokens_used": 1023159776, + "objective/train/unlikelihood_loss": 0.021943124011158943, + "theoretical_loss": 3.381191379152739, + "tokens_seen": 2652700672 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001981222917669716, + "loss": 1.254, + "theoretical_loss": 3.381191379152739, + "tokens_seen": 2652700672 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001980821697961804, + "loss": 1.2588, + "theoretical_loss": 3.381178321714797, + "tokens_seen": 2652831744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019804204782538918, + "loss": 1.3048, + "theoretical_loss": 3.3811652651026156, + "tokens_seen": 2652962816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019800192585459796, + "loss": 1.2565, + "theoretical_loss": 3.381152209316103, + "tokens_seen": 2653093888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019796180388380677, + "loss": 1.2082, + "theoretical_loss": 3.3811391543551648, + "tokens_seen": 2653224960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019792168191301557, + "loss": 1.2029, + "theoretical_loss": 3.381126100219709, + "tokens_seen": 2653356032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019788155994222438, + "loss": 1.3565, + "theoretical_loss": 3.3811130469096424, + "tokens_seen": 2653487104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019784143797143316, + "loss": 1.2096, + "theoretical_loss": 3.381099994424872, + "tokens_seen": 2653618176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019780131600064197, + "loss": 1.2328, + "theoretical_loss": 3.3810869427653047, + "tokens_seen": 2653749248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019776119402985075, + "loss": 1.2696, + "theoretical_loss": 3.381073891930848, + "tokens_seen": 2653880320 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019772107205905952, + "loss": 1.2514, + "theoretical_loss": 3.3810608419214088, + "tokens_seen": 2654011392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019768095008826833, + "loss": 1.264, + "theoretical_loss": 3.3810477927368936, + "tokens_seen": 2654142464 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019764082811747714, + "loss": 1.3031, + "theoretical_loss": 3.38103474437721, + "tokens_seen": 2654273536 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019760070614668594, + "loss": 1.2237, + "theoretical_loss": 3.381021696842266, + "tokens_seen": 2654404608 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019756058417589472, + "loss": 1.2511, + "theoretical_loss": 3.3810086501319674, + "tokens_seen": 2654535680 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019752046220510353, + "loss": 1.3428, + "theoretical_loss": 3.380995604246222, + "tokens_seen": 2654666752 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001974803402343123, + "loss": 1.2763, + "theoretical_loss": 3.3809825591849365, + "tokens_seen": 2654797824 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001974402182635211, + "loss": 1.3284, + "theoretical_loss": 3.380969514948019, + "tokens_seen": 2654928896 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001974000962927299, + "loss": 1.2617, + "theoretical_loss": 3.380956471535376, + "tokens_seen": 2655059968 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973599743219387, + "loss": 1.2128, + "theoretical_loss": 3.3809434289469147, + "tokens_seen": 2655191040 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973198523511475, + "loss": 1.2439, + "theoretical_loss": 3.3809303871825427, + "tokens_seen": 2655322112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019727973038035629, + "loss": 1.2205, + "theoretical_loss": 3.3809173462421667, + "tokens_seen": 2655453184 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001972396084095651, + "loss": 1.264, + "theoretical_loss": 3.380904306125695, + "tokens_seen": 2655584256 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001971994864387739, + "loss": 1.341, + "theoretical_loss": 3.380891266833034, + "tokens_seen": 2655715328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019715936446798265, + "loss": 1.2437, + "theoretical_loss": 3.3808782283640912, + "tokens_seen": 2655846400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019711924249719146, + "loss": 1.2862, + "theoretical_loss": 3.380865190718774, + "tokens_seen": 2655977472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019707912052640026, + "loss": 1.2965, + "theoretical_loss": 3.38085215389699, + "tokens_seen": 2656108544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019703899855560907, + "loss": 1.2743, + "theoretical_loss": 3.380839117898646, + "tokens_seen": 2656239616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019699887658481785, + "loss": 1.2712, + "theoretical_loss": 3.380826082723649, + "tokens_seen": 2656370688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019695875461402665, + "loss": 1.2558, + "theoretical_loss": 3.3808130483719077, + "tokens_seen": 2656501760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019691863264323546, + "loss": 1.3094, + "theoretical_loss": 3.380800014843329, + "tokens_seen": 2656632832 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001968785106724442, + "loss": 1.3, + "theoretical_loss": 3.3807869821378196, + "tokens_seen": 2656763904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019683838870165302, + "loss": 1.2778, + "theoretical_loss": 3.3807739502552874, + "tokens_seen": 2656894976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019679826673086183, + "loss": 1.2303, + "theoretical_loss": 3.38076091919564, + "tokens_seen": 2657026048 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001967581447600706, + "loss": 1.294, + "theoretical_loss": 3.3807478889587848, + "tokens_seen": 2657157120 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001967180227892794, + "loss": 1.2942, + "theoretical_loss": 3.3807348595446287, + "tokens_seen": 2657288192 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019667790081848822, + "loss": 1.2561, + "theoretical_loss": 3.38072183095308, + "tokens_seen": 2657419264 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019663777884769702, + "loss": 1.3183, + "theoretical_loss": 3.3807088031840458, + "tokens_seen": 2657550336 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001965976568769058, + "loss": 1.2363, + "theoretical_loss": 3.3806957762374337, + "tokens_seen": 2657681408 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019655753490611458, + "loss": 1.3466, + "theoretical_loss": 3.380682750113151, + "tokens_seen": 2657812480 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001965174129353234, + "loss": 1.1972, + "theoretical_loss": 3.3806697248111055, + "tokens_seen": 2657943552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019647729096453217, + "loss": 1.1914, + "theoretical_loss": 3.3806567003312047, + "tokens_seen": 2658074624 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019643716899374097, + "loss": 1.2223, + "theoretical_loss": 3.3806436766733556, + "tokens_seen": 2658205696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019639704702294978, + "loss": 1.2655, + "theoretical_loss": 3.380630653837467, + "tokens_seen": 2658336768 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001963569250521586, + "loss": 1.3173, + "theoretical_loss": 3.3806176318234455, + "tokens_seen": 2658467840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019631680308136737, + "loss": 1.2376, + "theoretical_loss": 3.380604610631199, + "tokens_seen": 2658598912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019627668111057615, + "loss": 1.2993, + "theoretical_loss": 3.3805915902606354, + "tokens_seen": 2658729984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019623655913978495, + "loss": 1.2366, + "theoretical_loss": 3.380578570711662, + "tokens_seen": 2658861056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019619643716899373, + "loss": 1.3435, + "theoretical_loss": 3.380565551984186, + "tokens_seen": 2658992128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019615631519820254, + "loss": 1.2556, + "theoretical_loss": 3.3805525340781166, + "tokens_seen": 2659123200 + }, + { + "epoch": 0.61, + "objective/train/aligned_token_count": 0.9953900575637817, + "objective/train/avg_token_score": 0.004609938710927963, + "objective/train/docs_used": 966064, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3384712934494019, + "objective/train/original_loss": 1.3450169563293457, + "objective/train/theoretical_loss": 3.38053951699336, + "objective/train/tokens_used": 1029713376, + "objective/train/unlikelihood_loss": 0.012781013734638691, + "theoretical_loss": 3.38053951699336, + "tokens_seen": 2659254272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019611619322741134, + "loss": 1.2778, + "theoretical_loss": 3.38053951699336, + "tokens_seen": 2659254272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019607607125662015, + "loss": 1.1701, + "theoretical_loss": 3.3805265007298244, + "tokens_seen": 2659385344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019603594928582893, + "loss": 1.3482, + "theoretical_loss": 3.380513485287418, + "tokens_seen": 2659516416 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001959958273150377, + "loss": 1.2507, + "theoretical_loss": 3.3805004706660475, + "tokens_seen": 2659647488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019595570534424651, + "loss": 1.1764, + "theoretical_loss": 3.3804874568656214, + "tokens_seen": 2659778560 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001959155833734553, + "loss": 1.2426, + "theoretical_loss": 3.3804744438860475, + "tokens_seen": 2659909632 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958754614026641, + "loss": 1.2734, + "theoretical_loss": 3.3804614317272335, + "tokens_seen": 2660040704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958353394318729, + "loss": 1.1568, + "theoretical_loss": 3.3804484203890866, + "tokens_seen": 2660171776 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019579521746108169, + "loss": 1.322, + "theoretical_loss": 3.3804354098715153, + "tokens_seen": 2660302848 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001957550954902905, + "loss": 1.2446, + "theoretical_loss": 3.380422400174427, + "tokens_seen": 2660433920 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001957149735194993, + "loss": 1.2963, + "theoretical_loss": 3.3804093912977304, + "tokens_seen": 2660564992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019567485154870808, + "loss": 1.2279, + "theoretical_loss": 3.380396383241332, + "tokens_seen": 2660696064 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019563472957791686, + "loss": 1.2072, + "theoretical_loss": 3.380383376005141, + "tokens_seen": 2660827136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019559460760712566, + "loss": 1.2807, + "theoretical_loss": 3.380370369589064, + "tokens_seen": 2660958208 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019555448563633447, + "loss": 1.2503, + "theoretical_loss": 3.38035736399301, + "tokens_seen": 2661089280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019551436366554325, + "loss": 1.2459, + "theoretical_loss": 3.380344359216886, + "tokens_seen": 2661220352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019547424169475205, + "loss": 1.2664, + "theoretical_loss": 3.380331355260601, + "tokens_seen": 2661351424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019543411972396086, + "loss": 1.2213, + "theoretical_loss": 3.380318352124062, + "tokens_seen": 2661482496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019539399775316964, + "loss": 1.2182, + "theoretical_loss": 3.3803053498071773, + "tokens_seen": 2661613568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019535387578237842, + "loss": 1.2403, + "theoretical_loss": 3.3802923483098546, + "tokens_seen": 2661744640 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019531375381158723, + "loss": 1.2635, + "theoretical_loss": 3.3802793476320026, + "tokens_seen": 2661875712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019527363184079603, + "loss": 1.2195, + "theoretical_loss": 3.380266347773529, + "tokens_seen": 2662006784 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001952335098700048, + "loss": 1.1857, + "theoretical_loss": 3.3802533487343407, + "tokens_seen": 2662137856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019519338789921362, + "loss": 1.2828, + "theoretical_loss": 3.380240350514348, + "tokens_seen": 2662268928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019515326592842242, + "loss": 1.2977, + "theoretical_loss": 3.3802273531134563, + "tokens_seen": 2662400000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001951131439576312, + "loss": 1.2574, + "theoretical_loss": 3.3802143565315763, + "tokens_seen": 2662531072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019507302198683998, + "loss": 1.2153, + "theoretical_loss": 3.380201360768614, + "tokens_seen": 2662662144 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001950329000160488, + "loss": 1.2667, + "theoretical_loss": 3.3801883658244782, + "tokens_seen": 2662793216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001949927780452576, + "loss": 1.2111, + "theoretical_loss": 3.3801753716990777, + "tokens_seen": 2662924288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019495265607446637, + "loss": 1.2673, + "theoretical_loss": 3.3801623783923196, + "tokens_seen": 2663055360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019491253410367518, + "loss": 1.2962, + "theoretical_loss": 3.3801493859041125, + "tokens_seen": 2663186432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019487241213288399, + "loss": 1.2528, + "theoretical_loss": 3.3801363942343645, + "tokens_seen": 2663317504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019483229016209277, + "loss": 1.1686, + "theoretical_loss": 3.3801234033829837, + "tokens_seen": 2663448576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019479216819130154, + "loss": 1.2429, + "theoretical_loss": 3.3801104133498785, + "tokens_seen": 2663579648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019475204622051035, + "loss": 1.1649, + "theoretical_loss": 3.3800974241349566, + "tokens_seen": 2663710720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019471192424971916, + "loss": 1.3451, + "theoretical_loss": 3.380084435738127, + "tokens_seen": 2663841792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019467180227892794, + "loss": 1.2926, + "theoretical_loss": 3.3800714481592973, + "tokens_seen": 2663972864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019463168030813674, + "loss": 1.2516, + "theoretical_loss": 3.380058461398376, + "tokens_seen": 2664103936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019459155833734555, + "loss": 1.2086, + "theoretical_loss": 3.380045475455271, + "tokens_seen": 2664235008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019455143636655433, + "loss": 1.2242, + "theoretical_loss": 3.380032490329891, + "tokens_seen": 2664366080 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001945113143957631, + "loss": 1.3285, + "theoretical_loss": 3.380019506022144, + "tokens_seen": 2664497152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019447119242497191, + "loss": 1.2603, + "theoretical_loss": 3.3800065225319385, + "tokens_seen": 2664628224 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019443107045418072, + "loss": 1.232, + "theoretical_loss": 3.379993539859183, + "tokens_seen": 2664759296 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001943909484833895, + "loss": 1.3231, + "theoretical_loss": 3.379980558003785, + "tokens_seen": 2664890368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001943508265125983, + "loss": 1.3072, + "theoretical_loss": 3.3799675769656536, + "tokens_seen": 2665021440 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001943107045418071, + "loss": 1.2661, + "theoretical_loss": 3.379954596744697, + "tokens_seen": 2665152512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001942705825710159, + "loss": 1.2314, + "theoretical_loss": 3.3799416173408234, + "tokens_seen": 2665283584 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019423046060022467, + "loss": 1.2943, + "theoretical_loss": 3.379928638753942, + "tokens_seen": 2665414656 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019419033862943348, + "loss": 1.2798, + "theoretical_loss": 3.37991566098396, + "tokens_seen": 2665545728 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019415021665864228, + "loss": 1.2832, + "theoretical_loss": 3.379902684030786, + "tokens_seen": 2665676800 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.992882251739502, + "objective/train/avg_token_score": 0.007117701694369316, + "objective/train/docs_used": 968517, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.162646770477295, + "objective/train/original_loss": 1.1720105409622192, + "objective/train/theoretical_loss": 3.3798897078943293, + "objective/train/tokens_used": 1036266976, + "objective/train/unlikelihood_loss": 0.024212360382080078, + "theoretical_loss": 3.3798897078943293, + "tokens_seen": 2665807872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019411009468785106, + "loss": 1.1767, + "theoretical_loss": 3.3798897078943293, + "tokens_seen": 2665807872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019406997271705987, + "loss": 1.2576, + "theoretical_loss": 3.379876732574498, + "tokens_seen": 2665938944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019402985074626867, + "loss": 1.2819, + "theoretical_loss": 3.3798637580712, + "tokens_seen": 2666070016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019398972877547745, + "loss": 1.2422, + "theoretical_loss": 3.379850784384344, + "tokens_seen": 2666201088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019394960680468626, + "loss": 1.3437, + "theoretical_loss": 3.379837811513839, + "tokens_seen": 2666332160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019390948483389504, + "loss": 1.2744, + "theoretical_loss": 3.3798248394595927, + "tokens_seen": 2666463232 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019386936286310382, + "loss": 1.2878, + "theoretical_loss": 3.3798118682215144, + "tokens_seen": 2666594304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019382924089231263, + "loss": 1.268, + "theoretical_loss": 3.3797988977995126, + "tokens_seen": 2666725376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019378911892152143, + "loss": 1.2039, + "theoretical_loss": 3.379785928193495, + "tokens_seen": 2666856448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019374899695073024, + "loss": 1.2261, + "theoretical_loss": 3.379772959403371, + "tokens_seen": 2666987520 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019370887497993902, + "loss": 1.206, + "theoretical_loss": 3.379759991429049, + "tokens_seen": 2667118592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019366875300914782, + "loss": 1.355, + "theoretical_loss": 3.3797470242704373, + "tokens_seen": 2667249664 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001936286310383566, + "loss": 1.2152, + "theoretical_loss": 3.379734057927445, + "tokens_seen": 2667380736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019358850906756538, + "loss": 1.2529, + "theoretical_loss": 3.37972109239998, + "tokens_seen": 2667511808 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001935483870967742, + "loss": 1.2829, + "theoretical_loss": 3.3797081276879517, + "tokens_seen": 2667642880 + }, + { + "epoch": 0.62, + "learning_rate": 0.000193508265125983, + "loss": 1.3096, + "theoretical_loss": 3.3796951637912684, + "tokens_seen": 2667773952 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001934681431551918, + "loss": 1.2891, + "theoretical_loss": 3.379682200709839, + "tokens_seen": 2667905024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019342802118440058, + "loss": 1.1537, + "theoretical_loss": 3.3796692384435714, + "tokens_seen": 2668036096 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019338789921360939, + "loss": 1.2862, + "theoretical_loss": 3.379656276992375, + "tokens_seen": 2668167168 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001933477772428182, + "loss": 1.235, + "theoretical_loss": 3.3796433163561588, + "tokens_seen": 2668298240 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019330765527202694, + "loss": 1.2793, + "theoretical_loss": 3.3796303565348307, + "tokens_seen": 2668429312 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019326753330123575, + "loss": 1.2034, + "theoretical_loss": 3.3796173975282997, + "tokens_seen": 2668560384 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019322741133044456, + "loss": 1.2811, + "theoretical_loss": 3.3796044393364753, + "tokens_seen": 2668691456 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019318728935965336, + "loss": 1.2533, + "theoretical_loss": 3.3795914819592654, + "tokens_seen": 2668822528 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019314716738886214, + "loss": 1.2356, + "theoretical_loss": 3.379578525396579, + "tokens_seen": 2668953600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019310704541807095, + "loss": 1.1675, + "theoretical_loss": 3.379565569648325, + "tokens_seen": 2669084672 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019306692344727976, + "loss": 1.2152, + "theoretical_loss": 3.379552614714412, + "tokens_seen": 2669215744 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001930268014764885, + "loss": 1.2433, + "theoretical_loss": 3.3795396605947494, + "tokens_seen": 2669346816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019298667950569731, + "loss": 1.2284, + "theoretical_loss": 3.379526707289245, + "tokens_seen": 2669477888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019294655753490612, + "loss": 1.2511, + "theoretical_loss": 3.3795137547978094, + "tokens_seen": 2669608960 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019290643556411493, + "loss": 1.2797, + "theoretical_loss": 3.3795008031203495, + "tokens_seen": 2669740032 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928663135933237, + "loss": 1.166, + "theoretical_loss": 3.3794878522567755, + "tokens_seen": 2669871104 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928261916225325, + "loss": 1.3729, + "theoretical_loss": 3.3794749022069954, + "tokens_seen": 2670002176 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019278606965174132, + "loss": 1.2326, + "theoretical_loss": 3.379461952970919, + "tokens_seen": 2670133248 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019274594768095007, + "loss": 1.297, + "theoretical_loss": 3.3794490045484546, + "tokens_seen": 2670264320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019270582571015888, + "loss": 1.1892, + "theoretical_loss": 3.3794360569395114, + "tokens_seen": 2670395392 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019266570373936768, + "loss": 1.2508, + "theoretical_loss": 3.3794231101439984, + "tokens_seen": 2670526464 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019262558176857646, + "loss": 1.2249, + "theoretical_loss": 3.3794101641618246, + "tokens_seen": 2670657536 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019258545979778527, + "loss": 1.2907, + "theoretical_loss": 3.379397218992899, + "tokens_seen": 2670788608 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019254533782699407, + "loss": 1.1958, + "theoretical_loss": 3.3793842746371303, + "tokens_seen": 2670919680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019250521585620288, + "loss": 1.2076, + "theoretical_loss": 3.3793713310944278, + "tokens_seen": 2671050752 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019246509388541166, + "loss": 1.3044, + "theoretical_loss": 3.3793583883647003, + "tokens_seen": 2671181824 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019242497191462044, + "loss": 1.2851, + "theoretical_loss": 3.379345446447857, + "tokens_seen": 2671312896 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019238484994382925, + "loss": 1.2716, + "theoretical_loss": 3.3793325053438075, + "tokens_seen": 2671443968 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019234472797303802, + "loss": 1.2978, + "theoretical_loss": 3.3793195650524597, + "tokens_seen": 2671575040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019230460600224683, + "loss": 1.22, + "theoretical_loss": 3.3793066255737236, + "tokens_seen": 2671706112 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019226448403145564, + "loss": 1.2801, + "theoretical_loss": 3.3792936869075083, + "tokens_seen": 2671837184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019222436206066444, + "loss": 1.2665, + "theoretical_loss": 3.379280749053722, + "tokens_seen": 2671968256 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019218424008987322, + "loss": 1.2101, + "theoretical_loss": 3.379267812012275, + "tokens_seen": 2672099328 + }, + { + "epoch": 0.62, + "learning_rate": 0.000192144118119082, + "loss": 1.3031, + "theoretical_loss": 3.3792548757830763, + "tokens_seen": 2672230400 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.9920362830162048, + "objective/train/avg_token_score": 0.007963757030665874, + "objective/train/docs_used": 970931, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.238418698310852, + "objective/train/original_loss": 1.2501087188720703, + "objective/train/theoretical_loss": 3.3792419403660343, + "objective/train/tokens_used": 1042820576, + "objective/train/unlikelihood_loss": 0.02000274322926998, + "theoretical_loss": 3.3792419403660343, + "tokens_seen": 2672361472 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921039961482908, + "loss": 1.2269, + "theoretical_loss": 3.3792419403660343, + "tokens_seen": 2672361472 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001920638741774996, + "loss": 1.2773, + "theoretical_loss": 3.379229005761059, + "tokens_seen": 2672492544 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001920237522067084, + "loss": 1.2552, + "theoretical_loss": 3.3792160719680586, + "tokens_seen": 2672623616 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001919836302359172, + "loss": 1.2808, + "theoretical_loss": 3.3792031389869432, + "tokens_seen": 2672754688 + }, + { + "epoch": 0.62, + "learning_rate": 0.000191943508265126, + "loss": 1.2475, + "theoretical_loss": 3.379190206817622, + "tokens_seen": 2672885760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019190338629433479, + "loss": 1.2224, + "theoretical_loss": 3.3791772754600036, + "tokens_seen": 2673016832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019186326432354356, + "loss": 1.3292, + "theoretical_loss": 3.379164344913998, + "tokens_seen": 2673147904 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019182314235275237, + "loss": 1.2404, + "theoretical_loss": 3.3791514151795137, + "tokens_seen": 2673278976 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019178302038196115, + "loss": 1.3365, + "theoretical_loss": 3.3791384862564606, + "tokens_seen": 2673410048 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019174289841116996, + "loss": 1.314, + "theoretical_loss": 3.379125558144748, + "tokens_seen": 2673541120 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019170277644037876, + "loss": 1.2722, + "theoretical_loss": 3.3791126308442845, + "tokens_seen": 2673672192 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019166265446958754, + "loss": 1.268, + "theoretical_loss": 3.3790997043549806, + "tokens_seen": 2673803264 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019162253249879635, + "loss": 1.2601, + "theoretical_loss": 3.3790867786767445, + "tokens_seen": 2673934336 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019158241052800515, + "loss": 1.2493, + "theoretical_loss": 3.379073853809486, + "tokens_seen": 2674065408 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019154228855721393, + "loss": 1.2902, + "theoretical_loss": 3.379060929753115, + "tokens_seen": 2674196480 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001915021665864227, + "loss": 1.3063, + "theoretical_loss": 3.37904800650754, + "tokens_seen": 2674327552 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019146204461563152, + "loss": 1.2825, + "theoretical_loss": 3.3790350840726706, + "tokens_seen": 2674458624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019142192264484033, + "loss": 1.2604, + "theoretical_loss": 3.3790221624484165, + "tokens_seen": 2674589696 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001913818006740491, + "loss": 1.2829, + "theoretical_loss": 3.3790092416346873, + "tokens_seen": 2674720768 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001913416787032579, + "loss": 1.1829, + "theoretical_loss": 3.378996321631392, + "tokens_seen": 2674851840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019130155673246672, + "loss": 1.2849, + "theoretical_loss": 3.37898340243844, + "tokens_seen": 2674982912 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001912614347616755, + "loss": 1.1583, + "theoretical_loss": 3.3789704840557415, + "tokens_seen": 2675113984 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019122131279088428, + "loss": 1.221, + "theoretical_loss": 3.378957566483205, + "tokens_seen": 2675245056 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019118119082009308, + "loss": 1.2533, + "theoretical_loss": 3.378944649720741, + "tokens_seen": 2675376128 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001911410688493019, + "loss": 1.2335, + "theoretical_loss": 3.3789317337682583, + "tokens_seen": 2675507200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019110094687851067, + "loss": 1.2485, + "theoretical_loss": 3.3789188186256665, + "tokens_seen": 2675638272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019106082490771947, + "loss": 1.2571, + "theoretical_loss": 3.3789059042928753, + "tokens_seen": 2675769344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019102070293692828, + "loss": 1.2803, + "theoretical_loss": 3.378892990769794, + "tokens_seen": 2675900416 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001909805809661371, + "loss": 1.3174, + "theoretical_loss": 3.378880078056333, + "tokens_seen": 2676031488 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019094045899534584, + "loss": 1.2661, + "theoretical_loss": 3.3788671661524012, + "tokens_seen": 2676162560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019090033702455465, + "loss": 1.3225, + "theoretical_loss": 3.378854255057908, + "tokens_seen": 2676293632 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019086021505376345, + "loss": 1.2331, + "theoretical_loss": 3.3788413447727637, + "tokens_seen": 2676424704 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019082009308297223, + "loss": 1.264, + "theoretical_loss": 3.378828435296877, + "tokens_seen": 2676555776 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019077997111218104, + "loss": 1.2169, + "theoretical_loss": 3.3788155266301585, + "tokens_seen": 2676686848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019073984914138984, + "loss": 1.2862, + "theoretical_loss": 3.3788026187725175, + "tokens_seen": 2676817920 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019069972717059862, + "loss": 1.2465, + "theoretical_loss": 3.3787897117238637, + "tokens_seen": 2676948992 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001906596051998074, + "loss": 1.2805, + "theoretical_loss": 3.3787768054841067, + "tokens_seen": 2677080064 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001906194832290162, + "loss": 1.2225, + "theoretical_loss": 3.378763900053156, + "tokens_seen": 2677211136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019057936125822501, + "loss": 1.1761, + "theoretical_loss": 3.378750995430922, + "tokens_seen": 2677342208 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001905392392874338, + "loss": 1.2112, + "theoretical_loss": 3.3787380916173135, + "tokens_seen": 2677473280 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904991173166426, + "loss": 1.2863, + "theoretical_loss": 3.3787251886122407, + "tokens_seen": 2677604352 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904589953458514, + "loss": 1.2169, + "theoretical_loss": 3.378712286415614, + "tokens_seen": 2677735424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019041887337506019, + "loss": 1.1766, + "theoretical_loss": 3.3786993850273426, + "tokens_seen": 2677866496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019037875140426896, + "loss": 1.2457, + "theoretical_loss": 3.3786864844473357, + "tokens_seen": 2677997568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019033862943347777, + "loss": 1.2702, + "theoretical_loss": 3.3786735846755036, + "tokens_seen": 2678128640 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019029850746268658, + "loss": 1.3223, + "theoretical_loss": 3.3786606857117567, + "tokens_seen": 2678259712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019025838549189536, + "loss": 1.3314, + "theoretical_loss": 3.3786477875560044, + "tokens_seen": 2678390784 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019021826352110416, + "loss": 1.2588, + "theoretical_loss": 3.378634890208156, + "tokens_seen": 2678521856 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019017814155031297, + "loss": 1.2726, + "theoretical_loss": 3.3786219936681223, + "tokens_seen": 2678652928 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019013801957952175, + "loss": 1.2834, + "theoretical_loss": 3.3786090979358123, + "tokens_seen": 2678784000 + }, + { + "epoch": 0.62, + "objective/train/aligned_token_count": 0.9920907020568848, + "objective/train/avg_token_score": 0.007909329608082771, + "objective/train/docs_used": 973380, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2642393112182617, + "objective/train/original_loss": 1.274808645248413, + "objective/train/theoretical_loss": 3.3785962030111367, + "objective/train/tokens_used": 1049374176, + "objective/train/unlikelihood_loss": 0.02184380404651165, + "theoretical_loss": 3.3785962030111367, + "tokens_seen": 2678915072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019009789760873055, + "loss": 1.2783, + "theoretical_loss": 3.3785962030111367, + "tokens_seen": 2678915072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019005777563793933, + "loss": 1.2121, + "theoretical_loss": 3.378583308894005, + "tokens_seen": 2679046144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019001765366714814, + "loss": 1.2174, + "theoretical_loss": 3.3785704155843272, + "tokens_seen": 2679177216 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018997753169635692, + "loss": 1.241, + "theoretical_loss": 3.378557523082013, + "tokens_seen": 2679308288 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018993740972556573, + "loss": 1.2545, + "theoretical_loss": 3.3785446313869727, + "tokens_seen": 2679439360 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018989728775477453, + "loss": 1.2309, + "theoretical_loss": 3.378531740499116, + "tokens_seen": 2679570432 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001898571657839833, + "loss": 1.2355, + "theoretical_loss": 3.378518850418353, + "tokens_seen": 2679701504 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018981704381319212, + "loss": 1.3168, + "theoretical_loss": 3.378505961144594, + "tokens_seen": 2679832576 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001897769218424009, + "loss": 1.1578, + "theoretical_loss": 3.378493072677748, + "tokens_seen": 2679963648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018973679987160968, + "loss": 1.2956, + "theoretical_loss": 3.378480185017726, + "tokens_seen": 2680094720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018969667790081848, + "loss": 1.366, + "theoretical_loss": 3.3784672981644386, + "tokens_seen": 2680225792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001896565559300273, + "loss": 1.2813, + "theoretical_loss": 3.378454412117794, + "tokens_seen": 2680356864 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001896164339592361, + "loss": 1.3069, + "theoretical_loss": 3.3784415268777037, + "tokens_seen": 2680487936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018957631198844487, + "loss": 1.2483, + "theoretical_loss": 3.378428642444077, + "tokens_seen": 2680619008 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018953619001765368, + "loss": 1.221, + "theoretical_loss": 3.378415758816825, + "tokens_seen": 2680750080 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018949606804686246, + "loss": 1.2696, + "theoretical_loss": 3.3784028759958566, + "tokens_seen": 2680881152 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018945594607607124, + "loss": 1.2354, + "theoretical_loss": 3.3783899939810826, + "tokens_seen": 2681012224 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018941582410528004, + "loss": 1.2313, + "theoretical_loss": 3.378377112772413, + "tokens_seen": 2681143296 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018937570213448885, + "loss": 1.2455, + "theoretical_loss": 3.3783642323697585, + "tokens_seen": 2681274368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018933558016369766, + "loss": 1.2533, + "theoretical_loss": 3.378351352773028, + "tokens_seen": 2681405440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018929545819290644, + "loss": 1.2803, + "theoretical_loss": 3.378338473982133, + "tokens_seen": 2681536512 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018925533622211524, + "loss": 1.2441, + "theoretical_loss": 3.3783255959969827, + "tokens_seen": 2681667584 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018921521425132405, + "loss": 1.221, + "theoretical_loss": 3.378312718817488, + "tokens_seen": 2681798656 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891750922805328, + "loss": 1.2643, + "theoretical_loss": 3.3782998424435586, + "tokens_seen": 2681929728 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891349703097416, + "loss": 1.2436, + "theoretical_loss": 3.3782869668751054, + "tokens_seen": 2682060800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018909484833895041, + "loss": 1.3098, + "theoretical_loss": 3.3782740921120378, + "tokens_seen": 2682191872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018905472636815922, + "loss": 1.2847, + "theoretical_loss": 3.378261218154267, + "tokens_seen": 2682322944 + }, + { + "epoch": 0.63, + "learning_rate": 0.000189014604397368, + "loss": 1.1922, + "theoretical_loss": 3.3782483450017025, + "tokens_seen": 2682454016 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001889744824265768, + "loss": 1.3117, + "theoretical_loss": 3.378235472654255, + "tokens_seen": 2682585088 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001889343604557856, + "loss": 1.1883, + "theoretical_loss": 3.3782226011118346, + "tokens_seen": 2682716160 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018889423848499436, + "loss": 1.1803, + "theoretical_loss": 3.3782097303743517, + "tokens_seen": 2682847232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018885411651420317, + "loss": 1.2872, + "theoretical_loss": 3.378196860441717, + "tokens_seen": 2682978304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018881399454341198, + "loss": 1.2652, + "theoretical_loss": 3.3781839913138403, + "tokens_seen": 2683109376 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018877387257262076, + "loss": 1.3128, + "theoretical_loss": 3.378171122990632, + "tokens_seen": 2683240448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018873375060182956, + "loss": 1.2511, + "theoretical_loss": 3.378158255472003, + "tokens_seen": 2683371520 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018869362863103837, + "loss": 1.266, + "theoretical_loss": 3.378145388757863, + "tokens_seen": 2683502592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018865350666024717, + "loss": 1.2541, + "theoretical_loss": 3.378132522848124, + "tokens_seen": 2683633664 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018861338468945595, + "loss": 1.2779, + "theoretical_loss": 3.378119657742694, + "tokens_seen": 2683764736 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018857326271866473, + "loss": 1.2922, + "theoretical_loss": 3.378106793441485, + "tokens_seen": 2683895808 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018853314074787354, + "loss": 1.2112, + "theoretical_loss": 3.378093929944407, + "tokens_seen": 2684026880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018849301877708232, + "loss": 1.2888, + "theoretical_loss": 3.378081067251371, + "tokens_seen": 2684157952 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018845289680629113, + "loss": 1.2405, + "theoretical_loss": 3.3780682053622866, + "tokens_seen": 2684289024 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018841277483549993, + "loss": 1.2841, + "theoretical_loss": 3.378055344277065, + "tokens_seen": 2684420096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018837265286470874, + "loss": 1.2698, + "theoretical_loss": 3.3780424839956167, + "tokens_seen": 2684551168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018833253089391752, + "loss": 1.311, + "theoretical_loss": 3.3780296245178514, + "tokens_seen": 2684682240 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001882924089231263, + "loss": 1.2795, + "theoretical_loss": 3.3780167658436806, + "tokens_seen": 2684813312 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001882522869523351, + "loss": 1.2742, + "theoretical_loss": 3.3780039079730146, + "tokens_seen": 2684944384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018821216498154388, + "loss": 1.1801, + "theoretical_loss": 3.377991050905764, + "tokens_seen": 2685075456 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001881720430107527, + "loss": 1.2417, + "theoretical_loss": 3.3779781946418392, + "tokens_seen": 2685206528 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001881319210399615, + "loss": 1.2699, + "theoretical_loss": 3.3779653391811504, + "tokens_seen": 2685337600 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.9927713871002197, + "objective/train/avg_token_score": 0.0072286115027964115, + "objective/train/docs_used": 975809, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1737419366836548, + "objective/train/original_loss": 1.181715965270996, + "objective/train/theoretical_loss": 3.3779524845236093, + "objective/train/tokens_used": 1055927776, + "objective/train/unlikelihood_loss": 0.026112541556358337, + "theoretical_loss": 3.3779524845236093, + "tokens_seen": 2685468672 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001880917990691703, + "loss": 1.1729, + "theoretical_loss": 3.3779524845236093, + "tokens_seen": 2685468672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018805167709837908, + "loss": 1.2463, + "theoretical_loss": 3.3779396306691254, + "tokens_seen": 2685599744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018801155512758786, + "loss": 1.3517, + "theoretical_loss": 3.3779267776176103, + "tokens_seen": 2685730816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018797143315679667, + "loss": 1.2456, + "theoretical_loss": 3.377913925368974, + "tokens_seen": 2685861888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018793131118600544, + "loss": 1.2719, + "theoretical_loss": 3.377901073923127, + "tokens_seen": 2685992960 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018789118921521425, + "loss": 1.225, + "theoretical_loss": 3.3778882232799807, + "tokens_seen": 2686124032 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018785106724442306, + "loss": 1.2587, + "theoretical_loss": 3.3778753734394455, + "tokens_seen": 2686255104 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018781094527363186, + "loss": 1.334, + "theoretical_loss": 3.377862524401432, + "tokens_seen": 2686386176 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018777082330284064, + "loss": 1.2258, + "theoretical_loss": 3.377849676165851, + "tokens_seen": 2686517248 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018773070133204945, + "loss": 1.2082, + "theoretical_loss": 3.3778368287326135, + "tokens_seen": 2686648320 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018769057936125823, + "loss": 1.2364, + "theoretical_loss": 3.3778239821016296, + "tokens_seen": 2686779392 + }, + { + "epoch": 0.63, + "learning_rate": 0.000187650457390467, + "loss": 1.3343, + "theoretical_loss": 3.3778111362728103, + "tokens_seen": 2686910464 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018761033541967581, + "loss": 1.2212, + "theoretical_loss": 3.377798291246067, + "tokens_seen": 2687041536 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018757021344888462, + "loss": 1.2804, + "theoretical_loss": 3.37778544702131, + "tokens_seen": 2687172608 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001875300914780934, + "loss": 1.2427, + "theoretical_loss": 3.3777726035984497, + "tokens_seen": 2687303680 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001874899695073022, + "loss": 1.1872, + "theoretical_loss": 3.3777597609773977, + "tokens_seen": 2687434752 + }, + { + "epoch": 0.63, + "learning_rate": 0.000187449847536511, + "loss": 1.3163, + "theoretical_loss": 3.3777469191580645, + "tokens_seen": 2687565824 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001874097255657198, + "loss": 1.3289, + "theoretical_loss": 3.377734078140361, + "tokens_seen": 2687696896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018736960359492857, + "loss": 1.2704, + "theoretical_loss": 3.377721237924198, + "tokens_seen": 2687827968 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018732948162413738, + "loss": 1.1716, + "theoretical_loss": 3.3777083985094865, + "tokens_seen": 2687959040 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018728935965334618, + "loss": 1.2412, + "theoretical_loss": 3.3776955598961376, + "tokens_seen": 2688090112 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018724923768255496, + "loss": 1.2461, + "theoretical_loss": 3.377682722084061, + "tokens_seen": 2688221184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018720911571176377, + "loss": 1.265, + "theoretical_loss": 3.3776698850731695, + "tokens_seen": 2688352256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018716899374097257, + "loss": 1.3423, + "theoretical_loss": 3.377657048863373, + "tokens_seen": 2688483328 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018712887177018135, + "loss": 1.2506, + "theoretical_loss": 3.377644213454582, + "tokens_seen": 2688614400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018708874979939013, + "loss": 1.3082, + "theoretical_loss": 3.3776313788467087, + "tokens_seen": 2688745472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018704862782859894, + "loss": 1.1883, + "theoretical_loss": 3.377618545039663, + "tokens_seen": 2688876544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018700850585780775, + "loss": 1.3438, + "theoretical_loss": 3.3776057120333567, + "tokens_seen": 2689007616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018696838388701652, + "loss": 1.188, + "theoretical_loss": 3.3775928798277, + "tokens_seen": 2689138688 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018692826191622533, + "loss": 1.3303, + "theoretical_loss": 3.377580048422604, + "tokens_seen": 2689269760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018688813994543414, + "loss": 1.1808, + "theoretical_loss": 3.377567217817981, + "tokens_seen": 2689400832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018684801797464294, + "loss": 1.251, + "theoretical_loss": 3.3775543880137406, + "tokens_seen": 2689531904 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001868078960038517, + "loss": 1.2053, + "theoretical_loss": 3.377541559009795, + "tokens_seen": 2689662976 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867677740330605, + "loss": 1.1949, + "theoretical_loss": 3.3775287308060538, + "tokens_seen": 2689794048 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867276520622693, + "loss": 1.267, + "theoretical_loss": 3.3775159034024296, + "tokens_seen": 2689925120 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001866875300914781, + "loss": 1.2728, + "theoretical_loss": 3.377503076798833, + "tokens_seen": 2690056192 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001866474081206869, + "loss": 1.3123, + "theoretical_loss": 3.3774902509951747, + "tokens_seen": 2690187264 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001866072861498957, + "loss": 1.2734, + "theoretical_loss": 3.3774774259913665, + "tokens_seen": 2690318336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018656716417910448, + "loss": 1.3474, + "theoretical_loss": 3.377464601787319, + "tokens_seen": 2690449408 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018652704220831326, + "loss": 1.2564, + "theoretical_loss": 3.3774517783829436, + "tokens_seen": 2690580480 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018648692023752206, + "loss": 1.2794, + "theoretical_loss": 3.3774389557781515, + "tokens_seen": 2690711552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018644679826673087, + "loss": 1.3102, + "theoretical_loss": 3.377426133972854, + "tokens_seen": 2690842624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018640667629593965, + "loss": 1.2983, + "theoretical_loss": 3.377413312966962, + "tokens_seen": 2690973696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018636655432514846, + "loss": 1.2352, + "theoretical_loss": 3.377400492760387, + "tokens_seen": 2691104768 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018632643235435726, + "loss": 1.272, + "theoretical_loss": 3.37738767335304, + "tokens_seen": 2691235840 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018628631038356604, + "loss": 1.2749, + "theoretical_loss": 3.377374854744833, + "tokens_seen": 2691366912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018624618841277485, + "loss": 1.3343, + "theoretical_loss": 3.377362036935676, + "tokens_seen": 2691497984 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018620606644198363, + "loss": 1.288, + "theoretical_loss": 3.377349219925481, + "tokens_seen": 2691629056 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018616594447119243, + "loss": 1.2856, + "theoretical_loss": 3.3773364037141596, + "tokens_seen": 2691760128 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001861258225004012, + "loss": 1.2593, + "theoretical_loss": 3.377323588301623, + "tokens_seen": 2691891200 + }, + { + "epoch": 0.63, + "objective/train/aligned_token_count": 0.9927344918251038, + "objective/train/avg_token_score": 0.007265455089509487, + "objective/train/docs_used": 978239, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2158987522125244, + "objective/train/original_loss": 1.2265266180038452, + "objective/train/theoretical_loss": 3.377310773687782, + "objective/train/tokens_used": 1062481376, + "objective/train/unlikelihood_loss": 0.018020931631326675, + "theoretical_loss": 3.377310773687782, + "tokens_seen": 2692022272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018608570052961002, + "loss": 1.2403, + "theoretical_loss": 3.377310773687782, + "tokens_seen": 2692022272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018604557855881883, + "loss": 1.2732, + "theoretical_loss": 3.377297959872548, + "tokens_seen": 2692153344 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001860054565880276, + "loss": 1.2679, + "theoretical_loss": 3.377285146855833, + "tokens_seen": 2692284416 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001859653346172364, + "loss": 1.2918, + "theoretical_loss": 3.3772723346375475, + "tokens_seen": 2692415488 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001859252126464452, + "loss": 1.3095, + "theoretical_loss": 3.3772595232176035, + "tokens_seen": 2692546560 + }, + { + "epoch": 0.63, + "learning_rate": 0.000185885090675654, + "loss": 1.307, + "theoretical_loss": 3.3772467125959125, + "tokens_seen": 2692677632 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018584496870486278, + "loss": 1.2853, + "theoretical_loss": 3.377233902772385, + "tokens_seen": 2692808704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018580484673407158, + "loss": 1.3792, + "theoretical_loss": 3.377221093746934, + "tokens_seen": 2692939776 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001857647247632804, + "loss": 1.2591, + "theoretical_loss": 3.377208285519469, + "tokens_seen": 2693070848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018572460279248917, + "loss": 1.256, + "theoretical_loss": 3.377195478089903, + "tokens_seen": 2693201920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018568448082169797, + "loss": 1.2998, + "theoretical_loss": 3.377182671458147, + "tokens_seen": 2693332992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018564435885090675, + "loss": 1.2946, + "theoretical_loss": 3.3771698656241123, + "tokens_seen": 2693464064 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018560423688011553, + "loss": 1.2232, + "theoretical_loss": 3.3771570605877104, + "tokens_seen": 2693595136 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018556411490932434, + "loss": 1.3804, + "theoretical_loss": 3.3771442563488527, + "tokens_seen": 2693726208 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018552399293853315, + "loss": 1.2344, + "theoretical_loss": 3.3771314529074514, + "tokens_seen": 2693857280 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018548387096774195, + "loss": 1.2603, + "theoretical_loss": 3.377118650263417, + "tokens_seen": 2693988352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018544374899695073, + "loss": 1.2546, + "theoretical_loss": 3.377105848416662, + "tokens_seen": 2694119424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018540362702615954, + "loss": 1.2706, + "theoretical_loss": 3.3770930473670977, + "tokens_seen": 2694250496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018536350505536834, + "loss": 1.3073, + "theoretical_loss": 3.3770802471146353, + "tokens_seen": 2694381568 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853233830845771, + "loss": 1.3202, + "theoretical_loss": 3.3770674476591864, + "tokens_seen": 2694512640 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001852832611137859, + "loss": 1.3267, + "theoretical_loss": 3.377054649000663, + "tokens_seen": 2694643712 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001852431391429947, + "loss": 1.2431, + "theoretical_loss": 3.3770418511389773, + "tokens_seen": 2694774784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018520301717220351, + "loss": 1.2361, + "theoretical_loss": 3.3770290540740393, + "tokens_seen": 2694905856 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001851628952014123, + "loss": 1.2387, + "theoretical_loss": 3.377016257805762, + "tokens_seen": 2695036928 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001851227732306211, + "loss": 1.315, + "theoretical_loss": 3.3770034623340566, + "tokens_seen": 2695168000 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001850826512598299, + "loss": 1.2616, + "theoretical_loss": 3.3769906676588346, + "tokens_seen": 2695299072 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018504252928903866, + "loss": 1.2141, + "theoretical_loss": 3.376977873780008, + "tokens_seen": 2695430144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018500240731824746, + "loss": 1.2879, + "theoretical_loss": 3.3769650806974885, + "tokens_seen": 2695561216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018496228534745627, + "loss": 1.1847, + "theoretical_loss": 3.3769522884111876, + "tokens_seen": 2695692288 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018492216337666508, + "loss": 1.2606, + "theoretical_loss": 3.3769394969210174, + "tokens_seen": 2695823360 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018488204140587386, + "loss": 1.2138, + "theoretical_loss": 3.3769267062268886, + "tokens_seen": 2695954432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018484191943508266, + "loss": 1.2077, + "theoretical_loss": 3.3769139163287147, + "tokens_seen": 2696085504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018480179746429147, + "loss": 1.2684, + "theoretical_loss": 3.376901127226406, + "tokens_seen": 2696216576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018476167549350022, + "loss": 1.3493, + "theoretical_loss": 3.376888338919875, + "tokens_seen": 2696347648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018472155352270903, + "loss": 1.2855, + "theoretical_loss": 3.376875551409033, + "tokens_seen": 2696478720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018468143155191783, + "loss": 1.2664, + "theoretical_loss": 3.3768627646937928, + "tokens_seen": 2696609792 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001846413095811266, + "loss": 1.2646, + "theoretical_loss": 3.3768499787740653, + "tokens_seen": 2696740864 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018460118761033542, + "loss": 1.2428, + "theoretical_loss": 3.3768371936497625, + "tokens_seen": 2696871936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018456106563954423, + "loss": 1.2828, + "theoretical_loss": 3.376824409320797, + "tokens_seen": 2697003008 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018452094366875303, + "loss": 1.233, + "theoretical_loss": 3.376811625787079, + "tokens_seen": 2697134080 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001844808216979618, + "loss": 1.2364, + "theoretical_loss": 3.3767988430485225, + "tokens_seen": 2697265152 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001844406997271706, + "loss": 1.2291, + "theoretical_loss": 3.3767860611050375, + "tokens_seen": 2697396224 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001844005777563794, + "loss": 1.2397, + "theoretical_loss": 3.376773279956537, + "tokens_seen": 2697527296 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018436045578558818, + "loss": 1.2124, + "theoretical_loss": 3.3767604996029332, + "tokens_seen": 2697658368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018432033381479698, + "loss": 1.2877, + "theoretical_loss": 3.376747720044137, + "tokens_seen": 2697789440 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001842802118440058, + "loss": 1.2218, + "theoretical_loss": 3.3767349412800614, + "tokens_seen": 2697920512 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001842400898732146, + "loss": 1.2873, + "theoretical_loss": 3.3767221633106175, + "tokens_seen": 2698051584 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018419996790242337, + "loss": 1.2799, + "theoretical_loss": 3.3767093861357176, + "tokens_seen": 2698182656 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018415984593163215, + "loss": 1.2247, + "theoretical_loss": 3.3766966097552737, + "tokens_seen": 2698313728 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018411972396084096, + "loss": 1.1704, + "theoretical_loss": 3.3766838341691985, + "tokens_seen": 2698444800 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.9915187358856201, + "objective/train/avg_token_score": 0.008481224998831749, + "objective/train/docs_used": 980656, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3968778848648071, + "objective/train/original_loss": 1.410049557685852, + "objective/train/theoretical_loss": 3.376671059377403, + "objective/train/tokens_used": 1069034976, + "objective/train/unlikelihood_loss": 0.019678620621562004, + "theoretical_loss": 3.376671059377403, + "tokens_seen": 2698575872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018407960199004974, + "loss": 1.3223, + "theoretical_loss": 3.376671059377403, + "tokens_seen": 2698575872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018403948001925854, + "loss": 1.216, + "theoretical_loss": 3.3766582853797997, + "tokens_seen": 2698706944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018399935804846735, + "loss": 1.3261, + "theoretical_loss": 3.3766455121763004, + "tokens_seen": 2698838016 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018395923607767616, + "loss": 1.2944, + "theoretical_loss": 3.376632739766818, + "tokens_seen": 2698969088 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018391911410688494, + "loss": 1.3347, + "theoretical_loss": 3.376619968151263, + "tokens_seen": 2699100160 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018387899213609374, + "loss": 1.2353, + "theoretical_loss": 3.376607197329549, + "tokens_seen": 2699231232 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018383887016530252, + "loss": 1.3379, + "theoretical_loss": 3.3765944273015878, + "tokens_seen": 2699362304 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001837987481945113, + "loss": 1.2699, + "theoretical_loss": 3.376581658067291, + "tokens_seen": 2699493376 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001837586262237201, + "loss": 1.2416, + "theoretical_loss": 3.3765688896265713, + "tokens_seen": 2699624448 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018371850425292891, + "loss": 1.3094, + "theoretical_loss": 3.3765561219793403, + "tokens_seen": 2699755520 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018367838228213772, + "loss": 1.2273, + "theoretical_loss": 3.376543355125511, + "tokens_seen": 2699886592 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001836382603113465, + "loss": 1.2485, + "theoretical_loss": 3.376530589064995, + "tokens_seen": 2700017664 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001835981383405553, + "loss": 1.2316, + "theoretical_loss": 3.376517823797704, + "tokens_seen": 2700148736 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018355801636976409, + "loss": 1.2602, + "theoretical_loss": 3.376505059323551, + "tokens_seen": 2700279808 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018351789439897286, + "loss": 1.2559, + "theoretical_loss": 3.3764922956424477, + "tokens_seen": 2700410880 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018347777242818167, + "loss": 1.2897, + "theoretical_loss": 3.376479532754307, + "tokens_seen": 2700541952 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018343765045739048, + "loss": 1.3001, + "theoretical_loss": 3.3764667706590408, + "tokens_seen": 2700673024 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018339752848659926, + "loss": 1.3302, + "theoretical_loss": 3.376454009356561, + "tokens_seen": 2700804096 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018335740651580806, + "loss": 1.2908, + "theoretical_loss": 3.3764412488467808, + "tokens_seen": 2700935168 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018331728454501687, + "loss": 1.3061, + "theoretical_loss": 3.3764284891296117, + "tokens_seen": 2701066240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018327716257422565, + "loss": 1.1898, + "theoretical_loss": 3.376415730204966, + "tokens_seen": 2701197312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018323704060343443, + "loss": 1.3608, + "theoretical_loss": 3.3764029720727566, + "tokens_seen": 2701328384 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018319691863264323, + "loss": 1.2621, + "theoretical_loss": 3.3763902147328952, + "tokens_seen": 2701459456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018315679666185204, + "loss": 1.2907, + "theoretical_loss": 3.3763774581852948, + "tokens_seen": 2701590528 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018311667469106082, + "loss": 1.2923, + "theoretical_loss": 3.376364702429867, + "tokens_seen": 2701721600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307655272026963, + "loss": 1.3158, + "theoretical_loss": 3.3763519474665245, + "tokens_seen": 2701852672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018303643074947843, + "loss": 1.2469, + "theoretical_loss": 3.37633919329518, + "tokens_seen": 2701983744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018299630877868724, + "loss": 1.2736, + "theoretical_loss": 3.3763264399157458, + "tokens_seen": 2702114816 + }, + { + "epoch": 0.64, + "learning_rate": 0.000182956186807896, + "loss": 1.2263, + "theoretical_loss": 3.376313687328134, + "tokens_seen": 2702245888 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001829160648371048, + "loss": 1.2504, + "theoretical_loss": 3.3763009355322575, + "tokens_seen": 2702376960 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001828759428663136, + "loss": 1.24, + "theoretical_loss": 3.376288184528028, + "tokens_seen": 2702508032 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018283582089552238, + "loss": 1.2816, + "theoretical_loss": 3.3762754343153594, + "tokens_seen": 2702639104 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001827956989247312, + "loss": 1.2762, + "theoretical_loss": 3.3762626848941624, + "tokens_seen": 2702770176 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018275557695394, + "loss": 1.3253, + "theoretical_loss": 3.3762499362643505, + "tokens_seen": 2702901248 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001827154549831488, + "loss": 1.2942, + "theoretical_loss": 3.376237188425836, + "tokens_seen": 2703032320 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018267533301235755, + "loss": 1.2331, + "theoretical_loss": 3.3762244413785316, + "tokens_seen": 2703163392 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018263521104156636, + "loss": 1.2806, + "theoretical_loss": 3.3762116951223495, + "tokens_seen": 2703294464 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018259508907077517, + "loss": 1.2247, + "theoretical_loss": 3.3761989496572022, + "tokens_seen": 2703425536 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018255496709998394, + "loss": 1.2997, + "theoretical_loss": 3.376186204983003, + "tokens_seen": 2703556608 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018251484512919275, + "loss": 1.2026, + "theoretical_loss": 3.376173461099664, + "tokens_seen": 2703687680 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018247472315840156, + "loss": 1.2445, + "theoretical_loss": 3.376160718007097, + "tokens_seen": 2703818752 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018243460118761034, + "loss": 1.3285, + "theoretical_loss": 3.3761479757052157, + "tokens_seen": 2703949824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018239447921681912, + "loss": 1.3246, + "theoretical_loss": 3.3761352341939324, + "tokens_seen": 2704080896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018235435724602792, + "loss": 1.2995, + "theoretical_loss": 3.3761224934731597, + "tokens_seen": 2704211968 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018231423527523673, + "loss": 1.3239, + "theoretical_loss": 3.3761097535428104, + "tokens_seen": 2704343040 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001822741133044455, + "loss": 1.3451, + "theoretical_loss": 3.376097014402797, + "tokens_seen": 2704474112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018223399133365431, + "loss": 1.2568, + "theoretical_loss": 3.3760842760530316, + "tokens_seen": 2704605184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018219386936286312, + "loss": 1.3204, + "theoretical_loss": 3.376071538493428, + "tokens_seen": 2704736256 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001821537473920719, + "loss": 1.3066, + "theoretical_loss": 3.376058801723898, + "tokens_seen": 2704867328 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001821136254212807, + "loss": 1.2392, + "theoretical_loss": 3.3760460657443545, + "tokens_seen": 2704998400 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.9928800463676453, + "objective/train/avg_token_score": 0.007119883317500353, + "objective/train/docs_used": 983087, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4089837074279785, + "objective/train/original_loss": 1.4195728302001953, + "objective/train/theoretical_loss": 3.3760333305547108, + "objective/train/tokens_used": 1075588576, + "objective/train/unlikelihood_loss": 0.01759984903037548, + "theoretical_loss": 3.3760333305547108, + "tokens_seen": 2705129472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018207350345048948, + "loss": 1.3986, + "theoretical_loss": 3.3760333305547108, + "tokens_seen": 2705129472 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001820333814796983, + "loss": 1.2784, + "theoretical_loss": 3.3760205961548793, + "tokens_seen": 2705260544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018199325950890707, + "loss": 1.265, + "theoretical_loss": 3.376007862544772, + "tokens_seen": 2705391616 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018195313753811588, + "loss": 1.2593, + "theoretical_loss": 3.375995129724303, + "tokens_seen": 2705522688 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018191301556732468, + "loss": 1.227, + "theoretical_loss": 3.375982397693384, + "tokens_seen": 2705653760 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018187289359653346, + "loss": 1.225, + "theoretical_loss": 3.3759696664519288, + "tokens_seen": 2705784832 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018183277162574227, + "loss": 1.2913, + "theoretical_loss": 3.3759569359998487, + "tokens_seen": 2705915904 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018179264965495105, + "loss": 1.2984, + "theoretical_loss": 3.375944206337058, + "tokens_seen": 2706046976 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018175252768415985, + "loss": 1.2215, + "theoretical_loss": 3.375931477463469, + "tokens_seen": 2706178048 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018171240571336863, + "loss": 1.2553, + "theoretical_loss": 3.3759187493789944, + "tokens_seen": 2706309120 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018167228374257744, + "loss": 1.2429, + "theoretical_loss": 3.375906022083547, + "tokens_seen": 2706440192 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018163216177178625, + "loss": 1.2999, + "theoretical_loss": 3.37589329557704, + "tokens_seen": 2706571264 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018159203980099502, + "loss": 1.3362, + "theoretical_loss": 3.375880569859387, + "tokens_seen": 2706702336 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018155191783020383, + "loss": 1.3365, + "theoretical_loss": 3.3758678449304993, + "tokens_seen": 2706833408 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018151179585941264, + "loss": 1.3606, + "theoretical_loss": 3.3758551207902903, + "tokens_seen": 2706964480 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001814716738886214, + "loss": 1.2958, + "theoretical_loss": 3.3758423974386735, + "tokens_seen": 2707095552 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001814315519178302, + "loss": 1.384, + "theoretical_loss": 3.3758296748755616, + "tokens_seen": 2707226624 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181391429947039, + "loss": 1.2496, + "theoretical_loss": 3.375816953100868, + "tokens_seen": 2707357696 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001813513079762478, + "loss": 1.383, + "theoretical_loss": 3.3758042321145045, + "tokens_seen": 2707488768 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001813111860054566, + "loss": 1.3372, + "theoretical_loss": 3.3757915119163853, + "tokens_seen": 2707619840 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001812710640346654, + "loss": 1.2649, + "theoretical_loss": 3.3757787925064227, + "tokens_seen": 2707750912 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001812309420638742, + "loss": 1.2648, + "theoretical_loss": 3.3757660738845296, + "tokens_seen": 2707881984 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018119082009308295, + "loss": 1.2403, + "theoretical_loss": 3.3757533560506197, + "tokens_seen": 2708013056 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018115069812229176, + "loss": 1.2509, + "theoretical_loss": 3.375740639004606, + "tokens_seen": 2708144128 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018111057615150057, + "loss": 1.3649, + "theoretical_loss": 3.3757279227464005, + "tokens_seen": 2708275200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018107045418070937, + "loss": 1.3112, + "theoretical_loss": 3.375715207275918, + "tokens_seen": 2708406272 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018103033220991815, + "loss": 1.3455, + "theoretical_loss": 3.37570249259307, + "tokens_seen": 2708537344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018099021023912696, + "loss": 1.2752, + "theoretical_loss": 3.3756897786977698, + "tokens_seen": 2708668416 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018095008826833576, + "loss": 1.1997, + "theoretical_loss": 3.3756770655899313, + "tokens_seen": 2708799488 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018090996629754452, + "loss": 1.385, + "theoretical_loss": 3.3756643532694675, + "tokens_seen": 2708930560 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018086984432675332, + "loss": 1.3558, + "theoretical_loss": 3.375651641736291, + "tokens_seen": 2709061632 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018082972235596213, + "loss": 1.3521, + "theoretical_loss": 3.3756389309903154, + "tokens_seen": 2709192704 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018078960038517093, + "loss": 1.3293, + "theoretical_loss": 3.375626221031454, + "tokens_seen": 2709323776 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001807494784143797, + "loss": 1.3162, + "theoretical_loss": 3.375613511859619, + "tokens_seen": 2709454848 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018070935644358852, + "loss": 1.3252, + "theoretical_loss": 3.375600803474725, + "tokens_seen": 2709585920 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018066923447279733, + "loss": 1.367, + "theoretical_loss": 3.3755880958766844, + "tokens_seen": 2709716992 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001806291125020061, + "loss": 1.3, + "theoretical_loss": 3.37557538906541, + "tokens_seen": 2709848064 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018058899053121488, + "loss": 1.2442, + "theoretical_loss": 3.375562683040816, + "tokens_seen": 2709979136 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001805488685604237, + "loss": 1.2833, + "theoretical_loss": 3.375549977802815, + "tokens_seen": 2710110208 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018050874658963247, + "loss": 1.3265, + "theoretical_loss": 3.3755372733513207, + "tokens_seen": 2710241280 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018046862461884128, + "loss": 1.3059, + "theoretical_loss": 3.375524569686246, + "tokens_seen": 2710372352 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018042850264805008, + "loss": 1.2896, + "theoretical_loss": 3.3755118668075044, + "tokens_seen": 2710503424 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001803883806772589, + "loss": 1.2503, + "theoretical_loss": 3.375499164715009, + "tokens_seen": 2710634496 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018034825870646767, + "loss": 1.381, + "theoretical_loss": 3.3754864634086736, + "tokens_seen": 2710765568 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018030813673567645, + "loss": 1.3173, + "theoretical_loss": 3.3754737628884115, + "tokens_seen": 2710896640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018026801476488525, + "loss": 1.3169, + "theoretical_loss": 3.375461063154135, + "tokens_seen": 2711027712 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018022789279409403, + "loss": 1.2927, + "theoretical_loss": 3.3754483642057584, + "tokens_seen": 2711158784 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018018777082330284, + "loss": 1.1865, + "theoretical_loss": 3.3754356660431952, + "tokens_seen": 2711289856 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018014764885251165, + "loss": 1.3275, + "theoretical_loss": 3.375422968666358, + "tokens_seen": 2711420928 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018010752688172045, + "loss": 1.3931, + "theoretical_loss": 3.3754102720751615, + "tokens_seen": 2711552000 + }, + { + "epoch": 0.64, + "objective/train/aligned_token_count": 0.9941312074661255, + "objective/train/avg_token_score": 0.005868761800229549, + "objective/train/docs_used": 985372, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.28545343875885, + "objective/train/original_loss": 1.2954440116882324, + "objective/train/theoretical_loss": 3.375397576269518, + "objective/train/tokens_used": 1082142176, + "objective/train/unlikelihood_loss": 0.017718344926834106, + "theoretical_loss": 3.375397576269518, + "tokens_seen": 2711683072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018006740491092923, + "loss": 1.3013, + "theoretical_loss": 3.375397576269518, + "tokens_seen": 2711683072 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180027282940138, + "loss": 1.2616, + "theoretical_loss": 3.3753848812493406, + "tokens_seen": 2711814144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017998716096934682, + "loss": 1.2643, + "theoretical_loss": 3.375372187014544, + "tokens_seen": 2711945216 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001799470389985556, + "loss": 1.2726, + "theoretical_loss": 3.375359493565041, + "tokens_seen": 2712076288 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001799069170277644, + "loss": 1.2781, + "theoretical_loss": 3.375346800900745, + "tokens_seen": 2712207360 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001798667950569732, + "loss": 1.2996, + "theoretical_loss": 3.375334109021569, + "tokens_seen": 2712338432 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017982667308618201, + "loss": 1.305, + "theoretical_loss": 3.3753214179274282, + "tokens_seen": 2712469504 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001797865511153908, + "loss": 1.316, + "theoretical_loss": 3.3753087276182345, + "tokens_seen": 2712600576 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001797464291445996, + "loss": 1.45, + "theoretical_loss": 3.375296038093902, + "tokens_seen": 2712731648 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017970630717380838, + "loss": 1.2999, + "theoretical_loss": 3.3752833493543437, + "tokens_seen": 2712862720 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017966618520301716, + "loss": 1.3099, + "theoretical_loss": 3.3752706613994743, + "tokens_seen": 2712993792 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017962606323222596, + "loss": 1.244, + "theoretical_loss": 3.375257974229206, + "tokens_seen": 2713124864 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017958594126143477, + "loss": 1.2259, + "theoretical_loss": 3.3752452878434536, + "tokens_seen": 2713255936 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017954581929064358, + "loss": 1.3872, + "theoretical_loss": 3.37523260224213, + "tokens_seen": 2713387008 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017950569731985236, + "loss": 1.3284, + "theoretical_loss": 3.3752199174251487, + "tokens_seen": 2713518080 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017946557534906116, + "loss": 1.2292, + "theoretical_loss": 3.375207233392424, + "tokens_seen": 2713649152 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017942545337826994, + "loss": 1.4044, + "theoretical_loss": 3.375194550143869, + "tokens_seen": 2713780224 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017938533140747872, + "loss": 1.3309, + "theoretical_loss": 3.375181867679397, + "tokens_seen": 2713911296 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017934520943668753, + "loss": 1.2824, + "theoretical_loss": 3.375169185998923, + "tokens_seen": 2714042368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017930508746589633, + "loss": 1.264, + "theoretical_loss": 3.3751565051023595, + "tokens_seen": 2714173440 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001792649654951051, + "loss": 1.2702, + "theoretical_loss": 3.3751438249896206, + "tokens_seen": 2714304512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017922484352431392, + "loss": 1.2964, + "theoretical_loss": 3.3751311456606197, + "tokens_seen": 2714435584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017918472155352273, + "loss": 1.3871, + "theoretical_loss": 3.3751184671152705, + "tokens_seen": 2714566656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017914459958273153, + "loss": 1.305, + "theoretical_loss": 3.3751057893534875, + "tokens_seen": 2714697728 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017910447761194028, + "loss": 1.3308, + "theoretical_loss": 3.3750931123751835, + "tokens_seen": 2714828800 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001790643556411491, + "loss": 1.2806, + "theoretical_loss": 3.375080436180273, + "tokens_seen": 2714959872 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001790242336703579, + "loss": 1.2721, + "theoretical_loss": 3.375067760768669, + "tokens_seen": 2715090944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017898411169956668, + "loss": 1.3129, + "theoretical_loss": 3.3750550861402857, + "tokens_seen": 2715222016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017894398972877548, + "loss": 1.1614, + "theoretical_loss": 3.3750424122950373, + "tokens_seen": 2715353088 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001789038677579843, + "loss": 1.287, + "theoretical_loss": 3.3750297392328372, + "tokens_seen": 2715484160 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001788637457871931, + "loss": 1.2956, + "theoretical_loss": 3.3750170669535993, + "tokens_seen": 2715615232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017882362381640185, + "loss": 1.1643, + "theoretical_loss": 3.375004395457237, + "tokens_seen": 2715746304 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017878350184561065, + "loss": 1.2784, + "theoretical_loss": 3.3749917247436647, + "tokens_seen": 2715877376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017874337987481946, + "loss": 1.2396, + "theoretical_loss": 3.3749790548127963, + "tokens_seen": 2716008448 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017870325790402824, + "loss": 1.2499, + "theoretical_loss": 3.3749663856645453, + "tokens_seen": 2716139520 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017866313593323704, + "loss": 1.2768, + "theoretical_loss": 3.374953717298826, + "tokens_seen": 2716270592 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017862301396244585, + "loss": 1.1963, + "theoretical_loss": 3.3749410497155523, + "tokens_seen": 2716401664 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017858289199165466, + "loss": 1.2671, + "theoretical_loss": 3.3749283829146375, + "tokens_seen": 2716532736 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001785427700208634, + "loss": 1.2611, + "theoretical_loss": 3.374915716895996, + "tokens_seen": 2716663808 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017850264805007222, + "loss": 1.2318, + "theoretical_loss": 3.374903051659542, + "tokens_seen": 2716794880 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017846252607928102, + "loss": 1.2636, + "theoretical_loss": 3.374890387205189, + "tokens_seen": 2716925952 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001784224041084898, + "loss": 1.3228, + "theoretical_loss": 3.374877723532851, + "tokens_seen": 2717057024 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001783822821376986, + "loss": 1.2758, + "theoretical_loss": 3.3748650606424424, + "tokens_seen": 2717188096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017834216016690741, + "loss": 1.2822, + "theoretical_loss": 3.3748523985338768, + "tokens_seen": 2717319168 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001783020381961162, + "loss": 1.2908, + "theoretical_loss": 3.3748397372070684, + "tokens_seen": 2717450240 + }, + { + "epoch": 0.65, + "learning_rate": 0.000178261916225325, + "loss": 1.2876, + "theoretical_loss": 3.3748270766619313, + "tokens_seen": 2717581312 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017822179425453378, + "loss": 1.2708, + "theoretical_loss": 3.374814416898379, + "tokens_seen": 2717712384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017818167228374259, + "loss": 1.1885, + "theoretical_loss": 3.3748017579163267, + "tokens_seen": 2717843456 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017814155031295136, + "loss": 1.3309, + "theoretical_loss": 3.374789099715687, + "tokens_seen": 2717974528 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017810142834216017, + "loss": 1.2455, + "theoretical_loss": 3.3747764422963753, + "tokens_seen": 2718105600 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.9934532642364502, + "objective/train/avg_token_score": 0.0065467203967273235, + "objective/train/docs_used": 987833, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3630428314208984, + "objective/train/original_loss": 1.3713628053665161, + "objective/train/theoretical_loss": 3.3747637856583053, + "objective/train/tokens_used": 1088695776, + "objective/train/unlikelihood_loss": 0.01987495645880699, + "theoretical_loss": 3.3747637856583053, + "tokens_seen": 2718236672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017806130637136898, + "loss": 1.2724, + "theoretical_loss": 3.3747637856583053, + "tokens_seen": 2718236672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017802118440057776, + "loss": 1.2933, + "theoretical_loss": 3.3747511298013904, + "tokens_seen": 2718367744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017798106242978656, + "loss": 1.2872, + "theoretical_loss": 3.374738474725546, + "tokens_seen": 2718498816 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017794094045899534, + "loss": 1.1834, + "theoretical_loss": 3.374725820430685, + "tokens_seen": 2718629888 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017790081848820415, + "loss": 1.2462, + "theoretical_loss": 3.374713166916722, + "tokens_seen": 2718760960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017786069651741293, + "loss": 1.1995, + "theoretical_loss": 3.3747005141835715, + "tokens_seen": 2718892032 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017782057454662173, + "loss": 1.3282, + "theoretical_loss": 3.3746878622311476, + "tokens_seen": 2719023104 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017778045257583054, + "loss": 1.2645, + "theoretical_loss": 3.3746752110593636, + "tokens_seen": 2719154176 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017774033060503932, + "loss": 1.238, + "theoretical_loss": 3.3746625606681353, + "tokens_seen": 2719285248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017770020863424813, + "loss": 1.2694, + "theoretical_loss": 3.3746499110573756, + "tokens_seen": 2719416320 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001776600866634569, + "loss": 1.2401, + "theoretical_loss": 3.374637262226999, + "tokens_seen": 2719547392 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001776199646926657, + "loss": 1.3264, + "theoretical_loss": 3.3746246141769207, + "tokens_seen": 2719678464 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001775798427218745, + "loss": 1.2988, + "theoretical_loss": 3.374611966907054, + "tokens_seen": 2719809536 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001775397207510833, + "loss": 1.2686, + "theoretical_loss": 3.3745993204173126, + "tokens_seen": 2719940608 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001774995987802921, + "loss": 1.3213, + "theoretical_loss": 3.3745866747076123, + "tokens_seen": 2720071680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017745947680950088, + "loss": 1.356, + "theoretical_loss": 3.3745740297778664, + "tokens_seen": 2720202752 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001774193548387097, + "loss": 1.3014, + "theoretical_loss": 3.3745613856279895, + "tokens_seen": 2720333824 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001773792328679185, + "loss": 1.2305, + "theoretical_loss": 3.374548742257896, + "tokens_seen": 2720464896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017733911089712725, + "loss": 1.3155, + "theoretical_loss": 3.3745360996675, + "tokens_seen": 2720595968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017729898892633605, + "loss": 1.2463, + "theoretical_loss": 3.374523457856716, + "tokens_seen": 2720727040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017725886695554486, + "loss": 1.277, + "theoretical_loss": 3.3745108168254587, + "tokens_seen": 2720858112 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017721874498475367, + "loss": 1.3365, + "theoretical_loss": 3.374498176573642, + "tokens_seen": 2720989184 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017717862301396244, + "loss": 1.2671, + "theoretical_loss": 3.3744855371011804, + "tokens_seen": 2721120256 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017713850104317125, + "loss": 1.2206, + "theoretical_loss": 3.3744728984079884, + "tokens_seen": 2721251328 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017709837907238006, + "loss": 1.3477, + "theoretical_loss": 3.37446026049398, + "tokens_seen": 2721382400 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001770582571015888, + "loss": 1.2249, + "theoretical_loss": 3.3744476233590706, + "tokens_seen": 2721513472 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017701813513079762, + "loss": 1.3022, + "theoretical_loss": 3.3744349870031742, + "tokens_seen": 2721644544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017697801316000642, + "loss": 1.3167, + "theoretical_loss": 3.3744223514262046, + "tokens_seen": 2721775616 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017693789118921523, + "loss": 1.2842, + "theoretical_loss": 3.3744097166280773, + "tokens_seen": 2721906688 + }, + { + "epoch": 0.65, + "learning_rate": 0.000176897769218424, + "loss": 1.2437, + "theoretical_loss": 3.374397082608706, + "tokens_seen": 2722037760 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017685764724763281, + "loss": 1.3345, + "theoretical_loss": 3.374384449368006, + "tokens_seen": 2722168832 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017681752527684162, + "loss": 1.3346, + "theoretical_loss": 3.374371816905891, + "tokens_seen": 2722299904 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001767774033060504, + "loss": 1.2238, + "theoretical_loss": 3.374359185222276, + "tokens_seen": 2722430976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017673728133525918, + "loss": 1.2766, + "theoretical_loss": 3.374346554317075, + "tokens_seen": 2722562048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017669715936446798, + "loss": 1.2014, + "theoretical_loss": 3.3743339241902035, + "tokens_seen": 2722693120 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001766570373936768, + "loss": 1.3913, + "theoretical_loss": 3.3743212948415753, + "tokens_seen": 2722824192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017661691542288557, + "loss": 1.2354, + "theoretical_loss": 3.374308666271105, + "tokens_seen": 2722955264 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017657679345209438, + "loss": 1.2104, + "theoretical_loss": 3.374296038478708, + "tokens_seen": 2723086336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017653667148130318, + "loss": 1.3076, + "theoretical_loss": 3.3742834114642983, + "tokens_seen": 2723217408 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017649654951051196, + "loss": 1.2338, + "theoretical_loss": 3.3742707852277904, + "tokens_seen": 2723348480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017645642753972074, + "loss": 1.172, + "theoretical_loss": 3.3742581597690986, + "tokens_seen": 2723479552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017641630556892955, + "loss": 1.2342, + "theoretical_loss": 3.374245535088139, + "tokens_seen": 2723610624 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017637618359813833, + "loss": 1.3009, + "theoretical_loss": 3.3742329111848246, + "tokens_seen": 2723741696 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017633606162734713, + "loss": 1.2423, + "theoretical_loss": 3.3742202880590715, + "tokens_seen": 2723872768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017629593965655594, + "loss": 1.2667, + "theoretical_loss": 3.3742076657107933, + "tokens_seen": 2724003840 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017625581768576475, + "loss": 1.3022, + "theoretical_loss": 3.3741950441399053, + "tokens_seen": 2724134912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017621569571497352, + "loss": 1.3199, + "theoretical_loss": 3.374182423346322, + "tokens_seen": 2724265984 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001761755737441823, + "loss": 1.3306, + "theoretical_loss": 3.3741698033299583, + "tokens_seen": 2724397056 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001761354517733911, + "loss": 1.2513, + "theoretical_loss": 3.3741571840907287, + "tokens_seen": 2724528128 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001760953298025999, + "loss": 1.2366, + "theoretical_loss": 3.374144565628548, + "tokens_seen": 2724659200 + }, + { + "epoch": 0.65, + "objective/train/aligned_token_count": 0.9909141063690186, + "objective/train/avg_token_score": 0.009085885249078274, + "objective/train/docs_used": 990115, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.302582025527954, + "objective/train/original_loss": 1.3165706396102905, + "objective/train/theoretical_loss": 3.374131947943331, + "objective/train/tokens_used": 1095249376, + "objective/train/unlikelihood_loss": 0.021186228841543198, + "theoretical_loss": 3.374131947943331, + "tokens_seen": 2724790272 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001760552078318087, + "loss": 1.2923, + "theoretical_loss": 3.374131947943331, + "tokens_seen": 2724790272 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001760150858610175, + "loss": 1.2528, + "theoretical_loss": 3.3741193310349926, + "tokens_seen": 2724921344 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001759749638902263, + "loss": 1.2084, + "theoretical_loss": 3.374106714903448, + "tokens_seen": 2725052416 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001759348419194351, + "loss": 1.2769, + "theoretical_loss": 3.374094099548611, + "tokens_seen": 2725183488 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758947199486439, + "loss": 1.2637, + "theoretical_loss": 3.3740814849703975, + "tokens_seen": 2725314560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017585459797785267, + "loss": 1.275, + "theoretical_loss": 3.3740688711687215, + "tokens_seen": 2725445632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017581447600706145, + "loss": 1.3088, + "theoretical_loss": 3.374056258143498, + "tokens_seen": 2725576704 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017577435403627026, + "loss": 1.2391, + "theoretical_loss": 3.3740436458946426, + "tokens_seen": 2725707776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017573423206547907, + "loss": 1.2978, + "theoretical_loss": 3.3740310344220696, + "tokens_seen": 2725838848 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017569411009468787, + "loss": 1.1914, + "theoretical_loss": 3.3740184237256936, + "tokens_seen": 2725969920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017565398812389665, + "loss": 1.3198, + "theoretical_loss": 3.37400581380543, + "tokens_seen": 2726100992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017561386615310546, + "loss": 1.2498, + "theoretical_loss": 3.373993204661194, + "tokens_seen": 2726232064 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017557374418231424, + "loss": 1.1789, + "theoretical_loss": 3.3739805962928995, + "tokens_seen": 2726363136 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017553362221152302, + "loss": 1.2993, + "theoretical_loss": 3.3739679887004623, + "tokens_seen": 2726494208 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017549350024073182, + "loss": 1.256, + "theoretical_loss": 3.373955381883797, + "tokens_seen": 2726625280 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017545337826994063, + "loss": 1.3319, + "theoretical_loss": 3.373942775842819, + "tokens_seen": 2726756352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017541325629914943, + "loss": 1.2362, + "theoretical_loss": 3.3739301705774425, + "tokens_seen": 2726887424 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001753731343283582, + "loss": 1.2593, + "theoretical_loss": 3.3739175660875835, + "tokens_seen": 2727018496 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017533301235756702, + "loss": 1.2727, + "theoretical_loss": 3.3739049623731563, + "tokens_seen": 2727149568 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001752928903867758, + "loss": 1.1756, + "theoretical_loss": 3.373892359434076, + "tokens_seen": 2727280640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017525276841598458, + "loss": 1.2647, + "theoretical_loss": 3.373879757270258, + "tokens_seen": 2727411712 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017521264644519338, + "loss": 1.2546, + "theoretical_loss": 3.373867155881617, + "tokens_seen": 2727542784 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001751725244744022, + "loss": 1.2866, + "theoretical_loss": 3.3738545552680685, + "tokens_seen": 2727673856 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017513240250361097, + "loss": 1.2396, + "theoretical_loss": 3.373841955429527, + "tokens_seen": 2727804928 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017509228053281978, + "loss": 1.2672, + "theoretical_loss": 3.373829356365908, + "tokens_seen": 2727936000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017505215856202858, + "loss": 1.289, + "theoretical_loss": 3.3738167580771266, + "tokens_seen": 2728067072 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001750120365912374, + "loss": 1.2293, + "theoretical_loss": 3.3738041605630977, + "tokens_seen": 2728198144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017497191462044614, + "loss": 1.2448, + "theoretical_loss": 3.3737915638237364, + "tokens_seen": 2728329216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017493179264965495, + "loss": 1.3212, + "theoretical_loss": 3.373778967858958, + "tokens_seen": 2728460288 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017489167067886375, + "loss": 1.2395, + "theoretical_loss": 3.373766372668678, + "tokens_seen": 2728591360 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017485154870807253, + "loss": 1.2702, + "theoretical_loss": 3.3737537782528113, + "tokens_seen": 2728722432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017481142673728134, + "loss": 1.3358, + "theoretical_loss": 3.373741184611273, + "tokens_seen": 2728853504 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017477130476649015, + "loss": 1.2836, + "theoretical_loss": 3.373728591743978, + "tokens_seen": 2728984576 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017473118279569895, + "loss": 1.325, + "theoretical_loss": 3.373715999650842, + "tokens_seen": 2729115648 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746910608249077, + "loss": 1.241, + "theoretical_loss": 3.3737034083317803, + "tokens_seen": 2729246720 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746509388541165, + "loss": 1.2665, + "theoretical_loss": 3.373690817786708, + "tokens_seen": 2729377792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017461081688332532, + "loss": 1.2644, + "theoretical_loss": 3.37367822801554, + "tokens_seen": 2729508864 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001745706949125341, + "loss": 1.2803, + "theoretical_loss": 3.373665639018192, + "tokens_seen": 2729639936 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001745305729417429, + "loss": 1.2352, + "theoretical_loss": 3.3736530507945792, + "tokens_seen": 2729771008 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001744904509709517, + "loss": 1.2715, + "theoretical_loss": 3.373640463344617, + "tokens_seen": 2729902080 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017445032900016051, + "loss": 1.2681, + "theoretical_loss": 3.37362787666822, + "tokens_seen": 2730033152 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001744102070293693, + "loss": 1.2957, + "theoretical_loss": 3.3736152907653043, + "tokens_seen": 2730164224 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017437008505857807, + "loss": 1.2769, + "theoretical_loss": 3.3736027056357853, + "tokens_seen": 2730295296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017432996308778688, + "loss": 1.2491, + "theoretical_loss": 3.373590121279578, + "tokens_seen": 2730426368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017428984111699566, + "loss": 1.3099, + "theoretical_loss": 3.373577537696598, + "tokens_seen": 2730557440 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017424971914620446, + "loss": 1.2191, + "theoretical_loss": 3.37356495488676, + "tokens_seen": 2730688512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017420959717541327, + "loss": 1.2541, + "theoretical_loss": 3.3735523728499803, + "tokens_seen": 2730819584 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017416947520462205, + "loss": 1.255, + "theoretical_loss": 3.373539791586174, + "tokens_seen": 2730950656 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017412935323383086, + "loss": 1.2488, + "theoretical_loss": 3.373527211095256, + "tokens_seen": 2731081728 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017408923126303964, + "loss": 1.2542, + "theoretical_loss": 3.3735146313771427, + "tokens_seen": 2731212800 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.9928444623947144, + "objective/train/avg_token_score": 0.007155559025704861, + "objective/train/docs_used": 992345, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.329649806022644, + "objective/train/original_loss": 1.3408503532409668, + "objective/train/theoretical_loss": 3.3735020524317485, + "objective/train/tokens_used": 1101802976, + "objective/train/unlikelihood_loss": 0.01693030446767807, + "theoretical_loss": 3.3735020524317485, + "tokens_seen": 2731343872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017404910929224844, + "loss": 1.2994, + "theoretical_loss": 3.3735020524317485, + "tokens_seen": 2731343872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017400898732145722, + "loss": 1.2767, + "theoretical_loss": 3.3734894742589896, + "tokens_seen": 2731474944 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017396886535066603, + "loss": 1.2475, + "theoretical_loss": 3.373476896858781, + "tokens_seen": 2731606016 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017392874337987483, + "loss": 1.3115, + "theoretical_loss": 3.3734643202310384, + "tokens_seen": 2731737088 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738886214090836, + "loss": 1.2094, + "theoretical_loss": 3.3734517443756777, + "tokens_seen": 2731868160 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017384849943829242, + "loss": 1.2479, + "theoretical_loss": 3.373439169292614, + "tokens_seen": 2731999232 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738083774675012, + "loss": 1.3093, + "theoretical_loss": 3.3734265949817623, + "tokens_seen": 2732130304 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017376825549671, + "loss": 1.2673, + "theoretical_loss": 3.3734140214430393, + "tokens_seen": 2732261376 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017372813352591878, + "loss": 1.2759, + "theoretical_loss": 3.3734014486763595, + "tokens_seen": 2732392448 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001736880115551276, + "loss": 1.2566, + "theoretical_loss": 3.373388876681639, + "tokens_seen": 2732523520 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001736478895843364, + "loss": 1.3042, + "theoretical_loss": 3.3733763054587933, + "tokens_seen": 2732654592 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017360776761354518, + "loss": 1.2981, + "theoretical_loss": 3.373363735007738, + "tokens_seen": 2732785664 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017356764564275398, + "loss": 1.2569, + "theoretical_loss": 3.3733511653283887, + "tokens_seen": 2732916736 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001735275236719628, + "loss": 1.3217, + "theoretical_loss": 3.373338596420661, + "tokens_seen": 2733047808 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017348740170117157, + "loss": 1.2372, + "theoretical_loss": 3.3733260282844704, + "tokens_seen": 2733178880 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017344727973038035, + "loss": 1.287, + "theoretical_loss": 3.3733134609197326, + "tokens_seen": 2733309952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017340715775958915, + "loss": 1.149, + "theoretical_loss": 3.3733008943263636, + "tokens_seen": 2733441024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017336703578879796, + "loss": 1.3187, + "theoretical_loss": 3.3732883285042785, + "tokens_seen": 2733572096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017332691381800674, + "loss": 1.2942, + "theoretical_loss": 3.3732757634533934, + "tokens_seen": 2733703168 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017328679184721555, + "loss": 1.277, + "theoretical_loss": 3.3732631991736235, + "tokens_seen": 2733834240 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017324666987642435, + "loss": 1.2751, + "theoretical_loss": 3.3732506356648853, + "tokens_seen": 2733965312 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001732065479056331, + "loss": 1.3258, + "theoretical_loss": 3.373238072927094, + "tokens_seen": 2734096384 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001731664259348419, + "loss": 1.1977, + "theoretical_loss": 3.3732255109601654, + "tokens_seen": 2734227456 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017312630396405072, + "loss": 1.2702, + "theoretical_loss": 3.373212949764015, + "tokens_seen": 2734358528 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017308618199325952, + "loss": 1.2953, + "theoretical_loss": 3.373200389338559, + "tokens_seen": 2734489600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001730460600224683, + "loss": 1.2987, + "theoretical_loss": 3.373187829683713, + "tokens_seen": 2734620672 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001730059380516771, + "loss": 1.2387, + "theoretical_loss": 3.373175270799393, + "tokens_seen": 2734751744 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017296581608088591, + "loss": 1.1859, + "theoretical_loss": 3.3731627126855144, + "tokens_seen": 2734882816 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017292569411009467, + "loss": 1.2734, + "theoretical_loss": 3.3731501553419934, + "tokens_seen": 2735013888 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017288557213930347, + "loss": 1.1843, + "theoretical_loss": 3.3731375987687455, + "tokens_seen": 2735144960 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017284545016851228, + "loss": 1.2204, + "theoretical_loss": 3.373125042965687, + "tokens_seen": 2735276032 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017280532819772109, + "loss": 1.275, + "theoretical_loss": 3.373112487932733, + "tokens_seen": 2735407104 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017276520622692986, + "loss": 1.2524, + "theoretical_loss": 3.3730999336698, + "tokens_seen": 2735538176 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017272508425613867, + "loss": 1.2407, + "theoretical_loss": 3.3730873801768038, + "tokens_seen": 2735669248 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017268496228534748, + "loss": 1.2073, + "theoretical_loss": 3.37307482745366, + "tokens_seen": 2735800320 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017264484031455626, + "loss": 1.3156, + "theoretical_loss": 3.373062275500285, + "tokens_seen": 2735931392 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017260471834376504, + "loss": 1.1835, + "theoretical_loss": 3.3730497243165942, + "tokens_seen": 2736062464 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017256459637297384, + "loss": 1.2985, + "theoretical_loss": 3.3730371739025036, + "tokens_seen": 2736193536 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017252447440218265, + "loss": 1.2253, + "theoretical_loss": 3.3730246242579294, + "tokens_seen": 2736324608 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017248435243139143, + "loss": 1.21, + "theoretical_loss": 3.3730120753827877, + "tokens_seen": 2736455680 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017244423046060023, + "loss": 1.2418, + "theoretical_loss": 3.3729995272769937, + "tokens_seen": 2736586752 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017240410848980904, + "loss": 1.1964, + "theoretical_loss": 3.3729869799404644, + "tokens_seen": 2736717824 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017236398651901782, + "loss": 1.2392, + "theoretical_loss": 3.3729744333731153, + "tokens_seen": 2736848896 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001723238645482266, + "loss": 1.335, + "theoretical_loss": 3.3729618875748626, + "tokens_seen": 2736979968 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001722837425774354, + "loss": 1.2857, + "theoretical_loss": 3.372949342545622, + "tokens_seen": 2737111040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017224362060664418, + "loss": 1.3041, + "theoretical_loss": 3.3729367982853096, + "tokens_seen": 2737242112 + }, + { + "epoch": 0.66, + "learning_rate": 0.000172203498635853, + "loss": 1.2605, + "theoretical_loss": 3.3729242547938414, + "tokens_seen": 2737373184 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001721633766650618, + "loss": 1.3931, + "theoretical_loss": 3.372911712071134, + "tokens_seen": 2737504256 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001721232546942706, + "loss": 1.2473, + "theoretical_loss": 3.3728991701171025, + "tokens_seen": 2737635328 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017208313272347938, + "loss": 1.2, + "theoretical_loss": 3.3728866289316644, + "tokens_seen": 2737766400 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.9949920177459717, + "objective/train/avg_token_score": 0.00500798923894763, + "objective/train/docs_used": 994721, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.153266429901123, + "objective/train/original_loss": 1.1602177619934082, + "objective/train/theoretical_loss": 3.3728740885147346, + "objective/train/tokens_used": 1108356576, + "objective/train/unlikelihood_loss": 0.015302805230021477, + "theoretical_loss": 3.3728740885147346, + "tokens_seen": 2737897472 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001720430107526882, + "loss": 1.2001, + "theoretical_loss": 3.3728740885147346, + "tokens_seen": 2737897472 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017200288878189697, + "loss": 1.2648, + "theoretical_loss": 3.3728615488662292, + "tokens_seen": 2738028544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017196276681110575, + "loss": 1.2478, + "theoretical_loss": 3.3728490099860653, + "tokens_seen": 2738159616 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017192264484031455, + "loss": 1.2628, + "theoretical_loss": 3.3728364718741584, + "tokens_seen": 2738290688 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017188252286952336, + "loss": 1.1863, + "theoretical_loss": 3.372823934530425, + "tokens_seen": 2738421760 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017184240089873217, + "loss": 1.2271, + "theoretical_loss": 3.3728113979547807, + "tokens_seen": 2738552832 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017180227892794094, + "loss": 1.1784, + "theoretical_loss": 3.3727988621471416, + "tokens_seen": 2738683904 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017176215695714975, + "loss": 1.3034, + "theoretical_loss": 3.372786327107425, + "tokens_seen": 2738814976 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017172203498635853, + "loss": 1.2416, + "theoretical_loss": 3.372773792835546, + "tokens_seen": 2738946048 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001716819130155673, + "loss": 1.2009, + "theoretical_loss": 3.372761259331422, + "tokens_seen": 2739077120 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017164179104477612, + "loss": 1.3291, + "theoretical_loss": 3.372748726594968, + "tokens_seen": 2739208192 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017160166907398492, + "loss": 1.316, + "theoretical_loss": 3.3727361946261003, + "tokens_seen": 2739339264 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017156154710319373, + "loss": 1.2809, + "theoretical_loss": 3.372723663424736, + "tokens_seen": 2739470336 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715214251324025, + "loss": 1.1989, + "theoretical_loss": 3.372711132990791, + "tokens_seen": 2739601408 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017148130316161131, + "loss": 1.2891, + "theoretical_loss": 3.3726986033241815, + "tokens_seen": 2739732480 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001714411811908201, + "loss": 1.2507, + "theoretical_loss": 3.372686074424824, + "tokens_seen": 2739863552 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017140105922002887, + "loss": 1.2936, + "theoretical_loss": 3.3726735462926345, + "tokens_seen": 2739994624 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017136093724923768, + "loss": 1.2934, + "theoretical_loss": 3.3726610189275297, + "tokens_seen": 2740125696 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017132081527844648, + "loss": 1.3423, + "theoretical_loss": 3.3726484923294255, + "tokens_seen": 2740256768 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017128069330765526, + "loss": 1.2172, + "theoretical_loss": 3.3726359664982386, + "tokens_seen": 2740387840 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017124057133686407, + "loss": 1.2182, + "theoretical_loss": 3.372623441433886, + "tokens_seen": 2740518912 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017120044936607288, + "loss": 1.2512, + "theoretical_loss": 3.3726109171362824, + "tokens_seen": 2740649984 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017116032739528168, + "loss": 1.2706, + "theoretical_loss": 3.3725983936053456, + "tokens_seen": 2740781056 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017112020542449044, + "loss": 1.1703, + "theoretical_loss": 3.3725858708409913, + "tokens_seen": 2740912128 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017108008345369924, + "loss": 1.1824, + "theoretical_loss": 3.3725733488431366, + "tokens_seen": 2741043200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017103996148290805, + "loss": 1.3272, + "theoretical_loss": 3.372560827611697, + "tokens_seen": 2741174272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017099983951211683, + "loss": 1.2583, + "theoretical_loss": 3.37254830714659, + "tokens_seen": 2741305344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017095971754132563, + "loss": 1.2629, + "theoretical_loss": 3.372535787447731, + "tokens_seen": 2741436416 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017091959557053444, + "loss": 1.254, + "theoretical_loss": 3.372523268515037, + "tokens_seen": 2741567488 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017087947359974325, + "loss": 1.305, + "theoretical_loss": 3.3725107503484253, + "tokens_seen": 2741698560 + }, + { + "epoch": 0.66, + "learning_rate": 0.000170839351628952, + "loss": 1.2247, + "theoretical_loss": 3.372498232947811, + "tokens_seen": 2741829632 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707992296581608, + "loss": 1.2522, + "theoretical_loss": 3.3724857163131112, + "tokens_seen": 2741960704 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707591076873696, + "loss": 1.1927, + "theoretical_loss": 3.3724732004442424, + "tokens_seen": 2742091776 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707189857165784, + "loss": 1.2363, + "theoretical_loss": 3.3724606853411214, + "tokens_seen": 2742222848 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001706788637457872, + "loss": 1.2407, + "theoretical_loss": 3.3724481710036645, + "tokens_seen": 2742353920 + }, + { + "epoch": 0.66, + "learning_rate": 0.000170638741774996, + "loss": 1.2413, + "theoretical_loss": 3.372435657431788, + "tokens_seen": 2742484992 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001705986198042048, + "loss": 1.2204, + "theoretical_loss": 3.372423144625409, + "tokens_seen": 2742616064 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017055849783341356, + "loss": 1.2302, + "theoretical_loss": 3.372410632584444, + "tokens_seen": 2742747136 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017051837586262237, + "loss": 1.2492, + "theoretical_loss": 3.3723981213088092, + "tokens_seen": 2742878208 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017047825389183117, + "loss": 1.2636, + "theoretical_loss": 3.3723856107984216, + "tokens_seen": 2743009280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017043813192103995, + "loss": 1.1691, + "theoretical_loss": 3.3723731010531974, + "tokens_seen": 2743140352 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017039800995024876, + "loss": 1.2618, + "theoretical_loss": 3.372360592073054, + "tokens_seen": 2743271424 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017035788797945757, + "loss": 1.2529, + "theoretical_loss": 3.3723480838579074, + "tokens_seen": 2743402496 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017031776600866637, + "loss": 1.2658, + "theoretical_loss": 3.3723355764076746, + "tokens_seen": 2743533568 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017027764403787515, + "loss": 1.1401, + "theoretical_loss": 3.372323069722272, + "tokens_seen": 2743664640 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017023752206708393, + "loss": 1.2628, + "theoretical_loss": 3.3723105638016166, + "tokens_seen": 2743795712 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017019740009629274, + "loss": 1.2337, + "theoretical_loss": 3.372298058645625, + "tokens_seen": 2743926784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017015727812550152, + "loss": 1.1763, + "theoretical_loss": 3.372285554254214, + "tokens_seen": 2744057856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017011715615471032, + "loss": 1.2396, + "theoretical_loss": 3.3722730506273, + "tokens_seen": 2744188928 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017007703418391913, + "loss": 1.2077, + "theoretical_loss": 3.3722605477648, + "tokens_seen": 2744320000 + }, + { + "epoch": 0.66, + "objective/train/aligned_token_count": 0.9928019046783447, + "objective/train/avg_token_score": 0.007198074366897345, + "objective/train/docs_used": 997363, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3255289793014526, + "objective/train/original_loss": 1.3361457586288452, + "objective/train/theoretical_loss": 3.372248045666631, + "objective/train/tokens_used": 1114910176, + "objective/train/unlikelihood_loss": 0.018424466252326965, + "theoretical_loss": 3.372248045666631, + "tokens_seen": 2744451072 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001700369122131279, + "loss": 1.3113, + "theoretical_loss": 3.372248045666631, + "tokens_seen": 2744451072 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001699967902423367, + "loss": 1.2212, + "theoretical_loss": 3.372235544332709, + "tokens_seen": 2744582144 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001699566682715455, + "loss": 1.248, + "theoretical_loss": 3.3722230437629515, + "tokens_seen": 2744713216 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001699165463007543, + "loss": 1.2343, + "theoretical_loss": 3.3722105439572756, + "tokens_seen": 2744844288 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016987642432996308, + "loss": 1.2059, + "theoretical_loss": 3.3721980449155966, + "tokens_seen": 2744975360 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016983630235917188, + "loss": 1.2623, + "theoretical_loss": 3.372185546637833, + "tokens_seen": 2745106432 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001697961803883807, + "loss": 1.2979, + "theoretical_loss": 3.372173049123901, + "tokens_seen": 2745237504 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016975605841758947, + "loss": 1.2872, + "theoretical_loss": 3.3721605523737175, + "tokens_seen": 2745368576 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016971593644679828, + "loss": 1.2828, + "theoretical_loss": 3.372148056387199, + "tokens_seen": 2745499648 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016967581447600708, + "loss": 1.2413, + "theoretical_loss": 3.3721355611642627, + "tokens_seen": 2745630720 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016963569250521586, + "loss": 1.3051, + "theoretical_loss": 3.372123066704826, + "tokens_seen": 2745761792 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016959557053442464, + "loss": 1.1788, + "theoretical_loss": 3.372110573008805, + "tokens_seen": 2745892864 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016955544856363345, + "loss": 1.2545, + "theoretical_loss": 3.3720980800761167, + "tokens_seen": 2746023936 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016951532659284225, + "loss": 1.1709, + "theoretical_loss": 3.3720855879066782, + "tokens_seen": 2746155008 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016947520462205103, + "loss": 1.2727, + "theoretical_loss": 3.372073096500406, + "tokens_seen": 2746286080 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016943508265125984, + "loss": 1.274, + "theoretical_loss": 3.3720606058572185, + "tokens_seen": 2746417152 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016939496068046865, + "loss": 1.2301, + "theoretical_loss": 3.372048115977031, + "tokens_seen": 2746548224 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016935483870967742, + "loss": 1.272, + "theoretical_loss": 3.3720356268597618, + "tokens_seen": 2746679296 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001693147167388862, + "loss": 1.2139, + "theoretical_loss": 3.3720231385053268, + "tokens_seen": 2746810368 + }, + { + "epoch": 0.66, + "learning_rate": 0.000169274594768095, + "loss": 1.2767, + "theoretical_loss": 3.3720106509136434, + "tokens_seen": 2746941440 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016923447279730382, + "loss": 1.2176, + "theoretical_loss": 3.371998164084629, + "tokens_seen": 2747072512 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001691943508265126, + "loss": 1.2926, + "theoretical_loss": 3.3719856780182003, + "tokens_seen": 2747203584 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691542288557214, + "loss": 1.2207, + "theoretical_loss": 3.3719731927142744, + "tokens_seen": 2747334656 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691141068849302, + "loss": 1.1765, + "theoretical_loss": 3.371960708172768, + "tokens_seen": 2747465728 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016907398491413896, + "loss": 1.2366, + "theoretical_loss": 3.3719482243935985, + "tokens_seen": 2747596800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016903386294334777, + "loss": 1.2358, + "theoretical_loss": 3.3719357413766833, + "tokens_seen": 2747727872 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016899374097255657, + "loss": 1.1902, + "theoretical_loss": 3.3719232591219392, + "tokens_seen": 2747858944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016895361900176538, + "loss": 1.2865, + "theoretical_loss": 3.3719107776292834, + "tokens_seen": 2747990016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016891349703097416, + "loss": 1.3093, + "theoretical_loss": 3.3718982968986326, + "tokens_seen": 2748121088 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016887337506018296, + "loss": 1.2497, + "theoretical_loss": 3.371885816929905, + "tokens_seen": 2748252160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016883325308939177, + "loss": 1.2618, + "theoretical_loss": 3.371873337723016, + "tokens_seen": 2748383232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016879313111860055, + "loss": 1.2182, + "theoretical_loss": 3.3718608592778847, + "tokens_seen": 2748514304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016875300914780933, + "loss": 1.2463, + "theoretical_loss": 3.3718483815944267, + "tokens_seen": 2748645376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016871288717701814, + "loss": 1.2273, + "theoretical_loss": 3.3718359046725603, + "tokens_seen": 2748776448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016867276520622694, + "loss": 1.2214, + "theoretical_loss": 3.3718234285122017, + "tokens_seen": 2748907520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016863264323543572, + "loss": 1.2464, + "theoretical_loss": 3.371810953113269, + "tokens_seen": 2749038592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016859252126464453, + "loss": 1.2331, + "theoretical_loss": 3.3717984784756796, + "tokens_seen": 2749169664 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016855239929385333, + "loss": 1.2884, + "theoretical_loss": 3.3717860045993495, + "tokens_seen": 2749300736 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001685122773230621, + "loss": 1.2227, + "theoretical_loss": 3.3717735314841972, + "tokens_seen": 2749431808 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001684721553522709, + "loss": 1.291, + "theoretical_loss": 3.3717610591301392, + "tokens_seen": 2749562880 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001684320333814797, + "loss": 1.2488, + "theoretical_loss": 3.3717485875370934, + "tokens_seen": 2749693952 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683919114106885, + "loss": 1.2716, + "theoretical_loss": 3.371736116704976, + "tokens_seen": 2749825024 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016835178943989728, + "loss": 1.2793, + "theoretical_loss": 3.371723646633706, + "tokens_seen": 2749956096 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683116674691061, + "loss": 1.2613, + "theoretical_loss": 3.371711177323199, + "tokens_seen": 2750087168 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001682715454983149, + "loss": 1.3218, + "theoretical_loss": 3.3716987087733736, + "tokens_seen": 2750218240 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016823142352752368, + "loss": 1.2238, + "theoretical_loss": 3.371686240984147, + "tokens_seen": 2750349312 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016819130155673246, + "loss": 1.3247, + "theoretical_loss": 3.371673773955435, + "tokens_seen": 2750480384 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016815117958594126, + "loss": 1.3473, + "theoretical_loss": 3.371661307687157, + "tokens_seen": 2750611456 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016811105761515004, + "loss": 1.3855, + "theoretical_loss": 3.3716488421792294, + "tokens_seen": 2750742528 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016807093564435885, + "loss": 1.1761, + "theoretical_loss": 3.37163637743157, + "tokens_seen": 2750873600 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.9917548298835754, + "objective/train/avg_token_score": 0.008245176635682583, + "objective/train/docs_used": 999260, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2166852951049805, + "objective/train/original_loss": 1.2273247241973877, + "objective/train/theoretical_loss": 3.3716239134440955, + "objective/train/tokens_used": 1121463776, + "objective/train/unlikelihood_loss": 0.023129120469093323, + "theoretical_loss": 3.3716239134440955, + "tokens_seen": 2751004672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016803081367356765, + "loss": 1.2656, + "theoretical_loss": 3.3716239134440955, + "tokens_seen": 2751004672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016799069170277646, + "loss": 1.1921, + "theoretical_loss": 3.3716114502167245, + "tokens_seen": 2751135744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016795056973198524, + "loss": 1.2022, + "theoretical_loss": 3.371598987749373, + "tokens_seen": 2751266816 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016791044776119405, + "loss": 1.2431, + "theoretical_loss": 3.3715865260419595, + "tokens_seen": 2751397888 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016787032579040282, + "loss": 1.2317, + "theoretical_loss": 3.3715740650944013, + "tokens_seen": 2751528960 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001678302038196116, + "loss": 1.2283, + "theoretical_loss": 3.3715616049066153, + "tokens_seen": 2751660032 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001677900818488204, + "loss": 1.1819, + "theoretical_loss": 3.37154914547852, + "tokens_seen": 2751791104 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016774995987802922, + "loss": 1.2241, + "theoretical_loss": 3.371536686810032, + "tokens_seen": 2751922176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016770983790723802, + "loss": 1.2711, + "theoretical_loss": 3.3715242289010687, + "tokens_seen": 2752053248 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676697159364468, + "loss": 1.3071, + "theoretical_loss": 3.3715117717515484, + "tokens_seen": 2752184320 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676295939656556, + "loss": 1.2538, + "theoretical_loss": 3.371499315361388, + "tokens_seen": 2752315392 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001675894719948644, + "loss": 1.2203, + "theoretical_loss": 3.371486859730506, + "tokens_seen": 2752446464 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016754935002407317, + "loss": 1.2268, + "theoretical_loss": 3.3714744048588186, + "tokens_seen": 2752577536 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016750922805328197, + "loss": 1.3857, + "theoretical_loss": 3.3714619507462444, + "tokens_seen": 2752708608 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016746910608249078, + "loss": 1.3307, + "theoretical_loss": 3.371449497392701, + "tokens_seen": 2752839680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016742898411169959, + "loss": 1.213, + "theoretical_loss": 3.371437044798105, + "tokens_seen": 2752970752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016738886214090836, + "loss": 1.2867, + "theoretical_loss": 3.371424592962375, + "tokens_seen": 2753101824 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016734874017011717, + "loss": 1.2308, + "theoretical_loss": 3.371412141885428, + "tokens_seen": 2753232896 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016730861819932598, + "loss": 1.2803, + "theoretical_loss": 3.3713996915671824, + "tokens_seen": 2753363968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016726849622853473, + "loss": 1.1923, + "theoretical_loss": 3.3713872420075557, + "tokens_seen": 2753495040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016722837425774354, + "loss": 1.3576, + "theoretical_loss": 3.371374793206465, + "tokens_seen": 2753626112 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016718825228695234, + "loss": 1.3004, + "theoretical_loss": 3.3713623451638277, + "tokens_seen": 2753757184 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016714813031616112, + "loss": 1.2713, + "theoretical_loss": 3.3713498978795626, + "tokens_seen": 2753888256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016710800834536993, + "loss": 1.2116, + "theoretical_loss": 3.371337451353587, + "tokens_seen": 2754019328 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016706788637457873, + "loss": 1.2269, + "theoretical_loss": 3.3713250055858177, + "tokens_seen": 2754150400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016702776440378754, + "loss": 1.284, + "theoretical_loss": 3.371312560576174, + "tokens_seen": 2754281472 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001669876424329963, + "loss": 1.24, + "theoretical_loss": 3.3713001163245724, + "tokens_seen": 2754412544 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001669475204622051, + "loss": 1.2311, + "theoretical_loss": 3.3712876728309316, + "tokens_seen": 2754543616 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001669073984914139, + "loss": 1.18, + "theoretical_loss": 3.3712752300951685, + "tokens_seen": 2754674688 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016686727652062268, + "loss": 1.2871, + "theoretical_loss": 3.371262788117201, + "tokens_seen": 2754805760 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001668271545498315, + "loss": 1.283, + "theoretical_loss": 3.3712503468969475, + "tokens_seen": 2754936832 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001667870325790403, + "loss": 1.2469, + "theoretical_loss": 3.3712379064343256, + "tokens_seen": 2755067904 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001667469106082491, + "loss": 1.2506, + "theoretical_loss": 3.371225466729253, + "tokens_seen": 2755198976 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016670678863745785, + "loss": 1.3157, + "theoretical_loss": 3.371213027781647, + "tokens_seen": 2755330048 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016666666666666666, + "loss": 1.3301, + "theoretical_loss": 3.3712005895914263, + "tokens_seen": 2755461120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016662654469587547, + "loss": 1.3149, + "theoretical_loss": 3.371188152158509, + "tokens_seen": 2755592192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016658642272508425, + "loss": 1.2808, + "theoretical_loss": 3.3711757154828117, + "tokens_seen": 2755723264 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016654630075429305, + "loss": 1.2413, + "theoretical_loss": 3.371163279564253, + "tokens_seen": 2755854336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016650617878350186, + "loss": 1.2523, + "theoretical_loss": 3.371150844402751, + "tokens_seen": 2755985408 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016646605681271067, + "loss": 1.2176, + "theoretical_loss": 3.3711384099982236, + "tokens_seen": 2756116480 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016642593484191944, + "loss": 1.2974, + "theoretical_loss": 3.371125976350588, + "tokens_seen": 2756247552 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016638581287112822, + "loss": 1.2187, + "theoretical_loss": 3.3711135434597628, + "tokens_seen": 2756378624 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016634569090033703, + "loss": 1.2402, + "theoretical_loss": 3.3711011113256664, + "tokens_seen": 2756509696 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001663055689295458, + "loss": 1.2702, + "theoretical_loss": 3.3710886799482154, + "tokens_seen": 2756640768 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016626544695875462, + "loss": 1.2758, + "theoretical_loss": 3.371076249327329, + "tokens_seen": 2756771840 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016622532498796342, + "loss": 1.2505, + "theoretical_loss": 3.3710638194629245, + "tokens_seen": 2756902912 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016618520301717223, + "loss": 1.2507, + "theoretical_loss": 3.3710513903549204, + "tokens_seen": 2757033984 + }, + { + "epoch": 0.67, + "learning_rate": 0.000166145081046381, + "loss": 1.2427, + "theoretical_loss": 3.3710389620032344, + "tokens_seen": 2757165056 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661049590755898, + "loss": 1.3033, + "theoretical_loss": 3.3710265344077843, + "tokens_seen": 2757296128 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001660648371047986, + "loss": 1.3277, + "theoretical_loss": 3.371014107568489, + "tokens_seen": 2757427200 + }, + { + "epoch": 0.67, + "objective/train/aligned_token_count": 0.9923155307769775, + "objective/train/avg_token_score": 0.007684460841119289, + "objective/train/docs_used": 1001599, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4049814939498901, + "objective/train/original_loss": 1.418095350265503, + "objective/train/theoretical_loss": 3.3710016814852652, + "objective/train/tokens_used": 1128017376, + "objective/train/unlikelihood_loss": 0.01980477385222912, + "theoretical_loss": 3.3710016814852652, + "tokens_seen": 2757558272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016602471513400737, + "loss": 1.3319, + "theoretical_loss": 3.3710016814852652, + "tokens_seen": 2757558272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016598459316321618, + "loss": 1.2704, + "theoretical_loss": 3.3709892561580324, + "tokens_seen": 2757689344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016594447119242498, + "loss": 1.2603, + "theoretical_loss": 3.3709768315867077, + "tokens_seen": 2757820416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016590434922163376, + "loss": 1.2663, + "theoretical_loss": 3.3709644077712095, + "tokens_seen": 2757951488 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016586422725084257, + "loss": 1.2, + "theoretical_loss": 3.370951984711456, + "tokens_seen": 2758082560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016582410528005135, + "loss": 1.2392, + "theoretical_loss": 3.370939562407365, + "tokens_seen": 2758213632 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016578398330926016, + "loss": 1.2225, + "theoretical_loss": 3.3709271408588553, + "tokens_seen": 2758344704 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016574386133846894, + "loss": 1.2474, + "theoretical_loss": 3.3709147200658443, + "tokens_seen": 2758475776 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016570373936767774, + "loss": 1.2689, + "theoretical_loss": 3.3709023000282503, + "tokens_seen": 2758606848 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016566361739688655, + "loss": 1.3146, + "theoretical_loss": 3.370889880745992, + "tokens_seen": 2758737920 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016562349542609533, + "loss": 1.3089, + "theoretical_loss": 3.3708774622189868, + "tokens_seen": 2758868992 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016558337345530413, + "loss": 1.3357, + "theoretical_loss": 3.3708650444471537, + "tokens_seen": 2759000064 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016554325148451294, + "loss": 1.2946, + "theoretical_loss": 3.37085262743041, + "tokens_seen": 2759131136 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016550312951372172, + "loss": 1.2038, + "theoretical_loss": 3.370840211168675, + "tokens_seen": 2759262208 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001654630075429305, + "loss": 1.2312, + "theoretical_loss": 3.370827795661866, + "tokens_seen": 2759393280 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001654228855721393, + "loss": 1.2815, + "theoretical_loss": 3.3708153809099013, + "tokens_seen": 2759524352 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653827636013481, + "loss": 1.172, + "theoretical_loss": 3.3708029669127, + "tokens_seen": 2759655424 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653426416305569, + "loss": 1.2523, + "theoretical_loss": 3.3707905536701794, + "tokens_seen": 2759786496 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653025196597657, + "loss": 1.2618, + "theoretical_loss": 3.3707781411822584, + "tokens_seen": 2759917568 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001652623976889745, + "loss": 1.2374, + "theoretical_loss": 3.370765729448855, + "tokens_seen": 2760048640 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016522227571818328, + "loss": 1.2264, + "theoretical_loss": 3.370753318469888, + "tokens_seen": 2760179712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016518215374739206, + "loss": 1.2343, + "theoretical_loss": 3.3707409082452746, + "tokens_seen": 2760310784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016514203177660087, + "loss": 1.2738, + "theoretical_loss": 3.3707284987749344, + "tokens_seen": 2760441856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016510190980580967, + "loss": 1.2249, + "theoretical_loss": 3.370716090058785, + "tokens_seen": 2760572928 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016506178783501845, + "loss": 1.2224, + "theoretical_loss": 3.370703682096745, + "tokens_seen": 2760704000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016502166586422726, + "loss": 1.3401, + "theoretical_loss": 3.3706912748887325, + "tokens_seen": 2760835072 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016498154389343607, + "loss": 1.3007, + "theoretical_loss": 3.3706788684346662, + "tokens_seen": 2760966144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016494142192264484, + "loss": 1.2034, + "theoretical_loss": 3.3706664627344645, + "tokens_seen": 2761097216 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016490129995185362, + "loss": 1.2373, + "theoretical_loss": 3.3706540577880455, + "tokens_seen": 2761228288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016486117798106243, + "loss": 1.2317, + "theoretical_loss": 3.370641653595328, + "tokens_seen": 2761359360 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016482105601027124, + "loss": 1.3198, + "theoretical_loss": 3.3706292501562305, + "tokens_seen": 2761490432 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016478093403948002, + "loss": 1.3084, + "theoretical_loss": 3.3706168474706706, + "tokens_seen": 2761621504 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016474081206868882, + "loss": 1.2804, + "theoretical_loss": 3.370604445538568, + "tokens_seen": 2761752576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016470069009789763, + "loss": 1.2492, + "theoretical_loss": 3.3705920443598405, + "tokens_seen": 2761883648 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001646605681271064, + "loss": 1.2707, + "theoretical_loss": 3.370579643934406, + "tokens_seen": 2762014720 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016462044615631519, + "loss": 1.2543, + "theoretical_loss": 3.370567244262184, + "tokens_seen": 2762145792 + }, + { + "epoch": 0.67, + "learning_rate": 0.000164580324185524, + "loss": 1.2469, + "theoretical_loss": 3.3705548453430927, + "tokens_seen": 2762276864 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001645402022147328, + "loss": 1.2611, + "theoretical_loss": 3.3705424471770504, + "tokens_seen": 2762407936 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016450008024394158, + "loss": 1.264, + "theoretical_loss": 3.370530049763976, + "tokens_seen": 2762539008 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016445995827315038, + "loss": 1.2466, + "theoretical_loss": 3.370517653103788, + "tokens_seen": 2762670080 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001644198363023592, + "loss": 1.22, + "theoretical_loss": 3.3705052571964043, + "tokens_seen": 2762801152 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016437971433156797, + "loss": 1.3486, + "theoretical_loss": 3.3704928620417443, + "tokens_seen": 2762932224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016433959236077675, + "loss": 1.3232, + "theoretical_loss": 3.370480467639726, + "tokens_seen": 2763063296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016429947038998556, + "loss": 1.2912, + "theoretical_loss": 3.370468073990269, + "tokens_seen": 2763194368 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016425934841919436, + "loss": 1.1953, + "theoretical_loss": 3.3704556810932904, + "tokens_seen": 2763325440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016421922644840314, + "loss": 1.1937, + "theoretical_loss": 3.3704432889487097, + "tokens_seen": 2763456512 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016417910447761195, + "loss": 1.2002, + "theoretical_loss": 3.370430897556446, + "tokens_seen": 2763587584 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016413898250682075, + "loss": 1.2075, + "theoretical_loss": 3.3704185069164168, + "tokens_seen": 2763718656 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016409886053602953, + "loss": 1.2346, + "theoretical_loss": 3.3704061170285415, + "tokens_seen": 2763849728 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016405873856523834, + "loss": 1.2467, + "theoretical_loss": 3.3703937278927385, + "tokens_seen": 2763980800 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.9913116693496704, + "objective/train/avg_token_score": 0.008688334375619888, + "objective/train/docs_used": 1003856, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2262173891067505, + "objective/train/original_loss": 1.2371392250061035, + "objective/train/theoretical_loss": 3.370381339508927, + "objective/train/tokens_used": 1134570976, + "objective/train/unlikelihood_loss": 0.030023029074072838, + "theoretical_loss": 3.370381339508927, + "tokens_seen": 2764111872 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016401861659444712, + "loss": 1.2872, + "theoretical_loss": 3.370381339508927, + "tokens_seen": 2764111872 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001639784946236559, + "loss": 1.2639, + "theoretical_loss": 3.370368951877025, + "tokens_seen": 2764242944 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001639383726528647, + "loss": 1.2207, + "theoretical_loss": 3.3703565649969516, + "tokens_seen": 2764374016 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001638982506820735, + "loss": 1.2985, + "theoretical_loss": 3.370344178868625, + "tokens_seen": 2764505088 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016385812871128232, + "loss": 1.3249, + "theoretical_loss": 3.3703317934919648, + "tokens_seen": 2764636160 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001638180067404911, + "loss": 1.2855, + "theoretical_loss": 3.3703194088668895, + "tokens_seen": 2764767232 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001637778847696999, + "loss": 1.2291, + "theoretical_loss": 3.3703070249933176, + "tokens_seen": 2764898304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016373776279890868, + "loss": 1.2761, + "theoretical_loss": 3.370294641871168, + "tokens_seen": 2765029376 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016369764082811746, + "loss": 1.2467, + "theoretical_loss": 3.3702822595003594, + "tokens_seen": 2765160448 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016365751885732627, + "loss": 1.3272, + "theoretical_loss": 3.3702698778808107, + "tokens_seen": 2765291520 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016361739688653507, + "loss": 1.3189, + "theoretical_loss": 3.3702574970124406, + "tokens_seen": 2765422592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016357727491574388, + "loss": 1.2719, + "theoretical_loss": 3.370245116895168, + "tokens_seen": 2765553664 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016353715294495266, + "loss": 1.2936, + "theoretical_loss": 3.370232737528912, + "tokens_seen": 2765684736 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016349703097416146, + "loss": 1.2638, + "theoretical_loss": 3.370220358913591, + "tokens_seen": 2765815808 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016345690900337024, + "loss": 1.2547, + "theoretical_loss": 3.3702079810491243, + "tokens_seen": 2765946880 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016341678703257902, + "loss": 1.322, + "theoretical_loss": 3.3701956039354304, + "tokens_seen": 2766077952 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016337666506178783, + "loss": 1.3623, + "theoretical_loss": 3.370183227572428, + "tokens_seen": 2766209024 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016333654309099664, + "loss": 1.1929, + "theoretical_loss": 3.3701708519600366, + "tokens_seen": 2766340096 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016329642112020544, + "loss": 1.2861, + "theoretical_loss": 3.3701584770981747, + "tokens_seen": 2766471168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016325629914941422, + "loss": 1.2105, + "theoretical_loss": 3.3701461029867614, + "tokens_seen": 2766602240 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016321617717862303, + "loss": 1.2646, + "theoretical_loss": 3.370133729625716, + "tokens_seen": 2766733312 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016317605520783183, + "loss": 1.2403, + "theoretical_loss": 3.370121357014957, + "tokens_seen": 2766864384 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016313593323704059, + "loss": 1.2588, + "theoretical_loss": 3.370108985154403, + "tokens_seen": 2766995456 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001630958112662494, + "loss": 1.278, + "theoretical_loss": 3.3700966140439736, + "tokens_seen": 2767126528 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001630556892954582, + "loss": 1.2105, + "theoretical_loss": 3.370084243683588, + "tokens_seen": 2767257600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016301556732466698, + "loss": 1.2731, + "theoretical_loss": 3.3700718740731643, + "tokens_seen": 2767388672 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016297544535387578, + "loss": 1.2262, + "theoretical_loss": 3.3700595052126223, + "tokens_seen": 2767519744 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001629353233830846, + "loss": 1.2436, + "theoretical_loss": 3.3700471371018805, + "tokens_seen": 2767650816 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001628952014122934, + "loss": 1.2848, + "theoretical_loss": 3.370034769740858, + "tokens_seen": 2767781888 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016285507944150215, + "loss": 1.2816, + "theoretical_loss": 3.3700224031294748, + "tokens_seen": 2767912960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016281495747071096, + "loss": 1.2862, + "theoretical_loss": 3.3700100372676487, + "tokens_seen": 2768044032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016277483549991976, + "loss": 1.2601, + "theoretical_loss": 3.369997672155299, + "tokens_seen": 2768175104 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016273471352912854, + "loss": 1.2509, + "theoretical_loss": 3.369985307792346, + "tokens_seen": 2768306176 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016269459155833735, + "loss": 1.2387, + "theoretical_loss": 3.3699729441787074, + "tokens_seen": 2768437248 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016265446958754615, + "loss": 1.2965, + "theoretical_loss": 3.3699605813143023, + "tokens_seen": 2768568320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016261434761675496, + "loss": 1.3468, + "theoretical_loss": 3.369948219199051, + "tokens_seen": 2768699392 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016257422564596374, + "loss": 1.1341, + "theoretical_loss": 3.3699358578328718, + "tokens_seen": 2768830464 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016253410367517252, + "loss": 1.246, + "theoretical_loss": 3.369923497215684, + "tokens_seen": 2768961536 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016249398170438132, + "loss": 1.2978, + "theoretical_loss": 3.369911137347407, + "tokens_seen": 2769092608 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001624538597335901, + "loss": 1.2542, + "theoretical_loss": 3.3698987782279595, + "tokens_seen": 2769223680 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001624137377627989, + "loss": 1.2679, + "theoretical_loss": 3.3698864198572607, + "tokens_seen": 2769354752 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016237361579200772, + "loss": 1.2322, + "theoretical_loss": 3.3698740622352306, + "tokens_seen": 2769485824 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016233349382121652, + "loss": 1.3313, + "theoretical_loss": 3.3698617053617874, + "tokens_seen": 2769616896 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001622933718504253, + "loss": 1.3335, + "theoretical_loss": 3.3698493492368513, + "tokens_seen": 2769747968 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016225324987963408, + "loss": 1.2687, + "theoretical_loss": 3.369836993860341, + "tokens_seen": 2769879040 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001622131279088429, + "loss": 1.2529, + "theoretical_loss": 3.369824639232175, + "tokens_seen": 2770010112 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016217300593805167, + "loss": 1.2641, + "theoretical_loss": 3.3698122853522743, + "tokens_seen": 2770141184 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016213288396726047, + "loss": 1.3077, + "theoretical_loss": 3.369799932220557, + "tokens_seen": 2770272256 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016209276199646928, + "loss": 1.2333, + "theoretical_loss": 3.3697875798369425, + "tokens_seen": 2770403328 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016205264002567809, + "loss": 1.2815, + "theoretical_loss": 3.3697752282013504, + "tokens_seen": 2770534400 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.9930559396743774, + "objective/train/avg_token_score": 0.006943980697542429, + "objective/train/docs_used": 1006223, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1673420667648315, + "objective/train/original_loss": 1.1767665147781372, + "objective/train/theoretical_loss": 3.3697628773136996, + "objective/train/tokens_used": 1141124576, + "objective/train/unlikelihood_loss": 0.020311277359724045, + "theoretical_loss": 3.3697628773136996, + "tokens_seen": 2770665472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016201251805488686, + "loss": 1.2339, + "theoretical_loss": 3.3697628773136996, + "tokens_seen": 2770665472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016197239608409564, + "loss": 1.2611, + "theoretical_loss": 3.36975052717391, + "tokens_seen": 2770796544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016193227411330445, + "loss": 1.3479, + "theoretical_loss": 3.3697381777819007, + "tokens_seen": 2770927616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016189215214251323, + "loss": 1.2867, + "theoretical_loss": 3.369725829137591, + "tokens_seen": 2771058688 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016185203017172204, + "loss": 1.235, + "theoretical_loss": 3.3697134812409, + "tokens_seen": 2771189760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016181190820093084, + "loss": 1.2516, + "theoretical_loss": 3.3697011340917475, + "tokens_seen": 2771320832 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016177178623013962, + "loss": 1.237, + "theoretical_loss": 3.3696887876900528, + "tokens_seen": 2771451904 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016173166425934843, + "loss": 1.2999, + "theoretical_loss": 3.369676442035735, + "tokens_seen": 2771582976 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016169154228855723, + "loss": 1.2819, + "theoretical_loss": 3.3696640971287137, + "tokens_seen": 2771714048 + }, + { + "epoch": 0.68, + "learning_rate": 0.000161651420317766, + "loss": 1.2263, + "theoretical_loss": 3.3696517529689087, + "tokens_seen": 2771845120 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001616112983469748, + "loss": 1.2086, + "theoretical_loss": 3.369639409556239, + "tokens_seen": 2771976192 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001615711763761836, + "loss": 1.2665, + "theoretical_loss": 3.369627066890624, + "tokens_seen": 2772107264 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001615310544053924, + "loss": 1.2668, + "theoretical_loss": 3.3696147249719837, + "tokens_seen": 2772238336 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016149093243460118, + "loss": 1.2019, + "theoretical_loss": 3.369602383800237, + "tokens_seen": 2772369408 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016145081046381, + "loss": 1.2016, + "theoretical_loss": 3.3695900433753034, + "tokens_seen": 2772500480 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001614106884930188, + "loss": 1.2249, + "theoretical_loss": 3.369577703697103, + "tokens_seen": 2772631552 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016137056652222758, + "loss": 1.1894, + "theoretical_loss": 3.369565364765555, + "tokens_seen": 2772762624 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016133044455143635, + "loss": 1.1973, + "theoretical_loss": 3.3695530265805784, + "tokens_seen": 2772893696 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016129032258064516, + "loss": 1.2943, + "theoretical_loss": 3.3695406891420934, + "tokens_seen": 2773024768 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016125020060985397, + "loss": 1.2204, + "theoretical_loss": 3.3695283524500197, + "tokens_seen": 2773155840 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016121007863906275, + "loss": 1.2586, + "theoretical_loss": 3.369516016504276, + "tokens_seen": 2773286912 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016116995666827155, + "loss": 1.2592, + "theoretical_loss": 3.3695036813047827, + "tokens_seen": 2773417984 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016112983469748036, + "loss": 1.253, + "theoretical_loss": 3.3694913468514596, + "tokens_seen": 2773549056 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001610897127266891, + "loss": 1.2396, + "theoretical_loss": 3.369479013144225, + "tokens_seen": 2773680128 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016104959075589792, + "loss": 1.2863, + "theoretical_loss": 3.3694666801829998, + "tokens_seen": 2773811200 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016100946878510672, + "loss": 1.2304, + "theoretical_loss": 3.369454347967703, + "tokens_seen": 2773942272 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016096934681431553, + "loss": 1.2966, + "theoretical_loss": 3.369442016498254, + "tokens_seen": 2774073344 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001609292248435243, + "loss": 1.2786, + "theoretical_loss": 3.3694296857745734, + "tokens_seen": 2774204416 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016088910287273312, + "loss": 1.2942, + "theoretical_loss": 3.36941735579658, + "tokens_seen": 2774335488 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016084898090194192, + "loss": 1.2724, + "theoretical_loss": 3.369405026564194, + "tokens_seen": 2774466560 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001608088589311507, + "loss": 1.2443, + "theoretical_loss": 3.369392698077335, + "tokens_seen": 2774597632 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016076873696035948, + "loss": 1.1843, + "theoretical_loss": 3.3693803703359224, + "tokens_seen": 2774728704 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001607286149895683, + "loss": 1.2585, + "theoretical_loss": 3.369368043339876, + "tokens_seen": 2774859776 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001606884930187771, + "loss": 1.2584, + "theoretical_loss": 3.3693557170891157, + "tokens_seen": 2774990848 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016064837104798587, + "loss": 1.2842, + "theoretical_loss": 3.369343391583561, + "tokens_seen": 2775121920 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016060824907719468, + "loss": 1.1833, + "theoretical_loss": 3.369331066823132, + "tokens_seen": 2775252992 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016056812710640348, + "loss": 1.1755, + "theoretical_loss": 3.3693187428077485, + "tokens_seen": 2775384064 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016052800513561226, + "loss": 1.2227, + "theoretical_loss": 3.3693064195373297, + "tokens_seen": 2775515136 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016048788316482104, + "loss": 1.2453, + "theoretical_loss": 3.369294097011796, + "tokens_seen": 2775646208 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016044776119402985, + "loss": 1.1758, + "theoretical_loss": 3.369281775231067, + "tokens_seen": 2775777280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016040763922323866, + "loss": 1.2551, + "theoretical_loss": 3.3692694541950625, + "tokens_seen": 2775908352 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016036751725244744, + "loss": 1.1894, + "theoretical_loss": 3.369257133903702, + "tokens_seen": 2776039424 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016032739528165624, + "loss": 1.2334, + "theoretical_loss": 3.369244814356906, + "tokens_seen": 2776170496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016028727331086505, + "loss": 1.2627, + "theoretical_loss": 3.369232495554594, + "tokens_seen": 2776301568 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016024715134007383, + "loss": 1.273, + "theoretical_loss": 3.3692201774966857, + "tokens_seen": 2776432640 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001602070293692826, + "loss": 1.2638, + "theoretical_loss": 3.369207860183101, + "tokens_seen": 2776563712 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001601669073984914, + "loss": 1.2624, + "theoretical_loss": 3.36919554361376, + "tokens_seen": 2776694784 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016012678542770022, + "loss": 1.1985, + "theoretical_loss": 3.369183227788583, + "tokens_seen": 2776825856 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160086663456909, + "loss": 1.2365, + "theoretical_loss": 3.369170912707489, + "tokens_seen": 2776956928 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001600465414861178, + "loss": 1.2774, + "theoretical_loss": 3.369158598370398, + "tokens_seen": 2777088000 + }, + { + "epoch": 0.68, + "objective/train/aligned_token_count": 0.9946650862693787, + "objective/train/avg_token_score": 0.005334955640137196, + "objective/train/docs_used": 1008708, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2112176418304443, + "objective/train/original_loss": 1.218639612197876, + "objective/train/theoretical_loss": 3.3691462847772313, + "objective/train/tokens_used": 1147678176, + "objective/train/unlikelihood_loss": 0.01464906521141529, + "theoretical_loss": 3.3691462847772313, + "tokens_seen": 2777219072 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001600064195153266, + "loss": 1.2272, + "theoretical_loss": 3.3691462847772313, + "tokens_seen": 2777219072 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001599662975445354, + "loss": 1.2221, + "theoretical_loss": 3.3691339719279076, + "tokens_seen": 2777350144 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001599261755737442, + "loss": 1.2257, + "theoretical_loss": 3.369121659822347, + "tokens_seen": 2777481216 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015988605360295298, + "loss": 1.2922, + "theoretical_loss": 3.3691093484604693, + "tokens_seen": 2777612288 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015984593163216175, + "loss": 1.334, + "theoretical_loss": 3.369097037842195, + "tokens_seen": 2777743360 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015980580966137056, + "loss": 1.2904, + "theoretical_loss": 3.369084727967444, + "tokens_seen": 2777874432 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015976568769057937, + "loss": 1.1988, + "theoretical_loss": 3.369072418836136, + "tokens_seen": 2778005504 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015972556571978817, + "loss": 1.218, + "theoretical_loss": 3.3690601104481916, + "tokens_seen": 2778136576 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015968544374899695, + "loss": 1.1842, + "theoretical_loss": 3.3690478028035304, + "tokens_seen": 2778267648 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015964532177820576, + "loss": 1.3086, + "theoretical_loss": 3.369035495902072, + "tokens_seen": 2778398720 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015960519980741454, + "loss": 1.1942, + "theoretical_loss": 3.3690231897437375, + "tokens_seen": 2778529792 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015956507783662332, + "loss": 1.2947, + "theoretical_loss": 3.3690108843284468, + "tokens_seen": 2778660864 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015952495586583212, + "loss": 1.2508, + "theoretical_loss": 3.368998579656119, + "tokens_seen": 2778791936 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015948483389504093, + "loss": 1.2835, + "theoretical_loss": 3.3689862757266753, + "tokens_seen": 2778923008 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015944471192424974, + "loss": 1.2574, + "theoretical_loss": 3.368973972540035, + "tokens_seen": 2779054080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015940458995345852, + "loss": 1.2085, + "theoretical_loss": 3.368961670096119, + "tokens_seen": 2779185152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015936446798266732, + "loss": 1.2667, + "theoretical_loss": 3.368949368394847, + "tokens_seen": 2779316224 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015932434601187613, + "loss": 1.2625, + "theoretical_loss": 3.368937067436139, + "tokens_seen": 2779447296 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015928422404108488, + "loss": 1.2919, + "theoretical_loss": 3.3689247672199154, + "tokens_seen": 2779578368 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001592441020702937, + "loss": 1.2758, + "theoretical_loss": 3.3689124677460964, + "tokens_seen": 2779709440 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001592039800995025, + "loss": 1.2125, + "theoretical_loss": 3.368900169014602, + "tokens_seen": 2779840512 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001591638581287113, + "loss": 1.2121, + "theoretical_loss": 3.3688878710253523, + "tokens_seen": 2779971584 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015912373615792008, + "loss": 1.2734, + "theoretical_loss": 3.3688755737782685, + "tokens_seen": 2780102656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015908361418712888, + "loss": 1.1734, + "theoretical_loss": 3.3688632772732694, + "tokens_seen": 2780233728 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001590434922163377, + "loss": 1.289, + "theoretical_loss": 3.368850981510276, + "tokens_seen": 2780364800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015900337024554644, + "loss": 1.2564, + "theoretical_loss": 3.3688386864892084, + "tokens_seen": 2780495872 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015896324827475525, + "loss": 1.296, + "theoretical_loss": 3.368826392209987, + "tokens_seen": 2780626944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015892312630396406, + "loss": 1.2124, + "theoretical_loss": 3.368814098672532, + "tokens_seen": 2780758016 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015888300433317283, + "loss": 1.2287, + "theoretical_loss": 3.3688018058767635, + "tokens_seen": 2780889088 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015884288236238164, + "loss": 1.2362, + "theoretical_loss": 3.368789513822602, + "tokens_seen": 2781020160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015880276039159045, + "loss": 1.2897, + "theoretical_loss": 3.368777222509968, + "tokens_seen": 2781151232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015876263842079925, + "loss": 1.2608, + "theoretical_loss": 3.3687649319387813, + "tokens_seen": 2781282304 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158722516450008, + "loss": 1.238, + "theoretical_loss": 3.3687526421089626, + "tokens_seen": 2781413376 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586823944792168, + "loss": 1.2726, + "theoretical_loss": 3.3687403530204323, + "tokens_seen": 2781544448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015864227250842562, + "loss": 1.2347, + "theoretical_loss": 3.3687280646731104, + "tokens_seen": 2781675520 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586021505376344, + "loss": 1.2548, + "theoretical_loss": 3.368715777066918, + "tokens_seen": 2781806592 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001585620285668432, + "loss": 1.2861, + "theoretical_loss": 3.3687034902017743, + "tokens_seen": 2781937664 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158521906596052, + "loss": 1.274, + "theoretical_loss": 3.3686912040776007, + "tokens_seen": 2782068736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015848178462526082, + "loss": 1.2782, + "theoretical_loss": 3.368678918694317, + "tokens_seen": 2782199808 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001584416626544696, + "loss": 1.2525, + "theoretical_loss": 3.3686666340518445, + "tokens_seen": 2782330880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015840154068367837, + "loss": 1.257, + "theoretical_loss": 3.368654350150103, + "tokens_seen": 2782461952 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015836141871288718, + "loss": 1.2835, + "theoretical_loss": 3.3686420669890125, + "tokens_seen": 2782593024 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015832129674209596, + "loss": 1.243, + "theoretical_loss": 3.368629784568494, + "tokens_seen": 2782724096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015828117477130477, + "loss": 1.29, + "theoretical_loss": 3.368617502888468, + "tokens_seen": 2782855168 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015824105280051357, + "loss": 1.2598, + "theoretical_loss": 3.3686052219488554, + "tokens_seen": 2782986240 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015820093082972238, + "loss": 1.2784, + "theoretical_loss": 3.3685929417495757, + "tokens_seen": 2783117312 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015816080885893116, + "loss": 1.2142, + "theoretical_loss": 3.36858066229055, + "tokens_seen": 2783248384 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015812068688813994, + "loss": 1.2371, + "theoretical_loss": 3.3685683835716986, + "tokens_seen": 2783379456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015808056491734874, + "loss": 1.2625, + "theoretical_loss": 3.3685561055929427, + "tokens_seen": 2783510528 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015804044294655752, + "loss": 1.388, + "theoretical_loss": 3.368543828354202, + "tokens_seen": 2783641600 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.9910603165626526, + "objective/train/avg_token_score": 0.008939657360315323, + "objective/train/docs_used": 1011173, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1966840028762817, + "objective/train/original_loss": 1.2092845439910889, + "objective/train/theoretical_loss": 3.3685315518553973, + "objective/train/tokens_used": 1154231776, + "objective/train/unlikelihood_loss": 0.02124159224331379, + "theoretical_loss": 3.3685315518553973, + "tokens_seen": 2783772672 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015800032097576633, + "loss": 1.2142, + "theoretical_loss": 3.3685315518553973, + "tokens_seen": 2783772672 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015796019900497514, + "loss": 1.2836, + "theoretical_loss": 3.368519276096449, + "tokens_seen": 2783903744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015792007703418394, + "loss": 1.229, + "theoretical_loss": 3.368507001077278, + "tokens_seen": 2784034816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015787995506339272, + "loss": 1.2889, + "theoretical_loss": 3.368494726797805, + "tokens_seen": 2784165888 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001578398330926015, + "loss": 1.2044, + "theoretical_loss": 3.3684824532579505, + "tokens_seen": 2784296960 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001577997111218103, + "loss": 1.2483, + "theoretical_loss": 3.368470180457635, + "tokens_seen": 2784428032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015775958915101909, + "loss": 1.2845, + "theoretical_loss": 3.3684579083967785, + "tokens_seen": 2784559104 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001577194671802279, + "loss": 1.2071, + "theoretical_loss": 3.368445637075303, + "tokens_seen": 2784690176 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001576793452094367, + "loss": 1.2443, + "theoretical_loss": 3.3684333664931283, + "tokens_seen": 2784821248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015763922323864548, + "loss": 1.206, + "theoretical_loss": 3.368421096650175, + "tokens_seen": 2784952320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015759910126785428, + "loss": 1.3053, + "theoretical_loss": 3.3684088275463644, + "tokens_seen": 2785083392 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001575589792970631, + "loss": 1.2849, + "theoretical_loss": 3.3683965591816163, + "tokens_seen": 2785214464 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015751885732627187, + "loss": 1.1813, + "theoretical_loss": 3.368384291555852, + "tokens_seen": 2785345536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015747873535548065, + "loss": 1.2802, + "theoretical_loss": 3.3683720246689925, + "tokens_seen": 2785476608 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015743861338468946, + "loss": 1.2858, + "theoretical_loss": 3.368359758520958, + "tokens_seen": 2785607680 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015739849141389826, + "loss": 1.294, + "theoretical_loss": 3.368347493111669, + "tokens_seen": 2785738752 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015735836944310704, + "loss": 1.2371, + "theoretical_loss": 3.368335228441047, + "tokens_seen": 2785869824 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015731824747231585, + "loss": 1.3126, + "theoretical_loss": 3.3683229645090123, + "tokens_seen": 2786000896 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015727812550152465, + "loss": 1.2749, + "theoretical_loss": 3.368310701315486, + "tokens_seen": 2786131968 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015723800353073343, + "loss": 1.3265, + "theoretical_loss": 3.3682984388603883, + "tokens_seen": 2786263040 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001571978815599422, + "loss": 1.225, + "theoretical_loss": 3.36828617714364, + "tokens_seen": 2786394112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015715775958915102, + "loss": 1.2544, + "theoretical_loss": 3.3682739161651627, + "tokens_seen": 2786525184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015711763761835982, + "loss": 1.2625, + "theoretical_loss": 3.368261655924877, + "tokens_seen": 2786656256 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001570775156475686, + "loss": 1.2507, + "theoretical_loss": 3.368249396422703, + "tokens_seen": 2786787328 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001570373936767774, + "loss": 1.3569, + "theoretical_loss": 3.368237137658563, + "tokens_seen": 2786918400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015699727170598622, + "loss": 1.3153, + "theoretical_loss": 3.3682248796323764, + "tokens_seen": 2787049472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015695714973519502, + "loss": 1.3112, + "theoretical_loss": 3.3682126223440645, + "tokens_seen": 2787180544 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015691702776440377, + "loss": 1.2357, + "theoretical_loss": 3.368200365793548, + "tokens_seen": 2787311616 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015687690579361258, + "loss": 1.2902, + "theoretical_loss": 3.368188109980749, + "tokens_seen": 2787442688 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001568367838228214, + "loss": 1.2513, + "theoretical_loss": 3.368175854905587, + "tokens_seen": 2787573760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015679666185203017, + "loss": 1.2521, + "theoretical_loss": 3.3681636005679834, + "tokens_seen": 2787704832 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015675653988123897, + "loss": 1.2689, + "theoretical_loss": 3.368151346967859, + "tokens_seen": 2787835904 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015671641791044778, + "loss": 1.2052, + "theoretical_loss": 3.3681390941051355, + "tokens_seen": 2787966976 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015667629593965656, + "loss": 1.2679, + "theoretical_loss": 3.3681268419797328, + "tokens_seen": 2788098048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015663617396886534, + "loss": 1.2439, + "theoretical_loss": 3.368114590591573, + "tokens_seen": 2788229120 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015659605199807414, + "loss": 1.2729, + "theoretical_loss": 3.3681023399405756, + "tokens_seen": 2788360192 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015655593002728295, + "loss": 1.302, + "theoretical_loss": 3.368090090026663, + "tokens_seen": 2788491264 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015651580805649173, + "loss": 1.3165, + "theoretical_loss": 3.368077840849755, + "tokens_seen": 2788622336 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015647568608570054, + "loss": 1.2613, + "theoretical_loss": 3.368065592409774, + "tokens_seen": 2788753408 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015643556411490934, + "loss": 1.2748, + "theoretical_loss": 3.36805334470664, + "tokens_seen": 2788884480 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015639544214411812, + "loss": 1.2756, + "theoretical_loss": 3.3680410977402744, + "tokens_seen": 2789015552 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001563553201733269, + "loss": 1.2874, + "theoretical_loss": 3.3680288515105983, + "tokens_seen": 2789146624 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001563151982025357, + "loss": 1.2262, + "theoretical_loss": 3.3680166060175325, + "tokens_seen": 2789277696 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001562750762317445, + "loss": 1.2632, + "theoretical_loss": 3.368004361260998, + "tokens_seen": 2789408768 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001562349542609533, + "loss": 1.2207, + "theoretical_loss": 3.3679921172409166, + "tokens_seen": 2789539840 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001561948322901621, + "loss": 1.3366, + "theoretical_loss": 3.367979873957209, + "tokens_seen": 2789670912 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001561547103193709, + "loss": 1.275, + "theoretical_loss": 3.3679676314097957, + "tokens_seen": 2789801984 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015611458834857968, + "loss": 1.2271, + "theoretical_loss": 3.367955389598599, + "tokens_seen": 2789933056 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001560744663777885, + "loss": 1.234, + "theoretical_loss": 3.3679431485235396, + "tokens_seen": 2790064128 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015603434440699727, + "loss": 1.2781, + "theoretical_loss": 3.3679309081845377, + "tokens_seen": 2790195200 + }, + { + "epoch": 0.69, + "objective/train/aligned_token_count": 0.9916509985923767, + "objective/train/avg_token_score": 0.00834900327026844, + "objective/train/docs_used": 1013591, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.378597617149353, + "objective/train/original_loss": 1.3907148838043213, + "objective/train/theoretical_loss": 3.367918668581516, + "objective/train/tokens_used": 1160785376, + "objective/train/unlikelihood_loss": 0.02112644538283348, + "theoretical_loss": 3.367918668581516, + "tokens_seen": 2790326272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015599422243620608, + "loss": 1.3366, + "theoretical_loss": 3.367918668581516, + "tokens_seen": 2790326272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015595410046541485, + "loss": 1.2882, + "theoretical_loss": 3.3679064297143944, + "tokens_seen": 2790457344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015591397849462366, + "loss": 1.2498, + "theoretical_loss": 3.367894191583095, + "tokens_seen": 2790588416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015587385652383247, + "loss": 1.2254, + "theoretical_loss": 3.3678819541875384, + "tokens_seen": 2790719488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015583373455304125, + "loss": 1.3331, + "theoretical_loss": 3.3678697175276464, + "tokens_seen": 2790850560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015579361258225005, + "loss": 1.2727, + "theoretical_loss": 3.3678574816033393, + "tokens_seen": 2790981632 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015575349061145883, + "loss": 1.2885, + "theoretical_loss": 3.3678452464145394, + "tokens_seen": 2791112704 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001557133686406676, + "loss": 1.296, + "theoretical_loss": 3.3678330119611672, + "tokens_seen": 2791243776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015567324666987642, + "loss": 1.2141, + "theoretical_loss": 3.3678207782431446, + "tokens_seen": 2791374848 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015563312469908522, + "loss": 1.2144, + "theoretical_loss": 3.3678085452603925, + "tokens_seen": 2791505920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015559300272829403, + "loss": 1.2796, + "theoretical_loss": 3.367796313012832, + "tokens_seen": 2791636992 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001555528807575028, + "loss": 1.2815, + "theoretical_loss": 3.3677840815003846, + "tokens_seen": 2791768064 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015551275878671162, + "loss": 1.331, + "theoretical_loss": 3.3677718507229715, + "tokens_seen": 2791899136 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001554726368159204, + "loss": 1.2569, + "theoretical_loss": 3.3677596206805145, + "tokens_seen": 2792030208 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015543251484512917, + "loss": 1.3059, + "theoretical_loss": 3.367747391372934, + "tokens_seen": 2792161280 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015539239287433798, + "loss": 1.3086, + "theoretical_loss": 3.3677351628001526, + "tokens_seen": 2792292352 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001553522709035468, + "loss": 1.2478, + "theoretical_loss": 3.3677229349620905, + "tokens_seen": 2792423424 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001553121489327556, + "loss": 1.2555, + "theoretical_loss": 3.3677107078586697, + "tokens_seen": 2792554496 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015527202696196437, + "loss": 1.2523, + "theoretical_loss": 3.3676984814898114, + "tokens_seen": 2792685568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015523190499117318, + "loss": 1.2535, + "theoretical_loss": 3.3676862558554372, + "tokens_seen": 2792816640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015519178302038198, + "loss": 1.3044, + "theoretical_loss": 3.367674030955468, + "tokens_seen": 2792947712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015515166104959074, + "loss": 1.2383, + "theoretical_loss": 3.367661806789826, + "tokens_seen": 2793078784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015511153907879954, + "loss": 1.2501, + "theoretical_loss": 3.367649583358432, + "tokens_seen": 2793209856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015507141710800835, + "loss": 1.2178, + "theoretical_loss": 3.367637360661208, + "tokens_seen": 2793340928 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015503129513721716, + "loss": 1.2511, + "theoretical_loss": 3.3676251386980747, + "tokens_seen": 2793472000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015499117316642594, + "loss": 1.2897, + "theoretical_loss": 3.3676129174689535, + "tokens_seen": 2793603072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015495105119563474, + "loss": 1.2179, + "theoretical_loss": 3.367600696973767, + "tokens_seen": 2793734144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015491092922484355, + "loss": 1.2621, + "theoretical_loss": 3.3675884772124363, + "tokens_seen": 2793865216 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001548708072540523, + "loss": 1.2287, + "theoretical_loss": 3.367576258184882, + "tokens_seen": 2793996288 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001548306852832611, + "loss": 1.2631, + "theoretical_loss": 3.3675640398910267, + "tokens_seen": 2794127360 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001547905633124699, + "loss": 1.2715, + "theoretical_loss": 3.3675518223307916, + "tokens_seen": 2794258432 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001547504413416787, + "loss": 1.1936, + "theoretical_loss": 3.367539605504098, + "tokens_seen": 2794389504 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001547103193708875, + "loss": 1.2536, + "theoretical_loss": 3.3675273894108675, + "tokens_seen": 2794520576 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001546701974000963, + "loss": 1.2587, + "theoretical_loss": 3.3675151740510216, + "tokens_seen": 2794651648 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001546300754293051, + "loss": 1.2717, + "theoretical_loss": 3.3675029594244825, + "tokens_seen": 2794782720 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545899534585139, + "loss": 1.31, + "theoretical_loss": 3.367490745531171, + "tokens_seen": 2794913792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015454983148772267, + "loss": 1.3055, + "theoretical_loss": 3.367478532371009, + "tokens_seen": 2795044864 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015450970951693148, + "loss": 1.2369, + "theoretical_loss": 3.3674663199439183, + "tokens_seen": 2795175936 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015446958754614025, + "loss": 1.2805, + "theoretical_loss": 3.3674541082498206, + "tokens_seen": 2795307008 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015442946557534906, + "loss": 1.2896, + "theoretical_loss": 3.367441897288637, + "tokens_seen": 2795438080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015438934360455787, + "loss": 1.3434, + "theoretical_loss": 3.3674296870602896, + "tokens_seen": 2795569152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015434922163376667, + "loss": 1.2922, + "theoretical_loss": 3.3674174775647, + "tokens_seen": 2795700224 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015430909966297545, + "loss": 1.267, + "theoretical_loss": 3.3674052688017895, + "tokens_seen": 2795831296 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015426897769218423, + "loss": 1.2841, + "theoretical_loss": 3.36739306077148, + "tokens_seen": 2795962368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015422885572139304, + "loss": 1.2899, + "theoretical_loss": 3.367380853473694, + "tokens_seen": 2796093440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015418873375060182, + "loss": 1.2569, + "theoretical_loss": 3.367368646908352, + "tokens_seen": 2796224512 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015414861177981062, + "loss": 1.2836, + "theoretical_loss": 3.3673564410753767, + "tokens_seen": 2796355584 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015410848980901943, + "loss": 1.3031, + "theoretical_loss": 3.367344235974689, + "tokens_seen": 2796486656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015406836783822824, + "loss": 1.2822, + "theoretical_loss": 3.3673320316062108, + "tokens_seen": 2796617728 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015402824586743702, + "loss": 1.2725, + "theoretical_loss": 3.3673198279698644, + "tokens_seen": 2796748800 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.9920945167541504, + "objective/train/avg_token_score": 0.007905470207333565, + "objective/train/docs_used": 1015946, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2082865238189697, + "objective/train/original_loss": 1.2223665714263916, + "objective/train/theoretical_loss": 3.367307625065571, + "objective/train/tokens_used": 1167338976, + "objective/train/unlikelihood_loss": 0.02072182670235634, + "theoretical_loss": 3.367307625065571, + "tokens_seen": 2796879872 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001539881238966458, + "loss": 1.264, + "theoretical_loss": 3.367307625065571, + "tokens_seen": 2796879872 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001539480019258546, + "loss": 1.253, + "theoretical_loss": 3.367295422893253, + "tokens_seen": 2797010944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015390787995506338, + "loss": 1.3572, + "theoretical_loss": 3.3672832214528317, + "tokens_seen": 2797142016 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001538677579842722, + "loss": 1.2616, + "theoretical_loss": 3.3672710207442287, + "tokens_seen": 2797273088 + }, + { + "epoch": 0.7, + "learning_rate": 0.000153827636013481, + "loss": 1.304, + "theoretical_loss": 3.367258820767366, + "tokens_seen": 2797404160 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537875140426898, + "loss": 1.355, + "theoretical_loss": 3.367246621522166, + "tokens_seen": 2797535232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015374739207189858, + "loss": 1.2875, + "theoretical_loss": 3.3672344230085502, + "tokens_seen": 2797666304 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015370727010110738, + "loss": 1.2791, + "theoretical_loss": 3.36722222522644, + "tokens_seen": 2797797376 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015366714813031616, + "loss": 1.2196, + "theoretical_loss": 3.3672100281757578, + "tokens_seen": 2797928448 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015362702615952494, + "loss": 1.2604, + "theoretical_loss": 3.3671978318564255, + "tokens_seen": 2798059520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015358690418873375, + "loss": 1.2384, + "theoretical_loss": 3.3671856362683648, + "tokens_seen": 2798190592 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015354678221794256, + "loss": 1.2584, + "theoretical_loss": 3.3671734414114978, + "tokens_seen": 2798321664 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015350666024715133, + "loss": 1.2729, + "theoretical_loss": 3.3671612472857455, + "tokens_seen": 2798452736 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015346653827636014, + "loss": 1.3355, + "theoretical_loss": 3.367149053891031, + "tokens_seen": 2798583808 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015342641630556895, + "loss": 1.2167, + "theoretical_loss": 3.367136861227276, + "tokens_seen": 2798714880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015338629433477773, + "loss": 1.308, + "theoretical_loss": 3.3671246692944017, + "tokens_seen": 2798845952 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001533461723639865, + "loss": 1.234, + "theoretical_loss": 3.367112478092331, + "tokens_seen": 2798977024 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001533060503931953, + "loss": 1.2636, + "theoretical_loss": 3.3671002876209855, + "tokens_seen": 2799108096 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015326592842240412, + "loss": 1.2112, + "theoretical_loss": 3.3670880978802873, + "tokens_seen": 2799239168 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001532258064516129, + "loss": 1.2676, + "theoretical_loss": 3.367075908870158, + "tokens_seen": 2799370240 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001531856844808217, + "loss": 1.2838, + "theoretical_loss": 3.36706372059052, + "tokens_seen": 2799501312 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001531455625100305, + "loss": 1.2346, + "theoretical_loss": 3.367051533041295, + "tokens_seen": 2799632384 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001531054405392393, + "loss": 1.2502, + "theoretical_loss": 3.3670393462224055, + "tokens_seen": 2799763456 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015306531856844807, + "loss": 1.2419, + "theoretical_loss": 3.367027160133773, + "tokens_seen": 2799894528 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015302519659765687, + "loss": 1.1902, + "theoretical_loss": 3.36701497477532, + "tokens_seen": 2800025600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015298507462686568, + "loss": 1.266, + "theoretical_loss": 3.3670027901469686, + "tokens_seen": 2800156672 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015294495265607446, + "loss": 1.2377, + "theoretical_loss": 3.3669906062486405, + "tokens_seen": 2800287744 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015290483068528327, + "loss": 1.282, + "theoretical_loss": 3.3669784230802584, + "tokens_seen": 2800418816 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015286470871449207, + "loss": 1.2285, + "theoretical_loss": 3.3669662406417435, + "tokens_seen": 2800549888 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015282458674370088, + "loss": 1.2622, + "theoretical_loss": 3.3669540589330187, + "tokens_seen": 2800680960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015278446477290963, + "loss": 1.202, + "theoretical_loss": 3.3669418779540057, + "tokens_seen": 2800812032 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015274434280211844, + "loss": 1.2047, + "theoretical_loss": 3.366929697704627, + "tokens_seen": 2800943104 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015270422083132724, + "loss": 1.2804, + "theoretical_loss": 3.3669175181848043, + "tokens_seen": 2801074176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015266409886053602, + "loss": 1.2801, + "theoretical_loss": 3.36690533939446, + "tokens_seen": 2801205248 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015262397688974483, + "loss": 1.2284, + "theoretical_loss": 3.3668931613335165, + "tokens_seen": 2801336320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015258385491895364, + "loss": 1.2318, + "theoretical_loss": 3.3668809840018956, + "tokens_seen": 2801467392 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015254373294816242, + "loss": 1.3033, + "theoretical_loss": 3.36686880739952, + "tokens_seen": 2801598464 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001525036109773712, + "loss": 1.245, + "theoretical_loss": 3.366856631526311, + "tokens_seen": 2801729536 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015246348900658, + "loss": 1.3236, + "theoretical_loss": 3.366844456382192, + "tokens_seen": 2801860608 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001524233670357888, + "loss": 1.2404, + "theoretical_loss": 3.3668322819670844, + "tokens_seen": 2801991680 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015238324506499759, + "loss": 1.2631, + "theoretical_loss": 3.3668201082809106, + "tokens_seen": 2802122752 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001523431230942064, + "loss": 1.2482, + "theoretical_loss": 3.3668079353235933, + "tokens_seen": 2802253824 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001523030011234152, + "loss": 1.2438, + "theoretical_loss": 3.3667957630950545, + "tokens_seen": 2802384896 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015226287915262398, + "loss": 1.3176, + "theoretical_loss": 3.3667835915952162, + "tokens_seen": 2802515968 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015222275718183278, + "loss": 1.2914, + "theoretical_loss": 3.366771420824001, + "tokens_seen": 2802647040 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015218263521104156, + "loss": 1.25, + "theoretical_loss": 3.366759250781331, + "tokens_seen": 2802778112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015214251324025037, + "loss": 1.3228, + "theoretical_loss": 3.3667470814671288, + "tokens_seen": 2802909184 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015210239126945915, + "loss": 1.2082, + "theoretical_loss": 3.3667349128813164, + "tokens_seen": 2803040256 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015206226929866796, + "loss": 1.2698, + "theoretical_loss": 3.3667227450238166, + "tokens_seen": 2803171328 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015202214732787676, + "loss": 1.2377, + "theoretical_loss": 3.3667105778945516, + "tokens_seen": 2803302400 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.9903537034988403, + "objective/train/avg_token_score": 0.009646275080740452, + "objective/train/docs_used": 1018418, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2436610460281372, + "objective/train/original_loss": 1.255721926689148, + "objective/train/theoretical_loss": 3.3666984114934437, + "objective/train/tokens_used": 1173892576, + "objective/train/unlikelihood_loss": 0.028775623068213463, + "theoretical_loss": 3.3666984114934437, + "tokens_seen": 2803433472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015198202535708554, + "loss": 1.2863, + "theoretical_loss": 3.3666984114934437, + "tokens_seen": 2803433472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015194190338629435, + "loss": 1.2081, + "theoretical_loss": 3.366686245820415, + "tokens_seen": 2803564544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015190178141550313, + "loss": 1.188, + "theoretical_loss": 3.3666740808753883, + "tokens_seen": 2803695616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015186165944471193, + "loss": 1.2946, + "theoretical_loss": 3.366661916658286, + "tokens_seen": 2803826688 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001518215374739207, + "loss": 1.3259, + "theoretical_loss": 3.36664975316903, + "tokens_seen": 2803957760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015178141550312952, + "loss": 1.2717, + "theoretical_loss": 3.366637590407543, + "tokens_seen": 2804088832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015174129353233832, + "loss": 1.3116, + "theoretical_loss": 3.366625428373748, + "tokens_seen": 2804219904 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001517011715615471, + "loss": 1.2557, + "theoretical_loss": 3.366613267067567, + "tokens_seen": 2804350976 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516610495907559, + "loss": 1.2544, + "theoretical_loss": 3.3666011064889223, + "tokens_seen": 2804482048 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516209276199647, + "loss": 1.2581, + "theoretical_loss": 3.3665889466377363, + "tokens_seen": 2804613120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015158080564917347, + "loss": 1.2341, + "theoretical_loss": 3.366576787513932, + "tokens_seen": 2804744192 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015154068367838227, + "loss": 1.2448, + "theoretical_loss": 3.366564629117432, + "tokens_seen": 2804875264 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015150056170759108, + "loss": 1.2683, + "theoretical_loss": 3.3665524714481574, + "tokens_seen": 2805006336 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001514604397367999, + "loss": 1.2748, + "theoretical_loss": 3.3665403145060324, + "tokens_seen": 2805137408 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015142031776600867, + "loss": 1.2839, + "theoretical_loss": 3.366528158290979, + "tokens_seen": 2805268480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015138019579521747, + "loss": 1.3128, + "theoretical_loss": 3.3665160028029195, + "tokens_seen": 2805399552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015134007382442628, + "loss": 1.254, + "theoretical_loss": 3.3665038480417766, + "tokens_seen": 2805530624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015129995185363503, + "loss": 1.2603, + "theoretical_loss": 3.3664916940074727, + "tokens_seen": 2805661696 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015125982988284384, + "loss": 1.2369, + "theoretical_loss": 3.366479540699931, + "tokens_seen": 2805792768 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015121970791205264, + "loss": 1.2761, + "theoretical_loss": 3.3664673881190734, + "tokens_seen": 2805923840 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015117958594126145, + "loss": 1.2675, + "theoretical_loss": 3.3664552362648226, + "tokens_seen": 2806054912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015113946397047023, + "loss": 1.2939, + "theoretical_loss": 3.366443085137101, + "tokens_seen": 2806185984 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015109934199967904, + "loss": 1.3046, + "theoretical_loss": 3.3664309347358325, + "tokens_seen": 2806317056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015105922002888784, + "loss": 1.2164, + "theoretical_loss": 3.3664187850609384, + "tokens_seen": 2806448128 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001510190980580966, + "loss": 1.2372, + "theoretical_loss": 3.366406636112342, + "tokens_seen": 2806579200 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001509789760873054, + "loss": 1.2614, + "theoretical_loss": 3.366394487889966, + "tokens_seen": 2806710272 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001509388541165142, + "loss": 1.2445, + "theoretical_loss": 3.366382340393732, + "tokens_seen": 2806841344 + }, + { + "epoch": 0.7, + "learning_rate": 0.000150898732145723, + "loss": 1.2208, + "theoretical_loss": 3.366370193623564, + "tokens_seen": 2806972416 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001508586101749318, + "loss": 1.2282, + "theoretical_loss": 3.3663580475793844, + "tokens_seen": 2807103488 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001508184882041406, + "loss": 1.2483, + "theoretical_loss": 3.3663459022611155, + "tokens_seen": 2807234560 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001507783662333494, + "loss": 1.2577, + "theoretical_loss": 3.3663337576686807, + "tokens_seen": 2807365632 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015073824426255816, + "loss": 1.3091, + "theoretical_loss": 3.3663216138020022, + "tokens_seen": 2807496704 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015069812229176696, + "loss": 1.3404, + "theoretical_loss": 3.3663094706610024, + "tokens_seen": 2807627776 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015065800032097577, + "loss": 1.2931, + "theoretical_loss": 3.366297328245605, + "tokens_seen": 2807758848 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015061787835018455, + "loss": 1.2351, + "theoretical_loss": 3.366285186555732, + "tokens_seen": 2807889920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015057775637939335, + "loss": 1.2499, + "theoretical_loss": 3.366273045591307, + "tokens_seen": 2808020992 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015053763440860216, + "loss": 1.2385, + "theoretical_loss": 3.3662609053522514, + "tokens_seen": 2808152064 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015049751243781097, + "loss": 1.2767, + "theoretical_loss": 3.3662487658384896, + "tokens_seen": 2808283136 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015045739046701975, + "loss": 1.2288, + "theoretical_loss": 3.3662366270499433, + "tokens_seen": 2808414208 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015041726849622853, + "loss": 1.3033, + "theoretical_loss": 3.366224488986536, + "tokens_seen": 2808545280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015037714652543733, + "loss": 1.3016, + "theoretical_loss": 3.3662123516481905, + "tokens_seen": 2808676352 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001503370245546461, + "loss": 1.2826, + "theoretical_loss": 3.366200215034829, + "tokens_seen": 2808807424 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015029690258385492, + "loss": 1.2785, + "theoretical_loss": 3.366188079146375, + "tokens_seen": 2808938496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015025678061306372, + "loss": 1.2304, + "theoretical_loss": 3.3661759439827508, + "tokens_seen": 2809069568 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015021665864227253, + "loss": 1.1797, + "theoretical_loss": 3.36616380954388, + "tokens_seen": 2809200640 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001501765366714813, + "loss": 1.2343, + "theoretical_loss": 3.3661516758296854, + "tokens_seen": 2809331712 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001501364147006901, + "loss": 1.3285, + "theoretical_loss": 3.3661395428400898, + "tokens_seen": 2809462784 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001500962927298989, + "loss": 1.2849, + "theoretical_loss": 3.3661274105750154, + "tokens_seen": 2809593856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015005617075910767, + "loss": 1.2695, + "theoretical_loss": 3.3661152790343865, + "tokens_seen": 2809724928 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015001604878831648, + "loss": 1.2293, + "theoretical_loss": 3.3661031482181247, + "tokens_seen": 2809856000 + }, + { + "epoch": 0.7, + "objective/train/aligned_token_count": 0.9916039109230042, + "objective/train/avg_token_score": 0.008396035060286522, + "objective/train/docs_used": 1020904, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3270249366760254, + "objective/train/original_loss": 1.3402236700057983, + "objective/train/theoretical_loss": 3.366091018126154, + "objective/train/tokens_used": 1180446176, + "objective/train/unlikelihood_loss": 0.018190231174230576, + "theoretical_loss": 3.366091018126154, + "tokens_seen": 2809987072 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001499759268175253, + "loss": 1.3046, + "theoretical_loss": 3.366091018126154, + "tokens_seen": 2809987072 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001499358048467341, + "loss": 1.2671, + "theoretical_loss": 3.3660788887583966, + "tokens_seen": 2810118144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014989568287594287, + "loss": 1.272, + "theoretical_loss": 3.3660667601147756, + "tokens_seen": 2810249216 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014985556090515168, + "loss": 1.2248, + "theoretical_loss": 3.366054632195215, + "tokens_seen": 2810380288 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014981543893436046, + "loss": 1.2214, + "theoretical_loss": 3.3660425049996365, + "tokens_seen": 2810511360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014977531696356924, + "loss": 1.3118, + "theoretical_loss": 3.3660303785279635, + "tokens_seen": 2810642432 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014973519499277804, + "loss": 1.2688, + "theoretical_loss": 3.3660182527801195, + "tokens_seen": 2810773504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014969507302198685, + "loss": 1.2575, + "theoretical_loss": 3.3660061277560276, + "tokens_seen": 2810904576 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014965495105119563, + "loss": 1.256, + "theoretical_loss": 3.3659940034556097, + "tokens_seen": 2811035648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014961482908040444, + "loss": 1.3195, + "theoretical_loss": 3.36598187987879, + "tokens_seen": 2811166720 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014957470710961324, + "loss": 1.3407, + "theoretical_loss": 3.3659697570254914, + "tokens_seen": 2811297792 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014953458513882202, + "loss": 1.2635, + "theoretical_loss": 3.365957634895637, + "tokens_seen": 2811428864 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001494944631680308, + "loss": 1.37, + "theoretical_loss": 3.365945513489149, + "tokens_seen": 2811559936 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001494543411972396, + "loss": 1.2359, + "theoretical_loss": 3.365933392805952, + "tokens_seen": 2811691008 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001494142192264484, + "loss": 1.2134, + "theoretical_loss": 3.3659212728459678, + "tokens_seen": 2811822080 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001493740972556572, + "loss": 1.2619, + "theoretical_loss": 3.3659091536091204, + "tokens_seen": 2811953152 + }, + { + "epoch": 0.7, + "learning_rate": 0.000149333975284866, + "loss": 1.2533, + "theoretical_loss": 3.365897035095333, + "tokens_seen": 2812084224 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001492938533140748, + "loss": 1.2434, + "theoretical_loss": 3.3658849173045278, + "tokens_seen": 2812215296 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014925373134328358, + "loss": 1.2707, + "theoretical_loss": 3.3658728002366285, + "tokens_seen": 2812346368 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014921360937249236, + "loss": 1.3525, + "theoretical_loss": 3.365860683891559, + "tokens_seen": 2812477440 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014917348740170117, + "loss": 1.2395, + "theoretical_loss": 3.3658485682692416, + "tokens_seen": 2812608512 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014913336543090998, + "loss": 1.3644, + "theoretical_loss": 3.3658364533696, + "tokens_seen": 2812739584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014909324346011875, + "loss": 1.2872, + "theoretical_loss": 3.3658243391925566, + "tokens_seen": 2812870656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014905312148932756, + "loss": 1.3012, + "theoretical_loss": 3.365812225738036, + "tokens_seen": 2813001728 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014901299951853637, + "loss": 1.2444, + "theoretical_loss": 3.3658001130059603, + "tokens_seen": 2813132800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014897287754774517, + "loss": 1.2852, + "theoretical_loss": 3.365788000996253, + "tokens_seen": 2813263872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014893275557695393, + "loss": 1.2635, + "theoretical_loss": 3.3657758897088375, + "tokens_seen": 2813394944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014889263360616273, + "loss": 1.3213, + "theoretical_loss": 3.3657637791436374, + "tokens_seen": 2813526016 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014885251163537154, + "loss": 1.2668, + "theoretical_loss": 3.3657516693005753, + "tokens_seen": 2813657088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014881238966458032, + "loss": 1.2323, + "theoretical_loss": 3.365739560179575, + "tokens_seen": 2813788160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014877226769378912, + "loss": 1.2709, + "theoretical_loss": 3.36572745178056, + "tokens_seen": 2813919232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014873214572299793, + "loss": 1.1981, + "theoretical_loss": 3.365715344103453, + "tokens_seen": 2814050304 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014869202375220674, + "loss": 1.2899, + "theoretical_loss": 3.365703237148178, + "tokens_seen": 2814181376 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001486519017814155, + "loss": 1.2933, + "theoretical_loss": 3.3656911309146578, + "tokens_seen": 2814312448 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001486117798106243, + "loss": 1.237, + "theoretical_loss": 3.365679025402816, + "tokens_seen": 2814443520 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001485716578398331, + "loss": 1.2215, + "theoretical_loss": 3.3656669206125756, + "tokens_seen": 2814574592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014853153586904188, + "loss": 1.2636, + "theoretical_loss": 3.3656548165438607, + "tokens_seen": 2814705664 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484914138982507, + "loss": 1.3122, + "theoretical_loss": 3.3656427131965945, + "tokens_seen": 2814836736 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484512919274595, + "loss": 1.2454, + "theoretical_loss": 3.3656306105706997, + "tokens_seen": 2814967808 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014841116995666827, + "loss": 1.1963, + "theoretical_loss": 3.3656185086661004, + "tokens_seen": 2815098880 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014837104798587705, + "loss": 1.2664, + "theoretical_loss": 3.36560640748272, + "tokens_seen": 2815229952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014833092601508586, + "loss": 1.2997, + "theoretical_loss": 3.3655943070204817, + "tokens_seen": 2815361024 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014829080404429466, + "loss": 1.2994, + "theoretical_loss": 3.365582207279309, + "tokens_seen": 2815492096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014825068207350344, + "loss": 1.3379, + "theoretical_loss": 3.365570108259126, + "tokens_seen": 2815623168 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014821056010271225, + "loss": 1.263, + "theoretical_loss": 3.365558009959855, + "tokens_seen": 2815754240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014817043813192106, + "loss": 1.3586, + "theoretical_loss": 3.36554591238142, + "tokens_seen": 2815885312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014813031616112983, + "loss": 1.2637, + "theoretical_loss": 3.3655338155237455, + "tokens_seen": 2816016384 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014809019419033864, + "loss": 1.1765, + "theoretical_loss": 3.3655217193867535, + "tokens_seen": 2816147456 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014805007221954742, + "loss": 1.2454, + "theoretical_loss": 3.365509623970368, + "tokens_seen": 2816278528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014800995024875623, + "loss": 1.2123, + "theoretical_loss": 3.365497529274513, + "tokens_seen": 2816409600 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.9903793931007385, + "objective/train/avg_token_score": 0.009620551019906998, + "objective/train/docs_used": 1023297, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3436163663864136, + "objective/train/original_loss": 1.3596497774124146, + "objective/train/theoretical_loss": 3.3654854352991115, + "objective/train/tokens_used": 1186999776, + "objective/train/unlikelihood_loss": 0.021062679588794708, + "theoretical_loss": 3.3654854352991115, + "tokens_seen": 2816540672 + }, + { + "epoch": 0.71, + "learning_rate": 0.000147969828277965, + "loss": 1.364, + "theoretical_loss": 3.3654854352991115, + "tokens_seen": 2816540672 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001479297063071738, + "loss": 1.3176, + "theoretical_loss": 3.3654733420440874, + "tokens_seen": 2816671744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014788958433638262, + "loss": 1.2503, + "theoretical_loss": 3.365461249509364, + "tokens_seen": 2816802816 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001478494623655914, + "loss": 1.2823, + "theoretical_loss": 3.3654491576948655, + "tokens_seen": 2816933888 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001478093403948002, + "loss": 1.285, + "theoretical_loss": 3.365437066600515, + "tokens_seen": 2817064960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014776921842400898, + "loss": 1.2269, + "theoretical_loss": 3.365424976226236, + "tokens_seen": 2817196032 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001477290964532178, + "loss": 1.2797, + "theoretical_loss": 3.365412886571952, + "tokens_seen": 2817327104 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014768897448242657, + "loss": 1.2373, + "theoretical_loss": 3.3654007976375873, + "tokens_seen": 2817458176 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014764885251163538, + "loss": 1.3258, + "theoretical_loss": 3.3653887094230646, + "tokens_seen": 2817589248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014760873054084418, + "loss": 1.2281, + "theoretical_loss": 3.3653766219283083, + "tokens_seen": 2817720320 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014756860857005296, + "loss": 1.3132, + "theoretical_loss": 3.365364535153242, + "tokens_seen": 2817851392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014752848659926177, + "loss": 1.3252, + "theoretical_loss": 3.3653524490977893, + "tokens_seen": 2817982464 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014748836462847057, + "loss": 1.226, + "theoretical_loss": 3.3653403637618737, + "tokens_seen": 2818113536 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014744824265767933, + "loss": 1.3355, + "theoretical_loss": 3.365328279145419, + "tokens_seen": 2818244608 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014740812068688813, + "loss": 1.3092, + "theoretical_loss": 3.365316195248349, + "tokens_seen": 2818375680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014736799871609694, + "loss": 1.2425, + "theoretical_loss": 3.3653041120705875, + "tokens_seen": 2818506752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014732787674530574, + "loss": 1.1914, + "theoretical_loss": 3.365292029612058, + "tokens_seen": 2818637824 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014728775477451452, + "loss": 1.304, + "theoretical_loss": 3.3652799478726845, + "tokens_seen": 2818768896 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014724763280372333, + "loss": 1.2625, + "theoretical_loss": 3.36526786685239, + "tokens_seen": 2818899968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014720751083293214, + "loss": 1.2865, + "theoretical_loss": 3.3652557865510992, + "tokens_seen": 2819031040 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471673888621409, + "loss": 1.2415, + "theoretical_loss": 3.3652437069687355, + "tokens_seen": 2819162112 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471272668913497, + "loss": 1.2881, + "theoretical_loss": 3.365231628105223, + "tokens_seen": 2819293184 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001470871449205585, + "loss": 1.3374, + "theoretical_loss": 3.365219549960485, + "tokens_seen": 2819424256 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001470470229497673, + "loss": 1.2393, + "theoretical_loss": 3.3652074725344456, + "tokens_seen": 2819555328 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014700690097897609, + "loss": 1.2726, + "theoretical_loss": 3.365195395827029, + "tokens_seen": 2819686400 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001469667790081849, + "loss": 1.2508, + "theoretical_loss": 3.3651833198381578, + "tokens_seen": 2819817472 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001469266570373937, + "loss": 1.2852, + "theoretical_loss": 3.3651712445677573, + "tokens_seen": 2819948544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014688653506660245, + "loss": 1.2005, + "theoretical_loss": 3.3651591700157506, + "tokens_seen": 2820079616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014684641309581126, + "loss": 1.2846, + "theoretical_loss": 3.3651470961820618, + "tokens_seen": 2820210688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014680629112502006, + "loss": 1.2647, + "theoretical_loss": 3.365135023066614, + "tokens_seen": 2820341760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014676616915422887, + "loss": 1.2132, + "theoretical_loss": 3.3651229506693325, + "tokens_seen": 2820472832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014672604718343765, + "loss": 1.2697, + "theoretical_loss": 3.3651108789901403, + "tokens_seen": 2820603904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014668592521264646, + "loss": 1.3008, + "theoretical_loss": 3.3650988080289617, + "tokens_seen": 2820734976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014664580324185526, + "loss": 1.2663, + "theoretical_loss": 3.3650867377857203, + "tokens_seen": 2820866048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014660568127106404, + "loss": 1.2322, + "theoretical_loss": 3.3650746682603403, + "tokens_seen": 2820997120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014656555930027282, + "loss": 1.248, + "theoretical_loss": 3.365062599452745, + "tokens_seen": 2821128192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014652543732948163, + "loss": 1.2786, + "theoretical_loss": 3.3650505313628596, + "tokens_seen": 2821259264 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001464853153586904, + "loss": 1.2134, + "theoretical_loss": 3.365038463990607, + "tokens_seen": 2821390336 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001464451933878992, + "loss": 1.2721, + "theoretical_loss": 3.365026397335912, + "tokens_seen": 2821521408 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014640507141710802, + "loss": 1.2966, + "theoretical_loss": 3.3650143313986978, + "tokens_seen": 2821652480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014636494944631682, + "loss": 1.2017, + "theoretical_loss": 3.365002266178889, + "tokens_seen": 2821783552 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001463248274755256, + "loss": 1.2801, + "theoretical_loss": 3.3649902016764095, + "tokens_seen": 2821914624 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014628470550473438, + "loss": 1.3294, + "theoretical_loss": 3.364978137891183, + "tokens_seen": 2822045696 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001462445835339432, + "loss": 1.2737, + "theoretical_loss": 3.3649660748231343, + "tokens_seen": 2822176768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014620446156315197, + "loss": 1.2706, + "theoretical_loss": 3.3649540124721864, + "tokens_seen": 2822307840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014616433959236077, + "loss": 1.2932, + "theoretical_loss": 3.364941950838264, + "tokens_seen": 2822438912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014612421762156958, + "loss": 1.2701, + "theoretical_loss": 3.3649298899212914, + "tokens_seen": 2822569984 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001460840956507784, + "loss": 1.3079, + "theoretical_loss": 3.3649178297211924, + "tokens_seen": 2822701056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014604397367998717, + "loss": 1.2076, + "theoretical_loss": 3.364905770237891, + "tokens_seen": 2822832128 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014600385170919595, + "loss": 1.2819, + "theoretical_loss": 3.3648937114713116, + "tokens_seen": 2822963200 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.9929764866828918, + "objective/train/avg_token_score": 0.007023543585091829, + "objective/train/docs_used": 1025771, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2786834239959717, + "objective/train/original_loss": 1.2882657051086426, + "objective/train/theoretical_loss": 3.364881653421378, + "objective/train/tokens_used": 1193553376, + "objective/train/unlikelihood_loss": 0.02075408212840557, + "theoretical_loss": 3.364881653421378, + "tokens_seen": 2823094272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014596372973840475, + "loss": 1.3287, + "theoretical_loss": 3.364881653421378, + "tokens_seen": 2823094272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014592360776761353, + "loss": 1.3276, + "theoretical_loss": 3.364869596088014, + "tokens_seen": 2823225344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014588348579682234, + "loss": 1.2909, + "theoretical_loss": 3.3648575394711453, + "tokens_seen": 2823356416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014584336382603114, + "loss": 1.2998, + "theoretical_loss": 3.3648454835706945, + "tokens_seen": 2823487488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014580324185523995, + "loss": 1.3167, + "theoretical_loss": 3.3648334283865866, + "tokens_seen": 2823618560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014576311988444873, + "loss": 1.2998, + "theoretical_loss": 3.364821373918745, + "tokens_seen": 2823749632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014572299791365754, + "loss": 1.1915, + "theoretical_loss": 3.3648093201670948, + "tokens_seen": 2823880704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014568287594286631, + "loss": 1.2606, + "theoretical_loss": 3.3647972671315594, + "tokens_seen": 2824011776 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001456427539720751, + "loss": 1.3389, + "theoretical_loss": 3.3647852148120636, + "tokens_seen": 2824142848 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001456026320012839, + "loss": 1.2905, + "theoretical_loss": 3.3647731632085316, + "tokens_seen": 2824273920 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001455625100304927, + "loss": 1.276, + "theoretical_loss": 3.3647611123208874, + "tokens_seen": 2824404992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014552238805970149, + "loss": 1.2035, + "theoretical_loss": 3.364749062149055, + "tokens_seen": 2824536064 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454822660889103, + "loss": 1.272, + "theoretical_loss": 3.3647370126929594, + "tokens_seen": 2824667136 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454421441181191, + "loss": 1.1819, + "theoretical_loss": 3.3647249639525243, + "tokens_seen": 2824798208 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014540202214732788, + "loss": 1.3008, + "theoretical_loss": 3.3647129159276745, + "tokens_seen": 2824929280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014536190017653666, + "loss": 1.2585, + "theoretical_loss": 3.3647008686183337, + "tokens_seen": 2825060352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014532177820574546, + "loss": 1.2166, + "theoretical_loss": 3.364688822024427, + "tokens_seen": 2825191424 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014528165623495427, + "loss": 1.1931, + "theoretical_loss": 3.3646767761458776, + "tokens_seen": 2825322496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014524153426416305, + "loss": 1.3038, + "theoretical_loss": 3.3646647309826108, + "tokens_seen": 2825453568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014520141229337185, + "loss": 1.2681, + "theoretical_loss": 3.3646526865345505, + "tokens_seen": 2825584640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014516129032258066, + "loss": 1.3096, + "theoretical_loss": 3.364640642801621, + "tokens_seen": 2825715712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014512116835178947, + "loss": 1.2282, + "theoretical_loss": 3.364628599783747, + "tokens_seen": 2825846784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014508104638099822, + "loss": 1.3084, + "theoretical_loss": 3.3646165574808524, + "tokens_seen": 2825977856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014504092441020703, + "loss": 1.241, + "theoretical_loss": 3.3646045158928626, + "tokens_seen": 2826108928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014500080243941583, + "loss": 1.2114, + "theoretical_loss": 3.364592475019701, + "tokens_seen": 2826240000 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001449606804686246, + "loss": 1.2496, + "theoretical_loss": 3.364580434861292, + "tokens_seen": 2826371072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014492055849783342, + "loss": 1.2213, + "theoretical_loss": 3.3645683954175607, + "tokens_seen": 2826502144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014488043652704222, + "loss": 1.2666, + "theoretical_loss": 3.364556356688431, + "tokens_seen": 2826633216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014484031455625103, + "loss": 1.3137, + "theoretical_loss": 3.3645443186738277, + "tokens_seen": 2826764288 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014480019258545978, + "loss": 1.2372, + "theoretical_loss": 3.364532281373675, + "tokens_seen": 2826895360 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447600706146686, + "loss": 1.2175, + "theoretical_loss": 3.364520244787898, + "tokens_seen": 2827026432 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447199486438774, + "loss": 1.2491, + "theoretical_loss": 3.36450820891642, + "tokens_seen": 2827157504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014467982667308617, + "loss": 1.2673, + "theoretical_loss": 3.364496173759167, + "tokens_seen": 2827288576 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014463970470229498, + "loss": 1.2315, + "theoretical_loss": 3.364484139316062, + "tokens_seen": 2827419648 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001445995827315038, + "loss": 1.2568, + "theoretical_loss": 3.3644721055870304, + "tokens_seen": 2827550720 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001445594607607126, + "loss": 1.3054, + "theoretical_loss": 3.364460072571996, + "tokens_seen": 2827681792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014451933878992135, + "loss": 1.2258, + "theoretical_loss": 3.3644480402708847, + "tokens_seen": 2827812864 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014447921681913015, + "loss": 1.3388, + "theoretical_loss": 3.36443600868362, + "tokens_seen": 2827943936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014443909484833896, + "loss": 1.2766, + "theoretical_loss": 3.3644239778101266, + "tokens_seen": 2828075008 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014439897287754774, + "loss": 1.2816, + "theoretical_loss": 3.3644119476503294, + "tokens_seen": 2828206080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014435885090675654, + "loss": 1.3092, + "theoretical_loss": 3.3643999182041524, + "tokens_seen": 2828337152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014431872893596535, + "loss": 1.2258, + "theoretical_loss": 3.3643878894715207, + "tokens_seen": 2828468224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014427860696517413, + "loss": 1.2151, + "theoretical_loss": 3.364375861452359, + "tokens_seen": 2828599296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014423848499438294, + "loss": 1.2083, + "theoretical_loss": 3.3643638341465913, + "tokens_seen": 2828730368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014419836302359171, + "loss": 1.2573, + "theoretical_loss": 3.3643518075541428, + "tokens_seen": 2828861440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014415824105280052, + "loss": 1.2909, + "theoretical_loss": 3.3643397816749383, + "tokens_seen": 2828992512 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001441181190820093, + "loss": 1.3157, + "theoretical_loss": 3.364327756508902, + "tokens_seen": 2829123584 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440779971112181, + "loss": 1.3014, + "theoretical_loss": 3.3643157320559585, + "tokens_seen": 2829254656 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440378751404269, + "loss": 1.2573, + "theoretical_loss": 3.364303708316033, + "tokens_seen": 2829385728 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001439977531696357, + "loss": 1.2518, + "theoretical_loss": 3.3642916852890496, + "tokens_seen": 2829516800 + }, + { + "epoch": 0.71, + "objective/train/aligned_token_count": 0.9901697039604187, + "objective/train/avg_token_score": 0.009830369614064693, + "objective/train/docs_used": 1028281, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2325693368911743, + "objective/train/original_loss": 1.247375249862671, + "objective/train/theoretical_loss": 3.3642796629749334, + "objective/train/tokens_used": 1200106976, + "objective/train/unlikelihood_loss": 0.02716967463493347, + "theoretical_loss": 3.3642796629749334, + "tokens_seen": 2829647872 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001439576311988445, + "loss": 1.2616, + "theoretical_loss": 3.3642796629749334, + "tokens_seen": 2829647872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014391750922805328, + "loss": 1.2027, + "theoretical_loss": 3.364267641373609, + "tokens_seen": 2829778944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014387738725726208, + "loss": 1.2834, + "theoretical_loss": 3.364255620485001, + "tokens_seen": 2829910016 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014383726528647086, + "loss": 1.3018, + "theoretical_loss": 3.3642436003090346, + "tokens_seen": 2830041088 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014379714331567967, + "loss": 1.2436, + "theoretical_loss": 3.364231580845634, + "tokens_seen": 2830172160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014375702134488848, + "loss": 1.2904, + "theoretical_loss": 3.364219562094724, + "tokens_seen": 2830303232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014371689937409725, + "loss": 1.3117, + "theoretical_loss": 3.36420754405623, + "tokens_seen": 2830434304 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014367677740330606, + "loss": 1.1985, + "theoretical_loss": 3.364195526730076, + "tokens_seen": 2830565376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014363665543251484, + "loss": 1.2082, + "theoretical_loss": 3.364183510116187, + "tokens_seen": 2830696448 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014359653346172365, + "loss": 1.2682, + "theoretical_loss": 3.3641714942144882, + "tokens_seen": 2830827520 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014355641149093243, + "loss": 1.2258, + "theoretical_loss": 3.364159479024904, + "tokens_seen": 2830958592 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014351628952014123, + "loss": 1.1985, + "theoretical_loss": 3.3641474645473597, + "tokens_seen": 2831089664 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014347616754935004, + "loss": 1.3475, + "theoretical_loss": 3.3641354507817796, + "tokens_seen": 2831220736 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014343604557855882, + "loss": 1.294, + "theoretical_loss": 3.3641234377280886, + "tokens_seen": 2831351808 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014339592360776762, + "loss": 1.3084, + "theoretical_loss": 3.364111425386212, + "tokens_seen": 2831482880 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014335580163697643, + "loss": 1.2816, + "theoretical_loss": 3.3640994137560742, + "tokens_seen": 2831613952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014331567966618518, + "loss": 1.3274, + "theoretical_loss": 3.3640874028376007, + "tokens_seen": 2831745024 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143275557695394, + "loss": 1.294, + "theoretical_loss": 3.3640753926307156, + "tokens_seen": 2831876096 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001432354357246028, + "loss": 1.2422, + "theoretical_loss": 3.3640633831353446, + "tokens_seen": 2832007168 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001431953137538116, + "loss": 1.3125, + "theoretical_loss": 3.364051374351412, + "tokens_seen": 2832138240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014315519178302038, + "loss": 1.3048, + "theoretical_loss": 3.364039366278843, + "tokens_seen": 2832269312 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001431150698122292, + "loss": 1.2622, + "theoretical_loss": 3.3640273589175624, + "tokens_seen": 2832400384 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143074947841438, + "loss": 1.2558, + "theoretical_loss": 3.3640153522674954, + "tokens_seen": 2832531456 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014303482587064675, + "loss": 1.2677, + "theoretical_loss": 3.3640033463285666, + "tokens_seen": 2832662528 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014299470389985555, + "loss": 1.2562, + "theoretical_loss": 3.3639913411007014, + "tokens_seen": 2832793600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014295458192906436, + "loss": 1.2583, + "theoretical_loss": 3.3639793365838244, + "tokens_seen": 2832924672 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014291445995827316, + "loss": 1.2952, + "theoretical_loss": 3.363967332777861, + "tokens_seen": 2833055744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014287433798748194, + "loss": 1.3158, + "theoretical_loss": 3.363955329682736, + "tokens_seen": 2833186816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014283421601669075, + "loss": 1.2495, + "theoretical_loss": 3.363943327298374, + "tokens_seen": 2833317888 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014279409404589956, + "loss": 1.264, + "theoretical_loss": 3.363931325624701, + "tokens_seen": 2833448960 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014275397207510833, + "loss": 1.2491, + "theoretical_loss": 3.3639193246616417, + "tokens_seen": 2833580032 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014271385010431711, + "loss": 1.2309, + "theoretical_loss": 3.36390732440912, + "tokens_seen": 2833711104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014267372813352592, + "loss": 1.3214, + "theoretical_loss": 3.363895324867063, + "tokens_seen": 2833842176 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014263360616273473, + "loss": 1.2594, + "theoretical_loss": 3.363883326035394, + "tokens_seen": 2833973248 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425934841919435, + "loss": 1.3084, + "theoretical_loss": 3.3638713279140395, + "tokens_seen": 2834104320 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425533622211523, + "loss": 1.2873, + "theoretical_loss": 3.3638593305029234, + "tokens_seen": 2834235392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014251324025036112, + "loss": 1.309, + "theoretical_loss": 3.363847333801971, + "tokens_seen": 2834366464 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001424731182795699, + "loss": 1.3445, + "theoretical_loss": 3.3638353378111083, + "tokens_seen": 2834497536 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014243299630877868, + "loss": 1.2889, + "theoretical_loss": 3.3638233425302597, + "tokens_seen": 2834628608 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014239287433798748, + "loss": 1.3041, + "theoretical_loss": 3.3638113479593508, + "tokens_seen": 2834759680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014235275236719626, + "loss": 1.3189, + "theoretical_loss": 3.363799354098306, + "tokens_seen": 2834890752 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014231263039640507, + "loss": 1.2734, + "theoretical_loss": 3.363787360947051, + "tokens_seen": 2835021824 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014227250842561388, + "loss": 1.3262, + "theoretical_loss": 3.3637753685055114, + "tokens_seen": 2835152896 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014223238645482268, + "loss": 1.2758, + "theoretical_loss": 3.363763376773612, + "tokens_seen": 2835283968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014219226448403146, + "loss": 1.2943, + "theoretical_loss": 3.3637513857512773, + "tokens_seen": 2835415040 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014215214251324024, + "loss": 1.2507, + "theoretical_loss": 3.363739395438434, + "tokens_seen": 2835546112 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014211202054244905, + "loss": 1.3302, + "theoretical_loss": 3.3637274058350055, + "tokens_seen": 2835677184 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014207189857165783, + "loss": 1.1956, + "theoretical_loss": 3.3637154169409187, + "tokens_seen": 2835808256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014203177660086663, + "loss": 1.2926, + "theoretical_loss": 3.363703428756098, + "tokens_seen": 2835939328 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014199165463007544, + "loss": 1.2637, + "theoretical_loss": 3.3636914412804684, + "tokens_seen": 2836070400 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.9904579520225525, + "objective/train/avg_token_score": 0.009542101062834263, + "objective/train/docs_used": 1030811, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.4069701433181763, + "objective/train/original_loss": 1.4220834970474243, + "objective/train/theoretical_loss": 3.3636794545139566, + "objective/train/tokens_used": 1206660576, + "objective/train/unlikelihood_loss": 0.024297814816236496, + "theoretical_loss": 3.3636794545139566, + "tokens_seen": 2836201472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014195153265928424, + "loss": 1.3207, + "theoretical_loss": 3.3636794545139566, + "tokens_seen": 2836201472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014191141068849302, + "loss": 1.2329, + "theoretical_loss": 3.363667468456486, + "tokens_seen": 2836332544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014187128871770183, + "loss": 1.2471, + "theoretical_loss": 3.363655483107983, + "tokens_seen": 2836463616 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001418311667469106, + "loss": 1.2382, + "theoretical_loss": 3.363643498468373, + "tokens_seen": 2836594688 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001417910447761194, + "loss": 1.3189, + "theoretical_loss": 3.363631514537581, + "tokens_seen": 2836725760 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001417509228053282, + "loss": 1.3544, + "theoretical_loss": 3.363619531315532, + "tokens_seen": 2836856832 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141710800834537, + "loss": 1.2789, + "theoretical_loss": 3.3636075488021517, + "tokens_seen": 2836987904 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001416706788637458, + "loss": 1.3143, + "theoretical_loss": 3.3635955669973656, + "tokens_seen": 2837118976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014163055689295459, + "loss": 1.2282, + "theoretical_loss": 3.363583585901099, + "tokens_seen": 2837250048 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001415904349221634, + "loss": 1.3123, + "theoretical_loss": 3.363571605513277, + "tokens_seen": 2837381120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014155031295137217, + "loss": 1.2273, + "theoretical_loss": 3.363559625833825, + "tokens_seen": 2837512192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014151019098058095, + "loss": 1.2833, + "theoretical_loss": 3.363547646862669, + "tokens_seen": 2837643264 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014147006900978976, + "loss": 1.2278, + "theoretical_loss": 3.3635356685997335, + "tokens_seen": 2837774336 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014142994703899856, + "loss": 1.3439, + "theoretical_loss": 3.3635236910449446, + "tokens_seen": 2837905408 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014138982506820734, + "loss": 1.3003, + "theoretical_loss": 3.3635117141982276, + "tokens_seen": 2838036480 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014134970309741615, + "loss": 1.2795, + "theoretical_loss": 3.363499738059508, + "tokens_seen": 2838167552 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014130958112662496, + "loss": 1.1825, + "theoretical_loss": 3.363487762628711, + "tokens_seen": 2838298624 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014126945915583373, + "loss": 1.2373, + "theoretical_loss": 3.363475787905762, + "tokens_seen": 2838429696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014122933718504251, + "loss": 1.2576, + "theoretical_loss": 3.363463813890587, + "tokens_seen": 2838560768 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014118921521425132, + "loss": 1.2192, + "theoretical_loss": 3.3634518405831106, + "tokens_seen": 2838691840 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014114909324346013, + "loss": 1.2106, + "theoretical_loss": 3.363439867983259, + "tokens_seen": 2838822912 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001411089712726689, + "loss": 1.1912, + "theoretical_loss": 3.363427896090958, + "tokens_seen": 2838953984 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001410688493018777, + "loss": 1.263, + "theoretical_loss": 3.3634159249061324, + "tokens_seen": 2839085056 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014102872733108652, + "loss": 1.3005, + "theoretical_loss": 3.363403954428708, + "tokens_seen": 2839216128 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014098860536029532, + "loss": 1.245, + "theoretical_loss": 3.3633919846586102, + "tokens_seen": 2839347200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014094848338950408, + "loss": 1.2883, + "theoretical_loss": 3.363380015595765, + "tokens_seen": 2839478272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014090836141871288, + "loss": 1.285, + "theoretical_loss": 3.3633680472400975, + "tokens_seen": 2839609344 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001408682394479217, + "loss": 1.2166, + "theoretical_loss": 3.3633560795915334, + "tokens_seen": 2839740416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014082811747713047, + "loss": 1.2539, + "theoretical_loss": 3.3633441126499983, + "tokens_seen": 2839871488 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014078799550633927, + "loss": 1.2025, + "theoretical_loss": 3.3633321464154182, + "tokens_seen": 2840002560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014074787353554808, + "loss": 1.358, + "theoretical_loss": 3.363320180887718, + "tokens_seen": 2840133632 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001407077515647569, + "loss": 1.2624, + "theoretical_loss": 3.363308216066824, + "tokens_seen": 2840264704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014066762959396564, + "loss": 1.3123, + "theoretical_loss": 3.363296251952661, + "tokens_seen": 2840395776 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014062750762317445, + "loss": 1.2075, + "theoretical_loss": 3.3632842885451555, + "tokens_seen": 2840526848 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014058738565238325, + "loss": 1.2485, + "theoretical_loss": 3.3632723258442327, + "tokens_seen": 2840657920 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014054726368159203, + "loss": 1.289, + "theoretical_loss": 3.363260363849818, + "tokens_seen": 2840788992 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014050714171080084, + "loss": 1.2226, + "theoretical_loss": 3.3632484025618385, + "tokens_seen": 2840920064 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014046701974000964, + "loss": 1.2376, + "theoretical_loss": 3.363236441980218, + "tokens_seen": 2841051136 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014042689776921845, + "loss": 1.3137, + "theoretical_loss": 3.3632244821048833, + "tokens_seen": 2841182208 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014038677579842723, + "loss": 1.3149, + "theoretical_loss": 3.36321252293576, + "tokens_seen": 2841313280 + }, + { + "epoch": 0.72, + "learning_rate": 0.000140346653827636, + "loss": 1.3052, + "theoretical_loss": 3.3632005644727734, + "tokens_seen": 2841444352 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014030653185684481, + "loss": 1.3244, + "theoretical_loss": 3.3631886067158496, + "tokens_seen": 2841575424 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001402664098860536, + "loss": 1.1946, + "theoretical_loss": 3.3631766496649145, + "tokens_seen": 2841706496 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001402262879152624, + "loss": 1.2884, + "theoretical_loss": 3.3631646933198933, + "tokens_seen": 2841837568 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001401861659444712, + "loss": 1.2465, + "theoretical_loss": 3.3631527376807124, + "tokens_seen": 2841968640 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014014604397367999, + "loss": 1.2444, + "theoretical_loss": 3.3631407827472968, + "tokens_seen": 2842099712 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001401059220028888, + "loss": 1.3222, + "theoretical_loss": 3.3631288285195735, + "tokens_seen": 2842230784 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014006580003209757, + "loss": 1.2745, + "theoretical_loss": 3.363116874997467, + "tokens_seen": 2842361856 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014002567806130638, + "loss": 1.3422, + "theoretical_loss": 3.363104922180904, + "tokens_seen": 2842492928 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013998555609051516, + "loss": 1.2605, + "theoretical_loss": 3.36309297006981, + "tokens_seen": 2842624000 + }, + { + "epoch": 0.72, + "objective/train/aligned_token_count": 0.9920662641525269, + "objective/train/avg_token_score": 0.007933743298053741, + "objective/train/docs_used": 1033097, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2498607635498047, + "objective/train/original_loss": 1.260568380355835, + "objective/train/theoretical_loss": 3.3630810186641105, + "objective/train/tokens_used": 1213214176, + "objective/train/unlikelihood_loss": 0.023088915273547173, + "theoretical_loss": 3.3630810186641105, + "tokens_seen": 2842755072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013994543411972396, + "loss": 1.2412, + "theoretical_loss": 3.3630810186641105, + "tokens_seen": 2842755072 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013990531214893277, + "loss": 1.2718, + "theoretical_loss": 3.363069067963732, + "tokens_seen": 2842886144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013986519017814155, + "loss": 1.273, + "theoretical_loss": 3.3630571179685997, + "tokens_seen": 2843017216 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013982506820735036, + "loss": 1.2352, + "theoretical_loss": 3.3630451686786405, + "tokens_seen": 2843148288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013978494623655913, + "loss": 1.2709, + "theoretical_loss": 3.363033220093779, + "tokens_seen": 2843279360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013974482426576794, + "loss": 1.2359, + "theoretical_loss": 3.363021272213942, + "tokens_seen": 2843410432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013970470229497672, + "loss": 1.2647, + "theoretical_loss": 3.3630093250390547, + "tokens_seen": 2843541504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013966458032418553, + "loss": 1.2842, + "theoretical_loss": 3.3629973785690437, + "tokens_seen": 2843672576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013962445835339433, + "loss": 1.2311, + "theoretical_loss": 3.3629854328038347, + "tokens_seen": 2843803648 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001395843363826031, + "loss": 1.2071, + "theoretical_loss": 3.3629734877433535, + "tokens_seen": 2843934720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013954421441181192, + "loss": 1.274, + "theoretical_loss": 3.3629615433875264, + "tokens_seen": 2844065792 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013950409244102072, + "loss": 1.2666, + "theoretical_loss": 3.362949599736279, + "tokens_seen": 2844196864 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013946397047022948, + "loss": 1.2781, + "theoretical_loss": 3.362937656789537, + "tokens_seen": 2844327936 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013942384849943828, + "loss": 1.2883, + "theoretical_loss": 3.362925714547227, + "tokens_seen": 2844459008 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001393837265286471, + "loss": 1.2256, + "theoretical_loss": 3.3629137730092746, + "tokens_seen": 2844590080 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001393436045578559, + "loss": 1.2952, + "theoretical_loss": 3.362901832175606, + "tokens_seen": 2844721152 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013930348258706467, + "loss": 1.2152, + "theoretical_loss": 3.362889892046147, + "tokens_seen": 2844852224 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013926336061627348, + "loss": 1.2493, + "theoretical_loss": 3.362877952620824, + "tokens_seen": 2844983296 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001392232386454823, + "loss": 1.2888, + "theoretical_loss": 3.3628660138995627, + "tokens_seen": 2845114368 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013918311667469104, + "loss": 1.2796, + "theoretical_loss": 3.3628540758822894, + "tokens_seen": 2845245440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013914299470389985, + "loss": 1.2327, + "theoretical_loss": 3.36284213856893, + "tokens_seen": 2845376512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013910287273310865, + "loss": 1.2554, + "theoretical_loss": 3.3628302019594107, + "tokens_seen": 2845507584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013906275076231746, + "loss": 1.2661, + "theoretical_loss": 3.362818266053657, + "tokens_seen": 2845638656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013902262879152624, + "loss": 1.2499, + "theoretical_loss": 3.3628063308515963, + "tokens_seen": 2845769728 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013898250682073504, + "loss": 1.2456, + "theoretical_loss": 3.3627943963531535, + "tokens_seen": 2845900800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013894238484994385, + "loss": 1.3036, + "theoretical_loss": 3.362782462558255, + "tokens_seen": 2846031872 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001389022628791526, + "loss": 1.2632, + "theoretical_loss": 3.362770529466827, + "tokens_seen": 2846162944 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001388621409083614, + "loss": 1.1977, + "theoretical_loss": 3.3627585970787957, + "tokens_seen": 2846294016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013882201893757021, + "loss": 1.2789, + "theoretical_loss": 3.362746665394087, + "tokens_seen": 2846425088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013878189696677902, + "loss": 1.1992, + "theoretical_loss": 3.3627347344126277, + "tokens_seen": 2846556160 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001387417749959878, + "loss": 1.2954, + "theoretical_loss": 3.3627228041343433, + "tokens_seen": 2846687232 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001387016530251966, + "loss": 1.2777, + "theoretical_loss": 3.36271087455916, + "tokens_seen": 2846818304 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001386615310544054, + "loss": 1.2481, + "theoretical_loss": 3.3626989456870047, + "tokens_seen": 2846949376 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001386214090836142, + "loss": 1.2543, + "theoretical_loss": 3.362687017517803, + "tokens_seen": 2847080448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013858128711282297, + "loss": 1.2025, + "theoretical_loss": 3.3626750900514812, + "tokens_seen": 2847211520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013854116514203178, + "loss": 1.2839, + "theoretical_loss": 3.3626631632879653, + "tokens_seen": 2847342592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013850104317124058, + "loss": 1.3051, + "theoretical_loss": 3.3626512372271824, + "tokens_seen": 2847473664 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013846092120044936, + "loss": 1.2606, + "theoretical_loss": 3.3626393118690574, + "tokens_seen": 2847604736 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013842079922965817, + "loss": 1.2342, + "theoretical_loss": 3.362627387213518, + "tokens_seen": 2847735808 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013838067725886698, + "loss": 1.2172, + "theoretical_loss": 3.3626154632604894, + "tokens_seen": 2847866880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013834055528807575, + "loss": 1.242, + "theoretical_loss": 3.3626035400098986, + "tokens_seen": 2847997952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013830043331728453, + "loss": 1.2487, + "theoretical_loss": 3.3625916174616712, + "tokens_seen": 2848129024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013826031134649334, + "loss": 1.1776, + "theoretical_loss": 3.362579695615734, + "tokens_seen": 2848260096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013822018937570212, + "loss": 1.2162, + "theoretical_loss": 3.3625677744720135, + "tokens_seen": 2848391168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013818006740491093, + "loss": 1.2734, + "theoretical_loss": 3.362555854030435, + "tokens_seen": 2848522240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013813994543411973, + "loss": 1.3103, + "theoretical_loss": 3.3625439342909265, + "tokens_seen": 2848653312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013809982346332854, + "loss": 1.3028, + "theoretical_loss": 3.3625320152534126, + "tokens_seen": 2848784384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013805970149253732, + "loss": 1.3014, + "theoretical_loss": 3.3625200969178213, + "tokens_seen": 2848915456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013801957952174612, + "loss": 1.2397, + "theoretical_loss": 3.3625081792840774, + "tokens_seen": 2849046528 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001379794575509549, + "loss": 1.2555, + "theoretical_loss": 3.362496262352108, + "tokens_seen": 2849177600 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.9938709139823914, + "objective/train/avg_token_score": 0.006129080429673195, + "objective/train/docs_used": 1035566, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2069975137710571, + "objective/train/original_loss": 1.215867519378662, + "objective/train/theoretical_loss": 3.3624843461218403, + "objective/train/tokens_used": 1219767776, + "objective/train/unlikelihood_loss": 0.015141197480261326, + "theoretical_loss": 3.3624843461218403, + "tokens_seen": 2849308672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013793933558016368, + "loss": 1.2505, + "theoretical_loss": 3.3624843461218403, + "tokens_seen": 2849308672 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001378992136093725, + "loss": 1.274, + "theoretical_loss": 3.3624724305931992, + "tokens_seen": 2849439744 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001378590916385813, + "loss": 1.2819, + "theoretical_loss": 3.3624605157661125, + "tokens_seen": 2849570816 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001378189696677901, + "loss": 1.2595, + "theoretical_loss": 3.3624486016405055, + "tokens_seen": 2849701888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013777884769699888, + "loss": 1.222, + "theoretical_loss": 3.362436688216305, + "tokens_seen": 2849832960 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001377387257262077, + "loss": 1.3433, + "theoretical_loss": 3.3624247754934378, + "tokens_seen": 2849964032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013769860375541647, + "loss": 1.2373, + "theoretical_loss": 3.36241286347183, + "tokens_seen": 2850095104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013765848178462525, + "loss": 1.2582, + "theoretical_loss": 3.3624009521514084, + "tokens_seen": 2850226176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013761835981383405, + "loss": 1.2235, + "theoretical_loss": 3.362389041532099, + "tokens_seen": 2850357248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013757823784304286, + "loss": 1.2962, + "theoretical_loss": 3.3623771316138287, + "tokens_seen": 2850488320 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013753811587225166, + "loss": 1.2932, + "theoretical_loss": 3.362365222396524, + "tokens_seen": 2850619392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013749799390146044, + "loss": 1.2982, + "theoretical_loss": 3.3623533138801114, + "tokens_seen": 2850750464 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013745787193066925, + "loss": 1.3098, + "theoretical_loss": 3.362341406064517, + "tokens_seen": 2850881536 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013741774995987803, + "loss": 1.2522, + "theoretical_loss": 3.3623294989496677, + "tokens_seen": 2851012608 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001373776279890868, + "loss": 1.1973, + "theoretical_loss": 3.36231759253549, + "tokens_seen": 2851143680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013733750601829561, + "loss": 1.2885, + "theoretical_loss": 3.362305686821911, + "tokens_seen": 2851274752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013729738404750442, + "loss": 1.3215, + "theoretical_loss": 3.362293781808856, + "tokens_seen": 2851405824 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001372572620767132, + "loss": 1.3403, + "theoretical_loss": 3.362281877496253, + "tokens_seen": 2851536896 + }, + { + "epoch": 0.73, + "learning_rate": 0.000137217140105922, + "loss": 1.2977, + "theoretical_loss": 3.362269973884027, + "tokens_seen": 2851667968 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001371770181351308, + "loss": 1.2001, + "theoretical_loss": 3.3622580709721066, + "tokens_seen": 2851799040 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013713689616433962, + "loss": 1.2442, + "theoretical_loss": 3.3622461687604166, + "tokens_seen": 2851930112 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013709677419354837, + "loss": 1.2826, + "theoretical_loss": 3.3622342672488847, + "tokens_seen": 2852061184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013705665222275718, + "loss": 1.3311, + "theoretical_loss": 3.3622223664374373, + "tokens_seen": 2852192256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013701653025196598, + "loss": 1.2077, + "theoretical_loss": 3.3622104663260006, + "tokens_seen": 2852323328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013697640828117476, + "loss": 1.2156, + "theoretical_loss": 3.362198566914502, + "tokens_seen": 2852454400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013693628631038357, + "loss": 1.2561, + "theoretical_loss": 3.3621866682028676, + "tokens_seen": 2852585472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013689616433959238, + "loss": 1.2625, + "theoretical_loss": 3.3621747701910243, + "tokens_seen": 2852716544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013685604236880118, + "loss": 1.3062, + "theoretical_loss": 3.3621628728788986, + "tokens_seen": 2852847616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013681592039800993, + "loss": 1.3083, + "theoretical_loss": 3.3621509762664177, + "tokens_seen": 2852978688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013677579842721874, + "loss": 1.3022, + "theoretical_loss": 3.362139080353508, + "tokens_seen": 2853109760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013673567645642755, + "loss": 1.2451, + "theoretical_loss": 3.362127185140096, + "tokens_seen": 2853240832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013669555448563633, + "loss": 1.2352, + "theoretical_loss": 3.3621152906261083, + "tokens_seen": 2853371904 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013665543251484513, + "loss": 1.2461, + "theoretical_loss": 3.3621033968114724, + "tokens_seen": 2853502976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013661531054405394, + "loss": 1.2708, + "theoretical_loss": 3.3620915036961145, + "tokens_seen": 2853634048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013657518857326274, + "loss": 1.223, + "theoretical_loss": 3.362079611279962, + "tokens_seen": 2853765120 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001365350666024715, + "loss": 1.2307, + "theoretical_loss": 3.3620677195629405, + "tokens_seen": 2853896192 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001364949446316803, + "loss": 1.1496, + "theoretical_loss": 3.362055828544978, + "tokens_seen": 2854027264 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001364548226608891, + "loss": 1.2872, + "theoretical_loss": 3.362043938226001, + "tokens_seen": 2854158336 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001364147006900979, + "loss": 1.2987, + "theoretical_loss": 3.3620320486059354, + "tokens_seen": 2854289408 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001363745787193067, + "loss": 1.2718, + "theoretical_loss": 3.3620201596847092, + "tokens_seen": 2854420480 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001363344567485155, + "loss": 1.311, + "theoretical_loss": 3.362008271462249, + "tokens_seen": 2854551552 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001362943347777243, + "loss": 1.3288, + "theoretical_loss": 3.361996383938481, + "tokens_seen": 2854682624 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013625421280693309, + "loss": 1.1835, + "theoretical_loss": 3.361984497113333, + "tokens_seen": 2854813696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013621409083614187, + "loss": 1.2333, + "theoretical_loss": 3.3619726109867307, + "tokens_seen": 2854944768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013617396886535067, + "loss": 1.2787, + "theoretical_loss": 3.3619607255586024, + "tokens_seen": 2855075840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013613384689455945, + "loss": 1.2636, + "theoretical_loss": 3.3619488408288736, + "tokens_seen": 2855206912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013609372492376826, + "loss": 1.2808, + "theoretical_loss": 3.361936956797472, + "tokens_seen": 2855337984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013605360295297706, + "loss": 1.2883, + "theoretical_loss": 3.3619250734643247, + "tokens_seen": 2855469056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013601348098218584, + "loss": 1.3438, + "theoretical_loss": 3.361913190829358, + "tokens_seen": 2855600128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013597335901139465, + "loss": 1.2389, + "theoretical_loss": 3.361901308892499, + "tokens_seen": 2855731200 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.9928230047225952, + "objective/train/avg_token_score": 0.007176929619163275, + "objective/train/docs_used": 1037860, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3377691507339478, + "objective/train/original_loss": 1.3499106168746948, + "objective/train/theoretical_loss": 3.361889427653675, + "objective/train/tokens_used": 1226321376, + "objective/train/unlikelihood_loss": 0.01918276958167553, + "theoretical_loss": 3.361889427653675, + "tokens_seen": 2855862272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013593323704060343, + "loss": 1.2888, + "theoretical_loss": 3.361889427653675, + "tokens_seen": 2855862272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013589311506981223, + "loss": 1.3003, + "theoretical_loss": 3.3618775471128126, + "tokens_seen": 2855993344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013585299309902101, + "loss": 1.3377, + "theoretical_loss": 3.361865667269839, + "tokens_seen": 2856124416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013581287112822982, + "loss": 1.3015, + "theoretical_loss": 3.361853788124681, + "tokens_seen": 2856255488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013577274915743863, + "loss": 1.292, + "theoretical_loss": 3.3618419096772656, + "tokens_seen": 2856386560 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001357326271866474, + "loss": 1.2834, + "theoretical_loss": 3.3618300319275196, + "tokens_seen": 2856517632 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001356925052158562, + "loss": 1.2843, + "theoretical_loss": 3.3618181548753707, + "tokens_seen": 2856648704 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013565238324506502, + "loss": 1.2835, + "theoretical_loss": 3.3618062785207456, + "tokens_seen": 2856779776 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001356122612742738, + "loss": 1.2576, + "theoretical_loss": 3.361794402863571, + "tokens_seen": 2856910848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013557213930348258, + "loss": 1.334, + "theoretical_loss": 3.3617825279037743, + "tokens_seen": 2857041920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013553201733269138, + "loss": 1.331, + "theoretical_loss": 3.3617706536412824, + "tokens_seen": 2857172992 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001354918953619002, + "loss": 1.3138, + "theoretical_loss": 3.3617587800760225, + "tokens_seen": 2857304064 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013545177339110897, + "loss": 1.2599, + "theoretical_loss": 3.3617469072079214, + "tokens_seen": 2857435136 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013541165142031777, + "loss": 1.2078, + "theoretical_loss": 3.361735035036906, + "tokens_seen": 2857566208 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013537152944952658, + "loss": 1.2047, + "theoretical_loss": 3.3617231635629046, + "tokens_seen": 2857697280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013533140747873533, + "loss": 1.2681, + "theoretical_loss": 3.3617112927858432, + "tokens_seen": 2857828352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013529128550794414, + "loss": 1.233, + "theoretical_loss": 3.361699422705649, + "tokens_seen": 2857959424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013525116353715295, + "loss": 1.2844, + "theoretical_loss": 3.3616875533222492, + "tokens_seen": 2858090496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013521104156636175, + "loss": 1.2108, + "theoretical_loss": 3.3616756846355718, + "tokens_seen": 2858221568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013517091959557053, + "loss": 1.2056, + "theoretical_loss": 3.3616638166455424, + "tokens_seen": 2858352640 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013513079762477934, + "loss": 1.2625, + "theoretical_loss": 3.3616519493520896, + "tokens_seen": 2858483712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013509067565398814, + "loss": 1.3001, + "theoretical_loss": 3.36164008275514, + "tokens_seen": 2858614784 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001350505536831969, + "loss": 1.2475, + "theoretical_loss": 3.3616282168546205, + "tokens_seen": 2858745856 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001350104317124057, + "loss": 1.2359, + "theoretical_loss": 3.3616163516504587, + "tokens_seen": 2858876928 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001349703097416145, + "loss": 1.2512, + "theoretical_loss": 3.3616044871425816, + "tokens_seen": 2859008000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013493018777082331, + "loss": 1.2312, + "theoretical_loss": 3.3615926233309166, + "tokens_seen": 2859139072 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001348900658000321, + "loss": 1.2542, + "theoretical_loss": 3.361580760215391, + "tokens_seen": 2859270144 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001348499438292409, + "loss": 1.2889, + "theoretical_loss": 3.3615688977959315, + "tokens_seen": 2859401216 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001348098218584497, + "loss": 1.1964, + "theoretical_loss": 3.361557036072466, + "tokens_seen": 2859532288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013476969988765849, + "loss": 1.2422, + "theoretical_loss": 3.3615451750449212, + "tokens_seen": 2859663360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013472957791686727, + "loss": 1.3353, + "theoretical_loss": 3.361533314713225, + "tokens_seen": 2859794432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013468945594607607, + "loss": 1.2747, + "theoretical_loss": 3.3615214550773045, + "tokens_seen": 2859925504 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013464933397528488, + "loss": 1.3072, + "theoretical_loss": 3.3615095961370867, + "tokens_seen": 2860056576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013460921200449366, + "loss": 1.3127, + "theoretical_loss": 3.361497737892499, + "tokens_seen": 2860187648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013456909003370246, + "loss": 1.3134, + "theoretical_loss": 3.361485880343469, + "tokens_seen": 2860318720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013452896806291127, + "loss": 1.2347, + "theoretical_loss": 3.3614740234899236, + "tokens_seen": 2860449792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013448884609212005, + "loss": 1.2245, + "theoretical_loss": 3.3614621673317906, + "tokens_seen": 2860580864 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013444872412132883, + "loss": 1.2139, + "theoretical_loss": 3.361450311868997, + "tokens_seen": 2860711936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013440860215053763, + "loss": 1.3146, + "theoretical_loss": 3.36143845710147, + "tokens_seen": 2860843008 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013436848017974644, + "loss": 1.2638, + "theoretical_loss": 3.3614266030291375, + "tokens_seen": 2860974080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013432835820895522, + "loss": 1.2551, + "theoretical_loss": 3.361414749651927, + "tokens_seen": 2861105152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013428823623816403, + "loss": 1.2917, + "theoretical_loss": 3.361402896969765, + "tokens_seen": 2861236224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013424811426737283, + "loss": 1.2536, + "theoretical_loss": 3.3613910449825797, + "tokens_seen": 2861367296 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001342079922965816, + "loss": 1.3336, + "theoretical_loss": 3.361379193690298, + "tokens_seen": 2861498368 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001341678703257904, + "loss": 1.2653, + "theoretical_loss": 3.361367343092848, + "tokens_seen": 2861629440 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001341277483549992, + "loss": 1.2743, + "theoretical_loss": 3.361355493190157, + "tokens_seen": 2861760512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013408762638420798, + "loss": 1.2404, + "theoretical_loss": 3.3613436439821514, + "tokens_seen": 2861891584 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013404750441341678, + "loss": 1.1862, + "theoretical_loss": 3.3613317954687596, + "tokens_seen": 2862022656 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001340073824426256, + "loss": 1.2793, + "theoretical_loss": 3.361319947649909, + "tokens_seen": 2862153728 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001339672604718344, + "loss": 1.2418, + "theoretical_loss": 3.361308100525527, + "tokens_seen": 2862284800 + }, + { + "epoch": 0.73, + "objective/train/aligned_token_count": 0.9913198947906494, + "objective/train/avg_token_score": 0.00868011824786663, + "objective/train/docs_used": 1040389, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3053922653198242, + "objective/train/original_loss": 1.317537546157837, + "objective/train/theoretical_loss": 3.3612962540955413, + "objective/train/tokens_used": 1232874976, + "objective/train/unlikelihood_loss": 0.022990930825471878, + "theoretical_loss": 3.3612962540955413, + "tokens_seen": 2862415872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013392713850104317, + "loss": 1.2764, + "theoretical_loss": 3.3612962540955413, + "tokens_seen": 2862415872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013388701653025198, + "loss": 1.2132, + "theoretical_loss": 3.3612844083598796, + "tokens_seen": 2862546944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013384689455946076, + "loss": 1.2406, + "theoretical_loss": 3.3612725633184684, + "tokens_seen": 2862678016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013380677258866954, + "loss": 1.2614, + "theoretical_loss": 3.361260718971236, + "tokens_seen": 2862809088 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013376665061787835, + "loss": 1.2537, + "theoretical_loss": 3.36124887531811, + "tokens_seen": 2862940160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013372652864708715, + "loss": 1.3219, + "theoretical_loss": 3.3612370323590173, + "tokens_seen": 2863071232 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013368640667629596, + "loss": 1.2958, + "theoretical_loss": 3.3612251900938857, + "tokens_seen": 2863202304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013364628470550474, + "loss": 1.1751, + "theoretical_loss": 3.3612133485226434, + "tokens_seen": 2863333376 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013360616273471354, + "loss": 1.2949, + "theoretical_loss": 3.3612015076452177, + "tokens_seen": 2863464448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013356604076392232, + "loss": 1.2163, + "theoretical_loss": 3.361189667461536, + "tokens_seen": 2863595520 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001335259187931311, + "loss": 1.3186, + "theoretical_loss": 3.3611778279715256, + "tokens_seen": 2863726592 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001334857968223399, + "loss": 1.2764, + "theoretical_loss": 3.361165989175115, + "tokens_seen": 2863857664 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013344567485154871, + "loss": 1.2454, + "theoretical_loss": 3.3611541510722307, + "tokens_seen": 2863988736 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013340555288075752, + "loss": 1.2904, + "theoretical_loss": 3.361142313662801, + "tokens_seen": 2864119808 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001333654309099663, + "loss": 1.2385, + "theoretical_loss": 3.3611304769467534, + "tokens_seen": 2864250880 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001333253089391751, + "loss": 1.276, + "theoretical_loss": 3.361118640924016, + "tokens_seen": 2864381952 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001332851869683839, + "loss": 1.2837, + "theoretical_loss": 3.361106805594516, + "tokens_seen": 2864513024 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013324506499759266, + "loss": 1.2064, + "theoretical_loss": 3.361094970958181, + "tokens_seen": 2864644096 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013320494302680147, + "loss": 1.3494, + "theoretical_loss": 3.3610831370149388, + "tokens_seen": 2864775168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013316482105601028, + "loss": 1.2851, + "theoretical_loss": 3.3610713037647173, + "tokens_seen": 2864906240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013312469908521906, + "loss": 1.3398, + "theoretical_loss": 3.361059471207444, + "tokens_seen": 2865037312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013308457711442786, + "loss": 1.2545, + "theoretical_loss": 3.361047639343047, + "tokens_seen": 2865168384 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013304445514363667, + "loss": 1.2709, + "theoretical_loss": 3.3610358081714535, + "tokens_seen": 2865299456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013300433317284548, + "loss": 1.2828, + "theoretical_loss": 3.361023977692591, + "tokens_seen": 2865430528 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013296421120205423, + "loss": 1.1947, + "theoretical_loss": 3.3610121479063886, + "tokens_seen": 2865561600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013292408923126303, + "loss": 1.3518, + "theoretical_loss": 3.3610003188127724, + "tokens_seen": 2865692672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013288396726047184, + "loss": 1.2889, + "theoretical_loss": 3.3609884904116716, + "tokens_seen": 2865823744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013284384528968062, + "loss": 1.2245, + "theoretical_loss": 3.3609766627030133, + "tokens_seen": 2865954816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013280372331888943, + "loss": 1.2594, + "theoretical_loss": 3.3609648356867248, + "tokens_seen": 2866085888 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013276360134809823, + "loss": 1.2447, + "theoretical_loss": 3.360953009362735, + "tokens_seen": 2866216960 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013272347937730704, + "loss": 1.2113, + "theoretical_loss": 3.360941183730971, + "tokens_seen": 2866348032 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001326833574065158, + "loss": 1.2958, + "theoretical_loss": 3.3609293587913607, + "tokens_seen": 2866479104 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001326432354357246, + "loss": 1.2766, + "theoretical_loss": 3.360917534543832, + "tokens_seen": 2866610176 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001326031134649334, + "loss": 1.3377, + "theoretical_loss": 3.3609057109883134, + "tokens_seen": 2866741248 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013256299149414218, + "loss": 1.2555, + "theoretical_loss": 3.3608938881247314, + "tokens_seen": 2866872320 + }, + { + "epoch": 0.74, + "learning_rate": 0.000132522869523351, + "loss": 1.293, + "theoretical_loss": 3.3608820659530148, + "tokens_seen": 2867003392 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001324827475525598, + "loss": 1.2223, + "theoretical_loss": 3.360870244473092, + "tokens_seen": 2867134464 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001324426255817686, + "loss": 1.2789, + "theoretical_loss": 3.3608584236848893, + "tokens_seen": 2867265536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013240250361097738, + "loss": 1.3274, + "theoretical_loss": 3.360846603588336, + "tokens_seen": 2867396608 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013236238164018616, + "loss": 1.2126, + "theoretical_loss": 3.3608347841833597, + "tokens_seen": 2867527680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013232225966939497, + "loss": 1.2108, + "theoretical_loss": 3.360822965469888, + "tokens_seen": 2867658752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013228213769860375, + "loss": 1.2517, + "theoretical_loss": 3.3608111474478486, + "tokens_seen": 2867789824 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013224201572781255, + "loss": 1.3147, + "theoretical_loss": 3.3607993301171706, + "tokens_seen": 2867920896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013220189375702136, + "loss": 1.27, + "theoretical_loss": 3.360787513477781, + "tokens_seen": 2868051968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013216177178623016, + "loss": 1.2828, + "theoretical_loss": 3.3607756975296077, + "tokens_seen": 2868183040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013212164981543894, + "loss": 1.3237, + "theoretical_loss": 3.3607638822725794, + "tokens_seen": 2868314112 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013208152784464772, + "loss": 1.262, + "theoretical_loss": 3.360752067706623, + "tokens_seen": 2868445184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013204140587385653, + "loss": 1.2122, + "theoretical_loss": 3.360740253831668, + "tokens_seen": 2868576256 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001320012839030653, + "loss": 1.2607, + "theoretical_loss": 3.360728440647641, + "tokens_seen": 2868707328 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013196116193227411, + "loss": 1.315, + "theoretical_loss": 3.360716628154471, + "tokens_seen": 2868838400 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.9911837577819824, + "objective/train/avg_token_score": 0.008816240355372429, + "objective/train/docs_used": 1042847, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3004045486450195, + "objective/train/original_loss": 1.3147457838058472, + "objective/train/theoretical_loss": 3.3607048163520856, + "objective/train/tokens_used": 1239428576, + "objective/train/unlikelihood_loss": 0.025474630296230316, + "theoretical_loss": 3.3607048163520856, + "tokens_seen": 2868969472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013192103996148292, + "loss": 1.3191, + "theoretical_loss": 3.3607048163520856, + "tokens_seen": 2868969472 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318809179906917, + "loss": 1.2914, + "theoretical_loss": 3.360693005240413, + "tokens_seen": 2869100544 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318407960199005, + "loss": 1.2699, + "theoretical_loss": 3.3606811948193807, + "tokens_seen": 2869231616 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013180067404910929, + "loss": 1.2522, + "theoretical_loss": 3.3606693850889178, + "tokens_seen": 2869362688 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001317605520783181, + "loss": 1.2556, + "theoretical_loss": 3.3606575760489514, + "tokens_seen": 2869493760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013172043010752687, + "loss": 1.2814, + "theoretical_loss": 3.36064576769941, + "tokens_seen": 2869624832 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013168030813673568, + "loss": 1.2676, + "theoretical_loss": 3.360633960040222, + "tokens_seen": 2869755904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013164018616594448, + "loss": 1.2678, + "theoretical_loss": 3.360622153071315, + "tokens_seen": 2869886976 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013160006419515326, + "loss": 1.2254, + "theoretical_loss": 3.360610346792617, + "tokens_seen": 2870018048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013155994222436207, + "loss": 1.2934, + "theoretical_loss": 3.360598541204057, + "tokens_seen": 2870149120 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013151982025357088, + "loss": 1.27, + "theoretical_loss": 3.360586736305563, + "tokens_seen": 2870280192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013147969828277965, + "loss": 1.2502, + "theoretical_loss": 3.360574932097062, + "tokens_seen": 2870411264 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013143957631198843, + "loss": 1.32, + "theoretical_loss": 3.360563128578483, + "tokens_seen": 2870542336 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013139945434119724, + "loss": 1.3129, + "theoretical_loss": 3.3605513257497543, + "tokens_seen": 2870673408 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013135933237040605, + "loss": 1.2462, + "theoretical_loss": 3.3605395236108038, + "tokens_seen": 2870804480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013131921039961483, + "loss": 1.2158, + "theoretical_loss": 3.3605277221615597, + "tokens_seen": 2870935552 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013127908842882363, + "loss": 1.2013, + "theoretical_loss": 3.3605159214019507, + "tokens_seen": 2871066624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013123896645803244, + "loss": 1.2632, + "theoretical_loss": 3.3605041213319042, + "tokens_seen": 2871197696 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001311988444872412, + "loss": 1.32, + "theoretical_loss": 3.360492321951349, + "tokens_seen": 2871328768 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013115872251645, + "loss": 1.2267, + "theoretical_loss": 3.3604805232602133, + "tokens_seen": 2871459840 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001311186005456588, + "loss": 1.2354, + "theoretical_loss": 3.360468725258425, + "tokens_seen": 2871590912 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001310784785748676, + "loss": 1.207, + "theoretical_loss": 3.360456927945913, + "tokens_seen": 2871721984 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001310383566040764, + "loss": 1.2767, + "theoretical_loss": 3.3604451313226047, + "tokens_seen": 2871853056 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001309982346332852, + "loss": 1.2941, + "theoretical_loss": 3.360433335388429, + "tokens_seen": 2871984128 + }, + { + "epoch": 0.74, + "learning_rate": 0.000130958112662494, + "loss": 1.272, + "theoretical_loss": 3.360421540143314, + "tokens_seen": 2872115200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013091799069170278, + "loss": 1.2711, + "theoretical_loss": 3.360409745587188, + "tokens_seen": 2872246272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013087786872091156, + "loss": 1.1939, + "theoretical_loss": 3.36039795171998, + "tokens_seen": 2872377344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013083774675012037, + "loss": 1.228, + "theoretical_loss": 3.3603861585416173, + "tokens_seen": 2872508416 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013079762477932917, + "loss": 1.2014, + "theoretical_loss": 3.360374366052029, + "tokens_seen": 2872639488 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013075750280853795, + "loss": 1.2767, + "theoretical_loss": 3.3603625742511425, + "tokens_seen": 2872770560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013071738083774676, + "loss": 1.2544, + "theoretical_loss": 3.3603507831388866, + "tokens_seen": 2872901632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013067725886695556, + "loss": 1.2164, + "theoretical_loss": 3.3603389927151905, + "tokens_seen": 2873032704 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013063713689616434, + "loss": 1.2562, + "theoretical_loss": 3.3603272029799816, + "tokens_seen": 2873163776 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013059701492537312, + "loss": 1.2534, + "theoretical_loss": 3.3603154139331886, + "tokens_seen": 2873294848 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013055689295458193, + "loss": 1.1799, + "theoretical_loss": 3.3603036255747396, + "tokens_seen": 2873425920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013051677098379073, + "loss": 1.33, + "theoretical_loss": 3.3602918379045636, + "tokens_seen": 2873556992 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013047664901299951, + "loss": 1.2979, + "theoretical_loss": 3.3602800509225887, + "tokens_seen": 2873688064 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013043652704220832, + "loss": 1.2489, + "theoretical_loss": 3.3602682646287434, + "tokens_seen": 2873819136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013039640507141713, + "loss": 1.229, + "theoretical_loss": 3.360256479022956, + "tokens_seen": 2873950208 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001303562831006259, + "loss": 1.3042, + "theoretical_loss": 3.3602446941051554, + "tokens_seen": 2874081280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013031616112983468, + "loss": 1.2092, + "theoretical_loss": 3.3602329098752692, + "tokens_seen": 2874212352 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001302760391590435, + "loss": 1.2743, + "theoretical_loss": 3.3602211263332267, + "tokens_seen": 2874343424 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001302359171882523, + "loss": 1.2857, + "theoretical_loss": 3.3602093434789557, + "tokens_seen": 2874474496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013019579521746108, + "loss": 1.1752, + "theoretical_loss": 3.3601975613123853, + "tokens_seen": 2874605568 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013015567324666988, + "loss": 1.2446, + "theoretical_loss": 3.3601857798334445, + "tokens_seen": 2874736640 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001301155512758787, + "loss": 1.2745, + "theoretical_loss": 3.3601739990420603, + "tokens_seen": 2874867712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013007542930508747, + "loss": 1.2629, + "theoretical_loss": 3.360162218938162, + "tokens_seen": 2874998784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013003530733429627, + "loss": 1.34, + "theoretical_loss": 3.3601504395216786, + "tokens_seen": 2875129856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012999518536350505, + "loss": 1.3353, + "theoretical_loss": 3.360138660792538, + "tokens_seen": 2875260928 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012995506339271383, + "loss": 1.307, + "theoretical_loss": 3.360126882750669, + "tokens_seen": 2875392000 + }, + { + "epoch": 0.74, + "objective/train/aligned_token_count": 0.9937359094619751, + "objective/train/avg_token_score": 0.0062640272080898285, + "objective/train/docs_used": 1045259, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1186366081237793, + "objective/train/original_loss": 1.1269221305847168, + "objective/train/theoretical_loss": 3.360115105396001, + "objective/train/tokens_used": 1245982176, + "objective/train/unlikelihood_loss": 0.01813659258186817, + "theoretical_loss": 3.360115105396001, + "tokens_seen": 2875523072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012991494142192264, + "loss": 1.1977, + "theoretical_loss": 3.360115105396001, + "tokens_seen": 2875523072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012987481945113145, + "loss": 1.281, + "theoretical_loss": 3.3601033287284605, + "tokens_seen": 2875654144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012983469748034025, + "loss": 1.291, + "theoretical_loss": 3.3600915527479778, + "tokens_seen": 2875785216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012979457550954903, + "loss": 1.2626, + "theoretical_loss": 3.360079777454481, + "tokens_seen": 2875916288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012975445353875784, + "loss": 1.2815, + "theoretical_loss": 3.3600680028478993, + "tokens_seen": 2876047360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012971433156796662, + "loss": 1.2979, + "theoretical_loss": 3.36005622892816, + "tokens_seen": 2876178432 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001296742095971754, + "loss": 1.2896, + "theoretical_loss": 3.360044455695193, + "tokens_seen": 2876309504 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001296340876263842, + "loss": 1.2733, + "theoretical_loss": 3.3600326831489267, + "tokens_seen": 2876440576 + }, + { + "epoch": 0.74, + "learning_rate": 0.000129593965655593, + "loss": 1.3161, + "theoretical_loss": 3.360020911289289, + "tokens_seen": 2876571648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012955384368480181, + "loss": 1.2326, + "theoretical_loss": 3.3600091401162095, + "tokens_seen": 2876702720 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001295137217140106, + "loss": 1.2875, + "theoretical_loss": 3.3599973696296166, + "tokens_seen": 2876833792 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001294735997432194, + "loss": 1.2728, + "theoretical_loss": 3.3599855998294386, + "tokens_seen": 2876964864 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012943347777242818, + "loss": 1.3409, + "theoretical_loss": 3.3599738307156044, + "tokens_seen": 2877095936 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012939335580163696, + "loss": 1.2466, + "theoretical_loss": 3.3599620622880426, + "tokens_seen": 2877227008 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012935323383084577, + "loss": 1.2969, + "theoretical_loss": 3.3599502945466826, + "tokens_seen": 2877358080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012931311186005457, + "loss": 1.2474, + "theoretical_loss": 3.3599385274914524, + "tokens_seen": 2877489152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012927298988926338, + "loss": 1.2689, + "theoretical_loss": 3.359926761122281, + "tokens_seen": 2877620224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012923286791847216, + "loss": 1.2775, + "theoretical_loss": 3.359914995439097, + "tokens_seen": 2877751296 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012919274594768096, + "loss": 1.2356, + "theoretical_loss": 3.3599032304418297, + "tokens_seen": 2877882368 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012915262397688977, + "loss": 1.3095, + "theoretical_loss": 3.3598914661304073, + "tokens_seen": 2878013440 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012911250200609852, + "loss": 1.2967, + "theoretical_loss": 3.3598797025047586, + "tokens_seen": 2878144512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012907238003530733, + "loss": 1.3006, + "theoretical_loss": 3.359867939564813, + "tokens_seen": 2878275584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012903225806451613, + "loss": 1.2856, + "theoretical_loss": 3.3598561773104985, + "tokens_seen": 2878406656 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001289921360937249, + "loss": 1.2407, + "theoretical_loss": 3.3598444157417444, + "tokens_seen": 2878537728 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012895201412293372, + "loss": 1.3165, + "theoretical_loss": 3.3598326548584794, + "tokens_seen": 2878668800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012891189215214253, + "loss": 1.3011, + "theoretical_loss": 3.3598208946606323, + "tokens_seen": 2878799872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012887177018135133, + "loss": 1.1923, + "theoretical_loss": 3.359809135148132, + "tokens_seen": 2878930944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012883164821056008, + "loss": 1.2384, + "theoretical_loss": 3.3597973763209072, + "tokens_seen": 2879062016 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001287915262397689, + "loss": 1.2961, + "theoretical_loss": 3.359785618178887, + "tokens_seen": 2879193088 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001287514042689777, + "loss": 1.2563, + "theoretical_loss": 3.359773860722001, + "tokens_seen": 2879324160 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012871128229818648, + "loss": 1.3115, + "theoretical_loss": 3.3597621039501764, + "tokens_seen": 2879455232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012867116032739528, + "loss": 1.2492, + "theoretical_loss": 3.3597503478633435, + "tokens_seen": 2879586304 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001286310383566041, + "loss": 1.1998, + "theoretical_loss": 3.3597385924614303, + "tokens_seen": 2879717376 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001285909163858129, + "loss": 1.2582, + "theoretical_loss": 3.3597268377443665, + "tokens_seen": 2879848448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012855079441502167, + "loss": 1.2878, + "theoretical_loss": 3.3597150837120804, + "tokens_seen": 2879979520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012851067244423045, + "loss": 1.2799, + "theoretical_loss": 3.3597033303645016, + "tokens_seen": 2880110592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012847055047343926, + "loss": 1.2333, + "theoretical_loss": 3.3596915777015584, + "tokens_seen": 2880241664 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012843042850264804, + "loss": 1.2696, + "theoretical_loss": 3.3596798257231804, + "tokens_seen": 2880372736 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012839030653185685, + "loss": 1.2907, + "theoretical_loss": 3.3596680744292957, + "tokens_seen": 2880503808 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012835018456106565, + "loss": 1.2693, + "theoretical_loss": 3.3596563238198343, + "tokens_seen": 2880634880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012831006259027446, + "loss": 1.2126, + "theoretical_loss": 3.359644573894725, + "tokens_seen": 2880765952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012826994061948324, + "loss": 1.2387, + "theoretical_loss": 3.359632824653896, + "tokens_seen": 2880897024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012822981864869202, + "loss": 1.2089, + "theoretical_loss": 3.3596210760972767, + "tokens_seen": 2881028096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012818969667790082, + "loss": 1.2644, + "theoretical_loss": 3.3596093282247965, + "tokens_seen": 2881159168 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001281495747071096, + "loss": 1.3088, + "theoretical_loss": 3.3595975810363843, + "tokens_seen": 2881290240 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001281094527363184, + "loss": 1.2612, + "theoretical_loss": 3.359585834531969, + "tokens_seen": 2881421312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012806933076552721, + "loss": 1.2633, + "theoretical_loss": 3.3595740887114798, + "tokens_seen": 2881552384 + }, + { + "epoch": 0.75, + "learning_rate": 0.000128029208794736, + "loss": 1.2884, + "theoretical_loss": 3.3595623435748454, + "tokens_seen": 2881683456 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001279890868239448, + "loss": 1.2404, + "theoretical_loss": 3.3595505991219956, + "tokens_seen": 2881814528 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012794896485315358, + "loss": 1.2635, + "theoretical_loss": 3.359538855352859, + "tokens_seen": 2881945600 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.9919629693031311, + "objective/train/avg_token_score": 0.008036998100578785, + "objective/train/docs_used": 1047629, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2448006868362427, + "objective/train/original_loss": 1.256359577178955, + "objective/train/theoretical_loss": 3.3595271122673642, + "objective/train/tokens_used": 1252535776, + "objective/train/unlikelihood_loss": 0.021942539140582085, + "theoretical_loss": 3.3595271122673642, + "tokens_seen": 2882076672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012790884288236239, + "loss": 1.2133, + "theoretical_loss": 3.3595271122673642, + "tokens_seen": 2882076672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012786872091157116, + "loss": 1.2313, + "theoretical_loss": 3.359515369865442, + "tokens_seen": 2882207744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012782859894077997, + "loss": 1.3188, + "theoretical_loss": 3.3595036281470194, + "tokens_seen": 2882338816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012778847696998878, + "loss": 1.1653, + "theoretical_loss": 3.359491887112027, + "tokens_seen": 2882469888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012774835499919756, + "loss": 1.2626, + "theoretical_loss": 3.3594801467603936, + "tokens_seen": 2882600960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012770823302840636, + "loss": 1.2871, + "theoretical_loss": 3.359468407092048, + "tokens_seen": 2882732032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012766811105761517, + "loss": 1.3156, + "theoretical_loss": 3.35945666810692, + "tokens_seen": 2882863104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012762798908682395, + "loss": 1.2814, + "theoretical_loss": 3.359444929804938, + "tokens_seen": 2882994176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012758786711603273, + "loss": 1.2599, + "theoretical_loss": 3.359433192186032, + "tokens_seen": 2883125248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012754774514524153, + "loss": 1.2799, + "theoretical_loss": 3.3594214552501307, + "tokens_seen": 2883256320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012750762317445034, + "loss": 1.1999, + "theoretical_loss": 3.359409718997163, + "tokens_seen": 2883387392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012746750120365912, + "loss": 1.338, + "theoretical_loss": 3.359397983427059, + "tokens_seen": 2883518464 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012742737923286793, + "loss": 1.251, + "theoretical_loss": 3.359386248539747, + "tokens_seen": 2883649536 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012738725726207673, + "loss": 1.2961, + "theoretical_loss": 3.3593745143351574, + "tokens_seen": 2883780608 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001273471352912855, + "loss": 1.2294, + "theoretical_loss": 3.359362780813218, + "tokens_seen": 2883911680 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001273070133204943, + "loss": 1.2572, + "theoretical_loss": 3.3593510479738597, + "tokens_seen": 2884042752 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001272668913497031, + "loss": 1.2738, + "theoretical_loss": 3.35933931581701, + "tokens_seen": 2884173824 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001272267693789119, + "loss": 1.296, + "theoretical_loss": 3.3593275843425996, + "tokens_seen": 2884304896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012718664740812068, + "loss": 1.2308, + "theoretical_loss": 3.359315853550557, + "tokens_seen": 2884435968 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001271465254373295, + "loss": 1.2233, + "theoretical_loss": 3.359304123440812, + "tokens_seen": 2884567040 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001271064034665383, + "loss": 1.2727, + "theoretical_loss": 3.3592923940132935, + "tokens_seen": 2884698112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012706628149574705, + "loss": 1.2381, + "theoretical_loss": 3.359280665267931, + "tokens_seen": 2884829184 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012702615952495585, + "loss": 1.2391, + "theoretical_loss": 3.359268937204654, + "tokens_seen": 2884960256 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012698603755416466, + "loss": 1.1524, + "theoretical_loss": 3.359257209823392, + "tokens_seen": 2885091328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012694591558337347, + "loss": 1.2537, + "theoretical_loss": 3.3592454831240732, + "tokens_seen": 2885222400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012690579361258225, + "loss": 1.3356, + "theoretical_loss": 3.3592337571066286, + "tokens_seen": 2885353472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012686567164179105, + "loss": 1.2559, + "theoretical_loss": 3.3592220317709867, + "tokens_seen": 2885484544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012682554967099986, + "loss": 1.2672, + "theoretical_loss": 3.3592103071170767, + "tokens_seen": 2885615616 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012678542770020864, + "loss": 1.2535, + "theoretical_loss": 3.359198583144828, + "tokens_seen": 2885746688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012674530572941742, + "loss": 1.2903, + "theoretical_loss": 3.359186859854171, + "tokens_seen": 2885877760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012670518375862622, + "loss": 1.1895, + "theoretical_loss": 3.359175137245034, + "tokens_seen": 2886008832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012666506178783503, + "loss": 1.2773, + "theoretical_loss": 3.359163415317347, + "tokens_seen": 2886139904 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001266249398170438, + "loss": 1.2378, + "theoretical_loss": 3.3591516940710395, + "tokens_seen": 2886270976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012658481784625261, + "loss": 1.2745, + "theoretical_loss": 3.35913997350604, + "tokens_seen": 2886402048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012654469587546142, + "loss": 1.2448, + "theoretical_loss": 3.3591282536222797, + "tokens_seen": 2886533120 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001265045739046702, + "loss": 1.2367, + "theoretical_loss": 3.3591165344196865, + "tokens_seen": 2886664192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012646445193387898, + "loss": 1.2529, + "theoretical_loss": 3.3591048158981907, + "tokens_seen": 2886795264 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012642432996308779, + "loss": 1.3084, + "theoretical_loss": 3.359093098057721, + "tokens_seen": 2886926336 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001263842079922966, + "loss": 1.2748, + "theoretical_loss": 3.359081380898208, + "tokens_seen": 2887057408 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012634408602150537, + "loss": 1.2345, + "theoretical_loss": 3.3590696644195805, + "tokens_seen": 2887188480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012630396405071418, + "loss": 1.2233, + "theoretical_loss": 3.3590579486217687, + "tokens_seen": 2887319552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012626384207992298, + "loss": 1.2649, + "theoretical_loss": 3.359046233504701, + "tokens_seen": 2887450624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012622372010913176, + "loss": 1.2555, + "theoretical_loss": 3.3590345190683077, + "tokens_seen": 2887581696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012618359813834057, + "loss": 1.2622, + "theoretical_loss": 3.3590228053125184, + "tokens_seen": 2887712768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012614347616754935, + "loss": 1.2248, + "theoretical_loss": 3.3590110922372625, + "tokens_seen": 2887843840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012610335419675815, + "loss": 1.2739, + "theoretical_loss": 3.3589993798424693, + "tokens_seen": 2887974912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012606323222596693, + "loss": 1.2128, + "theoretical_loss": 3.3589876681280693, + "tokens_seen": 2888105984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012602311025517574, + "loss": 1.3546, + "theoretical_loss": 3.358975957093991, + "tokens_seen": 2888237056 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012598298828438455, + "loss": 1.2366, + "theoretical_loss": 3.3589642467401646, + "tokens_seen": 2888368128 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012594286631359333, + "loss": 1.2667, + "theoretical_loss": 3.3589525370665196, + "tokens_seen": 2888499200 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.9933603405952454, + "objective/train/avg_token_score": 0.006639623083174229, + "objective/train/docs_used": 1050131, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3299700021743774, + "objective/train/original_loss": 1.339750051498413, + "objective/train/theoretical_loss": 3.3589408280729858, + "objective/train/tokens_used": 1259089376, + "objective/train/unlikelihood_loss": 0.0165520329028368, + "theoretical_loss": 3.3589408280729858, + "tokens_seen": 2888630272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012590274434280213, + "loss": 1.3052, + "theoretical_loss": 3.3589408280729858, + "tokens_seen": 2888630272 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001258626223720109, + "loss": 1.3231, + "theoretical_loss": 3.3589291197594924, + "tokens_seen": 2888761344 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001258225004012197, + "loss": 1.3243, + "theoretical_loss": 3.3589174121259693, + "tokens_seen": 2888892416 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001257823784304285, + "loss": 1.2302, + "theoretical_loss": 3.3589057051723463, + "tokens_seen": 2889023488 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001257422564596373, + "loss": 1.2881, + "theoretical_loss": 3.3588939988985533, + "tokens_seen": 2889154560 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001257021344888461, + "loss": 1.2729, + "theoretical_loss": 3.3588822933045193, + "tokens_seen": 2889285632 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001256620125180549, + "loss": 1.2343, + "theoretical_loss": 3.3588705883901744, + "tokens_seen": 2889416704 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001256218905472637, + "loss": 1.2581, + "theoretical_loss": 3.358858884155448, + "tokens_seen": 2889547776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012558176857647247, + "loss": 1.2791, + "theoretical_loss": 3.3588471806002707, + "tokens_seen": 2889678848 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012554164660568125, + "loss": 1.251, + "theoretical_loss": 3.358835477724571, + "tokens_seen": 2889809920 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012550152463489006, + "loss": 1.2461, + "theoretical_loss": 3.3588237755282795, + "tokens_seen": 2889940992 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012546140266409887, + "loss": 1.2592, + "theoretical_loss": 3.3588120740113254, + "tokens_seen": 2890072064 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012542128069330767, + "loss": 1.2423, + "theoretical_loss": 3.358800373173639, + "tokens_seen": 2890203136 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012538115872251645, + "loss": 1.2332, + "theoretical_loss": 3.35878867301515, + "tokens_seen": 2890334208 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012534103675172526, + "loss": 1.2818, + "theoretical_loss": 3.3587769735357873, + "tokens_seen": 2890465280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012530091478093406, + "loss": 1.2486, + "theoretical_loss": 3.3587652747354815, + "tokens_seen": 2890596352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012526079281014282, + "loss": 1.283, + "theoretical_loss": 3.3587535766141627, + "tokens_seen": 2890727424 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012522067083935162, + "loss": 1.174, + "theoretical_loss": 3.35874187917176, + "tokens_seen": 2890858496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012518054886856043, + "loss": 1.3237, + "theoretical_loss": 3.3587301824082036, + "tokens_seen": 2890989568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012514042689776923, + "loss": 1.2226, + "theoretical_loss": 3.358718486323423, + "tokens_seen": 2891120640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012510030492697801, + "loss": 1.2251, + "theoretical_loss": 3.3587067909173483, + "tokens_seen": 2891251712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012506018295618682, + "loss": 1.2685, + "theoretical_loss": 3.3586950961899094, + "tokens_seen": 2891382784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012502006098539563, + "loss": 1.2673, + "theoretical_loss": 3.3586834021410357, + "tokens_seen": 2891513856 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001249799390146044, + "loss": 1.2723, + "theoretical_loss": 3.3586717087706575, + "tokens_seen": 2891644928 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001249398170438132, + "loss": 1.2375, + "theoretical_loss": 3.358660016078705, + "tokens_seen": 2891776000 + }, + { + "epoch": 0.75, + "learning_rate": 0.000124899695073022, + "loss": 1.2759, + "theoretical_loss": 3.3586483240651077, + "tokens_seen": 2891907072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012485957310223077, + "loss": 1.2251, + "theoretical_loss": 3.3586366327297954, + "tokens_seen": 2892038144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012481945113143958, + "loss": 1.2602, + "theoretical_loss": 3.358624942072698, + "tokens_seen": 2892169216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012477932916064838, + "loss": 1.2762, + "theoretical_loss": 3.358613252093745, + "tokens_seen": 2892300288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012473920718985716, + "loss": 1.2608, + "theoretical_loss": 3.3586015627928676, + "tokens_seen": 2892431360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012469908521906597, + "loss": 1.2521, + "theoretical_loss": 3.358589874169995, + "tokens_seen": 2892562432 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012465896324827477, + "loss": 1.2826, + "theoretical_loss": 3.3585781862250568, + "tokens_seen": 2892693504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012461884127748355, + "loss": 1.2857, + "theoretical_loss": 3.3585664989579835, + "tokens_seen": 2892824576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012457871930669233, + "loss": 1.308, + "theoretical_loss": 3.358554812368705, + "tokens_seen": 2892955648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012453859733590114, + "loss": 1.1822, + "theoretical_loss": 3.358543126457151, + "tokens_seen": 2893086720 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012449847536510995, + "loss": 1.2669, + "theoretical_loss": 3.358531441223252, + "tokens_seen": 2893217792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012445835339431873, + "loss": 1.3172, + "theoretical_loss": 3.3585197566669374, + "tokens_seen": 2893348864 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012441823142352753, + "loss": 1.3107, + "theoretical_loss": 3.358508072788138, + "tokens_seen": 2893479936 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001243781094527363, + "loss": 1.2308, + "theoretical_loss": 3.3584963895867825, + "tokens_seen": 2893611008 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012433798748194512, + "loss": 1.2446, + "theoretical_loss": 3.3584847070628023, + "tokens_seen": 2893742080 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001242978655111539, + "loss": 1.2602, + "theoretical_loss": 3.358473025216127, + "tokens_seen": 2893873152 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001242577435403627, + "loss": 1.2051, + "theoretical_loss": 3.3584613440466864, + "tokens_seen": 2894004224 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001242176215695715, + "loss": 1.2156, + "theoretical_loss": 3.3584496635544108, + "tokens_seen": 2894135296 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241774995987803, + "loss": 1.2228, + "theoretical_loss": 3.35843798373923, + "tokens_seen": 2894266368 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241373776279891, + "loss": 1.295, + "theoretical_loss": 3.3584263046010747, + "tokens_seen": 2894397440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012409725565719787, + "loss": 1.3035, + "theoretical_loss": 3.3584146261398744, + "tokens_seen": 2894528512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012405713368640668, + "loss": 1.2763, + "theoretical_loss": 3.3584029483555593, + "tokens_seen": 2894659584 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012401701171561546, + "loss": 1.3023, + "theoretical_loss": 3.3583912712480597, + "tokens_seen": 2894790656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012397688974482427, + "loss": 1.3123, + "theoretical_loss": 3.358379594817306, + "tokens_seen": 2894921728 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012393676777403307, + "loss": 1.2631, + "theoretical_loss": 3.3583679190632276, + "tokens_seen": 2895052800 + }, + { + "epoch": 0.75, + "objective/train/aligned_token_count": 0.9946760535240173, + "objective/train/avg_token_score": 0.005323933903127909, + "objective/train/docs_used": 1052611, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.243948221206665, + "objective/train/original_loss": 1.2512774467468262, + "objective/train/theoretical_loss": 3.3583562439857557, + "objective/train/tokens_used": 1265642976, + "objective/train/unlikelihood_loss": 0.01454743929207325, + "theoretical_loss": 3.3583562439857557, + "tokens_seen": 2895183872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012389664580324185, + "loss": 1.2351, + "theoretical_loss": 3.3583562439857557, + "tokens_seen": 2895183872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012385652383245066, + "loss": 1.2378, + "theoretical_loss": 3.358344569584819, + "tokens_seen": 2895314944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012381640186165944, + "loss": 1.2443, + "theoretical_loss": 3.3583328958603493, + "tokens_seen": 2895446016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012377627989086824, + "loss": 1.2771, + "theoretical_loss": 3.3583212228122754, + "tokens_seen": 2895577088 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012373615792007702, + "loss": 1.2327, + "theoretical_loss": 3.3583095504405285, + "tokens_seen": 2895708160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012369603594928583, + "loss": 1.2151, + "theoretical_loss": 3.358297878745038, + "tokens_seen": 2895839232 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012365591397849463, + "loss": 1.2433, + "theoretical_loss": 3.358286207725735, + "tokens_seen": 2895970304 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012361579200770341, + "loss": 1.2641, + "theoretical_loss": 3.358274537382549, + "tokens_seen": 2896101376 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012357567003691222, + "loss": 1.2804, + "theoretical_loss": 3.35826286771541, + "tokens_seen": 2896232448 + }, + { + "epoch": 0.76, + "learning_rate": 0.000123535548066121, + "loss": 1.2597, + "theoretical_loss": 3.358251198724249, + "tokens_seen": 2896363520 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001234954260953298, + "loss": 1.2415, + "theoretical_loss": 3.358239530408996, + "tokens_seen": 2896494592 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001234553041245386, + "loss": 1.3102, + "theoretical_loss": 3.3582278627695814, + "tokens_seen": 2896625664 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001234151821537474, + "loss": 1.2448, + "theoretical_loss": 3.358216195805935, + "tokens_seen": 2896756736 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001233750601829562, + "loss": 1.279, + "theoretical_loss": 3.358204529517988, + "tokens_seen": 2896887808 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012333493821216498, + "loss": 1.1882, + "theoretical_loss": 3.3581928639056695, + "tokens_seen": 2897018880 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012329481624137378, + "loss": 1.3073, + "theoretical_loss": 3.3581811989689103, + "tokens_seen": 2897149952 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012325469427058256, + "loss": 1.1951, + "theoretical_loss": 3.358169534707641, + "tokens_seen": 2897281024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012321457229979137, + "loss": 1.2763, + "theoretical_loss": 3.358157871121792, + "tokens_seen": 2897412096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012317445032900017, + "loss": 1.2096, + "theoretical_loss": 3.3581462082112927, + "tokens_seen": 2897543168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012313432835820895, + "loss": 1.2137, + "theoretical_loss": 3.3581345459760747, + "tokens_seen": 2897674240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012309420638741776, + "loss": 1.2126, + "theoretical_loss": 3.3581228844160678, + "tokens_seen": 2897805312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012305408441662654, + "loss": 1.2462, + "theoretical_loss": 3.358111223531202, + "tokens_seen": 2897936384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012301396244583535, + "loss": 1.2966, + "theoretical_loss": 3.3580995633214084, + "tokens_seen": 2898067456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012297384047504412, + "loss": 1.2428, + "theoretical_loss": 3.3580879037866165, + "tokens_seen": 2898198528 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012293371850425293, + "loss": 1.3161, + "theoretical_loss": 3.3580762449267576, + "tokens_seen": 2898329600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012289359653346174, + "loss": 1.2486, + "theoretical_loss": 3.3580645867417616, + "tokens_seen": 2898460672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012285347456267052, + "loss": 1.2219, + "theoretical_loss": 3.358052929231559, + "tokens_seen": 2898591744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012281335259187932, + "loss": 1.2262, + "theoretical_loss": 3.358041272396081, + "tokens_seen": 2898722816 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001227732306210881, + "loss": 1.2456, + "theoretical_loss": 3.3580296162352563, + "tokens_seen": 2898853888 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001227331086502969, + "loss": 1.2307, + "theoretical_loss": 3.3580179607490166, + "tokens_seen": 2898984960 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226929866795057, + "loss": 1.3026, + "theoretical_loss": 3.358006305937293, + "tokens_seen": 2899116032 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226528647087145, + "loss": 1.2446, + "theoretical_loss": 3.357994651800014, + "tokens_seen": 2899247104 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226127427379233, + "loss": 1.1705, + "theoretical_loss": 3.3579829983371114, + "tokens_seen": 2899378176 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012257262076713208, + "loss": 1.3089, + "theoretical_loss": 3.357971345548516, + "tokens_seen": 2899509248 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012253249879634089, + "loss": 1.2289, + "theoretical_loss": 3.3579596934341573, + "tokens_seen": 2899640320 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012249237682554966, + "loss": 1.2304, + "theoretical_loss": 3.357948041993967, + "tokens_seen": 2899771392 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012245225485475847, + "loss": 1.2693, + "theoretical_loss": 3.357936391227874, + "tokens_seen": 2899902464 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012241213288396728, + "loss": 1.3367, + "theoretical_loss": 3.35792474113581, + "tokens_seen": 2900033536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012237201091317606, + "loss": 1.241, + "theoretical_loss": 3.357913091717706, + "tokens_seen": 2900164608 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012233188894238486, + "loss": 1.2878, + "theoretical_loss": 3.357901442973491, + "tokens_seen": 2900295680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012229176697159364, + "loss": 1.2659, + "theoretical_loss": 3.357889794903097, + "tokens_seen": 2900426752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012225164500080245, + "loss": 1.2236, + "theoretical_loss": 3.3578781475064536, + "tokens_seen": 2900557824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012221152303001123, + "loss": 1.2891, + "theoretical_loss": 3.3578665007834916, + "tokens_seen": 2900688896 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012217140105922003, + "loss": 1.2849, + "theoretical_loss": 3.3578548547341422, + "tokens_seen": 2900819968 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012213127908842884, + "loss": 1.2714, + "theoretical_loss": 3.3578432093583355, + "tokens_seen": 2900951040 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012209115711763762, + "loss": 1.2385, + "theoretical_loss": 3.3578315646560024, + "tokens_seen": 2901082112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012205103514684641, + "loss": 1.1756, + "theoretical_loss": 3.357819920627073, + "tokens_seen": 2901213184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012201091317605522, + "loss": 1.3194, + "theoretical_loss": 3.357808277271478, + "tokens_seen": 2901344256 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012197079120526401, + "loss": 1.27, + "theoretical_loss": 3.357796634589149, + "tokens_seen": 2901475328 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012193066923447279, + "loss": 1.2287, + "theoretical_loss": 3.3577849925800156, + "tokens_seen": 2901606400 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.9908344745635986, + "objective/train/avg_token_score": 0.00916551798582077, + "objective/train/docs_used": 1055165, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3219940662384033, + "objective/train/original_loss": 1.3357250690460205, + "objective/train/theoretical_loss": 3.357773351244009, + "objective/train/tokens_used": 1272196576, + "objective/train/unlikelihood_loss": 0.024815205484628677, + "theoretical_loss": 3.357773351244009, + "tokens_seen": 2901737472 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001218905472636816, + "loss": 1.2759, + "theoretical_loss": 3.357773351244009, + "tokens_seen": 2901737472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012185042529289039, + "loss": 1.2349, + "theoretical_loss": 3.3577617105810598, + "tokens_seen": 2901868544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012181030332209918, + "loss": 1.3632, + "theoretical_loss": 3.357750070591098, + "tokens_seen": 2901999616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012177018135130798, + "loss": 1.3085, + "theoretical_loss": 3.3577384312740555, + "tokens_seen": 2902130688 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012173005938051678, + "loss": 1.243, + "theoretical_loss": 3.357726792629862, + "tokens_seen": 2902261760 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012168993740972557, + "loss": 1.3192, + "theoretical_loss": 3.357715154658449, + "tokens_seen": 2902392832 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012164981543893435, + "loss": 1.3369, + "theoretical_loss": 3.357703517359747, + "tokens_seen": 2902523904 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012160969346814316, + "loss": 1.2726, + "theoretical_loss": 3.357691880733686, + "tokens_seen": 2902654976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012156957149735195, + "loss": 1.1983, + "theoretical_loss": 3.3576802447801977, + "tokens_seen": 2902786048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012152944952656076, + "loss": 1.2284, + "theoretical_loss": 3.3576686094992123, + "tokens_seen": 2902917120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012148932755576954, + "loss": 1.2941, + "theoretical_loss": 3.357656974890661, + "tokens_seen": 2903048192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012144920558497833, + "loss": 1.2826, + "theoretical_loss": 3.357645340954474, + "tokens_seen": 2903179264 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012140908361418714, + "loss": 1.2549, + "theoretical_loss": 3.357633707690583, + "tokens_seen": 2903310336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012136896164339592, + "loss": 1.1831, + "theoretical_loss": 3.357622075098918, + "tokens_seen": 2903441408 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012132883967260472, + "loss": 1.2587, + "theoretical_loss": 3.35761044317941, + "tokens_seen": 2903572480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012128871770181352, + "loss": 1.2651, + "theoretical_loss": 3.35759881193199, + "tokens_seen": 2903703552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012124859573102232, + "loss": 1.2803, + "theoretical_loss": 3.3575871813565885, + "tokens_seen": 2903834624 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001212084737602311, + "loss": 1.2237, + "theoretical_loss": 3.3575755514531367, + "tokens_seen": 2903965696 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001211683517894399, + "loss": 1.2929, + "theoretical_loss": 3.3575639222215656, + "tokens_seen": 2904096768 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001211282298186487, + "loss": 1.2628, + "theoretical_loss": 3.357552293661805, + "tokens_seen": 2904227840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012108810784785749, + "loss": 1.2376, + "theoretical_loss": 3.3575406657737874, + "tokens_seen": 2904358912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012104798587706629, + "loss": 1.2698, + "theoretical_loss": 3.357529038557442, + "tokens_seen": 2904489984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012100786390627508, + "loss": 1.2409, + "theoretical_loss": 3.3575174120127014, + "tokens_seen": 2904621056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012096774193548387, + "loss": 1.2278, + "theoretical_loss": 3.357505786139495, + "tokens_seen": 2904752128 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012092761996469266, + "loss": 1.265, + "theoretical_loss": 3.3574941609377547, + "tokens_seen": 2904883200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012088749799390146, + "loss": 1.2851, + "theoretical_loss": 3.357482536407411, + "tokens_seen": 2905014272 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012084737602311026, + "loss": 1.2421, + "theoretical_loss": 3.357470912548395, + "tokens_seen": 2905145344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012080725405231906, + "loss": 1.2516, + "theoretical_loss": 3.357459289360637, + "tokens_seen": 2905276416 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012076713208152785, + "loss": 1.2472, + "theoretical_loss": 3.357447666844069, + "tokens_seen": 2905407488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012072701011073664, + "loss": 1.2573, + "theoretical_loss": 3.357436044998621, + "tokens_seen": 2905538560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012068688813994543, + "loss": 1.263, + "theoretical_loss": 3.357424423824225, + "tokens_seen": 2905669632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012064676616915424, + "loss": 1.2805, + "theoretical_loss": 3.357412803320811, + "tokens_seen": 2905800704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012060664419836302, + "loss": 1.2474, + "theoretical_loss": 3.3574011834883106, + "tokens_seen": 2905931776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012056652222757183, + "loss": 1.3016, + "theoretical_loss": 3.3573895643266547, + "tokens_seen": 2906062848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012052640025678062, + "loss": 1.2793, + "theoretical_loss": 3.357377945835774, + "tokens_seen": 2906193920 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001204862782859894, + "loss": 1.2278, + "theoretical_loss": 3.3573663280156, + "tokens_seen": 2906324992 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001204461563151982, + "loss": 1.2495, + "theoretical_loss": 3.3573547108660633, + "tokens_seen": 2906456064 + }, + { + "epoch": 0.76, + "learning_rate": 0.000120406034344407, + "loss": 1.2366, + "theoretical_loss": 3.357343094387095, + "tokens_seen": 2906587136 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001203659123736158, + "loss": 1.2696, + "theoretical_loss": 3.357331478578627, + "tokens_seen": 2906718208 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012032579040282458, + "loss": 1.2801, + "theoretical_loss": 3.3573198634405887, + "tokens_seen": 2906849280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012028566843203339, + "loss": 1.2752, + "theoretical_loss": 3.357308248972913, + "tokens_seen": 2906980352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012024554646124218, + "loss": 1.2175, + "theoretical_loss": 3.35729663517553, + "tokens_seen": 2907111424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012020542449045097, + "loss": 1.24, + "theoretical_loss": 3.3572850220483703, + "tokens_seen": 2907242496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012016530251965977, + "loss": 1.2094, + "theoretical_loss": 3.3572734095913663, + "tokens_seen": 2907373568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012012518054886856, + "loss": 1.2691, + "theoretical_loss": 3.357261797804448, + "tokens_seen": 2907504640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012008505857807737, + "loss": 1.2337, + "theoretical_loss": 3.3572501866875477, + "tokens_seen": 2907635712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012004493660728616, + "loss": 1.2737, + "theoretical_loss": 3.357238576240595, + "tokens_seen": 2907766784 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012000481463649494, + "loss": 1.2711, + "theoretical_loss": 3.3572269664635224, + "tokens_seen": 2907897856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011996469266570374, + "loss": 1.2216, + "theoretical_loss": 3.35721535735626, + "tokens_seen": 2908028928 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011992457069491254, + "loss": 1.2105, + "theoretical_loss": 3.35720374891874, + "tokens_seen": 2908160000 + }, + { + "epoch": 0.76, + "objective/train/aligned_token_count": 0.9926728010177612, + "objective/train/avg_token_score": 0.007327239494770765, + "objective/train/docs_used": 1057433, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2782224416732788, + "objective/train/original_loss": 1.287460446357727, + "objective/train/theoretical_loss": 3.357192141150893, + "objective/train/tokens_used": 1278750176, + "objective/train/unlikelihood_loss": 0.02227126434445381, + "theoretical_loss": 3.357192141150893, + "tokens_seen": 2908291072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011988444872412133, + "loss": 1.2332, + "theoretical_loss": 3.357192141150893, + "tokens_seen": 2908291072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011984432675333012, + "loss": 1.2855, + "theoretical_loss": 3.35718053405265, + "tokens_seen": 2908422144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011980420478253893, + "loss": 1.2246, + "theoretical_loss": 3.3571689276239427, + "tokens_seen": 2908553216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011976408281174772, + "loss": 1.2304, + "theoretical_loss": 3.357157321864702, + "tokens_seen": 2908684288 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001197239608409565, + "loss": 1.2008, + "theoretical_loss": 3.3571457167748595, + "tokens_seen": 2908815360 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001196838388701653, + "loss": 1.242, + "theoretical_loss": 3.3571341123543457, + "tokens_seen": 2908946432 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001196437168993741, + "loss": 1.177, + "theoretical_loss": 3.3571225086030925, + "tokens_seen": 2909077504 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001196035949285829, + "loss": 1.3103, + "theoretical_loss": 3.357110905521031, + "tokens_seen": 2909208576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011956347295779169, + "loss": 1.2819, + "theoretical_loss": 3.3570993031080922, + "tokens_seen": 2909339648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011952335098700048, + "loss": 1.2237, + "theoretical_loss": 3.3570877013642075, + "tokens_seen": 2909470720 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011948322901620928, + "loss": 1.2688, + "theoretical_loss": 3.357076100289308, + "tokens_seen": 2909601792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011944310704541806, + "loss": 1.3106, + "theoretical_loss": 3.3570644998833257, + "tokens_seen": 2909732864 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011940298507462687, + "loss": 1.2089, + "theoretical_loss": 3.3570529001461913, + "tokens_seen": 2909863936 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011936286310383566, + "loss": 1.2034, + "theoretical_loss": 3.357041301077836, + "tokens_seen": 2909995008 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011932274113304447, + "loss": 1.3054, + "theoretical_loss": 3.357029702678192, + "tokens_seen": 2910126080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011928261916225325, + "loss": 1.1809, + "theoretical_loss": 3.357018104947189, + "tokens_seen": 2910257152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011924249719146204, + "loss": 1.3626, + "theoretical_loss": 3.35700650788476, + "tokens_seen": 2910388224 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011920237522067085, + "loss": 1.206, + "theoretical_loss": 3.3569949114908355, + "tokens_seen": 2910519296 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011916225324987964, + "loss": 1.2491, + "theoretical_loss": 3.356983315765347, + "tokens_seen": 2910650368 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011912213127908843, + "loss": 1.2832, + "theoretical_loss": 3.356971720708226, + "tokens_seen": 2910781440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011908200930829723, + "loss": 1.302, + "theoretical_loss": 3.3569601263194038, + "tokens_seen": 2910912512 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011904188733750602, + "loss": 1.3247, + "theoretical_loss": 3.3569485325988113, + "tokens_seen": 2911043584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011900176536671481, + "loss": 1.3044, + "theoretical_loss": 3.3569369395463813, + "tokens_seen": 2911174656 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001189616433959236, + "loss": 1.2556, + "theoretical_loss": 3.3569253471620435, + "tokens_seen": 2911305728 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011892152142513241, + "loss": 1.2467, + "theoretical_loss": 3.3569137554457305, + "tokens_seen": 2911436800 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001188813994543412, + "loss": 1.2644, + "theoretical_loss": 3.3569021643973733, + "tokens_seen": 2911567872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011884127748355, + "loss": 1.197, + "theoretical_loss": 3.3568905740169033, + "tokens_seen": 2911698944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011880115551275879, + "loss": 1.2388, + "theoretical_loss": 3.3568789843042524, + "tokens_seen": 2911830016 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011876103354196758, + "loss": 1.2383, + "theoretical_loss": 3.3568673952593513, + "tokens_seen": 2911961088 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011872091157117639, + "loss": 1.2673, + "theoretical_loss": 3.356855806882132, + "tokens_seen": 2912092160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011868078960038517, + "loss": 1.3615, + "theoretical_loss": 3.3568442191725256, + "tokens_seen": 2912223232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011864066762959397, + "loss": 1.2625, + "theoretical_loss": 3.3568326321304642, + "tokens_seen": 2912354304 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011860054565880277, + "loss": 1.2884, + "theoretical_loss": 3.356821045755879, + "tokens_seen": 2912485376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011856042368801154, + "loss": 1.2678, + "theoretical_loss": 3.3568094600487015, + "tokens_seen": 2912616448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011852030171722035, + "loss": 1.226, + "theoretical_loss": 3.356797875008863, + "tokens_seen": 2912747520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011848017974642914, + "loss": 1.2687, + "theoretical_loss": 3.3567862906362955, + "tokens_seen": 2912878592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011844005777563795, + "loss": 1.2127, + "theoretical_loss": 3.35677470693093, + "tokens_seen": 2913009664 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011839993580484673, + "loss": 1.1702, + "theoretical_loss": 3.3567631238926983, + "tokens_seen": 2913140736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011835981383405554, + "loss": 1.2828, + "theoretical_loss": 3.3567515415215325, + "tokens_seen": 2913271808 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011831969186326433, + "loss": 1.2894, + "theoretical_loss": 3.356739959817363, + "tokens_seen": 2913402880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011827956989247312, + "loss": 1.2204, + "theoretical_loss": 3.3567283787801223, + "tokens_seen": 2913533952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011823944792168191, + "loss": 1.3282, + "theoretical_loss": 3.356716798409742, + "tokens_seen": 2913665024 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001181993259508907, + "loss": 1.253, + "theoretical_loss": 3.3567052187061535, + "tokens_seen": 2913796096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011815920398009951, + "loss": 1.2393, + "theoretical_loss": 3.356693639669288, + "tokens_seen": 2913927168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011811908200930829, + "loss": 1.2145, + "theoretical_loss": 3.3566820612990775, + "tokens_seen": 2914058240 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001180789600385171, + "loss": 1.2451, + "theoretical_loss": 3.3566704835954537, + "tokens_seen": 2914189312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011803883806772589, + "loss": 1.229, + "theoretical_loss": 3.3566589065583483, + "tokens_seen": 2914320384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011799871609693468, + "loss": 1.2524, + "theoretical_loss": 3.356647330187693, + "tokens_seen": 2914451456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011795859412614348, + "loss": 1.3059, + "theoretical_loss": 3.356635754483419, + "tokens_seen": 2914582528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011791847215535227, + "loss": 1.2083, + "theoretical_loss": 3.3566241794454585, + "tokens_seen": 2914713600 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.9948214888572693, + "objective/train/avg_token_score": 0.005178466439247131, + "objective/train/docs_used": 1059748, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.321231722831726, + "objective/train/original_loss": 1.3285338878631592, + "objective/train/theoretical_loss": 3.3566126050737424, + "objective/train/tokens_used": 1285303776, + "objective/train/unlikelihood_loss": 0.014435913413763046, + "theoretical_loss": 3.3566126050737424, + "tokens_seen": 2914844672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011787835018456108, + "loss": 1.2938, + "theoretical_loss": 3.3566126050737424, + "tokens_seen": 2914844672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011783822821376987, + "loss": 1.2964, + "theoretical_loss": 3.3566010313682035, + "tokens_seen": 2914975744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011779810624297865, + "loss": 1.2313, + "theoretical_loss": 3.3565894583287728, + "tokens_seen": 2915106816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011775798427218745, + "loss": 1.22, + "theoretical_loss": 3.356577885955382, + "tokens_seen": 2915237888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011771786230139625, + "loss": 1.3706, + "theoretical_loss": 3.3565663142479636, + "tokens_seen": 2915368960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011767774033060505, + "loss": 1.2842, + "theoretical_loss": 3.3565547432064484, + "tokens_seen": 2915500032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011763761835981383, + "loss": 1.3176, + "theoretical_loss": 3.356543172830768, + "tokens_seen": 2915631104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011759749638902264, + "loss": 1.2552, + "theoretical_loss": 3.356531603120855, + "tokens_seen": 2915762176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011755737441823143, + "loss": 1.315, + "theoretical_loss": 3.356520034076641, + "tokens_seen": 2915893248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011751725244744021, + "loss": 1.1876, + "theoretical_loss": 3.3565084656980573, + "tokens_seen": 2916024320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011747713047664902, + "loss": 1.2768, + "theoretical_loss": 3.356496897985036, + "tokens_seen": 2916155392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011743700850585781, + "loss": 1.2564, + "theoretical_loss": 3.356485330937509, + "tokens_seen": 2916286464 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011739688653506662, + "loss": 1.1936, + "theoretical_loss": 3.356473764555408, + "tokens_seen": 2916417536 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001173567645642754, + "loss": 1.2173, + "theoretical_loss": 3.3564621988386647, + "tokens_seen": 2916548608 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011731664259348419, + "loss": 1.2465, + "theoretical_loss": 3.356450633787211, + "tokens_seen": 2916679680 + }, + { + "epoch": 0.77, + "learning_rate": 0.000117276520622693, + "loss": 1.313, + "theoretical_loss": 3.3564390694009787, + "tokens_seen": 2916810752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011723639865190179, + "loss": 1.2073, + "theoretical_loss": 3.3564275056799, + "tokens_seen": 2916941824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011719627668111058, + "loss": 1.269, + "theoretical_loss": 3.3564159426239057, + "tokens_seen": 2917072896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011715615471031937, + "loss": 1.283, + "theoretical_loss": 3.356404380232929, + "tokens_seen": 2917203968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011711603273952818, + "loss": 1.2513, + "theoretical_loss": 3.356392818506901, + "tokens_seen": 2917335040 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011707591076873696, + "loss": 1.2591, + "theoretical_loss": 3.3563812574457534, + "tokens_seen": 2917466112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011703578879794575, + "loss": 1.2976, + "theoretical_loss": 3.356369697049419, + "tokens_seen": 2917597184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011699566682715456, + "loss": 1.2343, + "theoretical_loss": 3.3563581373178293, + "tokens_seen": 2917728256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011695554485636335, + "loss": 1.2813, + "theoretical_loss": 3.356346578250916, + "tokens_seen": 2917859328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011691542288557214, + "loss": 1.2621, + "theoretical_loss": 3.356335019848611, + "tokens_seen": 2917990400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011687530091478094, + "loss": 1.3044, + "theoretical_loss": 3.356323462110846, + "tokens_seen": 2918121472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011683517894398973, + "loss": 1.2982, + "theoretical_loss": 3.3563119050375536, + "tokens_seen": 2918252544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011679505697319853, + "loss": 1.2271, + "theoretical_loss": 3.3563003486286656, + "tokens_seen": 2918383616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011675493500240731, + "loss": 1.2874, + "theoretical_loss": 3.3562887928841136, + "tokens_seen": 2918514688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011671481303161612, + "loss": 1.2458, + "theoretical_loss": 3.35627723780383, + "tokens_seen": 2918645760 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011667469106082491, + "loss": 1.2778, + "theoretical_loss": 3.3562656833877464, + "tokens_seen": 2918776832 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001166345690900337, + "loss": 1.3139, + "theoretical_loss": 3.3562541296357953, + "tokens_seen": 2918907904 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001165944471192425, + "loss": 1.2277, + "theoretical_loss": 3.356242576547908, + "tokens_seen": 2919038976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011655432514845129, + "loss": 1.2706, + "theoretical_loss": 3.3562310241240167, + "tokens_seen": 2919170048 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001165142031776601, + "loss": 1.3195, + "theoretical_loss": 3.3562194723640544, + "tokens_seen": 2919301120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011647408120686888, + "loss": 1.2694, + "theoretical_loss": 3.356207921267952, + "tokens_seen": 2919432192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011643395923607768, + "loss": 1.3021, + "theoretical_loss": 3.356196370835642, + "tokens_seen": 2919563264 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011639383726528648, + "loss": 1.2664, + "theoretical_loss": 3.356184821067056, + "tokens_seen": 2919694336 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011635371529449527, + "loss": 1.2769, + "theoretical_loss": 3.3561732719621267, + "tokens_seen": 2919825408 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011631359332370406, + "loss": 1.2304, + "theoretical_loss": 3.356161723520786, + "tokens_seen": 2919956480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011627347135291285, + "loss": 1.2512, + "theoretical_loss": 3.356150175742966, + "tokens_seen": 2920087552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011623334938212166, + "loss": 1.2974, + "theoretical_loss": 3.3561386286285986, + "tokens_seen": 2920218624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011619322741133044, + "loss": 1.3303, + "theoretical_loss": 3.3561270821776157, + "tokens_seen": 2920349696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011615310544053925, + "loss": 1.2946, + "theoretical_loss": 3.35611553638995, + "tokens_seen": 2920480768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011611298346974804, + "loss": 1.3113, + "theoretical_loss": 3.3561039912655333, + "tokens_seen": 2920611840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011607286149895683, + "loss": 1.2594, + "theoretical_loss": 3.356092446804298, + "tokens_seen": 2920742912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011603273952816562, + "loss": 1.2628, + "theoretical_loss": 3.3560809030061756, + "tokens_seen": 2920873984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011599261755737442, + "loss": 1.294, + "theoretical_loss": 3.356069359871099, + "tokens_seen": 2921005056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011595249558658322, + "loss": 1.2693, + "theoretical_loss": 3.356057817399, + "tokens_seen": 2921136128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011591237361579202, + "loss": 1.2149, + "theoretical_loss": 3.3560462755898106, + "tokens_seen": 2921267200 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.9905661940574646, + "objective/train/avg_token_score": 0.009433820843696594, + "objective/train/docs_used": 1062024, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3894973993301392, + "objective/train/original_loss": 1.4037880897521973, + "objective/train/theoretical_loss": 3.3560347344434636, + "objective/train/tokens_used": 1291857376, + "objective/train/unlikelihood_loss": 0.024809444323182106, + "theoretical_loss": 3.3560347344434636, + "tokens_seen": 2921398272 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001158722516450008, + "loss": 1.3036, + "theoretical_loss": 3.3560347344434636, + "tokens_seen": 2921398272 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001158321296742096, + "loss": 1.2748, + "theoretical_loss": 3.356023193959891, + "tokens_seen": 2921529344 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001157920077034184, + "loss": 1.2322, + "theoretical_loss": 3.3560116541390244, + "tokens_seen": 2921660416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011575188573262719, + "loss": 1.2444, + "theoretical_loss": 3.3560001149807963, + "tokens_seen": 2921791488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011571176376183598, + "loss": 1.2683, + "theoretical_loss": 3.3559885764851396, + "tokens_seen": 2921922560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011567164179104479, + "loss": 1.2805, + "theoretical_loss": 3.3559770386519854, + "tokens_seen": 2922053632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011563151982025358, + "loss": 1.2713, + "theoretical_loss": 3.3559655014812666, + "tokens_seen": 2922184704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011559139784946236, + "loss": 1.2829, + "theoretical_loss": 3.355953964972916, + "tokens_seen": 2922315776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011555127587867116, + "loss": 1.1875, + "theoretical_loss": 3.355942429126865, + "tokens_seen": 2922446848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011551115390787996, + "loss": 1.3224, + "theoretical_loss": 3.355930893943046, + "tokens_seen": 2922577920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011547103193708876, + "loss": 1.2894, + "theoretical_loss": 3.3559193594213914, + "tokens_seen": 2922708992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011543090996629754, + "loss": 1.2979, + "theoretical_loss": 3.3559078255618338, + "tokens_seen": 2922840064 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011539078799550633, + "loss": 1.2654, + "theoretical_loss": 3.3558962923643048, + "tokens_seen": 2922971136 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011535066602471514, + "loss": 1.2355, + "theoretical_loss": 3.3558847598287374, + "tokens_seen": 2923102208 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011531054405392392, + "loss": 1.2611, + "theoretical_loss": 3.355873227955064, + "tokens_seen": 2923233280 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011527042208313273, + "loss": 1.2932, + "theoretical_loss": 3.355861696743216, + "tokens_seen": 2923364352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011523030011234152, + "loss": 1.2463, + "theoretical_loss": 3.355850166193127, + "tokens_seen": 2923495424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011519017814155033, + "loss": 1.2623, + "theoretical_loss": 3.3558386363047283, + "tokens_seen": 2923626496 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001151500561707591, + "loss": 1.3299, + "theoretical_loss": 3.355827107077953, + "tokens_seen": 2923757568 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001151099341999679, + "loss": 1.2728, + "theoretical_loss": 3.3558155785127326, + "tokens_seen": 2923888640 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001150698122291767, + "loss": 1.224, + "theoretical_loss": 3.3558040506090006, + "tokens_seen": 2924019712 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001150296902583855, + "loss": 1.2408, + "theoretical_loss": 3.3557925233666888, + "tokens_seen": 2924150784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011498956828759429, + "loss": 1.338, + "theoretical_loss": 3.3557809967857293, + "tokens_seen": 2924281856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011494944631680308, + "loss": 1.3044, + "theoretical_loss": 3.355769470866055, + "tokens_seen": 2924412928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011490932434601187, + "loss": 1.4049, + "theoretical_loss": 3.355757945607598, + "tokens_seen": 2924544000 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011486920237522068, + "loss": 1.2461, + "theoretical_loss": 3.3557464210102914, + "tokens_seen": 2924675072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011482908040442946, + "loss": 1.299, + "theoretical_loss": 3.355734897074067, + "tokens_seen": 2924806144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011478895843363827, + "loss": 1.3373, + "theoretical_loss": 3.3557233737988574, + "tokens_seen": 2924937216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011474883646284706, + "loss": 1.3228, + "theoretical_loss": 3.355711851184595, + "tokens_seen": 2925068288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011470871449205585, + "loss": 1.3186, + "theoretical_loss": 3.3557003292312126, + "tokens_seen": 2925199360 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011466859252126464, + "loss": 1.2142, + "theoretical_loss": 3.355688807938642, + "tokens_seen": 2925330432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011462847055047344, + "loss": 1.2225, + "theoretical_loss": 3.3556772873068166, + "tokens_seen": 2925461504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011458834857968224, + "loss": 1.3291, + "theoretical_loss": 3.3556657673356685, + "tokens_seen": 2925592576 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011454822660889102, + "loss": 1.322, + "theoretical_loss": 3.3556542480251297, + "tokens_seen": 2925723648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011450810463809983, + "loss": 1.3435, + "theoretical_loss": 3.3556427293751336, + "tokens_seen": 2925854720 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011446798266730862, + "loss": 1.2859, + "theoretical_loss": 3.355631211385612, + "tokens_seen": 2925985792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011442786069651743, + "loss": 1.2639, + "theoretical_loss": 3.355619694056498, + "tokens_seen": 2926116864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011438773872572621, + "loss": 1.2356, + "theoretical_loss": 3.3556081773877238, + "tokens_seen": 2926247936 + }, + { + "epoch": 0.77, + "learning_rate": 0.000114347616754935, + "loss": 1.3059, + "theoretical_loss": 3.3555966613792223, + "tokens_seen": 2926379008 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011430749478414381, + "loss": 1.1959, + "theoretical_loss": 3.3555851460309256, + "tokens_seen": 2926510080 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011426737281335259, + "loss": 1.3257, + "theoretical_loss": 3.3555736313427666, + "tokens_seen": 2926641152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011422725084256139, + "loss": 1.2366, + "theoretical_loss": 3.355562117314678, + "tokens_seen": 2926772224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011418712887177019, + "loss": 1.2597, + "theoretical_loss": 3.3555506039465923, + "tokens_seen": 2926903296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011414700690097898, + "loss": 1.264, + "theoretical_loss": 3.355539091238442, + "tokens_seen": 2927034368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011410688493018777, + "loss": 1.2596, + "theoretical_loss": 3.3555275791901593, + "tokens_seen": 2927165440 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011406676295939656, + "loss": 1.2938, + "theoretical_loss": 3.3555160678016778, + "tokens_seen": 2927296512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011402664098860537, + "loss": 1.2537, + "theoretical_loss": 3.35550455707293, + "tokens_seen": 2927427584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011398651901781416, + "loss": 1.2346, + "theoretical_loss": 3.3554930470038475, + "tokens_seen": 2927558656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011394639704702294, + "loss": 1.2741, + "theoretical_loss": 3.355481537594364, + "tokens_seen": 2927689728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011390627507623175, + "loss": 1.2154, + "theoretical_loss": 3.3554700288444117, + "tokens_seen": 2927820800 + }, + { + "epoch": 0.77, + "objective/train/aligned_token_count": 0.9944932460784912, + "objective/train/avg_token_score": 0.005506803747266531, + "objective/train/docs_used": 1064246, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2894902229309082, + "objective/train/original_loss": 1.297324776649475, + "objective/train/theoretical_loss": 3.3554585207539236, + "objective/train/tokens_used": 1298410976, + "objective/train/unlikelihood_loss": 0.014317471534013748, + "theoretical_loss": 3.3554585207539236, + "tokens_seen": 2927951872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011386615310544054, + "loss": 1.2745, + "theoretical_loss": 3.3554585207539236, + "tokens_seen": 2927951872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011382603113464933, + "loss": 1.2659, + "theoretical_loss": 3.355447013322832, + "tokens_seen": 2928082944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011378590916385813, + "loss": 1.2854, + "theoretical_loss": 3.3554355065510704, + "tokens_seen": 2928214016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011374578719306693, + "loss": 1.1813, + "theoretical_loss": 3.3554240004385703, + "tokens_seen": 2928345088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011370566522227573, + "loss": 1.2718, + "theoretical_loss": 3.3554124949852655, + "tokens_seen": 2928476160 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001136655432514845, + "loss": 1.2806, + "theoretical_loss": 3.3554009901910877, + "tokens_seen": 2928607232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011362542128069331, + "loss": 1.2496, + "theoretical_loss": 3.3553894860559708, + "tokens_seen": 2928738304 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001135852993099021, + "loss": 1.2858, + "theoretical_loss": 3.355377982579847, + "tokens_seen": 2928869376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011354517733911091, + "loss": 1.2856, + "theoretical_loss": 3.355366479762649, + "tokens_seen": 2929000448 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011350505536831969, + "loss": 1.3056, + "theoretical_loss": 3.3553549776043097, + "tokens_seen": 2929131520 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001134649333975285, + "loss": 1.2021, + "theoretical_loss": 3.3553434761047614, + "tokens_seen": 2929262592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011342481142673729, + "loss": 1.2599, + "theoretical_loss": 3.3553319752639377, + "tokens_seen": 2929393664 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011338468945594607, + "loss": 1.2965, + "theoretical_loss": 3.3553204750817707, + "tokens_seen": 2929524736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011334456748515487, + "loss": 1.1963, + "theoretical_loss": 3.3553089755581937, + "tokens_seen": 2929655808 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011330444551436367, + "loss": 1.2926, + "theoretical_loss": 3.3552974766931394, + "tokens_seen": 2929786880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011326432354357247, + "loss": 1.2384, + "theoretical_loss": 3.35528597848654, + "tokens_seen": 2929917952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011322420157278125, + "loss": 1.2318, + "theoretical_loss": 3.3552744809383297, + "tokens_seen": 2930049024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011318407960199004, + "loss": 1.2735, + "theoretical_loss": 3.3552629840484403, + "tokens_seen": 2930180096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011314395763119885, + "loss": 1.227, + "theoretical_loss": 3.3552514878168047, + "tokens_seen": 2930311168 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011310383566040764, + "loss": 1.2249, + "theoretical_loss": 3.355239992243356, + "tokens_seen": 2930442240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011306371368961644, + "loss": 1.2979, + "theoretical_loss": 3.3552284973280275, + "tokens_seen": 2930573312 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011302359171882523, + "loss": 1.2792, + "theoretical_loss": 3.3552170030707513, + "tokens_seen": 2930704384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011298346974803404, + "loss": 1.2666, + "theoretical_loss": 3.355205509471461, + "tokens_seen": 2930835456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011294334777724281, + "loss": 1.2913, + "theoretical_loss": 3.355194016530089, + "tokens_seen": 2930966528 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011290322580645161, + "loss": 1.2196, + "theoretical_loss": 3.355182524246568, + "tokens_seen": 2931097600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011286310383566041, + "loss": 1.3575, + "theoretical_loss": 3.3551710326208317, + "tokens_seen": 2931228672 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001128229818648692, + "loss": 1.2736, + "theoretical_loss": 3.355159541652813, + "tokens_seen": 2931359744 + }, + { + "epoch": 0.78, + "learning_rate": 0.000112782859894078, + "loss": 1.275, + "theoretical_loss": 3.355148051342444, + "tokens_seen": 2931490816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011274273792328679, + "loss": 1.1974, + "theoretical_loss": 3.3551365616896582, + "tokens_seen": 2931621888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011270261595249558, + "loss": 1.3026, + "theoretical_loss": 3.3551250726943884, + "tokens_seen": 2931752960 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011266249398170439, + "loss": 1.2447, + "theoretical_loss": 3.3551135843565683, + "tokens_seen": 2931884032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011262237201091317, + "loss": 1.3396, + "theoretical_loss": 3.35510209667613, + "tokens_seen": 2932015104 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011258225004012198, + "loss": 1.1822, + "theoretical_loss": 3.355090609653007, + "tokens_seen": 2932146176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011254212806933077, + "loss": 1.1998, + "theoretical_loss": 3.3550791232871315, + "tokens_seen": 2932277248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011250200609853958, + "loss": 1.2359, + "theoretical_loss": 3.3550676375784376, + "tokens_seen": 2932408320 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011246188412774835, + "loss": 1.2253, + "theoretical_loss": 3.3550561525268576, + "tokens_seen": 2932539392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011242176215695715, + "loss": 1.2214, + "theoretical_loss": 3.355044668132325, + "tokens_seen": 2932670464 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011238164018616595, + "loss": 1.3173, + "theoretical_loss": 3.355033184394773, + "tokens_seen": 2932801536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011234151821537473, + "loss": 1.2194, + "theoretical_loss": 3.3550217013141337, + "tokens_seen": 2932932608 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011230139624458354, + "loss": 1.2148, + "theoretical_loss": 3.355010218890341, + "tokens_seen": 2933063680 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011226127427379233, + "loss": 1.2443, + "theoretical_loss": 3.354998737123328, + "tokens_seen": 2933194752 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011222115230300112, + "loss": 1.2149, + "theoretical_loss": 3.354987256013027, + "tokens_seen": 2933325824 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011218103033220992, + "loss": 1.3295, + "theoretical_loss": 3.354975775559372, + "tokens_seen": 2933456896 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011214090836141871, + "loss": 1.272, + "theoretical_loss": 3.3549642957622954, + "tokens_seen": 2933587968 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011210078639062752, + "loss": 1.2655, + "theoretical_loss": 3.354952816621731, + "tokens_seen": 2933719040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011206066441983631, + "loss": 1.2381, + "theoretical_loss": 3.3549413381376114, + "tokens_seen": 2933850112 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001120205424490451, + "loss": 1.258, + "theoretical_loss": 3.3549298603098703, + "tokens_seen": 2933981184 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001119804204782539, + "loss": 1.3181, + "theoretical_loss": 3.35491838313844, + "tokens_seen": 2934112256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011194029850746269, + "loss": 1.2868, + "theoretical_loss": 3.3549069066232544, + "tokens_seen": 2934243328 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011190017653667148, + "loss": 1.2255, + "theoretical_loss": 3.3548954307642465, + "tokens_seen": 2934374400 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.9888792037963867, + "objective/train/avg_token_score": 0.011120806448161602, + "objective/train/docs_used": 1066769, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3049077987670898, + "objective/train/original_loss": 1.321727991104126, + "objective/train/theoretical_loss": 3.354883955561349, + "objective/train/tokens_used": 1304964576, + "objective/train/unlikelihood_loss": 0.02587653137743473, + "theoretical_loss": 3.354883955561349, + "tokens_seen": 2934505472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011186005456588027, + "loss": 1.2638, + "theoretical_loss": 3.354883955561349, + "tokens_seen": 2934505472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011181993259508908, + "loss": 1.2223, + "theoretical_loss": 3.3548724810144956, + "tokens_seen": 2934636544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011177981062429787, + "loss": 1.2273, + "theoretical_loss": 3.3548610071236196, + "tokens_seen": 2934767616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011173968865350665, + "loss": 1.2824, + "theoretical_loss": 3.3548495338886535, + "tokens_seen": 2934898688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011169956668271546, + "loss": 1.2769, + "theoretical_loss": 3.354838061309531, + "tokens_seen": 2935029760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011165944471192425, + "loss": 1.2288, + "theoretical_loss": 3.3548265893861853, + "tokens_seen": 2935160832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011161932274113306, + "loss": 1.2384, + "theoretical_loss": 3.35481511811855, + "tokens_seen": 2935291904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011157920077034184, + "loss": 1.1953, + "theoretical_loss": 3.3548036475065577, + "tokens_seen": 2935422976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011153907879955064, + "loss": 1.1779, + "theoretical_loss": 3.354792177550142, + "tokens_seen": 2935554048 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011149895682875944, + "loss": 1.3442, + "theoretical_loss": 3.354780708249236, + "tokens_seen": 2935685120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011145883485796821, + "loss": 1.3042, + "theoretical_loss": 3.354769239603773, + "tokens_seen": 2935816192 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011141871288717702, + "loss": 1.2303, + "theoretical_loss": 3.354757771613686, + "tokens_seen": 2935947264 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011137859091638581, + "loss": 1.176, + "theoretical_loss": 3.354746304278909, + "tokens_seen": 2936078336 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011133846894559462, + "loss": 1.1929, + "theoretical_loss": 3.354734837599375, + "tokens_seen": 2936209408 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001112983469748034, + "loss": 1.2449, + "theoretical_loss": 3.3547233715750173, + "tokens_seen": 2936340480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011125822500401219, + "loss": 1.19, + "theoretical_loss": 3.354711906205769, + "tokens_seen": 2936471552 + }, + { + "epoch": 0.78, + "learning_rate": 0.000111218103033221, + "loss": 1.2055, + "theoretical_loss": 3.3547004414915635, + "tokens_seen": 2936602624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011117798106242979, + "loss": 1.2813, + "theoretical_loss": 3.3546889774323345, + "tokens_seen": 2936733696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011113785909163858, + "loss": 1.2394, + "theoretical_loss": 3.3546775140280145, + "tokens_seen": 2936864768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011109773712084738, + "loss": 1.1958, + "theoretical_loss": 3.354666051278538, + "tokens_seen": 2936995840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011105761515005618, + "loss": 1.2596, + "theoretical_loss": 3.3546545891838377, + "tokens_seen": 2937126912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011101749317926496, + "loss": 1.2278, + "theoretical_loss": 3.354643127743847, + "tokens_seen": 2937257984 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011097737120847375, + "loss": 1.3104, + "theoretical_loss": 3.3546316669584995, + "tokens_seen": 2937389056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011093724923768256, + "loss": 1.2809, + "theoretical_loss": 3.3546202068277284, + "tokens_seen": 2937520128 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011089712726689135, + "loss": 1.2978, + "theoretical_loss": 3.3546087473514676, + "tokens_seen": 2937651200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011085700529610015, + "loss": 1.2337, + "theoretical_loss": 3.3545972885296496, + "tokens_seen": 2937782272 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011081688332530894, + "loss": 1.2279, + "theoretical_loss": 3.3545858303622085, + "tokens_seen": 2937913344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011077676135451773, + "loss": 1.2597, + "theoretical_loss": 3.3545743728490773, + "tokens_seen": 2938044416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011073663938372654, + "loss": 1.2793, + "theoretical_loss": 3.35456291599019, + "tokens_seen": 2938175488 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011069651741293532, + "loss": 1.3252, + "theoretical_loss": 3.3545514597854798, + "tokens_seen": 2938306560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011065639544214412, + "loss": 1.3123, + "theoretical_loss": 3.35454000423488, + "tokens_seen": 2938437632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011061627347135292, + "loss": 1.2509, + "theoretical_loss": 3.354528549338325, + "tokens_seen": 2938568704 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011057615150056171, + "loss": 1.2441, + "theoretical_loss": 3.354517095095747, + "tokens_seen": 2938699776 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001105360295297705, + "loss": 1.2492, + "theoretical_loss": 3.3545056415070795, + "tokens_seen": 2938830848 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001104959075589793, + "loss": 1.2216, + "theoretical_loss": 3.354494188572257, + "tokens_seen": 2938961920 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001104557855881881, + "loss": 1.2879, + "theoretical_loss": 3.3544827362912124, + "tokens_seen": 2939092992 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011041566361739688, + "loss": 1.2263, + "theoretical_loss": 3.3544712846638793, + "tokens_seen": 2939224064 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011037554164660569, + "loss": 1.2944, + "theoretical_loss": 3.354459833690192, + "tokens_seen": 2939355136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011033541967581448, + "loss": 1.2904, + "theoretical_loss": 3.3544483833700824, + "tokens_seen": 2939486208 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011029529770502327, + "loss": 1.2663, + "theoretical_loss": 3.3544369337034854, + "tokens_seen": 2939617280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011025517573423206, + "loss": 1.3156, + "theoretical_loss": 3.354425484690334, + "tokens_seen": 2939748352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011021505376344086, + "loss": 1.2517, + "theoretical_loss": 3.3544140363305623, + "tokens_seen": 2939879424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011017493179264966, + "loss": 1.2552, + "theoretical_loss": 3.3544025886241036, + "tokens_seen": 2940010496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011013480982185846, + "loss": 1.2529, + "theoretical_loss": 3.354391141570891, + "tokens_seen": 2940141568 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011009468785106725, + "loss": 1.2568, + "theoretical_loss": 3.354379695170859, + "tokens_seen": 2940272640 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011005456588027604, + "loss": 1.2328, + "theoretical_loss": 3.35436824942394, + "tokens_seen": 2940403712 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011001444390948483, + "loss": 1.21, + "theoretical_loss": 3.3543568043300693, + "tokens_seen": 2940534784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010997432193869363, + "loss": 1.2511, + "theoretical_loss": 3.354345359889179, + "tokens_seen": 2940665856 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010993419996790242, + "loss": 1.2532, + "theoretical_loss": 3.3543339161012034, + "tokens_seen": 2940796928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010989407799711123, + "loss": 1.2532, + "theoretical_loss": 3.354322472966076, + "tokens_seen": 2940928000 + }, + { + "epoch": 0.78, + "objective/train/aligned_token_count": 0.9926429986953735, + "objective/train/avg_token_score": 0.0073569766245782375, + "objective/train/docs_used": 1069310, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2810592651367188, + "objective/train/original_loss": 1.291106939315796, + "objective/train/theoretical_loss": 3.354311030483731, + "objective/train/tokens_used": 1311518176, + "objective/train/unlikelihood_loss": 0.020033488050103188, + "theoretical_loss": 3.354311030483731, + "tokens_seen": 2941059072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010985395602632002, + "loss": 1.2777, + "theoretical_loss": 3.354311030483731, + "tokens_seen": 2941059072 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001098138340555288, + "loss": 1.205, + "theoretical_loss": 3.354299588654101, + "tokens_seen": 2941190144 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001097737120847376, + "loss": 1.2856, + "theoretical_loss": 3.3542881474771207, + "tokens_seen": 2941321216 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001097335901139464, + "loss": 1.2417, + "theoretical_loss": 3.3542767069527235, + "tokens_seen": 2941452288 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001096934681431552, + "loss": 1.3067, + "theoretical_loss": 3.354265267080843, + "tokens_seen": 2941583360 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010965334617236398, + "loss": 1.274, + "theoretical_loss": 3.3542538278614127, + "tokens_seen": 2941714432 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010961322420157279, + "loss": 1.1792, + "theoretical_loss": 3.3542423892943667, + "tokens_seen": 2941845504 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010957310223078158, + "loss": 1.2466, + "theoretical_loss": 3.3542309513796384, + "tokens_seen": 2941976576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010953298025999036, + "loss": 1.287, + "theoretical_loss": 3.354219514117162, + "tokens_seen": 2942107648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010949285828919917, + "loss": 1.3234, + "theoretical_loss": 3.3542080775068706, + "tokens_seen": 2942238720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010945273631840796, + "loss": 1.2646, + "theoretical_loss": 3.3541966415486986, + "tokens_seen": 2942369792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010941261434761677, + "loss": 1.187, + "theoretical_loss": 3.354185206242579, + "tokens_seen": 2942500864 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010937249237682555, + "loss": 1.2367, + "theoretical_loss": 3.3541737715884468, + "tokens_seen": 2942631936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010933237040603435, + "loss": 1.2669, + "theoretical_loss": 3.3541623375862346, + "tokens_seen": 2942763008 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010929224843524314, + "loss": 1.2422, + "theoretical_loss": 3.3541509042358766, + "tokens_seen": 2942894080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010925212646445194, + "loss": 1.2043, + "theoretical_loss": 3.354139471537307, + "tokens_seen": 2943025152 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010921200449366073, + "loss": 1.2583, + "theoretical_loss": 3.354128039490459, + "tokens_seen": 2943156224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010917188252286952, + "loss": 1.2437, + "theoretical_loss": 3.354116608095267, + "tokens_seen": 2943287296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010913176055207833, + "loss": 1.3111, + "theoretical_loss": 3.354105177351664, + "tokens_seen": 2943418368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010909163858128711, + "loss": 1.2513, + "theoretical_loss": 3.3540937472595846, + "tokens_seen": 2943549440 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001090515166104959, + "loss": 1.263, + "theoretical_loss": 3.3540823178189623, + "tokens_seen": 2943680512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010901139463970471, + "loss": 1.2468, + "theoretical_loss": 3.3540708890297317, + "tokens_seen": 2943811584 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001089712726689135, + "loss": 1.3018, + "theoretical_loss": 3.3540594608918255, + "tokens_seen": 2943942656 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001089311506981223, + "loss": 1.3594, + "theoretical_loss": 3.3540480334051783, + "tokens_seen": 2944073728 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010889102872733109, + "loss": 1.2573, + "theoretical_loss": 3.354036606569724, + "tokens_seen": 2944204800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010885090675653989, + "loss": 1.3372, + "theoretical_loss": 3.3540251803853964, + "tokens_seen": 2944335872 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010881078478574869, + "loss": 1.3226, + "theoretical_loss": 3.354013754852129, + "tokens_seen": 2944466944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010877066281495746, + "loss": 1.3411, + "theoretical_loss": 3.3540023299698563, + "tokens_seen": 2944598016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010873054084416627, + "loss": 1.3036, + "theoretical_loss": 3.353990905738512, + "tokens_seen": 2944729088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010869041887337506, + "loss": 1.2328, + "theoretical_loss": 3.3539794821580307, + "tokens_seen": 2944860160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010865029690258386, + "loss": 1.2715, + "theoretical_loss": 3.353968059228345, + "tokens_seen": 2944991232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010861017493179265, + "loss": 1.2766, + "theoretical_loss": 3.3539566369493894, + "tokens_seen": 2945122304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010857005296100144, + "loss": 1.3194, + "theoretical_loss": 3.3539452153210987, + "tokens_seen": 2945253376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010852993099021025, + "loss": 1.297, + "theoretical_loss": 3.3539337943434058, + "tokens_seen": 2945384448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010848980901941903, + "loss": 1.2462, + "theoretical_loss": 3.3539223740162454, + "tokens_seen": 2945515520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010844968704862783, + "loss": 1.2366, + "theoretical_loss": 3.3539109543395513, + "tokens_seen": 2945646592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010840956507783663, + "loss": 1.2893, + "theoretical_loss": 3.353899535313257, + "tokens_seen": 2945777664 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010836944310704543, + "loss": 1.1705, + "theoretical_loss": 3.3538881169372976, + "tokens_seen": 2945908736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010832932113625421, + "loss": 1.3105, + "theoretical_loss": 3.353876699211606, + "tokens_seen": 2946039808 + }, + { + "epoch": 0.79, + "learning_rate": 0.000108289199165463, + "loss": 1.2555, + "theoretical_loss": 3.353865282136117, + "tokens_seen": 2946170880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010824907719467181, + "loss": 1.2907, + "theoretical_loss": 3.3538538657107644, + "tokens_seen": 2946301952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010820895522388059, + "loss": 1.2698, + "theoretical_loss": 3.353842449935482, + "tokens_seen": 2946433024 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001081688332530894, + "loss": 1.2136, + "theoretical_loss": 3.353831034810204, + "tokens_seen": 2946564096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010812871128229819, + "loss": 1.2694, + "theoretical_loss": 3.353819620334865, + "tokens_seen": 2946695168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010808858931150698, + "loss": 1.2827, + "theoretical_loss": 3.3538082065093984, + "tokens_seen": 2946826240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010804846734071577, + "loss": 1.2335, + "theoretical_loss": 3.3537967933337387, + "tokens_seen": 2946957312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010800834536992457, + "loss": 1.2749, + "theoretical_loss": 3.3537853808078197, + "tokens_seen": 2947088384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010796822339913337, + "loss": 1.2582, + "theoretical_loss": 3.353773968931576, + "tokens_seen": 2947219456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010792810142834217, + "loss": 1.182, + "theoretical_loss": 3.3537625577049415, + "tokens_seen": 2947350528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010788797945755096, + "loss": 1.2494, + "theoretical_loss": 3.35375114712785, + "tokens_seen": 2947481600 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.9914300441741943, + "objective/train/avg_token_score": 0.008569997735321522, + "objective/train/docs_used": 1071710, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.351104974746704, + "objective/train/original_loss": 1.3623846769332886, + "objective/train/theoretical_loss": 3.3537397372002355, + "objective/train/tokens_used": 1318071776, + "objective/train/unlikelihood_loss": 0.023724064230918884, + "theoretical_loss": 3.3537397372002355, + "tokens_seen": 2947612672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010784785748675975, + "loss": 1.2784, + "theoretical_loss": 3.3537397372002355, + "tokens_seen": 2947612672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010780773551596854, + "loss": 1.1761, + "theoretical_loss": 3.353728327922033, + "tokens_seen": 2947743744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010776761354517735, + "loss": 1.2678, + "theoretical_loss": 3.353716919293176, + "tokens_seen": 2947874816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010772749157438613, + "loss": 1.1967, + "theoretical_loss": 3.353705511313599, + "tokens_seen": 2948005888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010768736960359494, + "loss": 1.2605, + "theoretical_loss": 3.3536941039832366, + "tokens_seen": 2948136960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010764724763280373, + "loss": 1.3067, + "theoretical_loss": 3.353682697302022, + "tokens_seen": 2948268032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010760712566201251, + "loss": 1.3105, + "theoretical_loss": 3.35367129126989, + "tokens_seen": 2948399104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010756700369122131, + "loss": 1.2206, + "theoretical_loss": 3.353659885886774, + "tokens_seen": 2948530176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010752688172043011, + "loss": 1.2555, + "theoretical_loss": 3.35364848115261, + "tokens_seen": 2948661248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010748675974963891, + "loss": 1.2568, + "theoretical_loss": 3.3536370770673303, + "tokens_seen": 2948792320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010744663777884769, + "loss": 1.1664, + "theoretical_loss": 3.3536256736308703, + "tokens_seen": 2948923392 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001074065158080565, + "loss": 1.2599, + "theoretical_loss": 3.3536142708431638, + "tokens_seen": 2949054464 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010736639383726529, + "loss": 1.3904, + "theoretical_loss": 3.3536028687041455, + "tokens_seen": 2949185536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010732627186647408, + "loss": 1.1787, + "theoretical_loss": 3.3535914672137492, + "tokens_seen": 2949316608 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010728614989568288, + "loss": 1.2784, + "theoretical_loss": 3.353580066371909, + "tokens_seen": 2949447680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010724602792489167, + "loss": 1.2471, + "theoretical_loss": 3.35356866617856, + "tokens_seen": 2949578752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010720590595410048, + "loss": 1.1957, + "theoretical_loss": 3.3535572666336355, + "tokens_seen": 2949709824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010716578398330926, + "loss": 1.3023, + "theoretical_loss": 3.353545867737071, + "tokens_seen": 2949840896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010712566201251805, + "loss": 1.1898, + "theoretical_loss": 3.3535344694887996, + "tokens_seen": 2949971968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010708554004172685, + "loss": 1.2028, + "theoretical_loss": 3.3535230718887563, + "tokens_seen": 2950103040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010704541807093565, + "loss": 1.199, + "theoretical_loss": 3.3535116749368754, + "tokens_seen": 2950234112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010700529610014444, + "loss": 1.288, + "theoretical_loss": 3.353500278633091, + "tokens_seen": 2950365184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010696517412935323, + "loss": 1.2343, + "theoretical_loss": 3.353488882977338, + "tokens_seen": 2950496256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010692505215856204, + "loss": 1.2817, + "theoretical_loss": 3.3534774879695504, + "tokens_seen": 2950627328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010688493018777083, + "loss": 1.2254, + "theoretical_loss": 3.3534660936096623, + "tokens_seen": 2950758400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010684480821697961, + "loss": 1.2921, + "theoretical_loss": 3.3534546998976085, + "tokens_seen": 2950889472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010680468624618842, + "loss": 1.2727, + "theoretical_loss": 3.3534433068333227, + "tokens_seen": 2951020544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010676456427539721, + "loss": 1.242, + "theoretical_loss": 3.35343191441674, + "tokens_seen": 2951151616 + }, + { + "epoch": 0.79, + "learning_rate": 0.000106724442304606, + "loss": 1.2209, + "theoretical_loss": 3.353420522647795, + "tokens_seen": 2951282688 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001066843203338148, + "loss": 1.308, + "theoretical_loss": 3.353409131526422, + "tokens_seen": 2951413760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010664419836302359, + "loss": 1.238, + "theoretical_loss": 3.3533977410525546, + "tokens_seen": 2951544832 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001066040763922324, + "loss": 1.2911, + "theoretical_loss": 3.353386351226128, + "tokens_seen": 2951675904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010656395442144117, + "loss": 1.228, + "theoretical_loss": 3.3533749620470767, + "tokens_seen": 2951806976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010652383245064998, + "loss": 1.2595, + "theoretical_loss": 3.3533635735153347, + "tokens_seen": 2951938048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010648371047985877, + "loss": 1.2927, + "theoretical_loss": 3.353352185630837, + "tokens_seen": 2952069120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010644358850906758, + "loss": 1.2154, + "theoretical_loss": 3.3533407983935177, + "tokens_seen": 2952200192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010640346653827636, + "loss": 1.3203, + "theoretical_loss": 3.3533294118033115, + "tokens_seen": 2952331264 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010636334456748515, + "loss": 1.1922, + "theoretical_loss": 3.3533180258601525, + "tokens_seen": 2952462336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010632322259669396, + "loss": 1.269, + "theoretical_loss": 3.3533066405639755, + "tokens_seen": 2952593408 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010628310062590274, + "loss": 1.212, + "theoretical_loss": 3.3532952559147153, + "tokens_seen": 2952724480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010624297865511154, + "loss": 1.325, + "theoretical_loss": 3.353283871912306, + "tokens_seen": 2952855552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010620285668432034, + "loss": 1.3058, + "theoretical_loss": 3.3532724885566827, + "tokens_seen": 2952986624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010616273471352913, + "loss": 1.3013, + "theoretical_loss": 3.353261105847779, + "tokens_seen": 2953117696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010612261274273792, + "loss": 1.1782, + "theoretical_loss": 3.35324972378553, + "tokens_seen": 2953248768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010608249077194671, + "loss": 1.3128, + "theoretical_loss": 3.3532383423698704, + "tokens_seen": 2953379840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010604236880115552, + "loss": 1.2485, + "theoretical_loss": 3.353226961600735, + "tokens_seen": 2953510912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010600224683036431, + "loss": 1.2761, + "theoretical_loss": 3.3532155814780573, + "tokens_seen": 2953641984 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001059621248595731, + "loss": 1.2434, + "theoretical_loss": 3.353204202001773, + "tokens_seen": 2953773056 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001059220028887819, + "loss": 1.3114, + "theoretical_loss": 3.3531928231718164, + "tokens_seen": 2953904128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010588188091799069, + "loss": 1.3243, + "theoretical_loss": 3.353181444988122, + "tokens_seen": 2954035200 + }, + { + "epoch": 0.79, + "objective/train/aligned_token_count": 0.9946377277374268, + "objective/train/avg_token_score": 0.0053623141720891, + "objective/train/docs_used": 1074062, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2568680047988892, + "objective/train/original_loss": 1.2641804218292236, + "objective/train/theoretical_loss": 3.353170067450624, + "objective/train/tokens_used": 1324625376, + "objective/train/unlikelihood_loss": 0.015389666892588139, + "theoretical_loss": 3.353170067450624, + "tokens_seen": 2954166272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010584175894719948, + "loss": 1.261, + "theoretical_loss": 3.353170067450624, + "tokens_seen": 2954166272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010580163697640828, + "loss": 1.2401, + "theoretical_loss": 3.3531586905592583, + "tokens_seen": 2954297344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010576151500561708, + "loss": 1.2367, + "theoretical_loss": 3.3531473143139583, + "tokens_seen": 2954428416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010572139303482588, + "loss": 1.2395, + "theoretical_loss": 3.353135938714659, + "tokens_seen": 2954559488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010568127106403466, + "loss": 1.2945, + "theoretical_loss": 3.353124563761295, + "tokens_seen": 2954690560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010564114909324346, + "loss": 1.2943, + "theoretical_loss": 3.3531131894538015, + "tokens_seen": 2954821632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010560102712245225, + "loss": 1.2924, + "theoretical_loss": 3.3531018157921126, + "tokens_seen": 2954952704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010556090515166106, + "loss": 1.2702, + "theoretical_loss": 3.353090442776163, + "tokens_seen": 2955083776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010552078318086984, + "loss": 1.2784, + "theoretical_loss": 3.3530790704058884, + "tokens_seen": 2955214848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010548066121007865, + "loss": 1.2455, + "theoretical_loss": 3.3530676986812216, + "tokens_seen": 2955345920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010544053923928744, + "loss": 1.2664, + "theoretical_loss": 3.353056327602099, + "tokens_seen": 2955476992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010540041726849623, + "loss": 1.192, + "theoretical_loss": 3.3530449571684544, + "tokens_seen": 2955608064 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010536029529770502, + "loss": 1.1997, + "theoretical_loss": 3.3530335873802235, + "tokens_seen": 2955739136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010532017332691382, + "loss": 1.2311, + "theoretical_loss": 3.3530222182373395, + "tokens_seen": 2955870208 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010528005135612262, + "loss": 1.2294, + "theoretical_loss": 3.353010849739739, + "tokens_seen": 2956001280 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001052399293853314, + "loss": 1.2526, + "theoretical_loss": 3.3529994818873554, + "tokens_seen": 2956132352 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001051998074145402, + "loss": 1.2106, + "theoretical_loss": 3.352988114680124, + "tokens_seen": 2956263424 + }, + { + "epoch": 0.79, + "learning_rate": 0.000105159685443749, + "loss": 1.3411, + "theoretical_loss": 3.352976748117979, + "tokens_seen": 2956394496 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001051195634729578, + "loss": 1.3201, + "theoretical_loss": 3.352965382200856, + "tokens_seen": 2956525568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010507944150216659, + "loss": 1.2885, + "theoretical_loss": 3.35295401692869, + "tokens_seen": 2956656640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010503931953137538, + "loss": 1.2396, + "theoretical_loss": 3.3529426523014147, + "tokens_seen": 2956787712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010499919756058419, + "loss": 1.2865, + "theoretical_loss": 3.3529312883189655, + "tokens_seen": 2956918784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010495907558979298, + "loss": 1.2668, + "theoretical_loss": 3.3529199249812778, + "tokens_seen": 2957049856 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010491895361900176, + "loss": 1.2875, + "theoretical_loss": 3.3529085622882855, + "tokens_seen": 2957180928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010487883164821056, + "loss": 1.2996, + "theoretical_loss": 3.3528972002399238, + "tokens_seen": 2957312000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010483870967741936, + "loss": 1.287, + "theoretical_loss": 3.352885838836128, + "tokens_seen": 2957443072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010479858770662815, + "loss": 1.2464, + "theoretical_loss": 3.3528744780768323, + "tokens_seen": 2957574144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010475846573583694, + "loss": 1.2474, + "theoretical_loss": 3.352863117961972, + "tokens_seen": 2957705216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010471834376504575, + "loss": 1.2025, + "theoretical_loss": 3.3528517584914814, + "tokens_seen": 2957836288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010467822179425454, + "loss": 1.2769, + "theoretical_loss": 3.352840399665296, + "tokens_seen": 2957967360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010463809982346332, + "loss": 1.1879, + "theoretical_loss": 3.352829041483351, + "tokens_seen": 2958098432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010459797785267213, + "loss": 1.2062, + "theoretical_loss": 3.3528176839455806, + "tokens_seen": 2958229504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010455785588188092, + "loss": 1.174, + "theoretical_loss": 3.35280632705192, + "tokens_seen": 2958360576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010451773391108973, + "loss": 1.2978, + "theoretical_loss": 3.352794970802304, + "tokens_seen": 2958491648 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001044776119402985, + "loss": 1.2705, + "theoretical_loss": 3.352783615196668, + "tokens_seen": 2958622720 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001044374899695073, + "loss": 1.3032, + "theoretical_loss": 3.3527722602349463, + "tokens_seen": 2958753792 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001043973679987161, + "loss": 1.2517, + "theoretical_loss": 3.3527609059170747, + "tokens_seen": 2958884864 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010435724602792488, + "loss": 1.2611, + "theoretical_loss": 3.3527495522429875, + "tokens_seen": 2959015936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010431712405713369, + "loss": 1.2141, + "theoretical_loss": 3.3527381992126193, + "tokens_seen": 2959147008 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010427700208634248, + "loss": 1.2915, + "theoretical_loss": 3.3527268468259064, + "tokens_seen": 2959278080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010423688011555129, + "loss": 1.2469, + "theoretical_loss": 3.3527154950827827, + "tokens_seen": 2959409152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010419675814476007, + "loss": 1.3218, + "theoretical_loss": 3.3527041439831833, + "tokens_seen": 2959540224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010415663617396886, + "loss": 1.2479, + "theoretical_loss": 3.352692793527044, + "tokens_seen": 2959671296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010411651420317767, + "loss": 1.2336, + "theoretical_loss": 3.352681443714299, + "tokens_seen": 2959802368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010407639223238646, + "loss": 1.3705, + "theoretical_loss": 3.3526700945448837, + "tokens_seen": 2959933440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010403627026159525, + "loss": 1.2725, + "theoretical_loss": 3.352658746018733, + "tokens_seen": 2960064512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010399614829080405, + "loss": 1.2365, + "theoretical_loss": 3.3526473981357823, + "tokens_seen": 2960195584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010395602632001284, + "loss": 1.2637, + "theoretical_loss": 3.352636050895966, + "tokens_seen": 2960326656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010391590434922163, + "loss": 1.2845, + "theoretical_loss": 3.3526247042992203, + "tokens_seen": 2960457728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010387578237843042, + "loss": 1.2186, + "theoretical_loss": 3.3526133583454794, + "tokens_seen": 2960588800 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7459160326454821, + "debugging/entropy-1-grams": 5.28426573657414, + "debugging/length": 471.5, + "debugging/num_segments": 12, + "debugging/score": 0.006897032404886403, + "debugging/score_std": 0.00540563131039379, + "epoch": 0.79, + "objective/train/aligned_token_count": 0.9946767687797546, + "objective/train/avg_token_score": 0.0053232889622449875, + "objective/train/docs_used": 1076444, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2198618650436401, + "objective/train/original_loss": 1.2265443801879883, + "objective/train/theoretical_loss": 3.3526020130346783, + "objective/train/tokens_used": 1331178976, + "objective/train/unlikelihood_loss": 0.017255859449505806, + "theoretical_loss": 3.3526020130346783, + "tokens_seen": 2960719872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010383566040763923, + "loss": 1.2406, + "theoretical_loss": 3.3526020130346783, + "tokens_seen": 2960719872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010379553843684802, + "loss": 1.2528, + "theoretical_loss": 3.3525906683667523, + "tokens_seen": 2960850944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010375541646605682, + "loss": 1.2824, + "theoretical_loss": 3.3525793243416366, + "tokens_seen": 2960982016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010371529449526561, + "loss": 1.2807, + "theoretical_loss": 3.352567980959267, + "tokens_seen": 2961113088 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001036751725244744, + "loss": 1.2719, + "theoretical_loss": 3.3525566382195775, + "tokens_seen": 2961244160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010363505055368321, + "loss": 1.1574, + "theoretical_loss": 3.3525452961225035, + "tokens_seen": 2961375232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010359492858289199, + "loss": 1.3135, + "theoretical_loss": 3.352533954667981, + "tokens_seen": 2961506304 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001035548066121008, + "loss": 1.2134, + "theoretical_loss": 3.352522613855944, + "tokens_seen": 2961637376 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010351468464130959, + "loss": 1.2799, + "theoretical_loss": 3.352511273686328, + "tokens_seen": 2961768448 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010347456267051837, + "loss": 1.2198, + "theoretical_loss": 3.352499934159069, + "tokens_seen": 2961899520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010343444069972717, + "loss": 1.2989, + "theoretical_loss": 3.3524885952741013, + "tokens_seen": 2962030592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010339431872893596, + "loss": 1.3085, + "theoretical_loss": 3.35247725703136, + "tokens_seen": 2962161664 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010335419675814477, + "loss": 1.1665, + "theoretical_loss": 3.3524659194307813, + "tokens_seen": 2962292736 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010331407478735355, + "loss": 1.3324, + "theoretical_loss": 3.3524545824722995, + "tokens_seen": 2962423808 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010327395281656236, + "loss": 1.2194, + "theoretical_loss": 3.3524432461558504, + "tokens_seen": 2962554880 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010323383084577115, + "loss": 1.2539, + "theoretical_loss": 3.3524319104813687, + "tokens_seen": 2962685952 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010319370887497994, + "loss": 1.1718, + "theoretical_loss": 3.35242057544879, + "tokens_seen": 2962817024 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010315358690418873, + "loss": 1.2434, + "theoretical_loss": 3.3524092410580493, + "tokens_seen": 2962948096 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010311346493339753, + "loss": 1.1562, + "theoretical_loss": 3.352397907309082, + "tokens_seen": 2963079168 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010307334296260633, + "loss": 1.1847, + "theoretical_loss": 3.352386574201824, + "tokens_seen": 2963210240 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010303322099181513, + "loss": 1.2396, + "theoretical_loss": 3.352375241736209, + "tokens_seen": 2963341312 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001029930990210239, + "loss": 1.2763, + "theoretical_loss": 3.3523639099121745, + "tokens_seen": 2963472384 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010295297705023271, + "loss": 1.2166, + "theoretical_loss": 3.3523525787296538, + "tokens_seen": 2963603456 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001029128550794415, + "loss": 1.2342, + "theoretical_loss": 3.352341248188583, + "tokens_seen": 2963734528 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001028727331086503, + "loss": 1.2301, + "theoretical_loss": 3.3523299182888975, + "tokens_seen": 2963865600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010283261113785909, + "loss": 1.2319, + "theoretical_loss": 3.3523185890305327, + "tokens_seen": 2963996672 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001027924891670679, + "loss": 1.2438, + "theoretical_loss": 3.352307260413424, + "tokens_seen": 2964127744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010275236719627669, + "loss": 1.2431, + "theoretical_loss": 3.352295932437506, + "tokens_seen": 2964258816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010271224522548547, + "loss": 1.3323, + "theoretical_loss": 3.352284605102715, + "tokens_seen": 2964389888 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010267212325469427, + "loss": 1.206, + "theoretical_loss": 3.352273278408986, + "tokens_seen": 2964520960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010263200128390307, + "loss": 1.2482, + "theoretical_loss": 3.3522619523562542, + "tokens_seen": 2964652032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010259187931311187, + "loss": 1.229, + "theoretical_loss": 3.352250626944455, + "tokens_seen": 2964783104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010255175734232065, + "loss": 1.198, + "theoretical_loss": 3.3522393021735244, + "tokens_seen": 2964914176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010251163537152945, + "loss": 1.2126, + "theoretical_loss": 3.352227978043397, + "tokens_seen": 2965045248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010247151340073825, + "loss": 1.3157, + "theoretical_loss": 3.3522166545540086, + "tokens_seen": 2965176320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010243139142994703, + "loss": 1.1989, + "theoretical_loss": 3.352205331705295, + "tokens_seen": 2965307392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010239126945915584, + "loss": 1.1414, + "theoretical_loss": 3.3521940094971905, + "tokens_seen": 2965438464 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010235114748836463, + "loss": 1.2571, + "theoretical_loss": 3.3521826879296315, + "tokens_seen": 2965569536 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010231102551757344, + "loss": 1.1731, + "theoretical_loss": 3.3521713670025535, + "tokens_seen": 2965700608 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010227090354678222, + "loss": 1.2173, + "theoretical_loss": 3.3521600467158916, + "tokens_seen": 2965831680 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010223078157599101, + "loss": 1.2235, + "theoretical_loss": 3.3521487270695816, + "tokens_seen": 2965962752 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010219065960519981, + "loss": 1.1718, + "theoretical_loss": 3.3521374080635584, + "tokens_seen": 2966093824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010215053763440861, + "loss": 1.2207, + "theoretical_loss": 3.352126089697758, + "tokens_seen": 2966224896 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001021104156636174, + "loss": 1.3372, + "theoretical_loss": 3.3521147719721154, + "tokens_seen": 2966355968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010207029369282619, + "loss": 1.2116, + "theoretical_loss": 3.3521034548865667, + "tokens_seen": 2966487040 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010203017172203499, + "loss": 1.2292, + "theoretical_loss": 3.352092138441047, + "tokens_seen": 2966618112 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010199004975124378, + "loss": 1.2853, + "theoretical_loss": 3.352080822635492, + "tokens_seen": 2966749184 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010194992778045257, + "loss": 1.2695, + "theoretical_loss": 3.3520695074698375, + "tokens_seen": 2966880256 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010190980580966138, + "loss": 1.3035, + "theoretical_loss": 3.3520581929440185, + "tokens_seen": 2967011328 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010186968383887017, + "loss": 1.2704, + "theoretical_loss": 3.352046879057971, + "tokens_seen": 2967142400 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.9933345317840576, + "objective/train/avg_token_score": 0.006665495224297047, + "objective/train/docs_used": 1078969, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.261391520500183, + "objective/train/original_loss": 1.2705096006393433, + "objective/train/theoretical_loss": 3.3520355658116303, + "objective/train/tokens_used": 1337732576, + "objective/train/unlikelihood_loss": 0.020438795909285545, + "theoretical_loss": 3.3520355658116303, + "tokens_seen": 2967273472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010182956186807896, + "loss": 1.2944, + "theoretical_loss": 3.3520355658116303, + "tokens_seen": 2967273472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010178943989728776, + "loss": 1.2722, + "theoretical_loss": 3.352024253204932, + "tokens_seen": 2967404544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010174931792649655, + "loss": 1.2162, + "theoretical_loss": 3.3520129412378115, + "tokens_seen": 2967535616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010170919595570535, + "loss": 1.2249, + "theoretical_loss": 3.352001629910205, + "tokens_seen": 2967666688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010166907398491413, + "loss": 1.1726, + "theoretical_loss": 3.3519903192220477, + "tokens_seen": 2967797760 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010162895201412294, + "loss": 1.2899, + "theoretical_loss": 3.3519790091732755, + "tokens_seen": 2967928832 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010158883004333173, + "loss": 1.2636, + "theoretical_loss": 3.351967699763823, + "tokens_seen": 2968059904 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010154870807254051, + "loss": 1.1836, + "theoretical_loss": 3.3519563909936276, + "tokens_seen": 2968190976 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010150858610174932, + "loss": 1.2261, + "theoretical_loss": 3.351945082862623, + "tokens_seen": 2968322048 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010146846413095811, + "loss": 1.2806, + "theoretical_loss": 3.3519337753707465, + "tokens_seen": 2968453120 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010142834216016692, + "loss": 1.277, + "theoretical_loss": 3.351922468517933, + "tokens_seen": 2968584192 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001013882201893757, + "loss": 1.2404, + "theoretical_loss": 3.351911162304118, + "tokens_seen": 2968715264 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001013480982185845, + "loss": 1.2219, + "theoretical_loss": 3.3518998567292373, + "tokens_seen": 2968846336 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001013079762477933, + "loss": 1.1942, + "theoretical_loss": 3.351888551793227, + "tokens_seen": 2968977408 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010126785427700209, + "loss": 1.2545, + "theoretical_loss": 3.3518772474960223, + "tokens_seen": 2969108480 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010122773230621088, + "loss": 1.2683, + "theoretical_loss": 3.3518659438375593, + "tokens_seen": 2969239552 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010118761033541967, + "loss": 1.2442, + "theoretical_loss": 3.3518546408177734, + "tokens_seen": 2969370624 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010114748836462848, + "loss": 1.2374, + "theoretical_loss": 3.3518433384366, + "tokens_seen": 2969501696 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010110736639383726, + "loss": 1.2431, + "theoretical_loss": 3.351832036693976, + "tokens_seen": 2969632768 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010106724442304605, + "loss": 1.2455, + "theoretical_loss": 3.351820735589836, + "tokens_seen": 2969763840 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010102712245225486, + "loss": 1.2235, + "theoretical_loss": 3.351809435124116, + "tokens_seen": 2969894912 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010098700048146365, + "loss": 1.319, + "theoretical_loss": 3.351798135296752, + "tokens_seen": 2970025984 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010094687851067244, + "loss": 1.2591, + "theoretical_loss": 3.35178683610768, + "tokens_seen": 2970157056 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010090675653988124, + "loss": 1.211, + "theoretical_loss": 3.3517755375568354, + "tokens_seen": 2970288128 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010086663456909004, + "loss": 1.2041, + "theoretical_loss": 3.351764239644154, + "tokens_seen": 2970419200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010082651259829884, + "loss": 1.2312, + "theoretical_loss": 3.3517529423695716, + "tokens_seen": 2970550272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010078639062750762, + "loss": 1.2138, + "theoretical_loss": 3.351741645733024, + "tokens_seen": 2970681344 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010074626865671642, + "loss": 1.2008, + "theoretical_loss": 3.351730349734447, + "tokens_seen": 2970812416 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010070614668592521, + "loss": 1.2495, + "theoretical_loss": 3.3517190543737767, + "tokens_seen": 2970943488 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010066602471513402, + "loss": 1.1586, + "theoretical_loss": 3.3517077596509486, + "tokens_seen": 2971074560 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001006259027443428, + "loss": 1.2349, + "theoretical_loss": 3.3516964655658987, + "tokens_seen": 2971205632 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001005857807735516, + "loss": 1.2825, + "theoretical_loss": 3.351685172118563, + "tokens_seen": 2971336704 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001005456588027604, + "loss": 1.2688, + "theoretical_loss": 3.351673879308877, + "tokens_seen": 2971467776 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010050553683196918, + "loss": 1.3024, + "theoretical_loss": 3.351662587136777, + "tokens_seen": 2971598848 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010046541486117798, + "loss": 1.2219, + "theoretical_loss": 3.3516512956021987, + "tokens_seen": 2971729920 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010042529289038678, + "loss": 1.2359, + "theoretical_loss": 3.351640004705078, + "tokens_seen": 2971860992 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010038517091959558, + "loss": 1.2334, + "theoretical_loss": 3.3516287144453503, + "tokens_seen": 2971992064 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010034504894880436, + "loss": 1.2375, + "theoretical_loss": 3.351617424822952, + "tokens_seen": 2972123136 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010030492697801316, + "loss": 1.1961, + "theoretical_loss": 3.3516061358378195, + "tokens_seen": 2972254208 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010026480500722196, + "loss": 1.3071, + "theoretical_loss": 3.3515948474898876, + "tokens_seen": 2972385280 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010022468303643075, + "loss": 1.1929, + "theoretical_loss": 3.351583559779093, + "tokens_seen": 2972516352 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010018456106563955, + "loss": 1.279, + "theoretical_loss": 3.351572272705372, + "tokens_seen": 2972647424 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010014443909484834, + "loss": 1.2597, + "theoretical_loss": 3.3515609862686597, + "tokens_seen": 2972778496 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010010431712405715, + "loss": 1.303, + "theoretical_loss": 3.351549700468892, + "tokens_seen": 2972909568 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010006419515326593, + "loss": 1.2171, + "theoretical_loss": 3.351538415306006, + "tokens_seen": 2973040640 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010002407318247472, + "loss": 1.1868, + "theoretical_loss": 3.3515271307799366, + "tokens_seen": 2973171712 + }, + { + "epoch": 0.8, + "learning_rate": 9.998395121168352e-05, + "loss": 1.2818, + "theoretical_loss": 3.3515158468906203, + "tokens_seen": 2973302784 + }, + { + "epoch": 0.8, + "learning_rate": 9.994382924089232e-05, + "loss": 1.3333, + "theoretical_loss": 3.3515045636379925, + "tokens_seen": 2973433856 + }, + { + "epoch": 0.8, + "learning_rate": 9.990370727010111e-05, + "loss": 1.2395, + "theoretical_loss": 3.35149328102199, + "tokens_seen": 2973564928 + }, + { + "epoch": 0.8, + "learning_rate": 9.98635852993099e-05, + "loss": 1.2059, + "theoretical_loss": 3.351481999042549, + "tokens_seen": 2973696000 + }, + { + "epoch": 0.8, + "objective/train/aligned_token_count": 0.9927148818969727, + "objective/train/avg_token_score": 0.00728512741625309, + "objective/train/docs_used": 1081215, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2071961164474487, + "objective/train/original_loss": 1.216953992843628, + "objective/train/theoretical_loss": 3.3514707176996046, + "objective/train/tokens_used": 1344286176, + "objective/train/unlikelihood_loss": 0.020546436309814453, + "theoretical_loss": 3.3514707176996046, + "tokens_seen": 2973827072 + }, + { + "epoch": 0.8, + "learning_rate": 9.98234633285187e-05, + "loss": 1.2932, + "theoretical_loss": 3.3514707176996046, + "tokens_seen": 2973827072 + }, + { + "epoch": 0.8, + "learning_rate": 9.97833413577275e-05, + "loss": 1.1841, + "theoretical_loss": 3.351459436993093, + "tokens_seen": 2973958144 + }, + { + "epoch": 0.8, + "learning_rate": 9.974321938693628e-05, + "loss": 1.2391, + "theoretical_loss": 3.3514481569229506, + "tokens_seen": 2974089216 + }, + { + "epoch": 0.8, + "learning_rate": 9.970309741614509e-05, + "loss": 1.2703, + "theoretical_loss": 3.351436877489114, + "tokens_seen": 2974220288 + }, + { + "epoch": 0.8, + "learning_rate": 9.966297544535388e-05, + "loss": 1.2664, + "theoretical_loss": 3.351425598691518, + "tokens_seen": 2974351360 + }, + { + "epoch": 0.8, + "learning_rate": 9.962285347456267e-05, + "loss": 1.2076, + "theoretical_loss": 3.3514143205300995, + "tokens_seen": 2974482432 + }, + { + "epoch": 0.8, + "learning_rate": 9.958273150377147e-05, + "loss": 1.2599, + "theoretical_loss": 3.3514030430047943, + "tokens_seen": 2974613504 + }, + { + "epoch": 0.8, + "learning_rate": 9.954260953298026e-05, + "loss": 1.2497, + "theoretical_loss": 3.3513917661155386, + "tokens_seen": 2974744576 + }, + { + "epoch": 0.8, + "learning_rate": 9.950248756218906e-05, + "loss": 1.2134, + "theoretical_loss": 3.3513804898622688, + "tokens_seen": 2974875648 + }, + { + "epoch": 0.8, + "learning_rate": 9.946236559139784e-05, + "loss": 1.2104, + "theoretical_loss": 3.3513692142449205, + "tokens_seen": 2975006720 + }, + { + "epoch": 0.8, + "learning_rate": 9.942224362060665e-05, + "loss": 1.3213, + "theoretical_loss": 3.35135793926343, + "tokens_seen": 2975137792 + }, + { + "epoch": 0.8, + "learning_rate": 9.938212164981544e-05, + "loss": 1.3279, + "theoretical_loss": 3.3513466649177337, + "tokens_seen": 2975268864 + }, + { + "epoch": 0.8, + "learning_rate": 9.934199967902424e-05, + "loss": 1.1493, + "theoretical_loss": 3.3513353912077672, + "tokens_seen": 2975399936 + }, + { + "epoch": 0.8, + "learning_rate": 9.930187770823303e-05, + "loss": 1.2334, + "theoretical_loss": 3.351324118133467, + "tokens_seen": 2975531008 + }, + { + "epoch": 0.8, + "learning_rate": 9.926175573744182e-05, + "loss": 1.2515, + "theoretical_loss": 3.3513128456947694, + "tokens_seen": 2975662080 + }, + { + "epoch": 0.8, + "learning_rate": 9.922163376665063e-05, + "loss": 1.2548, + "theoretical_loss": 3.35130157389161, + "tokens_seen": 2975793152 + }, + { + "epoch": 0.8, + "learning_rate": 9.918151179585941e-05, + "loss": 1.3009, + "theoretical_loss": 3.351290302723926, + "tokens_seen": 2975924224 + }, + { + "epoch": 0.8, + "learning_rate": 9.914138982506821e-05, + "loss": 1.2894, + "theoretical_loss": 3.3512790321916524, + "tokens_seen": 2976055296 + }, + { + "epoch": 0.8, + "learning_rate": 9.9101267854277e-05, + "loss": 1.2825, + "theoretical_loss": 3.3512677622947264, + "tokens_seen": 2976186368 + }, + { + "epoch": 0.8, + "learning_rate": 9.90611458834858e-05, + "loss": 1.1854, + "theoretical_loss": 3.3512564930330835, + "tokens_seen": 2976317440 + }, + { + "epoch": 0.8, + "learning_rate": 9.902102391269459e-05, + "loss": 1.2558, + "theoretical_loss": 3.35124522440666, + "tokens_seen": 2976448512 + }, + { + "epoch": 0.8, + "learning_rate": 9.898090194190338e-05, + "loss": 1.2743, + "theoretical_loss": 3.3512339564153923, + "tokens_seen": 2976579584 + }, + { + "epoch": 0.8, + "learning_rate": 9.894077997111219e-05, + "loss": 1.2721, + "theoretical_loss": 3.3512226890592167, + "tokens_seen": 2976710656 + }, + { + "epoch": 0.8, + "learning_rate": 9.890065800032098e-05, + "loss": 1.2536, + "theoretical_loss": 3.3512114223380696, + "tokens_seen": 2976841728 + }, + { + "epoch": 0.8, + "learning_rate": 9.886053602952976e-05, + "loss": 1.2066, + "theoretical_loss": 3.3512001562518874, + "tokens_seen": 2976972800 + }, + { + "epoch": 0.8, + "learning_rate": 9.882041405873857e-05, + "loss": 1.2371, + "theoretical_loss": 3.3511888908006053, + "tokens_seen": 2977103872 + }, + { + "epoch": 0.8, + "learning_rate": 9.878029208794736e-05, + "loss": 1.1695, + "theoretical_loss": 3.3511776259841604, + "tokens_seen": 2977234944 + }, + { + "epoch": 0.8, + "learning_rate": 9.874017011715615e-05, + "loss": 1.2765, + "theoretical_loss": 3.3511663618024885, + "tokens_seen": 2977366016 + }, + { + "epoch": 0.8, + "learning_rate": 9.870004814636495e-05, + "loss": 1.2146, + "theoretical_loss": 3.3511550982555267, + "tokens_seen": 2977497088 + }, + { + "epoch": 0.8, + "learning_rate": 9.865992617557375e-05, + "loss": 1.2788, + "theoretical_loss": 3.3511438353432106, + "tokens_seen": 2977628160 + }, + { + "epoch": 0.8, + "learning_rate": 9.861980420478255e-05, + "loss": 1.2358, + "theoretical_loss": 3.351132573065477, + "tokens_seen": 2977759232 + }, + { + "epoch": 0.8, + "learning_rate": 9.857968223399133e-05, + "loss": 1.223, + "theoretical_loss": 3.3511213114222618, + "tokens_seen": 2977890304 + }, + { + "epoch": 0.8, + "learning_rate": 9.853956026320013e-05, + "loss": 1.1989, + "theoretical_loss": 3.351110050413501, + "tokens_seen": 2978021376 + }, + { + "epoch": 0.8, + "learning_rate": 9.849943829240892e-05, + "loss": 1.27, + "theoretical_loss": 3.3510987900391322, + "tokens_seen": 2978152448 + }, + { + "epoch": 0.81, + "learning_rate": 9.845931632161773e-05, + "loss": 1.2506, + "theoretical_loss": 3.3510875302990906, + "tokens_seen": 2978283520 + }, + { + "epoch": 0.81, + "learning_rate": 9.841919435082651e-05, + "loss": 1.2255, + "theoretical_loss": 3.3510762711933126, + "tokens_seen": 2978414592 + }, + { + "epoch": 0.81, + "learning_rate": 9.83790723800353e-05, + "loss": 1.2691, + "theoretical_loss": 3.3510650127217354, + "tokens_seen": 2978545664 + }, + { + "epoch": 0.81, + "learning_rate": 9.833895040924411e-05, + "loss": 1.2535, + "theoretical_loss": 3.3510537548842945, + "tokens_seen": 2978676736 + }, + { + "epoch": 0.81, + "learning_rate": 9.82988284384529e-05, + "loss": 1.232, + "theoretical_loss": 3.3510424976809263, + "tokens_seen": 2978807808 + }, + { + "epoch": 0.81, + "learning_rate": 9.82587064676617e-05, + "loss": 1.173, + "theoretical_loss": 3.351031241111568, + "tokens_seen": 2978938880 + }, + { + "epoch": 0.81, + "learning_rate": 9.821858449687049e-05, + "loss": 1.2722, + "theoretical_loss": 3.3510199851761553, + "tokens_seen": 2979069952 + }, + { + "epoch": 0.81, + "learning_rate": 9.81784625260793e-05, + "loss": 1.2385, + "theoretical_loss": 3.351008729874625, + "tokens_seen": 2979201024 + }, + { + "epoch": 0.81, + "learning_rate": 9.813834055528807e-05, + "loss": 1.2965, + "theoretical_loss": 3.350997475206913, + "tokens_seen": 2979332096 + }, + { + "epoch": 0.81, + "learning_rate": 9.809821858449687e-05, + "loss": 1.2404, + "theoretical_loss": 3.350986221172956, + "tokens_seen": 2979463168 + }, + { + "epoch": 0.81, + "learning_rate": 9.805809661370567e-05, + "loss": 1.3037, + "theoretical_loss": 3.3509749677726908, + "tokens_seen": 2979594240 + }, + { + "epoch": 0.81, + "learning_rate": 9.801797464291446e-05, + "loss": 1.2587, + "theoretical_loss": 3.3509637150060536, + "tokens_seen": 2979725312 + }, + { + "epoch": 0.81, + "learning_rate": 9.797785267212326e-05, + "loss": 1.2963, + "theoretical_loss": 3.3509524628729803, + "tokens_seen": 2979856384 + }, + { + "epoch": 0.81, + "learning_rate": 9.793773070133205e-05, + "loss": 1.2816, + "theoretical_loss": 3.3509412113734083, + "tokens_seen": 2979987456 + }, + { + "epoch": 0.81, + "learning_rate": 9.789760873054084e-05, + "loss": 1.2869, + "theoretical_loss": 3.3509299605072735, + "tokens_seen": 2980118528 + }, + { + "epoch": 0.81, + "learning_rate": 9.785748675974965e-05, + "loss": 1.2957, + "theoretical_loss": 3.350918710274512, + "tokens_seen": 2980249600 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.9895803332328796, + "objective/train/avg_token_score": 0.010419688187539577, + "objective/train/docs_used": 1083727, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2724065780639648, + "objective/train/original_loss": 1.2870585918426514, + "objective/train/theoretical_loss": 3.3509074606750615, + "objective/train/tokens_used": 1350839776, + "objective/train/unlikelihood_loss": 0.027490951120853424, + "theoretical_loss": 3.3509074606750615, + "tokens_seen": 2980380672 + }, + { + "epoch": 0.81, + "learning_rate": 9.781736478895843e-05, + "loss": 1.2853, + "theoretical_loss": 3.3509074606750615, + "tokens_seen": 2980380672 + }, + { + "epoch": 0.81, + "learning_rate": 9.777724281816723e-05, + "loss": 1.2351, + "theoretical_loss": 3.3508962117088577, + "tokens_seen": 2980511744 + }, + { + "epoch": 0.81, + "learning_rate": 9.773712084737603e-05, + "loss": 1.2903, + "theoretical_loss": 3.350884963375837, + "tokens_seen": 2980642816 + }, + { + "epoch": 0.81, + "learning_rate": 9.769699887658482e-05, + "loss": 1.3048, + "theoretical_loss": 3.3508737156759363, + "tokens_seen": 2980773888 + }, + { + "epoch": 0.81, + "learning_rate": 9.765687690579361e-05, + "loss": 1.2007, + "theoretical_loss": 3.3508624686090913, + "tokens_seen": 2980904960 + }, + { + "epoch": 0.81, + "learning_rate": 9.76167549350024e-05, + "loss": 1.1954, + "theoretical_loss": 3.3508512221752396, + "tokens_seen": 2981036032 + }, + { + "epoch": 0.81, + "learning_rate": 9.757663296421121e-05, + "loss": 1.1813, + "theoretical_loss": 3.3508399763743175, + "tokens_seen": 2981167104 + }, + { + "epoch": 0.81, + "learning_rate": 9.753651099341999e-05, + "loss": 1.2744, + "theoretical_loss": 3.3508287312062612, + "tokens_seen": 2981298176 + }, + { + "epoch": 0.81, + "learning_rate": 9.74963890226288e-05, + "loss": 1.287, + "theoretical_loss": 3.3508174866710077, + "tokens_seen": 2981429248 + }, + { + "epoch": 0.81, + "learning_rate": 9.745626705183759e-05, + "loss": 1.2843, + "theoretical_loss": 3.350806242768493, + "tokens_seen": 2981560320 + }, + { + "epoch": 0.81, + "learning_rate": 9.741614508104638e-05, + "loss": 1.2323, + "theoretical_loss": 3.350794999498654, + "tokens_seen": 2981691392 + }, + { + "epoch": 0.81, + "learning_rate": 9.737602311025518e-05, + "loss": 1.2806, + "theoretical_loss": 3.350783756861427, + "tokens_seen": 2981822464 + }, + { + "epoch": 0.81, + "learning_rate": 9.733590113946397e-05, + "loss": 1.224, + "theoretical_loss": 3.3507725148567493, + "tokens_seen": 2981953536 + }, + { + "epoch": 0.81, + "learning_rate": 9.729577916867277e-05, + "loss": 1.3309, + "theoretical_loss": 3.350761273484557, + "tokens_seen": 2982084608 + }, + { + "epoch": 0.81, + "learning_rate": 9.725565719788155e-05, + "loss": 1.2992, + "theoretical_loss": 3.350750032744787, + "tokens_seen": 2982215680 + }, + { + "epoch": 0.81, + "learning_rate": 9.721553522709036e-05, + "loss": 1.2533, + "theoretical_loss": 3.3507387926373755, + "tokens_seen": 2982346752 + }, + { + "epoch": 0.81, + "learning_rate": 9.717541325629915e-05, + "loss": 1.1759, + "theoretical_loss": 3.3507275531622596, + "tokens_seen": 2982477824 + }, + { + "epoch": 0.81, + "learning_rate": 9.713529128550795e-05, + "loss": 1.2375, + "theoretical_loss": 3.350716314319375, + "tokens_seen": 2982608896 + }, + { + "epoch": 0.81, + "learning_rate": 9.709516931471674e-05, + "loss": 1.3156, + "theoretical_loss": 3.3507050761086603, + "tokens_seen": 2982739968 + }, + { + "epoch": 0.81, + "learning_rate": 9.705504734392553e-05, + "loss": 1.2261, + "theoretical_loss": 3.35069383853005, + "tokens_seen": 2982871040 + }, + { + "epoch": 0.81, + "learning_rate": 9.701492537313434e-05, + "loss": 1.2146, + "theoretical_loss": 3.350682601583482, + "tokens_seen": 2983002112 + }, + { + "epoch": 0.81, + "learning_rate": 9.697480340234313e-05, + "loss": 1.2359, + "theoretical_loss": 3.3506713652688926, + "tokens_seen": 2983133184 + }, + { + "epoch": 0.81, + "learning_rate": 9.693468143155191e-05, + "loss": 1.21, + "theoretical_loss": 3.3506601295862186, + "tokens_seen": 2983264256 + }, + { + "epoch": 0.81, + "learning_rate": 9.689455946076072e-05, + "loss": 1.217, + "theoretical_loss": 3.350648894535397, + "tokens_seen": 2983395328 + }, + { + "epoch": 0.81, + "learning_rate": 9.685443748996951e-05, + "loss": 1.1705, + "theoretical_loss": 3.350637660116364, + "tokens_seen": 2983526400 + }, + { + "epoch": 0.81, + "learning_rate": 9.68143155191783e-05, + "loss": 1.2742, + "theoretical_loss": 3.3506264263290566, + "tokens_seen": 2983657472 + }, + { + "epoch": 0.81, + "learning_rate": 9.67741935483871e-05, + "loss": 1.2114, + "theoretical_loss": 3.350615193173411, + "tokens_seen": 2983788544 + }, + { + "epoch": 0.81, + "learning_rate": 9.67340715775959e-05, + "loss": 1.2942, + "theoretical_loss": 3.350603960649365, + "tokens_seen": 2983919616 + }, + { + "epoch": 0.81, + "learning_rate": 9.669394960680469e-05, + "loss": 1.1918, + "theoretical_loss": 3.350592728756854, + "tokens_seen": 2984050688 + }, + { + "epoch": 0.81, + "learning_rate": 9.665382763601347e-05, + "loss": 1.2324, + "theoretical_loss": 3.350581497495816, + "tokens_seen": 2984181760 + }, + { + "epoch": 0.81, + "learning_rate": 9.661370566522228e-05, + "loss": 1.2174, + "theoretical_loss": 3.350570266866187, + "tokens_seen": 2984312832 + }, + { + "epoch": 0.81, + "learning_rate": 9.657358369443107e-05, + "loss": 1.2475, + "theoretical_loss": 3.3505590368679044, + "tokens_seen": 2984443904 + }, + { + "epoch": 0.81, + "learning_rate": 9.653346172363988e-05, + "loss": 1.2486, + "theoretical_loss": 3.3505478075009045, + "tokens_seen": 2984574976 + }, + { + "epoch": 0.81, + "learning_rate": 9.649333975284866e-05, + "loss": 1.2654, + "theoretical_loss": 3.350536578765124, + "tokens_seen": 2984706048 + }, + { + "epoch": 0.81, + "learning_rate": 9.645321778205746e-05, + "loss": 1.1937, + "theoretical_loss": 3.3505253506605, + "tokens_seen": 2984837120 + }, + { + "epoch": 0.81, + "learning_rate": 9.641309581126626e-05, + "loss": 1.2645, + "theoretical_loss": 3.350514123186969, + "tokens_seen": 2984968192 + }, + { + "epoch": 0.81, + "learning_rate": 9.637297384047504e-05, + "loss": 1.1462, + "theoretical_loss": 3.3505028963444685, + "tokens_seen": 2985099264 + }, + { + "epoch": 0.81, + "learning_rate": 9.633285186968384e-05, + "loss": 1.2105, + "theoretical_loss": 3.3504916701329344, + "tokens_seen": 2985230336 + }, + { + "epoch": 0.81, + "learning_rate": 9.629272989889263e-05, + "loss": 1.2344, + "theoretical_loss": 3.350480444552304, + "tokens_seen": 2985361408 + }, + { + "epoch": 0.81, + "learning_rate": 9.625260792810144e-05, + "loss": 1.266, + "theoretical_loss": 3.3504692196025143, + "tokens_seen": 2985492480 + }, + { + "epoch": 0.81, + "learning_rate": 9.621248595731022e-05, + "loss": 1.2325, + "theoretical_loss": 3.3504579952835023, + "tokens_seen": 2985623552 + }, + { + "epoch": 0.81, + "learning_rate": 9.617236398651901e-05, + "loss": 1.1459, + "theoretical_loss": 3.350446771595204, + "tokens_seen": 2985754624 + }, + { + "epoch": 0.81, + "learning_rate": 9.613224201572782e-05, + "loss": 1.2805, + "theoretical_loss": 3.350435548537557, + "tokens_seen": 2985885696 + }, + { + "epoch": 0.81, + "learning_rate": 9.609212004493661e-05, + "loss": 1.222, + "theoretical_loss": 3.350424326110498, + "tokens_seen": 2986016768 + }, + { + "epoch": 0.81, + "learning_rate": 9.60519980741454e-05, + "loss": 1.1951, + "theoretical_loss": 3.350413104313964, + "tokens_seen": 2986147840 + }, + { + "epoch": 0.81, + "learning_rate": 9.60118761033542e-05, + "loss": 1.2719, + "theoretical_loss": 3.350401883147892, + "tokens_seen": 2986278912 + }, + { + "epoch": 0.81, + "learning_rate": 9.5971754132563e-05, + "loss": 1.2298, + "theoretical_loss": 3.3503906626122184, + "tokens_seen": 2986409984 + }, + { + "epoch": 0.81, + "learning_rate": 9.593163216177178e-05, + "loss": 1.1825, + "theoretical_loss": 3.3503794427068803, + "tokens_seen": 2986541056 + }, + { + "epoch": 0.81, + "learning_rate": 9.589151019098058e-05, + "loss": 1.2811, + "theoretical_loss": 3.3503682234318153, + "tokens_seen": 2986672128 + }, + { + "epoch": 0.81, + "learning_rate": 9.585138822018938e-05, + "loss": 1.2364, + "theoretical_loss": 3.3503570047869595, + "tokens_seen": 2986803200 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.9937630891799927, + "objective/train/avg_token_score": 0.00623689778149128, + "objective/train/docs_used": 1086010, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3088970184326172, + "objective/train/original_loss": 1.3172991275787354, + "objective/train/theoretical_loss": 3.35034578677225, + "objective/train/tokens_used": 1357393376, + "objective/train/unlikelihood_loss": 0.01829712651669979, + "theoretical_loss": 3.35034578677225, + "tokens_seen": 2986934272 + }, + { + "epoch": 0.81, + "learning_rate": 9.581126624939817e-05, + "loss": 1.2699, + "theoretical_loss": 3.35034578677225, + "tokens_seen": 2986934272 + }, + { + "epoch": 0.81, + "learning_rate": 9.577114427860697e-05, + "loss": 1.2218, + "theoretical_loss": 3.3503345693876243, + "tokens_seen": 2987065344 + }, + { + "epoch": 0.81, + "learning_rate": 9.573102230781576e-05, + "loss": 1.31, + "theoretical_loss": 3.3503233526330183, + "tokens_seen": 2987196416 + }, + { + "epoch": 0.81, + "learning_rate": 9.569090033702455e-05, + "loss": 1.2455, + "theoretical_loss": 3.35031213650837, + "tokens_seen": 2987327488 + }, + { + "epoch": 0.81, + "learning_rate": 9.565077836623336e-05, + "loss": 1.2053, + "theoretical_loss": 3.3503009210136163, + "tokens_seen": 2987458560 + }, + { + "epoch": 0.81, + "learning_rate": 9.561065639544214e-05, + "loss": 1.2871, + "theoretical_loss": 3.3502897061486934, + "tokens_seen": 2987589632 + }, + { + "epoch": 0.81, + "learning_rate": 9.557053442465094e-05, + "loss": 1.2436, + "theoretical_loss": 3.3502784919135395, + "tokens_seen": 2987720704 + }, + { + "epoch": 0.81, + "learning_rate": 9.553041245385974e-05, + "loss": 1.2613, + "theoretical_loss": 3.3502672783080905, + "tokens_seen": 2987851776 + }, + { + "epoch": 0.81, + "learning_rate": 9.549029048306854e-05, + "loss": 1.2225, + "theoretical_loss": 3.3502560653322835, + "tokens_seen": 2987982848 + }, + { + "epoch": 0.81, + "learning_rate": 9.545016851227732e-05, + "loss": 1.232, + "theoretical_loss": 3.3502448529860565, + "tokens_seen": 2988113920 + }, + { + "epoch": 0.81, + "learning_rate": 9.541004654148612e-05, + "loss": 1.2039, + "theoretical_loss": 3.3502336412693454, + "tokens_seen": 2988244992 + }, + { + "epoch": 0.81, + "learning_rate": 9.536992457069492e-05, + "loss": 1.2939, + "theoretical_loss": 3.350222430182088, + "tokens_seen": 2988376064 + }, + { + "epoch": 0.81, + "learning_rate": 9.53298025999037e-05, + "loss": 1.1479, + "theoretical_loss": 3.3502112197242213, + "tokens_seen": 2988507136 + }, + { + "epoch": 0.81, + "learning_rate": 9.528968062911251e-05, + "loss": 1.257, + "theoretical_loss": 3.3502000098956817, + "tokens_seen": 2988638208 + }, + { + "epoch": 0.81, + "learning_rate": 9.52495586583213e-05, + "loss": 1.2233, + "theoretical_loss": 3.350188800696407, + "tokens_seen": 2988769280 + }, + { + "epoch": 0.81, + "learning_rate": 9.520943668753009e-05, + "loss": 1.2639, + "theoretical_loss": 3.350177592126334, + "tokens_seen": 2988900352 + }, + { + "epoch": 0.81, + "learning_rate": 9.516931471673889e-05, + "loss": 1.1834, + "theoretical_loss": 3.3501663841854, + "tokens_seen": 2989031424 + }, + { + "epoch": 0.81, + "learning_rate": 9.512919274594768e-05, + "loss": 1.2155, + "theoretical_loss": 3.3501551768735416, + "tokens_seen": 2989162496 + }, + { + "epoch": 0.81, + "learning_rate": 9.508907077515648e-05, + "loss": 1.241, + "theoretical_loss": 3.3501439701906968, + "tokens_seen": 2989293568 + }, + { + "epoch": 0.81, + "learning_rate": 9.504894880436528e-05, + "loss": 1.319, + "theoretical_loss": 3.3501327641368013, + "tokens_seen": 2989424640 + }, + { + "epoch": 0.81, + "learning_rate": 9.500882683357407e-05, + "loss": 1.1701, + "theoretical_loss": 3.350121558711794, + "tokens_seen": 2989555712 + }, + { + "epoch": 0.81, + "learning_rate": 9.496870486278286e-05, + "loss": 1.2624, + "theoretical_loss": 3.3501103539156105, + "tokens_seen": 2989686784 + }, + { + "epoch": 0.81, + "learning_rate": 9.492858289199166e-05, + "loss": 1.2695, + "theoretical_loss": 3.3500991497481887, + "tokens_seen": 2989817856 + }, + { + "epoch": 0.81, + "learning_rate": 9.488846092120045e-05, + "loss": 1.2048, + "theoretical_loss": 3.3500879462094657, + "tokens_seen": 2989948928 + }, + { + "epoch": 0.81, + "learning_rate": 9.484833895040924e-05, + "loss": 1.2249, + "theoretical_loss": 3.350076743299378, + "tokens_seen": 2990080000 + }, + { + "epoch": 0.81, + "learning_rate": 9.480821697961805e-05, + "loss": 1.3065, + "theoretical_loss": 3.350065541017864, + "tokens_seen": 2990211072 + }, + { + "epoch": 0.81, + "learning_rate": 9.476809500882684e-05, + "loss": 1.2304, + "theoretical_loss": 3.35005433936486, + "tokens_seen": 2990342144 + }, + { + "epoch": 0.81, + "learning_rate": 9.472797303803562e-05, + "loss": 1.2554, + "theoretical_loss": 3.3500431383403035, + "tokens_seen": 2990473216 + }, + { + "epoch": 0.81, + "learning_rate": 9.468785106724443e-05, + "loss": 1.1591, + "theoretical_loss": 3.3500319379441317, + "tokens_seen": 2990604288 + }, + { + "epoch": 0.81, + "learning_rate": 9.464772909645322e-05, + "loss": 1.2329, + "theoretical_loss": 3.3500207381762817, + "tokens_seen": 2990735360 + }, + { + "epoch": 0.81, + "learning_rate": 9.460760712566202e-05, + "loss": 1.3485, + "theoretical_loss": 3.3500095390366904, + "tokens_seen": 2990866432 + }, + { + "epoch": 0.81, + "learning_rate": 9.45674851548708e-05, + "loss": 1.2351, + "theoretical_loss": 3.3499983405252958, + "tokens_seen": 2990997504 + }, + { + "epoch": 0.81, + "learning_rate": 9.452736318407961e-05, + "loss": 1.2312, + "theoretical_loss": 3.3499871426420347, + "tokens_seen": 2991128576 + }, + { + "epoch": 0.81, + "learning_rate": 9.44872412132884e-05, + "loss": 1.1127, + "theoretical_loss": 3.349975945386844, + "tokens_seen": 2991259648 + }, + { + "epoch": 0.81, + "learning_rate": 9.444711924249718e-05, + "loss": 1.2183, + "theoretical_loss": 3.3499647487596613, + "tokens_seen": 2991390720 + }, + { + "epoch": 0.81, + "learning_rate": 9.440699727170599e-05, + "loss": 1.2449, + "theoretical_loss": 3.3499535527604243, + "tokens_seen": 2991521792 + }, + { + "epoch": 0.81, + "learning_rate": 9.436687530091478e-05, + "loss": 1.181, + "theoretical_loss": 3.3499423573890694, + "tokens_seen": 2991652864 + }, + { + "epoch": 0.81, + "learning_rate": 9.432675333012359e-05, + "loss": 1.3016, + "theoretical_loss": 3.3499311626455346, + "tokens_seen": 2991783936 + }, + { + "epoch": 0.81, + "learning_rate": 9.428663135933237e-05, + "loss": 1.2431, + "theoretical_loss": 3.3499199685297567, + "tokens_seen": 2991915008 + }, + { + "epoch": 0.81, + "learning_rate": 9.424650938854116e-05, + "loss": 1.2094, + "theoretical_loss": 3.3499087750416736, + "tokens_seen": 2992046080 + }, + { + "epoch": 0.81, + "learning_rate": 9.420638741774997e-05, + "loss": 1.174, + "theoretical_loss": 3.349897582181222, + "tokens_seen": 2992177152 + }, + { + "epoch": 0.81, + "learning_rate": 9.416626544695876e-05, + "loss": 1.2157, + "theoretical_loss": 3.349886389948339, + "tokens_seen": 2992308224 + }, + { + "epoch": 0.81, + "learning_rate": 9.412614347616755e-05, + "loss": 1.2114, + "theoretical_loss": 3.3498751983429633, + "tokens_seen": 2992439296 + }, + { + "epoch": 0.81, + "learning_rate": 9.408602150537634e-05, + "loss": 1.155, + "theoretical_loss": 3.3498640073650305, + "tokens_seen": 2992570368 + }, + { + "epoch": 0.81, + "learning_rate": 9.404589953458515e-05, + "loss": 1.2226, + "theoretical_loss": 3.349852817014479, + "tokens_seen": 2992701440 + }, + { + "epoch": 0.81, + "learning_rate": 9.400577756379393e-05, + "loss": 1.22, + "theoretical_loss": 3.349841627291246, + "tokens_seen": 2992832512 + }, + { + "epoch": 0.81, + "learning_rate": 9.396565559300272e-05, + "loss": 1.1493, + "theoretical_loss": 3.3498304381952684, + "tokens_seen": 2992963584 + }, + { + "epoch": 0.81, + "learning_rate": 9.392553362221153e-05, + "loss": 1.2602, + "theoretical_loss": 3.3498192497264845, + "tokens_seen": 2993094656 + }, + { + "epoch": 0.81, + "learning_rate": 9.388541165142032e-05, + "loss": 1.2024, + "theoretical_loss": 3.3498080618848305, + "tokens_seen": 2993225728 + }, + { + "epoch": 0.81, + "learning_rate": 9.384528968062911e-05, + "loss": 1.1887, + "theoretical_loss": 3.349796874670245, + "tokens_seen": 2993356800 + }, + { + "epoch": 0.81, + "objective/train/aligned_token_count": 0.9904163479804993, + "objective/train/avg_token_score": 0.00958362314850092, + "objective/train/docs_used": 1088408, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.267090082168579, + "objective/train/original_loss": 1.2797080278396606, + "objective/train/theoretical_loss": 3.3497856880826644, + "objective/train/tokens_used": 1363946976, + "objective/train/unlikelihood_loss": 0.029741233214735985, + "theoretical_loss": 3.3497856880826644, + "tokens_seen": 2993487872 + }, + { + "epoch": 0.81, + "learning_rate": 9.380516770983791e-05, + "loss": 1.346, + "theoretical_loss": 3.3497856880826644, + "tokens_seen": 2993487872 + }, + { + "epoch": 0.81, + "learning_rate": 9.37650457390467e-05, + "loss": 1.2935, + "theoretical_loss": 3.3497745021220267, + "tokens_seen": 2993618944 + }, + { + "epoch": 0.81, + "learning_rate": 9.37249237682555e-05, + "loss": 1.306, + "theoretical_loss": 3.3497633167882688, + "tokens_seen": 2993750016 + }, + { + "epoch": 0.81, + "learning_rate": 9.368480179746429e-05, + "loss": 1.2286, + "theoretical_loss": 3.3497521320813286, + "tokens_seen": 2993881088 + }, + { + "epoch": 0.81, + "learning_rate": 9.364467982667309e-05, + "loss": 1.2856, + "theoretical_loss": 3.3497409480011435, + "tokens_seen": 2994012160 + }, + { + "epoch": 0.81, + "learning_rate": 9.360455785588188e-05, + "loss": 1.2292, + "theoretical_loss": 3.3497297645476505, + "tokens_seen": 2994143232 + }, + { + "epoch": 0.81, + "learning_rate": 9.356443588509068e-05, + "loss": 1.3261, + "theoretical_loss": 3.3497185817207877, + "tokens_seen": 2994274304 + }, + { + "epoch": 0.81, + "learning_rate": 9.352431391429947e-05, + "loss": 1.1445, + "theoretical_loss": 3.3497073995204922, + "tokens_seen": 2994405376 + }, + { + "epoch": 0.81, + "learning_rate": 9.348419194350826e-05, + "loss": 1.2699, + "theoretical_loss": 3.3496962179467014, + "tokens_seen": 2994536448 + }, + { + "epoch": 0.81, + "learning_rate": 9.344406997271707e-05, + "loss": 1.2162, + "theoretical_loss": 3.349685036999353, + "tokens_seen": 2994667520 + }, + { + "epoch": 0.82, + "learning_rate": 9.340394800192585e-05, + "loss": 1.2867, + "theoretical_loss": 3.3496738566783844, + "tokens_seen": 2994798592 + }, + { + "epoch": 0.82, + "learning_rate": 9.336382603113465e-05, + "loss": 1.2565, + "theoretical_loss": 3.3496626769837325, + "tokens_seen": 2994929664 + }, + { + "epoch": 0.82, + "learning_rate": 9.332370406034345e-05, + "loss": 1.3458, + "theoretical_loss": 3.3496514979153362, + "tokens_seen": 2995060736 + }, + { + "epoch": 0.82, + "learning_rate": 9.328358208955224e-05, + "loss": 1.2707, + "theoretical_loss": 3.3496403194731315, + "tokens_seen": 2995191808 + }, + { + "epoch": 0.82, + "learning_rate": 9.324346011876103e-05, + "loss": 1.313, + "theoretical_loss": 3.349629141657057, + "tokens_seen": 2995322880 + }, + { + "epoch": 0.82, + "learning_rate": 9.320333814796983e-05, + "loss": 1.2171, + "theoretical_loss": 3.3496179644670496, + "tokens_seen": 2995453952 + }, + { + "epoch": 0.82, + "learning_rate": 9.316321617717863e-05, + "loss": 1.2563, + "theoretical_loss": 3.3496067879030473, + "tokens_seen": 2995585024 + }, + { + "epoch": 0.82, + "learning_rate": 9.312309420638742e-05, + "loss": 1.2274, + "theoretical_loss": 3.3495956119649875, + "tokens_seen": 2995716096 + }, + { + "epoch": 0.82, + "learning_rate": 9.308297223559622e-05, + "loss": 1.2791, + "theoretical_loss": 3.3495844366528074, + "tokens_seen": 2995847168 + }, + { + "epoch": 0.82, + "learning_rate": 9.304285026480501e-05, + "loss": 1.2203, + "theoretical_loss": 3.3495732619664453, + "tokens_seen": 2995978240 + }, + { + "epoch": 0.82, + "learning_rate": 9.30027282940138e-05, + "loss": 1.2407, + "theoretical_loss": 3.349562087905838, + "tokens_seen": 2996109312 + }, + { + "epoch": 0.82, + "learning_rate": 9.29626063232226e-05, + "loss": 1.1937, + "theoretical_loss": 3.3495509144709232, + "tokens_seen": 2996240384 + }, + { + "epoch": 0.82, + "learning_rate": 9.292248435243139e-05, + "loss": 1.1874, + "theoretical_loss": 3.3495397416616393, + "tokens_seen": 2996371456 + }, + { + "epoch": 0.82, + "learning_rate": 9.28823623816402e-05, + "loss": 1.2448, + "theoretical_loss": 3.349528569477923, + "tokens_seen": 2996502528 + }, + { + "epoch": 0.82, + "learning_rate": 9.284224041084899e-05, + "loss": 1.2872, + "theoretical_loss": 3.349517397919712, + "tokens_seen": 2996633600 + }, + { + "epoch": 0.82, + "learning_rate": 9.280211844005777e-05, + "loss": 1.2341, + "theoretical_loss": 3.3495062269869447, + "tokens_seen": 2996764672 + }, + { + "epoch": 0.82, + "learning_rate": 9.276199646926657e-05, + "loss": 1.3217, + "theoretical_loss": 3.3494950566795576, + "tokens_seen": 2996895744 + }, + { + "epoch": 0.82, + "learning_rate": 9.272187449847537e-05, + "loss": 1.2, + "theoretical_loss": 3.3494838869974894, + "tokens_seen": 2997026816 + }, + { + "epoch": 0.82, + "learning_rate": 9.268175252768417e-05, + "loss": 1.2094, + "theoretical_loss": 3.349472717940677, + "tokens_seen": 2997157888 + }, + { + "epoch": 0.82, + "learning_rate": 9.264163055689295e-05, + "loss": 1.2503, + "theoretical_loss": 3.3494615495090585, + "tokens_seen": 2997288960 + }, + { + "epoch": 0.82, + "learning_rate": 9.260150858610176e-05, + "loss": 1.2945, + "theoretical_loss": 3.3494503817025714, + "tokens_seen": 2997420032 + }, + { + "epoch": 0.82, + "learning_rate": 9.256138661531055e-05, + "loss": 1.2579, + "theoretical_loss": 3.349439214521153, + "tokens_seen": 2997551104 + }, + { + "epoch": 0.82, + "learning_rate": 9.252126464451933e-05, + "loss": 1.2128, + "theoretical_loss": 3.349428047964742, + "tokens_seen": 2997682176 + }, + { + "epoch": 0.82, + "learning_rate": 9.248114267372814e-05, + "loss": 1.2432, + "theoretical_loss": 3.349416882033275, + "tokens_seen": 2997813248 + }, + { + "epoch": 0.82, + "learning_rate": 9.244102070293693e-05, + "loss": 1.2381, + "theoretical_loss": 3.3494057167266904, + "tokens_seen": 2997944320 + }, + { + "epoch": 0.82, + "learning_rate": 9.240089873214573e-05, + "loss": 1.2355, + "theoretical_loss": 3.3493945520449255, + "tokens_seen": 2998075392 + }, + { + "epoch": 0.82, + "learning_rate": 9.236077676135451e-05, + "loss": 1.1536, + "theoretical_loss": 3.3493833879879182, + "tokens_seen": 2998206464 + }, + { + "epoch": 0.82, + "learning_rate": 9.23206547905633e-05, + "loss": 1.2847, + "theoretical_loss": 3.3493722245556063, + "tokens_seen": 2998337536 + }, + { + "epoch": 0.82, + "learning_rate": 9.228053281977211e-05, + "loss": 1.3017, + "theoretical_loss": 3.3493610617479277, + "tokens_seen": 2998468608 + }, + { + "epoch": 0.82, + "learning_rate": 9.22404108489809e-05, + "loss": 1.2531, + "theoretical_loss": 3.3493498995648197, + "tokens_seen": 2998599680 + }, + { + "epoch": 0.82, + "learning_rate": 9.22002888781897e-05, + "loss": 1.2639, + "theoretical_loss": 3.34933873800622, + "tokens_seen": 2998730752 + }, + { + "epoch": 0.82, + "learning_rate": 9.216016690739849e-05, + "loss": 1.2447, + "theoretical_loss": 3.349327577072067, + "tokens_seen": 2998861824 + }, + { + "epoch": 0.82, + "learning_rate": 9.21200449366073e-05, + "loss": 1.2062, + "theoretical_loss": 3.349316416762298, + "tokens_seen": 2998992896 + }, + { + "epoch": 0.82, + "learning_rate": 9.207992296581608e-05, + "loss": 1.2209, + "theoretical_loss": 3.3493052570768507, + "tokens_seen": 2999123968 + }, + { + "epoch": 0.82, + "learning_rate": 9.203980099502487e-05, + "loss": 1.2641, + "theoretical_loss": 3.3492940980156636, + "tokens_seen": 2999255040 + }, + { + "epoch": 0.82, + "learning_rate": 9.199967902423368e-05, + "loss": 1.2049, + "theoretical_loss": 3.3492829395786736, + "tokens_seen": 2999386112 + }, + { + "epoch": 0.82, + "learning_rate": 9.195955705344247e-05, + "loss": 1.1909, + "theoretical_loss": 3.3492717817658186, + "tokens_seen": 2999517184 + }, + { + "epoch": 0.82, + "learning_rate": 9.191943508265126e-05, + "loss": 1.1561, + "theoretical_loss": 3.3492606245770373, + "tokens_seen": 2999648256 + }, + { + "epoch": 0.82, + "learning_rate": 9.187931311186005e-05, + "loss": 1.1843, + "theoretical_loss": 3.3492494680122666, + "tokens_seen": 2999779328 + }, + { + "epoch": 0.82, + "learning_rate": 9.183919114106886e-05, + "loss": 1.2654, + "theoretical_loss": 3.349238312071445, + "tokens_seen": 2999910400 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.9921075701713562, + "objective/train/avg_token_score": 0.007892429828643799, + "objective/train/docs_used": 1090669, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3878148794174194, + "objective/train/original_loss": 1.3976670503616333, + "objective/train/theoretical_loss": 3.3492271567545093, + "objective/train/tokens_used": 1370500576, + "objective/train/unlikelihood_loss": 0.024516455829143524, + "theoretical_loss": 3.3492271567545093, + "tokens_seen": 3000041472 + }, + { + "epoch": 0.82, + "learning_rate": 9.179906917027765e-05, + "loss": 1.3005, + "theoretical_loss": 3.3492271567545093, + "tokens_seen": 3000041472 + }, + { + "epoch": 0.82, + "learning_rate": 9.175894719948643e-05, + "loss": 1.1628, + "theoretical_loss": 3.3492160020613984, + "tokens_seen": 3000172544 + }, + { + "epoch": 0.82, + "learning_rate": 9.171882522869524e-05, + "loss": 1.212, + "theoretical_loss": 3.34920484799205, + "tokens_seen": 3000303616 + }, + { + "epoch": 0.82, + "learning_rate": 9.167870325790403e-05, + "loss": 1.265, + "theoretical_loss": 3.3491936945464023, + "tokens_seen": 3000434688 + }, + { + "epoch": 0.82, + "learning_rate": 9.163858128711282e-05, + "loss": 1.2967, + "theoretical_loss": 3.349182541724392, + "tokens_seen": 3000565760 + }, + { + "epoch": 0.82, + "learning_rate": 9.159845931632162e-05, + "loss": 1.2208, + "theoretical_loss": 3.3491713895259574, + "tokens_seen": 3000696832 + }, + { + "epoch": 0.82, + "learning_rate": 9.155833734553041e-05, + "loss": 1.1367, + "theoretical_loss": 3.3491602379510375, + "tokens_seen": 3000827904 + }, + { + "epoch": 0.82, + "learning_rate": 9.151821537473922e-05, + "loss": 1.2681, + "theoretical_loss": 3.349149086999569, + "tokens_seen": 3000958976 + }, + { + "epoch": 0.82, + "learning_rate": 9.1478093403948e-05, + "loss": 1.3558, + "theoretical_loss": 3.34913793667149, + "tokens_seen": 3001090048 + }, + { + "epoch": 0.82, + "learning_rate": 9.14379714331568e-05, + "loss": 1.168, + "theoretical_loss": 3.349126786966739, + "tokens_seen": 3001221120 + }, + { + "epoch": 0.82, + "learning_rate": 9.13978494623656e-05, + "loss": 1.3, + "theoretical_loss": 3.3491156378852533, + "tokens_seen": 3001352192 + }, + { + "epoch": 0.82, + "learning_rate": 9.13577274915744e-05, + "loss": 1.1276, + "theoretical_loss": 3.349104489426971, + "tokens_seen": 3001483264 + }, + { + "epoch": 0.82, + "learning_rate": 9.131760552078318e-05, + "loss": 1.1673, + "theoretical_loss": 3.3490933415918303, + "tokens_seen": 3001614336 + }, + { + "epoch": 0.82, + "learning_rate": 9.127748354999197e-05, + "loss": 1.172, + "theoretical_loss": 3.3490821943797693, + "tokens_seen": 3001745408 + }, + { + "epoch": 0.82, + "learning_rate": 9.123736157920078e-05, + "loss": 1.2827, + "theoretical_loss": 3.3490710477907255, + "tokens_seen": 3001876480 + }, + { + "epoch": 0.82, + "learning_rate": 9.119723960840956e-05, + "loss": 1.2652, + "theoretical_loss": 3.3490599018246368, + "tokens_seen": 3002007552 + }, + { + "epoch": 0.82, + "learning_rate": 9.115711763761836e-05, + "loss": 1.2808, + "theoretical_loss": 3.3490487564814417, + "tokens_seen": 3002138624 + }, + { + "epoch": 0.82, + "learning_rate": 9.111699566682716e-05, + "loss": 1.2917, + "theoretical_loss": 3.3490376117610783, + "tokens_seen": 3002269696 + }, + { + "epoch": 0.82, + "learning_rate": 9.107687369603595e-05, + "loss": 1.2419, + "theoretical_loss": 3.3490264676634833, + "tokens_seen": 3002400768 + }, + { + "epoch": 0.82, + "learning_rate": 9.103675172524474e-05, + "loss": 1.3274, + "theoretical_loss": 3.3490153241885965, + "tokens_seen": 3002531840 + }, + { + "epoch": 0.82, + "learning_rate": 9.099662975445354e-05, + "loss": 1.2608, + "theoretical_loss": 3.3490041813363547, + "tokens_seen": 3002662912 + }, + { + "epoch": 0.82, + "learning_rate": 9.095650778366234e-05, + "loss": 1.2984, + "theoretical_loss": 3.3489930391066967, + "tokens_seen": 3002793984 + }, + { + "epoch": 0.82, + "learning_rate": 9.091638581287113e-05, + "loss": 1.2508, + "theoretical_loss": 3.34898189749956, + "tokens_seen": 3002925056 + }, + { + "epoch": 0.82, + "learning_rate": 9.087626384207993e-05, + "loss": 1.1868, + "theoretical_loss": 3.3489707565148827, + "tokens_seen": 3003056128 + }, + { + "epoch": 0.82, + "learning_rate": 9.083614187128872e-05, + "loss": 1.3128, + "theoretical_loss": 3.348959616152603, + "tokens_seen": 3003187200 + }, + { + "epoch": 0.82, + "learning_rate": 9.079601990049751e-05, + "loss": 1.236, + "theoretical_loss": 3.3489484764126587, + "tokens_seen": 3003318272 + }, + { + "epoch": 0.82, + "learning_rate": 9.075589792970632e-05, + "loss": 1.3349, + "theoretical_loss": 3.3489373372949887, + "tokens_seen": 3003449344 + }, + { + "epoch": 0.82, + "learning_rate": 9.07157759589151e-05, + "loss": 1.2334, + "theoretical_loss": 3.3489261987995302, + "tokens_seen": 3003580416 + }, + { + "epoch": 0.82, + "learning_rate": 9.06756539881239e-05, + "loss": 1.2635, + "theoretical_loss": 3.3489150609262217, + "tokens_seen": 3003711488 + }, + { + "epoch": 0.82, + "learning_rate": 9.06355320173327e-05, + "loss": 1.1829, + "theoretical_loss": 3.348903923675001, + "tokens_seen": 3003842560 + }, + { + "epoch": 0.82, + "learning_rate": 9.059541004654148e-05, + "loss": 1.2531, + "theoretical_loss": 3.3488927870458065, + "tokens_seen": 3003973632 + }, + { + "epoch": 0.82, + "learning_rate": 9.055528807575028e-05, + "loss": 1.2451, + "theoretical_loss": 3.3488816510385764, + "tokens_seen": 3004104704 + }, + { + "epoch": 0.82, + "learning_rate": 9.051516610495908e-05, + "loss": 1.1918, + "theoretical_loss": 3.3488705156532483, + "tokens_seen": 3004235776 + }, + { + "epoch": 0.82, + "learning_rate": 9.047504413416788e-05, + "loss": 1.2034, + "theoretical_loss": 3.348859380889761, + "tokens_seen": 3004366848 + }, + { + "epoch": 0.82, + "learning_rate": 9.043492216337666e-05, + "loss": 1.2129, + "theoretical_loss": 3.3488482467480525, + "tokens_seen": 3004497920 + }, + { + "epoch": 0.82, + "learning_rate": 9.039480019258547e-05, + "loss": 1.2197, + "theoretical_loss": 3.3488371132280603, + "tokens_seen": 3004628992 + }, + { + "epoch": 0.82, + "learning_rate": 9.035467822179426e-05, + "loss": 1.2242, + "theoretical_loss": 3.3488259803297233, + "tokens_seen": 3004760064 + }, + { + "epoch": 0.82, + "learning_rate": 9.031455625100305e-05, + "loss": 1.1885, + "theoretical_loss": 3.34881484805298, + "tokens_seen": 3004891136 + }, + { + "epoch": 0.82, + "learning_rate": 9.027443428021185e-05, + "loss": 1.1706, + "theoretical_loss": 3.348803716397767, + "tokens_seen": 3005022208 + }, + { + "epoch": 0.82, + "learning_rate": 9.023431230942064e-05, + "loss": 1.235, + "theoretical_loss": 3.3487925853640244, + "tokens_seen": 3005153280 + }, + { + "epoch": 0.82, + "learning_rate": 9.019419033862944e-05, + "loss": 1.2114, + "theoretical_loss": 3.348781454951689, + "tokens_seen": 3005284352 + }, + { + "epoch": 0.82, + "learning_rate": 9.015406836783822e-05, + "loss": 1.2218, + "theoretical_loss": 3.3487703251606993, + "tokens_seen": 3005415424 + }, + { + "epoch": 0.82, + "learning_rate": 9.011394639704702e-05, + "loss": 1.2533, + "theoretical_loss": 3.3487591959909944, + "tokens_seen": 3005546496 + }, + { + "epoch": 0.82, + "learning_rate": 9.007382442625582e-05, + "loss": 1.2206, + "theoretical_loss": 3.3487480674425116, + "tokens_seen": 3005677568 + }, + { + "epoch": 0.82, + "learning_rate": 9.003370245546462e-05, + "loss": 1.2547, + "theoretical_loss": 3.3487369395151894, + "tokens_seen": 3005808640 + }, + { + "epoch": 0.82, + "learning_rate": 8.999358048467341e-05, + "loss": 1.161, + "theoretical_loss": 3.348725812208966, + "tokens_seen": 3005939712 + }, + { + "epoch": 0.82, + "learning_rate": 8.99534585138822e-05, + "loss": 1.1503, + "theoretical_loss": 3.3487146855237797, + "tokens_seen": 3006070784 + }, + { + "epoch": 0.82, + "learning_rate": 8.991333654309101e-05, + "loss": 1.1536, + "theoretical_loss": 3.348703559459569, + "tokens_seen": 3006201856 + }, + { + "epoch": 0.82, + "learning_rate": 8.98732145722998e-05, + "loss": 1.2315, + "theoretical_loss": 3.3486924340162716, + "tokens_seen": 3006332928 + }, + { + "epoch": 0.82, + "learning_rate": 8.983309260150858e-05, + "loss": 1.2352, + "theoretical_loss": 3.3486813091938266, + "tokens_seen": 3006464000 + }, + { + "epoch": 0.82, + "objective/train/aligned_token_count": 0.9927536249160767, + "objective/train/avg_token_score": 0.007246372289955616, + "objective/train/docs_used": 1093144, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2512459754943848, + "objective/train/original_loss": 1.261361002922058, + "objective/train/theoretical_loss": 3.348670184992171, + "objective/train/tokens_used": 1377054176, + "objective/train/unlikelihood_loss": 0.0275878868997097, + "theoretical_loss": 3.348670184992171, + "tokens_seen": 3006595072 + }, + { + "epoch": 0.82, + "learning_rate": 8.979297063071739e-05, + "loss": 1.2452, + "theoretical_loss": 3.348670184992171, + "tokens_seen": 3006595072 + }, + { + "epoch": 0.82, + "learning_rate": 8.975284865992618e-05, + "loss": 1.2164, + "theoretical_loss": 3.3486590614112446, + "tokens_seen": 3006726144 + }, + { + "epoch": 0.82, + "learning_rate": 8.971272668913497e-05, + "loss": 1.2647, + "theoretical_loss": 3.348647938450985, + "tokens_seen": 3006857216 + }, + { + "epoch": 0.82, + "learning_rate": 8.967260471834376e-05, + "loss": 1.1928, + "theoretical_loss": 3.3486368161113305, + "tokens_seen": 3006988288 + }, + { + "epoch": 0.82, + "learning_rate": 8.963248274755256e-05, + "loss": 1.1842, + "theoretical_loss": 3.3486256943922195, + "tokens_seen": 3007119360 + }, + { + "epoch": 0.82, + "learning_rate": 8.959236077676136e-05, + "loss": 1.2568, + "theoretical_loss": 3.34861457329359, + "tokens_seen": 3007250432 + }, + { + "epoch": 0.82, + "learning_rate": 8.955223880597014e-05, + "loss": 1.1484, + "theoretical_loss": 3.3486034528153805, + "tokens_seen": 3007381504 + }, + { + "epoch": 0.82, + "learning_rate": 8.951211683517895e-05, + "loss": 1.2759, + "theoretical_loss": 3.34859233295753, + "tokens_seen": 3007512576 + }, + { + "epoch": 0.82, + "learning_rate": 8.947199486438774e-05, + "loss": 1.2751, + "theoretical_loss": 3.3485812137199757, + "tokens_seen": 3007643648 + }, + { + "epoch": 0.82, + "learning_rate": 8.943187289359655e-05, + "loss": 1.1793, + "theoretical_loss": 3.348570095102657, + "tokens_seen": 3007774720 + }, + { + "epoch": 0.82, + "learning_rate": 8.939175092280533e-05, + "loss": 1.266, + "theoretical_loss": 3.3485589771055118, + "tokens_seen": 3007905792 + }, + { + "epoch": 0.82, + "learning_rate": 8.935162895201412e-05, + "loss": 1.2109, + "theoretical_loss": 3.3485478597284786, + "tokens_seen": 3008036864 + }, + { + "epoch": 0.82, + "learning_rate": 8.931150698122293e-05, + "loss": 1.2019, + "theoretical_loss": 3.348536742971496, + "tokens_seen": 3008167936 + }, + { + "epoch": 0.82, + "learning_rate": 8.92713850104317e-05, + "loss": 1.2148, + "theoretical_loss": 3.348525626834502, + "tokens_seen": 3008299008 + }, + { + "epoch": 0.82, + "learning_rate": 8.923126303964051e-05, + "loss": 1.1654, + "theoretical_loss": 3.348514511317435, + "tokens_seen": 3008430080 + }, + { + "epoch": 0.82, + "learning_rate": 8.91911410688493e-05, + "loss": 1.1909, + "theoretical_loss": 3.3485033964202335, + "tokens_seen": 3008561152 + }, + { + "epoch": 0.82, + "learning_rate": 8.91510190980581e-05, + "loss": 1.2476, + "theoretical_loss": 3.3484922821428365, + "tokens_seen": 3008692224 + }, + { + "epoch": 0.82, + "learning_rate": 8.911089712726689e-05, + "loss": 1.2706, + "theoretical_loss": 3.3484811684851814, + "tokens_seen": 3008823296 + }, + { + "epoch": 0.82, + "learning_rate": 8.907077515647568e-05, + "loss": 1.236, + "theoretical_loss": 3.3484700554472075, + "tokens_seen": 3008954368 + }, + { + "epoch": 0.82, + "learning_rate": 8.903065318568449e-05, + "loss": 1.1679, + "theoretical_loss": 3.348458943028853, + "tokens_seen": 3009085440 + }, + { + "epoch": 0.82, + "learning_rate": 8.899053121489328e-05, + "loss": 1.2726, + "theoretical_loss": 3.3484478312300565, + "tokens_seen": 3009216512 + }, + { + "epoch": 0.82, + "learning_rate": 8.895040924410207e-05, + "loss": 1.2341, + "theoretical_loss": 3.3484367200507563, + "tokens_seen": 3009347584 + }, + { + "epoch": 0.82, + "learning_rate": 8.891028727331087e-05, + "loss": 1.2439, + "theoretical_loss": 3.3484256094908904, + "tokens_seen": 3009478656 + }, + { + "epoch": 0.82, + "learning_rate": 8.887016530251966e-05, + "loss": 1.2042, + "theoretical_loss": 3.3484144995503984, + "tokens_seen": 3009609728 + }, + { + "epoch": 0.82, + "learning_rate": 8.883004333172845e-05, + "loss": 1.2213, + "theoretical_loss": 3.3484033902292176, + "tokens_seen": 3009740800 + }, + { + "epoch": 0.82, + "learning_rate": 8.878992136093724e-05, + "loss": 1.2612, + "theoretical_loss": 3.3483922815272873, + "tokens_seen": 3009871872 + }, + { + "epoch": 0.82, + "learning_rate": 8.874979939014605e-05, + "loss": 1.2252, + "theoretical_loss": 3.3483811734445457, + "tokens_seen": 3010002944 + }, + { + "epoch": 0.82, + "learning_rate": 8.870967741935484e-05, + "loss": 1.1795, + "theoretical_loss": 3.348370065980932, + "tokens_seen": 3010134016 + }, + { + "epoch": 0.82, + "learning_rate": 8.866955544856362e-05, + "loss": 1.2076, + "theoretical_loss": 3.348358959136384, + "tokens_seen": 3010265088 + }, + { + "epoch": 0.82, + "learning_rate": 8.862943347777243e-05, + "loss": 1.2701, + "theoretical_loss": 3.3483478529108397, + "tokens_seen": 3010396160 + }, + { + "epoch": 0.82, + "learning_rate": 8.858931150698122e-05, + "loss": 1.2043, + "theoretical_loss": 3.348336747304239, + "tokens_seen": 3010527232 + }, + { + "epoch": 0.82, + "learning_rate": 8.854918953619003e-05, + "loss": 1.2907, + "theoretical_loss": 3.3483256423165195, + "tokens_seen": 3010658304 + }, + { + "epoch": 0.82, + "learning_rate": 8.850906756539881e-05, + "loss": 1.2127, + "theoretical_loss": 3.3483145379476205, + "tokens_seen": 3010789376 + }, + { + "epoch": 0.82, + "learning_rate": 8.846894559460761e-05, + "loss": 1.2265, + "theoretical_loss": 3.3483034341974798, + "tokens_seen": 3010920448 + }, + { + "epoch": 0.82, + "learning_rate": 8.842882362381641e-05, + "loss": 1.2747, + "theoretical_loss": 3.3482923310660366, + "tokens_seen": 3011051520 + }, + { + "epoch": 0.82, + "learning_rate": 8.83887016530252e-05, + "loss": 1.1863, + "theoretical_loss": 3.3482812285532293, + "tokens_seen": 3011182592 + }, + { + "epoch": 0.83, + "learning_rate": 8.834857968223399e-05, + "loss": 1.1767, + "theoretical_loss": 3.3482701266589965, + "tokens_seen": 3011313664 + }, + { + "epoch": 0.83, + "learning_rate": 8.830845771144279e-05, + "loss": 1.2409, + "theoretical_loss": 3.3482590253832765, + "tokens_seen": 3011444736 + }, + { + "epoch": 0.83, + "learning_rate": 8.826833574065159e-05, + "loss": 1.2602, + "theoretical_loss": 3.3482479247260084, + "tokens_seen": 3011575808 + }, + { + "epoch": 0.83, + "learning_rate": 8.822821376986037e-05, + "loss": 1.2468, + "theoretical_loss": 3.34823682468713, + "tokens_seen": 3011706880 + }, + { + "epoch": 0.83, + "learning_rate": 8.818809179906916e-05, + "loss": 1.2226, + "theoretical_loss": 3.3482257252665817, + "tokens_seen": 3011837952 + }, + { + "epoch": 0.83, + "learning_rate": 8.814796982827797e-05, + "loss": 1.2474, + "theoretical_loss": 3.3482146264643005, + "tokens_seen": 3011969024 + }, + { + "epoch": 0.83, + "learning_rate": 8.810784785748676e-05, + "loss": 1.2065, + "theoretical_loss": 3.348203528280225, + "tokens_seen": 3012100096 + }, + { + "epoch": 0.83, + "learning_rate": 8.806772588669556e-05, + "loss": 1.1819, + "theoretical_loss": 3.3481924307142954, + "tokens_seen": 3012231168 + }, + { + "epoch": 0.83, + "learning_rate": 8.802760391590435e-05, + "loss": 1.2127, + "theoretical_loss": 3.348181333766449, + "tokens_seen": 3012362240 + }, + { + "epoch": 0.83, + "learning_rate": 8.798748194511315e-05, + "loss": 1.1669, + "theoretical_loss": 3.3481702374366247, + "tokens_seen": 3012493312 + }, + { + "epoch": 0.83, + "learning_rate": 8.794735997432195e-05, + "loss": 1.2965, + "theoretical_loss": 3.3481591417247616, + "tokens_seen": 3012624384 + }, + { + "epoch": 0.83, + "learning_rate": 8.790723800353073e-05, + "loss": 1.2476, + "theoretical_loss": 3.3481480466307985, + "tokens_seen": 3012755456 + }, + { + "epoch": 0.83, + "learning_rate": 8.786711603273953e-05, + "loss": 1.2352, + "theoretical_loss": 3.3481369521546736, + "tokens_seen": 3012886528 + }, + { + "epoch": 0.83, + "learning_rate": 8.782699406194833e-05, + "loss": 1.291, + "theoretical_loss": 3.3481258582963256, + "tokens_seen": 3013017600 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.9914937615394592, + "objective/train/avg_token_score": 0.008506190031766891, + "objective/train/docs_used": 1095529, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2406266927719116, + "objective/train/original_loss": 1.2519199848175049, + "objective/train/theoretical_loss": 3.3481147650556937, + "objective/train/tokens_used": 1383607776, + "objective/train/unlikelihood_loss": 0.02390122599899769, + "theoretical_loss": 3.3481147650556937, + "tokens_seen": 3013148672 + }, + { + "epoch": 0.83, + "learning_rate": 8.778687209115712e-05, + "loss": 1.2063, + "theoretical_loss": 3.3481147650556937, + "tokens_seen": 3013148672 + }, + { + "epoch": 0.83, + "learning_rate": 8.774675012036591e-05, + "loss": 1.2533, + "theoretical_loss": 3.3481036724327167, + "tokens_seen": 3013279744 + }, + { + "epoch": 0.83, + "learning_rate": 8.770662814957472e-05, + "loss": 1.1972, + "theoretical_loss": 3.3480925804273323, + "tokens_seen": 3013410816 + }, + { + "epoch": 0.83, + "learning_rate": 8.766650617878351e-05, + "loss": 1.2195, + "theoretical_loss": 3.3480814890394806, + "tokens_seen": 3013541888 + }, + { + "epoch": 0.83, + "learning_rate": 8.762638420799229e-05, + "loss": 1.2329, + "theoretical_loss": 3.3480703982690994, + "tokens_seen": 3013672960 + }, + { + "epoch": 0.83, + "learning_rate": 8.75862622372011e-05, + "loss": 1.2232, + "theoretical_loss": 3.348059308116128, + "tokens_seen": 3013804032 + }, + { + "epoch": 0.83, + "learning_rate": 8.754614026640989e-05, + "loss": 1.2596, + "theoretical_loss": 3.348048218580505, + "tokens_seen": 3013935104 + }, + { + "epoch": 0.83, + "learning_rate": 8.75060182956187e-05, + "loss": 1.2478, + "theoretical_loss": 3.348037129662169, + "tokens_seen": 3014066176 + }, + { + "epoch": 0.83, + "learning_rate": 8.746589632482747e-05, + "loss": 1.282, + "theoretical_loss": 3.3480260413610594, + "tokens_seen": 3014197248 + }, + { + "epoch": 0.83, + "learning_rate": 8.742577435403627e-05, + "loss": 1.2117, + "theoretical_loss": 3.348014953677114, + "tokens_seen": 3014328320 + }, + { + "epoch": 0.83, + "learning_rate": 8.738565238324507e-05, + "loss": 1.3007, + "theoretical_loss": 3.3480038666102727, + "tokens_seen": 3014459392 + }, + { + "epoch": 0.83, + "learning_rate": 8.734553041245385e-05, + "loss": 1.2444, + "theoretical_loss": 3.347992780160474, + "tokens_seen": 3014590464 + }, + { + "epoch": 0.83, + "learning_rate": 8.730540844166266e-05, + "loss": 1.2133, + "theoretical_loss": 3.3479816943276566, + "tokens_seen": 3014721536 + }, + { + "epoch": 0.83, + "learning_rate": 8.726528647087145e-05, + "loss": 1.2509, + "theoretical_loss": 3.347970609111759, + "tokens_seen": 3014852608 + }, + { + "epoch": 0.83, + "learning_rate": 8.722516450008026e-05, + "loss": 1.2277, + "theoretical_loss": 3.3479595245127203, + "tokens_seen": 3014983680 + }, + { + "epoch": 0.83, + "learning_rate": 8.718504252928904e-05, + "loss": 1.3066, + "theoretical_loss": 3.3479484405304794, + "tokens_seen": 3015114752 + }, + { + "epoch": 0.83, + "learning_rate": 8.714492055849783e-05, + "loss": 1.27, + "theoretical_loss": 3.3479373571649753, + "tokens_seen": 3015245824 + }, + { + "epoch": 0.83, + "learning_rate": 8.710479858770664e-05, + "loss": 1.205, + "theoretical_loss": 3.347926274416147, + "tokens_seen": 3015376896 + }, + { + "epoch": 0.83, + "learning_rate": 8.706467661691543e-05, + "loss": 1.2447, + "theoretical_loss": 3.3479151922839328, + "tokens_seen": 3015507968 + }, + { + "epoch": 0.83, + "learning_rate": 8.702455464612422e-05, + "loss": 1.3156, + "theoretical_loss": 3.347904110768272, + "tokens_seen": 3015639040 + }, + { + "epoch": 0.83, + "learning_rate": 8.698443267533301e-05, + "loss": 1.1463, + "theoretical_loss": 3.3478930298691036, + "tokens_seen": 3015770112 + }, + { + "epoch": 0.83, + "learning_rate": 8.69443107045418e-05, + "loss": 1.2091, + "theoretical_loss": 3.3478819495863665, + "tokens_seen": 3015901184 + }, + { + "epoch": 0.83, + "learning_rate": 8.69041887337506e-05, + "loss": 1.1712, + "theoretical_loss": 3.3478708699199995, + "tokens_seen": 3016032256 + }, + { + "epoch": 0.83, + "learning_rate": 8.686406676295939e-05, + "loss": 1.1704, + "theoretical_loss": 3.3478597908699412, + "tokens_seen": 3016163328 + }, + { + "epoch": 0.83, + "learning_rate": 8.68239447921682e-05, + "loss": 1.1909, + "theoretical_loss": 3.347848712436131, + "tokens_seen": 3016294400 + }, + { + "epoch": 0.83, + "learning_rate": 8.678382282137699e-05, + "loss": 1.3013, + "theoretical_loss": 3.3478376346185077, + "tokens_seen": 3016425472 + }, + { + "epoch": 0.83, + "learning_rate": 8.674370085058578e-05, + "loss": 1.2376, + "theoretical_loss": 3.3478265574170103, + "tokens_seen": 3016556544 + }, + { + "epoch": 0.83, + "learning_rate": 8.670357887979458e-05, + "loss": 1.2446, + "theoretical_loss": 3.3478154808315774, + "tokens_seen": 3016687616 + }, + { + "epoch": 0.83, + "learning_rate": 8.666345690900337e-05, + "loss": 1.2176, + "theoretical_loss": 3.3478044048621487, + "tokens_seen": 3016818688 + }, + { + "epoch": 0.83, + "learning_rate": 8.662333493821218e-05, + "loss": 1.2504, + "theoretical_loss": 3.3477933295086624, + "tokens_seen": 3016949760 + }, + { + "epoch": 0.83, + "learning_rate": 8.658321296742095e-05, + "loss": 1.2328, + "theoretical_loss": 3.347782254771058, + "tokens_seen": 3017080832 + }, + { + "epoch": 0.83, + "learning_rate": 8.654309099662976e-05, + "loss": 1.2092, + "theoretical_loss": 3.3477711806492745, + "tokens_seen": 3017211904 + }, + { + "epoch": 0.83, + "learning_rate": 8.650296902583855e-05, + "loss": 1.1971, + "theoretical_loss": 3.347760107143251, + "tokens_seen": 3017342976 + }, + { + "epoch": 0.83, + "learning_rate": 8.646284705504733e-05, + "loss": 1.1321, + "theoretical_loss": 3.3477490342529257, + "tokens_seen": 3017474048 + }, + { + "epoch": 0.83, + "learning_rate": 8.642272508425614e-05, + "loss": 1.2213, + "theoretical_loss": 3.3477379619782384, + "tokens_seen": 3017605120 + }, + { + "epoch": 0.83, + "learning_rate": 8.638260311346493e-05, + "loss": 1.2677, + "theoretical_loss": 3.347726890319128, + "tokens_seen": 3017736192 + }, + { + "epoch": 0.83, + "learning_rate": 8.634248114267374e-05, + "loss": 1.3125, + "theoretical_loss": 3.3477158192755336, + "tokens_seen": 3017867264 + }, + { + "epoch": 0.83, + "learning_rate": 8.630235917188252e-05, + "loss": 1.1525, + "theoretical_loss": 3.3477047488473937, + "tokens_seen": 3017998336 + }, + { + "epoch": 0.83, + "learning_rate": 8.626223720109132e-05, + "loss": 1.2295, + "theoretical_loss": 3.347693679034648, + "tokens_seen": 3018129408 + }, + { + "epoch": 0.83, + "learning_rate": 8.622211523030012e-05, + "loss": 1.2314, + "theoretical_loss": 3.3476826098372356, + "tokens_seen": 3018260480 + }, + { + "epoch": 0.83, + "learning_rate": 8.618199325950891e-05, + "loss": 1.2594, + "theoretical_loss": 3.347671541255095, + "tokens_seen": 3018391552 + }, + { + "epoch": 0.83, + "learning_rate": 8.61418712887177e-05, + "loss": 1.2455, + "theoretical_loss": 3.347660473288166, + "tokens_seen": 3018522624 + }, + { + "epoch": 0.83, + "learning_rate": 8.61017493179265e-05, + "loss": 1.2306, + "theoretical_loss": 3.347649405936387, + "tokens_seen": 3018653696 + }, + { + "epoch": 0.83, + "learning_rate": 8.60616273471353e-05, + "loss": 1.1694, + "theoretical_loss": 3.347638339199697, + "tokens_seen": 3018784768 + }, + { + "epoch": 0.83, + "learning_rate": 8.60215053763441e-05, + "loss": 1.2138, + "theoretical_loss": 3.3476272730780363, + "tokens_seen": 3018915840 + }, + { + "epoch": 0.83, + "learning_rate": 8.598138340555287e-05, + "loss": 1.308, + "theoretical_loss": 3.3476162075713427, + "tokens_seen": 3019046912 + }, + { + "epoch": 0.83, + "learning_rate": 8.594126143476168e-05, + "loss": 1.1761, + "theoretical_loss": 3.3476051426795563, + "tokens_seen": 3019177984 + }, + { + "epoch": 0.83, + "learning_rate": 8.590113946397047e-05, + "loss": 1.2926, + "theoretical_loss": 3.3475940784026155, + "tokens_seen": 3019309056 + }, + { + "epoch": 0.83, + "learning_rate": 8.586101749317927e-05, + "loss": 1.2232, + "theoretical_loss": 3.3475830147404597, + "tokens_seen": 3019440128 + }, + { + "epoch": 0.83, + "learning_rate": 8.582089552238806e-05, + "loss": 1.2774, + "theoretical_loss": 3.3475719516930287, + "tokens_seen": 3019571200 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.9923089742660522, + "objective/train/avg_token_score": 0.007690987084060907, + "objective/train/docs_used": 1097674, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2262123823165894, + "objective/train/original_loss": 1.23773193359375, + "objective/train/theoretical_loss": 3.34756088926026, + "objective/train/tokens_used": 1390161376, + "objective/train/unlikelihood_loss": 0.020462170243263245, + "theoretical_loss": 3.34756088926026, + "tokens_seen": 3019702272 + }, + { + "epoch": 0.83, + "learning_rate": 8.578077355159686e-05, + "loss": 1.2186, + "theoretical_loss": 3.34756088926026, + "tokens_seen": 3019702272 + }, + { + "epoch": 0.83, + "learning_rate": 8.574065158080566e-05, + "loss": 1.2804, + "theoretical_loss": 3.3475498274420947, + "tokens_seen": 3019833344 + }, + { + "epoch": 0.83, + "learning_rate": 8.570052961001444e-05, + "loss": 1.2653, + "theoretical_loss": 3.347538766238471, + "tokens_seen": 3019964416 + }, + { + "epoch": 0.83, + "learning_rate": 8.566040763922324e-05, + "loss": 1.2017, + "theoretical_loss": 3.347527705649328, + "tokens_seen": 3020095488 + }, + { + "epoch": 0.83, + "learning_rate": 8.562028566843204e-05, + "loss": 1.245, + "theoretical_loss": 3.347516645674605, + "tokens_seen": 3020226560 + }, + { + "epoch": 0.83, + "learning_rate": 8.558016369764084e-05, + "loss": 1.2493, + "theoretical_loss": 3.3475055863142416, + "tokens_seen": 3020357632 + }, + { + "epoch": 0.83, + "learning_rate": 8.554004172684962e-05, + "loss": 1.2157, + "theoretical_loss": 3.3474945275681764, + "tokens_seen": 3020488704 + }, + { + "epoch": 0.83, + "learning_rate": 8.549991975605841e-05, + "loss": 1.2334, + "theoretical_loss": 3.3474834694363493, + "tokens_seen": 3020619776 + }, + { + "epoch": 0.83, + "learning_rate": 8.545979778526722e-05, + "loss": 1.2272, + "theoretical_loss": 3.3474724119186994, + "tokens_seen": 3020750848 + }, + { + "epoch": 0.83, + "learning_rate": 8.5419675814476e-05, + "loss": 1.2203, + "theoretical_loss": 3.3474613550151653, + "tokens_seen": 3020881920 + }, + { + "epoch": 0.83, + "learning_rate": 8.53795538436848e-05, + "loss": 1.236, + "theoretical_loss": 3.3474502987256867, + "tokens_seen": 3021012992 + }, + { + "epoch": 0.83, + "learning_rate": 8.53394318728936e-05, + "loss": 1.2665, + "theoretical_loss": 3.3474392430502027, + "tokens_seen": 3021144064 + }, + { + "epoch": 0.83, + "learning_rate": 8.52993099021024e-05, + "loss": 1.2473, + "theoretical_loss": 3.347428187988653, + "tokens_seen": 3021275136 + }, + { + "epoch": 0.83, + "learning_rate": 8.525918793131118e-05, + "loss": 1.2967, + "theoretical_loss": 3.3474171335409766, + "tokens_seen": 3021406208 + }, + { + "epoch": 0.83, + "learning_rate": 8.521906596051998e-05, + "loss": 1.207, + "theoretical_loss": 3.3474060797071123, + "tokens_seen": 3021537280 + }, + { + "epoch": 0.83, + "learning_rate": 8.517894398972878e-05, + "loss": 1.2322, + "theoretical_loss": 3.3473950264870003, + "tokens_seen": 3021668352 + }, + { + "epoch": 0.83, + "learning_rate": 8.513882201893758e-05, + "loss": 1.2614, + "theoretical_loss": 3.3473839738805795, + "tokens_seen": 3021799424 + }, + { + "epoch": 0.83, + "learning_rate": 8.509870004814637e-05, + "loss": 1.2691, + "theoretical_loss": 3.347372921887789, + "tokens_seen": 3021930496 + }, + { + "epoch": 0.83, + "learning_rate": 8.505857807735516e-05, + "loss": 1.1929, + "theoretical_loss": 3.347361870508568, + "tokens_seen": 3022061568 + }, + { + "epoch": 0.83, + "learning_rate": 8.501845610656395e-05, + "loss": 1.2337, + "theoretical_loss": 3.3473508197428563, + "tokens_seen": 3022192640 + }, + { + "epoch": 0.83, + "learning_rate": 8.497833413577275e-05, + "loss": 1.25, + "theoretical_loss": 3.3473397695905933, + "tokens_seen": 3022323712 + }, + { + "epoch": 0.83, + "learning_rate": 8.493821216498154e-05, + "loss": 1.2211, + "theoretical_loss": 3.347328720051718, + "tokens_seen": 3022454784 + }, + { + "epoch": 0.83, + "learning_rate": 8.489809019419035e-05, + "loss": 1.2099, + "theoretical_loss": 3.3473176711261696, + "tokens_seen": 3022585856 + }, + { + "epoch": 0.83, + "learning_rate": 8.485796822339914e-05, + "loss": 1.2142, + "theoretical_loss": 3.347306622813888, + "tokens_seen": 3022716928 + }, + { + "epoch": 0.83, + "learning_rate": 8.481784625260793e-05, + "loss": 1.1933, + "theoretical_loss": 3.347295575114812, + "tokens_seen": 3022848000 + }, + { + "epoch": 0.83, + "learning_rate": 8.477772428181672e-05, + "loss": 1.2075, + "theoretical_loss": 3.347284528028881, + "tokens_seen": 3022979072 + }, + { + "epoch": 0.83, + "learning_rate": 8.473760231102552e-05, + "loss": 1.207, + "theoretical_loss": 3.3472734815560354, + "tokens_seen": 3023110144 + }, + { + "epoch": 0.83, + "learning_rate": 8.469748034023432e-05, + "loss": 1.2482, + "theoretical_loss": 3.3472624356962135, + "tokens_seen": 3023241216 + }, + { + "epoch": 0.83, + "learning_rate": 8.46573583694431e-05, + "loss": 1.28, + "theoretical_loss": 3.347251390449355, + "tokens_seen": 3023372288 + }, + { + "epoch": 0.83, + "learning_rate": 8.461723639865191e-05, + "loss": 1.1759, + "theoretical_loss": 3.347240345815399, + "tokens_seen": 3023503360 + }, + { + "epoch": 0.83, + "learning_rate": 8.45771144278607e-05, + "loss": 1.2586, + "theoretical_loss": 3.347229301794286, + "tokens_seen": 3023634432 + }, + { + "epoch": 0.83, + "learning_rate": 8.453699245706948e-05, + "loss": 1.2288, + "theoretical_loss": 3.347218258385954, + "tokens_seen": 3023765504 + }, + { + "epoch": 0.83, + "learning_rate": 8.449687048627829e-05, + "loss": 1.1806, + "theoretical_loss": 3.347207215590344, + "tokens_seen": 3023896576 + }, + { + "epoch": 0.83, + "learning_rate": 8.445674851548708e-05, + "loss": 1.1792, + "theoretical_loss": 3.347196173407394, + "tokens_seen": 3024027648 + }, + { + "epoch": 0.83, + "learning_rate": 8.441662654469589e-05, + "loss": 1.2259, + "theoretical_loss": 3.347185131837044, + "tokens_seen": 3024158720 + }, + { + "epoch": 0.83, + "learning_rate": 8.437650457390466e-05, + "loss": 1.197, + "theoretical_loss": 3.347174090879234, + "tokens_seen": 3024289792 + }, + { + "epoch": 0.83, + "learning_rate": 8.433638260311347e-05, + "loss": 1.2264, + "theoretical_loss": 3.3471630505339025, + "tokens_seen": 3024420864 + }, + { + "epoch": 0.83, + "learning_rate": 8.429626063232226e-05, + "loss": 1.2775, + "theoretical_loss": 3.3471520108009893, + "tokens_seen": 3024551936 + }, + { + "epoch": 0.83, + "learning_rate": 8.425613866153106e-05, + "loss": 1.3045, + "theoretical_loss": 3.3471409716804343, + "tokens_seen": 3024683008 + }, + { + "epoch": 0.83, + "learning_rate": 8.421601669073985e-05, + "loss": 1.1611, + "theoretical_loss": 3.347129933172177, + "tokens_seen": 3024814080 + }, + { + "epoch": 0.83, + "learning_rate": 8.417589471994864e-05, + "loss": 1.2734, + "theoretical_loss": 3.347118895276157, + "tokens_seen": 3024945152 + }, + { + "epoch": 0.83, + "learning_rate": 8.413577274915745e-05, + "loss": 1.2673, + "theoretical_loss": 3.3471078579923126, + "tokens_seen": 3025076224 + }, + { + "epoch": 0.83, + "learning_rate": 8.409565077836623e-05, + "loss": 1.2051, + "theoretical_loss": 3.3470968213205845, + "tokens_seen": 3025207296 + }, + { + "epoch": 0.83, + "learning_rate": 8.405552880757502e-05, + "loss": 1.233, + "theoretical_loss": 3.347085785260912, + "tokens_seen": 3025338368 + }, + { + "epoch": 0.83, + "learning_rate": 8.401540683678383e-05, + "loss": 1.2147, + "theoretical_loss": 3.347074749813235, + "tokens_seen": 3025469440 + }, + { + "epoch": 0.83, + "learning_rate": 8.397528486599262e-05, + "loss": 1.3472, + "theoretical_loss": 3.347063714977492, + "tokens_seen": 3025600512 + }, + { + "epoch": 0.83, + "learning_rate": 8.393516289520141e-05, + "loss": 1.1637, + "theoretical_loss": 3.347052680753624, + "tokens_seen": 3025731584 + }, + { + "epoch": 0.83, + "learning_rate": 8.38950409244102e-05, + "loss": 1.2839, + "theoretical_loss": 3.347041647141569, + "tokens_seen": 3025862656 + }, + { + "epoch": 0.83, + "learning_rate": 8.385491895361901e-05, + "loss": 1.205, + "theoretical_loss": 3.347030614141268, + "tokens_seen": 3025993728 + }, + { + "epoch": 0.83, + "learning_rate": 8.38147969828278e-05, + "loss": 1.2844, + "theoretical_loss": 3.3470195817526593, + "tokens_seen": 3026124800 + }, + { + "epoch": 0.83, + "objective/train/aligned_token_count": 0.9922446012496948, + "objective/train/avg_token_score": 0.007755397818982601, + "objective/train/docs_used": 1099986, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.267446517944336, + "objective/train/original_loss": 1.280320167541504, + "objective/train/theoretical_loss": 3.3470085499756834, + "objective/train/tokens_used": 1396714976, + "objective/train/unlikelihood_loss": 0.019157670438289642, + "theoretical_loss": 3.3470085499756834, + "tokens_seen": 3026255872 + }, + { + "epoch": 0.83, + "learning_rate": 8.377467501203658e-05, + "loss": 1.2603, + "theoretical_loss": 3.3470085499756834, + "tokens_seen": 3026255872 + }, + { + "epoch": 0.83, + "learning_rate": 8.373455304124539e-05, + "loss": 1.3092, + "theoretical_loss": 3.3469975188102796, + "tokens_seen": 3026386944 + }, + { + "epoch": 0.83, + "learning_rate": 8.369443107045418e-05, + "loss": 1.3074, + "theoretical_loss": 3.3469864882563876, + "tokens_seen": 3026518016 + }, + { + "epoch": 0.83, + "learning_rate": 8.365430909966299e-05, + "loss": 1.2972, + "theoretical_loss": 3.346975458313947, + "tokens_seen": 3026649088 + }, + { + "epoch": 0.83, + "learning_rate": 8.361418712887177e-05, + "loss": 1.2105, + "theoretical_loss": 3.346964428982897, + "tokens_seen": 3026780160 + }, + { + "epoch": 0.83, + "learning_rate": 8.357406515808056e-05, + "loss": 1.2663, + "theoretical_loss": 3.3469534002631782, + "tokens_seen": 3026911232 + }, + { + "epoch": 0.83, + "learning_rate": 8.353394318728937e-05, + "loss": 1.194, + "theoretical_loss": 3.3469423721547296, + "tokens_seen": 3027042304 + }, + { + "epoch": 0.83, + "learning_rate": 8.349382121649815e-05, + "loss": 1.2572, + "theoretical_loss": 3.3469313446574906, + "tokens_seen": 3027173376 + }, + { + "epoch": 0.83, + "learning_rate": 8.345369924570695e-05, + "loss": 1.2755, + "theoretical_loss": 3.346920317771401, + "tokens_seen": 3027304448 + }, + { + "epoch": 0.83, + "learning_rate": 8.341357727491575e-05, + "loss": 1.285, + "theoretical_loss": 3.3469092914964014, + "tokens_seen": 3027435520 + }, + { + "epoch": 0.83, + "learning_rate": 8.337345530412455e-05, + "loss": 1.2521, + "theoretical_loss": 3.34689826583243, + "tokens_seen": 3027566592 + }, + { + "epoch": 0.84, + "learning_rate": 8.333333333333333e-05, + "loss": 1.2586, + "theoretical_loss": 3.3468872407794277, + "tokens_seen": 3027697664 + }, + { + "epoch": 0.84, + "learning_rate": 8.329321136254212e-05, + "loss": 1.251, + "theoretical_loss": 3.346876216337334, + "tokens_seen": 3027828736 + }, + { + "epoch": 0.84, + "learning_rate": 8.325308939175093e-05, + "loss": 1.2746, + "theoretical_loss": 3.3468651925060877, + "tokens_seen": 3027959808 + }, + { + "epoch": 0.84, + "learning_rate": 8.321296742095972e-05, + "loss": 1.2173, + "theoretical_loss": 3.3468541692856295, + "tokens_seen": 3028090880 + }, + { + "epoch": 0.84, + "learning_rate": 8.317284545016852e-05, + "loss": 1.179, + "theoretical_loss": 3.3468431466758983, + "tokens_seen": 3028221952 + }, + { + "epoch": 0.84, + "learning_rate": 8.313272347937731e-05, + "loss": 1.1712, + "theoretical_loss": 3.3468321246768347, + "tokens_seen": 3028353024 + }, + { + "epoch": 0.84, + "learning_rate": 8.309260150858611e-05, + "loss": 1.2335, + "theoretical_loss": 3.346821103288378, + "tokens_seen": 3028484096 + }, + { + "epoch": 0.84, + "learning_rate": 8.30524795377949e-05, + "loss": 1.1628, + "theoretical_loss": 3.346810082510468, + "tokens_seen": 3028615168 + }, + { + "epoch": 0.84, + "learning_rate": 8.301235756700369e-05, + "loss": 1.2544, + "theoretical_loss": 3.3467990623430444, + "tokens_seen": 3028746240 + }, + { + "epoch": 0.84, + "learning_rate": 8.297223559621249e-05, + "loss": 1.3207, + "theoretical_loss": 3.346788042786047, + "tokens_seen": 3028877312 + }, + { + "epoch": 0.84, + "learning_rate": 8.293211362542129e-05, + "loss": 1.2883, + "theoretical_loss": 3.3467770238394157, + "tokens_seen": 3029008384 + }, + { + "epoch": 0.84, + "learning_rate": 8.289199165463008e-05, + "loss": 1.2535, + "theoretical_loss": 3.34676600550309, + "tokens_seen": 3029139456 + }, + { + "epoch": 0.84, + "learning_rate": 8.285186968383887e-05, + "loss": 1.2557, + "theoretical_loss": 3.3467549877770097, + "tokens_seen": 3029270528 + }, + { + "epoch": 0.84, + "learning_rate": 8.281174771304766e-05, + "loss": 1.2286, + "theoretical_loss": 3.346743970661115, + "tokens_seen": 3029401600 + }, + { + "epoch": 0.84, + "learning_rate": 8.277162574225647e-05, + "loss": 1.2747, + "theoretical_loss": 3.3467329541553457, + "tokens_seen": 3029532672 + }, + { + "epoch": 0.84, + "learning_rate": 8.273150377146525e-05, + "loss": 1.2132, + "theoretical_loss": 3.346721938259641, + "tokens_seen": 3029663744 + }, + { + "epoch": 0.84, + "learning_rate": 8.269138180067406e-05, + "loss": 1.1701, + "theoretical_loss": 3.346710922973941, + "tokens_seen": 3029794816 + }, + { + "epoch": 0.84, + "learning_rate": 8.265125982988285e-05, + "loss": 1.2655, + "theoretical_loss": 3.346699908298186, + "tokens_seen": 3029925888 + }, + { + "epoch": 0.84, + "learning_rate": 8.261113785909164e-05, + "loss": 1.2596, + "theoretical_loss": 3.346688894232315, + "tokens_seen": 3030056960 + }, + { + "epoch": 0.84, + "learning_rate": 8.257101588830043e-05, + "loss": 1.2075, + "theoretical_loss": 3.3466778807762694, + "tokens_seen": 3030188032 + }, + { + "epoch": 0.84, + "learning_rate": 8.253089391750923e-05, + "loss": 1.2536, + "theoretical_loss": 3.346666867929987, + "tokens_seen": 3030319104 + }, + { + "epoch": 0.84, + "learning_rate": 8.249077194671803e-05, + "loss": 1.261, + "theoretical_loss": 3.3466558556934087, + "tokens_seen": 3030450176 + }, + { + "epoch": 0.84, + "learning_rate": 8.245064997592681e-05, + "loss": 1.188, + "theoretical_loss": 3.346644844066475, + "tokens_seen": 3030581248 + }, + { + "epoch": 0.84, + "learning_rate": 8.241052800513562e-05, + "loss": 1.2846, + "theoretical_loss": 3.3466338330491245, + "tokens_seen": 3030712320 + }, + { + "epoch": 0.84, + "learning_rate": 8.237040603434441e-05, + "loss": 1.2763, + "theoretical_loss": 3.3466228226412977, + "tokens_seen": 3030843392 + }, + { + "epoch": 0.84, + "learning_rate": 8.23302840635532e-05, + "loss": 1.2823, + "theoretical_loss": 3.346611812842935, + "tokens_seen": 3030974464 + }, + { + "epoch": 0.84, + "learning_rate": 8.2290162092762e-05, + "loss": 1.233, + "theoretical_loss": 3.3466008036539754, + "tokens_seen": 3031105536 + }, + { + "epoch": 0.84, + "learning_rate": 8.225004012197079e-05, + "loss": 1.2163, + "theoretical_loss": 3.3465897950743595, + "tokens_seen": 3031236608 + }, + { + "epoch": 0.84, + "learning_rate": 8.22099181511796e-05, + "loss": 1.2687, + "theoretical_loss": 3.346578787104027, + "tokens_seen": 3031367680 + }, + { + "epoch": 0.84, + "learning_rate": 8.216979618038837e-05, + "loss": 1.3374, + "theoretical_loss": 3.3465677797429176, + "tokens_seen": 3031498752 + }, + { + "epoch": 0.84, + "learning_rate": 8.212967420959718e-05, + "loss": 1.2473, + "theoretical_loss": 3.3465567729909713, + "tokens_seen": 3031629824 + }, + { + "epoch": 0.84, + "learning_rate": 8.208955223880597e-05, + "loss": 1.2895, + "theoretical_loss": 3.346545766848129, + "tokens_seen": 3031760896 + }, + { + "epoch": 0.84, + "learning_rate": 8.204943026801477e-05, + "loss": 1.2359, + "theoretical_loss": 3.346534761314329, + "tokens_seen": 3031891968 + }, + { + "epoch": 0.84, + "learning_rate": 8.200930829722356e-05, + "loss": 1.3368, + "theoretical_loss": 3.3465237563895123, + "tokens_seen": 3032023040 + }, + { + "epoch": 0.84, + "learning_rate": 8.196918632643235e-05, + "loss": 1.321, + "theoretical_loss": 3.346512752073619, + "tokens_seen": 3032154112 + }, + { + "epoch": 0.84, + "learning_rate": 8.192906435564116e-05, + "loss": 1.1863, + "theoretical_loss": 3.3465017483665886, + "tokens_seen": 3032285184 + }, + { + "epoch": 0.84, + "learning_rate": 8.188894238484995e-05, + "loss": 1.305, + "theoretical_loss": 3.3464907452683614, + "tokens_seen": 3032416256 + }, + { + "epoch": 0.84, + "learning_rate": 8.184882041405873e-05, + "loss": 1.2062, + "theoretical_loss": 3.3464797427788775, + "tokens_seen": 3032547328 + }, + { + "epoch": 0.84, + "learning_rate": 8.180869844326754e-05, + "loss": 1.2894, + "theoretical_loss": 3.3464687408980764, + "tokens_seen": 3032678400 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.9925715327262878, + "objective/train/avg_token_score": 0.007428442593663931, + "objective/train/docs_used": 1102459, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2367547750473022, + "objective/train/original_loss": 1.247039556503296, + "objective/train/theoretical_loss": 3.3464577396258983, + "objective/train/tokens_used": 1403268576, + "objective/train/unlikelihood_loss": 0.022024942561984062, + "theoretical_loss": 3.3464577396258983, + "tokens_seen": 3032809472 + }, + { + "epoch": 0.84, + "learning_rate": 8.176857647247633e-05, + "loss": 1.2486, + "theoretical_loss": 3.3464577396258983, + "tokens_seen": 3032809472 + }, + { + "epoch": 0.84, + "learning_rate": 8.172845450168512e-05, + "loss": 1.2217, + "theoretical_loss": 3.346446738962284, + "tokens_seen": 3032940544 + }, + { + "epoch": 0.84, + "learning_rate": 8.168833253089391e-05, + "loss": 1.2375, + "theoretical_loss": 3.3464357389071724, + "tokens_seen": 3033071616 + }, + { + "epoch": 0.84, + "learning_rate": 8.164821056010272e-05, + "loss": 1.2084, + "theoretical_loss": 3.3464247394605042, + "tokens_seen": 3033202688 + }, + { + "epoch": 0.84, + "learning_rate": 8.160808858931151e-05, + "loss": 1.2062, + "theoretical_loss": 3.346413740622219, + "tokens_seen": 3033333760 + }, + { + "epoch": 0.84, + "learning_rate": 8.156796661852029e-05, + "loss": 1.2389, + "theoretical_loss": 3.3464027423922578, + "tokens_seen": 3033464832 + }, + { + "epoch": 0.84, + "learning_rate": 8.15278446477291e-05, + "loss": 1.2159, + "theoretical_loss": 3.3463917447705596, + "tokens_seen": 3033595904 + }, + { + "epoch": 0.84, + "learning_rate": 8.148772267693789e-05, + "loss": 1.2698, + "theoretical_loss": 3.346380747757065, + "tokens_seen": 3033726976 + }, + { + "epoch": 0.84, + "learning_rate": 8.14476007061467e-05, + "loss": 1.2763, + "theoretical_loss": 3.3463697513517143, + "tokens_seen": 3033858048 + }, + { + "epoch": 0.84, + "learning_rate": 8.140747873535548e-05, + "loss": 1.2514, + "theoretical_loss": 3.346358755554447, + "tokens_seen": 3033989120 + }, + { + "epoch": 0.84, + "learning_rate": 8.136735676456427e-05, + "loss": 1.2953, + "theoretical_loss": 3.3463477603652034, + "tokens_seen": 3034120192 + }, + { + "epoch": 0.84, + "learning_rate": 8.132723479377308e-05, + "loss": 1.2692, + "theoretical_loss": 3.3463367657839242, + "tokens_seen": 3034251264 + }, + { + "epoch": 0.84, + "learning_rate": 8.128711282298187e-05, + "loss": 1.218, + "theoretical_loss": 3.3463257718105486, + "tokens_seen": 3034382336 + }, + { + "epoch": 0.84, + "learning_rate": 8.124699085219066e-05, + "loss": 1.2964, + "theoretical_loss": 3.3463147784450173, + "tokens_seen": 3034513408 + }, + { + "epoch": 0.84, + "learning_rate": 8.120686888139945e-05, + "loss": 1.1857, + "theoretical_loss": 3.3463037856872706, + "tokens_seen": 3034644480 + }, + { + "epoch": 0.84, + "learning_rate": 8.116674691060826e-05, + "loss": 1.3149, + "theoretical_loss": 3.3462927935372484, + "tokens_seen": 3034775552 + }, + { + "epoch": 0.84, + "learning_rate": 8.112662493981704e-05, + "loss": 1.1826, + "theoretical_loss": 3.3462818019948903, + "tokens_seen": 3034906624 + }, + { + "epoch": 0.84, + "learning_rate": 8.108650296902583e-05, + "loss": 1.262, + "theoretical_loss": 3.3462708110601374, + "tokens_seen": 3035037696 + }, + { + "epoch": 0.84, + "learning_rate": 8.104638099823464e-05, + "loss": 1.2393, + "theoretical_loss": 3.3462598207329295, + "tokens_seen": 3035168768 + }, + { + "epoch": 0.84, + "learning_rate": 8.100625902744343e-05, + "loss": 1.2918, + "theoretical_loss": 3.346248831013207, + "tokens_seen": 3035299840 + }, + { + "epoch": 0.84, + "learning_rate": 8.096613705665222e-05, + "loss": 1.2537, + "theoretical_loss": 3.3462378419009093, + "tokens_seen": 3035430912 + }, + { + "epoch": 0.84, + "learning_rate": 8.092601508586102e-05, + "loss": 1.2685, + "theoretical_loss": 3.346226853395977, + "tokens_seen": 3035561984 + }, + { + "epoch": 0.84, + "learning_rate": 8.088589311506981e-05, + "loss": 1.2538, + "theoretical_loss": 3.346215865498351, + "tokens_seen": 3035693056 + }, + { + "epoch": 0.84, + "learning_rate": 8.084577114427862e-05, + "loss": 1.2742, + "theoretical_loss": 3.346204878207971, + "tokens_seen": 3035824128 + }, + { + "epoch": 0.84, + "learning_rate": 8.08056491734874e-05, + "loss": 1.3393, + "theoretical_loss": 3.346193891524777, + "tokens_seen": 3035955200 + }, + { + "epoch": 0.84, + "learning_rate": 8.07655272026962e-05, + "loss": 1.288, + "theoretical_loss": 3.346182905448709, + "tokens_seen": 3036086272 + }, + { + "epoch": 0.84, + "learning_rate": 8.0725405231905e-05, + "loss": 1.3142, + "theoretical_loss": 3.3461719199797084, + "tokens_seen": 3036217344 + }, + { + "epoch": 0.84, + "learning_rate": 8.068528326111379e-05, + "loss": 1.2027, + "theoretical_loss": 3.346160935117714, + "tokens_seen": 3036348416 + }, + { + "epoch": 0.84, + "learning_rate": 8.064516129032258e-05, + "loss": 1.3047, + "theoretical_loss": 3.3461499508626673, + "tokens_seen": 3036479488 + }, + { + "epoch": 0.84, + "learning_rate": 8.060503931953137e-05, + "loss": 1.2898, + "theoretical_loss": 3.3461389672145083, + "tokens_seen": 3036610560 + }, + { + "epoch": 0.84, + "learning_rate": 8.056491734874018e-05, + "loss": 1.3202, + "theoretical_loss": 3.3461279841731764, + "tokens_seen": 3036741632 + }, + { + "epoch": 0.84, + "learning_rate": 8.052479537794896e-05, + "loss": 1.2519, + "theoretical_loss": 3.346117001738613, + "tokens_seen": 3036872704 + }, + { + "epoch": 0.84, + "learning_rate": 8.048467340715777e-05, + "loss": 1.1882, + "theoretical_loss": 3.3461060199107573, + "tokens_seen": 3037003776 + }, + { + "epoch": 0.84, + "learning_rate": 8.044455143636656e-05, + "loss": 1.2855, + "theoretical_loss": 3.3460950386895507, + "tokens_seen": 3037134848 + }, + { + "epoch": 0.84, + "learning_rate": 8.040442946557535e-05, + "loss": 1.3046, + "theoretical_loss": 3.3460840580749327, + "tokens_seen": 3037265920 + }, + { + "epoch": 0.84, + "learning_rate": 8.036430749478414e-05, + "loss": 1.2426, + "theoretical_loss": 3.346073078066844, + "tokens_seen": 3037396992 + }, + { + "epoch": 0.84, + "learning_rate": 8.032418552399294e-05, + "loss": 1.219, + "theoretical_loss": 3.346062098665225, + "tokens_seen": 3037528064 + }, + { + "epoch": 0.84, + "learning_rate": 8.028406355320174e-05, + "loss": 1.2783, + "theoretical_loss": 3.3460511198700162, + "tokens_seen": 3037659136 + }, + { + "epoch": 0.84, + "learning_rate": 8.024394158241052e-05, + "loss": 1.2926, + "theoretical_loss": 3.346040141681157, + "tokens_seen": 3037790208 + }, + { + "epoch": 0.84, + "learning_rate": 8.020381961161933e-05, + "loss": 1.2481, + "theoretical_loss": 3.3460291640985886, + "tokens_seen": 3037921280 + }, + { + "epoch": 0.84, + "learning_rate": 8.016369764082812e-05, + "loss": 1.2455, + "theoretical_loss": 3.3460181871222514, + "tokens_seen": 3038052352 + }, + { + "epoch": 0.84, + "learning_rate": 8.012357567003691e-05, + "loss": 1.2551, + "theoretical_loss": 3.3460072107520853, + "tokens_seen": 3038183424 + }, + { + "epoch": 0.84, + "learning_rate": 8.00834536992457e-05, + "loss": 1.2371, + "theoretical_loss": 3.345996234988031, + "tokens_seen": 3038314496 + }, + { + "epoch": 0.84, + "learning_rate": 8.00433317284545e-05, + "loss": 1.2088, + "theoretical_loss": 3.3459852598300284, + "tokens_seen": 3038445568 + }, + { + "epoch": 0.84, + "learning_rate": 8.00032097576633e-05, + "loss": 1.1538, + "theoretical_loss": 3.3459742852780185, + "tokens_seen": 3038576640 + }, + { + "epoch": 0.84, + "learning_rate": 7.99630877868721e-05, + "loss": 1.2406, + "theoretical_loss": 3.345963311331942, + "tokens_seen": 3038707712 + }, + { + "epoch": 0.84, + "learning_rate": 7.992296581608088e-05, + "loss": 1.2269, + "theoretical_loss": 3.345952337991738, + "tokens_seen": 3038838784 + }, + { + "epoch": 0.84, + "learning_rate": 7.988284384528968e-05, + "loss": 1.2379, + "theoretical_loss": 3.345941365257348, + "tokens_seen": 3038969856 + }, + { + "epoch": 0.84, + "learning_rate": 7.984272187449848e-05, + "loss": 1.1982, + "theoretical_loss": 3.345930393128712, + "tokens_seen": 3039100928 + }, + { + "epoch": 0.84, + "learning_rate": 7.980259990370727e-05, + "loss": 1.2975, + "theoretical_loss": 3.345919421605771, + "tokens_seen": 3039232000 + }, + { + "epoch": 0.84, + "objective/train/aligned_token_count": 0.9932354092597961, + "objective/train/avg_token_score": 0.006764549762010574, + "objective/train/docs_used": 1104383, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1933152675628662, + "objective/train/original_loss": 1.2015148401260376, + "objective/train/theoretical_loss": 3.3459084506884644, + "objective/train/tokens_used": 1409822176, + "objective/train/unlikelihood_loss": 0.021154677495360374, + "theoretical_loss": 3.3459084506884644, + "tokens_seen": 3039363072 + }, + { + "epoch": 0.84, + "learning_rate": 7.976247793291606e-05, + "loss": 1.1986, + "theoretical_loss": 3.3459084506884644, + "tokens_seen": 3039363072 + }, + { + "epoch": 0.84, + "learning_rate": 7.972235596212487e-05, + "loss": 1.2284, + "theoretical_loss": 3.345897480376734, + "tokens_seen": 3039494144 + }, + { + "epoch": 0.84, + "learning_rate": 7.968223399133366e-05, + "loss": 1.3054, + "theoretical_loss": 3.345886510670519, + "tokens_seen": 3039625216 + }, + { + "epoch": 0.84, + "learning_rate": 7.964211202054244e-05, + "loss": 1.2319, + "theoretical_loss": 3.34587554156976, + "tokens_seen": 3039756288 + }, + { + "epoch": 0.84, + "learning_rate": 7.960199004975125e-05, + "loss": 1.2838, + "theoretical_loss": 3.3458645730743988, + "tokens_seen": 3039887360 + }, + { + "epoch": 0.84, + "learning_rate": 7.956186807896004e-05, + "loss": 1.2116, + "theoretical_loss": 3.345853605184374, + "tokens_seen": 3040018432 + }, + { + "epoch": 0.84, + "learning_rate": 7.952174610816885e-05, + "loss": 1.1892, + "theoretical_loss": 3.345842637899628, + "tokens_seen": 3040149504 + }, + { + "epoch": 0.84, + "learning_rate": 7.948162413737762e-05, + "loss": 1.2659, + "theoretical_loss": 3.3458316712201, + "tokens_seen": 3040280576 + }, + { + "epoch": 0.84, + "learning_rate": 7.944150216658642e-05, + "loss": 1.1902, + "theoretical_loss": 3.3458207051457305, + "tokens_seen": 3040411648 + }, + { + "epoch": 0.84, + "learning_rate": 7.940138019579522e-05, + "loss": 1.2266, + "theoretical_loss": 3.3458097396764606, + "tokens_seen": 3040542720 + }, + { + "epoch": 0.84, + "learning_rate": 7.9361258225004e-05, + "loss": 1.2428, + "theoretical_loss": 3.3457987748122306, + "tokens_seen": 3040673792 + }, + { + "epoch": 0.84, + "learning_rate": 7.932113625421281e-05, + "loss": 1.2969, + "theoretical_loss": 3.3457878105529812, + "tokens_seen": 3040804864 + }, + { + "epoch": 0.84, + "learning_rate": 7.92810142834216e-05, + "loss": 1.251, + "theoretical_loss": 3.345776846898653, + "tokens_seen": 3040935936 + }, + { + "epoch": 0.84, + "learning_rate": 7.924089231263041e-05, + "loss": 1.2063, + "theoretical_loss": 3.345765883849186, + "tokens_seen": 3041067008 + }, + { + "epoch": 0.84, + "learning_rate": 7.920077034183919e-05, + "loss": 1.3191, + "theoretical_loss": 3.345754921404521, + "tokens_seen": 3041198080 + }, + { + "epoch": 0.84, + "learning_rate": 7.916064837104798e-05, + "loss": 1.2734, + "theoretical_loss": 3.345743959564599, + "tokens_seen": 3041329152 + }, + { + "epoch": 0.84, + "learning_rate": 7.912052640025679e-05, + "loss": 1.2091, + "theoretical_loss": 3.34573299832936, + "tokens_seen": 3041460224 + }, + { + "epoch": 0.84, + "learning_rate": 7.908040442946558e-05, + "loss": 1.2547, + "theoretical_loss": 3.345722037698745, + "tokens_seen": 3041591296 + }, + { + "epoch": 0.84, + "learning_rate": 7.904028245867437e-05, + "loss": 1.2616, + "theoretical_loss": 3.3457110776726946, + "tokens_seen": 3041722368 + }, + { + "epoch": 0.84, + "learning_rate": 7.900016048788316e-05, + "loss": 1.3388, + "theoretical_loss": 3.345700118251149, + "tokens_seen": 3041853440 + }, + { + "epoch": 0.84, + "learning_rate": 7.896003851709197e-05, + "loss": 1.2137, + "theoretical_loss": 3.345689159434049, + "tokens_seen": 3041984512 + }, + { + "epoch": 0.84, + "learning_rate": 7.891991654630075e-05, + "loss": 1.2051, + "theoretical_loss": 3.345678201221336, + "tokens_seen": 3042115584 + }, + { + "epoch": 0.84, + "learning_rate": 7.887979457550954e-05, + "loss": 1.4055, + "theoretical_loss": 3.345667243612949, + "tokens_seen": 3042246656 + }, + { + "epoch": 0.84, + "learning_rate": 7.883967260471835e-05, + "loss": 1.233, + "theoretical_loss": 3.34565628660883, + "tokens_seen": 3042377728 + }, + { + "epoch": 0.84, + "learning_rate": 7.879955063392714e-05, + "loss": 1.2816, + "theoretical_loss": 3.345645330208919, + "tokens_seen": 3042508800 + }, + { + "epoch": 0.84, + "learning_rate": 7.875942866313593e-05, + "loss": 1.2879, + "theoretical_loss": 3.345634374413157, + "tokens_seen": 3042639872 + }, + { + "epoch": 0.84, + "learning_rate": 7.871930669234473e-05, + "loss": 1.1583, + "theoretical_loss": 3.3456234192214844, + "tokens_seen": 3042770944 + }, + { + "epoch": 0.84, + "learning_rate": 7.867918472155352e-05, + "loss": 1.349, + "theoretical_loss": 3.345612464633842, + "tokens_seen": 3042902016 + }, + { + "epoch": 0.84, + "learning_rate": 7.863906275076233e-05, + "loss": 1.256, + "theoretical_loss": 3.345601510650171, + "tokens_seen": 3043033088 + }, + { + "epoch": 0.84, + "learning_rate": 7.85989407799711e-05, + "loss": 1.3195, + "theoretical_loss": 3.3455905572704108, + "tokens_seen": 3043164160 + }, + { + "epoch": 0.84, + "learning_rate": 7.855881880917991e-05, + "loss": 1.1572, + "theoretical_loss": 3.345579604494503, + "tokens_seen": 3043295232 + }, + { + "epoch": 0.84, + "learning_rate": 7.85186968383887e-05, + "loss": 1.231, + "theoretical_loss": 3.345568652322388, + "tokens_seen": 3043426304 + }, + { + "epoch": 0.84, + "learning_rate": 7.847857486759751e-05, + "loss": 1.317, + "theoretical_loss": 3.345557700754007, + "tokens_seen": 3043557376 + }, + { + "epoch": 0.84, + "learning_rate": 7.843845289680629e-05, + "loss": 1.2565, + "theoretical_loss": 3.3455467497893006, + "tokens_seen": 3043688448 + }, + { + "epoch": 0.84, + "learning_rate": 7.839833092601508e-05, + "loss": 1.2202, + "theoretical_loss": 3.345535799428209, + "tokens_seen": 3043819520 + }, + { + "epoch": 0.84, + "learning_rate": 7.835820895522389e-05, + "loss": 1.2756, + "theoretical_loss": 3.3455248496706735, + "tokens_seen": 3043950592 + }, + { + "epoch": 0.84, + "learning_rate": 7.831808698443267e-05, + "loss": 1.2694, + "theoretical_loss": 3.345513900516634, + "tokens_seen": 3044081664 + }, + { + "epoch": 0.85, + "learning_rate": 7.827796501364148e-05, + "loss": 1.2117, + "theoretical_loss": 3.345502951966033, + "tokens_seen": 3044212736 + }, + { + "epoch": 0.85, + "learning_rate": 7.823784304285027e-05, + "loss": 1.2565, + "theoretical_loss": 3.345492004018809, + "tokens_seen": 3044343808 + }, + { + "epoch": 0.85, + "learning_rate": 7.819772107205906e-05, + "loss": 1.2742, + "theoretical_loss": 3.3454810566749043, + "tokens_seen": 3044474880 + }, + { + "epoch": 0.85, + "learning_rate": 7.815759910126785e-05, + "loss": 1.2927, + "theoretical_loss": 3.3454701099342596, + "tokens_seen": 3044605952 + }, + { + "epoch": 0.85, + "learning_rate": 7.811747713047665e-05, + "loss": 1.2141, + "theoretical_loss": 3.3454591637968147, + "tokens_seen": 3044737024 + }, + { + "epoch": 0.85, + "learning_rate": 7.807735515968545e-05, + "loss": 1.2547, + "theoretical_loss": 3.3454482182625114, + "tokens_seen": 3044868096 + }, + { + "epoch": 0.85, + "learning_rate": 7.803723318889425e-05, + "loss": 1.2753, + "theoretical_loss": 3.3454372733312905, + "tokens_seen": 3044999168 + }, + { + "epoch": 0.85, + "learning_rate": 7.799711121810304e-05, + "loss": 1.3119, + "theoretical_loss": 3.345426329003092, + "tokens_seen": 3045130240 + }, + { + "epoch": 0.85, + "learning_rate": 7.795698924731183e-05, + "loss": 1.2471, + "theoretical_loss": 3.345415385277857, + "tokens_seen": 3045261312 + }, + { + "epoch": 0.85, + "learning_rate": 7.791686727652062e-05, + "loss": 1.189, + "theoretical_loss": 3.345404442155527, + "tokens_seen": 3045392384 + }, + { + "epoch": 0.85, + "learning_rate": 7.787674530572942e-05, + "loss": 1.1375, + "theoretical_loss": 3.3453934996360424, + "tokens_seen": 3045523456 + }, + { + "epoch": 0.85, + "learning_rate": 7.783662333493821e-05, + "loss": 1.2954, + "theoretical_loss": 3.345382557719344, + "tokens_seen": 3045654528 + }, + { + "epoch": 0.85, + "learning_rate": 7.779650136414702e-05, + "loss": 1.241, + "theoretical_loss": 3.345371616405372, + "tokens_seen": 3045785600 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.9917280673980713, + "objective/train/avg_token_score": 0.008271921426057816, + "objective/train/docs_used": 1106703, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2165675163269043, + "objective/train/original_loss": 1.228397250175476, + "objective/train/theoretical_loss": 3.345360675694069, + "objective/train/tokens_used": 1416375776, + "objective/train/unlikelihood_loss": 0.02200232818722725, + "theoretical_loss": 3.345360675694069, + "tokens_seen": 3045916672 + }, + { + "epoch": 0.85, + "learning_rate": 7.775637939335581e-05, + "loss": 1.2671, + "theoretical_loss": 3.345360675694069, + "tokens_seen": 3045916672 + }, + { + "epoch": 0.85, + "learning_rate": 7.771625742256459e-05, + "loss": 1.2645, + "theoretical_loss": 3.3453497355853745, + "tokens_seen": 3046047744 + }, + { + "epoch": 0.85, + "learning_rate": 7.76761354517734e-05, + "loss": 1.2512, + "theoretical_loss": 3.345338796079229, + "tokens_seen": 3046178816 + }, + { + "epoch": 0.85, + "learning_rate": 7.763601348098219e-05, + "loss": 1.2439, + "theoretical_loss": 3.345327857175575, + "tokens_seen": 3046309888 + }, + { + "epoch": 0.85, + "learning_rate": 7.759589151019099e-05, + "loss": 1.2354, + "theoretical_loss": 3.345316918874352, + "tokens_seen": 3046440960 + }, + { + "epoch": 0.85, + "learning_rate": 7.755576953939977e-05, + "loss": 1.2479, + "theoretical_loss": 3.3453059811755015, + "tokens_seen": 3046572032 + }, + { + "epoch": 0.85, + "learning_rate": 7.751564756860858e-05, + "loss": 1.255, + "theoretical_loss": 3.3452950440789637, + "tokens_seen": 3046703104 + }, + { + "epoch": 0.85, + "learning_rate": 7.747552559781737e-05, + "loss": 1.2512, + "theoretical_loss": 3.345284107584681, + "tokens_seen": 3046834176 + }, + { + "epoch": 0.85, + "learning_rate": 7.743540362702615e-05, + "loss": 1.2888, + "theoretical_loss": 3.3452731716925928, + "tokens_seen": 3046965248 + }, + { + "epoch": 0.85, + "learning_rate": 7.739528165623496e-05, + "loss": 1.2313, + "theoretical_loss": 3.345262236402641, + "tokens_seen": 3047096320 + }, + { + "epoch": 0.85, + "learning_rate": 7.735515968544375e-05, + "loss": 1.2848, + "theoretical_loss": 3.3452513017147663, + "tokens_seen": 3047227392 + }, + { + "epoch": 0.85, + "learning_rate": 7.731503771465256e-05, + "loss": 1.292, + "theoretical_loss": 3.3452403676289095, + "tokens_seen": 3047358464 + }, + { + "epoch": 0.85, + "learning_rate": 7.727491574386133e-05, + "loss": 1.2458, + "theoretical_loss": 3.3452294341450117, + "tokens_seen": 3047489536 + }, + { + "epoch": 0.85, + "learning_rate": 7.723479377307013e-05, + "loss": 1.2535, + "theoretical_loss": 3.345218501263014, + "tokens_seen": 3047620608 + }, + { + "epoch": 0.85, + "learning_rate": 7.719467180227893e-05, + "loss": 1.2789, + "theoretical_loss": 3.3452075689828566, + "tokens_seen": 3047751680 + }, + { + "epoch": 0.85, + "learning_rate": 7.715454983148773e-05, + "loss": 1.3149, + "theoretical_loss": 3.3451966373044817, + "tokens_seen": 3047882752 + }, + { + "epoch": 0.85, + "learning_rate": 7.711442786069652e-05, + "loss": 1.2818, + "theoretical_loss": 3.345185706227829, + "tokens_seen": 3048013824 + }, + { + "epoch": 0.85, + "learning_rate": 7.707430588990531e-05, + "loss": 1.2747, + "theoretical_loss": 3.3451747757528407, + "tokens_seen": 3048144896 + }, + { + "epoch": 0.85, + "learning_rate": 7.703418391911412e-05, + "loss": 1.2653, + "theoretical_loss": 3.3451638458794575, + "tokens_seen": 3048275968 + }, + { + "epoch": 0.85, + "learning_rate": 7.69940619483229e-05, + "loss": 1.215, + "theoretical_loss": 3.3451529166076197, + "tokens_seen": 3048407040 + }, + { + "epoch": 0.85, + "learning_rate": 7.695393997753169e-05, + "loss": 1.2203, + "theoretical_loss": 3.3451419879372692, + "tokens_seen": 3048538112 + }, + { + "epoch": 0.85, + "learning_rate": 7.69138180067405e-05, + "loss": 1.2521, + "theoretical_loss": 3.345131059868347, + "tokens_seen": 3048669184 + }, + { + "epoch": 0.85, + "learning_rate": 7.687369603594929e-05, + "loss": 1.2695, + "theoretical_loss": 3.345120132400793, + "tokens_seen": 3048800256 + }, + { + "epoch": 0.85, + "learning_rate": 7.683357406515808e-05, + "loss": 1.2684, + "theoretical_loss": 3.3451092055345497, + "tokens_seen": 3048931328 + }, + { + "epoch": 0.85, + "learning_rate": 7.679345209436687e-05, + "loss": 1.2357, + "theoretical_loss": 3.345098279269558, + "tokens_seen": 3049062400 + }, + { + "epoch": 0.85, + "learning_rate": 7.675333012357567e-05, + "loss": 1.1778, + "theoretical_loss": 3.3450873536057575, + "tokens_seen": 3049193472 + }, + { + "epoch": 0.85, + "learning_rate": 7.671320815278447e-05, + "loss": 1.2549, + "theoretical_loss": 3.345076428543091, + "tokens_seen": 3049324544 + }, + { + "epoch": 0.85, + "learning_rate": 7.667308618199325e-05, + "loss": 1.2692, + "theoretical_loss": 3.3450655040814987, + "tokens_seen": 3049455616 + }, + { + "epoch": 0.85, + "learning_rate": 7.663296421120206e-05, + "loss": 1.2672, + "theoretical_loss": 3.3450545802209217, + "tokens_seen": 3049586688 + }, + { + "epoch": 0.85, + "learning_rate": 7.659284224041085e-05, + "loss": 1.2205, + "theoretical_loss": 3.345043656961302, + "tokens_seen": 3049717760 + }, + { + "epoch": 0.85, + "learning_rate": 7.655272026961964e-05, + "loss": 1.2415, + "theoretical_loss": 3.3450327343025794, + "tokens_seen": 3049848832 + }, + { + "epoch": 0.85, + "learning_rate": 7.651259829882844e-05, + "loss": 1.208, + "theoretical_loss": 3.3450218122446955, + "tokens_seen": 3049979904 + }, + { + "epoch": 0.85, + "learning_rate": 7.647247632803723e-05, + "loss": 1.248, + "theoretical_loss": 3.345010890787592, + "tokens_seen": 3050110976 + }, + { + "epoch": 0.85, + "learning_rate": 7.643235435724604e-05, + "loss": 1.2111, + "theoretical_loss": 3.3449999699312096, + "tokens_seen": 3050242048 + }, + { + "epoch": 0.85, + "learning_rate": 7.639223238645482e-05, + "loss": 1.2749, + "theoretical_loss": 3.3449890496754895, + "tokens_seen": 3050373120 + }, + { + "epoch": 0.85, + "learning_rate": 7.635211041566362e-05, + "loss": 1.2273, + "theoretical_loss": 3.3449781300203725, + "tokens_seen": 3050504192 + }, + { + "epoch": 0.85, + "learning_rate": 7.631198844487241e-05, + "loss": 1.2829, + "theoretical_loss": 3.3449672109658, + "tokens_seen": 3050635264 + }, + { + "epoch": 0.85, + "learning_rate": 7.627186647408121e-05, + "loss": 1.2506, + "theoretical_loss": 3.3449562925117133, + "tokens_seen": 3050766336 + }, + { + "epoch": 0.85, + "learning_rate": 7.623174450329e-05, + "loss": 1.2657, + "theoretical_loss": 3.344945374658054, + "tokens_seen": 3050897408 + }, + { + "epoch": 0.85, + "learning_rate": 7.619162253249879e-05, + "loss": 1.1583, + "theoretical_loss": 3.3449344574047624, + "tokens_seen": 3051028480 + }, + { + "epoch": 0.85, + "learning_rate": 7.61515005617076e-05, + "loss": 1.2826, + "theoretical_loss": 3.3449235407517803, + "tokens_seen": 3051159552 + }, + { + "epoch": 0.85, + "learning_rate": 7.611137859091639e-05, + "loss": 1.2977, + "theoretical_loss": 3.344912624699049, + "tokens_seen": 3051290624 + }, + { + "epoch": 0.85, + "learning_rate": 7.607125662012518e-05, + "loss": 1.2158, + "theoretical_loss": 3.3449017092465088, + "tokens_seen": 3051421696 + }, + { + "epoch": 0.85, + "learning_rate": 7.603113464933398e-05, + "loss": 1.1363, + "theoretical_loss": 3.344890794394102, + "tokens_seen": 3051552768 + }, + { + "epoch": 0.85, + "learning_rate": 7.599101267854277e-05, + "loss": 1.2313, + "theoretical_loss": 3.344879880141769, + "tokens_seen": 3051683840 + }, + { + "epoch": 0.85, + "learning_rate": 7.595089070775156e-05, + "loss": 1.227, + "theoretical_loss": 3.3448689664894515, + "tokens_seen": 3051814912 + }, + { + "epoch": 0.85, + "learning_rate": 7.591076873696036e-05, + "loss": 1.256, + "theoretical_loss": 3.3448580534370906, + "tokens_seen": 3051945984 + }, + { + "epoch": 0.85, + "learning_rate": 7.587064676616916e-05, + "loss": 1.2908, + "theoretical_loss": 3.344847140984628, + "tokens_seen": 3052077056 + }, + { + "epoch": 0.85, + "learning_rate": 7.583052479537795e-05, + "loss": 1.2731, + "theoretical_loss": 3.3448362291320044, + "tokens_seen": 3052208128 + }, + { + "epoch": 0.85, + "learning_rate": 7.579040282458673e-05, + "loss": 1.1939, + "theoretical_loss": 3.3448253178791614, + "tokens_seen": 3052339200 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.9923360347747803, + "objective/train/avg_token_score": 0.007663935422897339, + "objective/train/docs_used": 1109048, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2499823570251465, + "objective/train/original_loss": 1.2624523639678955, + "objective/train/theoretical_loss": 3.34481440722604, + "objective/train/tokens_used": 1422929376, + "objective/train/unlikelihood_loss": 0.021119721233844757, + "theoretical_loss": 3.34481440722604, + "tokens_seen": 3052470272 + }, + { + "epoch": 0.85, + "learning_rate": 7.575028085379554e-05, + "loss": 1.2464, + "theoretical_loss": 3.34481440722604, + "tokens_seen": 3052470272 + }, + { + "epoch": 0.85, + "learning_rate": 7.571015888300433e-05, + "loss": 1.2489, + "theoretical_loss": 3.3448034971725815, + "tokens_seen": 3052601344 + }, + { + "epoch": 0.85, + "learning_rate": 7.567003691221314e-05, + "loss": 1.3119, + "theoretical_loss": 3.3447925877187275, + "tokens_seen": 3052732416 + }, + { + "epoch": 0.85, + "learning_rate": 7.562991494142192e-05, + "loss": 1.2529, + "theoretical_loss": 3.344781678864419, + "tokens_seen": 3052863488 + }, + { + "epoch": 0.85, + "learning_rate": 7.558979297063073e-05, + "loss": 1.1411, + "theoretical_loss": 3.3447707706095975, + "tokens_seen": 3052994560 + }, + { + "epoch": 0.85, + "learning_rate": 7.554967099983952e-05, + "loss": 1.2071, + "theoretical_loss": 3.344759862954204, + "tokens_seen": 3053125632 + }, + { + "epoch": 0.85, + "learning_rate": 7.55095490290483e-05, + "loss": 1.2793, + "theoretical_loss": 3.34474895589818, + "tokens_seen": 3053256704 + }, + { + "epoch": 0.85, + "learning_rate": 7.54694270582571e-05, + "loss": 1.2359, + "theoretical_loss": 3.3447380494414674, + "tokens_seen": 3053387776 + }, + { + "epoch": 0.85, + "learning_rate": 7.54293050874659e-05, + "loss": 1.2608, + "theoretical_loss": 3.344727143584007, + "tokens_seen": 3053518848 + }, + { + "epoch": 0.85, + "learning_rate": 7.53891831166747e-05, + "loss": 1.2915, + "theoretical_loss": 3.3447162383257405, + "tokens_seen": 3053649920 + }, + { + "epoch": 0.85, + "learning_rate": 7.534906114588348e-05, + "loss": 1.2035, + "theoretical_loss": 3.3447053336666084, + "tokens_seen": 3053780992 + }, + { + "epoch": 0.85, + "learning_rate": 7.530893917509227e-05, + "loss": 1.255, + "theoretical_loss": 3.3446944296065526, + "tokens_seen": 3053912064 + }, + { + "epoch": 0.85, + "learning_rate": 7.526881720430108e-05, + "loss": 1.239, + "theoretical_loss": 3.344683526145515, + "tokens_seen": 3054043136 + }, + { + "epoch": 0.85, + "learning_rate": 7.522869523350987e-05, + "loss": 1.1903, + "theoretical_loss": 3.344672623283436, + "tokens_seen": 3054174208 + }, + { + "epoch": 0.85, + "learning_rate": 7.518857326271867e-05, + "loss": 1.1952, + "theoretical_loss": 3.344661721020258, + "tokens_seen": 3054305280 + }, + { + "epoch": 0.85, + "learning_rate": 7.514845129192746e-05, + "loss": 1.2438, + "theoretical_loss": 3.3446508193559215, + "tokens_seen": 3054436352 + }, + { + "epoch": 0.85, + "learning_rate": 7.510832932113627e-05, + "loss": 1.1949, + "theoretical_loss": 3.3446399182903686, + "tokens_seen": 3054567424 + }, + { + "epoch": 0.85, + "learning_rate": 7.506820735034504e-05, + "loss": 1.2189, + "theoretical_loss": 3.3446290178235403, + "tokens_seen": 3054698496 + }, + { + "epoch": 0.85, + "learning_rate": 7.502808537955384e-05, + "loss": 1.1709, + "theoretical_loss": 3.344618117955378, + "tokens_seen": 3054829568 + }, + { + "epoch": 0.85, + "learning_rate": 7.498796340876264e-05, + "loss": 1.1783, + "theoretical_loss": 3.3446072186858236, + "tokens_seen": 3054960640 + }, + { + "epoch": 0.85, + "learning_rate": 7.494784143797144e-05, + "loss": 1.2129, + "theoretical_loss": 3.3445963200148183, + "tokens_seen": 3055091712 + }, + { + "epoch": 0.85, + "learning_rate": 7.490771946718023e-05, + "loss": 1.19, + "theoretical_loss": 3.3445854219423032, + "tokens_seen": 3055222784 + }, + { + "epoch": 0.85, + "learning_rate": 7.486759749638902e-05, + "loss": 1.2251, + "theoretical_loss": 3.34457452446822, + "tokens_seen": 3055353856 + }, + { + "epoch": 0.85, + "learning_rate": 7.482747552559781e-05, + "loss": 1.2622, + "theoretical_loss": 3.344563627592511, + "tokens_seen": 3055484928 + }, + { + "epoch": 0.85, + "learning_rate": 7.478735355480662e-05, + "loss": 1.2416, + "theoretical_loss": 3.344552731315116, + "tokens_seen": 3055616000 + }, + { + "epoch": 0.85, + "learning_rate": 7.47472315840154e-05, + "loss": 1.2843, + "theoretical_loss": 3.3445418356359777, + "tokens_seen": 3055747072 + }, + { + "epoch": 0.85, + "learning_rate": 7.47071096132242e-05, + "loss": 1.1518, + "theoretical_loss": 3.344530940555037, + "tokens_seen": 3055878144 + }, + { + "epoch": 0.85, + "learning_rate": 7.4666987642433e-05, + "loss": 1.2229, + "theoretical_loss": 3.3445200460722364, + "tokens_seen": 3056009216 + }, + { + "epoch": 0.85, + "learning_rate": 7.462686567164179e-05, + "loss": 1.2547, + "theoretical_loss": 3.3445091521875163, + "tokens_seen": 3056140288 + }, + { + "epoch": 0.85, + "learning_rate": 7.458674370085058e-05, + "loss": 1.2223, + "theoretical_loss": 3.3444982589008188, + "tokens_seen": 3056271360 + }, + { + "epoch": 0.85, + "learning_rate": 7.454662173005938e-05, + "loss": 1.2839, + "theoretical_loss": 3.344487366212085, + "tokens_seen": 3056402432 + }, + { + "epoch": 0.85, + "learning_rate": 7.450649975926818e-05, + "loss": 1.1953, + "theoretical_loss": 3.3444764741212567, + "tokens_seen": 3056533504 + }, + { + "epoch": 0.85, + "learning_rate": 7.446637778847696e-05, + "loss": 1.1813, + "theoretical_loss": 3.3444655826282754, + "tokens_seen": 3056664576 + }, + { + "epoch": 0.85, + "learning_rate": 7.442625581768577e-05, + "loss": 1.2221, + "theoretical_loss": 3.344454691733083, + "tokens_seen": 3056795648 + }, + { + "epoch": 0.85, + "learning_rate": 7.438613384689456e-05, + "loss": 1.2517, + "theoretical_loss": 3.3444438014356206, + "tokens_seen": 3056926720 + }, + { + "epoch": 0.85, + "learning_rate": 7.434601187610337e-05, + "loss": 1.2374, + "theoretical_loss": 3.3444329117358294, + "tokens_seen": 3057057792 + }, + { + "epoch": 0.85, + "learning_rate": 7.430588990531215e-05, + "loss": 1.2194, + "theoretical_loss": 3.344422022633652, + "tokens_seen": 3057188864 + }, + { + "epoch": 0.85, + "learning_rate": 7.426576793452094e-05, + "loss": 1.159, + "theoretical_loss": 3.3444111341290297, + "tokens_seen": 3057319936 + }, + { + "epoch": 0.85, + "learning_rate": 7.422564596372975e-05, + "loss": 1.2075, + "theoretical_loss": 3.344400246221903, + "tokens_seen": 3057451008 + }, + { + "epoch": 0.85, + "learning_rate": 7.418552399293853e-05, + "loss": 1.2651, + "theoretical_loss": 3.344389358912215, + "tokens_seen": 3057582080 + }, + { + "epoch": 0.85, + "learning_rate": 7.414540202214733e-05, + "loss": 1.1495, + "theoretical_loss": 3.3443784721999066, + "tokens_seen": 3057713152 + }, + { + "epoch": 0.85, + "learning_rate": 7.410528005135612e-05, + "loss": 1.1616, + "theoretical_loss": 3.3443675860849194, + "tokens_seen": 3057844224 + }, + { + "epoch": 0.85, + "learning_rate": 7.406515808056492e-05, + "loss": 1.2328, + "theoretical_loss": 3.344356700567195, + "tokens_seen": 3057975296 + }, + { + "epoch": 0.85, + "learning_rate": 7.402503610977371e-05, + "loss": 1.2138, + "theoretical_loss": 3.344345815646675, + "tokens_seen": 3058106368 + }, + { + "epoch": 0.85, + "learning_rate": 7.39849141389825e-05, + "loss": 1.1768, + "theoretical_loss": 3.344334931323302, + "tokens_seen": 3058237440 + }, + { + "epoch": 0.85, + "learning_rate": 7.394479216819131e-05, + "loss": 1.2023, + "theoretical_loss": 3.344324047597016, + "tokens_seen": 3058368512 + }, + { + "epoch": 0.85, + "learning_rate": 7.39046701974001e-05, + "loss": 1.2893, + "theoretical_loss": 3.34431316446776, + "tokens_seen": 3058499584 + }, + { + "epoch": 0.85, + "learning_rate": 7.38645482266089e-05, + "loss": 1.3198, + "theoretical_loss": 3.3443022819354744, + "tokens_seen": 3058630656 + }, + { + "epoch": 0.85, + "learning_rate": 7.382442625581769e-05, + "loss": 1.0898, + "theoretical_loss": 3.3442914000001025, + "tokens_seen": 3058761728 + }, + { + "epoch": 0.85, + "learning_rate": 7.378430428502648e-05, + "loss": 1.2374, + "theoretical_loss": 3.3442805186615843, + "tokens_seen": 3058892800 + }, + { + "epoch": 0.85, + "objective/train/aligned_token_count": 0.9947852492332458, + "objective/train/avg_token_score": 0.0052147796377539635, + "objective/train/docs_used": 1111330, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1514041423797607, + "objective/train/original_loss": 1.1570816040039062, + "objective/train/theoretical_loss": 3.344269637919863, + "objective/train/tokens_used": 1429482976, + "objective/train/unlikelihood_loss": 0.016995318233966827, + "theoretical_loss": 3.344269637919863, + "tokens_seen": 3059023872 + }, + { + "epoch": 0.85, + "learning_rate": 7.374418231423529e-05, + "loss": 1.1693, + "theoretical_loss": 3.344269637919863, + "tokens_seen": 3059023872 + }, + { + "epoch": 0.85, + "learning_rate": 7.370406034344407e-05, + "loss": 1.2672, + "theoretical_loss": 3.3442587577748792, + "tokens_seen": 3059154944 + }, + { + "epoch": 0.85, + "learning_rate": 7.366393837265287e-05, + "loss": 1.2281, + "theoretical_loss": 3.344247878226575, + "tokens_seen": 3059286016 + }, + { + "epoch": 0.85, + "learning_rate": 7.362381640186166e-05, + "loss": 1.2629, + "theoretical_loss": 3.3442369992748926, + "tokens_seen": 3059417088 + }, + { + "epoch": 0.85, + "learning_rate": 7.358369443107044e-05, + "loss": 1.1777, + "theoretical_loss": 3.344226120919773, + "tokens_seen": 3059548160 + }, + { + "epoch": 0.85, + "learning_rate": 7.354357246027925e-05, + "loss": 1.1931, + "theoretical_loss": 3.344215243161158, + "tokens_seen": 3059679232 + }, + { + "epoch": 0.85, + "learning_rate": 7.350345048948804e-05, + "loss": 1.1996, + "theoretical_loss": 3.3442043659989893, + "tokens_seen": 3059810304 + }, + { + "epoch": 0.85, + "learning_rate": 7.346332851869685e-05, + "loss": 1.2061, + "theoretical_loss": 3.3441934894332093, + "tokens_seen": 3059941376 + }, + { + "epoch": 0.85, + "learning_rate": 7.342320654790563e-05, + "loss": 1.2291, + "theoretical_loss": 3.344182613463759, + "tokens_seen": 3060072448 + }, + { + "epoch": 0.85, + "learning_rate": 7.338308457711443e-05, + "loss": 1.2966, + "theoretical_loss": 3.344171738090581, + "tokens_seen": 3060203520 + }, + { + "epoch": 0.85, + "learning_rate": 7.334296260632323e-05, + "loss": 1.1727, + "theoretical_loss": 3.344160863313616, + "tokens_seen": 3060334592 + }, + { + "epoch": 0.85, + "learning_rate": 7.330284063553202e-05, + "loss": 1.2422, + "theoretical_loss": 3.3441499891328066, + "tokens_seen": 3060465664 + }, + { + "epoch": 0.85, + "learning_rate": 7.326271866474081e-05, + "loss": 1.2436, + "theoretical_loss": 3.3441391155480944, + "tokens_seen": 3060596736 + }, + { + "epoch": 0.86, + "learning_rate": 7.32225966939496e-05, + "loss": 1.1828, + "theoretical_loss": 3.344128242559421, + "tokens_seen": 3060727808 + }, + { + "epoch": 0.86, + "learning_rate": 7.318247472315841e-05, + "loss": 1.2819, + "theoretical_loss": 3.344117370166728, + "tokens_seen": 3060858880 + }, + { + "epoch": 0.86, + "learning_rate": 7.314235275236719e-05, + "loss": 1.1808, + "theoretical_loss": 3.344106498369958, + "tokens_seen": 3060989952 + }, + { + "epoch": 0.86, + "learning_rate": 7.310223078157598e-05, + "loss": 1.2023, + "theoretical_loss": 3.3440956271690525, + "tokens_seen": 3061121024 + }, + { + "epoch": 0.86, + "learning_rate": 7.306210881078479e-05, + "loss": 1.2202, + "theoretical_loss": 3.344084756563953, + "tokens_seen": 3061252096 + }, + { + "epoch": 0.86, + "learning_rate": 7.302198683999358e-05, + "loss": 1.1075, + "theoretical_loss": 3.3440738865546016, + "tokens_seen": 3061383168 + }, + { + "epoch": 0.86, + "learning_rate": 7.298186486920238e-05, + "loss": 1.2676, + "theoretical_loss": 3.34406301714094, + "tokens_seen": 3061514240 + }, + { + "epoch": 0.86, + "learning_rate": 7.294174289841117e-05, + "loss": 1.2499, + "theoretical_loss": 3.34405214832291, + "tokens_seen": 3061645312 + }, + { + "epoch": 0.86, + "learning_rate": 7.290162092761998e-05, + "loss": 1.2298, + "theoretical_loss": 3.3440412801004538, + "tokens_seen": 3061776384 + }, + { + "epoch": 0.86, + "learning_rate": 7.286149895682877e-05, + "loss": 1.228, + "theoretical_loss": 3.344030412473513, + "tokens_seen": 3061907456 + }, + { + "epoch": 0.86, + "learning_rate": 7.282137698603755e-05, + "loss": 1.3238, + "theoretical_loss": 3.3440195454420296, + "tokens_seen": 3062038528 + }, + { + "epoch": 0.86, + "learning_rate": 7.278125501524635e-05, + "loss": 1.2372, + "theoretical_loss": 3.3440086790059453, + "tokens_seen": 3062169600 + }, + { + "epoch": 0.86, + "learning_rate": 7.274113304445515e-05, + "loss": 1.196, + "theoretical_loss": 3.343997813165202, + "tokens_seen": 3062300672 + }, + { + "epoch": 0.86, + "learning_rate": 7.270101107366394e-05, + "loss": 1.2975, + "theoretical_loss": 3.343986947919742, + "tokens_seen": 3062431744 + }, + { + "epoch": 0.86, + "learning_rate": 7.266088910287273e-05, + "loss": 1.2644, + "theoretical_loss": 3.343976083269507, + "tokens_seen": 3062562816 + }, + { + "epoch": 0.86, + "learning_rate": 7.262076713208152e-05, + "loss": 1.3584, + "theoretical_loss": 3.343965219214439, + "tokens_seen": 3062693888 + }, + { + "epoch": 0.86, + "learning_rate": 7.258064516129033e-05, + "loss": 1.2155, + "theoretical_loss": 3.3439543557544793, + "tokens_seen": 3062824960 + }, + { + "epoch": 0.86, + "learning_rate": 7.254052319049911e-05, + "loss": 1.2452, + "theoretical_loss": 3.343943492889571, + "tokens_seen": 3062956032 + }, + { + "epoch": 0.86, + "learning_rate": 7.250040121970792e-05, + "loss": 1.2033, + "theoretical_loss": 3.343932630619655, + "tokens_seen": 3063087104 + }, + { + "epoch": 0.86, + "learning_rate": 7.246027924891671e-05, + "loss": 1.2464, + "theoretical_loss": 3.343921768944673, + "tokens_seen": 3063218176 + }, + { + "epoch": 0.86, + "learning_rate": 7.242015727812552e-05, + "loss": 1.2672, + "theoretical_loss": 3.3439109078645686, + "tokens_seen": 3063349248 + }, + { + "epoch": 0.86, + "learning_rate": 7.23800353073343e-05, + "loss": 1.2935, + "theoretical_loss": 3.3439000473792824, + "tokens_seen": 3063480320 + }, + { + "epoch": 0.86, + "learning_rate": 7.233991333654309e-05, + "loss": 1.2436, + "theoretical_loss": 3.3438891874887564, + "tokens_seen": 3063611392 + }, + { + "epoch": 0.86, + "learning_rate": 7.22997913657519e-05, + "loss": 1.2744, + "theoretical_loss": 3.3438783281929334, + "tokens_seen": 3063742464 + }, + { + "epoch": 0.86, + "learning_rate": 7.225966939496067e-05, + "loss": 1.2259, + "theoretical_loss": 3.3438674694917543, + "tokens_seen": 3063873536 + }, + { + "epoch": 0.86, + "learning_rate": 7.221954742416948e-05, + "loss": 1.2289, + "theoretical_loss": 3.343856611385162, + "tokens_seen": 3064004608 + }, + { + "epoch": 0.86, + "learning_rate": 7.217942545337827e-05, + "loss": 1.3127, + "theoretical_loss": 3.3438457538730986, + "tokens_seen": 3064135680 + }, + { + "epoch": 0.86, + "learning_rate": 7.213930348258706e-05, + "loss": 1.259, + "theoretical_loss": 3.3438348969555056, + "tokens_seen": 3064266752 + }, + { + "epoch": 0.86, + "learning_rate": 7.209918151179586e-05, + "loss": 1.1842, + "theoretical_loss": 3.3438240406323247, + "tokens_seen": 3064397824 + }, + { + "epoch": 0.86, + "learning_rate": 7.205905954100465e-05, + "loss": 1.1695, + "theoretical_loss": 3.3438131849034987, + "tokens_seen": 3064528896 + }, + { + "epoch": 0.86, + "learning_rate": 7.201893757021346e-05, + "loss": 1.2622, + "theoretical_loss": 3.3438023297689696, + "tokens_seen": 3064659968 + }, + { + "epoch": 0.86, + "learning_rate": 7.197881559942225e-05, + "loss": 1.1972, + "theoretical_loss": 3.343791475228679, + "tokens_seen": 3064791040 + }, + { + "epoch": 0.86, + "learning_rate": 7.193869362863104e-05, + "loss": 1.1955, + "theoretical_loss": 3.3437806212825687, + "tokens_seen": 3064922112 + }, + { + "epoch": 0.86, + "learning_rate": 7.189857165783983e-05, + "loss": 1.2608, + "theoretical_loss": 3.343769767930582, + "tokens_seen": 3065053184 + }, + { + "epoch": 0.86, + "learning_rate": 7.185844968704863e-05, + "loss": 1.2596, + "theoretical_loss": 3.3437589151726597, + "tokens_seen": 3065184256 + }, + { + "epoch": 0.86, + "learning_rate": 7.181832771625742e-05, + "loss": 1.1974, + "theoretical_loss": 3.343748063008744, + "tokens_seen": 3065315328 + }, + { + "epoch": 0.86, + "learning_rate": 7.177820574546621e-05, + "loss": 1.2296, + "theoretical_loss": 3.343737211438778, + "tokens_seen": 3065446400 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.9946680068969727, + "objective/train/avg_token_score": 0.005332026164978743, + "objective/train/docs_used": 1113675, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.218103051185608, + "objective/train/original_loss": 1.224928379058838, + "objective/train/theoretical_loss": 3.343726360462703, + "objective/train/tokens_used": 1436036576, + "objective/train/unlikelihood_loss": 0.016749244183301926, + "theoretical_loss": 3.343726360462703, + "tokens_seen": 3065577472 + }, + { + "epoch": 0.86, + "learning_rate": 7.173808377467502e-05, + "loss": 1.2498, + "theoretical_loss": 3.343726360462703, + "tokens_seen": 3065577472 + }, + { + "epoch": 0.86, + "learning_rate": 7.169796180388381e-05, + "loss": 1.229, + "theoretical_loss": 3.3437155100804614, + "tokens_seen": 3065708544 + }, + { + "epoch": 0.86, + "learning_rate": 7.165783983309259e-05, + "loss": 1.2545, + "theoretical_loss": 3.343704660291995, + "tokens_seen": 3065839616 + }, + { + "epoch": 0.86, + "learning_rate": 7.16177178623014e-05, + "loss": 1.2475, + "theoretical_loss": 3.3436938110972463, + "tokens_seen": 3065970688 + }, + { + "epoch": 0.86, + "learning_rate": 7.157759589151019e-05, + "loss": 1.3011, + "theoretical_loss": 3.343682962496157, + "tokens_seen": 3066101760 + }, + { + "epoch": 0.86, + "learning_rate": 7.1537473920719e-05, + "loss": 1.1888, + "theoretical_loss": 3.3436721144886694, + "tokens_seen": 3066232832 + }, + { + "epoch": 0.86, + "learning_rate": 7.149735194992778e-05, + "loss": 1.2395, + "theoretical_loss": 3.343661267074726, + "tokens_seen": 3066363904 + }, + { + "epoch": 0.86, + "learning_rate": 7.145722997913658e-05, + "loss": 1.2155, + "theoretical_loss": 3.3436504202542685, + "tokens_seen": 3066494976 + }, + { + "epoch": 0.86, + "learning_rate": 7.141710800834537e-05, + "loss": 1.1636, + "theoretical_loss": 3.343639574027239, + "tokens_seen": 3066626048 + }, + { + "epoch": 0.86, + "learning_rate": 7.137698603755417e-05, + "loss": 1.2432, + "theoretical_loss": 3.3436287283935804, + "tokens_seen": 3066757120 + }, + { + "epoch": 0.86, + "learning_rate": 7.133686406676296e-05, + "loss": 1.2289, + "theoretical_loss": 3.343617883353234, + "tokens_seen": 3066888192 + }, + { + "epoch": 0.86, + "learning_rate": 7.129674209597175e-05, + "loss": 1.2223, + "theoretical_loss": 3.3436070389061427, + "tokens_seen": 3067019264 + }, + { + "epoch": 0.86, + "learning_rate": 7.125662012518056e-05, + "loss": 1.264, + "theoretical_loss": 3.343596195052248, + "tokens_seen": 3067150336 + }, + { + "epoch": 0.86, + "learning_rate": 7.121649815438934e-05, + "loss": 1.2422, + "theoretical_loss": 3.343585351791493, + "tokens_seen": 3067281408 + }, + { + "epoch": 0.86, + "learning_rate": 7.117637618359813e-05, + "loss": 1.2503, + "theoretical_loss": 3.343574509123819, + "tokens_seen": 3067412480 + }, + { + "epoch": 0.86, + "learning_rate": 7.113625421280694e-05, + "loss": 1.2431, + "theoretical_loss": 3.3435636670491684, + "tokens_seen": 3067543552 + }, + { + "epoch": 0.86, + "learning_rate": 7.109613224201573e-05, + "loss": 1.2259, + "theoretical_loss": 3.3435528255674845, + "tokens_seen": 3067674624 + }, + { + "epoch": 0.86, + "learning_rate": 7.105601027122452e-05, + "loss": 1.1489, + "theoretical_loss": 3.3435419846787076, + "tokens_seen": 3067805696 + }, + { + "epoch": 0.86, + "learning_rate": 7.101588830043332e-05, + "loss": 1.1809, + "theoretical_loss": 3.3435311443827818, + "tokens_seen": 3067936768 + }, + { + "epoch": 0.86, + "learning_rate": 7.097576632964212e-05, + "loss": 1.2512, + "theoretical_loss": 3.3435203046796484, + "tokens_seen": 3068067840 + }, + { + "epoch": 0.86, + "learning_rate": 7.093564435885091e-05, + "loss": 1.2079, + "theoretical_loss": 3.34350946556925, + "tokens_seen": 3068198912 + }, + { + "epoch": 0.86, + "learning_rate": 7.08955223880597e-05, + "loss": 1.2055, + "theoretical_loss": 3.343498627051528, + "tokens_seen": 3068329984 + }, + { + "epoch": 0.86, + "learning_rate": 7.08554004172685e-05, + "loss": 1.3143, + "theoretical_loss": 3.343487789126426, + "tokens_seen": 3068461056 + }, + { + "epoch": 0.86, + "learning_rate": 7.081527844647729e-05, + "loss": 1.2529, + "theoretical_loss": 3.3434769517938854, + "tokens_seen": 3068592128 + }, + { + "epoch": 0.86, + "learning_rate": 7.077515647568609e-05, + "loss": 1.2532, + "theoretical_loss": 3.343466115053849, + "tokens_seen": 3068723200 + }, + { + "epoch": 0.86, + "learning_rate": 7.073503450489488e-05, + "loss": 1.1982, + "theoretical_loss": 3.3434552789062586, + "tokens_seen": 3068854272 + }, + { + "epoch": 0.86, + "learning_rate": 7.069491253410367e-05, + "loss": 1.1734, + "theoretical_loss": 3.343444443351057, + "tokens_seen": 3068985344 + }, + { + "epoch": 0.86, + "learning_rate": 7.065479056331248e-05, + "loss": 1.2484, + "theoretical_loss": 3.343433608388186, + "tokens_seen": 3069116416 + }, + { + "epoch": 0.86, + "learning_rate": 7.061466859252126e-05, + "loss": 1.2678, + "theoretical_loss": 3.3434227740175886, + "tokens_seen": 3069247488 + }, + { + "epoch": 0.86, + "learning_rate": 7.057454662173006e-05, + "loss": 1.2275, + "theoretical_loss": 3.3434119402392066, + "tokens_seen": 3069378560 + }, + { + "epoch": 0.86, + "learning_rate": 7.053442465093886e-05, + "loss": 1.244, + "theoretical_loss": 3.3434011070529825, + "tokens_seen": 3069509632 + }, + { + "epoch": 0.86, + "learning_rate": 7.049430268014766e-05, + "loss": 1.2133, + "theoretical_loss": 3.3433902744588586, + "tokens_seen": 3069640704 + }, + { + "epoch": 0.86, + "learning_rate": 7.045418070935644e-05, + "loss": 1.2872, + "theoretical_loss": 3.343379442456777, + "tokens_seen": 3069771776 + }, + { + "epoch": 0.86, + "learning_rate": 7.041405873856523e-05, + "loss": 1.2942, + "theoretical_loss": 3.3433686110466807, + "tokens_seen": 3069902848 + }, + { + "epoch": 0.86, + "learning_rate": 7.037393676777404e-05, + "loss": 1.2931, + "theoretical_loss": 3.3433577802285117, + "tokens_seen": 3070033920 + }, + { + "epoch": 0.86, + "learning_rate": 7.033381479698282e-05, + "loss": 1.2665, + "theoretical_loss": 3.3433469500022124, + "tokens_seen": 3070164992 + }, + { + "epoch": 0.86, + "learning_rate": 7.029369282619163e-05, + "loss": 1.2782, + "theoretical_loss": 3.343336120367725, + "tokens_seen": 3070296064 + }, + { + "epoch": 0.86, + "learning_rate": 7.025357085540042e-05, + "loss": 1.2268, + "theoretical_loss": 3.3433252913249927, + "tokens_seen": 3070427136 + }, + { + "epoch": 0.86, + "learning_rate": 7.021344888460923e-05, + "loss": 1.2527, + "theoretical_loss": 3.3433144628739564, + "tokens_seen": 3070558208 + }, + { + "epoch": 0.86, + "learning_rate": 7.0173326913818e-05, + "loss": 1.2443, + "theoretical_loss": 3.34330363501456, + "tokens_seen": 3070689280 + }, + { + "epoch": 0.86, + "learning_rate": 7.01332049430268e-05, + "loss": 1.2546, + "theoretical_loss": 3.3432928077467454, + "tokens_seen": 3070820352 + }, + { + "epoch": 0.86, + "learning_rate": 7.00930829722356e-05, + "loss": 1.2822, + "theoretical_loss": 3.343281981070455, + "tokens_seen": 3070951424 + }, + { + "epoch": 0.86, + "learning_rate": 7.00529610014444e-05, + "loss": 1.2321, + "theoretical_loss": 3.343271154985631, + "tokens_seen": 3071082496 + }, + { + "epoch": 0.86, + "learning_rate": 7.001283903065319e-05, + "loss": 1.2294, + "theoretical_loss": 3.343260329492216, + "tokens_seen": 3071213568 + }, + { + "epoch": 0.86, + "learning_rate": 6.997271705986198e-05, + "loss": 1.2451, + "theoretical_loss": 3.343249504590153, + "tokens_seen": 3071344640 + }, + { + "epoch": 0.86, + "learning_rate": 6.993259508907077e-05, + "loss": 1.2141, + "theoretical_loss": 3.3432386802793834, + "tokens_seen": 3071475712 + }, + { + "epoch": 0.86, + "learning_rate": 6.989247311827957e-05, + "loss": 1.2492, + "theoretical_loss": 3.3432278565598508, + "tokens_seen": 3071606784 + }, + { + "epoch": 0.86, + "learning_rate": 6.985235114748836e-05, + "loss": 1.3209, + "theoretical_loss": 3.343217033431497, + "tokens_seen": 3071737856 + }, + { + "epoch": 0.86, + "learning_rate": 6.981222917669717e-05, + "loss": 1.2505, + "theoretical_loss": 3.343206210894264, + "tokens_seen": 3071868928 + }, + { + "epoch": 0.86, + "learning_rate": 6.977210720590596e-05, + "loss": 1.295, + "theoretical_loss": 3.3431953889480956, + "tokens_seen": 3072000000 + }, + { + "epoch": 0.86, + "objective/train/aligned_token_count": 0.994194507598877, + "objective/train/avg_token_score": 0.005805479362607002, + "objective/train/docs_used": 1116127, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.198793888092041, + "objective/train/original_loss": 1.2065520286560059, + "objective/train/theoretical_loss": 3.3431845675929335, + "objective/train/tokens_used": 1442590176, + "objective/train/unlikelihood_loss": 0.01822221837937832, + "theoretical_loss": 3.3431845675929335, + "tokens_seen": 3072131072 + }, + { + "epoch": 0.86, + "learning_rate": 6.973198523511474e-05, + "loss": 1.2408, + "theoretical_loss": 3.3431845675929335, + "tokens_seen": 3072131072 + }, + { + "epoch": 0.86, + "learning_rate": 6.969186326432354e-05, + "loss": 1.2295, + "theoretical_loss": 3.34317374682872, + "tokens_seen": 3072262144 + }, + { + "epoch": 0.86, + "learning_rate": 6.965174129353234e-05, + "loss": 1.2643, + "theoretical_loss": 3.343162926655398, + "tokens_seen": 3072393216 + }, + { + "epoch": 0.86, + "learning_rate": 6.961161932274114e-05, + "loss": 1.2013, + "theoretical_loss": 3.3431521070729104, + "tokens_seen": 3072524288 + }, + { + "epoch": 0.86, + "learning_rate": 6.957149735194992e-05, + "loss": 1.2456, + "theoretical_loss": 3.343141288081199, + "tokens_seen": 3072655360 + }, + { + "epoch": 0.86, + "learning_rate": 6.953137538115873e-05, + "loss": 1.2565, + "theoretical_loss": 3.3431304696802067, + "tokens_seen": 3072786432 + }, + { + "epoch": 0.86, + "learning_rate": 6.949125341036752e-05, + "loss": 1.2653, + "theoretical_loss": 3.3431196518698756, + "tokens_seen": 3072917504 + }, + { + "epoch": 0.86, + "learning_rate": 6.94511314395763e-05, + "loss": 1.2171, + "theoretical_loss": 3.3431088346501494, + "tokens_seen": 3073048576 + }, + { + "epoch": 0.86, + "learning_rate": 6.941100946878511e-05, + "loss": 1.1835, + "theoretical_loss": 3.3430980180209695, + "tokens_seen": 3073179648 + }, + { + "epoch": 0.86, + "learning_rate": 6.93708874979939e-05, + "loss": 1.2698, + "theoretical_loss": 3.343087201982279, + "tokens_seen": 3073310720 + }, + { + "epoch": 0.86, + "learning_rate": 6.93307655272027e-05, + "loss": 1.1807, + "theoretical_loss": 3.3430763865340203, + "tokens_seen": 3073441792 + }, + { + "epoch": 0.86, + "learning_rate": 6.929064355641149e-05, + "loss": 1.2835, + "theoretical_loss": 3.343065571676136, + "tokens_seen": 3073572864 + }, + { + "epoch": 0.86, + "learning_rate": 6.925052158562029e-05, + "loss": 1.2924, + "theoretical_loss": 3.343054757408569, + "tokens_seen": 3073703936 + }, + { + "epoch": 0.86, + "learning_rate": 6.921039961482908e-05, + "loss": 1.1675, + "theoretical_loss": 3.343043943731262, + "tokens_seen": 3073835008 + }, + { + "epoch": 0.86, + "learning_rate": 6.917027764403788e-05, + "loss": 1.2409, + "theoretical_loss": 3.3430331306441565, + "tokens_seen": 3073966080 + }, + { + "epoch": 0.86, + "learning_rate": 6.913015567324667e-05, + "loss": 1.2522, + "theoretical_loss": 3.343022318147196, + "tokens_seen": 3074097152 + }, + { + "epoch": 0.86, + "learning_rate": 6.909003370245546e-05, + "loss": 1.2476, + "theoretical_loss": 3.3430115062403236, + "tokens_seen": 3074228224 + }, + { + "epoch": 0.86, + "learning_rate": 6.904991173166427e-05, + "loss": 1.1909, + "theoretical_loss": 3.3430006949234814, + "tokens_seen": 3074359296 + }, + { + "epoch": 0.86, + "learning_rate": 6.900978976087306e-05, + "loss": 1.2795, + "theoretical_loss": 3.3429898841966117, + "tokens_seen": 3074490368 + }, + { + "epoch": 0.86, + "learning_rate": 6.896966779008184e-05, + "loss": 1.2317, + "theoretical_loss": 3.3429790740596577, + "tokens_seen": 3074621440 + }, + { + "epoch": 0.86, + "learning_rate": 6.892954581929065e-05, + "loss": 1.2403, + "theoretical_loss": 3.3429682645125616, + "tokens_seen": 3074752512 + }, + { + "epoch": 0.86, + "learning_rate": 6.888942384849944e-05, + "loss": 1.1568, + "theoretical_loss": 3.3429574555552666, + "tokens_seen": 3074883584 + }, + { + "epoch": 0.86, + "learning_rate": 6.884930187770823e-05, + "loss": 1.2286, + "theoretical_loss": 3.342946647187715, + "tokens_seen": 3075014656 + }, + { + "epoch": 0.86, + "learning_rate": 6.880917990691703e-05, + "loss": 1.287, + "theoretical_loss": 3.3429358394098494, + "tokens_seen": 3075145728 + }, + { + "epoch": 0.86, + "learning_rate": 6.876905793612583e-05, + "loss": 1.2577, + "theoretical_loss": 3.342925032221613, + "tokens_seen": 3075276800 + }, + { + "epoch": 0.86, + "learning_rate": 6.872893596533462e-05, + "loss": 1.2393, + "theoretical_loss": 3.3429142256229483, + "tokens_seen": 3075407872 + }, + { + "epoch": 0.86, + "learning_rate": 6.86888139945434e-05, + "loss": 1.2482, + "theoretical_loss": 3.3429034196137977, + "tokens_seen": 3075538944 + }, + { + "epoch": 0.86, + "learning_rate": 6.864869202375221e-05, + "loss": 1.2806, + "theoretical_loss": 3.342892614194104, + "tokens_seen": 3075670016 + }, + { + "epoch": 0.86, + "learning_rate": 6.8608570052961e-05, + "loss": 1.2103, + "theoretical_loss": 3.34288180936381, + "tokens_seen": 3075801088 + }, + { + "epoch": 0.86, + "learning_rate": 6.856844808216981e-05, + "loss": 1.1578, + "theoretical_loss": 3.3428710051228587, + "tokens_seen": 3075932160 + }, + { + "epoch": 0.86, + "learning_rate": 6.852832611137859e-05, + "loss": 1.2255, + "theoretical_loss": 3.3428602014711926, + "tokens_seen": 3076063232 + }, + { + "epoch": 0.86, + "learning_rate": 6.848820414058738e-05, + "loss": 1.2302, + "theoretical_loss": 3.3428493984087546, + "tokens_seen": 3076194304 + }, + { + "epoch": 0.86, + "learning_rate": 6.844808216979619e-05, + "loss": 1.2868, + "theoretical_loss": 3.342838595935487, + "tokens_seen": 3076325376 + }, + { + "epoch": 0.86, + "learning_rate": 6.840796019900497e-05, + "loss": 1.2148, + "theoretical_loss": 3.3428277940513333, + "tokens_seen": 3076456448 + }, + { + "epoch": 0.86, + "learning_rate": 6.836783822821377e-05, + "loss": 1.2402, + "theoretical_loss": 3.342816992756236, + "tokens_seen": 3076587520 + }, + { + "epoch": 0.86, + "learning_rate": 6.832771625742257e-05, + "loss": 1.1565, + "theoretical_loss": 3.3428061920501375, + "tokens_seen": 3076718592 + }, + { + "epoch": 0.86, + "learning_rate": 6.828759428663137e-05, + "loss": 1.1882, + "theoretical_loss": 3.342795391932981, + "tokens_seen": 3076849664 + }, + { + "epoch": 0.86, + "learning_rate": 6.824747231584015e-05, + "loss": 1.2333, + "theoretical_loss": 3.342784592404709, + "tokens_seen": 3076980736 + }, + { + "epoch": 0.86, + "learning_rate": 6.820735034504894e-05, + "loss": 1.2138, + "theoretical_loss": 3.3427737934652644, + "tokens_seen": 3077111808 + }, + { + "epoch": 0.87, + "learning_rate": 6.816722837425775e-05, + "loss": 1.2675, + "theoretical_loss": 3.3427629951145903, + "tokens_seen": 3077242880 + }, + { + "epoch": 0.87, + "learning_rate": 6.812710640346654e-05, + "loss": 1.2352, + "theoretical_loss": 3.3427521973526293, + "tokens_seen": 3077373952 + }, + { + "epoch": 0.87, + "learning_rate": 6.808698443267534e-05, + "loss": 1.2619, + "theoretical_loss": 3.342741400179324, + "tokens_seen": 3077505024 + }, + { + "epoch": 0.87, + "learning_rate": 6.804686246188413e-05, + "loss": 1.2377, + "theoretical_loss": 3.342730603594618, + "tokens_seen": 3077636096 + }, + { + "epoch": 0.87, + "learning_rate": 6.800674049109292e-05, + "loss": 1.2709, + "theoretical_loss": 3.342719807598453, + "tokens_seen": 3077767168 + }, + { + "epoch": 0.87, + "learning_rate": 6.796661852030171e-05, + "loss": 1.2313, + "theoretical_loss": 3.342709012190773, + "tokens_seen": 3077898240 + }, + { + "epoch": 0.87, + "learning_rate": 6.792649654951051e-05, + "loss": 1.2675, + "theoretical_loss": 3.3426982173715203, + "tokens_seen": 3078029312 + }, + { + "epoch": 0.87, + "learning_rate": 6.788637457871931e-05, + "loss": 1.2089, + "theoretical_loss": 3.3426874231406374, + "tokens_seen": 3078160384 + }, + { + "epoch": 0.87, + "learning_rate": 6.78462526079281e-05, + "loss": 1.2448, + "theoretical_loss": 3.342676629498068, + "tokens_seen": 3078291456 + }, + { + "epoch": 0.87, + "learning_rate": 6.78061306371369e-05, + "loss": 1.1891, + "theoretical_loss": 3.3426658364437545, + "tokens_seen": 3078422528 + }, + { + "epoch": 0.87, + "learning_rate": 6.776600866634569e-05, + "loss": 1.2686, + "theoretical_loss": 3.34265504397764, + "tokens_seen": 3078553600 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.9918553829193115, + "objective/train/avg_token_score": 0.008144626393914223, + "objective/train/docs_used": 1118389, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.275383710861206, + "objective/train/original_loss": 1.2875779867172241, + "objective/train/theoretical_loss": 3.342644252099667, + "objective/train/tokens_used": 1449143776, + "objective/train/unlikelihood_loss": 0.022032946348190308, + "theoretical_loss": 3.342644252099667, + "tokens_seen": 3078684672 + }, + { + "epoch": 0.87, + "learning_rate": 6.772588669555448e-05, + "loss": 1.2715, + "theoretical_loss": 3.342644252099667, + "tokens_seen": 3078684672 + }, + { + "epoch": 0.87, + "learning_rate": 6.768576472476329e-05, + "loss": 1.2153, + "theoretical_loss": 3.3426334608097785, + "tokens_seen": 3078815744 + }, + { + "epoch": 0.87, + "learning_rate": 6.764564275397207e-05, + "loss": 1.1911, + "theoretical_loss": 3.342622670107918, + "tokens_seen": 3078946816 + }, + { + "epoch": 0.87, + "learning_rate": 6.760552078318088e-05, + "loss": 1.1524, + "theoretical_loss": 3.342611879994028, + "tokens_seen": 3079077888 + }, + { + "epoch": 0.87, + "learning_rate": 6.756539881238967e-05, + "loss": 1.2546, + "theoretical_loss": 3.3426010904680514, + "tokens_seen": 3079208960 + }, + { + "epoch": 0.87, + "learning_rate": 6.752527684159845e-05, + "loss": 1.2676, + "theoretical_loss": 3.342590301529931, + "tokens_seen": 3079340032 + }, + { + "epoch": 0.87, + "learning_rate": 6.748515487080725e-05, + "loss": 1.208, + "theoretical_loss": 3.3425795131796106, + "tokens_seen": 3079471104 + }, + { + "epoch": 0.87, + "learning_rate": 6.744503290001605e-05, + "loss": 1.26, + "theoretical_loss": 3.342568725417032, + "tokens_seen": 3079602176 + }, + { + "epoch": 0.87, + "learning_rate": 6.740491092922485e-05, + "loss": 1.2682, + "theoretical_loss": 3.3425579382421393, + "tokens_seen": 3079733248 + }, + { + "epoch": 0.87, + "learning_rate": 6.736478895843363e-05, + "loss": 1.2547, + "theoretical_loss": 3.342547151654874, + "tokens_seen": 3079864320 + }, + { + "epoch": 0.87, + "learning_rate": 6.732466698764244e-05, + "loss": 1.2512, + "theoretical_loss": 3.3425363656551808, + "tokens_seen": 3079995392 + }, + { + "epoch": 0.87, + "learning_rate": 6.728454501685123e-05, + "loss": 1.2357, + "theoretical_loss": 3.3425255802430014, + "tokens_seen": 3080126464 + }, + { + "epoch": 0.87, + "learning_rate": 6.724442304606002e-05, + "loss": 1.2848, + "theoretical_loss": 3.3425147954182792, + "tokens_seen": 3080257536 + }, + { + "epoch": 0.87, + "learning_rate": 6.720430107526882e-05, + "loss": 1.1845, + "theoretical_loss": 3.3425040111809574, + "tokens_seen": 3080388608 + }, + { + "epoch": 0.87, + "learning_rate": 6.716417910447761e-05, + "loss": 1.2598, + "theoretical_loss": 3.342493227530979, + "tokens_seen": 3080519680 + }, + { + "epoch": 0.87, + "learning_rate": 6.712405713368642e-05, + "loss": 1.2706, + "theoretical_loss": 3.3424824444682866, + "tokens_seen": 3080650752 + }, + { + "epoch": 0.87, + "learning_rate": 6.70839351628952e-05, + "loss": 1.2011, + "theoretical_loss": 3.342471661992824, + "tokens_seen": 3080781824 + }, + { + "epoch": 0.87, + "learning_rate": 6.704381319210399e-05, + "loss": 1.2378, + "theoretical_loss": 3.3424608801045332, + "tokens_seen": 3080912896 + }, + { + "epoch": 0.87, + "learning_rate": 6.70036912213128e-05, + "loss": 1.245, + "theoretical_loss": 3.342450098803358, + "tokens_seen": 3081043968 + }, + { + "epoch": 0.87, + "learning_rate": 6.696356925052159e-05, + "loss": 1.2902, + "theoretical_loss": 3.3424393180892413, + "tokens_seen": 3081175040 + }, + { + "epoch": 0.87, + "learning_rate": 6.692344727973038e-05, + "loss": 1.2411, + "theoretical_loss": 3.342428537962126, + "tokens_seen": 3081306112 + }, + { + "epoch": 0.87, + "learning_rate": 6.688332530893917e-05, + "loss": 1.2831, + "theoretical_loss": 3.3424177584219557, + "tokens_seen": 3081437184 + }, + { + "epoch": 0.87, + "learning_rate": 6.684320333814798e-05, + "loss": 1.2443, + "theoretical_loss": 3.342406979468673, + "tokens_seen": 3081568256 + }, + { + "epoch": 0.87, + "learning_rate": 6.680308136735677e-05, + "loss": 1.3155, + "theoretical_loss": 3.3423962011022206, + "tokens_seen": 3081699328 + }, + { + "epoch": 0.87, + "learning_rate": 6.676295939656555e-05, + "loss": 1.2022, + "theoretical_loss": 3.342385423322542, + "tokens_seen": 3081830400 + }, + { + "epoch": 0.87, + "learning_rate": 6.672283742577436e-05, + "loss": 1.2054, + "theoretical_loss": 3.342374646129581, + "tokens_seen": 3081961472 + }, + { + "epoch": 0.87, + "learning_rate": 6.668271545498315e-05, + "loss": 1.1454, + "theoretical_loss": 3.34236386952328, + "tokens_seen": 3082092544 + }, + { + "epoch": 0.87, + "learning_rate": 6.664259348419196e-05, + "loss": 1.2199, + "theoretical_loss": 3.3423530935035815, + "tokens_seen": 3082223616 + }, + { + "epoch": 0.87, + "learning_rate": 6.660247151340074e-05, + "loss": 1.1969, + "theoretical_loss": 3.34234231807043, + "tokens_seen": 3082354688 + }, + { + "epoch": 0.87, + "learning_rate": 6.656234954260953e-05, + "loss": 1.2366, + "theoretical_loss": 3.3423315432237675, + "tokens_seen": 3082485760 + }, + { + "epoch": 0.87, + "learning_rate": 6.652222757181833e-05, + "loss": 1.2704, + "theoretical_loss": 3.342320768963538, + "tokens_seen": 3082616832 + }, + { + "epoch": 0.87, + "learning_rate": 6.648210560102711e-05, + "loss": 1.2209, + "theoretical_loss": 3.3423099952896838, + "tokens_seen": 3082747904 + }, + { + "epoch": 0.87, + "learning_rate": 6.644198363023592e-05, + "loss": 1.2355, + "theoretical_loss": 3.342299222202149, + "tokens_seen": 3082878976 + }, + { + "epoch": 0.87, + "learning_rate": 6.640186165944471e-05, + "loss": 1.1892, + "theoretical_loss": 3.3422884497008756, + "tokens_seen": 3083010048 + }, + { + "epoch": 0.87, + "learning_rate": 6.636173968865352e-05, + "loss": 1.254, + "theoretical_loss": 3.342277677785808, + "tokens_seen": 3083141120 + }, + { + "epoch": 0.87, + "learning_rate": 6.63216177178623e-05, + "loss": 1.2289, + "theoretical_loss": 3.3422669064568886, + "tokens_seen": 3083272192 + }, + { + "epoch": 0.87, + "learning_rate": 6.628149574707109e-05, + "loss": 1.2349, + "theoretical_loss": 3.3422561357140608, + "tokens_seen": 3083403264 + }, + { + "epoch": 0.87, + "learning_rate": 6.62413737762799e-05, + "loss": 1.2316, + "theoretical_loss": 3.342245365557268, + "tokens_seen": 3083534336 + }, + { + "epoch": 0.87, + "learning_rate": 6.620125180548869e-05, + "loss": 1.257, + "theoretical_loss": 3.342234595986453, + "tokens_seen": 3083665408 + }, + { + "epoch": 0.87, + "learning_rate": 6.616112983469748e-05, + "loss": 1.2127, + "theoretical_loss": 3.3422238270015594, + "tokens_seen": 3083796480 + }, + { + "epoch": 0.87, + "learning_rate": 6.612100786390628e-05, + "loss": 1.1813, + "theoretical_loss": 3.34221305860253, + "tokens_seen": 3083927552 + }, + { + "epoch": 0.87, + "learning_rate": 6.608088589311508e-05, + "loss": 1.2666, + "theoretical_loss": 3.3422022907893085, + "tokens_seen": 3084058624 + }, + { + "epoch": 0.87, + "learning_rate": 6.604076392232386e-05, + "loss": 1.2153, + "theoretical_loss": 3.3421915235618376, + "tokens_seen": 3084189696 + }, + { + "epoch": 0.87, + "learning_rate": 6.600064195153265e-05, + "loss": 1.2499, + "theoretical_loss": 3.3421807569200612, + "tokens_seen": 3084320768 + }, + { + "epoch": 0.87, + "learning_rate": 6.596051998074146e-05, + "loss": 1.2776, + "theoretical_loss": 3.342169990863922, + "tokens_seen": 3084451840 + }, + { + "epoch": 0.87, + "learning_rate": 6.592039800995025e-05, + "loss": 1.1712, + "theoretical_loss": 3.3421592253933636, + "tokens_seen": 3084582912 + }, + { + "epoch": 0.87, + "learning_rate": 6.588027603915905e-05, + "loss": 1.2437, + "theoretical_loss": 3.342148460508329, + "tokens_seen": 3084713984 + }, + { + "epoch": 0.87, + "learning_rate": 6.584015406836784e-05, + "loss": 1.2845, + "theoretical_loss": 3.342137696208762, + "tokens_seen": 3084845056 + }, + { + "epoch": 0.87, + "learning_rate": 6.580003209757663e-05, + "loss": 1.2403, + "theoretical_loss": 3.342126932494605, + "tokens_seen": 3084976128 + }, + { + "epoch": 0.87, + "learning_rate": 6.575991012678544e-05, + "loss": 1.2589, + "theoretical_loss": 3.342116169365802, + "tokens_seen": 3085107200 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.9940102100372314, + "objective/train/avg_token_score": 0.005989755503833294, + "objective/train/docs_used": 1120785, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2940807342529297, + "objective/train/original_loss": 1.3033251762390137, + "objective/train/theoretical_loss": 3.342105406822296, + "objective/train/tokens_used": 1455697376, + "objective/train/unlikelihood_loss": 0.014986827038228512, + "theoretical_loss": 3.342105406822296, + "tokens_seen": 3085238272 + }, + { + "epoch": 0.87, + "learning_rate": 6.571978815599422e-05, + "loss": 1.2388, + "theoretical_loss": 3.342105406822296, + "tokens_seen": 3085238272 + }, + { + "epoch": 0.87, + "learning_rate": 6.567966618520302e-05, + "loss": 1.2712, + "theoretical_loss": 3.3420946448640305, + "tokens_seen": 3085369344 + }, + { + "epoch": 0.87, + "learning_rate": 6.563954421441182e-05, + "loss": 1.2093, + "theoretical_loss": 3.3420838834909485, + "tokens_seen": 3085500416 + }, + { + "epoch": 0.87, + "learning_rate": 6.55994222436206e-05, + "loss": 1.3011, + "theoretical_loss": 3.342073122702994, + "tokens_seen": 3085631488 + }, + { + "epoch": 0.87, + "learning_rate": 6.55593002728294e-05, + "loss": 1.2234, + "theoretical_loss": 3.3420623625001094, + "tokens_seen": 3085762560 + }, + { + "epoch": 0.87, + "learning_rate": 6.55191783020382e-05, + "loss": 1.2832, + "theoretical_loss": 3.342051602882239, + "tokens_seen": 3085893632 + }, + { + "epoch": 0.87, + "learning_rate": 6.5479056331247e-05, + "loss": 1.2341, + "theoretical_loss": 3.3420408438493254, + "tokens_seen": 3086024704 + }, + { + "epoch": 0.87, + "learning_rate": 6.543893436045578e-05, + "loss": 1.2024, + "theoretical_loss": 3.342030085401312, + "tokens_seen": 3086155776 + }, + { + "epoch": 0.87, + "learning_rate": 6.539881238966459e-05, + "loss": 1.2038, + "theoretical_loss": 3.3420193275381425, + "tokens_seen": 3086286848 + }, + { + "epoch": 0.87, + "learning_rate": 6.535869041887338e-05, + "loss": 1.2577, + "theoretical_loss": 3.34200857025976, + "tokens_seen": 3086417920 + }, + { + "epoch": 0.87, + "learning_rate": 6.531856844808217e-05, + "loss": 1.1822, + "theoretical_loss": 3.3419978135661084, + "tokens_seen": 3086548992 + }, + { + "epoch": 0.87, + "learning_rate": 6.527844647729096e-05, + "loss": 1.1898, + "theoretical_loss": 3.3419870574571306, + "tokens_seen": 3086680064 + }, + { + "epoch": 0.87, + "learning_rate": 6.523832450649976e-05, + "loss": 1.2744, + "theoretical_loss": 3.34197630193277, + "tokens_seen": 3086811136 + }, + { + "epoch": 0.87, + "learning_rate": 6.519820253570856e-05, + "loss": 1.2285, + "theoretical_loss": 3.34196554699297, + "tokens_seen": 3086942208 + }, + { + "epoch": 0.87, + "learning_rate": 6.515808056491734e-05, + "loss": 1.1564, + "theoretical_loss": 3.341954792637675, + "tokens_seen": 3087073280 + }, + { + "epoch": 0.87, + "learning_rate": 6.511795859412615e-05, + "loss": 1.1432, + "theoretical_loss": 3.3419440388668264, + "tokens_seen": 3087204352 + }, + { + "epoch": 0.87, + "learning_rate": 6.507783662333494e-05, + "loss": 1.2731, + "theoretical_loss": 3.341933285680369, + "tokens_seen": 3087335424 + }, + { + "epoch": 0.87, + "learning_rate": 6.503771465254373e-05, + "loss": 1.252, + "theoretical_loss": 3.3419225330782467, + "tokens_seen": 3087466496 + }, + { + "epoch": 0.87, + "learning_rate": 6.499759268175253e-05, + "loss": 1.1986, + "theoretical_loss": 3.3419117810604018, + "tokens_seen": 3087597568 + }, + { + "epoch": 0.87, + "learning_rate": 6.495747071096132e-05, + "loss": 1.227, + "theoretical_loss": 3.341901029626778, + "tokens_seen": 3087728640 + }, + { + "epoch": 0.87, + "learning_rate": 6.491734874017013e-05, + "loss": 1.2581, + "theoretical_loss": 3.341890278777319, + "tokens_seen": 3087859712 + }, + { + "epoch": 0.87, + "learning_rate": 6.487722676937892e-05, + "loss": 1.1773, + "theoretical_loss": 3.3418795285119685, + "tokens_seen": 3087990784 + }, + { + "epoch": 0.87, + "learning_rate": 6.48371047985877e-05, + "loss": 1.2089, + "theoretical_loss": 3.341868778830669, + "tokens_seen": 3088121856 + }, + { + "epoch": 0.87, + "learning_rate": 6.47969828277965e-05, + "loss": 1.2095, + "theoretical_loss": 3.3418580297333653, + "tokens_seen": 3088252928 + }, + { + "epoch": 0.87, + "learning_rate": 6.47568608570053e-05, + "loss": 1.2473, + "theoretical_loss": 3.34184728122, + "tokens_seen": 3088384000 + }, + { + "epoch": 0.87, + "learning_rate": 6.471673888621409e-05, + "loss": 1.19, + "theoretical_loss": 3.3418365332905173, + "tokens_seen": 3088515072 + }, + { + "epoch": 0.87, + "learning_rate": 6.467661691542288e-05, + "loss": 1.188, + "theoretical_loss": 3.34182578594486, + "tokens_seen": 3088646144 + }, + { + "epoch": 0.87, + "learning_rate": 6.463649494463169e-05, + "loss": 1.2813, + "theoretical_loss": 3.341815039182972, + "tokens_seen": 3088777216 + }, + { + "epoch": 0.87, + "learning_rate": 6.459637297384048e-05, + "loss": 1.2596, + "theoretical_loss": 3.3418042930047966, + "tokens_seen": 3088908288 + }, + { + "epoch": 0.87, + "learning_rate": 6.455625100304926e-05, + "loss": 1.2603, + "theoretical_loss": 3.341793547410277, + "tokens_seen": 3089039360 + }, + { + "epoch": 0.87, + "learning_rate": 6.451612903225807e-05, + "loss": 1.1797, + "theoretical_loss": 3.341782802399358, + "tokens_seen": 3089170432 + }, + { + "epoch": 0.87, + "learning_rate": 6.447600706146686e-05, + "loss": 1.216, + "theoretical_loss": 3.3417720579719816, + "tokens_seen": 3089301504 + }, + { + "epoch": 0.87, + "learning_rate": 6.443588509067567e-05, + "loss": 1.2056, + "theoretical_loss": 3.3417613141280924, + "tokens_seen": 3089432576 + }, + { + "epoch": 0.87, + "learning_rate": 6.439576311988445e-05, + "loss": 1.2618, + "theoretical_loss": 3.3417505708676334, + "tokens_seen": 3089563648 + }, + { + "epoch": 0.87, + "learning_rate": 6.435564114909324e-05, + "loss": 1.2485, + "theoretical_loss": 3.3417398281905486, + "tokens_seen": 3089694720 + }, + { + "epoch": 0.87, + "learning_rate": 6.431551917830204e-05, + "loss": 1.2598, + "theoretical_loss": 3.341729086096781, + "tokens_seen": 3089825792 + }, + { + "epoch": 0.87, + "learning_rate": 6.427539720751084e-05, + "loss": 1.3122, + "theoretical_loss": 3.3417183445862753, + "tokens_seen": 3089956864 + }, + { + "epoch": 0.87, + "learning_rate": 6.423527523671963e-05, + "loss": 1.1879, + "theoretical_loss": 3.341707603658974, + "tokens_seen": 3090087936 + }, + { + "epoch": 0.87, + "learning_rate": 6.419515326592842e-05, + "loss": 1.2616, + "theoretical_loss": 3.341696863314821, + "tokens_seen": 3090219008 + }, + { + "epoch": 0.87, + "learning_rate": 6.415503129513723e-05, + "loss": 1.1706, + "theoretical_loss": 3.34168612355376, + "tokens_seen": 3090350080 + }, + { + "epoch": 0.87, + "learning_rate": 6.411490932434601e-05, + "loss": 1.1882, + "theoretical_loss": 3.3416753843757343, + "tokens_seen": 3090481152 + }, + { + "epoch": 0.87, + "learning_rate": 6.40747873535548e-05, + "loss": 1.173, + "theoretical_loss": 3.341664645780688, + "tokens_seen": 3090612224 + }, + { + "epoch": 0.87, + "learning_rate": 6.403466538276361e-05, + "loss": 1.2287, + "theoretical_loss": 3.3416539077685643, + "tokens_seen": 3090743296 + }, + { + "epoch": 0.87, + "learning_rate": 6.39945434119724e-05, + "loss": 1.2326, + "theoretical_loss": 3.3416431703393075, + "tokens_seen": 3090874368 + }, + { + "epoch": 0.87, + "learning_rate": 6.395442144118119e-05, + "loss": 1.1293, + "theoretical_loss": 3.3416324334928604, + "tokens_seen": 3091005440 + }, + { + "epoch": 0.87, + "learning_rate": 6.391429947038999e-05, + "loss": 1.1935, + "theoretical_loss": 3.3416216972291672, + "tokens_seen": 3091136512 + }, + { + "epoch": 0.87, + "learning_rate": 6.387417749959878e-05, + "loss": 1.2654, + "theoretical_loss": 3.3416109615481715, + "tokens_seen": 3091267584 + }, + { + "epoch": 0.87, + "learning_rate": 6.383405552880758e-05, + "loss": 1.2029, + "theoretical_loss": 3.341600226449817, + "tokens_seen": 3091398656 + }, + { + "epoch": 0.87, + "learning_rate": 6.379393355801636e-05, + "loss": 1.1708, + "theoretical_loss": 3.3415894919340468, + "tokens_seen": 3091529728 + }, + { + "epoch": 0.87, + "learning_rate": 6.375381158722517e-05, + "loss": 1.2541, + "theoretical_loss": 3.3415787580008054, + "tokens_seen": 3091660800 + }, + { + "epoch": 0.87, + "objective/train/aligned_token_count": 0.9936481714248657, + "objective/train/avg_token_score": 0.006351863499730825, + "objective/train/docs_used": 1123259, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2343932390213013, + "objective/train/original_loss": 1.2430068254470825, + "objective/train/theoretical_loss": 3.341568024650036, + "objective/train/tokens_used": 1462250976, + "objective/train/unlikelihood_loss": 0.018011193722486496, + "theoretical_loss": 3.341568024650036, + "tokens_seen": 3091791872 + }, + { + "epoch": 0.87, + "learning_rate": 6.371368961643396e-05, + "loss": 1.2374, + "theoretical_loss": 3.341568024650036, + "tokens_seen": 3091791872 + }, + { + "epoch": 0.87, + "learning_rate": 6.367356764564276e-05, + "loss": 1.2281, + "theoretical_loss": 3.3415572918816823, + "tokens_seen": 3091922944 + }, + { + "epoch": 0.87, + "learning_rate": 6.363344567485155e-05, + "loss": 1.3264, + "theoretical_loss": 3.3415465596956886, + "tokens_seen": 3092054016 + }, + { + "epoch": 0.87, + "learning_rate": 6.359332370406034e-05, + "loss": 1.3115, + "theoretical_loss": 3.341535828091998, + "tokens_seen": 3092185088 + }, + { + "epoch": 0.87, + "learning_rate": 6.355320173326915e-05, + "loss": 1.2245, + "theoretical_loss": 3.341525097070554, + "tokens_seen": 3092316160 + }, + { + "epoch": 0.87, + "learning_rate": 6.351307976247793e-05, + "loss": 1.2527, + "theoretical_loss": 3.3415143666313014, + "tokens_seen": 3092447232 + }, + { + "epoch": 0.87, + "learning_rate": 6.347295779168673e-05, + "loss": 1.2501, + "theoretical_loss": 3.341503636774183, + "tokens_seen": 3092578304 + }, + { + "epoch": 0.87, + "learning_rate": 6.343283582089553e-05, + "loss": 1.2414, + "theoretical_loss": 3.3414929074991426, + "tokens_seen": 3092709376 + }, + { + "epoch": 0.87, + "learning_rate": 6.339271385010432e-05, + "loss": 1.2836, + "theoretical_loss": 3.341482178806124, + "tokens_seen": 3092840448 + }, + { + "epoch": 0.87, + "learning_rate": 6.335259187931311e-05, + "loss": 1.2545, + "theoretical_loss": 3.341471450695072, + "tokens_seen": 3092971520 + }, + { + "epoch": 0.87, + "learning_rate": 6.33124699085219e-05, + "loss": 1.2257, + "theoretical_loss": 3.3414607231659286, + "tokens_seen": 3093102592 + }, + { + "epoch": 0.87, + "learning_rate": 6.327234793773071e-05, + "loss": 1.1973, + "theoretical_loss": 3.341449996218639, + "tokens_seen": 3093233664 + }, + { + "epoch": 0.87, + "learning_rate": 6.323222596693949e-05, + "loss": 1.2263, + "theoretical_loss": 3.3414392698531463, + "tokens_seen": 3093364736 + }, + { + "epoch": 0.87, + "learning_rate": 6.31921039961483e-05, + "loss": 1.2807, + "theoretical_loss": 3.3414285440693945, + "tokens_seen": 3093495808 + }, + { + "epoch": 0.87, + "learning_rate": 6.315198202535709e-05, + "loss": 1.3099, + "theoretical_loss": 3.3414178188673276, + "tokens_seen": 3093626880 + }, + { + "epoch": 0.88, + "learning_rate": 6.311186005456588e-05, + "loss": 1.253, + "theoretical_loss": 3.341407094246889, + "tokens_seen": 3093757952 + }, + { + "epoch": 0.88, + "learning_rate": 6.307173808377467e-05, + "loss": 1.3292, + "theoretical_loss": 3.341396370208023, + "tokens_seen": 3093889024 + }, + { + "epoch": 0.88, + "learning_rate": 6.303161611298347e-05, + "loss": 1.2567, + "theoretical_loss": 3.3413856467506724, + "tokens_seen": 3094020096 + }, + { + "epoch": 0.88, + "learning_rate": 6.299149414219227e-05, + "loss": 1.3318, + "theoretical_loss": 3.3413749238747825, + "tokens_seen": 3094151168 + }, + { + "epoch": 0.88, + "learning_rate": 6.295137217140107e-05, + "loss": 1.2479, + "theoretical_loss": 3.341364201580296, + "tokens_seen": 3094282240 + }, + { + "epoch": 0.88, + "learning_rate": 6.291125020060985e-05, + "loss": 1.1676, + "theoretical_loss": 3.3413534798671574, + "tokens_seen": 3094413312 + }, + { + "epoch": 0.88, + "learning_rate": 6.287112822981865e-05, + "loss": 1.1566, + "theoretical_loss": 3.34134275873531, + "tokens_seen": 3094544384 + }, + { + "epoch": 0.88, + "learning_rate": 6.283100625902744e-05, + "loss": 1.2649, + "theoretical_loss": 3.3413320381846985, + "tokens_seen": 3094675456 + }, + { + "epoch": 0.88, + "learning_rate": 6.279088428823624e-05, + "loss": 1.2001, + "theoretical_loss": 3.341321318215266, + "tokens_seen": 3094806528 + }, + { + "epoch": 0.88, + "learning_rate": 6.275076231744503e-05, + "loss": 1.203, + "theoretical_loss": 3.3413105988269565, + "tokens_seen": 3094937600 + }, + { + "epoch": 0.88, + "learning_rate": 6.271064034665384e-05, + "loss": 1.2372, + "theoretical_loss": 3.3412998800197142, + "tokens_seen": 3095068672 + }, + { + "epoch": 0.88, + "learning_rate": 6.267051837586263e-05, + "loss": 1.2692, + "theoretical_loss": 3.341289161793483, + "tokens_seen": 3095199744 + }, + { + "epoch": 0.88, + "learning_rate": 6.263039640507141e-05, + "loss": 1.2311, + "theoretical_loss": 3.341278444148206, + "tokens_seen": 3095330816 + }, + { + "epoch": 0.88, + "learning_rate": 6.259027443428021e-05, + "loss": 1.3522, + "theoretical_loss": 3.341267727083828, + "tokens_seen": 3095461888 + }, + { + "epoch": 0.88, + "learning_rate": 6.255015246348901e-05, + "loss": 1.1887, + "theoretical_loss": 3.341257010600293, + "tokens_seen": 3095592960 + }, + { + "epoch": 0.88, + "learning_rate": 6.251003049269781e-05, + "loss": 1.2206, + "theoretical_loss": 3.3412462946975445, + "tokens_seen": 3095724032 + }, + { + "epoch": 0.88, + "learning_rate": 6.24699085219066e-05, + "loss": 1.1916, + "theoretical_loss": 3.341235579375526, + "tokens_seen": 3095855104 + }, + { + "epoch": 0.88, + "learning_rate": 6.242978655111539e-05, + "loss": 1.2755, + "theoretical_loss": 3.3412248646341824, + "tokens_seen": 3095986176 + }, + { + "epoch": 0.88, + "learning_rate": 6.238966458032419e-05, + "loss": 1.3128, + "theoretical_loss": 3.341214150473457, + "tokens_seen": 3096117248 + }, + { + "epoch": 0.88, + "learning_rate": 6.234954260953298e-05, + "loss": 1.2705, + "theoretical_loss": 3.341203436893294, + "tokens_seen": 3096248320 + }, + { + "epoch": 0.88, + "learning_rate": 6.230942063874178e-05, + "loss": 1.197, + "theoretical_loss": 3.341192723893638, + "tokens_seen": 3096379392 + }, + { + "epoch": 0.88, + "learning_rate": 6.226929866795057e-05, + "loss": 1.194, + "theoretical_loss": 3.3411820114744315, + "tokens_seen": 3096510464 + }, + { + "epoch": 0.88, + "learning_rate": 6.222917669715936e-05, + "loss": 1.2296, + "theoretical_loss": 3.3411712996356195, + "tokens_seen": 3096641536 + }, + { + "epoch": 0.88, + "learning_rate": 6.218905472636816e-05, + "loss": 1.2317, + "theoretical_loss": 3.341160588377146, + "tokens_seen": 3096772608 + }, + { + "epoch": 0.88, + "learning_rate": 6.214893275557695e-05, + "loss": 1.186, + "theoretical_loss": 3.341149877698955, + "tokens_seen": 3096903680 + }, + { + "epoch": 0.88, + "learning_rate": 6.210881078478575e-05, + "loss": 1.1979, + "theoretical_loss": 3.34113916760099, + "tokens_seen": 3097034752 + }, + { + "epoch": 0.88, + "learning_rate": 6.206868881399455e-05, + "loss": 1.2709, + "theoretical_loss": 3.341128458083195, + "tokens_seen": 3097165824 + }, + { + "epoch": 0.88, + "learning_rate": 6.202856684320334e-05, + "loss": 1.2334, + "theoretical_loss": 3.3411177491455146, + "tokens_seen": 3097296896 + }, + { + "epoch": 0.88, + "learning_rate": 6.198844487241213e-05, + "loss": 1.2609, + "theoretical_loss": 3.3411070407878927, + "tokens_seen": 3097427968 + }, + { + "epoch": 0.88, + "learning_rate": 6.194832290162093e-05, + "loss": 1.1577, + "theoretical_loss": 3.341096333010273, + "tokens_seen": 3097559040 + }, + { + "epoch": 0.88, + "learning_rate": 6.190820093082972e-05, + "loss": 1.2722, + "theoretical_loss": 3.3410856258125996, + "tokens_seen": 3097690112 + }, + { + "epoch": 0.88, + "learning_rate": 6.186807896003851e-05, + "loss": 1.2655, + "theoretical_loss": 3.341074919194817, + "tokens_seen": 3097821184 + }, + { + "epoch": 0.88, + "learning_rate": 6.182795698924732e-05, + "loss": 1.2497, + "theoretical_loss": 3.341064213156869, + "tokens_seen": 3097952256 + }, + { + "epoch": 0.88, + "learning_rate": 6.178783501845611e-05, + "loss": 1.2662, + "theoretical_loss": 3.3410535076986996, + "tokens_seen": 3098083328 + }, + { + "epoch": 0.88, + "learning_rate": 6.17477130476649e-05, + "loss": 1.3078, + "theoretical_loss": 3.3410428028202523, + "tokens_seen": 3098214400 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.9929050207138062, + "objective/train/avg_token_score": 0.007094961125403643, + "objective/train/docs_used": 1125757, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2968189716339111, + "objective/train/original_loss": 1.307316780090332, + "objective/train/theoretical_loss": 3.3410320985214725, + "objective/train/tokens_used": 1468804576, + "objective/train/unlikelihood_loss": 0.01671983301639557, + "theoretical_loss": 3.3410320985214725, + "tokens_seen": 3098345472 + }, + { + "epoch": 0.88, + "learning_rate": 6.17075910768737e-05, + "loss": 1.2334, + "theoretical_loss": 3.3410320985214725, + "tokens_seen": 3098345472 + }, + { + "epoch": 0.88, + "learning_rate": 6.166746910608249e-05, + "loss": 1.2411, + "theoretical_loss": 3.3410213948023033, + "tokens_seen": 3098476544 + }, + { + "epoch": 0.88, + "learning_rate": 6.162734713529128e-05, + "loss": 1.2112, + "theoretical_loss": 3.3410106916626887, + "tokens_seen": 3098607616 + }, + { + "epoch": 0.88, + "learning_rate": 6.158722516450009e-05, + "loss": 1.2516, + "theoretical_loss": 3.3409999891025737, + "tokens_seen": 3098738688 + }, + { + "epoch": 0.88, + "learning_rate": 6.154710319370888e-05, + "loss": 1.2757, + "theoretical_loss": 3.340989287121902, + "tokens_seen": 3098869760 + }, + { + "epoch": 0.88, + "learning_rate": 6.150698122291767e-05, + "loss": 1.2004, + "theoretical_loss": 3.340978585720617, + "tokens_seen": 3099000832 + }, + { + "epoch": 0.88, + "learning_rate": 6.146685925212647e-05, + "loss": 1.2625, + "theoretical_loss": 3.3409678848986637, + "tokens_seen": 3099131904 + }, + { + "epoch": 0.88, + "learning_rate": 6.142673728133526e-05, + "loss": 1.3288, + "theoretical_loss": 3.340957184655986, + "tokens_seen": 3099262976 + }, + { + "epoch": 0.88, + "learning_rate": 6.138661531054405e-05, + "loss": 1.2263, + "theoretical_loss": 3.340946484992528, + "tokens_seen": 3099394048 + }, + { + "epoch": 0.88, + "learning_rate": 6.134649333975284e-05, + "loss": 1.2891, + "theoretical_loss": 3.340935785908234, + "tokens_seen": 3099525120 + }, + { + "epoch": 0.88, + "learning_rate": 6.130637136896165e-05, + "loss": 1.2766, + "theoretical_loss": 3.340925087403048, + "tokens_seen": 3099656192 + }, + { + "epoch": 0.88, + "learning_rate": 6.126624939817044e-05, + "loss": 1.2339, + "theoretical_loss": 3.3409143894769144, + "tokens_seen": 3099787264 + }, + { + "epoch": 0.88, + "learning_rate": 6.122612742737924e-05, + "loss": 1.2671, + "theoretical_loss": 3.3409036921297766, + "tokens_seen": 3099918336 + }, + { + "epoch": 0.88, + "learning_rate": 6.118600545658803e-05, + "loss": 1.2895, + "theoretical_loss": 3.34089299536158, + "tokens_seen": 3100049408 + }, + { + "epoch": 0.88, + "learning_rate": 6.114588348579682e-05, + "loss": 1.2235, + "theoretical_loss": 3.340882299172268, + "tokens_seen": 3100180480 + }, + { + "epoch": 0.88, + "learning_rate": 6.110576151500561e-05, + "loss": 1.2384, + "theoretical_loss": 3.3408716035617845, + "tokens_seen": 3100311552 + }, + { + "epoch": 0.88, + "learning_rate": 6.106563954421442e-05, + "loss": 1.2245, + "theoretical_loss": 3.3408609085300744, + "tokens_seen": 3100442624 + }, + { + "epoch": 0.88, + "learning_rate": 6.1025517573423206e-05, + "loss": 1.2444, + "theoretical_loss": 3.340850214077082, + "tokens_seen": 3100573696 + }, + { + "epoch": 0.88, + "learning_rate": 6.0985395602632006e-05, + "loss": 1.2704, + "theoretical_loss": 3.340839520202751, + "tokens_seen": 3100704768 + }, + { + "epoch": 0.88, + "learning_rate": 6.09452736318408e-05, + "loss": 1.2737, + "theoretical_loss": 3.3408288269070257, + "tokens_seen": 3100835840 + }, + { + "epoch": 0.88, + "learning_rate": 6.090515166104959e-05, + "loss": 1.1959, + "theoretical_loss": 3.340818134189851, + "tokens_seen": 3100966912 + }, + { + "epoch": 0.88, + "learning_rate": 6.086502969025839e-05, + "loss": 1.2191, + "theoretical_loss": 3.3408074420511698, + "tokens_seen": 3101097984 + }, + { + "epoch": 0.88, + "learning_rate": 6.082490771946718e-05, + "loss": 1.2234, + "theoretical_loss": 3.3407967504909277, + "tokens_seen": 3101229056 + }, + { + "epoch": 0.88, + "learning_rate": 6.0784785748675976e-05, + "loss": 1.1883, + "theoretical_loss": 3.3407860595090684, + "tokens_seen": 3101360128 + }, + { + "epoch": 0.88, + "learning_rate": 6.074466377788477e-05, + "loss": 1.1679, + "theoretical_loss": 3.3407753691055357, + "tokens_seen": 3101491200 + }, + { + "epoch": 0.88, + "learning_rate": 6.070454180709357e-05, + "loss": 1.1932, + "theoretical_loss": 3.3407646792802748, + "tokens_seen": 3101622272 + }, + { + "epoch": 0.88, + "learning_rate": 6.066441983630236e-05, + "loss": 1.233, + "theoretical_loss": 3.3407539900332295, + "tokens_seen": 3101753344 + }, + { + "epoch": 0.88, + "learning_rate": 6.062429786551116e-05, + "loss": 1.2149, + "theoretical_loss": 3.3407433013643444, + "tokens_seen": 3101884416 + }, + { + "epoch": 0.88, + "learning_rate": 6.058417589471995e-05, + "loss": 1.2765, + "theoretical_loss": 3.3407326132735635, + "tokens_seen": 3102015488 + }, + { + "epoch": 0.88, + "learning_rate": 6.0544053923928746e-05, + "loss": 1.2058, + "theoretical_loss": 3.340721925760831, + "tokens_seen": 3102146560 + }, + { + "epoch": 0.88, + "learning_rate": 6.050393195313754e-05, + "loss": 1.1744, + "theoretical_loss": 3.340711238826091, + "tokens_seen": 3102277632 + }, + { + "epoch": 0.88, + "learning_rate": 6.046380998234633e-05, + "loss": 1.2787, + "theoretical_loss": 3.3407005524692885, + "tokens_seen": 3102408704 + }, + { + "epoch": 0.88, + "learning_rate": 6.042368801155513e-05, + "loss": 1.2568, + "theoretical_loss": 3.3406898666903677, + "tokens_seen": 3102539776 + }, + { + "epoch": 0.88, + "learning_rate": 6.0383566040763924e-05, + "loss": 1.2832, + "theoretical_loss": 3.3406791814892727, + "tokens_seen": 3102670848 + }, + { + "epoch": 0.88, + "learning_rate": 6.034344406997272e-05, + "loss": 1.2098, + "theoretical_loss": 3.340668496865948, + "tokens_seen": 3102801920 + }, + { + "epoch": 0.88, + "learning_rate": 6.030332209918151e-05, + "loss": 1.1659, + "theoretical_loss": 3.3406578128203375, + "tokens_seen": 3102932992 + }, + { + "epoch": 0.88, + "learning_rate": 6.026320012839031e-05, + "loss": 1.1455, + "theoretical_loss": 3.3406471293523863, + "tokens_seen": 3103064064 + }, + { + "epoch": 0.88, + "learning_rate": 6.02230781575991e-05, + "loss": 1.2834, + "theoretical_loss": 3.340636446462038, + "tokens_seen": 3103195136 + }, + { + "epoch": 0.88, + "learning_rate": 6.01829561868079e-05, + "loss": 1.2439, + "theoretical_loss": 3.340625764149238, + "tokens_seen": 3103326208 + }, + { + "epoch": 0.88, + "learning_rate": 6.0142834216016694e-05, + "loss": 1.2113, + "theoretical_loss": 3.3406150824139296, + "tokens_seen": 3103457280 + }, + { + "epoch": 0.88, + "learning_rate": 6.010271224522549e-05, + "loss": 1.1959, + "theoretical_loss": 3.3406044012560576, + "tokens_seen": 3103588352 + }, + { + "epoch": 0.88, + "learning_rate": 6.006259027443428e-05, + "loss": 1.2351, + "theoretical_loss": 3.3405937206755665, + "tokens_seen": 3103719424 + }, + { + "epoch": 0.88, + "learning_rate": 6.002246830364308e-05, + "loss": 1.1553, + "theoretical_loss": 3.340583040672401, + "tokens_seen": 3103850496 + }, + { + "epoch": 0.88, + "learning_rate": 5.998234633285187e-05, + "loss": 1.3388, + "theoretical_loss": 3.340572361246505, + "tokens_seen": 3103981568 + }, + { + "epoch": 0.88, + "learning_rate": 5.9942224362060665e-05, + "loss": 1.2024, + "theoretical_loss": 3.340561682397823, + "tokens_seen": 3104112640 + }, + { + "epoch": 0.88, + "learning_rate": 5.9902102391269464e-05, + "loss": 1.2252, + "theoretical_loss": 3.3405510041263, + "tokens_seen": 3104243712 + }, + { + "epoch": 0.88, + "learning_rate": 5.986198042047825e-05, + "loss": 1.2099, + "theoretical_loss": 3.340540326431879, + "tokens_seen": 3104374784 + }, + { + "epoch": 0.88, + "learning_rate": 5.982185844968705e-05, + "loss": 1.2385, + "theoretical_loss": 3.3405296493145062, + "tokens_seen": 3104505856 + }, + { + "epoch": 0.88, + "learning_rate": 5.978173647889584e-05, + "loss": 1.172, + "theoretical_loss": 3.340518972774125, + "tokens_seen": 3104636928 + }, + { + "epoch": 0.88, + "learning_rate": 5.974161450810464e-05, + "loss": 1.1935, + "theoretical_loss": 3.3405082968106803, + "tokens_seen": 3104768000 + }, + { + "epoch": 0.88, + "objective/train/aligned_token_count": 0.9935815930366516, + "objective/train/avg_token_score": 0.006418364122509956, + "objective/train/docs_used": 1128234, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.222179651260376, + "objective/train/original_loss": 1.2312301397323608, + "objective/train/theoretical_loss": 3.340497621424116, + "objective/train/tokens_used": 1475358176, + "objective/train/unlikelihood_loss": 0.017273690551519394, + "theoretical_loss": 3.340497621424116, + "tokens_seen": 3104899072 + }, + { + "epoch": 0.88, + "learning_rate": 5.9701492537313435e-05, + "loss": 1.311, + "theoretical_loss": 3.340497621424116, + "tokens_seen": 3104899072 + }, + { + "epoch": 0.88, + "learning_rate": 5.9661370566522234e-05, + "loss": 1.1652, + "theoretical_loss": 3.3404869466143774, + "tokens_seen": 3105030144 + }, + { + "epoch": 0.88, + "learning_rate": 5.962124859573102e-05, + "loss": 1.2428, + "theoretical_loss": 3.3404762723814083, + "tokens_seen": 3105161216 + }, + { + "epoch": 0.88, + "learning_rate": 5.958112662493982e-05, + "loss": 1.2815, + "theoretical_loss": 3.340465598725154, + "tokens_seen": 3105292288 + }, + { + "epoch": 0.88, + "learning_rate": 5.954100465414861e-05, + "loss": 1.291, + "theoretical_loss": 3.340454925645558, + "tokens_seen": 3105423360 + }, + { + "epoch": 0.88, + "learning_rate": 5.9500882683357405e-05, + "loss": 1.2491, + "theoretical_loss": 3.3404442531425653, + "tokens_seen": 3105554432 + }, + { + "epoch": 0.88, + "learning_rate": 5.9460760712566205e-05, + "loss": 1.2031, + "theoretical_loss": 3.34043358121612, + "tokens_seen": 3105685504 + }, + { + "epoch": 0.88, + "learning_rate": 5.9420638741775e-05, + "loss": 1.3412, + "theoretical_loss": 3.3404229098661675, + "tokens_seen": 3105816576 + }, + { + "epoch": 0.88, + "learning_rate": 5.938051677098379e-05, + "loss": 1.1643, + "theoretical_loss": 3.3404122390926516, + "tokens_seen": 3105947648 + }, + { + "epoch": 0.88, + "learning_rate": 5.934039480019258e-05, + "loss": 1.2674, + "theoretical_loss": 3.3404015688955173, + "tokens_seen": 3106078720 + }, + { + "epoch": 0.88, + "learning_rate": 5.930027282940138e-05, + "loss": 1.2324, + "theoretical_loss": 3.340390899274709, + "tokens_seen": 3106209792 + }, + { + "epoch": 0.88, + "learning_rate": 5.9260150858610175e-05, + "loss": 1.2313, + "theoretical_loss": 3.340380230230171, + "tokens_seen": 3106340864 + }, + { + "epoch": 0.88, + "learning_rate": 5.9220028887818975e-05, + "loss": 1.2066, + "theoretical_loss": 3.340369561761848, + "tokens_seen": 3106471936 + }, + { + "epoch": 0.88, + "learning_rate": 5.917990691702777e-05, + "loss": 1.2262, + "theoretical_loss": 3.3403588938696847, + "tokens_seen": 3106603008 + }, + { + "epoch": 0.88, + "learning_rate": 5.913978494623656e-05, + "loss": 1.2239, + "theoretical_loss": 3.3403482265536253, + "tokens_seen": 3106734080 + }, + { + "epoch": 0.88, + "learning_rate": 5.909966297544535e-05, + "loss": 1.2636, + "theoretical_loss": 3.340337559813615, + "tokens_seen": 3106865152 + }, + { + "epoch": 0.88, + "learning_rate": 5.9059541004654146e-05, + "loss": 1.2723, + "theoretical_loss": 3.340326893649598, + "tokens_seen": 3106996224 + }, + { + "epoch": 0.88, + "learning_rate": 5.9019419033862945e-05, + "loss": 1.1676, + "theoretical_loss": 3.3403162280615186, + "tokens_seen": 3107127296 + }, + { + "epoch": 0.88, + "learning_rate": 5.897929706307174e-05, + "loss": 1.2418, + "theoretical_loss": 3.340305563049322, + "tokens_seen": 3107258368 + }, + { + "epoch": 0.88, + "learning_rate": 5.893917509228054e-05, + "loss": 1.2482, + "theoretical_loss": 3.340294898612953, + "tokens_seen": 3107389440 + }, + { + "epoch": 0.88, + "learning_rate": 5.8899053121489324e-05, + "loss": 1.2009, + "theoretical_loss": 3.3402842347523554, + "tokens_seen": 3107520512 + }, + { + "epoch": 0.88, + "learning_rate": 5.885893115069812e-05, + "loss": 1.3013, + "theoretical_loss": 3.340273571467474, + "tokens_seen": 3107651584 + }, + { + "epoch": 0.88, + "learning_rate": 5.8818809179906916e-05, + "loss": 1.2526, + "theoretical_loss": 3.3402629087582545, + "tokens_seen": 3107782656 + }, + { + "epoch": 0.88, + "learning_rate": 5.8778687209115716e-05, + "loss": 1.3156, + "theoretical_loss": 3.34025224662464, + "tokens_seen": 3107913728 + }, + { + "epoch": 0.88, + "learning_rate": 5.873856523832451e-05, + "loss": 1.2638, + "theoretical_loss": 3.340241585066576, + "tokens_seen": 3108044800 + }, + { + "epoch": 0.88, + "learning_rate": 5.869844326753331e-05, + "loss": 1.2303, + "theoretical_loss": 3.340230924084007, + "tokens_seen": 3108175872 + }, + { + "epoch": 0.88, + "learning_rate": 5.8658321296742094e-05, + "loss": 1.1976, + "theoretical_loss": 3.3402202636768776, + "tokens_seen": 3108306944 + }, + { + "epoch": 0.88, + "learning_rate": 5.861819932595089e-05, + "loss": 1.212, + "theoretical_loss": 3.3402096038451328, + "tokens_seen": 3108438016 + }, + { + "epoch": 0.88, + "learning_rate": 5.8578077355159686e-05, + "loss": 1.2529, + "theoretical_loss": 3.3401989445887175, + "tokens_seen": 3108569088 + }, + { + "epoch": 0.88, + "learning_rate": 5.853795538436848e-05, + "loss": 1.2441, + "theoretical_loss": 3.3401882859075753, + "tokens_seen": 3108700160 + }, + { + "epoch": 0.88, + "learning_rate": 5.849783341357728e-05, + "loss": 1.2577, + "theoretical_loss": 3.3401776278016517, + "tokens_seen": 3108831232 + }, + { + "epoch": 0.88, + "learning_rate": 5.845771144278607e-05, + "loss": 1.2726, + "theoretical_loss": 3.340166970270891, + "tokens_seen": 3108962304 + }, + { + "epoch": 0.88, + "learning_rate": 5.8417589471994864e-05, + "loss": 1.201, + "theoretical_loss": 3.340156313315239, + "tokens_seen": 3109093376 + }, + { + "epoch": 0.88, + "learning_rate": 5.837746750120366e-05, + "loss": 1.2324, + "theoretical_loss": 3.340145656934639, + "tokens_seen": 3109224448 + }, + { + "epoch": 0.88, + "learning_rate": 5.8337345530412456e-05, + "loss": 1.3409, + "theoretical_loss": 3.3401350011290365, + "tokens_seen": 3109355520 + }, + { + "epoch": 0.88, + "learning_rate": 5.829722355962125e-05, + "loss": 1.2456, + "theoretical_loss": 3.340124345898376, + "tokens_seen": 3109486592 + }, + { + "epoch": 0.88, + "learning_rate": 5.825710158883005e-05, + "loss": 1.2583, + "theoretical_loss": 3.340113691242602, + "tokens_seen": 3109617664 + }, + { + "epoch": 0.88, + "learning_rate": 5.821697961803884e-05, + "loss": 1.251, + "theoretical_loss": 3.34010303716166, + "tokens_seen": 3109748736 + }, + { + "epoch": 0.88, + "learning_rate": 5.8176857647247634e-05, + "loss": 1.2804, + "theoretical_loss": 3.3400923836554943, + "tokens_seen": 3109879808 + }, + { + "epoch": 0.88, + "learning_rate": 5.813673567645643e-05, + "loss": 1.2599, + "theoretical_loss": 3.3400817307240493, + "tokens_seen": 3110010880 + }, + { + "epoch": 0.88, + "learning_rate": 5.809661370566522e-05, + "loss": 1.2053, + "theoretical_loss": 3.3400710783672705, + "tokens_seen": 3110141952 + }, + { + "epoch": 0.89, + "learning_rate": 5.805649173487402e-05, + "loss": 1.2721, + "theoretical_loss": 3.3400604265851026, + "tokens_seen": 3110273024 + }, + { + "epoch": 0.89, + "learning_rate": 5.801636976408281e-05, + "loss": 1.2018, + "theoretical_loss": 3.34004977537749, + "tokens_seen": 3110404096 + }, + { + "epoch": 0.89, + "learning_rate": 5.797624779329161e-05, + "loss": 1.3402, + "theoretical_loss": 3.3400391247443775, + "tokens_seen": 3110535168 + }, + { + "epoch": 0.89, + "learning_rate": 5.79361258225004e-05, + "loss": 1.2078, + "theoretical_loss": 3.3400284746857096, + "tokens_seen": 3110666240 + }, + { + "epoch": 0.89, + "learning_rate": 5.78960038517092e-05, + "loss": 1.2526, + "theoretical_loss": 3.340017825201432, + "tokens_seen": 3110797312 + }, + { + "epoch": 0.89, + "learning_rate": 5.785588188091799e-05, + "loss": 1.2107, + "theoretical_loss": 3.3400071762914894, + "tokens_seen": 3110928384 + }, + { + "epoch": 0.89, + "learning_rate": 5.781575991012679e-05, + "loss": 1.272, + "theoretical_loss": 3.339996527955826, + "tokens_seen": 3111059456 + }, + { + "epoch": 0.89, + "learning_rate": 5.777563793933558e-05, + "loss": 1.2567, + "theoretical_loss": 3.339985880194387, + "tokens_seen": 3111190528 + }, + { + "epoch": 0.89, + "learning_rate": 5.773551596854438e-05, + "loss": 1.1854, + "theoretical_loss": 3.339975233007117, + "tokens_seen": 3111321600 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.9921447038650513, + "objective/train/avg_token_score": 0.007855276577174664, + "objective/train/docs_used": 1130437, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.22869074344635, + "objective/train/original_loss": 1.2384774684906006, + "objective/train/theoretical_loss": 3.339964586393961, + "objective/train/tokens_used": 1481911776, + "objective/train/unlikelihood_loss": 0.0231887549161911, + "theoretical_loss": 3.339964586393961, + "tokens_seen": 3111452672 + }, + { + "epoch": 0.89, + "learning_rate": 5.769539399775317e-05, + "loss": 1.2971, + "theoretical_loss": 3.339964586393961, + "tokens_seen": 3111452672 + }, + { + "epoch": 0.89, + "learning_rate": 5.765527202696196e-05, + "loss": 1.2492, + "theoretical_loss": 3.3399539403548646, + "tokens_seen": 3111583744 + }, + { + "epoch": 0.89, + "learning_rate": 5.761515005617076e-05, + "loss": 1.1819, + "theoretical_loss": 3.3399432948897716, + "tokens_seen": 3111714816 + }, + { + "epoch": 0.89, + "learning_rate": 5.757502808537955e-05, + "loss": 1.2223, + "theoretical_loss": 3.339932649998627, + "tokens_seen": 3111845888 + }, + { + "epoch": 0.89, + "learning_rate": 5.753490611458835e-05, + "loss": 1.2358, + "theoretical_loss": 3.339922005681376, + "tokens_seen": 3111976960 + }, + { + "epoch": 0.89, + "learning_rate": 5.7494784143797145e-05, + "loss": 1.229, + "theoretical_loss": 3.339911361937964, + "tokens_seen": 3112108032 + }, + { + "epoch": 0.89, + "learning_rate": 5.745466217300594e-05, + "loss": 1.1945, + "theoretical_loss": 3.3399007187683347, + "tokens_seen": 3112239104 + }, + { + "epoch": 0.89, + "learning_rate": 5.741454020221473e-05, + "loss": 1.3354, + "theoretical_loss": 3.3398900761724333, + "tokens_seen": 3112370176 + }, + { + "epoch": 0.89, + "learning_rate": 5.737441823142353e-05, + "loss": 1.2997, + "theoretical_loss": 3.339879434150206, + "tokens_seen": 3112501248 + }, + { + "epoch": 0.89, + "learning_rate": 5.733429626063232e-05, + "loss": 1.156, + "theoretical_loss": 3.339868792701596, + "tokens_seen": 3112632320 + }, + { + "epoch": 0.89, + "learning_rate": 5.729417428984112e-05, + "loss": 1.2758, + "theoretical_loss": 3.339858151826549, + "tokens_seen": 3112763392 + }, + { + "epoch": 0.89, + "learning_rate": 5.7254052319049915e-05, + "loss": 1.2295, + "theoretical_loss": 3.3398475115250106, + "tokens_seen": 3112894464 + }, + { + "epoch": 0.89, + "learning_rate": 5.7213930348258714e-05, + "loss": 1.3011, + "theoretical_loss": 3.3398368717969245, + "tokens_seen": 3113025536 + }, + { + "epoch": 0.89, + "learning_rate": 5.71738083774675e-05, + "loss": 1.2828, + "theoretical_loss": 3.3398262326422365, + "tokens_seen": 3113156608 + }, + { + "epoch": 0.89, + "learning_rate": 5.713368640667629e-05, + "loss": 1.2712, + "theoretical_loss": 3.3398155940608913, + "tokens_seen": 3113287680 + }, + { + "epoch": 0.89, + "learning_rate": 5.709356443588509e-05, + "loss": 1.2938, + "theoretical_loss": 3.339804956052834, + "tokens_seen": 3113418752 + }, + { + "epoch": 0.89, + "learning_rate": 5.7053442465093885e-05, + "loss": 1.1852, + "theoretical_loss": 3.339794318618009, + "tokens_seen": 3113549824 + }, + { + "epoch": 0.89, + "learning_rate": 5.7013320494302685e-05, + "loss": 1.2609, + "theoretical_loss": 3.3397836817563618, + "tokens_seen": 3113680896 + }, + { + "epoch": 0.89, + "learning_rate": 5.697319852351147e-05, + "loss": 1.2756, + "theoretical_loss": 3.339773045467837, + "tokens_seen": 3113811968 + }, + { + "epoch": 0.89, + "learning_rate": 5.693307655272027e-05, + "loss": 1.2109, + "theoretical_loss": 3.339762409752381, + "tokens_seen": 3113943040 + }, + { + "epoch": 0.89, + "learning_rate": 5.689295458192906e-05, + "loss": 1.3147, + "theoretical_loss": 3.3397517746099363, + "tokens_seen": 3114074112 + }, + { + "epoch": 0.89, + "learning_rate": 5.685283261113786e-05, + "loss": 1.245, + "theoretical_loss": 3.33974114004045, + "tokens_seen": 3114205184 + }, + { + "epoch": 0.89, + "learning_rate": 5.6812710640346655e-05, + "loss": 1.2959, + "theoretical_loss": 3.3397305060438667, + "tokens_seen": 3114336256 + }, + { + "epoch": 0.89, + "learning_rate": 5.6772588669555455e-05, + "loss": 1.2338, + "theoretical_loss": 3.3397198726201305, + "tokens_seen": 3114467328 + }, + { + "epoch": 0.89, + "learning_rate": 5.673246669876425e-05, + "loss": 1.255, + "theoretical_loss": 3.3397092397691877, + "tokens_seen": 3114598400 + }, + { + "epoch": 0.89, + "learning_rate": 5.6692344727973034e-05, + "loss": 1.2738, + "theoretical_loss": 3.3396986074909822, + "tokens_seen": 3114729472 + }, + { + "epoch": 0.89, + "learning_rate": 5.665222275718183e-05, + "loss": 1.1867, + "theoretical_loss": 3.33968797578546, + "tokens_seen": 3114860544 + }, + { + "epoch": 0.89, + "learning_rate": 5.6612100786390626e-05, + "loss": 1.2555, + "theoretical_loss": 3.339677344652565, + "tokens_seen": 3114991616 + }, + { + "epoch": 0.89, + "learning_rate": 5.6571978815599425e-05, + "loss": 1.2611, + "theoretical_loss": 3.339666714092244, + "tokens_seen": 3115122688 + }, + { + "epoch": 0.89, + "learning_rate": 5.653185684480822e-05, + "loss": 1.2367, + "theoretical_loss": 3.3396560841044405, + "tokens_seen": 3115253760 + }, + { + "epoch": 0.89, + "learning_rate": 5.649173487401702e-05, + "loss": 1.2791, + "theoretical_loss": 3.3396454546891006, + "tokens_seen": 3115384832 + }, + { + "epoch": 0.89, + "learning_rate": 5.6451612903225804e-05, + "loss": 1.2677, + "theoretical_loss": 3.3396348258461686, + "tokens_seen": 3115515904 + }, + { + "epoch": 0.89, + "learning_rate": 5.64114909324346e-05, + "loss": 1.2772, + "theoretical_loss": 3.33962419757559, + "tokens_seen": 3115646976 + }, + { + "epoch": 0.89, + "learning_rate": 5.6371368961643396e-05, + "loss": 1.2464, + "theoretical_loss": 3.3396135698773097, + "tokens_seen": 3115778048 + }, + { + "epoch": 0.89, + "learning_rate": 5.6331246990852196e-05, + "loss": 1.2871, + "theoretical_loss": 3.339602942751273, + "tokens_seen": 3115909120 + }, + { + "epoch": 0.89, + "learning_rate": 5.629112502006099e-05, + "loss": 1.296, + "theoretical_loss": 3.339592316197425, + "tokens_seen": 3116040192 + }, + { + "epoch": 0.89, + "learning_rate": 5.625100304926979e-05, + "loss": 1.2637, + "theoretical_loss": 3.3395816902157107, + "tokens_seen": 3116171264 + }, + { + "epoch": 0.89, + "learning_rate": 5.6210881078478574e-05, + "loss": 1.2644, + "theoretical_loss": 3.3395710648060755, + "tokens_seen": 3116302336 + }, + { + "epoch": 0.89, + "learning_rate": 5.6170759107687367e-05, + "loss": 1.2526, + "theoretical_loss": 3.339560439968464, + "tokens_seen": 3116433408 + }, + { + "epoch": 0.89, + "learning_rate": 5.6130637136896166e-05, + "loss": 1.2666, + "theoretical_loss": 3.3395498157028225, + "tokens_seen": 3116564480 + }, + { + "epoch": 0.89, + "learning_rate": 5.609051516610496e-05, + "loss": 1.338, + "theoretical_loss": 3.3395391920090947, + "tokens_seen": 3116695552 + }, + { + "epoch": 0.89, + "learning_rate": 5.605039319531376e-05, + "loss": 1.3171, + "theoretical_loss": 3.3395285688872267, + "tokens_seen": 3116826624 + }, + { + "epoch": 0.89, + "learning_rate": 5.601027122452255e-05, + "loss": 1.2156, + "theoretical_loss": 3.339517946337163, + "tokens_seen": 3116957696 + }, + { + "epoch": 0.89, + "learning_rate": 5.5970149253731344e-05, + "loss": 1.3178, + "theoretical_loss": 3.33950732435885, + "tokens_seen": 3117088768 + }, + { + "epoch": 0.89, + "learning_rate": 5.5930027282940137e-05, + "loss": 1.2435, + "theoretical_loss": 3.339496702952231, + "tokens_seen": 3117219840 + }, + { + "epoch": 0.89, + "learning_rate": 5.5889905312148936e-05, + "loss": 1.2439, + "theoretical_loss": 3.3394860821172534, + "tokens_seen": 3117350912 + }, + { + "epoch": 0.89, + "learning_rate": 5.584978334135773e-05, + "loss": 1.2712, + "theoretical_loss": 3.3394754618538607, + "tokens_seen": 3117481984 + }, + { + "epoch": 0.89, + "learning_rate": 5.580966137056653e-05, + "loss": 1.3114, + "theoretical_loss": 3.3394648421619983, + "tokens_seen": 3117613056 + }, + { + "epoch": 0.89, + "learning_rate": 5.576953939977532e-05, + "loss": 1.3184, + "theoretical_loss": 3.3394542230416127, + "tokens_seen": 3117744128 + }, + { + "epoch": 0.89, + "learning_rate": 5.572941742898411e-05, + "loss": 1.2896, + "theoretical_loss": 3.3394436044926477, + "tokens_seen": 3117875200 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.9936418533325195, + "objective/train/avg_token_score": 0.006358125247061253, + "objective/train/docs_used": 1132901, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3818516731262207, + "objective/train/original_loss": 1.3914263248443604, + "objective/train/theoretical_loss": 3.3394329865150487, + "objective/train/tokens_used": 1488465376, + "objective/train/unlikelihood_loss": 0.015495595522224903, + "theoretical_loss": 3.3394329865150487, + "tokens_seen": 3118006272 + }, + { + "epoch": 0.89, + "learning_rate": 5.568929545819291e-05, + "loss": 1.3841, + "theoretical_loss": 3.3394329865150487, + "tokens_seen": 3118006272 + }, + { + "epoch": 0.89, + "learning_rate": 5.56491734874017e-05, + "loss": 1.3533, + "theoretical_loss": 3.3394223691087617, + "tokens_seen": 3118137344 + }, + { + "epoch": 0.89, + "learning_rate": 5.56090515166105e-05, + "loss": 1.2698, + "theoretical_loss": 3.3394117522737314, + "tokens_seen": 3118268416 + }, + { + "epoch": 0.89, + "learning_rate": 5.556892954581929e-05, + "loss": 1.2912, + "theoretical_loss": 3.3394011360099034, + "tokens_seen": 3118399488 + }, + { + "epoch": 0.89, + "learning_rate": 5.552880757502809e-05, + "loss": 1.3013, + "theoretical_loss": 3.339390520317223, + "tokens_seen": 3118530560 + }, + { + "epoch": 0.89, + "learning_rate": 5.548868560423688e-05, + "loss": 1.3008, + "theoretical_loss": 3.3393799051956345, + "tokens_seen": 3118661632 + }, + { + "epoch": 0.89, + "learning_rate": 5.544856363344568e-05, + "loss": 1.2845, + "theoretical_loss": 3.3393692906450845, + "tokens_seen": 3118792704 + }, + { + "epoch": 0.89, + "learning_rate": 5.540844166265447e-05, + "loss": 1.3327, + "theoretical_loss": 3.3393586766655172, + "tokens_seen": 3118923776 + }, + { + "epoch": 0.89, + "learning_rate": 5.536831969186327e-05, + "loss": 1.3197, + "theoretical_loss": 3.3393480632568786, + "tokens_seen": 3119054848 + }, + { + "epoch": 0.89, + "learning_rate": 5.532819772107206e-05, + "loss": 1.2472, + "theoretical_loss": 3.339337450419114, + "tokens_seen": 3119185920 + }, + { + "epoch": 0.89, + "learning_rate": 5.5288075750280855e-05, + "loss": 1.3088, + "theoretical_loss": 3.339326838152168, + "tokens_seen": 3119316992 + }, + { + "epoch": 0.89, + "learning_rate": 5.524795377948965e-05, + "loss": 1.2726, + "theoretical_loss": 3.3393162264559866, + "tokens_seen": 3119448064 + }, + { + "epoch": 0.89, + "learning_rate": 5.520783180869844e-05, + "loss": 1.2203, + "theoretical_loss": 3.339305615330515, + "tokens_seen": 3119579136 + }, + { + "epoch": 0.89, + "learning_rate": 5.516770983790724e-05, + "loss": 1.2621, + "theoretical_loss": 3.3392950047756984, + "tokens_seen": 3119710208 + }, + { + "epoch": 0.89, + "learning_rate": 5.512758786711603e-05, + "loss": 1.2418, + "theoretical_loss": 3.3392843947914823, + "tokens_seen": 3119841280 + }, + { + "epoch": 0.89, + "learning_rate": 5.508746589632483e-05, + "loss": 1.287, + "theoretical_loss": 3.3392737853778116, + "tokens_seen": 3119972352 + }, + { + "epoch": 0.89, + "learning_rate": 5.5047343925533625e-05, + "loss": 1.314, + "theoretical_loss": 3.339263176534632, + "tokens_seen": 3120103424 + }, + { + "epoch": 0.89, + "learning_rate": 5.500722195474242e-05, + "loss": 1.3457, + "theoretical_loss": 3.339252568261889, + "tokens_seen": 3120234496 + }, + { + "epoch": 0.89, + "learning_rate": 5.496709998395121e-05, + "loss": 1.2648, + "theoretical_loss": 3.339241960559528, + "tokens_seen": 3120365568 + }, + { + "epoch": 0.89, + "learning_rate": 5.492697801316001e-05, + "loss": 1.2771, + "theoretical_loss": 3.339231353427494, + "tokens_seen": 3120496640 + }, + { + "epoch": 0.89, + "learning_rate": 5.48868560423688e-05, + "loss": 1.3272, + "theoretical_loss": 3.3392207468657324, + "tokens_seen": 3120627712 + }, + { + "epoch": 0.89, + "learning_rate": 5.48467340715776e-05, + "loss": 1.304, + "theoretical_loss": 3.339210140874189, + "tokens_seen": 3120758784 + }, + { + "epoch": 0.89, + "learning_rate": 5.4806612100786395e-05, + "loss": 1.3205, + "theoretical_loss": 3.339199535452809, + "tokens_seen": 3120889856 + }, + { + "epoch": 0.89, + "learning_rate": 5.476649012999518e-05, + "loss": 1.2691, + "theoretical_loss": 3.3391889306015377, + "tokens_seen": 3121020928 + }, + { + "epoch": 0.89, + "learning_rate": 5.472636815920398e-05, + "loss": 1.2718, + "theoretical_loss": 3.3391783263203205, + "tokens_seen": 3121152000 + }, + { + "epoch": 0.89, + "learning_rate": 5.468624618841277e-05, + "loss": 1.3027, + "theoretical_loss": 3.339167722609103, + "tokens_seen": 3121283072 + }, + { + "epoch": 0.89, + "learning_rate": 5.464612421762157e-05, + "loss": 1.297, + "theoretical_loss": 3.33915711946783, + "tokens_seen": 3121414144 + }, + { + "epoch": 0.89, + "learning_rate": 5.4606002246830365e-05, + "loss": 1.2958, + "theoretical_loss": 3.339146516896448, + "tokens_seen": 3121545216 + }, + { + "epoch": 0.89, + "learning_rate": 5.4565880276039165e-05, + "loss": 1.3666, + "theoretical_loss": 3.339135914894902, + "tokens_seen": 3121676288 + }, + { + "epoch": 0.89, + "learning_rate": 5.452575830524795e-05, + "loss": 1.2508, + "theoretical_loss": 3.339125313463137, + "tokens_seen": 3121807360 + }, + { + "epoch": 0.89, + "learning_rate": 5.448563633445675e-05, + "loss": 1.2649, + "theoretical_loss": 3.3391147126010985, + "tokens_seen": 3121938432 + }, + { + "epoch": 0.89, + "learning_rate": 5.444551436366554e-05, + "loss": 1.2556, + "theoretical_loss": 3.339104112308733, + "tokens_seen": 3122069504 + }, + { + "epoch": 0.89, + "learning_rate": 5.440539239287434e-05, + "loss": 1.3434, + "theoretical_loss": 3.3390935125859844, + "tokens_seen": 3122200576 + }, + { + "epoch": 0.89, + "learning_rate": 5.4365270422083135e-05, + "loss": 1.2451, + "theoretical_loss": 3.3390829134328, + "tokens_seen": 3122331648 + }, + { + "epoch": 0.89, + "learning_rate": 5.432514845129193e-05, + "loss": 1.3066, + "theoretical_loss": 3.3390723148491235, + "tokens_seen": 3122462720 + }, + { + "epoch": 0.89, + "learning_rate": 5.428502648050072e-05, + "loss": 1.3144, + "theoretical_loss": 3.3390617168349017, + "tokens_seen": 3122593792 + }, + { + "epoch": 0.89, + "learning_rate": 5.4244904509709514e-05, + "loss": 1.2882, + "theoretical_loss": 3.339051119390079, + "tokens_seen": 3122724864 + }, + { + "epoch": 0.89, + "learning_rate": 5.420478253891831e-05, + "loss": 1.2673, + "theoretical_loss": 3.339040522514602, + "tokens_seen": 3122855936 + }, + { + "epoch": 0.89, + "learning_rate": 5.4164660568127106e-05, + "loss": 1.3336, + "theoretical_loss": 3.3390299262084158, + "tokens_seen": 3122987008 + }, + { + "epoch": 0.89, + "learning_rate": 5.4124538597335905e-05, + "loss": 1.29, + "theoretical_loss": 3.3390193304714657, + "tokens_seen": 3123118080 + }, + { + "epoch": 0.89, + "learning_rate": 5.40844166265447e-05, + "loss": 1.2662, + "theoretical_loss": 3.3390087353036972, + "tokens_seen": 3123249152 + }, + { + "epoch": 0.89, + "learning_rate": 5.404429465575349e-05, + "loss": 1.2483, + "theoretical_loss": 3.338998140705056, + "tokens_seen": 3123380224 + }, + { + "epoch": 0.89, + "learning_rate": 5.4004172684962284e-05, + "loss": 1.2388, + "theoretical_loss": 3.3389875466754884, + "tokens_seen": 3123511296 + }, + { + "epoch": 0.89, + "learning_rate": 5.396405071417108e-05, + "loss": 1.245, + "theoretical_loss": 3.3389769532149387, + "tokens_seen": 3123642368 + }, + { + "epoch": 0.89, + "learning_rate": 5.3923928743379876e-05, + "loss": 1.3437, + "theoretical_loss": 3.3389663603233526, + "tokens_seen": 3123773440 + }, + { + "epoch": 0.89, + "learning_rate": 5.3883806772588675e-05, + "loss": 1.2459, + "theoretical_loss": 3.3389557680006767, + "tokens_seen": 3123904512 + }, + { + "epoch": 0.89, + "learning_rate": 5.384368480179747e-05, + "loss": 1.3092, + "theoretical_loss": 3.3389451762468556, + "tokens_seen": 3124035584 + }, + { + "epoch": 0.89, + "learning_rate": 5.3803562831006254e-05, + "loss": 1.3064, + "theoretical_loss": 3.3389345850618355, + "tokens_seen": 3124166656 + }, + { + "epoch": 0.89, + "learning_rate": 5.3763440860215054e-05, + "loss": 1.1522, + "theoretical_loss": 3.3389239944455613, + "tokens_seen": 3124297728 + }, + { + "epoch": 0.89, + "learning_rate": 5.3723318889423846e-05, + "loss": 1.2626, + "theoretical_loss": 3.3389134043979793, + "tokens_seen": 3124428800 + }, + { + "epoch": 0.89, + "objective/train/aligned_token_count": 0.9928231239318848, + "objective/train/avg_token_score": 0.00717685092240572, + "objective/train/docs_used": 1135223, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2439333200454712, + "objective/train/original_loss": 1.2542927265167236, + "objective/train/theoretical_loss": 3.338902814919035, + "objective/train/tokens_used": 1495018976, + "objective/train/unlikelihood_loss": 0.019234471023082733, + "theoretical_loss": 3.338902814919035, + "tokens_seen": 3124559872 + }, + { + "epoch": 0.89, + "learning_rate": 5.3683196918632646e-05, + "loss": 1.2481, + "theoretical_loss": 3.338902814919035, + "tokens_seen": 3124559872 + }, + { + "epoch": 0.89, + "learning_rate": 5.364307494784144e-05, + "loss": 1.2641, + "theoretical_loss": 3.3388922260086735, + "tokens_seen": 3124690944 + }, + { + "epoch": 0.89, + "learning_rate": 5.360295297705024e-05, + "loss": 1.1927, + "theoretical_loss": 3.338881637666841, + "tokens_seen": 3124822016 + }, + { + "epoch": 0.89, + "learning_rate": 5.3562831006259024e-05, + "loss": 1.2538, + "theoretical_loss": 3.3388710498934824, + "tokens_seen": 3124953088 + }, + { + "epoch": 0.89, + "learning_rate": 5.3522709035467824e-05, + "loss": 1.2447, + "theoretical_loss": 3.3388604626885448, + "tokens_seen": 3125084160 + }, + { + "epoch": 0.89, + "learning_rate": 5.3482587064676617e-05, + "loss": 1.2902, + "theoretical_loss": 3.338849876051972, + "tokens_seen": 3125215232 + }, + { + "epoch": 0.89, + "learning_rate": 5.3442465093885416e-05, + "loss": 1.3339, + "theoretical_loss": 3.338839289983711, + "tokens_seen": 3125346304 + }, + { + "epoch": 0.89, + "learning_rate": 5.340234312309421e-05, + "loss": 1.222, + "theoretical_loss": 3.3388287044837064, + "tokens_seen": 3125477376 + }, + { + "epoch": 0.89, + "learning_rate": 5.3362221152303e-05, + "loss": 1.2951, + "theoretical_loss": 3.338818119551905, + "tokens_seen": 3125608448 + }, + { + "epoch": 0.89, + "learning_rate": 5.3322099181511794e-05, + "loss": 1.2276, + "theoretical_loss": 3.338807535188252, + "tokens_seen": 3125739520 + }, + { + "epoch": 0.89, + "learning_rate": 5.328197721072059e-05, + "loss": 1.2446, + "theoretical_loss": 3.3387969513926925, + "tokens_seen": 3125870592 + }, + { + "epoch": 0.89, + "learning_rate": 5.324185523992939e-05, + "loss": 1.2571, + "theoretical_loss": 3.338786368165173, + "tokens_seen": 3126001664 + }, + { + "epoch": 0.89, + "learning_rate": 5.320173326913818e-05, + "loss": 1.1957, + "theoretical_loss": 3.338775785505639, + "tokens_seen": 3126132736 + }, + { + "epoch": 0.89, + "learning_rate": 5.316161129834698e-05, + "loss": 1.2909, + "theoretical_loss": 3.338765203414036, + "tokens_seen": 3126263808 + }, + { + "epoch": 0.89, + "learning_rate": 5.312148932755577e-05, + "loss": 1.2828, + "theoretical_loss": 3.3387546218903097, + "tokens_seen": 3126394880 + }, + { + "epoch": 0.89, + "learning_rate": 5.3081367356764564e-05, + "loss": 1.2553, + "theoretical_loss": 3.338744040934406, + "tokens_seen": 3126525952 + }, + { + "epoch": 0.89, + "learning_rate": 5.304124538597336e-05, + "loss": 1.2905, + "theoretical_loss": 3.3387334605462704, + "tokens_seen": 3126657024 + }, + { + "epoch": 0.9, + "learning_rate": 5.300112341518216e-05, + "loss": 1.3221, + "theoretical_loss": 3.3387228807258493, + "tokens_seen": 3126788096 + }, + { + "epoch": 0.9, + "learning_rate": 5.296100144439095e-05, + "loss": 1.3157, + "theoretical_loss": 3.3387123014730875, + "tokens_seen": 3126919168 + }, + { + "epoch": 0.9, + "learning_rate": 5.292087947359974e-05, + "loss": 1.2431, + "theoretical_loss": 3.338701722787931, + "tokens_seen": 3127050240 + }, + { + "epoch": 0.9, + "learning_rate": 5.288075750280854e-05, + "loss": 1.2533, + "theoretical_loss": 3.3386911446703262, + "tokens_seen": 3127181312 + }, + { + "epoch": 0.9, + "learning_rate": 5.284063553201733e-05, + "loss": 1.2968, + "theoretical_loss": 3.3386805671202184, + "tokens_seen": 3127312384 + }, + { + "epoch": 0.9, + "learning_rate": 5.280051356122613e-05, + "loss": 1.2811, + "theoretical_loss": 3.338669990137553, + "tokens_seen": 3127443456 + }, + { + "epoch": 0.9, + "learning_rate": 5.276039159043492e-05, + "loss": 1.2756, + "theoretical_loss": 3.3386594137222767, + "tokens_seen": 3127574528 + }, + { + "epoch": 0.9, + "learning_rate": 5.272026961964372e-05, + "loss": 1.2984, + "theoretical_loss": 3.338648837874334, + "tokens_seen": 3127705600 + }, + { + "epoch": 0.9, + "learning_rate": 5.268014764885251e-05, + "loss": 1.3136, + "theoretical_loss": 3.338638262593672, + "tokens_seen": 3127836672 + }, + { + "epoch": 0.9, + "learning_rate": 5.264002567806131e-05, + "loss": 1.2811, + "theoretical_loss": 3.338627687880236, + "tokens_seen": 3127967744 + }, + { + "epoch": 0.9, + "learning_rate": 5.25999037072701e-05, + "loss": 1.1886, + "theoretical_loss": 3.3386171137339713, + "tokens_seen": 3128098816 + }, + { + "epoch": 0.9, + "learning_rate": 5.25597817364789e-05, + "loss": 1.2214, + "theoretical_loss": 3.3386065401548244, + "tokens_seen": 3128229888 + }, + { + "epoch": 0.9, + "learning_rate": 5.251965976568769e-05, + "loss": 1.2203, + "theoretical_loss": 3.338595967142741, + "tokens_seen": 3128360960 + }, + { + "epoch": 0.9, + "learning_rate": 5.247953779489649e-05, + "loss": 1.3369, + "theoretical_loss": 3.3385853946976667, + "tokens_seen": 3128492032 + }, + { + "epoch": 0.9, + "learning_rate": 5.243941582410528e-05, + "loss": 1.2479, + "theoretical_loss": 3.3385748228195475, + "tokens_seen": 3128623104 + }, + { + "epoch": 0.9, + "learning_rate": 5.2399293853314075e-05, + "loss": 1.2331, + "theoretical_loss": 3.338564251508329, + "tokens_seen": 3128754176 + }, + { + "epoch": 0.9, + "learning_rate": 5.2359171882522875e-05, + "loss": 1.2719, + "theoretical_loss": 3.3385536807639578, + "tokens_seen": 3128885248 + }, + { + "epoch": 0.9, + "learning_rate": 5.231904991173166e-05, + "loss": 1.2642, + "theoretical_loss": 3.3385431105863788, + "tokens_seen": 3129016320 + }, + { + "epoch": 0.9, + "learning_rate": 5.227892794094046e-05, + "loss": 1.299, + "theoretical_loss": 3.3385325409755384, + "tokens_seen": 3129147392 + }, + { + "epoch": 0.9, + "learning_rate": 5.223880597014925e-05, + "loss": 1.2242, + "theoretical_loss": 3.338521971931382, + "tokens_seen": 3129278464 + }, + { + "epoch": 0.9, + "learning_rate": 5.219868399935805e-05, + "loss": 1.2007, + "theoretical_loss": 3.3385114034538566, + "tokens_seen": 3129409536 + }, + { + "epoch": 0.9, + "learning_rate": 5.2158562028566845e-05, + "loss": 1.2424, + "theoretical_loss": 3.3385008355429067, + "tokens_seen": 3129540608 + }, + { + "epoch": 0.9, + "learning_rate": 5.2118440057775645e-05, + "loss": 1.2577, + "theoretical_loss": 3.338490268198479, + "tokens_seen": 3129671680 + }, + { + "epoch": 0.9, + "learning_rate": 5.207831808698443e-05, + "loss": 1.2989, + "theoretical_loss": 3.3384797014205194, + "tokens_seen": 3129802752 + }, + { + "epoch": 0.9, + "learning_rate": 5.203819611619323e-05, + "loss": 1.3584, + "theoretical_loss": 3.3384691352089737, + "tokens_seen": 3129933824 + }, + { + "epoch": 0.9, + "learning_rate": 5.199807414540202e-05, + "loss": 1.3458, + "theoretical_loss": 3.3384585695637874, + "tokens_seen": 3130064896 + }, + { + "epoch": 0.9, + "learning_rate": 5.1957952174610816e-05, + "loss": 1.2099, + "theoretical_loss": 3.338448004484907, + "tokens_seen": 3130195968 + }, + { + "epoch": 0.9, + "learning_rate": 5.1917830203819615e-05, + "loss": 1.3044, + "theoretical_loss": 3.3384374399722785, + "tokens_seen": 3130327040 + }, + { + "epoch": 0.9, + "learning_rate": 5.187770823302841e-05, + "loss": 1.3261, + "theoretical_loss": 3.3384268760258475, + "tokens_seen": 3130458112 + }, + { + "epoch": 0.9, + "learning_rate": 5.18375862622372e-05, + "loss": 1.2301, + "theoretical_loss": 3.3384163126455597, + "tokens_seen": 3130589184 + }, + { + "epoch": 0.9, + "learning_rate": 5.1797464291445994e-05, + "loss": 1.2285, + "theoretical_loss": 3.3384057498313617, + "tokens_seen": 3130720256 + }, + { + "epoch": 0.9, + "learning_rate": 5.175734232065479e-05, + "loss": 1.277, + "theoretical_loss": 3.338395187583199, + "tokens_seen": 3130851328 + }, + { + "epoch": 0.9, + "learning_rate": 5.1717220349863586e-05, + "loss": 1.2603, + "theoretical_loss": 3.3383846259010177, + "tokens_seen": 3130982400 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.9953626990318298, + "objective/train/avg_token_score": 0.004637277219444513, + "objective/train/docs_used": 1137606, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.17179274559021, + "objective/train/original_loss": 1.177140474319458, + "objective/train/theoretical_loss": 3.338374064784764, + "objective/train/tokens_used": 1501572576, + "objective/train/unlikelihood_loss": 0.015586144290864468, + "theoretical_loss": 3.338374064784764, + "tokens_seen": 3131113472 + }, + { + "epoch": 0.9, + "learning_rate": 5.1677098379072385e-05, + "loss": 1.2279, + "theoretical_loss": 3.338374064784764, + "tokens_seen": 3131113472 + }, + { + "epoch": 0.9, + "learning_rate": 5.163697640828118e-05, + "loss": 1.2231, + "theoretical_loss": 3.3383635042343833, + "tokens_seen": 3131244544 + }, + { + "epoch": 0.9, + "learning_rate": 5.159685443748997e-05, + "loss": 1.2821, + "theoretical_loss": 3.3383529442498228, + "tokens_seen": 3131375616 + }, + { + "epoch": 0.9, + "learning_rate": 5.1556732466698764e-05, + "loss": 1.1728, + "theoretical_loss": 3.338342384831027, + "tokens_seen": 3131506688 + }, + { + "epoch": 0.9, + "learning_rate": 5.151661049590756e-05, + "loss": 1.286, + "theoretical_loss": 3.3383318259779426, + "tokens_seen": 3131637760 + }, + { + "epoch": 0.9, + "learning_rate": 5.1476488525116356e-05, + "loss": 1.254, + "theoretical_loss": 3.3383212676905156, + "tokens_seen": 3131768832 + }, + { + "epoch": 0.9, + "learning_rate": 5.143636655432515e-05, + "loss": 1.2281, + "theoretical_loss": 3.3383107099686926, + "tokens_seen": 3131899904 + }, + { + "epoch": 0.9, + "learning_rate": 5.139624458353395e-05, + "loss": 1.1871, + "theoretical_loss": 3.3383001528124185, + "tokens_seen": 3132030976 + }, + { + "epoch": 0.9, + "learning_rate": 5.1356122612742734e-05, + "loss": 1.2564, + "theoretical_loss": 3.3382895962216406, + "tokens_seen": 3132162048 + }, + { + "epoch": 0.9, + "learning_rate": 5.1316000641951534e-05, + "loss": 1.2225, + "theoretical_loss": 3.3382790401963036, + "tokens_seen": 3132293120 + }, + { + "epoch": 0.9, + "learning_rate": 5.1275878671160326e-05, + "loss": 1.2259, + "theoretical_loss": 3.3382684847363544, + "tokens_seen": 3132424192 + }, + { + "epoch": 0.9, + "learning_rate": 5.1235756700369126e-05, + "loss": 1.2612, + "theoretical_loss": 3.3382579298417387, + "tokens_seen": 3132555264 + }, + { + "epoch": 0.9, + "learning_rate": 5.119563472957792e-05, + "loss": 1.2418, + "theoretical_loss": 3.338247375512403, + "tokens_seen": 3132686336 + }, + { + "epoch": 0.9, + "learning_rate": 5.115551275878672e-05, + "loss": 1.2935, + "theoretical_loss": 3.338236821748293, + "tokens_seen": 3132817408 + }, + { + "epoch": 0.9, + "learning_rate": 5.1115390787995504e-05, + "loss": 1.2478, + "theoretical_loss": 3.338226268549355, + "tokens_seen": 3132948480 + }, + { + "epoch": 0.9, + "learning_rate": 5.1075268817204304e-05, + "loss": 1.3065, + "theoretical_loss": 3.3382157159155352, + "tokens_seen": 3133079552 + }, + { + "epoch": 0.9, + "learning_rate": 5.1035146846413096e-05, + "loss": 1.2154, + "theoretical_loss": 3.338205163846779, + "tokens_seen": 3133210624 + }, + { + "epoch": 0.9, + "learning_rate": 5.099502487562189e-05, + "loss": 1.3651, + "theoretical_loss": 3.3381946123430337, + "tokens_seen": 3133341696 + }, + { + "epoch": 0.9, + "learning_rate": 5.095490290483069e-05, + "loss": 1.2782, + "theoretical_loss": 3.338184061404244, + "tokens_seen": 3133472768 + }, + { + "epoch": 0.9, + "learning_rate": 5.091478093403948e-05, + "loss": 1.1962, + "theoretical_loss": 3.338173511030357, + "tokens_seen": 3133603840 + }, + { + "epoch": 0.9, + "learning_rate": 5.0874658963248274e-05, + "loss": 1.2924, + "theoretical_loss": 3.3381629612213186, + "tokens_seen": 3133734912 + }, + { + "epoch": 0.9, + "learning_rate": 5.083453699245707e-05, + "loss": 1.371, + "theoretical_loss": 3.338152411977075, + "tokens_seen": 3133865984 + }, + { + "epoch": 0.9, + "learning_rate": 5.0794415021665867e-05, + "loss": 1.2886, + "theoretical_loss": 3.3381418632975723, + "tokens_seen": 3133997056 + }, + { + "epoch": 0.9, + "learning_rate": 5.075429305087466e-05, + "loss": 1.2547, + "theoretical_loss": 3.338131315182756, + "tokens_seen": 3134128128 + }, + { + "epoch": 0.9, + "learning_rate": 5.071417108008346e-05, + "loss": 1.2677, + "theoretical_loss": 3.3381207676325735, + "tokens_seen": 3134259200 + }, + { + "epoch": 0.9, + "learning_rate": 5.067404910929225e-05, + "loss": 1.3306, + "theoretical_loss": 3.3381102206469704, + "tokens_seen": 3134390272 + }, + { + "epoch": 0.9, + "learning_rate": 5.0633927138501044e-05, + "loss": 1.2883, + "theoretical_loss": 3.338099674225892, + "tokens_seen": 3134521344 + }, + { + "epoch": 0.9, + "learning_rate": 5.059380516770984e-05, + "loss": 1.2118, + "theoretical_loss": 3.338089128369286, + "tokens_seen": 3134652416 + }, + { + "epoch": 0.9, + "learning_rate": 5.055368319691863e-05, + "loss": 1.2571, + "theoretical_loss": 3.3380785830770976, + "tokens_seen": 3134783488 + }, + { + "epoch": 0.9, + "learning_rate": 5.051356122612743e-05, + "loss": 1.2027, + "theoretical_loss": 3.3380680383492733, + "tokens_seen": 3134914560 + }, + { + "epoch": 0.9, + "learning_rate": 5.047343925533622e-05, + "loss": 1.3354, + "theoretical_loss": 3.338057494185759, + "tokens_seen": 3135045632 + }, + { + "epoch": 0.9, + "learning_rate": 5.043331728454502e-05, + "loss": 1.3637, + "theoretical_loss": 3.338046950586502, + "tokens_seen": 3135176704 + }, + { + "epoch": 0.9, + "learning_rate": 5.039319531375381e-05, + "loss": 1.2944, + "theoretical_loss": 3.3380364075514466, + "tokens_seen": 3135307776 + }, + { + "epoch": 0.9, + "learning_rate": 5.035307334296261e-05, + "loss": 1.2221, + "theoretical_loss": 3.3380258650805406, + "tokens_seen": 3135438848 + }, + { + "epoch": 0.9, + "learning_rate": 5.03129513721714e-05, + "loss": 1.2927, + "theoretical_loss": 3.33801532317373, + "tokens_seen": 3135569920 + }, + { + "epoch": 0.9, + "learning_rate": 5.02728294013802e-05, + "loss": 1.2293, + "theoretical_loss": 3.33800478183096, + "tokens_seen": 3135700992 + }, + { + "epoch": 0.9, + "learning_rate": 5.023270743058899e-05, + "loss": 1.2415, + "theoretical_loss": 3.3379942410521783, + "tokens_seen": 3135832064 + }, + { + "epoch": 0.9, + "learning_rate": 5.019258545979779e-05, + "loss": 1.2612, + "theoretical_loss": 3.3379837008373303, + "tokens_seen": 3135963136 + }, + { + "epoch": 0.9, + "learning_rate": 5.015246348900658e-05, + "loss": 1.2034, + "theoretical_loss": 3.3379731611863623, + "tokens_seen": 3136094208 + }, + { + "epoch": 0.9, + "learning_rate": 5.011234151821538e-05, + "loss": 1.2358, + "theoretical_loss": 3.337962622099221, + "tokens_seen": 3136225280 + }, + { + "epoch": 0.9, + "learning_rate": 5.007221954742417e-05, + "loss": 1.2632, + "theoretical_loss": 3.337952083575852, + "tokens_seen": 3136356352 + }, + { + "epoch": 0.9, + "learning_rate": 5.003209757663296e-05, + "loss": 1.2539, + "theoretical_loss": 3.3379415456162023, + "tokens_seen": 3136487424 + }, + { + "epoch": 0.9, + "learning_rate": 4.999197560584176e-05, + "loss": 1.2503, + "theoretical_loss": 3.3379310082202176, + "tokens_seen": 3136618496 + }, + { + "epoch": 0.9, + "learning_rate": 4.9951853635050555e-05, + "loss": 1.2168, + "theoretical_loss": 3.3379204713878448, + "tokens_seen": 3136749568 + }, + { + "epoch": 0.9, + "learning_rate": 4.991173166425935e-05, + "loss": 1.1654, + "theoretical_loss": 3.337909935119029, + "tokens_seen": 3136880640 + }, + { + "epoch": 0.9, + "learning_rate": 4.987160969346814e-05, + "loss": 1.2043, + "theoretical_loss": 3.3378993994137183, + "tokens_seen": 3137011712 + }, + { + "epoch": 0.9, + "learning_rate": 4.983148772267694e-05, + "loss": 1.2025, + "theoretical_loss": 3.3378888642718576, + "tokens_seen": 3137142784 + }, + { + "epoch": 0.9, + "learning_rate": 4.979136575188573e-05, + "loss": 1.2561, + "theoretical_loss": 3.337878329693394, + "tokens_seen": 3137273856 + }, + { + "epoch": 0.9, + "learning_rate": 4.975124378109453e-05, + "loss": 1.1916, + "theoretical_loss": 3.337867795678273, + "tokens_seen": 3137404928 + }, + { + "epoch": 0.9, + "learning_rate": 4.9711121810303325e-05, + "loss": 1.242, + "theoretical_loss": 3.337857262226442, + "tokens_seen": 3137536000 + }, + { + "epoch": 0.9, + "objective/train/aligned_token_count": 0.9934064149856567, + "objective/train/avg_token_score": 0.006593545898795128, + "objective/train/docs_used": 1140026, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2882694005966187, + "objective/train/original_loss": 1.2971916198730469, + "objective/train/theoretical_loss": 3.337846729337847, + "objective/train/tokens_used": 1508126176, + "objective/train/unlikelihood_loss": 0.018268000334501266, + "theoretical_loss": 3.337846729337847, + "tokens_seen": 3137667072 + }, + { + "epoch": 0.9, + "learning_rate": 4.967099983951212e-05, + "loss": 1.2531, + "theoretical_loss": 3.337846729337847, + "tokens_seen": 3137667072 + }, + { + "epoch": 0.9, + "learning_rate": 4.963087786872091e-05, + "loss": 1.253, + "theoretical_loss": 3.3378361970124337, + "tokens_seen": 3137798144 + }, + { + "epoch": 0.9, + "learning_rate": 4.9590755897929703e-05, + "loss": 1.254, + "theoretical_loss": 3.3378256652501492, + "tokens_seen": 3137929216 + }, + { + "epoch": 0.9, + "learning_rate": 4.95506339271385e-05, + "loss": 1.2653, + "theoretical_loss": 3.3378151340509397, + "tokens_seen": 3138060288 + }, + { + "epoch": 0.9, + "learning_rate": 4.9510511956347296e-05, + "loss": 1.2208, + "theoretical_loss": 3.3378046034147513, + "tokens_seen": 3138191360 + }, + { + "epoch": 0.9, + "learning_rate": 4.9470389985556095e-05, + "loss": 1.2957, + "theoretical_loss": 3.3377940733415308, + "tokens_seen": 3138322432 + }, + { + "epoch": 0.9, + "learning_rate": 4.943026801476488e-05, + "loss": 1.2276, + "theoretical_loss": 3.3377835438312244, + "tokens_seen": 3138453504 + }, + { + "epoch": 0.9, + "learning_rate": 4.939014604397368e-05, + "loss": 1.2815, + "theoretical_loss": 3.337773014883778, + "tokens_seen": 3138584576 + }, + { + "epoch": 0.9, + "learning_rate": 4.9350024073182473e-05, + "loss": 1.2348, + "theoretical_loss": 3.3377624864991393, + "tokens_seen": 3138715648 + }, + { + "epoch": 0.9, + "learning_rate": 4.930990210239127e-05, + "loss": 1.2622, + "theoretical_loss": 3.3377519586772535, + "tokens_seen": 3138846720 + }, + { + "epoch": 0.9, + "learning_rate": 4.9269780131600066e-05, + "loss": 1.2551, + "theoretical_loss": 3.3377414314180673, + "tokens_seen": 3138977792 + }, + { + "epoch": 0.9, + "learning_rate": 4.9229658160808865e-05, + "loss": 1.2554, + "theoretical_loss": 3.3377309047215276, + "tokens_seen": 3139108864 + }, + { + "epoch": 0.9, + "learning_rate": 4.918953619001765e-05, + "loss": 1.2282, + "theoretical_loss": 3.33772037858758, + "tokens_seen": 3139239936 + }, + { + "epoch": 0.9, + "learning_rate": 4.914941421922645e-05, + "loss": 1.2865, + "theoretical_loss": 3.3377098530161717, + "tokens_seen": 3139371008 + }, + { + "epoch": 0.9, + "learning_rate": 4.9109292248435244e-05, + "loss": 1.287, + "theoretical_loss": 3.337699328007249, + "tokens_seen": 3139502080 + }, + { + "epoch": 0.9, + "learning_rate": 4.9069170277644036e-05, + "loss": 1.2699, + "theoretical_loss": 3.3376888035607584, + "tokens_seen": 3139633152 + }, + { + "epoch": 0.9, + "learning_rate": 4.9029048306852836e-05, + "loss": 1.2891, + "theoretical_loss": 3.3376782796766458, + "tokens_seen": 3139764224 + }, + { + "epoch": 0.9, + "learning_rate": 4.898892633606163e-05, + "loss": 1.2469, + "theoretical_loss": 3.3376677563548585, + "tokens_seen": 3139895296 + }, + { + "epoch": 0.9, + "learning_rate": 4.894880436527042e-05, + "loss": 1.2916, + "theoretical_loss": 3.3376572335953423, + "tokens_seen": 3140026368 + }, + { + "epoch": 0.9, + "learning_rate": 4.8908682394479214e-05, + "loss": 1.1703, + "theoretical_loss": 3.337646711398044, + "tokens_seen": 3140157440 + }, + { + "epoch": 0.9, + "learning_rate": 4.8868560423688014e-05, + "loss": 1.321, + "theoretical_loss": 3.33763618976291, + "tokens_seen": 3140288512 + }, + { + "epoch": 0.9, + "learning_rate": 4.8828438452896806e-05, + "loss": 1.1915, + "theoretical_loss": 3.337625668689887, + "tokens_seen": 3140419584 + }, + { + "epoch": 0.9, + "learning_rate": 4.8788316482105606e-05, + "loss": 1.2446, + "theoretical_loss": 3.3376151481789216, + "tokens_seen": 3140550656 + }, + { + "epoch": 0.9, + "learning_rate": 4.87481945113144e-05, + "loss": 1.201, + "theoretical_loss": 3.33760462822996, + "tokens_seen": 3140681728 + }, + { + "epoch": 0.9, + "learning_rate": 4.870807254052319e-05, + "loss": 1.2248, + "theoretical_loss": 3.3375941088429486, + "tokens_seen": 3140812800 + }, + { + "epoch": 0.9, + "learning_rate": 4.8667950569731984e-05, + "loss": 1.3203, + "theoretical_loss": 3.337583590017834, + "tokens_seen": 3140943872 + }, + { + "epoch": 0.9, + "learning_rate": 4.862782859894078e-05, + "loss": 1.2773, + "theoretical_loss": 3.337573071754563, + "tokens_seen": 3141074944 + }, + { + "epoch": 0.9, + "learning_rate": 4.8587706628149576e-05, + "loss": 1.2465, + "theoretical_loss": 3.3375625540530827, + "tokens_seen": 3141206016 + }, + { + "epoch": 0.9, + "learning_rate": 4.854758465735837e-05, + "loss": 1.2709, + "theoretical_loss": 3.3375520369133382, + "tokens_seen": 3141337088 + }, + { + "epoch": 0.9, + "learning_rate": 4.850746268656717e-05, + "loss": 1.2867, + "theoretical_loss": 3.337541520335277, + "tokens_seen": 3141468160 + }, + { + "epoch": 0.9, + "learning_rate": 4.8467340715775955e-05, + "loss": 1.2557, + "theoretical_loss": 3.337531004318846, + "tokens_seen": 3141599232 + }, + { + "epoch": 0.9, + "learning_rate": 4.8427218744984754e-05, + "loss": 1.3258, + "theoretical_loss": 3.3375204888639907, + "tokens_seen": 3141730304 + }, + { + "epoch": 0.9, + "learning_rate": 4.838709677419355e-05, + "loss": 1.2476, + "theoretical_loss": 3.3375099739706586, + "tokens_seen": 3141861376 + }, + { + "epoch": 0.9, + "learning_rate": 4.8346974803402347e-05, + "loss": 1.2538, + "theoretical_loss": 3.337499459638796, + "tokens_seen": 3141992448 + }, + { + "epoch": 0.9, + "learning_rate": 4.830685283261114e-05, + "loss": 1.2485, + "theoretical_loss": 3.3374889458683494, + "tokens_seen": 3142123520 + }, + { + "epoch": 0.9, + "learning_rate": 4.826673086181994e-05, + "loss": 1.2173, + "theoretical_loss": 3.3374784326592657, + "tokens_seen": 3142254592 + }, + { + "epoch": 0.9, + "learning_rate": 4.822660889102873e-05, + "loss": 1.2966, + "theoretical_loss": 3.337467920011491, + "tokens_seen": 3142385664 + }, + { + "epoch": 0.9, + "learning_rate": 4.818648692023752e-05, + "loss": 1.1918, + "theoretical_loss": 3.337457407924972, + "tokens_seen": 3142516736 + }, + { + "epoch": 0.9, + "learning_rate": 4.814636494944632e-05, + "loss": 1.2678, + "theoretical_loss": 3.337446896399656, + "tokens_seen": 3142647808 + }, + { + "epoch": 0.9, + "learning_rate": 4.810624297865511e-05, + "loss": 1.2289, + "theoretical_loss": 3.337436385435489, + "tokens_seen": 3142778880 + }, + { + "epoch": 0.9, + "learning_rate": 4.806612100786391e-05, + "loss": 1.2224, + "theoretical_loss": 3.337425875032418, + "tokens_seen": 3142909952 + }, + { + "epoch": 0.9, + "learning_rate": 4.80259990370727e-05, + "loss": 1.2297, + "theoretical_loss": 3.337415365190389, + "tokens_seen": 3143041024 + }, + { + "epoch": 0.9, + "learning_rate": 4.79858770662815e-05, + "loss": 1.3679, + "theoretical_loss": 3.33740485590935, + "tokens_seen": 3143172096 + }, + { + "epoch": 0.91, + "learning_rate": 4.794575509549029e-05, + "loss": 1.241, + "theoretical_loss": 3.337394347189246, + "tokens_seen": 3143303168 + }, + { + "epoch": 0.91, + "learning_rate": 4.790563312469909e-05, + "loss": 1.2327, + "theoretical_loss": 3.3373838390300246, + "tokens_seen": 3143434240 + }, + { + "epoch": 0.91, + "learning_rate": 4.786551115390788e-05, + "loss": 1.1974, + "theoretical_loss": 3.3373733314316323, + "tokens_seen": 3143565312 + }, + { + "epoch": 0.91, + "learning_rate": 4.782538918311668e-05, + "loss": 1.2189, + "theoretical_loss": 3.3373628243940163, + "tokens_seen": 3143696384 + }, + { + "epoch": 0.91, + "learning_rate": 4.778526721232547e-05, + "loss": 1.2913, + "theoretical_loss": 3.3373523179171225, + "tokens_seen": 3143827456 + }, + { + "epoch": 0.91, + "learning_rate": 4.774514524153427e-05, + "loss": 1.293, + "theoretical_loss": 3.3373418120008975, + "tokens_seen": 3143958528 + }, + { + "epoch": 0.91, + "learning_rate": 4.770502327074306e-05, + "loss": 1.2862, + "theoretical_loss": 3.337331306645289, + "tokens_seen": 3144089600 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.9939411878585815, + "objective/train/avg_token_score": 0.006058818195015192, + "objective/train/docs_used": 1142279, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2240748405456543, + "objective/train/original_loss": 1.2318867444992065, + "objective/train/theoretical_loss": 3.337320801850243, + "objective/train/tokens_used": 1514679776, + "objective/train/unlikelihood_loss": 0.020201629027724266, + "theoretical_loss": 3.337320801850243, + "tokens_seen": 3144220672 + }, + { + "epoch": 0.91, + "learning_rate": 4.766490129995185e-05, + "loss": 1.254, + "theoretical_loss": 3.337320801850243, + "tokens_seen": 3144220672 + }, + { + "epoch": 0.91, + "learning_rate": 4.762477932916065e-05, + "loss": 1.1576, + "theoretical_loss": 3.3373102976157063, + "tokens_seen": 3144351744 + }, + { + "epoch": 0.91, + "learning_rate": 4.758465735836944e-05, + "loss": 1.2881, + "theoretical_loss": 3.337299793941626, + "tokens_seen": 3144482816 + }, + { + "epoch": 0.91, + "learning_rate": 4.754453538757824e-05, + "loss": 1.2111, + "theoretical_loss": 3.337289290827948, + "tokens_seen": 3144613888 + }, + { + "epoch": 0.91, + "learning_rate": 4.7504413416787035e-05, + "loss": 1.2872, + "theoretical_loss": 3.33727878827462, + "tokens_seen": 3144744960 + }, + { + "epoch": 0.91, + "learning_rate": 4.746429144599583e-05, + "loss": 1.2731, + "theoretical_loss": 3.337268286281588, + "tokens_seen": 3144876032 + }, + { + "epoch": 0.91, + "learning_rate": 4.742416947520462e-05, + "loss": 1.2365, + "theoretical_loss": 3.3372577848487994, + "tokens_seen": 3145007104 + }, + { + "epoch": 0.91, + "learning_rate": 4.738404750441342e-05, + "loss": 1.3013, + "theoretical_loss": 3.3372472839762004, + "tokens_seen": 3145138176 + }, + { + "epoch": 0.91, + "learning_rate": 4.734392553362221e-05, + "loss": 1.2427, + "theoretical_loss": 3.337236783663738, + "tokens_seen": 3145269248 + }, + { + "epoch": 0.91, + "learning_rate": 4.730380356283101e-05, + "loss": 1.2124, + "theoretical_loss": 3.3372262839113596, + "tokens_seen": 3145400320 + }, + { + "epoch": 0.91, + "learning_rate": 4.7263681592039805e-05, + "loss": 1.1997, + "theoretical_loss": 3.3372157847190107, + "tokens_seen": 3145531392 + }, + { + "epoch": 0.91, + "learning_rate": 4.722355962124859e-05, + "loss": 1.2329, + "theoretical_loss": 3.3372052860866392, + "tokens_seen": 3145662464 + }, + { + "epoch": 0.91, + "learning_rate": 4.718343765045739e-05, + "loss": 1.2412, + "theoretical_loss": 3.337194788014191, + "tokens_seen": 3145793536 + }, + { + "epoch": 0.91, + "learning_rate": 4.714331567966618e-05, + "loss": 1.296, + "theoretical_loss": 3.337184290501614, + "tokens_seen": 3145924608 + }, + { + "epoch": 0.91, + "learning_rate": 4.710319370887498e-05, + "loss": 1.2507, + "theoretical_loss": 3.337173793548854, + "tokens_seen": 3146055680 + }, + { + "epoch": 0.91, + "learning_rate": 4.7063071738083776e-05, + "loss": 1.2595, + "theoretical_loss": 3.3371632971558585, + "tokens_seen": 3146186752 + }, + { + "epoch": 0.91, + "learning_rate": 4.7022949767292575e-05, + "loss": 1.2267, + "theoretical_loss": 3.337152801322574, + "tokens_seen": 3146317824 + }, + { + "epoch": 0.91, + "learning_rate": 4.698282779650136e-05, + "loss": 1.2001, + "theoretical_loss": 3.3371423060489476, + "tokens_seen": 3146448896 + }, + { + "epoch": 0.91, + "learning_rate": 4.694270582571016e-05, + "loss": 1.2757, + "theoretical_loss": 3.3371318113349258, + "tokens_seen": 3146579968 + }, + { + "epoch": 0.91, + "learning_rate": 4.6902583854918953e-05, + "loss": 1.2117, + "theoretical_loss": 3.3371213171804555, + "tokens_seen": 3146711040 + }, + { + "epoch": 0.91, + "learning_rate": 4.686246188412775e-05, + "loss": 1.2127, + "theoretical_loss": 3.3371108235854834, + "tokens_seen": 3146842112 + }, + { + "epoch": 0.91, + "learning_rate": 4.6822339913336546e-05, + "loss": 1.2124, + "theoretical_loss": 3.337100330549957, + "tokens_seen": 3146973184 + }, + { + "epoch": 0.91, + "learning_rate": 4.678221794254534e-05, + "loss": 1.2359, + "theoretical_loss": 3.337089838073823, + "tokens_seen": 3147104256 + }, + { + "epoch": 0.91, + "learning_rate": 4.674209597175413e-05, + "loss": 1.2553, + "theoretical_loss": 3.337079346157028, + "tokens_seen": 3147235328 + }, + { + "epoch": 0.91, + "learning_rate": 4.6701974000962924e-05, + "loss": 1.2664, + "theoretical_loss": 3.3370688547995186, + "tokens_seen": 3147366400 + }, + { + "epoch": 0.91, + "learning_rate": 4.6661852030171723e-05, + "loss": 1.2646, + "theoretical_loss": 3.337058364001243, + "tokens_seen": 3147497472 + }, + { + "epoch": 0.91, + "learning_rate": 4.6621730059380516e-05, + "loss": 1.3132, + "theoretical_loss": 3.3370478737621463, + "tokens_seen": 3147628544 + }, + { + "epoch": 0.91, + "learning_rate": 4.6581608088589316e-05, + "loss": 1.2409, + "theoretical_loss": 3.3370373840821763, + "tokens_seen": 3147759616 + }, + { + "epoch": 0.91, + "learning_rate": 4.654148611779811e-05, + "loss": 1.211, + "theoretical_loss": 3.3370268949612805, + "tokens_seen": 3147890688 + }, + { + "epoch": 0.91, + "learning_rate": 4.65013641470069e-05, + "loss": 1.2279, + "theoretical_loss": 3.337016406399405, + "tokens_seen": 3148021760 + }, + { + "epoch": 0.91, + "learning_rate": 4.6461242176215694e-05, + "loss": 1.2077, + "theoretical_loss": 3.337005918396497, + "tokens_seen": 3148152832 + }, + { + "epoch": 0.91, + "learning_rate": 4.6421120205424494e-05, + "loss": 1.1973, + "theoretical_loss": 3.336995430952503, + "tokens_seen": 3148283904 + }, + { + "epoch": 0.91, + "learning_rate": 4.6380998234633286e-05, + "loss": 1.2314, + "theoretical_loss": 3.336984944067371, + "tokens_seen": 3148414976 + }, + { + "epoch": 0.91, + "learning_rate": 4.6340876263842086e-05, + "loss": 1.2436, + "theoretical_loss": 3.336974457741047, + "tokens_seen": 3148546048 + }, + { + "epoch": 0.91, + "learning_rate": 4.630075429305088e-05, + "loss": 1.21, + "theoretical_loss": 3.336963971973478, + "tokens_seen": 3148677120 + }, + { + "epoch": 0.91, + "learning_rate": 4.6260632322259665e-05, + "loss": 1.2137, + "theoretical_loss": 3.336953486764612, + "tokens_seen": 3148808192 + }, + { + "epoch": 0.91, + "learning_rate": 4.6220510351468464e-05, + "loss": 1.2924, + "theoretical_loss": 3.3369430021143947, + "tokens_seen": 3148939264 + }, + { + "epoch": 0.91, + "learning_rate": 4.618038838067726e-05, + "loss": 1.3122, + "theoretical_loss": 3.3369325180227736, + "tokens_seen": 3149070336 + }, + { + "epoch": 0.91, + "learning_rate": 4.6140266409886056e-05, + "loss": 1.2684, + "theoretical_loss": 3.336922034489696, + "tokens_seen": 3149201408 + }, + { + "epoch": 0.91, + "learning_rate": 4.610014443909485e-05, + "loss": 1.2852, + "theoretical_loss": 3.3369115515151084, + "tokens_seen": 3149332480 + }, + { + "epoch": 0.91, + "learning_rate": 4.606002246830365e-05, + "loss": 1.3167, + "theoretical_loss": 3.3369010690989582, + "tokens_seen": 3149463552 + }, + { + "epoch": 0.91, + "learning_rate": 4.6019900497512435e-05, + "loss": 1.2611, + "theoretical_loss": 3.336890587241192, + "tokens_seen": 3149594624 + }, + { + "epoch": 0.91, + "learning_rate": 4.5979778526721234e-05, + "loss": 1.2241, + "theoretical_loss": 3.336880105941757, + "tokens_seen": 3149725696 + }, + { + "epoch": 0.91, + "learning_rate": 4.593965655593003e-05, + "loss": 1.3124, + "theoretical_loss": 3.3368696252006003, + "tokens_seen": 3149856768 + }, + { + "epoch": 0.91, + "learning_rate": 4.5899534585138826e-05, + "loss": 1.228, + "theoretical_loss": 3.336859145017669, + "tokens_seen": 3149987840 + }, + { + "epoch": 0.91, + "learning_rate": 4.585941261434762e-05, + "loss": 1.2187, + "theoretical_loss": 3.3368486653929104, + "tokens_seen": 3150118912 + }, + { + "epoch": 0.91, + "learning_rate": 4.581929064355641e-05, + "loss": 1.3155, + "theoretical_loss": 3.3368381863262706, + "tokens_seen": 3150249984 + }, + { + "epoch": 0.91, + "learning_rate": 4.5779168672765205e-05, + "loss": 1.1925, + "theoretical_loss": 3.3368277078176973, + "tokens_seen": 3150381056 + }, + { + "epoch": 0.91, + "learning_rate": 4.5739046701974e-05, + "loss": 1.2433, + "theoretical_loss": 3.336817229867138, + "tokens_seen": 3150512128 + }, + { + "epoch": 0.91, + "learning_rate": 4.56989247311828e-05, + "loss": 1.3265, + "theoretical_loss": 3.336806752474539, + "tokens_seen": 3150643200 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.9914184212684631, + "objective/train/avg_token_score": 0.008581587113440037, + "objective/train/docs_used": 1144741, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1823221445083618, + "objective/train/original_loss": 1.193099021911621, + "objective/train/theoretical_loss": 3.3367962756398475, + "objective/train/tokens_used": 1521233376, + "objective/train/unlikelihood_loss": 0.02356528304517269, + "theoretical_loss": 3.3367962756398475, + "tokens_seen": 3150774272 + }, + { + "epoch": 0.91, + "learning_rate": 4.565880276039159e-05, + "loss": 1.2176, + "theoretical_loss": 3.3367962756398475, + "tokens_seen": 3150774272 + }, + { + "epoch": 0.91, + "learning_rate": 4.561868078960039e-05, + "loss": 1.2877, + "theoretical_loss": 3.3367857993630112, + "tokens_seen": 3150905344 + }, + { + "epoch": 0.91, + "learning_rate": 4.557855881880918e-05, + "loss": 1.142, + "theoretical_loss": 3.3367753236439763, + "tokens_seen": 3151036416 + }, + { + "epoch": 0.91, + "learning_rate": 4.5538436848017975e-05, + "loss": 1.2908, + "theoretical_loss": 3.3367648484826904, + "tokens_seen": 3151167488 + }, + { + "epoch": 0.91, + "learning_rate": 4.549831487722677e-05, + "loss": 1.256, + "theoretical_loss": 3.3367543738791006, + "tokens_seen": 3151298560 + }, + { + "epoch": 0.91, + "learning_rate": 4.545819290643557e-05, + "loss": 1.1891, + "theoretical_loss": 3.336743899833154, + "tokens_seen": 3151429632 + }, + { + "epoch": 0.91, + "learning_rate": 4.541807093564436e-05, + "loss": 1.3185, + "theoretical_loss": 3.336733426344798, + "tokens_seen": 3151560704 + }, + { + "epoch": 0.91, + "learning_rate": 4.537794896485316e-05, + "loss": 1.2743, + "theoretical_loss": 3.336722953413979, + "tokens_seen": 3151691776 + }, + { + "epoch": 0.91, + "learning_rate": 4.533782699406195e-05, + "loss": 1.2403, + "theoretical_loss": 3.3367124810406446, + "tokens_seen": 3151822848 + }, + { + "epoch": 0.91, + "learning_rate": 4.529770502327074e-05, + "loss": 1.2236, + "theoretical_loss": 3.3367020092247417, + "tokens_seen": 3151953920 + }, + { + "epoch": 0.91, + "learning_rate": 4.525758305247954e-05, + "loss": 1.2654, + "theoretical_loss": 3.336691537966218, + "tokens_seen": 3152084992 + }, + { + "epoch": 0.91, + "learning_rate": 4.521746108168833e-05, + "loss": 1.2101, + "theoretical_loss": 3.3366810672650202, + "tokens_seen": 3152216064 + }, + { + "epoch": 0.91, + "learning_rate": 4.517733911089713e-05, + "loss": 1.2556, + "theoretical_loss": 3.3366705971210955, + "tokens_seen": 3152347136 + }, + { + "epoch": 0.91, + "learning_rate": 4.513721714010592e-05, + "loss": 1.2245, + "theoretical_loss": 3.3366601275343912, + "tokens_seen": 3152478208 + }, + { + "epoch": 0.91, + "learning_rate": 4.509709516931472e-05, + "loss": 1.1968, + "theoretical_loss": 3.336649658504854, + "tokens_seen": 3152609280 + }, + { + "epoch": 0.91, + "learning_rate": 4.505697319852351e-05, + "loss": 1.3173, + "theoretical_loss": 3.3366391900324324, + "tokens_seen": 3152740352 + }, + { + "epoch": 0.91, + "learning_rate": 4.501685122773231e-05, + "loss": 1.2444, + "theoretical_loss": 3.336628722117072, + "tokens_seen": 3152871424 + }, + { + "epoch": 0.91, + "learning_rate": 4.49767292569411e-05, + "loss": 1.3664, + "theoretical_loss": 3.336618254758721, + "tokens_seen": 3153002496 + }, + { + "epoch": 0.91, + "learning_rate": 4.49366072861499e-05, + "loss": 1.1816, + "theoretical_loss": 3.336607787957326, + "tokens_seen": 3153133568 + }, + { + "epoch": 0.91, + "learning_rate": 4.489648531535869e-05, + "loss": 1.2368, + "theoretical_loss": 3.3365973217128344, + "tokens_seen": 3153264640 + }, + { + "epoch": 0.91, + "learning_rate": 4.4856363344567486e-05, + "loss": 1.2518, + "theoretical_loss": 3.336586856025194, + "tokens_seen": 3153395712 + }, + { + "epoch": 0.91, + "learning_rate": 4.481624137377628e-05, + "loss": 1.2644, + "theoretical_loss": 3.3365763908943515, + "tokens_seen": 3153526784 + }, + { + "epoch": 0.91, + "learning_rate": 4.477611940298507e-05, + "loss": 1.2639, + "theoretical_loss": 3.3365659263202536, + "tokens_seen": 3153657856 + }, + { + "epoch": 0.91, + "learning_rate": 4.473599743219387e-05, + "loss": 1.2456, + "theoretical_loss": 3.336555462302849, + "tokens_seen": 3153788928 + }, + { + "epoch": 0.91, + "learning_rate": 4.469587546140266e-05, + "loss": 1.2948, + "theoretical_loss": 3.3365449988420837, + "tokens_seen": 3153920000 + }, + { + "epoch": 0.91, + "learning_rate": 4.465575349061146e-05, + "loss": 1.2143, + "theoretical_loss": 3.336534535937905, + "tokens_seen": 3154051072 + }, + { + "epoch": 0.91, + "learning_rate": 4.4615631519820256e-05, + "loss": 1.284, + "theoretical_loss": 3.3365240735902613, + "tokens_seen": 3154182144 + }, + { + "epoch": 0.91, + "learning_rate": 4.457550954902905e-05, + "loss": 1.1677, + "theoretical_loss": 3.3365136117990986, + "tokens_seen": 3154313216 + }, + { + "epoch": 0.91, + "learning_rate": 4.453538757823784e-05, + "loss": 1.2334, + "theoretical_loss": 3.3365031505643645, + "tokens_seen": 3154444288 + }, + { + "epoch": 0.91, + "learning_rate": 4.449526560744664e-05, + "loss": 1.2308, + "theoretical_loss": 3.336492689886007, + "tokens_seen": 3154575360 + }, + { + "epoch": 0.91, + "learning_rate": 4.445514363665543e-05, + "loss": 1.2469, + "theoretical_loss": 3.3364822297639725, + "tokens_seen": 3154706432 + }, + { + "epoch": 0.91, + "learning_rate": 4.4415021665864226e-05, + "loss": 1.2851, + "theoretical_loss": 3.3364717701982083, + "tokens_seen": 3154837504 + }, + { + "epoch": 0.91, + "learning_rate": 4.4374899695073026e-05, + "loss": 1.2906, + "theoretical_loss": 3.3364613111886627, + "tokens_seen": 3154968576 + }, + { + "epoch": 0.91, + "learning_rate": 4.433477772428181e-05, + "loss": 1.31, + "theoretical_loss": 3.336450852735282, + "tokens_seen": 3155099648 + }, + { + "epoch": 0.91, + "learning_rate": 4.429465575349061e-05, + "loss": 1.1993, + "theoretical_loss": 3.3364403948380144, + "tokens_seen": 3155230720 + }, + { + "epoch": 0.91, + "learning_rate": 4.4254533782699404e-05, + "loss": 1.2614, + "theoretical_loss": 3.3364299374968063, + "tokens_seen": 3155361792 + }, + { + "epoch": 0.91, + "learning_rate": 4.4214411811908203e-05, + "loss": 1.2231, + "theoretical_loss": 3.336419480711605, + "tokens_seen": 3155492864 + }, + { + "epoch": 0.91, + "learning_rate": 4.4174289841116996e-05, + "loss": 1.3246, + "theoretical_loss": 3.336409024482359, + "tokens_seen": 3155623936 + }, + { + "epoch": 0.91, + "learning_rate": 4.4134167870325796e-05, + "loss": 1.2069, + "theoretical_loss": 3.336398568809015, + "tokens_seen": 3155755008 + }, + { + "epoch": 0.91, + "learning_rate": 4.409404589953458e-05, + "loss": 1.2625, + "theoretical_loss": 3.33638811369152, + "tokens_seen": 3155886080 + }, + { + "epoch": 0.91, + "learning_rate": 4.405392392874338e-05, + "loss": 1.239, + "theoretical_loss": 3.336377659129822, + "tokens_seen": 3156017152 + }, + { + "epoch": 0.91, + "learning_rate": 4.4013801957952174e-05, + "loss": 1.203, + "theoretical_loss": 3.3363672051238678, + "tokens_seen": 3156148224 + }, + { + "epoch": 0.91, + "learning_rate": 4.3973679987160974e-05, + "loss": 1.1913, + "theoretical_loss": 3.336356751673605, + "tokens_seen": 3156279296 + }, + { + "epoch": 0.91, + "learning_rate": 4.3933558016369766e-05, + "loss": 1.237, + "theoretical_loss": 3.336346298778981, + "tokens_seen": 3156410368 + }, + { + "epoch": 0.91, + "learning_rate": 4.389343604557856e-05, + "loss": 1.2231, + "theoretical_loss": 3.3363358464399435, + "tokens_seen": 3156541440 + }, + { + "epoch": 0.91, + "learning_rate": 4.385331407478736e-05, + "loss": 1.2233, + "theoretical_loss": 3.3363253946564395, + "tokens_seen": 3156672512 + }, + { + "epoch": 0.91, + "learning_rate": 4.3813192103996145e-05, + "loss": 1.3435, + "theoretical_loss": 3.3363149434284165, + "tokens_seen": 3156803584 + }, + { + "epoch": 0.91, + "learning_rate": 4.3773070133204944e-05, + "loss": 1.1919, + "theoretical_loss": 3.336304492755822, + "tokens_seen": 3156934656 + }, + { + "epoch": 0.91, + "learning_rate": 4.373294816241374e-05, + "loss": 1.2882, + "theoretical_loss": 3.3362940426386034, + "tokens_seen": 3157065728 + }, + { + "epoch": 0.91, + "learning_rate": 4.3692826191622536e-05, + "loss": 1.288, + "theoretical_loss": 3.3362835930767076, + "tokens_seen": 3157196800 + }, + { + "epoch": 0.91, + "objective/train/aligned_token_count": 0.991966724395752, + "objective/train/avg_token_score": 0.008033297955989838, + "objective/train/docs_used": 1147172, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3391764163970947, + "objective/train/original_loss": 1.351239800453186, + "objective/train/theoretical_loss": 3.3362731440700832, + "objective/train/tokens_used": 1527786976, + "objective/train/unlikelihood_loss": 0.01976098120212555, + "theoretical_loss": 3.3362731440700832, + "tokens_seen": 3157327872 + }, + { + "epoch": 0.91, + "learning_rate": 4.365270422083133e-05, + "loss": 1.3095, + "theoretical_loss": 3.3362731440700832, + "tokens_seen": 3157327872 + }, + { + "epoch": 0.91, + "learning_rate": 4.361258225004013e-05, + "loss": 1.287, + "theoretical_loss": 3.3362626956186765, + "tokens_seen": 3157458944 + }, + { + "epoch": 0.91, + "learning_rate": 4.3572460279248915e-05, + "loss": 1.2589, + "theoretical_loss": 3.336252247722436, + "tokens_seen": 3157590016 + }, + { + "epoch": 0.91, + "learning_rate": 4.3532338308457714e-05, + "loss": 1.2703, + "theoretical_loss": 3.336241800381308, + "tokens_seen": 3157721088 + }, + { + "epoch": 0.91, + "learning_rate": 4.349221633766651e-05, + "loss": 1.2437, + "theoretical_loss": 3.336231353595241, + "tokens_seen": 3157852160 + }, + { + "epoch": 0.91, + "learning_rate": 4.34520943668753e-05, + "loss": 1.2122, + "theoretical_loss": 3.3362209073641815, + "tokens_seen": 3157983232 + }, + { + "epoch": 0.91, + "learning_rate": 4.34119723960841e-05, + "loss": 1.2484, + "theoretical_loss": 3.336210461688078, + "tokens_seen": 3158114304 + }, + { + "epoch": 0.91, + "learning_rate": 4.337185042529289e-05, + "loss": 1.3123, + "theoretical_loss": 3.3362000165668775, + "tokens_seen": 3158245376 + }, + { + "epoch": 0.91, + "learning_rate": 4.3331728454501685e-05, + "loss": 1.212, + "theoretical_loss": 3.336189572000527, + "tokens_seen": 3158376448 + }, + { + "epoch": 0.91, + "learning_rate": 4.329160648371048e-05, + "loss": 1.2016, + "theoretical_loss": 3.336179127988975, + "tokens_seen": 3158507520 + }, + { + "epoch": 0.91, + "learning_rate": 4.325148451291928e-05, + "loss": 1.2586, + "theoretical_loss": 3.3361686845321685, + "tokens_seen": 3158638592 + }, + { + "epoch": 0.91, + "learning_rate": 4.321136254212807e-05, + "loss": 1.2504, + "theoretical_loss": 3.336158241630055, + "tokens_seen": 3158769664 + }, + { + "epoch": 0.91, + "learning_rate": 4.317124057133687e-05, + "loss": 1.2197, + "theoretical_loss": 3.3361477992825814, + "tokens_seen": 3158900736 + }, + { + "epoch": 0.91, + "learning_rate": 4.313111860054566e-05, + "loss": 1.2408, + "theoretical_loss": 3.3361373574896964, + "tokens_seen": 3159031808 + }, + { + "epoch": 0.91, + "learning_rate": 4.3090996629754455e-05, + "loss": 1.252, + "theoretical_loss": 3.336126916251347, + "tokens_seen": 3159162880 + }, + { + "epoch": 0.91, + "learning_rate": 4.305087465896325e-05, + "loss": 1.2491, + "theoretical_loss": 3.336116475567481, + "tokens_seen": 3159293952 + }, + { + "epoch": 0.91, + "learning_rate": 4.301075268817205e-05, + "loss": 1.2522, + "theoretical_loss": 3.3361060354380454, + "tokens_seen": 3159425024 + }, + { + "epoch": 0.91, + "learning_rate": 4.297063071738084e-05, + "loss": 1.2632, + "theoretical_loss": 3.3360955958629877, + "tokens_seen": 3159556096 + }, + { + "epoch": 0.91, + "learning_rate": 4.293050874658963e-05, + "loss": 1.206, + "theoretical_loss": 3.3360851568422563, + "tokens_seen": 3159687168 + }, + { + "epoch": 0.92, + "learning_rate": 4.289038677579843e-05, + "loss": 1.2863, + "theoretical_loss": 3.3360747183757984, + "tokens_seen": 3159818240 + }, + { + "epoch": 0.92, + "learning_rate": 4.285026480500722e-05, + "loss": 1.272, + "theoretical_loss": 3.336064280463561, + "tokens_seen": 3159949312 + }, + { + "epoch": 0.92, + "learning_rate": 4.281014283421602e-05, + "loss": 1.2629, + "theoretical_loss": 3.3360538431054927, + "tokens_seen": 3160080384 + }, + { + "epoch": 0.92, + "learning_rate": 4.277002086342481e-05, + "loss": 1.205, + "theoretical_loss": 3.3360434063015405, + "tokens_seen": 3160211456 + }, + { + "epoch": 0.92, + "learning_rate": 4.272989889263361e-05, + "loss": 1.3631, + "theoretical_loss": 3.336032970051652, + "tokens_seen": 3160342528 + }, + { + "epoch": 0.92, + "learning_rate": 4.26897769218424e-05, + "loss": 1.1429, + "theoretical_loss": 3.3360225343557746, + "tokens_seen": 3160473600 + }, + { + "epoch": 0.92, + "learning_rate": 4.26496549510512e-05, + "loss": 1.2574, + "theoretical_loss": 3.3360120992138564, + "tokens_seen": 3160604672 + }, + { + "epoch": 0.92, + "learning_rate": 4.260953298025999e-05, + "loss": 1.225, + "theoretical_loss": 3.3360016646258446, + "tokens_seen": 3160735744 + }, + { + "epoch": 0.92, + "learning_rate": 4.256941100946879e-05, + "loss": 1.2208, + "theoretical_loss": 3.3359912305916875, + "tokens_seen": 3160866816 + }, + { + "epoch": 0.92, + "learning_rate": 4.252928903867758e-05, + "loss": 1.2451, + "theoretical_loss": 3.3359807971113318, + "tokens_seen": 3160997888 + }, + { + "epoch": 0.92, + "learning_rate": 4.248916706788637e-05, + "loss": 1.226, + "theoretical_loss": 3.335970364184726, + "tokens_seen": 3161128960 + }, + { + "epoch": 0.92, + "learning_rate": 4.244904509709517e-05, + "loss": 1.2169, + "theoretical_loss": 3.3359599318118174, + "tokens_seen": 3161260032 + }, + { + "epoch": 0.92, + "learning_rate": 4.2408923126303965e-05, + "loss": 1.2051, + "theoretical_loss": 3.3359494999925534, + "tokens_seen": 3161391104 + }, + { + "epoch": 0.92, + "learning_rate": 4.236880115551276e-05, + "loss": 1.2166, + "theoretical_loss": 3.3359390687268817, + "tokens_seen": 3161522176 + }, + { + "epoch": 0.92, + "learning_rate": 4.232867918472155e-05, + "loss": 1.2809, + "theoretical_loss": 3.3359286380147504, + "tokens_seen": 3161653248 + }, + { + "epoch": 0.92, + "learning_rate": 4.228855721393035e-05, + "loss": 1.3134, + "theoretical_loss": 3.335918207856107, + "tokens_seen": 3161784320 + }, + { + "epoch": 0.92, + "learning_rate": 4.224843524313914e-05, + "loss": 1.2705, + "theoretical_loss": 3.3359077782508995, + "tokens_seen": 3161915392 + }, + { + "epoch": 0.92, + "learning_rate": 4.220831327234794e-05, + "loss": 1.3131, + "theoretical_loss": 3.3358973491990747, + "tokens_seen": 3162046464 + }, + { + "epoch": 0.92, + "learning_rate": 4.2168191301556736e-05, + "loss": 1.3363, + "theoretical_loss": 3.335886920700581, + "tokens_seen": 3162177536 + }, + { + "epoch": 0.92, + "learning_rate": 4.212806933076553e-05, + "loss": 1.2255, + "theoretical_loss": 3.335876492755366, + "tokens_seen": 3162308608 + }, + { + "epoch": 0.92, + "learning_rate": 4.208794735997432e-05, + "loss": 1.2427, + "theoretical_loss": 3.335866065363377, + "tokens_seen": 3162439680 + }, + { + "epoch": 0.92, + "learning_rate": 4.2047825389183114e-05, + "loss": 1.2212, + "theoretical_loss": 3.3358556385245626, + "tokens_seen": 3162570752 + }, + { + "epoch": 0.92, + "learning_rate": 4.200770341839191e-05, + "loss": 1.269, + "theoretical_loss": 3.33584521223887, + "tokens_seen": 3162701824 + }, + { + "epoch": 0.92, + "learning_rate": 4.1967581447600706e-05, + "loss": 1.2678, + "theoretical_loss": 3.335834786506247, + "tokens_seen": 3162832896 + }, + { + "epoch": 0.92, + "learning_rate": 4.1927459476809506e-05, + "loss": 1.2083, + "theoretical_loss": 3.3358243613266407, + "tokens_seen": 3162963968 + }, + { + "epoch": 0.92, + "learning_rate": 4.188733750601829e-05, + "loss": 1.2583, + "theoretical_loss": 3.3358139366999997, + "tokens_seen": 3163095040 + }, + { + "epoch": 0.92, + "learning_rate": 4.184721553522709e-05, + "loss": 1.3102, + "theoretical_loss": 3.3358035126262715, + "tokens_seen": 3163226112 + }, + { + "epoch": 0.92, + "learning_rate": 4.1807093564435884e-05, + "loss": 1.1758, + "theoretical_loss": 3.335793089105404, + "tokens_seen": 3163357184 + }, + { + "epoch": 0.92, + "learning_rate": 4.1766971593644683e-05, + "loss": 1.2227, + "theoretical_loss": 3.3357826661373453, + "tokens_seen": 3163488256 + }, + { + "epoch": 0.92, + "learning_rate": 4.1726849622853476e-05, + "loss": 1.2481, + "theoretical_loss": 3.335772243722042, + "tokens_seen": 3163619328 + }, + { + "epoch": 0.92, + "learning_rate": 4.1686727652062276e-05, + "loss": 1.26, + "theoretical_loss": 3.335761821859443, + "tokens_seen": 3163750400 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.99249267578125, + "objective/train/avg_token_score": 0.0075073144398629665, + "objective/train/docs_used": 1149760, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3148897886276245, + "objective/train/original_loss": 1.3264377117156982, + "objective/train/theoretical_loss": 3.3357514005494955, + "objective/train/tokens_used": 1534340576, + "objective/train/unlikelihood_loss": 0.017631400376558304, + "theoretical_loss": 3.3357514005494955, + "tokens_seen": 3163881472 + }, + { + "epoch": 0.92, + "learning_rate": 4.164660568127106e-05, + "loss": 1.2796, + "theoretical_loss": 3.3357514005494955, + "tokens_seen": 3163881472 + }, + { + "epoch": 0.92, + "learning_rate": 4.160648371047986e-05, + "loss": 1.2053, + "theoretical_loss": 3.3357409797921473, + "tokens_seen": 3164012544 + }, + { + "epoch": 0.92, + "learning_rate": 4.1566361739688654e-05, + "loss": 1.2462, + "theoretical_loss": 3.335730559587347, + "tokens_seen": 3164143616 + }, + { + "epoch": 0.92, + "learning_rate": 4.152623976889745e-05, + "loss": 1.1936, + "theoretical_loss": 3.3357201399350416, + "tokens_seen": 3164274688 + }, + { + "epoch": 0.92, + "learning_rate": 4.1486117798106246e-05, + "loss": 1.2042, + "theoretical_loss": 3.3357097208351787, + "tokens_seen": 3164405760 + }, + { + "epoch": 0.92, + "learning_rate": 4.144599582731504e-05, + "loss": 1.2533, + "theoretical_loss": 3.335699302287707, + "tokens_seen": 3164536832 + }, + { + "epoch": 0.92, + "learning_rate": 4.140587385652383e-05, + "loss": 1.2387, + "theoretical_loss": 3.335688884292574, + "tokens_seen": 3164667904 + }, + { + "epoch": 0.92, + "learning_rate": 4.1365751885732624e-05, + "loss": 1.1863, + "theoretical_loss": 3.335678466849727, + "tokens_seen": 3164798976 + }, + { + "epoch": 0.92, + "learning_rate": 4.1325629914941424e-05, + "loss": 1.2168, + "theoretical_loss": 3.3356680499591147, + "tokens_seen": 3164930048 + }, + { + "epoch": 0.92, + "learning_rate": 4.128550794415022e-05, + "loss": 1.2686, + "theoretical_loss": 3.3356576336206842, + "tokens_seen": 3165061120 + }, + { + "epoch": 0.92, + "learning_rate": 4.1245385973359016e-05, + "loss": 1.203, + "theoretical_loss": 3.335647217834384, + "tokens_seen": 3165192192 + }, + { + "epoch": 0.92, + "learning_rate": 4.120526400256781e-05, + "loss": 1.2017, + "theoretical_loss": 3.335636802600162, + "tokens_seen": 3165323264 + }, + { + "epoch": 0.92, + "learning_rate": 4.11651420317766e-05, + "loss": 1.2049, + "theoretical_loss": 3.3356263879179653, + "tokens_seen": 3165454336 + }, + { + "epoch": 0.92, + "learning_rate": 4.1125020060985395e-05, + "loss": 1.1716, + "theoretical_loss": 3.3356159737877427, + "tokens_seen": 3165585408 + }, + { + "epoch": 0.92, + "learning_rate": 4.108489809019419e-05, + "loss": 1.202, + "theoretical_loss": 3.335605560209441, + "tokens_seen": 3165716480 + }, + { + "epoch": 0.92, + "learning_rate": 4.104477611940299e-05, + "loss": 1.2325, + "theoretical_loss": 3.3355951471830094, + "tokens_seen": 3165847552 + }, + { + "epoch": 0.92, + "learning_rate": 4.100465414861178e-05, + "loss": 1.2118, + "theoretical_loss": 3.335584734708395, + "tokens_seen": 3165978624 + }, + { + "epoch": 0.92, + "learning_rate": 4.096453217782058e-05, + "loss": 1.2347, + "theoretical_loss": 3.3355743227855457, + "tokens_seen": 3166109696 + }, + { + "epoch": 0.92, + "learning_rate": 4.0924410207029365e-05, + "loss": 1.2948, + "theoretical_loss": 3.3355639114144093, + "tokens_seen": 3166240768 + }, + { + "epoch": 0.92, + "learning_rate": 4.0884288236238165e-05, + "loss": 1.2615, + "theoretical_loss": 3.3355535005949344, + "tokens_seen": 3166371840 + }, + { + "epoch": 0.92, + "learning_rate": 4.084416626544696e-05, + "loss": 1.215, + "theoretical_loss": 3.335543090327069, + "tokens_seen": 3166502912 + }, + { + "epoch": 0.92, + "learning_rate": 4.080404429465576e-05, + "loss": 1.2382, + "theoretical_loss": 3.33553268061076, + "tokens_seen": 3166633984 + }, + { + "epoch": 0.92, + "learning_rate": 4.076392232386455e-05, + "loss": 1.326, + "theoretical_loss": 3.335522271445956, + "tokens_seen": 3166765056 + }, + { + "epoch": 0.92, + "learning_rate": 4.072380035307335e-05, + "loss": 1.2255, + "theoretical_loss": 3.335511862832605, + "tokens_seen": 3166896128 + }, + { + "epoch": 0.92, + "learning_rate": 4.0683678382282135e-05, + "loss": 1.2171, + "theoretical_loss": 3.3355014547706547, + "tokens_seen": 3167027200 + }, + { + "epoch": 0.92, + "learning_rate": 4.0643556411490935e-05, + "loss": 1.1901, + "theoretical_loss": 3.335491047260054, + "tokens_seen": 3167158272 + }, + { + "epoch": 0.92, + "learning_rate": 4.060343444069973e-05, + "loss": 1.2738, + "theoretical_loss": 3.3354806403007493, + "tokens_seen": 3167289344 + }, + { + "epoch": 0.92, + "learning_rate": 4.056331246990852e-05, + "loss": 1.2218, + "theoretical_loss": 3.3354702338926896, + "tokens_seen": 3167420416 + }, + { + "epoch": 0.92, + "learning_rate": 4.052319049911732e-05, + "loss": 1.3115, + "theoretical_loss": 3.3354598280358227, + "tokens_seen": 3167551488 + }, + { + "epoch": 0.92, + "learning_rate": 4.048306852832611e-05, + "loss": 1.2562, + "theoretical_loss": 3.335449422730097, + "tokens_seen": 3167682560 + }, + { + "epoch": 0.92, + "learning_rate": 4.0442946557534905e-05, + "loss": 1.2086, + "theoretical_loss": 3.3354390179754594, + "tokens_seen": 3167813632 + }, + { + "epoch": 0.92, + "learning_rate": 4.04028245867437e-05, + "loss": 1.2282, + "theoretical_loss": 3.335428613771859, + "tokens_seen": 3167944704 + }, + { + "epoch": 0.92, + "learning_rate": 4.03627026159525e-05, + "loss": 1.2403, + "theoretical_loss": 3.3354182101192436, + "tokens_seen": 3168075776 + }, + { + "epoch": 0.92, + "learning_rate": 4.032258064516129e-05, + "loss": 1.2535, + "theoretical_loss": 3.335407807017561, + "tokens_seen": 3168206848 + }, + { + "epoch": 0.92, + "learning_rate": 4.028245867437009e-05, + "loss": 1.3073, + "theoretical_loss": 3.335397404466759, + "tokens_seen": 3168337920 + }, + { + "epoch": 0.92, + "learning_rate": 4.024233670357888e-05, + "loss": 1.2654, + "theoretical_loss": 3.3353870024667858, + "tokens_seen": 3168468992 + }, + { + "epoch": 0.92, + "learning_rate": 4.0202214732787675e-05, + "loss": 1.3371, + "theoretical_loss": 3.33537660101759, + "tokens_seen": 3168600064 + }, + { + "epoch": 0.92, + "learning_rate": 4.016209276199647e-05, + "loss": 1.3027, + "theoretical_loss": 3.335366200119119, + "tokens_seen": 3168731136 + }, + { + "epoch": 0.92, + "learning_rate": 4.012197079120526e-05, + "loss": 1.2194, + "theoretical_loss": 3.3353557997713215, + "tokens_seen": 3168862208 + }, + { + "epoch": 0.92, + "learning_rate": 4.008184882041406e-05, + "loss": 1.3049, + "theoretical_loss": 3.3353453999741447, + "tokens_seen": 3168993280 + }, + { + "epoch": 0.92, + "learning_rate": 4.004172684962285e-05, + "loss": 1.3062, + "theoretical_loss": 3.3353350007275377, + "tokens_seen": 3169124352 + }, + { + "epoch": 0.92, + "learning_rate": 4.000160487883165e-05, + "loss": 1.3468, + "theoretical_loss": 3.3353246020314478, + "tokens_seen": 3169255424 + }, + { + "epoch": 0.92, + "learning_rate": 3.996148290804044e-05, + "loss": 1.2384, + "theoretical_loss": 3.3353142038858232, + "tokens_seen": 3169386496 + }, + { + "epoch": 0.92, + "learning_rate": 3.992136093724924e-05, + "loss": 1.2223, + "theoretical_loss": 3.3353038062906117, + "tokens_seen": 3169517568 + }, + { + "epoch": 0.92, + "learning_rate": 3.988123896645803e-05, + "loss": 1.2648, + "theoretical_loss": 3.3352934092457627, + "tokens_seen": 3169648640 + }, + { + "epoch": 0.92, + "learning_rate": 3.984111699566683e-05, + "loss": 1.2342, + "theoretical_loss": 3.3352830127512227, + "tokens_seen": 3169779712 + }, + { + "epoch": 0.92, + "learning_rate": 3.980099502487562e-05, + "loss": 1.264, + "theoretical_loss": 3.335272616806941, + "tokens_seen": 3169910784 + }, + { + "epoch": 0.92, + "learning_rate": 3.976087305408442e-05, + "loss": 1.3398, + "theoretical_loss": 3.335262221412865, + "tokens_seen": 3170041856 + }, + { + "epoch": 0.92, + "learning_rate": 3.972075108329321e-05, + "loss": 1.2103, + "theoretical_loss": 3.3352518265689435, + "tokens_seen": 3170172928 + }, + { + "epoch": 0.92, + "learning_rate": 3.9680629112502e-05, + "loss": 1.2605, + "theoretical_loss": 3.335241432275124, + "tokens_seen": 3170304000 + }, + { + "epoch": 0.92, + "objective/train/aligned_token_count": 0.9928733706474304, + "objective/train/avg_token_score": 0.007126656360924244, + "objective/train/docs_used": 1152259, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2139619588851929, + "objective/train/original_loss": 1.2258747816085815, + "objective/train/theoretical_loss": 3.335231038531355, + "objective/train/tokens_used": 1540894176, + "objective/train/unlikelihood_loss": 0.018221450969576836, + "theoretical_loss": 3.335231038531355, + "tokens_seen": 3170435072 + }, + { + "epoch": 0.92, + "learning_rate": 3.96405071417108e-05, + "loss": 1.2391, + "theoretical_loss": 3.335231038531355, + "tokens_seen": 3170435072 + }, + { + "epoch": 0.92, + "learning_rate": 3.9600385170919594e-05, + "loss": 1.3649, + "theoretical_loss": 3.335220645337584, + "tokens_seen": 3170566144 + }, + { + "epoch": 0.92, + "learning_rate": 3.956026320012839e-05, + "loss": 1.1727, + "theoretical_loss": 3.33521025269376, + "tokens_seen": 3170697216 + }, + { + "epoch": 0.92, + "learning_rate": 3.9520141229337186e-05, + "loss": 1.2595, + "theoretical_loss": 3.3351998605998316, + "tokens_seen": 3170828288 + }, + { + "epoch": 0.92, + "learning_rate": 3.9480019258545986e-05, + "loss": 1.2439, + "theoretical_loss": 3.3351894690557455, + "tokens_seen": 3170959360 + }, + { + "epoch": 0.92, + "learning_rate": 3.943989728775477e-05, + "loss": 1.2833, + "theoretical_loss": 3.3351790780614508, + "tokens_seen": 3171090432 + }, + { + "epoch": 0.92, + "learning_rate": 3.939977531696357e-05, + "loss": 1.2244, + "theoretical_loss": 3.3351686876168958, + "tokens_seen": 3171221504 + }, + { + "epoch": 0.92, + "learning_rate": 3.9359653346172364e-05, + "loss": 1.2734, + "theoretical_loss": 3.335158297722028, + "tokens_seen": 3171352576 + }, + { + "epoch": 0.92, + "learning_rate": 3.931953137538116e-05, + "loss": 1.257, + "theoretical_loss": 3.335147908376796, + "tokens_seen": 3171483648 + }, + { + "epoch": 0.92, + "learning_rate": 3.9279409404589956e-05, + "loss": 1.2768, + "theoretical_loss": 3.3351375195811483, + "tokens_seen": 3171614720 + }, + { + "epoch": 0.92, + "learning_rate": 3.9239287433798756e-05, + "loss": 1.2396, + "theoretical_loss": 3.335127131335033, + "tokens_seen": 3171745792 + }, + { + "epoch": 0.92, + "learning_rate": 3.919916546300754e-05, + "loss": 1.2273, + "theoretical_loss": 3.335116743638398, + "tokens_seen": 3171876864 + }, + { + "epoch": 0.92, + "learning_rate": 3.9159043492216334e-05, + "loss": 1.1973, + "theoretical_loss": 3.335106356491192, + "tokens_seen": 3172007936 + }, + { + "epoch": 0.92, + "learning_rate": 3.9118921521425134e-05, + "loss": 1.2473, + "theoretical_loss": 3.3350959698933624, + "tokens_seen": 3172139008 + }, + { + "epoch": 0.92, + "learning_rate": 3.907879955063393e-05, + "loss": 1.2708, + "theoretical_loss": 3.3350855838448585, + "tokens_seen": 3172270080 + }, + { + "epoch": 0.92, + "learning_rate": 3.9038677579842726e-05, + "loss": 1.2545, + "theoretical_loss": 3.3350751983456277, + "tokens_seen": 3172401152 + }, + { + "epoch": 0.92, + "learning_rate": 3.899855560905152e-05, + "loss": 1.1945, + "theoretical_loss": 3.3350648133956184, + "tokens_seen": 3172532224 + }, + { + "epoch": 0.92, + "learning_rate": 3.895843363826031e-05, + "loss": 1.2484, + "theoretical_loss": 3.3350544289947797, + "tokens_seen": 3172663296 + }, + { + "epoch": 0.92, + "learning_rate": 3.8918311667469104e-05, + "loss": 1.3417, + "theoretical_loss": 3.335044045143059, + "tokens_seen": 3172794368 + }, + { + "epoch": 0.92, + "learning_rate": 3.8878189696677904e-05, + "loss": 1.2377, + "theoretical_loss": 3.335033661840405, + "tokens_seen": 3172925440 + }, + { + "epoch": 0.92, + "learning_rate": 3.88380677258867e-05, + "loss": 1.2447, + "theoretical_loss": 3.3350232790867653, + "tokens_seen": 3173056512 + }, + { + "epoch": 0.92, + "learning_rate": 3.8797945755095496e-05, + "loss": 1.2239, + "theoretical_loss": 3.335012896882089, + "tokens_seen": 3173187584 + }, + { + "epoch": 0.92, + "learning_rate": 3.875782378430429e-05, + "loss": 1.2389, + "theoretical_loss": 3.335002515226324, + "tokens_seen": 3173318656 + }, + { + "epoch": 0.92, + "learning_rate": 3.8717701813513075e-05, + "loss": 1.2631, + "theoretical_loss": 3.3349921341194184, + "tokens_seen": 3173449728 + }, + { + "epoch": 0.92, + "learning_rate": 3.8677579842721875e-05, + "loss": 1.2406, + "theoretical_loss": 3.3349817535613218, + "tokens_seen": 3173580800 + }, + { + "epoch": 0.92, + "learning_rate": 3.863745787193067e-05, + "loss": 1.2769, + "theoretical_loss": 3.334971373551981, + "tokens_seen": 3173711872 + }, + { + "epoch": 0.92, + "learning_rate": 3.859733590113947e-05, + "loss": 1.2353, + "theoretical_loss": 3.3349609940913445, + "tokens_seen": 3173842944 + }, + { + "epoch": 0.92, + "learning_rate": 3.855721393034826e-05, + "loss": 1.2698, + "theoretical_loss": 3.3349506151793613, + "tokens_seen": 3173974016 + }, + { + "epoch": 0.92, + "learning_rate": 3.851709195955706e-05, + "loss": 1.2864, + "theoretical_loss": 3.3349402368159797, + "tokens_seen": 3174105088 + }, + { + "epoch": 0.92, + "learning_rate": 3.8476969988765845e-05, + "loss": 1.2126, + "theoretical_loss": 3.3349298590011474, + "tokens_seen": 3174236160 + }, + { + "epoch": 0.92, + "learning_rate": 3.8436848017974645e-05, + "loss": 1.2678, + "theoretical_loss": 3.3349194817348136, + "tokens_seen": 3174367232 + }, + { + "epoch": 0.92, + "learning_rate": 3.839672604718344e-05, + "loss": 1.1982, + "theoretical_loss": 3.334909105016926, + "tokens_seen": 3174498304 + }, + { + "epoch": 0.92, + "learning_rate": 3.835660407639224e-05, + "loss": 1.2155, + "theoretical_loss": 3.334898728847433, + "tokens_seen": 3174629376 + }, + { + "epoch": 0.92, + "learning_rate": 3.831648210560103e-05, + "loss": 1.2448, + "theoretical_loss": 3.3348883532262836, + "tokens_seen": 3174760448 + }, + { + "epoch": 0.92, + "learning_rate": 3.827636013480982e-05, + "loss": 1.2894, + "theoretical_loss": 3.3348779781534255, + "tokens_seen": 3174891520 + }, + { + "epoch": 0.92, + "learning_rate": 3.8236238164018615e-05, + "loss": 1.2001, + "theoretical_loss": 3.3348676036288074, + "tokens_seen": 3175022592 + }, + { + "epoch": 0.92, + "learning_rate": 3.819611619322741e-05, + "loss": 1.2061, + "theoretical_loss": 3.3348572296523775, + "tokens_seen": 3175153664 + }, + { + "epoch": 0.92, + "learning_rate": 3.815599422243621e-05, + "loss": 1.2213, + "theoretical_loss": 3.3348468562240847, + "tokens_seen": 3175284736 + }, + { + "epoch": 0.92, + "learning_rate": 3.8115872251645e-05, + "loss": 1.2692, + "theoretical_loss": 3.334836483343877, + "tokens_seen": 3175415808 + }, + { + "epoch": 0.92, + "learning_rate": 3.80757502808538e-05, + "loss": 1.2761, + "theoretical_loss": 3.3348261110117026, + "tokens_seen": 3175546880 + }, + { + "epoch": 0.92, + "learning_rate": 3.803562831006259e-05, + "loss": 1.228, + "theoretical_loss": 3.3348157392275106, + "tokens_seen": 3175677952 + }, + { + "epoch": 0.92, + "learning_rate": 3.7995506339271385e-05, + "loss": 1.306, + "theoretical_loss": 3.3348053679912493, + "tokens_seen": 3175809024 + }, + { + "epoch": 0.92, + "learning_rate": 3.795538436848018e-05, + "loss": 1.2827, + "theoretical_loss": 3.334794997302866, + "tokens_seen": 3175940096 + }, + { + "epoch": 0.92, + "learning_rate": 3.791526239768898e-05, + "loss": 1.2567, + "theoretical_loss": 3.334784627162311, + "tokens_seen": 3176071168 + }, + { + "epoch": 0.93, + "learning_rate": 3.787514042689777e-05, + "loss": 1.2405, + "theoretical_loss": 3.3347742575695314, + "tokens_seen": 3176202240 + }, + { + "epoch": 0.93, + "learning_rate": 3.783501845610657e-05, + "loss": 1.1698, + "theoretical_loss": 3.334763888524476, + "tokens_seen": 3176333312 + }, + { + "epoch": 0.93, + "learning_rate": 3.779489648531536e-05, + "loss": 1.2266, + "theoretical_loss": 3.334753520027093, + "tokens_seen": 3176464384 + }, + { + "epoch": 0.93, + "learning_rate": 3.775477451452415e-05, + "loss": 1.2267, + "theoretical_loss": 3.3347431520773316, + "tokens_seen": 3176595456 + }, + { + "epoch": 0.93, + "learning_rate": 3.771465254373295e-05, + "loss": 1.2197, + "theoretical_loss": 3.33473278467514, + "tokens_seen": 3176726528 + }, + { + "epoch": 0.93, + "learning_rate": 3.767453057294174e-05, + "loss": 1.2656, + "theoretical_loss": 3.3347224178204664, + "tokens_seen": 3176857600 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.9929877519607544, + "objective/train/avg_token_score": 0.007012245245277882, + "objective/train/docs_used": 1154349, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2578418254852295, + "objective/train/original_loss": 1.2668776512145996, + "objective/train/theoretical_loss": 3.3347120515132596, + "objective/train/tokens_used": 1547447776, + "objective/train/unlikelihood_loss": 0.020432723686099052, + "theoretical_loss": 3.3347120515132596, + "tokens_seen": 3176988672 + }, + { + "epoch": 0.93, + "learning_rate": 3.763440860215054e-05, + "loss": 1.2748, + "theoretical_loss": 3.3347120515132596, + "tokens_seen": 3176988672 + }, + { + "epoch": 0.93, + "learning_rate": 3.759428663135933e-05, + "loss": 1.2786, + "theoretical_loss": 3.3347016857534677, + "tokens_seen": 3177119744 + }, + { + "epoch": 0.93, + "learning_rate": 3.755416466056813e-05, + "loss": 1.2384, + "theoretical_loss": 3.33469132054104, + "tokens_seen": 3177250816 + }, + { + "epoch": 0.93, + "learning_rate": 3.751404268977692e-05, + "loss": 1.2644, + "theoretical_loss": 3.334680955875924, + "tokens_seen": 3177381888 + }, + { + "epoch": 0.93, + "learning_rate": 3.747392071898572e-05, + "loss": 1.2982, + "theoretical_loss": 3.334670591758069, + "tokens_seen": 3177512960 + }, + { + "epoch": 0.93, + "learning_rate": 3.743379874819451e-05, + "loss": 1.2531, + "theoretical_loss": 3.334660228187423, + "tokens_seen": 3177644032 + }, + { + "epoch": 0.93, + "learning_rate": 3.739367677740331e-05, + "loss": 1.231, + "theoretical_loss": 3.334649865163935, + "tokens_seen": 3177775104 + }, + { + "epoch": 0.93, + "learning_rate": 3.73535548066121e-05, + "loss": 1.279, + "theoretical_loss": 3.3346395026875535, + "tokens_seen": 3177906176 + }, + { + "epoch": 0.93, + "learning_rate": 3.7313432835820896e-05, + "loss": 1.2684, + "theoretical_loss": 3.334629140758227, + "tokens_seen": 3178037248 + }, + { + "epoch": 0.93, + "learning_rate": 3.727331086502969e-05, + "loss": 1.2383, + "theoretical_loss": 3.3346187793759032, + "tokens_seen": 3178168320 + }, + { + "epoch": 0.93, + "learning_rate": 3.723318889423848e-05, + "loss": 1.2379, + "theoretical_loss": 3.3346084185405322, + "tokens_seen": 3178299392 + }, + { + "epoch": 0.93, + "learning_rate": 3.719306692344728e-05, + "loss": 1.2343, + "theoretical_loss": 3.3345980582520616, + "tokens_seen": 3178430464 + }, + { + "epoch": 0.93, + "learning_rate": 3.7152944952656074e-05, + "loss": 1.2197, + "theoretical_loss": 3.33458769851044, + "tokens_seen": 3178561536 + }, + { + "epoch": 0.93, + "learning_rate": 3.711282298186487e-05, + "loss": 1.2164, + "theoretical_loss": 3.3345773393156164, + "tokens_seen": 3178692608 + }, + { + "epoch": 0.93, + "learning_rate": 3.7072701011073666e-05, + "loss": 1.2746, + "theoretical_loss": 3.334566980667539, + "tokens_seen": 3178823680 + }, + { + "epoch": 0.93, + "learning_rate": 3.703257904028246e-05, + "loss": 1.2732, + "theoretical_loss": 3.334556622566157, + "tokens_seen": 3178954752 + }, + { + "epoch": 0.93, + "learning_rate": 3.699245706949125e-05, + "loss": 1.2127, + "theoretical_loss": 3.3345462650114177, + "tokens_seen": 3179085824 + }, + { + "epoch": 0.93, + "learning_rate": 3.695233509870005e-05, + "loss": 1.2607, + "theoretical_loss": 3.3345359080032715, + "tokens_seen": 3179216896 + }, + { + "epoch": 0.93, + "learning_rate": 3.6912213127908844e-05, + "loss": 1.227, + "theoretical_loss": 3.3345255515416654, + "tokens_seen": 3179347968 + }, + { + "epoch": 0.93, + "learning_rate": 3.687209115711764e-05, + "loss": 1.3199, + "theoretical_loss": 3.3345151956265493, + "tokens_seen": 3179479040 + }, + { + "epoch": 0.93, + "learning_rate": 3.6831969186326436e-05, + "loss": 1.2204, + "theoretical_loss": 3.3345048402578708, + "tokens_seen": 3179610112 + }, + { + "epoch": 0.93, + "learning_rate": 3.679184721553522e-05, + "loss": 1.2914, + "theoretical_loss": 3.334494485435579, + "tokens_seen": 3179741184 + }, + { + "epoch": 0.93, + "learning_rate": 3.675172524474402e-05, + "loss": 1.3016, + "theoretical_loss": 3.334484131159623, + "tokens_seen": 3179872256 + }, + { + "epoch": 0.93, + "learning_rate": 3.6711603273952814e-05, + "loss": 1.2833, + "theoretical_loss": 3.334473777429951, + "tokens_seen": 3180003328 + }, + { + "epoch": 0.93, + "learning_rate": 3.6671481303161614e-05, + "loss": 1.2658, + "theoretical_loss": 3.3344634242465117, + "tokens_seen": 3180134400 + }, + { + "epoch": 0.93, + "learning_rate": 3.6631359332370407e-05, + "loss": 1.3294, + "theoretical_loss": 3.3344530716092535, + "tokens_seen": 3180265472 + }, + { + "epoch": 0.93, + "learning_rate": 3.6591237361579206e-05, + "loss": 1.2685, + "theoretical_loss": 3.334442719518125, + "tokens_seen": 3180396544 + }, + { + "epoch": 0.93, + "learning_rate": 3.655111539078799e-05, + "loss": 1.2421, + "theoretical_loss": 3.334432367973076, + "tokens_seen": 3180527616 + }, + { + "epoch": 0.93, + "learning_rate": 3.651099341999679e-05, + "loss": 1.2681, + "theoretical_loss": 3.334422016974054, + "tokens_seen": 3180658688 + }, + { + "epoch": 0.93, + "learning_rate": 3.6470871449205584e-05, + "loss": 1.2607, + "theoretical_loss": 3.3344116665210084, + "tokens_seen": 3180789760 + }, + { + "epoch": 0.93, + "learning_rate": 3.6430749478414384e-05, + "loss": 1.158, + "theoretical_loss": 3.3344013166138873, + "tokens_seen": 3180920832 + }, + { + "epoch": 0.93, + "learning_rate": 3.639062750762318e-05, + "loss": 1.2044, + "theoretical_loss": 3.3343909672526397, + "tokens_seen": 3181051904 + }, + { + "epoch": 0.93, + "learning_rate": 3.635050553683197e-05, + "loss": 1.2714, + "theoretical_loss": 3.3343806184372147, + "tokens_seen": 3181182976 + }, + { + "epoch": 0.93, + "learning_rate": 3.631038356604076e-05, + "loss": 1.2504, + "theoretical_loss": 3.33437027016756, + "tokens_seen": 3181314048 + }, + { + "epoch": 0.93, + "learning_rate": 3.6270261595249555e-05, + "loss": 1.267, + "theoretical_loss": 3.3343599224436256, + "tokens_seen": 3181445120 + }, + { + "epoch": 0.93, + "learning_rate": 3.6230139624458354e-05, + "loss": 1.2366, + "theoretical_loss": 3.3343495752653594, + "tokens_seen": 3181576192 + }, + { + "epoch": 0.93, + "learning_rate": 3.619001765366715e-05, + "loss": 1.283, + "theoretical_loss": 3.334339228632711, + "tokens_seen": 3181707264 + }, + { + "epoch": 0.93, + "learning_rate": 3.614989568287595e-05, + "loss": 1.2302, + "theoretical_loss": 3.334328882545628, + "tokens_seen": 3181838336 + }, + { + "epoch": 0.93, + "learning_rate": 3.610977371208474e-05, + "loss": 1.2244, + "theoretical_loss": 3.3343185370040596, + "tokens_seen": 3181969408 + }, + { + "epoch": 0.93, + "learning_rate": 3.606965174129353e-05, + "loss": 1.2623, + "theoretical_loss": 3.3343081920079545, + "tokens_seen": 3182100480 + }, + { + "epoch": 0.93, + "learning_rate": 3.6029529770502325e-05, + "loss": 1.28, + "theoretical_loss": 3.334297847557262, + "tokens_seen": 3182231552 + }, + { + "epoch": 0.93, + "learning_rate": 3.5989407799711125e-05, + "loss": 1.2, + "theoretical_loss": 3.3342875036519306, + "tokens_seen": 3182362624 + }, + { + "epoch": 0.93, + "learning_rate": 3.594928582891992e-05, + "loss": 1.2752, + "theoretical_loss": 3.334277160291909, + "tokens_seen": 3182493696 + }, + { + "epoch": 0.93, + "learning_rate": 3.590916385812871e-05, + "loss": 1.3234, + "theoretical_loss": 3.334266817477146, + "tokens_seen": 3182624768 + }, + { + "epoch": 0.93, + "learning_rate": 3.586904188733751e-05, + "loss": 1.2874, + "theoretical_loss": 3.3342564752075905, + "tokens_seen": 3182755840 + }, + { + "epoch": 0.93, + "learning_rate": 3.5828919916546296e-05, + "loss": 1.2715, + "theoretical_loss": 3.334246133483191, + "tokens_seen": 3182886912 + }, + { + "epoch": 0.93, + "learning_rate": 3.5788797945755095e-05, + "loss": 1.2218, + "theoretical_loss": 3.3342357923038968, + "tokens_seen": 3183017984 + }, + { + "epoch": 0.93, + "learning_rate": 3.574867597496389e-05, + "loss": 1.2692, + "theoretical_loss": 3.3342254516696563, + "tokens_seen": 3183149056 + }, + { + "epoch": 0.93, + "learning_rate": 3.570855400417269e-05, + "loss": 1.2218, + "theoretical_loss": 3.334215111580418, + "tokens_seen": 3183280128 + }, + { + "epoch": 0.93, + "learning_rate": 3.566843203338148e-05, + "loss": 1.2882, + "theoretical_loss": 3.334204772036132, + "tokens_seen": 3183411200 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.994452953338623, + "objective/train/avg_token_score": 0.005547053646296263, + "objective/train/docs_used": 1156704, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2935713529586792, + "objective/train/original_loss": 1.3016667366027832, + "objective/train/theoretical_loss": 3.334194433036746, + "objective/train/tokens_used": 1554001376, + "objective/train/unlikelihood_loss": 0.014097167178988457, + "theoretical_loss": 3.334194433036746, + "tokens_seen": 3183542272 + }, + { + "epoch": 0.93, + "learning_rate": 3.562831006259028e-05, + "loss": 1.3037, + "theoretical_loss": 3.334194433036746, + "tokens_seen": 3183542272 + }, + { + "epoch": 0.93, + "learning_rate": 3.5588188091799066e-05, + "loss": 1.24, + "theoretical_loss": 3.3341840945822097, + "tokens_seen": 3183673344 + }, + { + "epoch": 0.93, + "learning_rate": 3.5548066121007865e-05, + "loss": 1.2559, + "theoretical_loss": 3.334173756672471, + "tokens_seen": 3183804416 + }, + { + "epoch": 0.93, + "learning_rate": 3.550794415021666e-05, + "loss": 1.2086, + "theoretical_loss": 3.3341634193074796, + "tokens_seen": 3183935488 + }, + { + "epoch": 0.93, + "learning_rate": 3.546782217942546e-05, + "loss": 1.2183, + "theoretical_loss": 3.334153082487184, + "tokens_seen": 3184066560 + }, + { + "epoch": 0.93, + "learning_rate": 3.542770020863425e-05, + "loss": 1.2122, + "theoretical_loss": 3.3341427462115325, + "tokens_seen": 3184197632 + }, + { + "epoch": 0.93, + "learning_rate": 3.538757823784304e-05, + "loss": 1.2893, + "theoretical_loss": 3.3341324104804753, + "tokens_seen": 3184328704 + }, + { + "epoch": 0.93, + "learning_rate": 3.5347456267051836e-05, + "loss": 1.2997, + "theoretical_loss": 3.33412207529396, + "tokens_seen": 3184459776 + }, + { + "epoch": 0.93, + "learning_rate": 3.530733429626063e-05, + "loss": 1.2511, + "theoretical_loss": 3.3341117406519367, + "tokens_seen": 3184590848 + }, + { + "epoch": 0.93, + "learning_rate": 3.526721232546943e-05, + "loss": 1.2573, + "theoretical_loss": 3.3341014065543533, + "tokens_seen": 3184721920 + }, + { + "epoch": 0.93, + "learning_rate": 3.522709035467822e-05, + "loss": 1.2338, + "theoretical_loss": 3.334091073001159, + "tokens_seen": 3184852992 + }, + { + "epoch": 0.93, + "learning_rate": 3.518696838388702e-05, + "loss": 1.244, + "theoretical_loss": 3.334080739992303, + "tokens_seen": 3184984064 + }, + { + "epoch": 0.93, + "learning_rate": 3.514684641309581e-05, + "loss": 1.2431, + "theoretical_loss": 3.334070407527734, + "tokens_seen": 3185115136 + }, + { + "epoch": 0.93, + "learning_rate": 3.510672444230461e-05, + "loss": 1.2778, + "theoretical_loss": 3.3340600756074013, + "tokens_seen": 3185246208 + }, + { + "epoch": 0.93, + "learning_rate": 3.50666024715134e-05, + "loss": 1.261, + "theoretical_loss": 3.334049744231253, + "tokens_seen": 3185377280 + }, + { + "epoch": 0.93, + "learning_rate": 3.50264805007222e-05, + "loss": 1.3121, + "theoretical_loss": 3.3340394133992386, + "tokens_seen": 3185508352 + }, + { + "epoch": 0.93, + "learning_rate": 3.498635852993099e-05, + "loss": 1.2787, + "theoretical_loss": 3.3340290831113073, + "tokens_seen": 3185639424 + }, + { + "epoch": 0.93, + "learning_rate": 3.4946236559139784e-05, + "loss": 1.2606, + "theoretical_loss": 3.3340187533674075, + "tokens_seen": 3185770496 + }, + { + "epoch": 0.93, + "learning_rate": 3.490611458834858e-05, + "loss": 1.252, + "theoretical_loss": 3.3340084241674885, + "tokens_seen": 3185901568 + }, + { + "epoch": 0.93, + "learning_rate": 3.486599261755737e-05, + "loss": 1.3044, + "theoretical_loss": 3.3339980955114994, + "tokens_seen": 3186032640 + }, + { + "epoch": 0.93, + "learning_rate": 3.482587064676617e-05, + "loss": 1.2396, + "theoretical_loss": 3.333987767399389, + "tokens_seen": 3186163712 + }, + { + "epoch": 0.93, + "learning_rate": 3.478574867597496e-05, + "loss": 1.2822, + "theoretical_loss": 3.333977439831106, + "tokens_seen": 3186294784 + }, + { + "epoch": 0.93, + "learning_rate": 3.474562670518376e-05, + "loss": 1.3068, + "theoretical_loss": 3.3339671128066004, + "tokens_seen": 3186425856 + }, + { + "epoch": 0.93, + "learning_rate": 3.4705504734392554e-05, + "loss": 1.2868, + "theoretical_loss": 3.3339567863258197, + "tokens_seen": 3186556928 + }, + { + "epoch": 0.93, + "learning_rate": 3.466538276360135e-05, + "loss": 1.1829, + "theoretical_loss": 3.3339464603887143, + "tokens_seen": 3186688000 + }, + { + "epoch": 0.93, + "learning_rate": 3.4625260792810146e-05, + "loss": 1.2727, + "theoretical_loss": 3.3339361349952323, + "tokens_seen": 3186819072 + }, + { + "epoch": 0.93, + "learning_rate": 3.458513882201894e-05, + "loss": 1.281, + "theoretical_loss": 3.333925810145323, + "tokens_seen": 3186950144 + }, + { + "epoch": 0.93, + "learning_rate": 3.454501685122773e-05, + "loss": 1.2543, + "theoretical_loss": 3.3339154858389355, + "tokens_seen": 3187081216 + }, + { + "epoch": 0.93, + "learning_rate": 3.450489488043653e-05, + "loss": 1.2643, + "theoretical_loss": 3.333905162076019, + "tokens_seen": 3187212288 + }, + { + "epoch": 0.93, + "learning_rate": 3.4464772909645324e-05, + "loss": 1.2998, + "theoretical_loss": 3.333894838856522, + "tokens_seen": 3187343360 + }, + { + "epoch": 0.93, + "learning_rate": 3.4424650938854116e-05, + "loss": 1.2321, + "theoretical_loss": 3.333884516180394, + "tokens_seen": 3187474432 + }, + { + "epoch": 0.93, + "learning_rate": 3.4384528968062916e-05, + "loss": 1.2845, + "theoretical_loss": 3.333874194047584, + "tokens_seen": 3187605504 + }, + { + "epoch": 0.93, + "learning_rate": 3.43444069972717e-05, + "loss": 1.3103, + "theoretical_loss": 3.3338638724580414, + "tokens_seen": 3187736576 + }, + { + "epoch": 0.93, + "learning_rate": 3.43042850264805e-05, + "loss": 1.2831, + "theoretical_loss": 3.3338535514117145, + "tokens_seen": 3187867648 + }, + { + "epoch": 0.93, + "learning_rate": 3.4264163055689294e-05, + "loss": 1.3031, + "theoretical_loss": 3.3338432309085526, + "tokens_seen": 3187998720 + }, + { + "epoch": 0.93, + "learning_rate": 3.4224041084898094e-05, + "loss": 1.285, + "theoretical_loss": 3.333832910948505, + "tokens_seen": 3188129792 + }, + { + "epoch": 0.93, + "learning_rate": 3.4183919114106887e-05, + "loss": 1.2436, + "theoretical_loss": 3.333822591531521, + "tokens_seen": 3188260864 + }, + { + "epoch": 0.93, + "learning_rate": 3.4143797143315686e-05, + "loss": 1.2712, + "theoretical_loss": 3.3338122726575494, + "tokens_seen": 3188391936 + }, + { + "epoch": 0.93, + "learning_rate": 3.410367517252447e-05, + "loss": 1.2594, + "theoretical_loss": 3.3338019543265394, + "tokens_seen": 3188523008 + }, + { + "epoch": 0.93, + "learning_rate": 3.406355320173327e-05, + "loss": 1.2049, + "theoretical_loss": 3.33379163653844, + "tokens_seen": 3188654080 + }, + { + "epoch": 0.93, + "learning_rate": 3.4023431230942064e-05, + "loss": 1.2554, + "theoretical_loss": 3.3337813192932004, + "tokens_seen": 3188785152 + }, + { + "epoch": 0.93, + "learning_rate": 3.398330926015086e-05, + "loss": 1.2017, + "theoretical_loss": 3.33377100259077, + "tokens_seen": 3188916224 + }, + { + "epoch": 0.93, + "learning_rate": 3.394318728935966e-05, + "loss": 1.1958, + "theoretical_loss": 3.333760686431097, + "tokens_seen": 3189047296 + }, + { + "epoch": 0.93, + "learning_rate": 3.390306531856845e-05, + "loss": 1.2221, + "theoretical_loss": 3.3337503708141316, + "tokens_seen": 3189178368 + }, + { + "epoch": 0.93, + "learning_rate": 3.386294334777724e-05, + "loss": 1.2509, + "theoretical_loss": 3.3337400557398222, + "tokens_seen": 3189309440 + }, + { + "epoch": 0.93, + "learning_rate": 3.3822821376986035e-05, + "loss": 1.2055, + "theoretical_loss": 3.333729741208119, + "tokens_seen": 3189440512 + }, + { + "epoch": 0.93, + "learning_rate": 3.3782699406194834e-05, + "loss": 1.316, + "theoretical_loss": 3.3337194272189694, + "tokens_seen": 3189571584 + }, + { + "epoch": 0.93, + "learning_rate": 3.374257743540363e-05, + "loss": 1.2684, + "theoretical_loss": 3.3337091137723243, + "tokens_seen": 3189702656 + }, + { + "epoch": 0.93, + "learning_rate": 3.370245546461243e-05, + "loss": 1.2373, + "theoretical_loss": 3.333698800868132, + "tokens_seen": 3189833728 + }, + { + "epoch": 0.93, + "learning_rate": 3.366233349382122e-05, + "loss": 1.3193, + "theoretical_loss": 3.3336884885063416, + "tokens_seen": 3189964800 + }, + { + "epoch": 0.93, + "objective/train/aligned_token_count": 0.990990400314331, + "objective/train/avg_token_score": 0.009009545668959618, + "objective/train/docs_used": 1159129, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.290852427482605, + "objective/train/original_loss": 1.3036293983459473, + "objective/train/theoretical_loss": 3.3336781766869032, + "objective/train/tokens_used": 1560554976, + "objective/train/unlikelihood_loss": 0.022918444126844406, + "theoretical_loss": 3.3336781766869032, + "tokens_seen": 3190095872 + }, + { + "epoch": 0.93, + "learning_rate": 3.362221152303001e-05, + "loss": 1.2886, + "theoretical_loss": 3.3336781766869032, + "tokens_seen": 3190095872 + }, + { + "epoch": 0.93, + "learning_rate": 3.3582089552238805e-05, + "loss": 1.2144, + "theoretical_loss": 3.333667865409765, + "tokens_seen": 3190226944 + }, + { + "epoch": 0.93, + "learning_rate": 3.35419675814476e-05, + "loss": 1.2687, + "theoretical_loss": 3.3336575546748763, + "tokens_seen": 3190358016 + }, + { + "epoch": 0.93, + "learning_rate": 3.35018456106564e-05, + "loss": 1.2865, + "theoretical_loss": 3.333647244482187, + "tokens_seen": 3190489088 + }, + { + "epoch": 0.93, + "learning_rate": 3.346172363986519e-05, + "loss": 1.207, + "theoretical_loss": 3.3336369348316457, + "tokens_seen": 3190620160 + }, + { + "epoch": 0.93, + "learning_rate": 3.342160166907399e-05, + "loss": 1.2644, + "theoretical_loss": 3.3336266257232015, + "tokens_seen": 3190751232 + }, + { + "epoch": 0.93, + "learning_rate": 3.3381479698282775e-05, + "loss": 1.1832, + "theoretical_loss": 3.333616317156804, + "tokens_seen": 3190882304 + }, + { + "epoch": 0.93, + "learning_rate": 3.3341357727491575e-05, + "loss": 1.2519, + "theoretical_loss": 3.333606009132403, + "tokens_seen": 3191013376 + }, + { + "epoch": 0.93, + "learning_rate": 3.330123575670037e-05, + "loss": 1.2956, + "theoretical_loss": 3.3335957016499465, + "tokens_seen": 3191144448 + }, + { + "epoch": 0.93, + "learning_rate": 3.326111378590917e-05, + "loss": 1.2379, + "theoretical_loss": 3.3335853947093845, + "tokens_seen": 3191275520 + }, + { + "epoch": 0.93, + "learning_rate": 3.322099181511796e-05, + "loss": 1.2737, + "theoretical_loss": 3.333575088310666, + "tokens_seen": 3191406592 + }, + { + "epoch": 0.93, + "learning_rate": 3.318086984432676e-05, + "loss": 1.2565, + "theoretical_loss": 3.3335647824537404, + "tokens_seen": 3191537664 + }, + { + "epoch": 0.93, + "learning_rate": 3.3140747873535546e-05, + "loss": 1.2194, + "theoretical_loss": 3.333554477138557, + "tokens_seen": 3191668736 + }, + { + "epoch": 0.93, + "learning_rate": 3.3100625902744345e-05, + "loss": 1.2423, + "theoretical_loss": 3.3335441723650647, + "tokens_seen": 3191799808 + }, + { + "epoch": 0.93, + "learning_rate": 3.306050393195314e-05, + "loss": 1.3242, + "theoretical_loss": 3.3335338681332134, + "tokens_seen": 3191930880 + }, + { + "epoch": 0.93, + "learning_rate": 3.302038196116193e-05, + "loss": 1.1924, + "theoretical_loss": 3.3335235644429524, + "tokens_seen": 3192061952 + }, + { + "epoch": 0.93, + "learning_rate": 3.298025999037073e-05, + "loss": 1.1965, + "theoretical_loss": 3.33351326129423, + "tokens_seen": 3192193024 + }, + { + "epoch": 0.93, + "learning_rate": 3.294013801957952e-05, + "loss": 1.2548, + "theoretical_loss": 3.3335029586869966, + "tokens_seen": 3192324096 + }, + { + "epoch": 0.93, + "learning_rate": 3.2900016048788316e-05, + "loss": 1.2385, + "theoretical_loss": 3.333492656621201, + "tokens_seen": 3192455168 + }, + { + "epoch": 0.93, + "learning_rate": 3.285989407799711e-05, + "loss": 1.2263, + "theoretical_loss": 3.333482355096793, + "tokens_seen": 3192586240 + }, + { + "epoch": 0.94, + "learning_rate": 3.281977210720591e-05, + "loss": 1.2969, + "theoretical_loss": 3.333472054113721, + "tokens_seen": 3192717312 + }, + { + "epoch": 0.94, + "learning_rate": 3.27796501364147e-05, + "loss": 1.2759, + "theoretical_loss": 3.3334617536719353, + "tokens_seen": 3192848384 + }, + { + "epoch": 0.94, + "learning_rate": 3.27395281656235e-05, + "loss": 1.2536, + "theoretical_loss": 3.333451453771384, + "tokens_seen": 3192979456 + }, + { + "epoch": 0.94, + "learning_rate": 3.269940619483229e-05, + "loss": 1.2032, + "theoretical_loss": 3.3334411544120184, + "tokens_seen": 3193110528 + }, + { + "epoch": 0.94, + "learning_rate": 3.2659284224041086e-05, + "loss": 1.2169, + "theoretical_loss": 3.333430855593786, + "tokens_seen": 3193241600 + }, + { + "epoch": 0.94, + "learning_rate": 3.261916225324988e-05, + "loss": 1.2414, + "theoretical_loss": 3.333420557316637, + "tokens_seen": 3193372672 + }, + { + "epoch": 0.94, + "learning_rate": 3.257904028245867e-05, + "loss": 1.3045, + "theoretical_loss": 3.3334102595805204, + "tokens_seen": 3193503744 + }, + { + "epoch": 0.94, + "learning_rate": 3.253891831166747e-05, + "loss": 1.2992, + "theoretical_loss": 3.3333999623853865, + "tokens_seen": 3193634816 + }, + { + "epoch": 0.94, + "learning_rate": 3.2498796340876264e-05, + "loss": 1.2307, + "theoretical_loss": 3.3333896657311834, + "tokens_seen": 3193765888 + }, + { + "epoch": 0.94, + "learning_rate": 3.245867437008506e-05, + "loss": 1.2464, + "theoretical_loss": 3.333379369617861, + "tokens_seen": 3193896960 + }, + { + "epoch": 0.94, + "learning_rate": 3.241855239929385e-05, + "loss": 1.2134, + "theoretical_loss": 3.333369074045369, + "tokens_seen": 3194028032 + }, + { + "epoch": 0.94, + "learning_rate": 3.237843042850265e-05, + "loss": 1.2316, + "theoretical_loss": 3.3333587790136567, + "tokens_seen": 3194159104 + }, + { + "epoch": 0.94, + "learning_rate": 3.233830845771144e-05, + "loss": 1.2102, + "theoretical_loss": 3.333348484522673, + "tokens_seen": 3194290176 + }, + { + "epoch": 0.94, + "learning_rate": 3.229818648692024e-05, + "loss": 1.2301, + "theoretical_loss": 3.333338190572368, + "tokens_seen": 3194421248 + }, + { + "epoch": 0.94, + "learning_rate": 3.2258064516129034e-05, + "loss": 1.2466, + "theoretical_loss": 3.3333278971626905, + "tokens_seen": 3194552320 + }, + { + "epoch": 0.94, + "learning_rate": 3.221794254533783e-05, + "loss": 1.358, + "theoretical_loss": 3.3333176042935904, + "tokens_seen": 3194683392 + }, + { + "epoch": 0.94, + "learning_rate": 3.217782057454662e-05, + "loss": 1.2727, + "theoretical_loss": 3.333307311965017, + "tokens_seen": 3194814464 + }, + { + "epoch": 0.94, + "learning_rate": 3.213769860375542e-05, + "loss": 1.1983, + "theoretical_loss": 3.3332970201769196, + "tokens_seen": 3194945536 + }, + { + "epoch": 0.94, + "learning_rate": 3.209757663296421e-05, + "loss": 1.2826, + "theoretical_loss": 3.333286728929248, + "tokens_seen": 3195076608 + }, + { + "epoch": 0.94, + "learning_rate": 3.2057454662173004e-05, + "loss": 1.2461, + "theoretical_loss": 3.333276438221951, + "tokens_seen": 3195207680 + }, + { + "epoch": 0.94, + "learning_rate": 3.2017332691381804e-05, + "loss": 1.2327, + "theoretical_loss": 3.3332661480549786, + "tokens_seen": 3195338752 + }, + { + "epoch": 0.94, + "learning_rate": 3.1977210720590596e-05, + "loss": 1.2123, + "theoretical_loss": 3.3332558584282803, + "tokens_seen": 3195469824 + }, + { + "epoch": 0.94, + "learning_rate": 3.193708874979939e-05, + "loss": 1.2502, + "theoretical_loss": 3.333245569341805, + "tokens_seen": 3195600896 + }, + { + "epoch": 0.94, + "learning_rate": 3.189696677900818e-05, + "loss": 1.258, + "theoretical_loss": 3.333235280795503, + "tokens_seen": 3195731968 + }, + { + "epoch": 0.94, + "learning_rate": 3.185684480821698e-05, + "loss": 1.2902, + "theoretical_loss": 3.333224992789323, + "tokens_seen": 3195863040 + }, + { + "epoch": 0.94, + "learning_rate": 3.1816722837425774e-05, + "loss": 1.2205, + "theoretical_loss": 3.3332147053232153, + "tokens_seen": 3195994112 + }, + { + "epoch": 0.94, + "learning_rate": 3.1776600866634574e-05, + "loss": 1.2204, + "theoretical_loss": 3.3332044183971288, + "tokens_seen": 3196125184 + }, + { + "epoch": 0.94, + "learning_rate": 3.1736478895843366e-05, + "loss": 1.2503, + "theoretical_loss": 3.333194132011013, + "tokens_seen": 3196256256 + }, + { + "epoch": 0.94, + "learning_rate": 3.169635692505216e-05, + "loss": 1.2901, + "theoretical_loss": 3.3331838461648178, + "tokens_seen": 3196387328 + }, + { + "epoch": 0.94, + "learning_rate": 3.165623495426095e-05, + "loss": 1.2527, + "theoretical_loss": 3.3331735608584925, + "tokens_seen": 3196518400 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.9943403601646423, + "objective/train/avg_token_score": 0.005659641232341528, + "objective/train/docs_used": 1161767, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.217198133468628, + "objective/train/original_loss": 1.2239303588867188, + "objective/train/theoretical_loss": 3.3331632760919865, + "objective/train/tokens_used": 1567108576, + "objective/train/unlikelihood_loss": 0.018422242254018784, + "theoretical_loss": 3.3331632760919865, + "tokens_seen": 3196649472 + }, + { + "epoch": 0.94, + "learning_rate": 3.1616112983469745e-05, + "loss": 1.1946, + "theoretical_loss": 3.3331632760919865, + "tokens_seen": 3196649472 + }, + { + "epoch": 0.94, + "learning_rate": 3.1575991012678544e-05, + "loss": 1.1715, + "theoretical_loss": 3.3331529918652496, + "tokens_seen": 3196780544 + }, + { + "epoch": 0.94, + "learning_rate": 3.153586904188734e-05, + "loss": 1.2277, + "theoretical_loss": 3.333142708178231, + "tokens_seen": 3196911616 + }, + { + "epoch": 0.94, + "learning_rate": 3.1495747071096137e-05, + "loss": 1.1819, + "theoretical_loss": 3.3331324250308807, + "tokens_seen": 3197042688 + }, + { + "epoch": 0.94, + "learning_rate": 3.145562510030492e-05, + "loss": 1.248, + "theoretical_loss": 3.3331221424231483, + "tokens_seen": 3197173760 + }, + { + "epoch": 0.94, + "learning_rate": 3.141550312951372e-05, + "loss": 1.2539, + "theoretical_loss": 3.3331118603549825, + "tokens_seen": 3197304832 + }, + { + "epoch": 0.94, + "learning_rate": 3.1375381158722515e-05, + "loss": 1.1942, + "theoretical_loss": 3.3331015788263336, + "tokens_seen": 3197435904 + }, + { + "epoch": 0.94, + "learning_rate": 3.1335259187931314e-05, + "loss": 1.2283, + "theoretical_loss": 3.3330912978371514, + "tokens_seen": 3197566976 + }, + { + "epoch": 0.94, + "learning_rate": 3.129513721714011e-05, + "loss": 1.2492, + "theoretical_loss": 3.333081017387385, + "tokens_seen": 3197698048 + }, + { + "epoch": 0.94, + "learning_rate": 3.125501524634891e-05, + "loss": 1.2409, + "theoretical_loss": 3.3330707374769837, + "tokens_seen": 3197829120 + }, + { + "epoch": 0.94, + "learning_rate": 3.121489327555769e-05, + "loss": 1.2494, + "theoretical_loss": 3.333060458105898, + "tokens_seen": 3197960192 + }, + { + "epoch": 0.94, + "learning_rate": 3.117477130476649e-05, + "loss": 1.3329, + "theoretical_loss": 3.3330501792740765, + "tokens_seen": 3198091264 + }, + { + "epoch": 0.94, + "learning_rate": 3.1134649333975285e-05, + "loss": 1.2086, + "theoretical_loss": 3.3330399009814693, + "tokens_seen": 3198222336 + }, + { + "epoch": 0.94, + "learning_rate": 3.109452736318408e-05, + "loss": 1.2397, + "theoretical_loss": 3.3330296232280263, + "tokens_seen": 3198353408 + }, + { + "epoch": 0.94, + "learning_rate": 3.105440539239288e-05, + "loss": 1.2414, + "theoretical_loss": 3.333019346013697, + "tokens_seen": 3198484480 + }, + { + "epoch": 0.94, + "learning_rate": 3.101428342160167e-05, + "loss": 1.2886, + "theoretical_loss": 3.3330090693384307, + "tokens_seen": 3198615552 + }, + { + "epoch": 0.94, + "learning_rate": 3.097416145081046e-05, + "loss": 1.2084, + "theoretical_loss": 3.3329987932021776, + "tokens_seen": 3198746624 + }, + { + "epoch": 0.94, + "learning_rate": 3.0934039480019255e-05, + "loss": 1.2646, + "theoretical_loss": 3.3329885176048863, + "tokens_seen": 3198877696 + }, + { + "epoch": 0.94, + "learning_rate": 3.0893917509228055e-05, + "loss": 1.2067, + "theoretical_loss": 3.3329782425465075, + "tokens_seen": 3199008768 + }, + { + "epoch": 0.94, + "learning_rate": 3.085379553843685e-05, + "loss": 1.2397, + "theoretical_loss": 3.3329679680269906, + "tokens_seen": 3199139840 + }, + { + "epoch": 0.94, + "learning_rate": 3.081367356764564e-05, + "loss": 1.2239, + "theoretical_loss": 3.332957694046285, + "tokens_seen": 3199270912 + }, + { + "epoch": 0.94, + "learning_rate": 3.077355159685444e-05, + "loss": 1.2722, + "theoretical_loss": 3.33294742060434, + "tokens_seen": 3199401984 + }, + { + "epoch": 0.94, + "learning_rate": 3.073342962606323e-05, + "loss": 1.1894, + "theoretical_loss": 3.3329371477011067, + "tokens_seen": 3199533056 + }, + { + "epoch": 0.94, + "learning_rate": 3.0693307655272026e-05, + "loss": 1.2144, + "theoretical_loss": 3.332926875336533, + "tokens_seen": 3199664128 + }, + { + "epoch": 0.94, + "learning_rate": 3.0653185684480825e-05, + "loss": 1.28, + "theoretical_loss": 3.33291660351057, + "tokens_seen": 3199795200 + }, + { + "epoch": 0.94, + "learning_rate": 3.061306371368962e-05, + "loss": 1.2331, + "theoretical_loss": 3.3329063322231667, + "tokens_seen": 3199926272 + }, + { + "epoch": 0.94, + "learning_rate": 3.057294174289841e-05, + "loss": 1.1961, + "theoretical_loss": 3.332896061474273, + "tokens_seen": 3200057344 + }, + { + "epoch": 0.94, + "learning_rate": 3.053281977210721e-05, + "loss": 1.2889, + "theoretical_loss": 3.3328857912638385, + "tokens_seen": 3200188416 + }, + { + "epoch": 0.94, + "learning_rate": 3.0492697801316003e-05, + "loss": 1.264, + "theoretical_loss": 3.3328755215918133, + "tokens_seen": 3200319488 + }, + { + "epoch": 0.94, + "learning_rate": 3.0452575830524796e-05, + "loss": 1.2593, + "theoretical_loss": 3.3328652524581464, + "tokens_seen": 3200450560 + }, + { + "epoch": 0.94, + "learning_rate": 3.041245385973359e-05, + "loss": 1.2731, + "theoretical_loss": 3.3328549838627883, + "tokens_seen": 3200581632 + }, + { + "epoch": 0.94, + "learning_rate": 3.0372331888942384e-05, + "loss": 1.1916, + "theoretical_loss": 3.332844715805688, + "tokens_seen": 3200712704 + }, + { + "epoch": 0.94, + "learning_rate": 3.033220991815118e-05, + "loss": 1.264, + "theoretical_loss": 3.332834448286796, + "tokens_seen": 3200843776 + }, + { + "epoch": 0.94, + "learning_rate": 3.0292087947359973e-05, + "loss": 1.2177, + "theoretical_loss": 3.3328241813060617, + "tokens_seen": 3200974848 + }, + { + "epoch": 0.94, + "learning_rate": 3.025196597656877e-05, + "loss": 1.2976, + "theoretical_loss": 3.3328139148634346, + "tokens_seen": 3201105920 + }, + { + "epoch": 0.94, + "learning_rate": 3.0211844005777566e-05, + "loss": 1.2236, + "theoretical_loss": 3.3328036489588646, + "tokens_seen": 3201236992 + }, + { + "epoch": 0.94, + "learning_rate": 3.017172203498636e-05, + "loss": 1.2296, + "theoretical_loss": 3.332793383592302, + "tokens_seen": 3201368064 + }, + { + "epoch": 0.94, + "learning_rate": 3.0131600064195155e-05, + "loss": 1.2856, + "theoretical_loss": 3.332783118763696, + "tokens_seen": 3201499136 + }, + { + "epoch": 0.94, + "learning_rate": 3.009147809340395e-05, + "loss": 1.2351, + "theoretical_loss": 3.332772854472996, + "tokens_seen": 3201630208 + }, + { + "epoch": 0.94, + "learning_rate": 3.0051356122612743e-05, + "loss": 1.2595, + "theoretical_loss": 3.332762590720153, + "tokens_seen": 3201761280 + }, + { + "epoch": 0.94, + "learning_rate": 3.001123415182154e-05, + "loss": 1.2052, + "theoretical_loss": 3.332752327505116, + "tokens_seen": 3201892352 + }, + { + "epoch": 0.94, + "learning_rate": 2.9971112181030332e-05, + "loss": 1.254, + "theoretical_loss": 3.332742064827835, + "tokens_seen": 3202023424 + }, + { + "epoch": 0.94, + "learning_rate": 2.9930990210239125e-05, + "loss": 1.2916, + "theoretical_loss": 3.3327318026882597, + "tokens_seen": 3202154496 + }, + { + "epoch": 0.94, + "learning_rate": 2.989086823944792e-05, + "loss": 1.2223, + "theoretical_loss": 3.33272154108634, + "tokens_seen": 3202285568 + }, + { + "epoch": 0.94, + "learning_rate": 2.9850746268656717e-05, + "loss": 1.167, + "theoretical_loss": 3.3327112800220258, + "tokens_seen": 3202416640 + }, + { + "epoch": 0.94, + "learning_rate": 2.981062429786551e-05, + "loss": 1.2883, + "theoretical_loss": 3.3327010194952664, + "tokens_seen": 3202547712 + }, + { + "epoch": 0.94, + "learning_rate": 2.9770502327074306e-05, + "loss": 1.2198, + "theoretical_loss": 3.3326907595060122, + "tokens_seen": 3202678784 + }, + { + "epoch": 0.94, + "learning_rate": 2.9730380356283102e-05, + "loss": 1.1211, + "theoretical_loss": 3.3326805000542135, + "tokens_seen": 3202809856 + }, + { + "epoch": 0.94, + "learning_rate": 2.9690258385491895e-05, + "loss": 1.2789, + "theoretical_loss": 3.332670241139819, + "tokens_seen": 3202940928 + }, + { + "epoch": 0.94, + "learning_rate": 2.965013641470069e-05, + "loss": 1.2196, + "theoretical_loss": 3.332659982762779, + "tokens_seen": 3203072000 + }, + { + "epoch": 0.94, + "objective/train/aligned_token_count": 0.9910057187080383, + "objective/train/avg_token_score": 0.008994284085929394, + "objective/train/docs_used": 1164087, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2822940349578857, + "objective/train/original_loss": 1.2958301305770874, + "objective/train/theoretical_loss": 3.3326497249230442, + "objective/train/tokens_used": 1573662176, + "objective/train/unlikelihood_loss": 0.023340929299592972, + "theoretical_loss": 3.3326497249230442, + "tokens_seen": 3203203072 + }, + { + "epoch": 0.94, + "learning_rate": 2.9610014443909487e-05, + "loss": 1.2508, + "theoretical_loss": 3.3326497249230442, + "tokens_seen": 3203203072 + }, + { + "epoch": 0.94, + "learning_rate": 2.956989247311828e-05, + "loss": 1.1709, + "theoretical_loss": 3.332639467620563, + "tokens_seen": 3203334144 + }, + { + "epoch": 0.94, + "learning_rate": 2.9529770502327073e-05, + "loss": 1.2352, + "theoretical_loss": 3.3326292108552864, + "tokens_seen": 3203465216 + }, + { + "epoch": 0.94, + "learning_rate": 2.948964853153587e-05, + "loss": 1.1808, + "theoretical_loss": 3.332618954627164, + "tokens_seen": 3203596288 + }, + { + "epoch": 0.94, + "learning_rate": 2.9449526560744662e-05, + "loss": 1.242, + "theoretical_loss": 3.332608698936146, + "tokens_seen": 3203727360 + }, + { + "epoch": 0.94, + "learning_rate": 2.9409404589953458e-05, + "loss": 1.2067, + "theoretical_loss": 3.332598443782181, + "tokens_seen": 3203858432 + }, + { + "epoch": 0.94, + "learning_rate": 2.9369282619162254e-05, + "loss": 1.2356, + "theoretical_loss": 3.3325881891652207, + "tokens_seen": 3203989504 + }, + { + "epoch": 0.94, + "learning_rate": 2.9329160648371047e-05, + "loss": 1.2777, + "theoretical_loss": 3.332577935085214, + "tokens_seen": 3204120576 + }, + { + "epoch": 0.94, + "learning_rate": 2.9289038677579843e-05, + "loss": 1.2449, + "theoretical_loss": 3.332567681542111, + "tokens_seen": 3204251648 + }, + { + "epoch": 0.94, + "learning_rate": 2.924891670678864e-05, + "loss": 1.2698, + "theoretical_loss": 3.332557428535861, + "tokens_seen": 3204382720 + }, + { + "epoch": 0.94, + "learning_rate": 2.9208794735997432e-05, + "loss": 1.248, + "theoretical_loss": 3.3325471760664147, + "tokens_seen": 3204513792 + }, + { + "epoch": 0.94, + "learning_rate": 2.9168672765206228e-05, + "loss": 1.231, + "theoretical_loss": 3.332536924133722, + "tokens_seen": 3204644864 + }, + { + "epoch": 0.94, + "learning_rate": 2.9128550794415024e-05, + "loss": 1.2447, + "theoretical_loss": 3.332526672737733, + "tokens_seen": 3204775936 + }, + { + "epoch": 0.94, + "learning_rate": 2.9088428823623817e-05, + "loss": 1.2767, + "theoretical_loss": 3.332516421878397, + "tokens_seen": 3204907008 + }, + { + "epoch": 0.94, + "learning_rate": 2.904830685283261e-05, + "loss": 1.2714, + "theoretical_loss": 3.3325061715556648, + "tokens_seen": 3205038080 + }, + { + "epoch": 0.94, + "learning_rate": 2.9008184882041406e-05, + "loss": 1.2492, + "theoretical_loss": 3.3324959217694854, + "tokens_seen": 3205169152 + }, + { + "epoch": 0.94, + "learning_rate": 2.89680629112502e-05, + "loss": 1.2038, + "theoretical_loss": 3.3324856725198098, + "tokens_seen": 3205300224 + }, + { + "epoch": 0.94, + "learning_rate": 2.8927940940458995e-05, + "loss": 1.2592, + "theoretical_loss": 3.3324754238065872, + "tokens_seen": 3205431296 + }, + { + "epoch": 0.94, + "learning_rate": 2.888781896966779e-05, + "loss": 1.2167, + "theoretical_loss": 3.3324651756297676, + "tokens_seen": 3205562368 + }, + { + "epoch": 0.94, + "learning_rate": 2.8847696998876584e-05, + "loss": 1.296, + "theoretical_loss": 3.3324549279893017, + "tokens_seen": 3205693440 + }, + { + "epoch": 0.94, + "learning_rate": 2.880757502808538e-05, + "loss": 1.2335, + "theoretical_loss": 3.3324446808851387, + "tokens_seen": 3205824512 + }, + { + "epoch": 0.94, + "learning_rate": 2.8767453057294176e-05, + "loss": 1.1584, + "theoretical_loss": 3.3324344343172294, + "tokens_seen": 3205955584 + }, + { + "epoch": 0.94, + "learning_rate": 2.872733108650297e-05, + "loss": 1.2216, + "theoretical_loss": 3.332424188285523, + "tokens_seen": 3206086656 + }, + { + "epoch": 0.94, + "learning_rate": 2.8687209115711765e-05, + "loss": 1.226, + "theoretical_loss": 3.33241394278997, + "tokens_seen": 3206217728 + }, + { + "epoch": 0.94, + "learning_rate": 2.864708714492056e-05, + "loss": 1.2282, + "theoretical_loss": 3.33240369783052, + "tokens_seen": 3206348800 + }, + { + "epoch": 0.94, + "learning_rate": 2.8606965174129357e-05, + "loss": 1.2434, + "theoretical_loss": 3.3323934534071236, + "tokens_seen": 3206479872 + }, + { + "epoch": 0.94, + "learning_rate": 2.8566843203338146e-05, + "loss": 1.2058, + "theoretical_loss": 3.3323832095197305, + "tokens_seen": 3206610944 + }, + { + "epoch": 0.94, + "learning_rate": 2.8526721232546943e-05, + "loss": 1.2186, + "theoretical_loss": 3.332372966168291, + "tokens_seen": 3206742016 + }, + { + "epoch": 0.94, + "learning_rate": 2.8486599261755735e-05, + "loss": 1.1954, + "theoretical_loss": 3.332362723352755, + "tokens_seen": 3206873088 + }, + { + "epoch": 0.94, + "learning_rate": 2.844647729096453e-05, + "loss": 1.1849, + "theoretical_loss": 3.3323524810730722, + "tokens_seen": 3207004160 + }, + { + "epoch": 0.94, + "learning_rate": 2.8406355320173328e-05, + "loss": 1.2063, + "theoretical_loss": 3.3323422393291935, + "tokens_seen": 3207135232 + }, + { + "epoch": 0.94, + "learning_rate": 2.8366233349382124e-05, + "loss": 1.259, + "theoretical_loss": 3.3323319981210684, + "tokens_seen": 3207266304 + }, + { + "epoch": 0.94, + "learning_rate": 2.8326111378590917e-05, + "loss": 1.1819, + "theoretical_loss": 3.332321757448647, + "tokens_seen": 3207397376 + }, + { + "epoch": 0.94, + "learning_rate": 2.8285989407799713e-05, + "loss": 1.2707, + "theoretical_loss": 3.3323115173118794, + "tokens_seen": 3207528448 + }, + { + "epoch": 0.94, + "learning_rate": 2.824586743700851e-05, + "loss": 1.2644, + "theoretical_loss": 3.3323012777107155, + "tokens_seen": 3207659520 + }, + { + "epoch": 0.94, + "learning_rate": 2.82057454662173e-05, + "loss": 1.1679, + "theoretical_loss": 3.332291038645106, + "tokens_seen": 3207790592 + }, + { + "epoch": 0.94, + "learning_rate": 2.8165623495426098e-05, + "loss": 1.2088, + "theoretical_loss": 3.332280800115001, + "tokens_seen": 3207921664 + }, + { + "epoch": 0.94, + "learning_rate": 2.8125501524634894e-05, + "loss": 1.2826, + "theoretical_loss": 3.3322705621203497, + "tokens_seen": 3208052736 + }, + { + "epoch": 0.94, + "learning_rate": 2.8085379553843683e-05, + "loss": 1.2767, + "theoretical_loss": 3.3322603246611027, + "tokens_seen": 3208183808 + }, + { + "epoch": 0.94, + "learning_rate": 2.804525758305248e-05, + "loss": 1.1521, + "theoretical_loss": 3.3322500877372105, + "tokens_seen": 3208314880 + }, + { + "epoch": 0.94, + "learning_rate": 2.8005135612261276e-05, + "loss": 1.1207, + "theoretical_loss": 3.332239851348623, + "tokens_seen": 3208445952 + }, + { + "epoch": 0.94, + "learning_rate": 2.7965013641470068e-05, + "loss": 1.2614, + "theoretical_loss": 3.33222961549529, + "tokens_seen": 3208577024 + }, + { + "epoch": 0.94, + "learning_rate": 2.7924891670678864e-05, + "loss": 1.2578, + "theoretical_loss": 3.332219380177163, + "tokens_seen": 3208708096 + }, + { + "epoch": 0.94, + "learning_rate": 2.788476969988766e-05, + "loss": 1.163, + "theoretical_loss": 3.3322091453941898, + "tokens_seen": 3208839168 + }, + { + "epoch": 0.94, + "learning_rate": 2.7844647729096453e-05, + "loss": 1.2649, + "theoretical_loss": 3.332198911146323, + "tokens_seen": 3208970240 + }, + { + "epoch": 0.94, + "learning_rate": 2.780452575830525e-05, + "loss": 1.2161, + "theoretical_loss": 3.3321886774335105, + "tokens_seen": 3209101312 + }, + { + "epoch": 0.95, + "learning_rate": 2.7764403787514046e-05, + "loss": 1.177, + "theoretical_loss": 3.3321784442557045, + "tokens_seen": 3209232384 + }, + { + "epoch": 0.95, + "learning_rate": 2.772428181672284e-05, + "loss": 1.2257, + "theoretical_loss": 3.3321682116128537, + "tokens_seen": 3209363456 + }, + { + "epoch": 0.95, + "learning_rate": 2.7684159845931635e-05, + "loss": 1.2537, + "theoretical_loss": 3.3321579795049097, + "tokens_seen": 3209494528 + }, + { + "epoch": 0.95, + "learning_rate": 2.7644037875140427e-05, + "loss": 1.2328, + "theoretical_loss": 3.3321477479318213, + "tokens_seen": 3209625600 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.9925563931465149, + "objective/train/avg_token_score": 0.007443617098033428, + "objective/train/docs_used": 1166593, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.265540599822998, + "objective/train/original_loss": 1.2758839130401611, + "objective/train/theoretical_loss": 3.332137516893539, + "objective/train/tokens_used": 1580215776, + "objective/train/unlikelihood_loss": 0.020632222294807434, + "theoretical_loss": 3.332137516893539, + "tokens_seen": 3209756672 + }, + { + "epoch": 0.95, + "learning_rate": 2.760391590434922e-05, + "loss": 1.2572, + "theoretical_loss": 3.332137516893539, + "tokens_seen": 3209756672 + }, + { + "epoch": 0.95, + "learning_rate": 2.7563793933558016e-05, + "loss": 1.2279, + "theoretical_loss": 3.3321272863900138, + "tokens_seen": 3209887744 + }, + { + "epoch": 0.95, + "learning_rate": 2.7523671962766812e-05, + "loss": 1.1961, + "theoretical_loss": 3.332117056421195, + "tokens_seen": 3210018816 + }, + { + "epoch": 0.95, + "learning_rate": 2.7483549991975605e-05, + "loss": 1.2071, + "theoretical_loss": 3.3321068269870335, + "tokens_seen": 3210149888 + }, + { + "epoch": 0.95, + "learning_rate": 2.74434280211844e-05, + "loss": 1.1548, + "theoretical_loss": 3.3320965980874795, + "tokens_seen": 3210280960 + }, + { + "epoch": 0.95, + "learning_rate": 2.7403306050393197e-05, + "loss": 1.2355, + "theoretical_loss": 3.3320863697224823, + "tokens_seen": 3210412032 + }, + { + "epoch": 0.95, + "learning_rate": 2.736318407960199e-05, + "loss": 1.2559, + "theoretical_loss": 3.3320761418919935, + "tokens_seen": 3210543104 + }, + { + "epoch": 0.95, + "learning_rate": 2.7323062108810786e-05, + "loss": 1.1395, + "theoretical_loss": 3.3320659145959626, + "tokens_seen": 3210674176 + }, + { + "epoch": 0.95, + "learning_rate": 2.7282940138019582e-05, + "loss": 1.2133, + "theoretical_loss": 3.3320556878343397, + "tokens_seen": 3210805248 + }, + { + "epoch": 0.95, + "learning_rate": 2.7242818167228375e-05, + "loss": 1.2817, + "theoretical_loss": 3.332045461607075, + "tokens_seen": 3210936320 + }, + { + "epoch": 0.95, + "learning_rate": 2.720269619643717e-05, + "loss": 1.1926, + "theoretical_loss": 3.3320352359141197, + "tokens_seen": 3211067392 + }, + { + "epoch": 0.95, + "learning_rate": 2.7162574225645964e-05, + "loss": 1.2184, + "theoretical_loss": 3.332025010755423, + "tokens_seen": 3211198464 + }, + { + "epoch": 0.95, + "learning_rate": 2.7122452254854757e-05, + "loss": 1.198, + "theoretical_loss": 3.332014786130936, + "tokens_seen": 3211329536 + }, + { + "epoch": 0.95, + "learning_rate": 2.7082330284063553e-05, + "loss": 1.2925, + "theoretical_loss": 3.3320045620406082, + "tokens_seen": 3211460608 + }, + { + "epoch": 0.95, + "learning_rate": 2.704220831327235e-05, + "loss": 1.3043, + "theoretical_loss": 3.3319943384843906, + "tokens_seen": 3211591680 + }, + { + "epoch": 0.95, + "learning_rate": 2.7002086342481142e-05, + "loss": 1.2045, + "theoretical_loss": 3.331984115462233, + "tokens_seen": 3211722752 + }, + { + "epoch": 0.95, + "learning_rate": 2.6961964371689938e-05, + "loss": 1.2197, + "theoretical_loss": 3.331973892974086, + "tokens_seen": 3211853824 + }, + { + "epoch": 0.95, + "learning_rate": 2.6921842400898734e-05, + "loss": 1.2454, + "theoretical_loss": 3.3319636710198997, + "tokens_seen": 3211984896 + }, + { + "epoch": 0.95, + "learning_rate": 2.6881720430107527e-05, + "loss": 1.2347, + "theoretical_loss": 3.3319534495996246, + "tokens_seen": 3212115968 + }, + { + "epoch": 0.95, + "learning_rate": 2.6841598459316323e-05, + "loss": 1.2623, + "theoretical_loss": 3.3319432287132114, + "tokens_seen": 3212247040 + }, + { + "epoch": 0.95, + "learning_rate": 2.680147648852512e-05, + "loss": 1.2226, + "theoretical_loss": 3.3319330083606093, + "tokens_seen": 3212378112 + }, + { + "epoch": 0.95, + "learning_rate": 2.6761354517733912e-05, + "loss": 1.2024, + "theoretical_loss": 3.33192278854177, + "tokens_seen": 3212509184 + }, + { + "epoch": 0.95, + "learning_rate": 2.6721232546942708e-05, + "loss": 1.2874, + "theoretical_loss": 3.3319125692566427, + "tokens_seen": 3212640256 + }, + { + "epoch": 0.95, + "learning_rate": 2.66811105761515e-05, + "loss": 1.2624, + "theoretical_loss": 3.3319023505051786, + "tokens_seen": 3212771328 + }, + { + "epoch": 0.95, + "learning_rate": 2.6640988605360294e-05, + "loss": 1.2739, + "theoretical_loss": 3.3318921322873276, + "tokens_seen": 3212902400 + }, + { + "epoch": 0.95, + "learning_rate": 2.660086663456909e-05, + "loss": 1.3168, + "theoretical_loss": 3.33188191460304, + "tokens_seen": 3213033472 + }, + { + "epoch": 0.95, + "learning_rate": 2.6560744663777886e-05, + "loss": 1.2698, + "theoretical_loss": 3.3318716974522666, + "tokens_seen": 3213164544 + }, + { + "epoch": 0.95, + "learning_rate": 2.652062269298668e-05, + "loss": 1.3274, + "theoretical_loss": 3.3318614808349576, + "tokens_seen": 3213295616 + }, + { + "epoch": 0.95, + "learning_rate": 2.6480500722195475e-05, + "loss": 1.2343, + "theoretical_loss": 3.331851264751063, + "tokens_seen": 3213426688 + }, + { + "epoch": 0.95, + "learning_rate": 2.644037875140427e-05, + "loss": 1.2454, + "theoretical_loss": 3.331841049200534, + "tokens_seen": 3213557760 + }, + { + "epoch": 0.95, + "learning_rate": 2.6400256780613064e-05, + "loss": 1.1829, + "theoretical_loss": 3.3318308341833203, + "tokens_seen": 3213688832 + }, + { + "epoch": 0.95, + "learning_rate": 2.636013480982186e-05, + "loss": 1.2119, + "theoretical_loss": 3.3318206196993723, + "tokens_seen": 3213819904 + }, + { + "epoch": 0.95, + "learning_rate": 2.6320012839030656e-05, + "loss": 1.2306, + "theoretical_loss": 3.331810405748641, + "tokens_seen": 3213950976 + }, + { + "epoch": 0.95, + "learning_rate": 2.627989086823945e-05, + "loss": 1.2337, + "theoretical_loss": 3.3318001923310763, + "tokens_seen": 3214082048 + }, + { + "epoch": 0.95, + "learning_rate": 2.6239768897448245e-05, + "loss": 1.2201, + "theoretical_loss": 3.331789979446629, + "tokens_seen": 3214213120 + }, + { + "epoch": 0.95, + "learning_rate": 2.6199646926657038e-05, + "loss": 1.3061, + "theoretical_loss": 3.3317797670952487, + "tokens_seen": 3214344192 + }, + { + "epoch": 0.95, + "learning_rate": 2.615952495586583e-05, + "loss": 1.2099, + "theoretical_loss": 3.331769555276887, + "tokens_seen": 3214475264 + }, + { + "epoch": 0.95, + "learning_rate": 2.6119402985074626e-05, + "loss": 1.2489, + "theoretical_loss": 3.331759343991494, + "tokens_seen": 3214606336 + }, + { + "epoch": 0.95, + "learning_rate": 2.6079281014283423e-05, + "loss": 1.2324, + "theoretical_loss": 3.3317491332390192, + "tokens_seen": 3214737408 + }, + { + "epoch": 0.95, + "learning_rate": 2.6039159043492215e-05, + "loss": 1.2563, + "theoretical_loss": 3.3317389230194143, + "tokens_seen": 3214868480 + }, + { + "epoch": 0.95, + "learning_rate": 2.599903707270101e-05, + "loss": 1.1833, + "theoretical_loss": 3.3317287133326294, + "tokens_seen": 3214999552 + }, + { + "epoch": 0.95, + "learning_rate": 2.5958915101909808e-05, + "loss": 1.1174, + "theoretical_loss": 3.331718504178615, + "tokens_seen": 3215130624 + }, + { + "epoch": 0.95, + "learning_rate": 2.59187931311186e-05, + "loss": 1.2191, + "theoretical_loss": 3.331708295557321, + "tokens_seen": 3215261696 + }, + { + "epoch": 0.95, + "learning_rate": 2.5878671160327397e-05, + "loss": 1.221, + "theoretical_loss": 3.3316980874686988, + "tokens_seen": 3215392768 + }, + { + "epoch": 0.95, + "learning_rate": 2.5838549189536193e-05, + "loss": 1.1884, + "theoretical_loss": 3.331687879912698, + "tokens_seen": 3215523840 + }, + { + "epoch": 0.95, + "learning_rate": 2.5798427218744985e-05, + "loss": 1.2433, + "theoretical_loss": 3.3316776728892696, + "tokens_seen": 3215654912 + }, + { + "epoch": 0.95, + "learning_rate": 2.575830524795378e-05, + "loss": 1.2252, + "theoretical_loss": 3.3316674663983648, + "tokens_seen": 3215785984 + }, + { + "epoch": 0.95, + "learning_rate": 2.5718183277162574e-05, + "loss": 1.2321, + "theoretical_loss": 3.3316572604399326, + "tokens_seen": 3215917056 + }, + { + "epoch": 0.95, + "learning_rate": 2.5678061306371367e-05, + "loss": 1.2252, + "theoretical_loss": 3.3316470550139243, + "tokens_seen": 3216048128 + }, + { + "epoch": 0.95, + "learning_rate": 2.5637939335580163e-05, + "loss": 1.2528, + "theoretical_loss": 3.33163685012029, + "tokens_seen": 3216179200 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.9951158165931702, + "objective/train/avg_token_score": 0.004884195514023304, + "objective/train/docs_used": 1168814, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.0988843441009521, + "objective/train/original_loss": 1.106846809387207, + "objective/train/theoretical_loss": 3.3316266457589814, + "objective/train/tokens_used": 1586769376, + "objective/train/unlikelihood_loss": 0.014917505905032158, + "theoretical_loss": 3.3316266457589814, + "tokens_seen": 3216310272 + }, + { + "epoch": 0.95, + "learning_rate": 2.559781736478896e-05, + "loss": 1.1368, + "theoretical_loss": 3.3316266457589814, + "tokens_seen": 3216310272 + }, + { + "epoch": 0.95, + "learning_rate": 2.5557695393997752e-05, + "loss": 1.3076, + "theoretical_loss": 3.331616441929948, + "tokens_seen": 3216441344 + }, + { + "epoch": 0.95, + "learning_rate": 2.5517573423206548e-05, + "loss": 1.196, + "theoretical_loss": 3.3316062386331406, + "tokens_seen": 3216572416 + }, + { + "epoch": 0.95, + "learning_rate": 2.5477451452415344e-05, + "loss": 1.256, + "theoretical_loss": 3.33159603586851, + "tokens_seen": 3216703488 + }, + { + "epoch": 0.95, + "learning_rate": 2.5437329481624137e-05, + "loss": 1.2788, + "theoretical_loss": 3.3315858336360065, + "tokens_seen": 3216834560 + }, + { + "epoch": 0.95, + "learning_rate": 2.5397207510832933e-05, + "loss": 1.2139, + "theoretical_loss": 3.3315756319355807, + "tokens_seen": 3216965632 + }, + { + "epoch": 0.95, + "learning_rate": 2.535708554004173e-05, + "loss": 1.2569, + "theoretical_loss": 3.331565430767183, + "tokens_seen": 3217096704 + }, + { + "epoch": 0.95, + "learning_rate": 2.5316963569250522e-05, + "loss": 1.2861, + "theoretical_loss": 3.331555230130764, + "tokens_seen": 3217227776 + }, + { + "epoch": 0.95, + "learning_rate": 2.5276841598459315e-05, + "loss": 1.2078, + "theoretical_loss": 3.331545030026275, + "tokens_seen": 3217358848 + }, + { + "epoch": 0.95, + "learning_rate": 2.523671962766811e-05, + "loss": 1.1675, + "theoretical_loss": 3.3315348304536654, + "tokens_seen": 3217489920 + }, + { + "epoch": 0.95, + "learning_rate": 2.5196597656876904e-05, + "loss": 1.2479, + "theoretical_loss": 3.3315246314128872, + "tokens_seen": 3217620992 + }, + { + "epoch": 0.95, + "learning_rate": 2.51564756860857e-05, + "loss": 1.256, + "theoretical_loss": 3.3315144329038895, + "tokens_seen": 3217752064 + }, + { + "epoch": 0.95, + "learning_rate": 2.5116353715294496e-05, + "loss": 1.2114, + "theoretical_loss": 3.331504234926624, + "tokens_seen": 3217883136 + }, + { + "epoch": 0.95, + "learning_rate": 2.507623174450329e-05, + "loss": 1.2661, + "theoretical_loss": 3.331494037481041, + "tokens_seen": 3218014208 + }, + { + "epoch": 0.95, + "learning_rate": 2.5036109773712085e-05, + "loss": 1.2091, + "theoretical_loss": 3.331483840567091, + "tokens_seen": 3218145280 + }, + { + "epoch": 0.95, + "learning_rate": 2.499598780292088e-05, + "loss": 1.2152, + "theoretical_loss": 3.331473644184725, + "tokens_seen": 3218276352 + }, + { + "epoch": 0.95, + "learning_rate": 2.4955865832129674e-05, + "loss": 1.2296, + "theoretical_loss": 3.331463448333893, + "tokens_seen": 3218407424 + }, + { + "epoch": 0.95, + "learning_rate": 2.491574386133847e-05, + "loss": 1.2735, + "theoretical_loss": 3.331453253014546, + "tokens_seen": 3218538496 + }, + { + "epoch": 0.95, + "learning_rate": 2.4875621890547266e-05, + "loss": 1.1851, + "theoretical_loss": 3.3314430582266352, + "tokens_seen": 3218669568 + }, + { + "epoch": 0.95, + "learning_rate": 2.483549991975606e-05, + "loss": 1.1591, + "theoretical_loss": 3.33143286397011, + "tokens_seen": 3218800640 + }, + { + "epoch": 0.95, + "learning_rate": 2.4795377948964852e-05, + "loss": 1.2792, + "theoretical_loss": 3.331422670244922, + "tokens_seen": 3218931712 + }, + { + "epoch": 0.95, + "learning_rate": 2.4755255978173648e-05, + "loss": 1.2445, + "theoretical_loss": 3.331412477051022, + "tokens_seen": 3219062784 + }, + { + "epoch": 0.95, + "learning_rate": 2.471513400738244e-05, + "loss": 1.1905, + "theoretical_loss": 3.33140228438836, + "tokens_seen": 3219193856 + }, + { + "epoch": 0.95, + "learning_rate": 2.4675012036591237e-05, + "loss": 1.2988, + "theoretical_loss": 3.3313920922568876, + "tokens_seen": 3219324928 + }, + { + "epoch": 0.95, + "learning_rate": 2.4634890065800033e-05, + "loss": 1.1915, + "theoretical_loss": 3.3313819006565546, + "tokens_seen": 3219456000 + }, + { + "epoch": 0.95, + "learning_rate": 2.4594768095008826e-05, + "loss": 1.2111, + "theoretical_loss": 3.331371709587312, + "tokens_seen": 3219587072 + }, + { + "epoch": 0.95, + "learning_rate": 2.4554646124217622e-05, + "loss": 1.2075, + "theoretical_loss": 3.33136151904911, + "tokens_seen": 3219718144 + }, + { + "epoch": 0.95, + "learning_rate": 2.4514524153426418e-05, + "loss": 1.2029, + "theoretical_loss": 3.3313513290419006, + "tokens_seen": 3219849216 + }, + { + "epoch": 0.95, + "learning_rate": 2.447440218263521e-05, + "loss": 1.2154, + "theoretical_loss": 3.3313411395656334, + "tokens_seen": 3219980288 + }, + { + "epoch": 0.95, + "learning_rate": 2.4434280211844007e-05, + "loss": 1.2192, + "theoretical_loss": 3.3313309506202597, + "tokens_seen": 3220111360 + }, + { + "epoch": 0.95, + "learning_rate": 2.4394158241052803e-05, + "loss": 1.2177, + "theoretical_loss": 3.33132076220573, + "tokens_seen": 3220242432 + }, + { + "epoch": 0.95, + "learning_rate": 2.4354036270261596e-05, + "loss": 1.1961, + "theoretical_loss": 3.3313105743219946, + "tokens_seen": 3220373504 + }, + { + "epoch": 0.95, + "learning_rate": 2.431391429947039e-05, + "loss": 1.1752, + "theoretical_loss": 3.331300386969005, + "tokens_seen": 3220504576 + }, + { + "epoch": 0.95, + "learning_rate": 2.4273792328679185e-05, + "loss": 1.298, + "theoretical_loss": 3.3312902001467117, + "tokens_seen": 3220635648 + }, + { + "epoch": 0.95, + "learning_rate": 2.4233670357887977e-05, + "loss": 1.2758, + "theoretical_loss": 3.3312800138550656, + "tokens_seen": 3220766720 + }, + { + "epoch": 0.95, + "learning_rate": 2.4193548387096773e-05, + "loss": 1.3095, + "theoretical_loss": 3.331269828094017, + "tokens_seen": 3220897792 + }, + { + "epoch": 0.95, + "learning_rate": 2.415342641630557e-05, + "loss": 1.218, + "theoretical_loss": 3.3312596428635173, + "tokens_seen": 3221028864 + }, + { + "epoch": 0.95, + "learning_rate": 2.4113304445514366e-05, + "loss": 1.1722, + "theoretical_loss": 3.3312494581635166, + "tokens_seen": 3221159936 + }, + { + "epoch": 0.95, + "learning_rate": 2.407318247472316e-05, + "loss": 1.3261, + "theoretical_loss": 3.331239273993966, + "tokens_seen": 3221291008 + }, + { + "epoch": 0.95, + "learning_rate": 2.4033060503931955e-05, + "loss": 1.1615, + "theoretical_loss": 3.331229090354816, + "tokens_seen": 3221422080 + }, + { + "epoch": 0.95, + "learning_rate": 2.399293853314075e-05, + "loss": 1.263, + "theoretical_loss": 3.331218907246018, + "tokens_seen": 3221553152 + }, + { + "epoch": 0.95, + "learning_rate": 2.3952816562349544e-05, + "loss": 1.2452, + "theoretical_loss": 3.3312087246675226, + "tokens_seen": 3221684224 + }, + { + "epoch": 0.95, + "learning_rate": 2.391269459155834e-05, + "loss": 1.223, + "theoretical_loss": 3.3311985426192807, + "tokens_seen": 3221815296 + }, + { + "epoch": 0.95, + "learning_rate": 2.3872572620767136e-05, + "loss": 1.2506, + "theoretical_loss": 3.3311883611012423, + "tokens_seen": 3221946368 + }, + { + "epoch": 0.95, + "learning_rate": 2.3832450649975925e-05, + "loss": 1.2147, + "theoretical_loss": 3.3311781801133593, + "tokens_seen": 3222077440 + }, + { + "epoch": 0.95, + "learning_rate": 2.379232867918472e-05, + "loss": 1.2893, + "theoretical_loss": 3.331167999655582, + "tokens_seen": 3222208512 + }, + { + "epoch": 0.95, + "learning_rate": 2.3752206708393518e-05, + "loss": 1.2057, + "theoretical_loss": 3.3311578197278613, + "tokens_seen": 3222339584 + }, + { + "epoch": 0.95, + "learning_rate": 2.371208473760231e-05, + "loss": 1.1682, + "theoretical_loss": 3.331147640330148, + "tokens_seen": 3222470656 + }, + { + "epoch": 0.95, + "learning_rate": 2.3671962766811106e-05, + "loss": 1.2065, + "theoretical_loss": 3.3311374614623928, + "tokens_seen": 3222601728 + }, + { + "epoch": 0.95, + "learning_rate": 2.3631840796019903e-05, + "loss": 1.2455, + "theoretical_loss": 3.3311272831245473, + "tokens_seen": 3222732800 + }, + { + "epoch": 0.95, + "objective/train/aligned_token_count": 0.9934417605400085, + "objective/train/avg_token_score": 0.0065582506358623505, + "objective/train/docs_used": 1170779, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2450804710388184, + "objective/train/original_loss": 1.2538104057312012, + "objective/train/theoretical_loss": 3.3311171053165616, + "objective/train/tokens_used": 1593322976, + "objective/train/unlikelihood_loss": 0.020034944638609886, + "theoretical_loss": 3.3311171053165616, + "tokens_seen": 3222863872 + }, + { + "epoch": 0.95, + "learning_rate": 2.3591718825228695e-05, + "loss": 1.2338, + "theoretical_loss": 3.3311171053165616, + "tokens_seen": 3222863872 + }, + { + "epoch": 0.95, + "learning_rate": 2.355159685443749e-05, + "loss": 1.2337, + "theoretical_loss": 3.3311069280383867, + "tokens_seen": 3222994944 + }, + { + "epoch": 0.95, + "learning_rate": 2.3511474883646288e-05, + "loss": 1.2814, + "theoretical_loss": 3.3310967512899734, + "tokens_seen": 3223126016 + }, + { + "epoch": 0.95, + "learning_rate": 2.347135291285508e-05, + "loss": 1.3179, + "theoretical_loss": 3.331086575071273, + "tokens_seen": 3223257088 + }, + { + "epoch": 0.95, + "learning_rate": 2.3431230942063876e-05, + "loss": 1.2681, + "theoretical_loss": 3.331076399382236, + "tokens_seen": 3223388160 + }, + { + "epoch": 0.95, + "learning_rate": 2.339110897127267e-05, + "loss": 1.2657, + "theoretical_loss": 3.3310662242228135, + "tokens_seen": 3223519232 + }, + { + "epoch": 0.95, + "learning_rate": 2.3350987000481462e-05, + "loss": 1.16, + "theoretical_loss": 3.331056049592956, + "tokens_seen": 3223650304 + }, + { + "epoch": 0.95, + "learning_rate": 2.3310865029690258e-05, + "loss": 1.278, + "theoretical_loss": 3.3310458754926153, + "tokens_seen": 3223781376 + }, + { + "epoch": 0.95, + "learning_rate": 2.3270743058899054e-05, + "loss": 1.1238, + "theoretical_loss": 3.3310357019217416, + "tokens_seen": 3223912448 + }, + { + "epoch": 0.95, + "learning_rate": 2.3230621088107847e-05, + "loss": 1.2183, + "theoretical_loss": 3.331025528880286, + "tokens_seen": 3224043520 + }, + { + "epoch": 0.95, + "learning_rate": 2.3190499117316643e-05, + "loss": 1.2375, + "theoretical_loss": 3.3310153563681992, + "tokens_seen": 3224174592 + }, + { + "epoch": 0.95, + "learning_rate": 2.315037714652544e-05, + "loss": 1.2118, + "theoretical_loss": 3.3310051843854325, + "tokens_seen": 3224305664 + }, + { + "epoch": 0.95, + "learning_rate": 2.3110255175734232e-05, + "loss": 1.2615, + "theoretical_loss": 3.330995012931937, + "tokens_seen": 3224436736 + }, + { + "epoch": 0.95, + "learning_rate": 2.3070133204943028e-05, + "loss": 1.2828, + "theoretical_loss": 3.3309848420076626, + "tokens_seen": 3224567808 + }, + { + "epoch": 0.95, + "learning_rate": 2.3030011234151824e-05, + "loss": 1.3072, + "theoretical_loss": 3.3309746716125614, + "tokens_seen": 3224698880 + }, + { + "epoch": 0.95, + "learning_rate": 2.2989889263360617e-05, + "loss": 1.2565, + "theoretical_loss": 3.330964501746584, + "tokens_seen": 3224829952 + }, + { + "epoch": 0.95, + "learning_rate": 2.2949767292569413e-05, + "loss": 1.2564, + "theoretical_loss": 3.3309543324096813, + "tokens_seen": 3224961024 + }, + { + "epoch": 0.95, + "learning_rate": 2.2909645321778206e-05, + "loss": 1.2225, + "theoretical_loss": 3.3309441636018047, + "tokens_seen": 3225092096 + }, + { + "epoch": 0.95, + "learning_rate": 2.2869523350987e-05, + "loss": 1.218, + "theoretical_loss": 3.330933995322904, + "tokens_seen": 3225223168 + }, + { + "epoch": 0.95, + "learning_rate": 2.2829401380195795e-05, + "loss": 1.3032, + "theoretical_loss": 3.3309238275729314, + "tokens_seen": 3225354240 + }, + { + "epoch": 0.95, + "learning_rate": 2.278927940940459e-05, + "loss": 1.218, + "theoretical_loss": 3.3309136603518374, + "tokens_seen": 3225485312 + }, + { + "epoch": 0.95, + "learning_rate": 2.2749157438613384e-05, + "loss": 1.2362, + "theoretical_loss": 3.330903493659573, + "tokens_seen": 3225616384 + }, + { + "epoch": 0.96, + "learning_rate": 2.270903546782218e-05, + "loss": 1.2361, + "theoretical_loss": 3.3308933274960895, + "tokens_seen": 3225747456 + }, + { + "epoch": 0.96, + "learning_rate": 2.2668913497030976e-05, + "loss": 1.2749, + "theoretical_loss": 3.3308831618613373, + "tokens_seen": 3225878528 + }, + { + "epoch": 0.96, + "learning_rate": 2.262879152623977e-05, + "loss": 1.2577, + "theoretical_loss": 3.330872996755268, + "tokens_seen": 3226009600 + }, + { + "epoch": 0.96, + "learning_rate": 2.2588669555448565e-05, + "loss": 1.2135, + "theoretical_loss": 3.330862832177832, + "tokens_seen": 3226140672 + }, + { + "epoch": 0.96, + "learning_rate": 2.254854758465736e-05, + "loss": 1.2259, + "theoretical_loss": 3.330852668128981, + "tokens_seen": 3226271744 + }, + { + "epoch": 0.96, + "learning_rate": 2.2508425613866154e-05, + "loss": 1.3149, + "theoretical_loss": 3.330842504608666, + "tokens_seen": 3226402816 + }, + { + "epoch": 0.96, + "learning_rate": 2.246830364307495e-05, + "loss": 1.2319, + "theoretical_loss": 3.330832341616838, + "tokens_seen": 3226533888 + }, + { + "epoch": 0.96, + "learning_rate": 2.2428181672283743e-05, + "loss": 1.2939, + "theoretical_loss": 3.3308221791534476, + "tokens_seen": 3226664960 + }, + { + "epoch": 0.96, + "learning_rate": 2.2388059701492536e-05, + "loss": 1.2172, + "theoretical_loss": 3.330812017218446, + "tokens_seen": 3226796032 + }, + { + "epoch": 0.96, + "learning_rate": 2.234793773070133e-05, + "loss": 1.1988, + "theoretical_loss": 3.3308018558117847, + "tokens_seen": 3226927104 + }, + { + "epoch": 0.96, + "learning_rate": 2.2307815759910128e-05, + "loss": 1.302, + "theoretical_loss": 3.3307916949334144, + "tokens_seen": 3227058176 + }, + { + "epoch": 0.96, + "learning_rate": 2.226769378911892e-05, + "loss": 1.2785, + "theoretical_loss": 3.330781534583286, + "tokens_seen": 3227189248 + }, + { + "epoch": 0.96, + "learning_rate": 2.2227571818327717e-05, + "loss": 1.2708, + "theoretical_loss": 3.330771374761351, + "tokens_seen": 3227320320 + }, + { + "epoch": 0.96, + "learning_rate": 2.2187449847536513e-05, + "loss": 1.1905, + "theoretical_loss": 3.330761215467561, + "tokens_seen": 3227451392 + }, + { + "epoch": 0.96, + "learning_rate": 2.2147327876745306e-05, + "loss": 1.2265, + "theoretical_loss": 3.3307510567018657, + "tokens_seen": 3227582464 + }, + { + "epoch": 0.96, + "learning_rate": 2.2107205905954102e-05, + "loss": 1.2349, + "theoretical_loss": 3.3307408984642173, + "tokens_seen": 3227713536 + }, + { + "epoch": 0.96, + "learning_rate": 2.2067083935162898e-05, + "loss": 1.2468, + "theoretical_loss": 3.330730740754566, + "tokens_seen": 3227844608 + }, + { + "epoch": 0.96, + "learning_rate": 2.202696196437169e-05, + "loss": 1.2858, + "theoretical_loss": 3.330720583572864, + "tokens_seen": 3227975680 + }, + { + "epoch": 0.96, + "learning_rate": 2.1986839993580487e-05, + "loss": 1.3055, + "theoretical_loss": 3.3307104269190617, + "tokens_seen": 3228106752 + }, + { + "epoch": 0.96, + "learning_rate": 2.194671802278928e-05, + "loss": 1.2166, + "theoretical_loss": 3.330700270793111, + "tokens_seen": 3228237824 + }, + { + "epoch": 0.96, + "learning_rate": 2.1906596051998072e-05, + "loss": 1.2611, + "theoretical_loss": 3.3306901151949617, + "tokens_seen": 3228368896 + }, + { + "epoch": 0.96, + "learning_rate": 2.186647408120687e-05, + "loss": 1.1629, + "theoretical_loss": 3.330679960124566, + "tokens_seen": 3228499968 + }, + { + "epoch": 0.96, + "learning_rate": 2.1826352110415665e-05, + "loss": 1.2474, + "theoretical_loss": 3.3306698055818744, + "tokens_seen": 3228631040 + }, + { + "epoch": 0.96, + "learning_rate": 2.1786230139624457e-05, + "loss": 1.2895, + "theoretical_loss": 3.330659651566839, + "tokens_seen": 3228762112 + }, + { + "epoch": 0.96, + "learning_rate": 2.1746108168833253e-05, + "loss": 1.3218, + "theoretical_loss": 3.33064949807941, + "tokens_seen": 3228893184 + }, + { + "epoch": 0.96, + "learning_rate": 2.170598619804205e-05, + "loss": 1.1842, + "theoretical_loss": 3.3306393451195393, + "tokens_seen": 3229024256 + }, + { + "epoch": 0.96, + "learning_rate": 2.1665864227250842e-05, + "loss": 1.1779, + "theoretical_loss": 3.3306291926871774, + "tokens_seen": 3229155328 + }, + { + "epoch": 0.96, + "learning_rate": 2.162574225645964e-05, + "loss": 1.2595, + "theoretical_loss": 3.3306190407822758, + "tokens_seen": 3229286400 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.9933333396911621, + "objective/train/avg_token_score": 0.006666652392596006, + "objective/train/docs_used": 1173331, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1803648471832275, + "objective/train/original_loss": 1.1896822452545166, + "objective/train/theoretical_loss": 3.330608889404786, + "objective/train/tokens_used": 1599876576, + "objective/train/unlikelihood_loss": 0.01913047581911087, + "theoretical_loss": 3.330608889404786, + "tokens_seen": 3229417472 + }, + { + "epoch": 0.96, + "learning_rate": 2.1585620285668435e-05, + "loss": 1.2295, + "theoretical_loss": 3.330608889404786, + "tokens_seen": 3229417472 + }, + { + "epoch": 0.96, + "learning_rate": 2.1545498314877227e-05, + "loss": 1.2461, + "theoretical_loss": 3.3305987385546585, + "tokens_seen": 3229548544 + }, + { + "epoch": 0.96, + "learning_rate": 2.1505376344086024e-05, + "loss": 1.2703, + "theoretical_loss": 3.330588588231845, + "tokens_seen": 3229679616 + }, + { + "epoch": 0.96, + "learning_rate": 2.1465254373294816e-05, + "loss": 1.1723, + "theoretical_loss": 3.330578438436297, + "tokens_seen": 3229810688 + }, + { + "epoch": 0.96, + "learning_rate": 2.142513240250361e-05, + "loss": 1.1343, + "theoretical_loss": 3.3305682891679647, + "tokens_seen": 3229941760 + }, + { + "epoch": 0.96, + "learning_rate": 2.1385010431712405e-05, + "loss": 1.238, + "theoretical_loss": 3.3305581404268003, + "tokens_seen": 3230072832 + }, + { + "epoch": 0.96, + "learning_rate": 2.13448884609212e-05, + "loss": 1.2168, + "theoretical_loss": 3.3305479922127548, + "tokens_seen": 3230203904 + }, + { + "epoch": 0.96, + "learning_rate": 2.1304766490129994e-05, + "loss": 1.286, + "theoretical_loss": 3.3305378445257787, + "tokens_seen": 3230334976 + }, + { + "epoch": 0.96, + "learning_rate": 2.126464451933879e-05, + "loss": 1.2029, + "theoretical_loss": 3.3305276973658247, + "tokens_seen": 3230466048 + }, + { + "epoch": 0.96, + "learning_rate": 2.1224522548547586e-05, + "loss": 1.239, + "theoretical_loss": 3.3305175507328424, + "tokens_seen": 3230597120 + }, + { + "epoch": 0.96, + "learning_rate": 2.118440057775638e-05, + "loss": 1.2485, + "theoretical_loss": 3.3305074046267844, + "tokens_seen": 3230728192 + }, + { + "epoch": 0.96, + "learning_rate": 2.1144278606965175e-05, + "loss": 1.1884, + "theoretical_loss": 3.3304972590476014, + "tokens_seen": 3230859264 + }, + { + "epoch": 0.96, + "learning_rate": 2.110415663617397e-05, + "loss": 1.2909, + "theoretical_loss": 3.330487113995244, + "tokens_seen": 3230990336 + }, + { + "epoch": 0.96, + "learning_rate": 2.1064034665382764e-05, + "loss": 1.2381, + "theoretical_loss": 3.330476969469665, + "tokens_seen": 3231121408 + }, + { + "epoch": 0.96, + "learning_rate": 2.1023912694591557e-05, + "loss": 1.2784, + "theoretical_loss": 3.3304668254708143, + "tokens_seen": 3231252480 + }, + { + "epoch": 0.96, + "learning_rate": 2.0983790723800353e-05, + "loss": 1.2578, + "theoretical_loss": 3.3304566819986436, + "tokens_seen": 3231383552 + }, + { + "epoch": 0.96, + "learning_rate": 2.0943668753009146e-05, + "loss": 1.2138, + "theoretical_loss": 3.3304465390531046, + "tokens_seen": 3231514624 + }, + { + "epoch": 0.96, + "learning_rate": 2.0903546782217942e-05, + "loss": 1.2367, + "theoretical_loss": 3.3304363966341484, + "tokens_seen": 3231645696 + }, + { + "epoch": 0.96, + "learning_rate": 2.0863424811426738e-05, + "loss": 1.257, + "theoretical_loss": 3.330426254741726, + "tokens_seen": 3231776768 + }, + { + "epoch": 0.96, + "learning_rate": 2.082330284063553e-05, + "loss": 1.3106, + "theoretical_loss": 3.3304161133757892, + "tokens_seen": 3231907840 + }, + { + "epoch": 0.96, + "learning_rate": 2.0783180869844327e-05, + "loss": 1.2323, + "theoretical_loss": 3.3304059725362887, + "tokens_seen": 3232038912 + }, + { + "epoch": 0.96, + "learning_rate": 2.0743058899053123e-05, + "loss": 1.1965, + "theoretical_loss": 3.3303958322231764, + "tokens_seen": 3232169984 + }, + { + "epoch": 0.96, + "learning_rate": 2.0702936928261916e-05, + "loss": 1.1785, + "theoretical_loss": 3.330385692436403, + "tokens_seen": 3232301056 + }, + { + "epoch": 0.96, + "learning_rate": 2.0662814957470712e-05, + "loss": 1.2768, + "theoretical_loss": 3.3303755531759207, + "tokens_seen": 3232432128 + }, + { + "epoch": 0.96, + "learning_rate": 2.0622692986679508e-05, + "loss": 1.2659, + "theoretical_loss": 3.33036541444168, + "tokens_seen": 3232563200 + }, + { + "epoch": 0.96, + "learning_rate": 2.05825710158883e-05, + "loss": 1.284, + "theoretical_loss": 3.3303552762336333, + "tokens_seen": 3232694272 + }, + { + "epoch": 0.96, + "learning_rate": 2.0542449045097094e-05, + "loss": 1.2304, + "theoretical_loss": 3.3303451385517304, + "tokens_seen": 3232825344 + }, + { + "epoch": 0.96, + "learning_rate": 2.050232707430589e-05, + "loss": 1.2979, + "theoretical_loss": 3.330335001395924, + "tokens_seen": 3232956416 + }, + { + "epoch": 0.96, + "learning_rate": 2.0462205103514683e-05, + "loss": 1.2001, + "theoretical_loss": 3.330324864766165, + "tokens_seen": 3233087488 + }, + { + "epoch": 0.96, + "learning_rate": 2.042208313272348e-05, + "loss": 1.2585, + "theoretical_loss": 3.330314728662405, + "tokens_seen": 3233218560 + }, + { + "epoch": 0.96, + "learning_rate": 2.0381961161932275e-05, + "loss": 1.2341, + "theoretical_loss": 3.3303045930845947, + "tokens_seen": 3233349632 + }, + { + "epoch": 0.96, + "learning_rate": 2.0341839191141068e-05, + "loss": 1.2379, + "theoretical_loss": 3.3302944580326868, + "tokens_seen": 3233480704 + }, + { + "epoch": 0.96, + "learning_rate": 2.0301717220349864e-05, + "loss": 1.2282, + "theoretical_loss": 3.330284323506631, + "tokens_seen": 3233611776 + }, + { + "epoch": 0.96, + "learning_rate": 2.026159524955866e-05, + "loss": 1.1809, + "theoretical_loss": 3.3302741895063797, + "tokens_seen": 3233742848 + }, + { + "epoch": 0.96, + "learning_rate": 2.0221473278767453e-05, + "loss": 1.219, + "theoretical_loss": 3.3302640560318846, + "tokens_seen": 3233873920 + }, + { + "epoch": 0.96, + "learning_rate": 2.018135130797625e-05, + "loss": 1.2508, + "theoretical_loss": 3.3302539230830965, + "tokens_seen": 3234004992 + }, + { + "epoch": 0.96, + "learning_rate": 2.0141229337185045e-05, + "loss": 1.2186, + "theoretical_loss": 3.330243790659967, + "tokens_seen": 3234136064 + }, + { + "epoch": 0.96, + "learning_rate": 2.0101107366393838e-05, + "loss": 1.23, + "theoretical_loss": 3.330233658762448, + "tokens_seen": 3234267136 + }, + { + "epoch": 0.96, + "learning_rate": 2.006098539560263e-05, + "loss": 1.1994, + "theoretical_loss": 3.3302235273904897, + "tokens_seen": 3234398208 + }, + { + "epoch": 0.96, + "learning_rate": 2.0020863424811427e-05, + "loss": 1.2603, + "theoretical_loss": 3.330213396544045, + "tokens_seen": 3234529280 + }, + { + "epoch": 0.96, + "learning_rate": 1.998074145402022e-05, + "loss": 1.2323, + "theoretical_loss": 3.330203266223064, + "tokens_seen": 3234660352 + }, + { + "epoch": 0.96, + "learning_rate": 1.9940619483229015e-05, + "loss": 1.2669, + "theoretical_loss": 3.3301931364274995, + "tokens_seen": 3234791424 + }, + { + "epoch": 0.96, + "learning_rate": 1.990049751243781e-05, + "loss": 1.2526, + "theoretical_loss": 3.3301830071573018, + "tokens_seen": 3234922496 + }, + { + "epoch": 0.96, + "learning_rate": 1.9860375541646604e-05, + "loss": 1.2809, + "theoretical_loss": 3.330172878412423, + "tokens_seen": 3235053568 + }, + { + "epoch": 0.96, + "learning_rate": 1.98202535708554e-05, + "loss": 1.2453, + "theoretical_loss": 3.3301627501928146, + "tokens_seen": 3235184640 + }, + { + "epoch": 0.96, + "learning_rate": 1.9780131600064197e-05, + "loss": 1.1991, + "theoretical_loss": 3.330152622498428, + "tokens_seen": 3235315712 + }, + { + "epoch": 0.96, + "learning_rate": 1.9740009629272993e-05, + "loss": 1.3151, + "theoretical_loss": 3.3301424953292145, + "tokens_seen": 3235446784 + }, + { + "epoch": 0.96, + "learning_rate": 1.9699887658481786e-05, + "loss": 1.1655, + "theoretical_loss": 3.3301323686851254, + "tokens_seen": 3235577856 + }, + { + "epoch": 0.96, + "learning_rate": 1.965976568769058e-05, + "loss": 1.297, + "theoretical_loss": 3.3301222425661123, + "tokens_seen": 3235708928 + }, + { + "epoch": 0.96, + "learning_rate": 1.9619643716899378e-05, + "loss": 1.2352, + "theoretical_loss": 3.3301121169721277, + "tokens_seen": 3235840000 + }, + { + "epoch": 0.96, + "objective/train/aligned_token_count": 0.9917211532592773, + "objective/train/avg_token_score": 0.008278782479465008, + "objective/train/docs_used": 1175644, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2416274547576904, + "objective/train/original_loss": 1.2537846565246582, + "objective/train/theoretical_loss": 3.330101991903122, + "objective/train/tokens_used": 1606430176, + "objective/train/unlikelihood_loss": 0.02301987074315548, + "theoretical_loss": 3.330101991903122, + "tokens_seen": 3235971072 + }, + { + "epoch": 0.96, + "learning_rate": 1.9579521746108167e-05, + "loss": 1.2333, + "theoretical_loss": 3.330101991903122, + "tokens_seen": 3235971072 + }, + { + "epoch": 0.96, + "learning_rate": 1.9539399775316963e-05, + "loss": 1.2268, + "theoretical_loss": 3.3300918673590467, + "tokens_seen": 3236102144 + }, + { + "epoch": 0.96, + "learning_rate": 1.949927780452576e-05, + "loss": 1.2592, + "theoretical_loss": 3.330081743339854, + "tokens_seen": 3236233216 + }, + { + "epoch": 0.96, + "learning_rate": 1.9459155833734552e-05, + "loss": 1.1881, + "theoretical_loss": 3.3300716198454947, + "tokens_seen": 3236364288 + }, + { + "epoch": 0.96, + "learning_rate": 1.941903386294335e-05, + "loss": 1.2595, + "theoretical_loss": 3.3300614968759215, + "tokens_seen": 3236495360 + }, + { + "epoch": 0.96, + "learning_rate": 1.9378911892152144e-05, + "loss": 1.2372, + "theoretical_loss": 3.3300513744310845, + "tokens_seen": 3236626432 + }, + { + "epoch": 0.96, + "learning_rate": 1.9338789921360937e-05, + "loss": 1.1697, + "theoretical_loss": 3.330041252510936, + "tokens_seen": 3236757504 + }, + { + "epoch": 0.96, + "learning_rate": 1.9298667950569733e-05, + "loss": 1.2331, + "theoretical_loss": 3.330031131115428, + "tokens_seen": 3236888576 + }, + { + "epoch": 0.96, + "learning_rate": 1.925854597977853e-05, + "loss": 1.2513, + "theoretical_loss": 3.3300210102445105, + "tokens_seen": 3237019648 + }, + { + "epoch": 0.96, + "learning_rate": 1.9218424008987322e-05, + "loss": 1.2201, + "theoretical_loss": 3.330010889898137, + "tokens_seen": 3237150720 + }, + { + "epoch": 0.96, + "learning_rate": 1.917830203819612e-05, + "loss": 1.1759, + "theoretical_loss": 3.330000770076258, + "tokens_seen": 3237281792 + }, + { + "epoch": 0.96, + "learning_rate": 1.913818006740491e-05, + "loss": 1.2096, + "theoretical_loss": 3.329990650778825, + "tokens_seen": 3237412864 + }, + { + "epoch": 0.96, + "learning_rate": 1.9098058096613704e-05, + "loss": 1.2876, + "theoretical_loss": 3.3299805320057905, + "tokens_seen": 3237543936 + }, + { + "epoch": 0.96, + "learning_rate": 1.90579361258225e-05, + "loss": 1.2983, + "theoretical_loss": 3.329970413757105, + "tokens_seen": 3237675008 + }, + { + "epoch": 0.96, + "learning_rate": 1.9017814155031296e-05, + "loss": 1.3114, + "theoretical_loss": 3.3299602960327204, + "tokens_seen": 3237806080 + }, + { + "epoch": 0.96, + "learning_rate": 1.897769218424009e-05, + "loss": 1.1683, + "theoretical_loss": 3.329950178832589, + "tokens_seen": 3237937152 + }, + { + "epoch": 0.96, + "learning_rate": 1.8937570213448885e-05, + "loss": 1.2577, + "theoretical_loss": 3.3299400621566613, + "tokens_seen": 3238068224 + }, + { + "epoch": 0.96, + "learning_rate": 1.889744824265768e-05, + "loss": 1.2215, + "theoretical_loss": 3.32992994600489, + "tokens_seen": 3238199296 + }, + { + "epoch": 0.96, + "learning_rate": 1.8857326271866474e-05, + "loss": 1.2214, + "theoretical_loss": 3.329919830377226, + "tokens_seen": 3238330368 + }, + { + "epoch": 0.96, + "learning_rate": 1.881720430107527e-05, + "loss": 1.2328, + "theoretical_loss": 3.329909715273621, + "tokens_seen": 3238461440 + }, + { + "epoch": 0.96, + "learning_rate": 1.8777082330284066e-05, + "loss": 1.2629, + "theoretical_loss": 3.329899600694027, + "tokens_seen": 3238592512 + }, + { + "epoch": 0.96, + "learning_rate": 1.873696035949286e-05, + "loss": 1.2802, + "theoretical_loss": 3.3298894866383955, + "tokens_seen": 3238723584 + }, + { + "epoch": 0.96, + "learning_rate": 1.8696838388701655e-05, + "loss": 1.3004, + "theoretical_loss": 3.329879373106678, + "tokens_seen": 3238854656 + }, + { + "epoch": 0.96, + "learning_rate": 1.8656716417910448e-05, + "loss": 1.1553, + "theoretical_loss": 3.3298692600988264, + "tokens_seen": 3238985728 + }, + { + "epoch": 0.96, + "learning_rate": 1.861659444711924e-05, + "loss": 1.2513, + "theoretical_loss": 3.3298591476147923, + "tokens_seen": 3239116800 + }, + { + "epoch": 0.96, + "learning_rate": 1.8576472476328037e-05, + "loss": 1.2619, + "theoretical_loss": 3.3298490356545276, + "tokens_seen": 3239247872 + }, + { + "epoch": 0.96, + "learning_rate": 1.8536350505536833e-05, + "loss": 1.2884, + "theoretical_loss": 3.329838924217983, + "tokens_seen": 3239378944 + }, + { + "epoch": 0.96, + "learning_rate": 1.8496228534745626e-05, + "loss": 1.292, + "theoretical_loss": 3.3298288133051113, + "tokens_seen": 3239510016 + }, + { + "epoch": 0.96, + "learning_rate": 1.8456106563954422e-05, + "loss": 1.2122, + "theoretical_loss": 3.3298187029158637, + "tokens_seen": 3239641088 + }, + { + "epoch": 0.96, + "learning_rate": 1.8415984593163218e-05, + "loss": 1.1944, + "theoretical_loss": 3.329808593050192, + "tokens_seen": 3239772160 + }, + { + "epoch": 0.96, + "learning_rate": 1.837586262237201e-05, + "loss": 1.207, + "theoretical_loss": 3.329798483708048, + "tokens_seen": 3239903232 + }, + { + "epoch": 0.96, + "learning_rate": 1.8335740651580807e-05, + "loss": 1.2315, + "theoretical_loss": 3.3297883748893833, + "tokens_seen": 3240034304 + }, + { + "epoch": 0.96, + "learning_rate": 1.8295618680789603e-05, + "loss": 1.2759, + "theoretical_loss": 3.3297782665941495, + "tokens_seen": 3240165376 + }, + { + "epoch": 0.96, + "learning_rate": 1.8255496709998396e-05, + "loss": 1.2704, + "theoretical_loss": 3.329768158822298, + "tokens_seen": 3240296448 + }, + { + "epoch": 0.96, + "learning_rate": 1.8215374739207192e-05, + "loss": 1.3329, + "theoretical_loss": 3.3297580515737817, + "tokens_seen": 3240427520 + }, + { + "epoch": 0.96, + "learning_rate": 1.8175252768415985e-05, + "loss": 1.2655, + "theoretical_loss": 3.329747944848551, + "tokens_seen": 3240558592 + }, + { + "epoch": 0.96, + "learning_rate": 1.8135130797624777e-05, + "loss": 1.2436, + "theoretical_loss": 3.3297378386465586, + "tokens_seen": 3240689664 + }, + { + "epoch": 0.96, + "learning_rate": 1.8095008826833574e-05, + "loss": 1.2094, + "theoretical_loss": 3.329727732967756, + "tokens_seen": 3240820736 + }, + { + "epoch": 0.96, + "learning_rate": 1.805488685604237e-05, + "loss": 1.1723, + "theoretical_loss": 3.329717627812095, + "tokens_seen": 3240951808 + }, + { + "epoch": 0.96, + "learning_rate": 1.8014764885251163e-05, + "loss": 1.2589, + "theoretical_loss": 3.3297075231795263, + "tokens_seen": 3241082880 + }, + { + "epoch": 0.96, + "learning_rate": 1.797464291445996e-05, + "loss": 1.2581, + "theoretical_loss": 3.3296974190700035, + "tokens_seen": 3241213952 + }, + { + "epoch": 0.96, + "learning_rate": 1.7934520943668755e-05, + "loss": 1.2384, + "theoretical_loss": 3.329687315483477, + "tokens_seen": 3241345024 + }, + { + "epoch": 0.96, + "learning_rate": 1.7894398972877548e-05, + "loss": 1.1527, + "theoretical_loss": 3.3296772124198992, + "tokens_seen": 3241476096 + }, + { + "epoch": 0.96, + "learning_rate": 1.7854277002086344e-05, + "loss": 1.152, + "theoretical_loss": 3.329667109879222, + "tokens_seen": 3241607168 + }, + { + "epoch": 0.96, + "learning_rate": 1.781415503129514e-05, + "loss": 1.2611, + "theoretical_loss": 3.3296570078613965, + "tokens_seen": 3241738240 + }, + { + "epoch": 0.96, + "learning_rate": 1.7774033060503933e-05, + "loss": 1.2401, + "theoretical_loss": 3.329646906366375, + "tokens_seen": 3241869312 + }, + { + "epoch": 0.96, + "learning_rate": 1.773391108971273e-05, + "loss": 1.1839, + "theoretical_loss": 3.3296368053941094, + "tokens_seen": 3242000384 + }, + { + "epoch": 0.96, + "learning_rate": 1.769378911892152e-05, + "loss": 1.2031, + "theoretical_loss": 3.3296267049445514, + "tokens_seen": 3242131456 + }, + { + "epoch": 0.97, + "learning_rate": 1.7653667148130314e-05, + "loss": 1.1773, + "theoretical_loss": 3.3296166050176526, + "tokens_seen": 3242262528 + }, + { + "epoch": 0.97, + "learning_rate": 1.761354517733911e-05, + "loss": 1.207, + "theoretical_loss": 3.329606505613365, + "tokens_seen": 3242393600 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.9925158023834229, + "objective/train/avg_token_score": 0.007484156172722578, + "objective/train/docs_used": 1177712, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2684705257415771, + "objective/train/original_loss": 1.2790627479553223, + "objective/train/theoretical_loss": 3.329596406731641, + "objective/train/tokens_used": 1612983776, + "objective/train/unlikelihood_loss": 0.0210325475782156, + "theoretical_loss": 3.329596406731641, + "tokens_seen": 3242524672 + }, + { + "epoch": 0.97, + "learning_rate": 1.7573423206547907e-05, + "loss": 1.2335, + "theoretical_loss": 3.329596406731641, + "tokens_seen": 3242524672 + }, + { + "epoch": 0.97, + "learning_rate": 1.75333012357567e-05, + "loss": 1.2656, + "theoretical_loss": 3.329586308372431, + "tokens_seen": 3242655744 + }, + { + "epoch": 0.97, + "learning_rate": 1.7493179264965495e-05, + "loss": 1.331, + "theoretical_loss": 3.329576210535688, + "tokens_seen": 3242786816 + }, + { + "epoch": 0.97, + "learning_rate": 1.745305729417429e-05, + "loss": 1.2208, + "theoretical_loss": 3.3295661132213636, + "tokens_seen": 3242917888 + }, + { + "epoch": 0.97, + "learning_rate": 1.7412935323383084e-05, + "loss": 1.2082, + "theoretical_loss": 3.32955601642941, + "tokens_seen": 3243048960 + }, + { + "epoch": 0.97, + "learning_rate": 1.737281335259188e-05, + "loss": 1.3357, + "theoretical_loss": 3.329545920159778, + "tokens_seen": 3243180032 + }, + { + "epoch": 0.97, + "learning_rate": 1.7332691381800677e-05, + "loss": 1.2169, + "theoretical_loss": 3.329535824412421, + "tokens_seen": 3243311104 + }, + { + "epoch": 0.97, + "learning_rate": 1.729256941100947e-05, + "loss": 1.2183, + "theoretical_loss": 3.3295257291872895, + "tokens_seen": 3243442176 + }, + { + "epoch": 0.97, + "learning_rate": 1.7252447440218265e-05, + "loss": 1.3086, + "theoretical_loss": 3.329515634484336, + "tokens_seen": 3243573248 + }, + { + "epoch": 0.97, + "learning_rate": 1.7212325469427058e-05, + "loss": 1.19, + "theoretical_loss": 3.3295055403035123, + "tokens_seen": 3243704320 + }, + { + "epoch": 0.97, + "learning_rate": 1.717220349863585e-05, + "loss": 1.2458, + "theoretical_loss": 3.3294954466447706, + "tokens_seen": 3243835392 + }, + { + "epoch": 0.97, + "learning_rate": 1.7132081527844647e-05, + "loss": 1.2387, + "theoretical_loss": 3.329485353508062, + "tokens_seen": 3243966464 + }, + { + "epoch": 0.97, + "learning_rate": 1.7091959557053443e-05, + "loss": 1.1873, + "theoretical_loss": 3.3294752608933393, + "tokens_seen": 3244097536 + }, + { + "epoch": 0.97, + "learning_rate": 1.7051837586262236e-05, + "loss": 1.2232, + "theoretical_loss": 3.329465168800554, + "tokens_seen": 3244228608 + }, + { + "epoch": 0.97, + "learning_rate": 1.7011715615471032e-05, + "loss": 1.2166, + "theoretical_loss": 3.329455077229658, + "tokens_seen": 3244359680 + }, + { + "epoch": 0.97, + "learning_rate": 1.697159364467983e-05, + "loss": 1.2044, + "theoretical_loss": 3.3294449861806035, + "tokens_seen": 3244490752 + }, + { + "epoch": 0.97, + "learning_rate": 1.693147167388862e-05, + "loss": 1.2156, + "theoretical_loss": 3.329434895653342, + "tokens_seen": 3244621824 + }, + { + "epoch": 0.97, + "learning_rate": 1.6891349703097417e-05, + "loss": 1.2661, + "theoretical_loss": 3.3294248056478257, + "tokens_seen": 3244752896 + }, + { + "epoch": 0.97, + "learning_rate": 1.6851227732306213e-05, + "loss": 1.2255, + "theoretical_loss": 3.3294147161640066, + "tokens_seen": 3244883968 + }, + { + "epoch": 0.97, + "learning_rate": 1.6811105761515006e-05, + "loss": 1.274, + "theoretical_loss": 3.3294046272018365, + "tokens_seen": 3245015040 + }, + { + "epoch": 0.97, + "learning_rate": 1.67709837907238e-05, + "loss": 1.2681, + "theoretical_loss": 3.3293945387612673, + "tokens_seen": 3245146112 + }, + { + "epoch": 0.97, + "learning_rate": 1.6730861819932595e-05, + "loss": 1.2682, + "theoretical_loss": 3.3293844508422517, + "tokens_seen": 3245277184 + }, + { + "epoch": 0.97, + "learning_rate": 1.6690739849141388e-05, + "loss": 1.2733, + "theoretical_loss": 3.3293743634447406, + "tokens_seen": 3245408256 + }, + { + "epoch": 0.97, + "learning_rate": 1.6650617878350184e-05, + "loss": 1.2172, + "theoretical_loss": 3.3293642765686866, + "tokens_seen": 3245539328 + }, + { + "epoch": 0.97, + "learning_rate": 1.661049590755898e-05, + "loss": 1.2009, + "theoretical_loss": 3.3293541902140413, + "tokens_seen": 3245670400 + }, + { + "epoch": 0.97, + "learning_rate": 1.6570373936767773e-05, + "loss": 1.2499, + "theoretical_loss": 3.3293441043807572, + "tokens_seen": 3245801472 + }, + { + "epoch": 0.97, + "learning_rate": 1.653025196597657e-05, + "loss": 1.1946, + "theoretical_loss": 3.329334019068786, + "tokens_seen": 3245932544 + }, + { + "epoch": 0.97, + "learning_rate": 1.6490129995185365e-05, + "loss": 1.2851, + "theoretical_loss": 3.32932393427808, + "tokens_seen": 3246063616 + }, + { + "epoch": 0.97, + "learning_rate": 1.6450008024394158e-05, + "loss": 1.1805, + "theoretical_loss": 3.3293138500085906, + "tokens_seen": 3246194688 + }, + { + "epoch": 0.97, + "learning_rate": 1.6409886053602954e-05, + "loss": 1.1769, + "theoretical_loss": 3.3293037662602702, + "tokens_seen": 3246325760 + }, + { + "epoch": 0.97, + "learning_rate": 1.636976408281175e-05, + "loss": 1.2512, + "theoretical_loss": 3.329293683033071, + "tokens_seen": 3246456832 + }, + { + "epoch": 0.97, + "learning_rate": 1.6329642112020543e-05, + "loss": 1.2219, + "theoretical_loss": 3.3292836003269444, + "tokens_seen": 3246587904 + }, + { + "epoch": 0.97, + "learning_rate": 1.6289520141229336e-05, + "loss": 1.2002, + "theoretical_loss": 3.3292735181418434, + "tokens_seen": 3246718976 + }, + { + "epoch": 0.97, + "learning_rate": 1.6249398170438132e-05, + "loss": 1.2558, + "theoretical_loss": 3.329263436477719, + "tokens_seen": 3246850048 + }, + { + "epoch": 0.97, + "learning_rate": 1.6209276199646925e-05, + "loss": 1.292, + "theoretical_loss": 3.329253355334524, + "tokens_seen": 3246981120 + }, + { + "epoch": 0.97, + "learning_rate": 1.616915422885572e-05, + "loss": 1.2269, + "theoretical_loss": 3.3292432747122103, + "tokens_seen": 3247112192 + }, + { + "epoch": 0.97, + "learning_rate": 1.6129032258064517e-05, + "loss": 1.3375, + "theoretical_loss": 3.3292331946107296, + "tokens_seen": 3247243264 + }, + { + "epoch": 0.97, + "learning_rate": 1.608891028727331e-05, + "loss": 1.2718, + "theoretical_loss": 3.3292231150300347, + "tokens_seen": 3247374336 + }, + { + "epoch": 0.97, + "learning_rate": 1.6048788316482106e-05, + "loss": 1.2454, + "theoretical_loss": 3.329213035970077, + "tokens_seen": 3247505408 + }, + { + "epoch": 0.97, + "learning_rate": 1.6008666345690902e-05, + "loss": 1.2226, + "theoretical_loss": 3.329202957430809, + "tokens_seen": 3247636480 + }, + { + "epoch": 0.97, + "learning_rate": 1.5968544374899695e-05, + "loss": 1.3171, + "theoretical_loss": 3.329192879412182, + "tokens_seen": 3247767552 + }, + { + "epoch": 0.97, + "learning_rate": 1.592842240410849e-05, + "loss": 1.21, + "theoretical_loss": 3.3291828019141496, + "tokens_seen": 3247898624 + }, + { + "epoch": 0.97, + "learning_rate": 1.5888300433317287e-05, + "loss": 1.2543, + "theoretical_loss": 3.329172724936662, + "tokens_seen": 3248029696 + }, + { + "epoch": 0.97, + "learning_rate": 1.584817846252608e-05, + "loss": 1.2599, + "theoretical_loss": 3.329162648479673, + "tokens_seen": 3248160768 + }, + { + "epoch": 0.97, + "learning_rate": 1.5808056491734872e-05, + "loss": 1.2707, + "theoretical_loss": 3.3291525725431335, + "tokens_seen": 3248291840 + }, + { + "epoch": 0.97, + "learning_rate": 1.576793452094367e-05, + "loss": 1.2754, + "theoretical_loss": 3.3291424971269965, + "tokens_seen": 3248422912 + }, + { + "epoch": 0.97, + "learning_rate": 1.572781255015246e-05, + "loss": 1.2326, + "theoretical_loss": 3.329132422231214, + "tokens_seen": 3248553984 + }, + { + "epoch": 0.97, + "learning_rate": 1.5687690579361257e-05, + "loss": 1.2652, + "theoretical_loss": 3.3291223478557375, + "tokens_seen": 3248685056 + }, + { + "epoch": 0.97, + "learning_rate": 1.5647568608570054e-05, + "loss": 1.2078, + "theoretical_loss": 3.3291122740005195, + "tokens_seen": 3248816128 + }, + { + "epoch": 0.97, + "learning_rate": 1.5607446637778846e-05, + "loss": 1.2686, + "theoretical_loss": 3.329102200665512, + "tokens_seen": 3248947200 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.9940483570098877, + "objective/train/avg_token_score": 0.005951669067144394, + "objective/train/docs_used": 1180177, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.206506371498108, + "objective/train/original_loss": 1.2146599292755127, + "objective/train/theoretical_loss": 3.3290921278506675, + "objective/train/tokens_used": 1619537376, + "objective/train/unlikelihood_loss": 0.017567142844200134, + "theoretical_loss": 3.3290921278506675, + "tokens_seen": 3249078272 + }, + { + "epoch": 0.97, + "learning_rate": 1.5567324666987642e-05, + "loss": 1.2754, + "theoretical_loss": 3.3290921278506675, + "tokens_seen": 3249078272 + }, + { + "epoch": 0.97, + "learning_rate": 1.552720269619644e-05, + "loss": 1.2822, + "theoretical_loss": 3.329082055555938, + "tokens_seen": 3249209344 + }, + { + "epoch": 0.97, + "learning_rate": 1.548708072540523e-05, + "loss": 1.2466, + "theoretical_loss": 3.329071983781276, + "tokens_seen": 3249340416 + }, + { + "epoch": 0.97, + "learning_rate": 1.5446958754614027e-05, + "loss": 1.2852, + "theoretical_loss": 3.3290619125266327, + "tokens_seen": 3249471488 + }, + { + "epoch": 0.97, + "learning_rate": 1.540683678382282e-05, + "loss": 1.2222, + "theoretical_loss": 3.3290518417919612, + "tokens_seen": 3249602560 + }, + { + "epoch": 0.97, + "learning_rate": 1.5366714813031616e-05, + "loss": 1.2795, + "theoretical_loss": 3.3290417715772134, + "tokens_seen": 3249733632 + }, + { + "epoch": 0.97, + "learning_rate": 1.5326592842240413e-05, + "loss": 1.1824, + "theoretical_loss": 3.329031701882341, + "tokens_seen": 3249864704 + }, + { + "epoch": 0.97, + "learning_rate": 1.5286470871449205e-05, + "loss": 1.2944, + "theoretical_loss": 3.3290216327072972, + "tokens_seen": 3249995776 + }, + { + "epoch": 0.97, + "learning_rate": 1.5246348900658001e-05, + "loss": 1.2475, + "theoretical_loss": 3.329011564052034, + "tokens_seen": 3250126848 + }, + { + "epoch": 0.97, + "learning_rate": 1.5206226929866794e-05, + "loss": 1.2331, + "theoretical_loss": 3.3290014959165024, + "tokens_seen": 3250257920 + }, + { + "epoch": 0.97, + "learning_rate": 1.516610495907559e-05, + "loss": 1.2353, + "theoretical_loss": 3.3289914283006556, + "tokens_seen": 3250388992 + }, + { + "epoch": 0.97, + "learning_rate": 1.5125982988284385e-05, + "loss": 1.1302, + "theoretical_loss": 3.3289813612044465, + "tokens_seen": 3250520064 + }, + { + "epoch": 0.97, + "learning_rate": 1.508586101749318e-05, + "loss": 1.2314, + "theoretical_loss": 3.328971294627826, + "tokens_seen": 3250651136 + }, + { + "epoch": 0.97, + "learning_rate": 1.5045739046701975e-05, + "loss": 1.2918, + "theoretical_loss": 3.328961228570747, + "tokens_seen": 3250782208 + }, + { + "epoch": 0.97, + "learning_rate": 1.500561707591077e-05, + "loss": 1.2792, + "theoretical_loss": 3.328951163033161, + "tokens_seen": 3250913280 + }, + { + "epoch": 0.97, + "learning_rate": 1.4965495105119563e-05, + "loss": 1.2934, + "theoretical_loss": 3.3289410980150214, + "tokens_seen": 3251044352 + }, + { + "epoch": 0.97, + "learning_rate": 1.4925373134328359e-05, + "loss": 1.2495, + "theoretical_loss": 3.32893103351628, + "tokens_seen": 3251175424 + }, + { + "epoch": 0.97, + "learning_rate": 1.4885251163537153e-05, + "loss": 1.2217, + "theoretical_loss": 3.3289209695368887, + "tokens_seen": 3251306496 + }, + { + "epoch": 0.97, + "learning_rate": 1.4845129192745948e-05, + "loss": 1.2444, + "theoretical_loss": 3.3289109060768, + "tokens_seen": 3251437568 + }, + { + "epoch": 0.97, + "learning_rate": 1.4805007221954744e-05, + "loss": 1.2003, + "theoretical_loss": 3.3289008431359663, + "tokens_seen": 3251568640 + }, + { + "epoch": 0.97, + "learning_rate": 1.4764885251163536e-05, + "loss": 1.2138, + "theoretical_loss": 3.3288907807143397, + "tokens_seen": 3251699712 + }, + { + "epoch": 0.97, + "learning_rate": 1.4724763280372331e-05, + "loss": 1.2267, + "theoretical_loss": 3.3288807188118725, + "tokens_seen": 3251830784 + }, + { + "epoch": 0.97, + "learning_rate": 1.4684641309581127e-05, + "loss": 1.3018, + "theoretical_loss": 3.328870657428517, + "tokens_seen": 3251961856 + }, + { + "epoch": 0.97, + "learning_rate": 1.4644519338789922e-05, + "loss": 1.215, + "theoretical_loss": 3.328860596564226, + "tokens_seen": 3252092928 + }, + { + "epoch": 0.97, + "learning_rate": 1.4604397367998716e-05, + "loss": 1.1713, + "theoretical_loss": 3.328850536218951, + "tokens_seen": 3252224000 + }, + { + "epoch": 0.97, + "learning_rate": 1.4564275397207512e-05, + "loss": 1.2046, + "theoretical_loss": 3.328840476392645, + "tokens_seen": 3252355072 + }, + { + "epoch": 0.97, + "learning_rate": 1.4524153426416305e-05, + "loss": 1.321, + "theoretical_loss": 3.3288304170852596, + "tokens_seen": 3252486144 + }, + { + "epoch": 0.97, + "learning_rate": 1.44840314556251e-05, + "loss": 1.2438, + "theoretical_loss": 3.3288203582967473, + "tokens_seen": 3252617216 + }, + { + "epoch": 0.97, + "learning_rate": 1.4443909484833895e-05, + "loss": 1.3082, + "theoretical_loss": 3.328810300027061, + "tokens_seen": 3252748288 + }, + { + "epoch": 0.97, + "learning_rate": 1.440378751404269e-05, + "loss": 1.1691, + "theoretical_loss": 3.3288002422761527, + "tokens_seen": 3252879360 + }, + { + "epoch": 0.97, + "learning_rate": 1.4363665543251484e-05, + "loss": 1.2343, + "theoretical_loss": 3.3287901850439745, + "tokens_seen": 3253010432 + }, + { + "epoch": 0.97, + "learning_rate": 1.432354357246028e-05, + "loss": 1.2841, + "theoretical_loss": 3.3287801283304788, + "tokens_seen": 3253141504 + }, + { + "epoch": 0.97, + "learning_rate": 1.4283421601669073e-05, + "loss": 1.2621, + "theoretical_loss": 3.3287700721356184, + "tokens_seen": 3253272576 + }, + { + "epoch": 0.97, + "learning_rate": 1.4243299630877868e-05, + "loss": 1.2501, + "theoretical_loss": 3.3287600164593454, + "tokens_seen": 3253403648 + }, + { + "epoch": 0.97, + "learning_rate": 1.4203177660086664e-05, + "loss": 1.2099, + "theoretical_loss": 3.328749961301612, + "tokens_seen": 3253534720 + }, + { + "epoch": 0.97, + "learning_rate": 1.4163055689295458e-05, + "loss": 1.3058, + "theoretical_loss": 3.3287399066623706, + "tokens_seen": 3253665792 + }, + { + "epoch": 0.97, + "learning_rate": 1.4122933718504254e-05, + "loss": 1.2275, + "theoretical_loss": 3.328729852541574, + "tokens_seen": 3253796864 + }, + { + "epoch": 0.97, + "learning_rate": 1.4082811747713049e-05, + "loss": 1.1944, + "theoretical_loss": 3.328719798939174, + "tokens_seen": 3253927936 + }, + { + "epoch": 0.97, + "learning_rate": 1.4042689776921842e-05, + "loss": 1.2025, + "theoretical_loss": 3.328709745855123, + "tokens_seen": 3254059008 + }, + { + "epoch": 0.97, + "learning_rate": 1.4002567806130638e-05, + "loss": 1.3347, + "theoretical_loss": 3.3286996932893738, + "tokens_seen": 3254190080 + }, + { + "epoch": 0.97, + "learning_rate": 1.3962445835339432e-05, + "loss": 1.3152, + "theoretical_loss": 3.328689641241879, + "tokens_seen": 3254321152 + }, + { + "epoch": 0.97, + "learning_rate": 1.3922323864548227e-05, + "loss": 1.2551, + "theoretical_loss": 3.3286795897125905, + "tokens_seen": 3254452224 + }, + { + "epoch": 0.97, + "learning_rate": 1.3882201893757023e-05, + "loss": 1.2783, + "theoretical_loss": 3.328669538701461, + "tokens_seen": 3254583296 + }, + { + "epoch": 0.97, + "learning_rate": 1.3842079922965817e-05, + "loss": 1.2366, + "theoretical_loss": 3.3286594882084426, + "tokens_seen": 3254714368 + }, + { + "epoch": 0.97, + "learning_rate": 1.380195795217461e-05, + "loss": 1.2546, + "theoretical_loss": 3.3286494382334877, + "tokens_seen": 3254845440 + }, + { + "epoch": 0.97, + "learning_rate": 1.3761835981383406e-05, + "loss": 1.3184, + "theoretical_loss": 3.3286393887765495, + "tokens_seen": 3254976512 + }, + { + "epoch": 0.97, + "learning_rate": 1.37217140105922e-05, + "loss": 1.2041, + "theoretical_loss": 3.3286293398375792, + "tokens_seen": 3255107584 + }, + { + "epoch": 0.97, + "learning_rate": 1.3681592039800995e-05, + "loss": 1.2805, + "theoretical_loss": 3.3286192914165307, + "tokens_seen": 3255238656 + }, + { + "epoch": 0.97, + "learning_rate": 1.3641470069009791e-05, + "loss": 1.2125, + "theoretical_loss": 3.3286092435133554, + "tokens_seen": 3255369728 + }, + { + "epoch": 0.97, + "learning_rate": 1.3601348098218586e-05, + "loss": 1.218, + "theoretical_loss": 3.328599196128006, + "tokens_seen": 3255500800 + }, + { + "epoch": 0.97, + "objective/train/aligned_token_count": 0.9918806552886963, + "objective/train/avg_token_score": 0.00811933632940054, + "objective/train/docs_used": 1182603, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2924766540527344, + "objective/train/original_loss": 1.307032585144043, + "objective/train/theoretical_loss": 3.328589149260435, + "objective/train/tokens_used": 1626090976, + "objective/train/unlikelihood_loss": 0.01887030154466629, + "theoretical_loss": 3.328589149260435, + "tokens_seen": 3255631872 + }, + { + "epoch": 0.97, + "learning_rate": 1.3561226127427378e-05, + "loss": 1.2598, + "theoretical_loss": 3.328589149260435, + "tokens_seen": 3255631872 + }, + { + "epoch": 0.97, + "learning_rate": 1.3521104156636175e-05, + "loss": 1.2407, + "theoretical_loss": 3.328579102910595, + "tokens_seen": 3255762944 + }, + { + "epoch": 0.97, + "learning_rate": 1.3480982185844969e-05, + "loss": 1.1228, + "theoretical_loss": 3.3285690570784383, + "tokens_seen": 3255894016 + }, + { + "epoch": 0.97, + "learning_rate": 1.3440860215053763e-05, + "loss": 1.2551, + "theoretical_loss": 3.3285590117639177, + "tokens_seen": 3256025088 + }, + { + "epoch": 0.97, + "learning_rate": 1.340073824426256e-05, + "loss": 1.328, + "theoretical_loss": 3.328548966966985, + "tokens_seen": 3256156160 + }, + { + "epoch": 0.97, + "learning_rate": 1.3360616273471354e-05, + "loss": 1.2034, + "theoretical_loss": 3.328538922687594, + "tokens_seen": 3256287232 + }, + { + "epoch": 0.97, + "learning_rate": 1.3320494302680147e-05, + "loss": 1.2468, + "theoretical_loss": 3.3285288789256957, + "tokens_seen": 3256418304 + }, + { + "epoch": 0.97, + "learning_rate": 1.3280372331888943e-05, + "loss": 1.2801, + "theoretical_loss": 3.328518835681243, + "tokens_seen": 3256549376 + }, + { + "epoch": 0.97, + "learning_rate": 1.3240250361097737e-05, + "loss": 1.2867, + "theoretical_loss": 3.328508792954189, + "tokens_seen": 3256680448 + }, + { + "epoch": 0.97, + "learning_rate": 1.3200128390306532e-05, + "loss": 1.3335, + "theoretical_loss": 3.3284987507444863, + "tokens_seen": 3256811520 + }, + { + "epoch": 0.97, + "learning_rate": 1.3160006419515328e-05, + "loss": 1.2838, + "theoretical_loss": 3.328488709052087, + "tokens_seen": 3256942592 + }, + { + "epoch": 0.97, + "learning_rate": 1.3119884448724122e-05, + "loss": 1.2009, + "theoretical_loss": 3.328478667876943, + "tokens_seen": 3257073664 + }, + { + "epoch": 0.97, + "learning_rate": 1.3079762477932915e-05, + "loss": 1.2195, + "theoretical_loss": 3.3284686272190083, + "tokens_seen": 3257204736 + }, + { + "epoch": 0.97, + "learning_rate": 1.3039640507141711e-05, + "loss": 1.2702, + "theoretical_loss": 3.3284585870782344, + "tokens_seen": 3257335808 + }, + { + "epoch": 0.97, + "learning_rate": 1.2999518536350506e-05, + "loss": 1.2289, + "theoretical_loss": 3.328448547454574, + "tokens_seen": 3257466880 + }, + { + "epoch": 0.97, + "learning_rate": 1.29593965655593e-05, + "loss": 1.2665, + "theoretical_loss": 3.32843850834798, + "tokens_seen": 3257597952 + }, + { + "epoch": 0.97, + "learning_rate": 1.2919274594768096e-05, + "loss": 1.2467, + "theoretical_loss": 3.3284284697584043, + "tokens_seen": 3257729024 + }, + { + "epoch": 0.97, + "learning_rate": 1.287915262397689e-05, + "loss": 1.2375, + "theoretical_loss": 3.3284184316858, + "tokens_seen": 3257860096 + }, + { + "epoch": 0.97, + "learning_rate": 1.2839030653185684e-05, + "loss": 1.2578, + "theoretical_loss": 3.32840839413012, + "tokens_seen": 3257991168 + }, + { + "epoch": 0.97, + "learning_rate": 1.279890868239448e-05, + "loss": 1.2669, + "theoretical_loss": 3.3283983570913165, + "tokens_seen": 3258122240 + }, + { + "epoch": 0.97, + "learning_rate": 1.2758786711603274e-05, + "loss": 1.2185, + "theoretical_loss": 3.328388320569342, + "tokens_seen": 3258253312 + }, + { + "epoch": 0.97, + "learning_rate": 1.2718664740812069e-05, + "loss": 1.2283, + "theoretical_loss": 3.328378284564149, + "tokens_seen": 3258384384 + }, + { + "epoch": 0.97, + "learning_rate": 1.2678542770020865e-05, + "loss": 1.2093, + "theoretical_loss": 3.32836824907569, + "tokens_seen": 3258515456 + }, + { + "epoch": 0.97, + "learning_rate": 1.2638420799229657e-05, + "loss": 1.1966, + "theoretical_loss": 3.3283582141039187, + "tokens_seen": 3258646528 + }, + { + "epoch": 0.98, + "learning_rate": 1.2598298828438452e-05, + "loss": 1.1485, + "theoretical_loss": 3.3283481796487866, + "tokens_seen": 3258777600 + }, + { + "epoch": 0.98, + "learning_rate": 1.2558176857647248e-05, + "loss": 1.1922, + "theoretical_loss": 3.328338145710246, + "tokens_seen": 3258908672 + }, + { + "epoch": 0.98, + "learning_rate": 1.2518054886856043e-05, + "loss": 1.1948, + "theoretical_loss": 3.328328112288251, + "tokens_seen": 3259039744 + }, + { + "epoch": 0.98, + "learning_rate": 1.2477932916064837e-05, + "loss": 1.2041, + "theoretical_loss": 3.328318079382753, + "tokens_seen": 3259170816 + }, + { + "epoch": 0.98, + "learning_rate": 1.2437810945273633e-05, + "loss": 1.2235, + "theoretical_loss": 3.328308046993705, + "tokens_seen": 3259301888 + }, + { + "epoch": 0.98, + "learning_rate": 1.2397688974482426e-05, + "loss": 1.2524, + "theoretical_loss": 3.32829801512106, + "tokens_seen": 3259432960 + }, + { + "epoch": 0.98, + "learning_rate": 1.235756700369122e-05, + "loss": 1.1725, + "theoretical_loss": 3.32828798376477, + "tokens_seen": 3259564032 + }, + { + "epoch": 0.98, + "learning_rate": 1.2317445032900016e-05, + "loss": 1.2978, + "theoretical_loss": 3.3282779529247883, + "tokens_seen": 3259695104 + }, + { + "epoch": 0.98, + "learning_rate": 1.2277323062108811e-05, + "loss": 1.3068, + "theoretical_loss": 3.328267922601067, + "tokens_seen": 3259826176 + }, + { + "epoch": 0.98, + "learning_rate": 1.2237201091317605e-05, + "loss": 1.1817, + "theoretical_loss": 3.3282578927935593, + "tokens_seen": 3259957248 + }, + { + "epoch": 0.98, + "learning_rate": 1.2197079120526401e-05, + "loss": 1.2217, + "theoretical_loss": 3.3282478635022175, + "tokens_seen": 3260088320 + }, + { + "epoch": 0.98, + "learning_rate": 1.2156957149735194e-05, + "loss": 1.1634, + "theoretical_loss": 3.3282378347269943, + "tokens_seen": 3260219392 + }, + { + "epoch": 0.98, + "learning_rate": 1.2116835178943989e-05, + "loss": 1.2025, + "theoretical_loss": 3.3282278064678428, + "tokens_seen": 3260350464 + }, + { + "epoch": 0.98, + "learning_rate": 1.2076713208152785e-05, + "loss": 1.1253, + "theoretical_loss": 3.328217778724715, + "tokens_seen": 3260481536 + }, + { + "epoch": 0.98, + "learning_rate": 1.203659123736158e-05, + "loss": 1.1869, + "theoretical_loss": 3.3282077514975645, + "tokens_seen": 3260612608 + }, + { + "epoch": 0.98, + "learning_rate": 1.1996469266570375e-05, + "loss": 1.235, + "theoretical_loss": 3.3281977247863432, + "tokens_seen": 3260743680 + }, + { + "epoch": 0.98, + "learning_rate": 1.195634729577917e-05, + "loss": 1.181, + "theoretical_loss": 3.328187698591004, + "tokens_seen": 3260874752 + }, + { + "epoch": 0.98, + "learning_rate": 1.1916225324987963e-05, + "loss": 1.1676, + "theoretical_loss": 3.3281776729115, + "tokens_seen": 3261005824 + }, + { + "epoch": 0.98, + "learning_rate": 1.1876103354196759e-05, + "loss": 1.2625, + "theoretical_loss": 3.3281676477477835, + "tokens_seen": 3261136896 + }, + { + "epoch": 0.98, + "learning_rate": 1.1835981383405553e-05, + "loss": 1.1781, + "theoretical_loss": 3.3281576230998073, + "tokens_seen": 3261267968 + }, + { + "epoch": 0.98, + "learning_rate": 1.1795859412614348e-05, + "loss": 1.176, + "theoretical_loss": 3.328147598967525, + "tokens_seen": 3261399040 + }, + { + "epoch": 0.98, + "learning_rate": 1.1755737441823144e-05, + "loss": 1.2175, + "theoretical_loss": 3.3281375753508877, + "tokens_seen": 3261530112 + }, + { + "epoch": 0.98, + "learning_rate": 1.1715615471031938e-05, + "loss": 1.2107, + "theoretical_loss": 3.3281275522498497, + "tokens_seen": 3261661184 + }, + { + "epoch": 0.98, + "learning_rate": 1.1675493500240731e-05, + "loss": 1.2138, + "theoretical_loss": 3.328117529664363, + "tokens_seen": 3261792256 + }, + { + "epoch": 0.98, + "learning_rate": 1.1635371529449527e-05, + "loss": 1.2351, + "theoretical_loss": 3.32810750759438, + "tokens_seen": 3261923328 + }, + { + "epoch": 0.98, + "learning_rate": 1.1595249558658322e-05, + "loss": 1.2007, + "theoretical_loss": 3.3280974860398542, + "tokens_seen": 3262054400 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.992702305316925, + "objective/train/avg_token_score": 0.007297667674720287, + "objective/train/docs_used": 1184948, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.3182915449142456, + "objective/train/original_loss": 1.3293194770812988, + "objective/train/theoretical_loss": 3.3280874650007384, + "objective/train/tokens_used": 1632644576, + "objective/train/unlikelihood_loss": 0.02034706436097622, + "theoretical_loss": 3.3280874650007384, + "tokens_seen": 3262185472 + }, + { + "epoch": 0.98, + "learning_rate": 1.1555127587867116e-05, + "loss": 1.2897, + "theoretical_loss": 3.3280874650007384, + "tokens_seen": 3262185472 + }, + { + "epoch": 0.98, + "learning_rate": 1.1515005617075912e-05, + "loss": 1.2852, + "theoretical_loss": 3.328077444476985, + "tokens_seen": 3262316544 + }, + { + "epoch": 0.98, + "learning_rate": 1.1474883646284707e-05, + "loss": 1.257, + "theoretical_loss": 3.328067424468547, + "tokens_seen": 3262447616 + }, + { + "epoch": 0.98, + "learning_rate": 1.14347616754935e-05, + "loss": 1.2318, + "theoretical_loss": 3.328057404975377, + "tokens_seen": 3262578688 + }, + { + "epoch": 0.98, + "learning_rate": 1.1394639704702296e-05, + "loss": 1.2, + "theoretical_loss": 3.3280473859974276, + "tokens_seen": 3262709760 + }, + { + "epoch": 0.98, + "learning_rate": 1.135451773391109e-05, + "loss": 1.3209, + "theoretical_loss": 3.3280373675346526, + "tokens_seen": 3262840832 + }, + { + "epoch": 0.98, + "learning_rate": 1.1314395763119884e-05, + "loss": 1.288, + "theoretical_loss": 3.328027349587004, + "tokens_seen": 3262971904 + }, + { + "epoch": 0.98, + "learning_rate": 1.127427379232868e-05, + "loss": 1.1199, + "theoretical_loss": 3.328017332154434, + "tokens_seen": 3263102976 + }, + { + "epoch": 0.98, + "learning_rate": 1.1234151821537475e-05, + "loss": 1.1965, + "theoretical_loss": 3.328007315236897, + "tokens_seen": 3263234048 + }, + { + "epoch": 0.98, + "learning_rate": 1.1194029850746268e-05, + "loss": 1.2232, + "theoretical_loss": 3.3279972988343447, + "tokens_seen": 3263365120 + }, + { + "epoch": 0.98, + "learning_rate": 1.1153907879955064e-05, + "loss": 1.1746, + "theoretical_loss": 3.3279872829467307, + "tokens_seen": 3263496192 + }, + { + "epoch": 0.98, + "learning_rate": 1.1113785909163858e-05, + "loss": 1.2316, + "theoretical_loss": 3.327977267574007, + "tokens_seen": 3263627264 + }, + { + "epoch": 0.98, + "learning_rate": 1.1073663938372653e-05, + "loss": 1.2273, + "theoretical_loss": 3.327967252716127, + "tokens_seen": 3263758336 + }, + { + "epoch": 0.98, + "learning_rate": 1.1033541967581449e-05, + "loss": 1.2355, + "theoretical_loss": 3.3279572383730436, + "tokens_seen": 3263889408 + }, + { + "epoch": 0.98, + "learning_rate": 1.0993419996790243e-05, + "loss": 1.2451, + "theoretical_loss": 3.327947224544709, + "tokens_seen": 3264020480 + }, + { + "epoch": 0.98, + "learning_rate": 1.0953298025999036e-05, + "loss": 1.246, + "theoretical_loss": 3.327937211231077, + "tokens_seen": 3264151552 + }, + { + "epoch": 0.98, + "learning_rate": 1.0913176055207832e-05, + "loss": 1.2348, + "theoretical_loss": 3.3279271984321, + "tokens_seen": 3264282624 + }, + { + "epoch": 0.98, + "learning_rate": 1.0873054084416627e-05, + "loss": 1.2538, + "theoretical_loss": 3.327917186147731, + "tokens_seen": 3264413696 + }, + { + "epoch": 0.98, + "learning_rate": 1.0832932113625421e-05, + "loss": 1.2589, + "theoretical_loss": 3.3279071743779225, + "tokens_seen": 3264544768 + }, + { + "epoch": 0.98, + "learning_rate": 1.0792810142834217e-05, + "loss": 1.2362, + "theoretical_loss": 3.327897163122628, + "tokens_seen": 3264675840 + }, + { + "epoch": 0.98, + "learning_rate": 1.0752688172043012e-05, + "loss": 1.2777, + "theoretical_loss": 3.3278871523818, + "tokens_seen": 3264806912 + }, + { + "epoch": 0.98, + "learning_rate": 1.0712566201251805e-05, + "loss": 1.2679, + "theoretical_loss": 3.3278771421553914, + "tokens_seen": 3264937984 + }, + { + "epoch": 0.98, + "learning_rate": 1.06724442304606e-05, + "loss": 1.199, + "theoretical_loss": 3.3278671324433557, + "tokens_seen": 3265069056 + }, + { + "epoch": 0.98, + "learning_rate": 1.0632322259669395e-05, + "loss": 1.2055, + "theoretical_loss": 3.327857123245645, + "tokens_seen": 3265200128 + }, + { + "epoch": 0.98, + "learning_rate": 1.059220028887819e-05, + "loss": 1.2435, + "theoretical_loss": 3.327847114562213, + "tokens_seen": 3265331200 + }, + { + "epoch": 0.98, + "learning_rate": 1.0552078318086986e-05, + "loss": 1.2826, + "theoretical_loss": 3.3278371063930114, + "tokens_seen": 3265462272 + }, + { + "epoch": 0.98, + "learning_rate": 1.0511956347295778e-05, + "loss": 1.2853, + "theoretical_loss": 3.3278270987379948, + "tokens_seen": 3265593344 + }, + { + "epoch": 0.98, + "learning_rate": 1.0471834376504573e-05, + "loss": 1.2085, + "theoretical_loss": 3.3278170915971153, + "tokens_seen": 3265724416 + }, + { + "epoch": 0.98, + "learning_rate": 1.0431712405713369e-05, + "loss": 1.1691, + "theoretical_loss": 3.3278070849703254, + "tokens_seen": 3265855488 + }, + { + "epoch": 0.98, + "learning_rate": 1.0391590434922163e-05, + "loss": 1.2172, + "theoretical_loss": 3.3277970788575786, + "tokens_seen": 3265986560 + }, + { + "epoch": 0.98, + "learning_rate": 1.0351468464130958e-05, + "loss": 1.284, + "theoretical_loss": 3.327787073258828, + "tokens_seen": 3266117632 + }, + { + "epoch": 0.98, + "learning_rate": 1.0311346493339754e-05, + "loss": 1.1536, + "theoretical_loss": 3.3277770681740266, + "tokens_seen": 3266248704 + }, + { + "epoch": 0.98, + "learning_rate": 1.0271224522548547e-05, + "loss": 1.2782, + "theoretical_loss": 3.327767063603127, + "tokens_seen": 3266379776 + }, + { + "epoch": 0.98, + "learning_rate": 1.0231102551757341e-05, + "loss": 1.228, + "theoretical_loss": 3.327757059546082, + "tokens_seen": 3266510848 + }, + { + "epoch": 0.98, + "learning_rate": 1.0190980580966137e-05, + "loss": 1.2147, + "theoretical_loss": 3.327747056002845, + "tokens_seen": 3266641920 + }, + { + "epoch": 0.98, + "learning_rate": 1.0150858610174932e-05, + "loss": 1.2644, + "theoretical_loss": 3.327737052973369, + "tokens_seen": 3266772992 + }, + { + "epoch": 0.98, + "learning_rate": 1.0110736639383726e-05, + "loss": 1.324, + "theoretical_loss": 3.3277270504576073, + "tokens_seen": 3266904064 + }, + { + "epoch": 0.98, + "learning_rate": 1.0070614668592522e-05, + "loss": 1.2416, + "theoretical_loss": 3.3277170484555123, + "tokens_seen": 3267035136 + }, + { + "epoch": 0.98, + "learning_rate": 1.0030492697801315e-05, + "loss": 1.2268, + "theoretical_loss": 3.327707046967037, + "tokens_seen": 3267166208 + }, + { + "epoch": 0.98, + "learning_rate": 9.99037072701011e-06, + "loss": 1.1619, + "theoretical_loss": 3.327697045992135, + "tokens_seen": 3267297280 + }, + { + "epoch": 0.98, + "learning_rate": 9.950248756218906e-06, + "loss": 1.2641, + "theoretical_loss": 3.327687045530759, + "tokens_seen": 3267428352 + }, + { + "epoch": 0.98, + "learning_rate": 9.9101267854277e-06, + "loss": 1.2118, + "theoretical_loss": 3.3276770455828615, + "tokens_seen": 3267559424 + }, + { + "epoch": 0.98, + "learning_rate": 9.870004814636496e-06, + "loss": 1.2375, + "theoretical_loss": 3.3276670461483966, + "tokens_seen": 3267690496 + }, + { + "epoch": 0.98, + "learning_rate": 9.82988284384529e-06, + "loss": 1.2536, + "theoretical_loss": 3.3276570472273166, + "tokens_seen": 3267821568 + }, + { + "epoch": 0.98, + "learning_rate": 9.789760873054084e-06, + "loss": 1.2976, + "theoretical_loss": 3.327647048819575, + "tokens_seen": 3267952640 + }, + { + "epoch": 0.98, + "learning_rate": 9.74963890226288e-06, + "loss": 1.2145, + "theoretical_loss": 3.3276370509251243, + "tokens_seen": 3268083712 + }, + { + "epoch": 0.98, + "learning_rate": 9.709516931471674e-06, + "loss": 1.2408, + "theoretical_loss": 3.327627053543918, + "tokens_seen": 3268214784 + }, + { + "epoch": 0.98, + "learning_rate": 9.669394960680469e-06, + "loss": 1.2516, + "theoretical_loss": 3.3276170566759093, + "tokens_seen": 3268345856 + }, + { + "epoch": 0.98, + "learning_rate": 9.629272989889265e-06, + "loss": 1.2479, + "theoretical_loss": 3.327607060321051, + "tokens_seen": 3268476928 + }, + { + "epoch": 0.98, + "learning_rate": 9.58915101909806e-06, + "loss": 1.1387, + "theoretical_loss": 3.3275970644792956, + "tokens_seen": 3268608000 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.9905927181243896, + "objective/train/avg_token_score": 0.009407276287674904, + "objective/train/docs_used": 1187457, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2274799346923828, + "objective/train/original_loss": 1.241201400756836, + "objective/train/theoretical_loss": 3.327587069150597, + "objective/train/tokens_used": 1639198176, + "objective/train/unlikelihood_loss": 0.025450611487030983, + "theoretical_loss": 3.327587069150597, + "tokens_seen": 3268739072 + }, + { + "epoch": 0.98, + "learning_rate": 9.549029048306852e-06, + "loss": 1.2486, + "theoretical_loss": 3.327587069150597, + "tokens_seen": 3268739072 + }, + { + "epoch": 0.98, + "learning_rate": 9.508907077515648e-06, + "loss": 1.2414, + "theoretical_loss": 3.3275770743349087, + "tokens_seen": 3268870144 + }, + { + "epoch": 0.98, + "learning_rate": 9.468785106724443e-06, + "loss": 1.2784, + "theoretical_loss": 3.327567080032183, + "tokens_seen": 3269001216 + }, + { + "epoch": 0.98, + "learning_rate": 9.428663135933237e-06, + "loss": 1.1675, + "theoretical_loss": 3.327557086242373, + "tokens_seen": 3269132288 + }, + { + "epoch": 0.98, + "learning_rate": 9.388541165142033e-06, + "loss": 1.3873, + "theoretical_loss": 3.327547092965432, + "tokens_seen": 3269263360 + }, + { + "epoch": 0.98, + "learning_rate": 9.348419194350828e-06, + "loss": 1.2376, + "theoretical_loss": 3.327537100201313, + "tokens_seen": 3269394432 + }, + { + "epoch": 0.98, + "learning_rate": 9.30829722355962e-06, + "loss": 1.1957, + "theoretical_loss": 3.3275271079499693, + "tokens_seen": 3269525504 + }, + { + "epoch": 0.98, + "learning_rate": 9.268175252768416e-06, + "loss": 1.2418, + "theoretical_loss": 3.3275171162113546, + "tokens_seen": 3269656576 + }, + { + "epoch": 0.98, + "learning_rate": 9.228053281977211e-06, + "loss": 1.2953, + "theoretical_loss": 3.327507124985421, + "tokens_seen": 3269787648 + }, + { + "epoch": 0.98, + "learning_rate": 9.187931311186005e-06, + "loss": 1.1938, + "theoretical_loss": 3.327497134272122, + "tokens_seen": 3269918720 + }, + { + "epoch": 0.98, + "learning_rate": 9.147809340394802e-06, + "loss": 1.2314, + "theoretical_loss": 3.327487144071411, + "tokens_seen": 3270049792 + }, + { + "epoch": 0.98, + "learning_rate": 9.107687369603596e-06, + "loss": 1.1906, + "theoretical_loss": 3.3274771543832404, + "tokens_seen": 3270180864 + }, + { + "epoch": 0.98, + "learning_rate": 9.067565398812389e-06, + "loss": 1.1751, + "theoretical_loss": 3.327467165207565, + "tokens_seen": 3270311936 + }, + { + "epoch": 0.98, + "learning_rate": 9.027443428021185e-06, + "loss": 1.155, + "theoretical_loss": 3.3274571765443364, + "tokens_seen": 3270443008 + }, + { + "epoch": 0.98, + "learning_rate": 8.98732145722998e-06, + "loss": 1.2212, + "theoretical_loss": 3.327447188393508, + "tokens_seen": 3270574080 + }, + { + "epoch": 0.98, + "learning_rate": 8.947199486438774e-06, + "loss": 1.1698, + "theoretical_loss": 3.3274372007550337, + "tokens_seen": 3270705152 + }, + { + "epoch": 0.98, + "learning_rate": 8.90707751564757e-06, + "loss": 1.2288, + "theoretical_loss": 3.327427213628866, + "tokens_seen": 3270836224 + }, + { + "epoch": 0.98, + "learning_rate": 8.866955544856364e-06, + "loss": 1.2622, + "theoretical_loss": 3.3274172270149585, + "tokens_seen": 3270967296 + }, + { + "epoch": 0.98, + "learning_rate": 8.826833574065157e-06, + "loss": 1.202, + "theoretical_loss": 3.3274072409132645, + "tokens_seen": 3271098368 + }, + { + "epoch": 0.98, + "learning_rate": 8.786711603273953e-06, + "loss": 1.1973, + "theoretical_loss": 3.3273972553237363, + "tokens_seen": 3271229440 + }, + { + "epoch": 0.98, + "learning_rate": 8.746589632482748e-06, + "loss": 1.2327, + "theoretical_loss": 3.3273872702463283, + "tokens_seen": 3271360512 + }, + { + "epoch": 0.98, + "learning_rate": 8.706467661691542e-06, + "loss": 1.2347, + "theoretical_loss": 3.327377285680993, + "tokens_seen": 3271491584 + }, + { + "epoch": 0.98, + "learning_rate": 8.666345690900338e-06, + "loss": 1.2209, + "theoretical_loss": 3.327367301627684, + "tokens_seen": 3271622656 + }, + { + "epoch": 0.98, + "learning_rate": 8.626223720109133e-06, + "loss": 1.2655, + "theoretical_loss": 3.3273573180863543, + "tokens_seen": 3271753728 + }, + { + "epoch": 0.98, + "learning_rate": 8.586101749317925e-06, + "loss": 1.1759, + "theoretical_loss": 3.327347335056957, + "tokens_seen": 3271884800 + }, + { + "epoch": 0.98, + "learning_rate": 8.545979778526722e-06, + "loss": 1.1021, + "theoretical_loss": 3.3273373525394456, + "tokens_seen": 3272015872 + }, + { + "epoch": 0.98, + "learning_rate": 8.505857807735516e-06, + "loss": 1.2447, + "theoretical_loss": 3.327327370533774, + "tokens_seen": 3272146944 + }, + { + "epoch": 0.98, + "learning_rate": 8.46573583694431e-06, + "loss": 1.2025, + "theoretical_loss": 3.3273173890398935, + "tokens_seen": 3272278016 + }, + { + "epoch": 0.98, + "learning_rate": 8.425613866153107e-06, + "loss": 1.3094, + "theoretical_loss": 3.3273074080577594, + "tokens_seen": 3272409088 + }, + { + "epoch": 0.98, + "learning_rate": 8.3854918953619e-06, + "loss": 1.2858, + "theoretical_loss": 3.327297427587324, + "tokens_seen": 3272540160 + }, + { + "epoch": 0.98, + "learning_rate": 8.345369924570694e-06, + "loss": 1.2498, + "theoretical_loss": 3.3272874476285406, + "tokens_seen": 3272671232 + }, + { + "epoch": 0.98, + "learning_rate": 8.30524795377949e-06, + "loss": 1.2738, + "theoretical_loss": 3.3272774681813626, + "tokens_seen": 3272802304 + }, + { + "epoch": 0.98, + "learning_rate": 8.265125982988284e-06, + "loss": 1.2392, + "theoretical_loss": 3.3272674892457434, + "tokens_seen": 3272933376 + }, + { + "epoch": 0.98, + "learning_rate": 8.225004012197079e-06, + "loss": 1.1772, + "theoretical_loss": 3.3272575108216365, + "tokens_seen": 3273064448 + }, + { + "epoch": 0.98, + "learning_rate": 8.184882041405875e-06, + "loss": 1.2026, + "theoretical_loss": 3.3272475329089946, + "tokens_seen": 3273195520 + }, + { + "epoch": 0.98, + "learning_rate": 8.144760070614668e-06, + "loss": 1.24, + "theoretical_loss": 3.327237555507771, + "tokens_seen": 3273326592 + }, + { + "epoch": 0.98, + "learning_rate": 8.104638099823462e-06, + "loss": 1.247, + "theoretical_loss": 3.3272275786179195, + "tokens_seen": 3273457664 + }, + { + "epoch": 0.98, + "learning_rate": 8.064516129032258e-06, + "loss": 1.2796, + "theoretical_loss": 3.3272176022393936, + "tokens_seen": 3273588736 + }, + { + "epoch": 0.98, + "learning_rate": 8.024394158241053e-06, + "loss": 1.1996, + "theoretical_loss": 3.3272076263721457, + "tokens_seen": 3273719808 + }, + { + "epoch": 0.98, + "learning_rate": 7.984272187449847e-06, + "loss": 1.328, + "theoretical_loss": 3.32719765101613, + "tokens_seen": 3273850880 + }, + { + "epoch": 0.98, + "learning_rate": 7.944150216658643e-06, + "loss": 1.2606, + "theoretical_loss": 3.3271876761712993, + "tokens_seen": 3273981952 + }, + { + "epoch": 0.98, + "learning_rate": 7.904028245867436e-06, + "loss": 1.2091, + "theoretical_loss": 3.3271777018376074, + "tokens_seen": 3274113024 + }, + { + "epoch": 0.98, + "learning_rate": 7.86390627507623e-06, + "loss": 1.1662, + "theoretical_loss": 3.3271677280150076, + "tokens_seen": 3274244096 + }, + { + "epoch": 0.98, + "learning_rate": 7.823784304285027e-06, + "loss": 1.3026, + "theoretical_loss": 3.3271577547034523, + "tokens_seen": 3274375168 + }, + { + "epoch": 0.98, + "learning_rate": 7.783662333493821e-06, + "loss": 1.2265, + "theoretical_loss": 3.3271477819028963, + "tokens_seen": 3274506240 + }, + { + "epoch": 0.98, + "learning_rate": 7.743540362702616e-06, + "loss": 1.2419, + "theoretical_loss": 3.327137809613292, + "tokens_seen": 3274637312 + }, + { + "epoch": 0.98, + "learning_rate": 7.70341839191141e-06, + "loss": 1.1786, + "theoretical_loss": 3.3271278378345928, + "tokens_seen": 3274768384 + }, + { + "epoch": 0.98, + "learning_rate": 7.663296421120206e-06, + "loss": 1.2579, + "theoretical_loss": 3.327117866566753, + "tokens_seen": 3274899456 + }, + { + "epoch": 0.98, + "learning_rate": 7.623174450329001e-06, + "loss": 1.2698, + "theoretical_loss": 3.327107895809725, + "tokens_seen": 3275030528 + }, + { + "epoch": 0.98, + "learning_rate": 7.583052479537795e-06, + "loss": 1.3013, + "theoretical_loss": 3.327097925563462, + "tokens_seen": 3275161600 + }, + { + "epoch": 0.98, + "objective/train/aligned_token_count": 0.9902477860450745, + "objective/train/avg_token_score": 0.009752204641699791, + "objective/train/docs_used": 1189805, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2036964893341064, + "objective/train/original_loss": 1.2179144620895386, + "objective/train/theoretical_loss": 3.3270879558279183, + "objective/train/tokens_used": 1645751776, + "objective/train/unlikelihood_loss": 0.026720257475972176, + "theoretical_loss": 3.3270879558279183, + "tokens_seen": 3275292672 + }, + { + "epoch": 0.99, + "learning_rate": 7.54293050874659e-06, + "loss": 1.1881, + "theoretical_loss": 3.3270879558279183, + "tokens_seen": 3275292672 + }, + { + "epoch": 0.99, + "learning_rate": 7.502808537955385e-06, + "loss": 1.2544, + "theoretical_loss": 3.327077986603047, + "tokens_seen": 3275423744 + }, + { + "epoch": 0.99, + "learning_rate": 7.462686567164179e-06, + "loss": 1.2295, + "theoretical_loss": 3.3270680178888012, + "tokens_seen": 3275554816 + }, + { + "epoch": 0.99, + "learning_rate": 7.422564596372974e-06, + "loss": 1.2171, + "theoretical_loss": 3.3270580496851347, + "tokens_seen": 3275685888 + }, + { + "epoch": 0.99, + "learning_rate": 7.382442625581768e-06, + "loss": 1.2404, + "theoretical_loss": 3.3270480819920003, + "tokens_seen": 3275816960 + }, + { + "epoch": 0.99, + "learning_rate": 7.3423206547905635e-06, + "loss": 1.2556, + "theoretical_loss": 3.327038114809352, + "tokens_seen": 3275948032 + }, + { + "epoch": 0.99, + "learning_rate": 7.302198683999358e-06, + "loss": 1.2703, + "theoretical_loss": 3.3270281481371438, + "tokens_seen": 3276079104 + }, + { + "epoch": 0.99, + "learning_rate": 7.2620767132081524e-06, + "loss": 1.2069, + "theoretical_loss": 3.327018181975328, + "tokens_seen": 3276210176 + }, + { + "epoch": 0.99, + "learning_rate": 7.221954742416948e-06, + "loss": 1.2964, + "theoretical_loss": 3.3270082163238586, + "tokens_seen": 3276341248 + }, + { + "epoch": 0.99, + "learning_rate": 7.181832771625742e-06, + "loss": 1.1017, + "theoretical_loss": 3.3269982511826885, + "tokens_seen": 3276472320 + }, + { + "epoch": 0.99, + "learning_rate": 7.141710800834537e-06, + "loss": 1.2508, + "theoretical_loss": 3.3269882865517717, + "tokens_seen": 3276603392 + }, + { + "epoch": 0.99, + "learning_rate": 7.101588830043332e-06, + "loss": 1.204, + "theoretical_loss": 3.326978322431062, + "tokens_seen": 3276734464 + }, + { + "epoch": 0.99, + "learning_rate": 7.061466859252127e-06, + "loss": 1.1696, + "theoretical_loss": 3.326968358820512, + "tokens_seen": 3276865536 + }, + { + "epoch": 0.99, + "learning_rate": 7.021344888460921e-06, + "loss": 1.2168, + "theoretical_loss": 3.3269583957200757, + "tokens_seen": 3276996608 + }, + { + "epoch": 0.99, + "learning_rate": 6.981222917669716e-06, + "loss": 1.201, + "theoretical_loss": 3.3269484331297066, + "tokens_seen": 3277127680 + }, + { + "epoch": 0.99, + "learning_rate": 6.941100946878511e-06, + "loss": 1.2477, + "theoretical_loss": 3.326938471049358, + "tokens_seen": 3277258752 + }, + { + "epoch": 0.99, + "learning_rate": 6.900978976087305e-06, + "loss": 1.2134, + "theoretical_loss": 3.326928509478983, + "tokens_seen": 3277389824 + }, + { + "epoch": 0.99, + "learning_rate": 6.8608570052961e-06, + "loss": 1.2275, + "theoretical_loss": 3.3269185484185364, + "tokens_seen": 3277520896 + }, + { + "epoch": 0.99, + "learning_rate": 6.820735034504896e-06, + "loss": 1.173, + "theoretical_loss": 3.3269085878679703, + "tokens_seen": 3277651968 + }, + { + "epoch": 0.99, + "learning_rate": 6.780613063713689e-06, + "loss": 1.2164, + "theoretical_loss": 3.326898627827239, + "tokens_seen": 3277783040 + }, + { + "epoch": 0.99, + "learning_rate": 6.7404910929224845e-06, + "loss": 1.2468, + "theoretical_loss": 3.326888668296296, + "tokens_seen": 3277914112 + }, + { + "epoch": 0.99, + "learning_rate": 6.70036912213128e-06, + "loss": 1.2685, + "theoretical_loss": 3.326878709275094, + "tokens_seen": 3278045184 + }, + { + "epoch": 0.99, + "learning_rate": 6.660247151340073e-06, + "loss": 1.2229, + "theoretical_loss": 3.3268687507635875, + "tokens_seen": 3278176256 + }, + { + "epoch": 0.99, + "learning_rate": 6.620125180548869e-06, + "loss": 1.2546, + "theoretical_loss": 3.3268587927617297, + "tokens_seen": 3278307328 + }, + { + "epoch": 0.99, + "learning_rate": 6.580003209757664e-06, + "loss": 1.2269, + "theoretical_loss": 3.326848835269474, + "tokens_seen": 3278438400 + }, + { + "epoch": 0.99, + "learning_rate": 6.539881238966458e-06, + "loss": 1.241, + "theoretical_loss": 3.3268388782867744, + "tokens_seen": 3278569472 + }, + { + "epoch": 0.99, + "learning_rate": 6.499759268175253e-06, + "loss": 1.2425, + "theoretical_loss": 3.326828921813584, + "tokens_seen": 3278700544 + }, + { + "epoch": 0.99, + "learning_rate": 6.459637297384048e-06, + "loss": 1.2121, + "theoretical_loss": 3.3268189658498564, + "tokens_seen": 3278831616 + }, + { + "epoch": 0.99, + "learning_rate": 6.419515326592842e-06, + "loss": 1.2358, + "theoretical_loss": 3.3268090103955457, + "tokens_seen": 3278962688 + }, + { + "epoch": 0.99, + "learning_rate": 6.379393355801637e-06, + "loss": 1.2266, + "theoretical_loss": 3.3267990554506044, + "tokens_seen": 3279093760 + }, + { + "epoch": 0.99, + "learning_rate": 6.339271385010432e-06, + "loss": 1.2936, + "theoretical_loss": 3.3267891010149873, + "tokens_seen": 3279224832 + }, + { + "epoch": 0.99, + "learning_rate": 6.299149414219226e-06, + "loss": 1.2617, + "theoretical_loss": 3.3267791470886467, + "tokens_seen": 3279355904 + }, + { + "epoch": 0.99, + "learning_rate": 6.259027443428021e-06, + "loss": 1.23, + "theoretical_loss": 3.326769193671537, + "tokens_seen": 3279486976 + }, + { + "epoch": 0.99, + "learning_rate": 6.2189054726368165e-06, + "loss": 1.1627, + "theoretical_loss": 3.326759240763612, + "tokens_seen": 3279618048 + }, + { + "epoch": 0.99, + "learning_rate": 6.17878350184561e-06, + "loss": 1.1304, + "theoretical_loss": 3.3267492883648253, + "tokens_seen": 3279749120 + }, + { + "epoch": 0.99, + "learning_rate": 6.1386615310544054e-06, + "loss": 1.3509, + "theoretical_loss": 3.32673933647513, + "tokens_seen": 3279880192 + }, + { + "epoch": 0.99, + "learning_rate": 6.098539560263201e-06, + "loss": 1.2203, + "theoretical_loss": 3.3267293850944797, + "tokens_seen": 3280011264 + }, + { + "epoch": 0.99, + "learning_rate": 6.058417589471994e-06, + "loss": 1.3206, + "theoretical_loss": 3.326719434222828, + "tokens_seen": 3280142336 + }, + { + "epoch": 0.99, + "learning_rate": 6.01829561868079e-06, + "loss": 1.2743, + "theoretical_loss": 3.326709483860129, + "tokens_seen": 3280273408 + }, + { + "epoch": 0.99, + "learning_rate": 5.978173647889585e-06, + "loss": 1.1551, + "theoretical_loss": 3.3266995340063366, + "tokens_seen": 3280404480 + }, + { + "epoch": 0.99, + "learning_rate": 5.938051677098379e-06, + "loss": 1.3043, + "theoretical_loss": 3.3266895846614033, + "tokens_seen": 3280535552 + }, + { + "epoch": 0.99, + "learning_rate": 5.897929706307174e-06, + "loss": 1.2007, + "theoretical_loss": 3.3266796358252835, + "tokens_seen": 3280666624 + }, + { + "epoch": 0.99, + "learning_rate": 5.857807735515969e-06, + "loss": 1.234, + "theoretical_loss": 3.3266696874979313, + "tokens_seen": 3280797696 + }, + { + "epoch": 0.99, + "learning_rate": 5.8176857647247636e-06, + "loss": 1.1641, + "theoretical_loss": 3.326659739679299, + "tokens_seen": 3280928768 + }, + { + "epoch": 0.99, + "learning_rate": 5.777563793933558e-06, + "loss": 1.2683, + "theoretical_loss": 3.3266497923693414, + "tokens_seen": 3281059840 + }, + { + "epoch": 0.99, + "learning_rate": 5.737441823142353e-06, + "loss": 1.2693, + "theoretical_loss": 3.326639845568012, + "tokens_seen": 3281190912 + }, + { + "epoch": 0.99, + "learning_rate": 5.697319852351148e-06, + "loss": 1.1684, + "theoretical_loss": 3.326629899275264, + "tokens_seen": 3281321984 + }, + { + "epoch": 0.99, + "learning_rate": 5.657197881559942e-06, + "loss": 1.2453, + "theoretical_loss": 3.326619953491052, + "tokens_seen": 3281453056 + }, + { + "epoch": 0.99, + "learning_rate": 5.6170759107687375e-06, + "loss": 1.2186, + "theoretical_loss": 3.3266100082153285, + "tokens_seen": 3281584128 + }, + { + "epoch": 0.99, + "learning_rate": 5.576953939977532e-06, + "loss": 1.2361, + "theoretical_loss": 3.3266000634480477, + "tokens_seen": 3281715200 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.9948239922523499, + "objective/train/avg_token_score": 0.005176020320504904, + "objective/train/docs_used": 1192226, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2487382888793945, + "objective/train/original_loss": 1.2558767795562744, + "objective/train/theoretical_loss": 3.326590119189164, + "objective/train/tokens_used": 1652305376, + "objective/train/unlikelihood_loss": 0.01480909250676632, + "theoretical_loss": 3.326590119189164, + "tokens_seen": 3281846272 + }, + { + "epoch": 0.99, + "learning_rate": 5.536831969186326e-06, + "loss": 1.2235, + "theoretical_loss": 3.326590119189164, + "tokens_seen": 3281846272 + }, + { + "epoch": 0.99, + "learning_rate": 5.496709998395122e-06, + "loss": 1.3084, + "theoretical_loss": 3.3265801754386297, + "tokens_seen": 3281977344 + }, + { + "epoch": 0.99, + "learning_rate": 5.456588027603916e-06, + "loss": 1.2016, + "theoretical_loss": 3.3265702321963997, + "tokens_seen": 3282108416 + }, + { + "epoch": 0.99, + "learning_rate": 5.416466056812711e-06, + "loss": 1.2166, + "theoretical_loss": 3.326560289462427, + "tokens_seen": 3282239488 + }, + { + "epoch": 0.99, + "learning_rate": 5.376344086021506e-06, + "loss": 1.2301, + "theoretical_loss": 3.326550347236666, + "tokens_seen": 3282370560 + }, + { + "epoch": 0.99, + "learning_rate": 5.3362221152303e-06, + "loss": 1.2538, + "theoretical_loss": 3.3265404055190704, + "tokens_seen": 3282501632 + }, + { + "epoch": 0.99, + "learning_rate": 5.296100144439095e-06, + "loss": 1.1902, + "theoretical_loss": 3.326530464309593, + "tokens_seen": 3282632704 + }, + { + "epoch": 0.99, + "learning_rate": 5.255978173647889e-06, + "loss": 1.2804, + "theoretical_loss": 3.3265205236081883, + "tokens_seen": 3282763776 + }, + { + "epoch": 0.99, + "learning_rate": 5.2158562028566845e-06, + "loss": 1.2523, + "theoretical_loss": 3.3265105834148097, + "tokens_seen": 3282894848 + }, + { + "epoch": 0.99, + "learning_rate": 5.175734232065479e-06, + "loss": 1.251, + "theoretical_loss": 3.3265006437294113, + "tokens_seen": 3283025920 + }, + { + "epoch": 0.99, + "learning_rate": 5.135612261274273e-06, + "loss": 1.2406, + "theoretical_loss": 3.326490704551947, + "tokens_seen": 3283156992 + }, + { + "epoch": 0.99, + "learning_rate": 5.095490290483069e-06, + "loss": 1.2554, + "theoretical_loss": 3.32648076588237, + "tokens_seen": 3283288064 + }, + { + "epoch": 0.99, + "learning_rate": 5.055368319691863e-06, + "loss": 1.2192, + "theoretical_loss": 3.3264708277206347, + "tokens_seen": 3283419136 + }, + { + "epoch": 0.99, + "learning_rate": 5.015246348900658e-06, + "loss": 1.2638, + "theoretical_loss": 3.326460890066694, + "tokens_seen": 3283550208 + }, + { + "epoch": 0.99, + "learning_rate": 4.975124378109453e-06, + "loss": 1.185, + "theoretical_loss": 3.3264509529205024, + "tokens_seen": 3283681280 + }, + { + "epoch": 0.99, + "learning_rate": 4.935002407318248e-06, + "loss": 1.2334, + "theoretical_loss": 3.326441016282014, + "tokens_seen": 3283812352 + }, + { + "epoch": 0.99, + "learning_rate": 4.894880436527042e-06, + "loss": 1.2226, + "theoretical_loss": 3.3264310801511816, + "tokens_seen": 3283943424 + }, + { + "epoch": 0.99, + "learning_rate": 4.854758465735837e-06, + "loss": 1.1238, + "theoretical_loss": 3.3264211445279592, + "tokens_seen": 3284074496 + }, + { + "epoch": 0.99, + "learning_rate": 4.814636494944632e-06, + "loss": 1.0895, + "theoretical_loss": 3.3264112094123015, + "tokens_seen": 3284205568 + }, + { + "epoch": 0.99, + "learning_rate": 4.774514524153426e-06, + "loss": 1.1691, + "theoretical_loss": 3.326401274804162, + "tokens_seen": 3284336640 + }, + { + "epoch": 0.99, + "learning_rate": 4.734392553362221e-06, + "loss": 1.1866, + "theoretical_loss": 3.3263913407034935, + "tokens_seen": 3284467712 + }, + { + "epoch": 0.99, + "learning_rate": 4.6942705825710166e-06, + "loss": 1.2498, + "theoretical_loss": 3.326381407110251, + "tokens_seen": 3284598784 + }, + { + "epoch": 0.99, + "learning_rate": 4.65414861177981e-06, + "loss": 1.262, + "theoretical_loss": 3.3263714740243877, + "tokens_seen": 3284729856 + }, + { + "epoch": 0.99, + "learning_rate": 4.6140266409886055e-06, + "loss": 1.2418, + "theoretical_loss": 3.326361541445858, + "tokens_seen": 3284860928 + }, + { + "epoch": 0.99, + "learning_rate": 4.573904670197401e-06, + "loss": 1.112, + "theoretical_loss": 3.3263516093746155, + "tokens_seen": 3284992000 + }, + { + "epoch": 0.99, + "learning_rate": 4.533782699406194e-06, + "loss": 1.2609, + "theoretical_loss": 3.3263416778106136, + "tokens_seen": 3285123072 + }, + { + "epoch": 0.99, + "learning_rate": 4.49366072861499e-06, + "loss": 1.1826, + "theoretical_loss": 3.3263317467538065, + "tokens_seen": 3285254144 + }, + { + "epoch": 0.99, + "learning_rate": 4.453538757823785e-06, + "loss": 1.242, + "theoretical_loss": 3.3263218162041484, + "tokens_seen": 3285385216 + }, + { + "epoch": 0.99, + "learning_rate": 4.4134167870325786e-06, + "loss": 1.2307, + "theoretical_loss": 3.3263118861615926, + "tokens_seen": 3285516288 + }, + { + "epoch": 0.99, + "learning_rate": 4.373294816241374e-06, + "loss": 1.2954, + "theoretical_loss": 3.326301956626093, + "tokens_seen": 3285647360 + }, + { + "epoch": 0.99, + "learning_rate": 4.333172845450169e-06, + "loss": 1.2385, + "theoretical_loss": 3.326292027597604, + "tokens_seen": 3285778432 + }, + { + "epoch": 0.99, + "learning_rate": 4.293050874658963e-06, + "loss": 1.2467, + "theoretical_loss": 3.3262820990760793, + "tokens_seen": 3285909504 + }, + { + "epoch": 0.99, + "learning_rate": 4.252928903867758e-06, + "loss": 1.1863, + "theoretical_loss": 3.3262721710614724, + "tokens_seen": 3286040576 + }, + { + "epoch": 0.99, + "learning_rate": 4.212806933076553e-06, + "loss": 1.1887, + "theoretical_loss": 3.326262243553738, + "tokens_seen": 3286171648 + }, + { + "epoch": 0.99, + "learning_rate": 4.172684962285347e-06, + "loss": 1.2615, + "theoretical_loss": 3.326252316552829, + "tokens_seen": 3286302720 + }, + { + "epoch": 0.99, + "learning_rate": 4.132562991494142e-06, + "loss": 1.2193, + "theoretical_loss": 3.3262423900586997, + "tokens_seen": 3286433792 + }, + { + "epoch": 0.99, + "learning_rate": 4.0924410207029375e-06, + "loss": 1.2052, + "theoretical_loss": 3.3262324640713046, + "tokens_seen": 3286564864 + }, + { + "epoch": 0.99, + "learning_rate": 4.052319049911731e-06, + "loss": 1.2268, + "theoretical_loss": 3.326222538590597, + "tokens_seen": 3286695936 + }, + { + "epoch": 0.99, + "learning_rate": 4.012197079120526e-06, + "loss": 1.1918, + "theoretical_loss": 3.326212613616531, + "tokens_seen": 3286827008 + }, + { + "epoch": 0.99, + "learning_rate": 3.972075108329322e-06, + "loss": 1.2991, + "theoretical_loss": 3.3262026891490604, + "tokens_seen": 3286958080 + }, + { + "epoch": 0.99, + "learning_rate": 3.931953137538115e-06, + "loss": 1.2066, + "theoretical_loss": 3.3261927651881393, + "tokens_seen": 3287089152 + }, + { + "epoch": 0.99, + "learning_rate": 3.891831166746911e-06, + "loss": 1.2408, + "theoretical_loss": 3.3261828417337216, + "tokens_seen": 3287220224 + }, + { + "epoch": 0.99, + "learning_rate": 3.851709195955705e-06, + "loss": 1.1856, + "theoretical_loss": 3.3261729187857614, + "tokens_seen": 3287351296 + }, + { + "epoch": 0.99, + "learning_rate": 3.8115872251645004e-06, + "loss": 1.1665, + "theoretical_loss": 3.3261629963442125, + "tokens_seen": 3287482368 + }, + { + "epoch": 0.99, + "learning_rate": 3.771465254373295e-06, + "loss": 1.2061, + "theoretical_loss": 3.326153074409029, + "tokens_seen": 3287613440 + }, + { + "epoch": 0.99, + "learning_rate": 3.7313432835820897e-06, + "loss": 1.1711, + "theoretical_loss": 3.326143152980164, + "tokens_seen": 3287744512 + }, + { + "epoch": 0.99, + "learning_rate": 3.691221312790884e-06, + "loss": 1.1889, + "theoretical_loss": 3.326133232057573, + "tokens_seen": 3287875584 + }, + { + "epoch": 0.99, + "learning_rate": 3.651099341999679e-06, + "loss": 1.1674, + "theoretical_loss": 3.3261233116412092, + "tokens_seen": 3288006656 + }, + { + "epoch": 0.99, + "learning_rate": 3.610977371208474e-06, + "loss": 1.2354, + "theoretical_loss": 3.3261133917310266, + "tokens_seen": 3288137728 + }, + { + "epoch": 0.99, + "learning_rate": 3.5708554004172683e-06, + "loss": 1.164, + "theoretical_loss": 3.326103472326979, + "tokens_seen": 3288268800 + }, + { + "epoch": 0.99, + "objective/train/aligned_token_count": 0.9924721121788025, + "objective/train/avg_token_score": 0.007527893874794245, + "objective/train/docs_used": 1194491, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.2073938846588135, + "objective/train/original_loss": 1.2173194885253906, + "objective/train/theoretical_loss": 3.3260935534290206, + "objective/train/tokens_used": 1658858976, + "objective/train/unlikelihood_loss": 0.02140188403427601, + "theoretical_loss": 3.3260935534290206, + "tokens_seen": 3288399872 + }, + { + "epoch": 0.99, + "learning_rate": 3.5307334296260636e-06, + "loss": 1.1916, + "theoretical_loss": 3.3260935534290206, + "tokens_seen": 3288399872 + }, + { + "epoch": 0.99, + "learning_rate": 3.490611458834858e-06, + "loss": 1.1868, + "theoretical_loss": 3.3260836350371057, + "tokens_seen": 3288530944 + }, + { + "epoch": 0.99, + "learning_rate": 3.4504894880436525e-06, + "loss": 1.299, + "theoretical_loss": 3.326073717151188, + "tokens_seen": 3288662016 + }, + { + "epoch": 0.99, + "learning_rate": 3.410367517252448e-06, + "loss": 1.3378, + "theoretical_loss": 3.3260637997712212, + "tokens_seen": 3288793088 + }, + { + "epoch": 0.99, + "learning_rate": 3.3702455464612422e-06, + "loss": 1.2387, + "theoretical_loss": 3.32605388289716, + "tokens_seen": 3288924160 + }, + { + "epoch": 0.99, + "learning_rate": 3.3301235756700367e-06, + "loss": 1.2164, + "theoretical_loss": 3.326043966528958, + "tokens_seen": 3289055232 + }, + { + "epoch": 0.99, + "learning_rate": 3.290001604878832e-06, + "loss": 1.2771, + "theoretical_loss": 3.3260340506665695, + "tokens_seen": 3289186304 + }, + { + "epoch": 0.99, + "learning_rate": 3.2498796340876264e-06, + "loss": 1.1797, + "theoretical_loss": 3.3260241353099484, + "tokens_seen": 3289317376 + }, + { + "epoch": 0.99, + "learning_rate": 3.209757663296421e-06, + "loss": 1.2101, + "theoretical_loss": 3.3260142204590486, + "tokens_seen": 3289448448 + }, + { + "epoch": 0.99, + "learning_rate": 3.169635692505216e-06, + "loss": 1.1919, + "theoretical_loss": 3.3260043061138247, + "tokens_seen": 3289579520 + }, + { + "epoch": 0.99, + "learning_rate": 3.1295137217140106e-06, + "loss": 1.2726, + "theoretical_loss": 3.32599439227423, + "tokens_seen": 3289710592 + }, + { + "epoch": 0.99, + "learning_rate": 3.089391750922805e-06, + "loss": 1.1732, + "theoretical_loss": 3.325984478940219, + "tokens_seen": 3289841664 + }, + { + "epoch": 0.99, + "learning_rate": 3.0492697801316004e-06, + "loss": 1.2361, + "theoretical_loss": 3.325974566111746, + "tokens_seen": 3289972736 + }, + { + "epoch": 0.99, + "learning_rate": 3.009147809340395e-06, + "loss": 1.2073, + "theoretical_loss": 3.3259646537887644, + "tokens_seen": 3290103808 + }, + { + "epoch": 0.99, + "learning_rate": 2.9690258385491897e-06, + "loss": 1.2476, + "theoretical_loss": 3.3259547419712288, + "tokens_seen": 3290234880 + }, + { + "epoch": 0.99, + "learning_rate": 2.9289038677579846e-06, + "loss": 1.1926, + "theoretical_loss": 3.3259448306590933, + "tokens_seen": 3290365952 + }, + { + "epoch": 0.99, + "learning_rate": 2.888781896966779e-06, + "loss": 1.2231, + "theoretical_loss": 3.325934919852312, + "tokens_seen": 3290497024 + }, + { + "epoch": 0.99, + "learning_rate": 2.848659926175574e-06, + "loss": 1.2258, + "theoretical_loss": 3.325925009550838, + "tokens_seen": 3290628096 + }, + { + "epoch": 0.99, + "learning_rate": 2.8085379553843687e-06, + "loss": 1.2626, + "theoretical_loss": 3.3259150997546274, + "tokens_seen": 3290759168 + }, + { + "epoch": 0.99, + "learning_rate": 2.768415984593163e-06, + "loss": 1.2029, + "theoretical_loss": 3.3259051904636325, + "tokens_seen": 3290890240 + }, + { + "epoch": 0.99, + "learning_rate": 2.728294013801958e-06, + "loss": 1.2098, + "theoretical_loss": 3.3258952816778087, + "tokens_seen": 3291021312 + }, + { + "epoch": 0.99, + "learning_rate": 2.688172043010753e-06, + "loss": 1.2062, + "theoretical_loss": 3.325885373397109, + "tokens_seen": 3291152384 + }, + { + "epoch": 0.99, + "learning_rate": 2.6480500722195474e-06, + "loss": 1.2228, + "theoretical_loss": 3.325875465621489, + "tokens_seen": 3291283456 + }, + { + "epoch": 0.99, + "learning_rate": 2.6079281014283423e-06, + "loss": 1.2505, + "theoretical_loss": 3.325865558350901, + "tokens_seen": 3291414528 + }, + { + "epoch": 0.99, + "learning_rate": 2.5678061306371367e-06, + "loss": 1.2198, + "theoretical_loss": 3.3258556515853, + "tokens_seen": 3291545600 + }, + { + "epoch": 0.99, + "learning_rate": 2.5276841598459316e-06, + "loss": 1.2051, + "theoretical_loss": 3.325845745324641, + "tokens_seen": 3291676672 + }, + { + "epoch": 1.0, + "learning_rate": 2.4875621890547264e-06, + "loss": 1.2006, + "theoretical_loss": 3.325835839568877, + "tokens_seen": 3291807744 + }, + { + "epoch": 1.0, + "learning_rate": 2.447440218263521e-06, + "loss": 1.1497, + "theoretical_loss": 3.3258259343179626, + "tokens_seen": 3291938816 + }, + { + "epoch": 1.0, + "learning_rate": 2.407318247472316e-06, + "loss": 1.2129, + "theoretical_loss": 3.3258160295718517, + "tokens_seen": 3292069888 + }, + { + "epoch": 1.0, + "learning_rate": 2.3671962766811106e-06, + "loss": 1.2089, + "theoretical_loss": 3.325806125330499, + "tokens_seen": 3292200960 + }, + { + "epoch": 1.0, + "learning_rate": 2.327074305889905e-06, + "loss": 1.2614, + "theoretical_loss": 3.3257962215938583, + "tokens_seen": 3292332032 + }, + { + "epoch": 1.0, + "learning_rate": 2.2869523350987004e-06, + "loss": 1.2351, + "theoretical_loss": 3.325786318361884, + "tokens_seen": 3292463104 + }, + { + "epoch": 1.0, + "learning_rate": 2.246830364307495e-06, + "loss": 1.3051, + "theoretical_loss": 3.32577641563453, + "tokens_seen": 3292594176 + }, + { + "epoch": 1.0, + "learning_rate": 2.2067083935162893e-06, + "loss": 1.2012, + "theoretical_loss": 3.3257665134117507, + "tokens_seen": 3292725248 + }, + { + "epoch": 1.0, + "learning_rate": 2.1665864227250846e-06, + "loss": 1.2086, + "theoretical_loss": 3.3257566116935005, + "tokens_seen": 3292856320 + }, + { + "epoch": 1.0, + "learning_rate": 2.126464451933879e-06, + "loss": 1.2112, + "theoretical_loss": 3.325746710479733, + "tokens_seen": 3292987392 + }, + { + "epoch": 1.0, + "learning_rate": 2.0863424811426735e-06, + "loss": 1.1005, + "theoretical_loss": 3.325736809770403, + "tokens_seen": 3293118464 + }, + { + "epoch": 1.0, + "learning_rate": 2.0462205103514688e-06, + "loss": 1.2373, + "theoretical_loss": 3.3257269095654647, + "tokens_seen": 3293249536 + }, + { + "epoch": 1.0, + "learning_rate": 2.006098539560263e-06, + "loss": 1.1641, + "theoretical_loss": 3.325717009864872, + "tokens_seen": 3293380608 + }, + { + "epoch": 1.0, + "learning_rate": 1.9659765687690577e-06, + "loss": 1.2243, + "theoretical_loss": 3.325707110668579, + "tokens_seen": 3293511680 + }, + { + "epoch": 1.0, + "learning_rate": 1.9258545979778525e-06, + "loss": 1.195, + "theoretical_loss": 3.325697211976541, + "tokens_seen": 3293642752 + }, + { + "epoch": 1.0, + "learning_rate": 1.8857326271866474e-06, + "loss": 1.2977, + "theoretical_loss": 3.325687313788711, + "tokens_seen": 3293773824 + }, + { + "epoch": 1.0, + "learning_rate": 1.845610656395442e-06, + "loss": 1.2359, + "theoretical_loss": 3.325677416105044, + "tokens_seen": 3293904896 + }, + { + "epoch": 1.0, + "learning_rate": 1.805488685604237e-06, + "loss": 1.1541, + "theoretical_loss": 3.325667518925494, + "tokens_seen": 3294035968 + }, + { + "epoch": 1.0, + "learning_rate": 1.7653667148130318e-06, + "loss": 1.2199, + "theoretical_loss": 3.3256576222500147, + "tokens_seen": 3294167040 + }, + { + "epoch": 1.0, + "learning_rate": 1.7252447440218263e-06, + "loss": 1.2452, + "theoretical_loss": 3.3256477260785613, + "tokens_seen": 3294298112 + }, + { + "epoch": 1.0, + "learning_rate": 1.6851227732306211e-06, + "loss": 1.2793, + "theoretical_loss": 3.325637830411088, + "tokens_seen": 3294429184 + }, + { + "epoch": 1.0, + "learning_rate": 1.645000802439416e-06, + "loss": 1.175, + "theoretical_loss": 3.3256279352475486, + "tokens_seen": 3294560256 + }, + { + "epoch": 1.0, + "learning_rate": 1.6048788316482104e-06, + "loss": 1.1699, + "theoretical_loss": 3.3256180405878975, + "tokens_seen": 3294691328 + }, + { + "epoch": 1.0, + "learning_rate": 1.5647568608570053e-06, + "loss": 1.2223, + "theoretical_loss": 3.325608146432089, + "tokens_seen": 3294822400 + }, + { + "epoch": 1.0, + "objective/train/aligned_token_count": 0.9926229119300842, + "objective/train/avg_token_score": 0.007377032656222582, + "objective/train/docs_used": 1196765, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/likelihood_loss": 1.1668452024459839, + "objective/train/original_loss": 1.1744681596755981, + "objective/train/theoretical_loss": 3.325598252780078, + "objective/train/tokens_used": 1665412576, + "objective/train/unlikelihood_loss": 0.025721674785017967, + "theoretical_loss": 3.325598252780078, + "tokens_seen": 3294953472 + }, + { + "epoch": 1.0, + "learning_rate": 1.5246348900658002e-06, + "loss": 1.258, + "theoretical_loss": 3.325598252780078, + "tokens_seen": 3294953472 + }, + { + "epoch": 1.0, + "learning_rate": 1.4845129192745948e-06, + "loss": 1.2555, + "theoretical_loss": 3.325588359631818, + "tokens_seen": 3295084544 + }, + { + "epoch": 1.0, + "learning_rate": 1.4443909484833895e-06, + "loss": 1.1576, + "theoretical_loss": 3.3255784669872632, + "tokens_seen": 3295215616 + }, + { + "epoch": 1.0, + "learning_rate": 1.4042689776921844e-06, + "loss": 1.2213, + "theoretical_loss": 3.325568574846369, + "tokens_seen": 3295346688 + }, + { + "epoch": 1.0, + "learning_rate": 1.364147006900979e-06, + "loss": 1.2217, + "theoretical_loss": 3.3255586832090884, + "tokens_seen": 3295477760 + }, + { + "epoch": 1.0, + "learning_rate": 1.3240250361097737e-06, + "loss": 1.2037, + "theoretical_loss": 3.3255487920753772, + "tokens_seen": 3295608832 + }, + { + "epoch": 1.0, + "learning_rate": 1.2839030653185684e-06, + "loss": 1.1906, + "theoretical_loss": 3.325538901445188, + "tokens_seen": 3295739904 + }, + { + "epoch": 1.0, + "learning_rate": 1.2437810945273632e-06, + "loss": 1.198, + "theoretical_loss": 3.325529011318477, + "tokens_seen": 3295870976 + }, + { + "epoch": 1.0, + "learning_rate": 1.203659123736158e-06, + "loss": 1.2412, + "theoretical_loss": 3.325519121695197, + "tokens_seen": 3296002048 + }, + { + "epoch": 1.0, + "learning_rate": 1.1635371529449525e-06, + "loss": 1.2456, + "theoretical_loss": 3.3255092325753033, + "tokens_seen": 3296133120 + }, + { + "epoch": 1.0, + "learning_rate": 1.1234151821537474e-06, + "loss": 1.2328, + "theoretical_loss": 3.32549934395875, + "tokens_seen": 3296264192 + }, + { + "epoch": 1.0, + "learning_rate": 1.0832932113625423e-06, + "loss": 1.2469, + "theoretical_loss": 3.3254894558454913, + "tokens_seen": 3296395264 + }, + { + "epoch": 1.0, + "learning_rate": 1.0431712405713367e-06, + "loss": 1.2742, + "theoretical_loss": 3.3254795682354814, + "tokens_seen": 3296526336 + }, + { + "epoch": 1.0, + "learning_rate": 1.0030492697801316e-06, + "loss": 1.2397, + "theoretical_loss": 3.3254696811286752, + "tokens_seen": 3296657408 + }, + { + "epoch": 1.0, + "learning_rate": 9.629272989889263e-07, + "loss": 1.2546, + "theoretical_loss": 3.325459794525027, + "tokens_seen": 3296788480 + }, + { + "epoch": 1.0, + "learning_rate": 9.22805328197721e-07, + "loss": 1.2718, + "theoretical_loss": 3.325449908424491, + "tokens_seen": 3296919552 + }, + { + "epoch": 1.0, + "learning_rate": 8.826833574065159e-07, + "loss": 1.2116, + "theoretical_loss": 3.3254400228270216, + "tokens_seen": 3297050624 + }, + { + "epoch": 1.0, + "learning_rate": 8.425613866153106e-07, + "loss": 1.211, + "theoretical_loss": 3.3254301377325732, + "tokens_seen": 3297181696 + }, + { + "epoch": 1.0, + "learning_rate": 8.024394158241052e-07, + "loss": 1.2247, + "theoretical_loss": 3.3254202531411003, + "tokens_seen": 3297312768 + }, + { + "epoch": 1.0, + "learning_rate": 7.623174450329001e-07, + "loss": 1.2415, + "theoretical_loss": 3.325410369052557, + "tokens_seen": 3297443840 + }, + { + "epoch": 1.0, + "learning_rate": 7.221954742416948e-07, + "loss": 1.2072, + "theoretical_loss": 3.3254004854668984, + "tokens_seen": 3297574912 + }, + { + "epoch": 1.0, + "learning_rate": 6.820735034504895e-07, + "loss": 1.1877, + "theoretical_loss": 3.3253906023840782, + "tokens_seen": 3297705984 + }, + { + "epoch": 1.0, + "learning_rate": 6.419515326592842e-07, + "loss": 1.1692, + "theoretical_loss": 3.3253807198040515, + "tokens_seen": 3297837056 + }, + { + "epoch": 1.0, + "learning_rate": 6.01829561868079e-07, + "loss": 1.2335, + "theoretical_loss": 3.325370837726772, + "tokens_seen": 3297968128 + }, + { + "epoch": 1.0, + "learning_rate": 5.617075910768737e-07, + "loss": 1.2278, + "theoretical_loss": 3.3253609561521946, + "tokens_seen": 3298099200 + }, + { + "epoch": 1.0, + "learning_rate": 5.215856202856684e-07, + "loss": 1.243, + "theoretical_loss": 3.325351075080274, + "tokens_seen": 3298230272 + }, + { + "epoch": 1.0, + "learning_rate": 4.814636494944631e-07, + "loss": 1.163, + "theoretical_loss": 3.325341194510964, + "tokens_seen": 3298361344 + }, + { + "epoch": 1.0, + "learning_rate": 4.4134167870325795e-07, + "loss": 1.2038, + "theoretical_loss": 3.3253313144442194, + "tokens_seen": 3298492416 + }, + { + "epoch": 1.0, + "learning_rate": 4.012197079120526e-07, + "loss": 1.2958, + "theoretical_loss": 3.325321434879995, + "tokens_seen": 3298623488 + }, + { + "epoch": 1.0, + "learning_rate": 3.610977371208474e-07, + "loss": 1.231, + "theoretical_loss": 3.325311555818245, + "tokens_seen": 3298754560 + }, + { + "epoch": 1.0, + "learning_rate": 3.209757663296421e-07, + "loss": 1.2278, + "theoretical_loss": 3.3253016772589232, + "tokens_seen": 3298885632 + }, + { + "epoch": 1.0, + "learning_rate": 2.8085379553843685e-07, + "loss": 1.2052, + "theoretical_loss": 3.325291799201985, + "tokens_seen": 3299016704 + }, + { + "epoch": 1.0, + "learning_rate": 2.4073182474723157e-07, + "loss": 1.2491, + "theoretical_loss": 3.3252819216473846, + "tokens_seen": 3299147776 + }, + { + "epoch": 1.0, + "learning_rate": 2.006098539560263e-07, + "loss": 1.1698, + "theoretical_loss": 3.3252720445950765, + "tokens_seen": 3299278848 + }, + { + "epoch": 1.0, + "learning_rate": 1.6048788316482104e-07, + "loss": 1.2697, + "theoretical_loss": 3.3252621680450156, + "tokens_seen": 3299409920 + }, + { + "epoch": 1.0, + "learning_rate": 1.2036591237361578e-07, + "loss": 1.2325, + "theoretical_loss": 3.3252522919971557, + "tokens_seen": 3299540992 + }, + { + "epoch": 1.0, + "learning_rate": 8.024394158241052e-08, + "loss": 1.2428, + "theoretical_loss": 3.3252424164514514, + "tokens_seen": 3299672064 + }, + { + "epoch": 1.0, + "learning_rate": 4.012197079120526e-08, + "loss": 1.2135, + "theoretical_loss": 3.325232541407858, + "tokens_seen": 3299803136 + }, + { + "epoch": 1.0, + "learning_rate": 0.0, + "loss": 1.1913, + "theoretical_loss": 3.325222666866329, + "tokens_seen": 3299934208 + } + ], + "max_steps": 12588, + "num_train_epochs": 9223372036854775807, + "total_flos": 8.42020889296896e+17, + "trial_name": null, + "trial_params": null +}